2020. 1. 3. 14:41
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | <code class = "hljs" > <span class = "hljs-keyword" > public </span> <span class = "hljs-class" ><span class = "hljs-keyword" > class </span> <span class = "hljs-title" >Population</span> </span>{ <span class = "hljs-function" ><span class = "hljs-keyword" > public </span> <span class = "hljs-keyword" > static </span> <span class = "hljs-keyword" > void </span> <span class = "hljs-title" >main</span><span class = "hljs-params" >(String[] args)</span> <span class = "hljs-keyword" > throws </span> Exception </span>{ <span class = "hljs-keyword" > if </span> (args.length != <span class = "hljs-number" > 2 </span>) { System.err.println(<span class = "hljs-string" > "Usage:Population " </span>); System.exit(-<span class = "hljs-number" > 1 </span>); } Job job = <span class = "hljs-keyword" > new </span> Job(); job.setJarByClass(Population. class ); job.setJobName(<span class = "hljs-string" > "Population" </span>); job.setMapperClass(PopulationMapper. class ); job.setReducerClass(PopulationReducer. class ); job.setOutputKeyClass(Text. class ); job.setOutputValueClass(IntWritable. class ); FileInputFormat.addInputPath(job, <span class = "hljs-keyword" > new </span> Path(args[<span class = "hljs-number" > 0 </span>])); FileOutputFormat.setOutputPath(job, <span class = "hljs-keyword" > new </span> Path(args[<span class = "hljs-number" > 1 </span>])); System.exit(job.waitForCompletion(<span class = "hljs-keyword" > true </span>) ? <span class = "hljs-number" > 0 </span> : <span class = "hljs-number" > 1 </span>); } } </code> |
mapper class
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 | <code class = "hljs" ><span class = "hljs-keyword" > public </span> <span class = "hljs-class" ><span class = "hljs-keyword" > class </span> <span class = "hljs-title" >PopulationMapper</span> <span class = "hljs-keyword" > extends </span> <span class = "hljs-title" >Mapper</span><<span class = "hljs-title" >LongWritable</span>, <span class = "hljs-title" >Text</span>, <span class = "hljs-title" >Text</span>, <span class = "hljs-title" >IntWritable</span>> </span>{ <span class = "hljs-keyword" > private </span> <span class = "hljs-keyword" > static </span> <span class = "hljs-keyword" > final </span> <span class = "hljs-keyword" > int </span> MISSING=<span class = "hljs-number" > 9999 </span>; <span class = "hljs-function" ><span class = "hljs-keyword" > public </span> <span class = "hljs-keyword" > void </span> <span class = "hljs-title" >map</span><span class = "hljs-params" >(LongWritable key, Text value, Context context)</span> <span class = "hljs-keyword" > throws </span> IOException, InterruptedException </span>{ String line = value.toString(); <span class = "hljs-comment" > //line = line.replaceAll(" ", "");</span> String year = line.substring(<span class = "hljs-number" > 0 </span>,<span class = "hljs-number" > 4 </span>); <span class = "hljs-keyword" > int </span> manpop = <span class = "hljs-number" > 0 </span>; <span class = "hljs-keyword" > if </span>(line.charAt(<span class = "hljs-number" > 7 </span>) == <span class = "hljs-string" > ' ' </span>) { manpop = Integer.parseInt(line.substring(<span class = "hljs-number" > 8 </span>, <span class = "hljs-number" > 14 </span>)); } String quality = line.substring(<span class = "hljs-number" > 56 </span>, <span class = "hljs-number" > 57 </span>); <span class = "hljs-keyword" > if </span> (manpop != MISSING && quality.matches(<span class = "hljs-string" > "[30]" </span>)) { context.write(<span class = "hljs-keyword" > new </span> Text(year), <span class = "hljs-keyword" > new </span> IntWritable(manpop)); } } } </code> |
reducer class
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | <code class = "hljs" ><span class = "hljs-keyword" > public </span> <span class = "hljs-class" ><span class = "hljs-keyword" > class </span> <span class = "hljs-title" >PopulationReducer</span> <span class = "hljs-keyword" > extends </span> <span class = "hljs-title" >Reducer</span><<span class = "hljs-title" >Text</span>, <span class = "hljs-title" >IntWritable</span>, <span class = "hljs-title" >Text</span>, <span class = "hljs-title" >DoubleWritable</span>> </span>{ <span class = "hljs-function" ><span class = "hljs-keyword" > public </span> <span class = "hljs-keyword" > void </span> <span class = "hljs-title" >reduce</span><span class = "hljs-params" >(Text key, Iterable values, Context context)</span> <span class = "hljs-keyword" > throws </span> IOException, InterruptedException </span>{ <span class = "hljs-keyword" > int </span> minValue = Integer.MAX_VALUE; <span class = "hljs-keyword" > int </span> maxValue = Integer.MIN_VALUE; <span class = "hljs-keyword" > int </span> number = <span class = "hljs-number" > 0 </span>; <span class = "hljs-keyword" > int </span> sum = <span class = "hljs-number" > 0 </span>; ArrayList valueList = <span class = "hljs-keyword" > new </span> ArrayList(); <span class = "hljs-keyword" > for </span> (IntWritable value : values) { minValue = Math.min(minValue, value.get()); maxValue = Math.max(maxValue, value.get()); sum += value.get(); valueList.add(value.get()); number++; } <span class = "hljs-keyword" > double </span> value1 = <span class = "hljs-number" > 0 </span>; <span class = "hljs-keyword" > double </span> value2 = <span class = "hljs-number" > 0 </span>; <span class = "hljs-keyword" > for </span>(<span class = "hljs-keyword" > int </span> i = <span class = "hljs-number" > 0 </span>; i<number; i++){ value1 = (<span class = "hljs-keyword" > double </span>)valueList.get(i)-sum/number; value2 = Math.pow(value1, <span class = "hljs-number" > 2 </span>)+value2; } context.write(key, <span class = "hljs-keyword" > new </span> DoubleWritable(minValue)); context.write(key, <span class = "hljs-keyword" > new </span> DoubleWritable(maxValue)); context.write(key, <span class = "hljs-keyword" > new </span> DoubleWritable(sum / number)); context.write(key, <span class = "hljs-keyword" > new </span> DoubleWritable(Math.sqrt(value2/number))); } } </code> |
'빅데이터' 카테고리의 다른 글
[mapreduce]교통사고 발생건수 통계 (0) | 2020.01.03 |
---|---|
[mapreduce]ncdc 연도별 기온 통계 (0) | 2020.01.03 |
[mapreduce] wordcount (0) | 2020.01.03 |