[mapreduce]인구수 통계

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26

<code class="hljs">  <span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Population</span> </span>{      <span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> Exception </span>{                  <span class="hljs-keyword">if</span> (args.length != <span class="hljs-number">2</span>) {             System.err.println(<span class="hljs-string">"Usage:Population  "</span>);             System.exit(-<span class="hljs-number">1</span>);         }                  Job job = <span class="hljs-keyword">new</span> Job();         job.setJarByClass(Population.class);         job.setJobName(<span class="hljs-string">"Population"</span>);                  job.setMapperClass(PopulationMapper.class);         job.setReducerClass(PopulationReducer.class);                  job.setOutputKeyClass(Text.class);         job.setOutputValueClass(IntWritable.class);                  FileInputFormat.addInputPath(job, <span class="hljs-keyword">new</span> Path(args[<span class="hljs-number">0</span>]));         FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> Path(args[<span class="hljs-number">1</span>]));         System.exit(job.waitForCompletion(<span class="hljs-keyword">true</span>) ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);     }   } </code>

mapper class

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21

<code class="hljs"><span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">PopulationMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title">Mapper</span><<span class="hljs-title">LongWritable</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">IntWritable</span>> </span>{      <span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">final</span> <span class="hljs-keyword">int</span> MISSING=<span class="hljs-number">9999</span>;          <span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException </span>{         String line = value.toString();         <span class="hljs-comment">//line = line.replaceAll(" ", "");</span>         String year = line.substring(<span class="hljs-number">0</span>,<span class="hljs-number">4</span>);         <span class="hljs-keyword">int</span> manpop = <span class="hljs-number">0</span>;                  <span class="hljs-keyword">if</span>(line.charAt(<span class="hljs-number">7</span>) == <span class="hljs-string">'    '</span>) {             manpop = Integer.parseInt(line.substring(<span class="hljs-number">8</span>, <span class="hljs-number">14</span>));         }                  String quality = line.substring(<span class="hljs-number">56</span>, <span class="hljs-number">57</span>);         <span class="hljs-keyword">if</span> (manpop != MISSING && quality.matches(<span class="hljs-string">"[30]"</span>)) {             context.write(<span class="hljs-keyword">new</span> Text(year), <span class="hljs-keyword">new</span> IntWritable(manpop));         }     } } </code>

reducer class

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37

<code class="hljs"><span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">PopulationReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title">Reducer</span><<span class="hljs-title">Text</span>, <span class="hljs-title">IntWritable</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">DoubleWritable</span>> </span>{          <span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title">reduce</span><span class="hljs-params">(Text key, Iterable values, Context context)</span>             <span class="hljs-keyword">throws</span> IOException, InterruptedException </span>{         <span class="hljs-keyword">int</span> minValue = Integer.MAX_VALUE;         <span class="hljs-keyword">int</span> maxValue = Integer.MIN_VALUE;         <span class="hljs-keyword">int</span> number = <span class="hljs-number">0</span>;         <span class="hljs-keyword">int</span> sum = <span class="hljs-number">0</span>;         ArrayList valueList = <span class="hljs-keyword">new</span> ArrayList();                    <span class="hljs-keyword">for</span> (IntWritable value : values) {             minValue = Math.min(minValue, value.get());             maxValue = Math.max(maxValue, value.get());             sum += value.get();             valueList.add(value.get());             number++;         }                  <span class="hljs-keyword">double</span> value1 = <span class="hljs-number">0</span>;         <span class="hljs-keyword">double</span> value2 = <span class="hljs-number">0</span>;         <span class="hljs-keyword">for</span>(<span class="hljs-keyword">int</span> i = <span class="hljs-number">0</span>; i<number; i++){             value1 = (<span class="hljs-keyword">double</span>)valueList.get(i)-sum/number;             value2 = Math.pow(value1, <span class="hljs-number">2</span>)+value2;         }                       context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(minValue));         context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(maxValue));         context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(sum / number));         context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(Math.sqrt(value2/number)));     }             } </code>