posted by 코딩 공부중 2020. 1. 3. 14:41
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
<code class="hljs">  <span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">Population</span> </span>{
    
<span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">void</span> <span class="hljs-title">main</span><span class="hljs-params">(String[] args)</span> <span class="hljs-keyword">throws</span> Exception </span>{
        
        <span class="hljs-keyword">if</span> (args.length != <span class="hljs-number">2</span>) {
            System.err.println(<span class="hljs-string">"Usage:Population  "</span>);
            System.exit(-<span class="hljs-number">1</span>);
        }
        
        Job job = <span class="hljs-keyword">new</span> Job();
        job.setJarByClass(Population.class);
        job.setJobName(<span class="hljs-string">"Population"</span>);
        
        job.setMapperClass(PopulationMapper.class);
        job.setReducerClass(PopulationReducer.class);
        
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        
        FileInputFormat.addInputPath(job, <span class="hljs-keyword">new</span> Path(args[<span class="hljs-number">0</span>]));
        FileOutputFormat.setOutputPath(job, <span class="hljs-keyword">new</span> Path(args[<span class="hljs-number">1</span>]));
        System.exit(job.waitForCompletion(<span class="hljs-keyword">true</span>) ? <span class="hljs-number">0</span> : <span class="hljs-number">1</span>);
    }
 
}
</code>

mapper class

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
<code class="hljs"><span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">PopulationMapper</span> <span class="hljs-keyword">extends</span> <span class="hljs-title">Mapper</span><<span class="hljs-title">LongWritable</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">IntWritable</span>> </span>{
    
<span class="hljs-keyword">private</span> <span class="hljs-keyword">static</span> <span class="hljs-keyword">final</span> <span class="hljs-keyword">int</span> MISSING=<span class="hljs-number">9999</span>;
    
    <span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title">map</span><span class="hljs-params">(LongWritable key, Text value, Context context)</span> <span class="hljs-keyword">throws</span> IOException, InterruptedException </span>{
        String line = value.toString();
        <span class="hljs-comment">//line = line.replaceAll(" ", "");</span>
        String year = line.substring(<span class="hljs-number">0</span>,<span class="hljs-number">4</span>);
        <span class="hljs-keyword">int</span> manpop = <span class="hljs-number">0</span>;
        
        <span class="hljs-keyword">if</span>(line.charAt(<span class="hljs-number">7</span>) == <span class="hljs-string">'    '</span>) {
            manpop = Integer.parseInt(line.substring(<span class="hljs-number">8</span>, <span class="hljs-number">14</span>));
        }
        
        String quality = line.substring(<span class="hljs-number">56</span>, <span class="hljs-number">57</span>);
        <span class="hljs-keyword">if</span> (manpop != MISSING && quality.matches(<span class="hljs-string">"[30]"</span>)) {
            context.write(<span class="hljs-keyword">new</span> Text(year), <span class="hljs-keyword">new</span> IntWritable(manpop));
        }
    }
}
</code>

reducer class

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
<code class="hljs"><span class="hljs-keyword">public</span> <span class="hljs-class"><span class="hljs-keyword">class</span> <span class="hljs-title">PopulationReducer</span> <span class="hljs-keyword">extends</span> <span class="hljs-title">Reducer</span><<span class="hljs-title">Text</span>, <span class="hljs-title">IntWritable</span>, <span class="hljs-title">Text</span>, <span class="hljs-title">DoubleWritable</span>> </span>{
    
    <span class="hljs-function"><span class="hljs-keyword">public</span> <span class="hljs-keyword">void</span> <span class="hljs-title">reduce</span><span class="hljs-params">(Text key, Iterable values, Context context)</span>
            <span class="hljs-keyword">throws</span> IOException, InterruptedException </span>{
        <span class="hljs-keyword">int</span> minValue = Integer.MAX_VALUE;
        <span class="hljs-keyword">int</span> maxValue = Integer.MIN_VALUE;
        <span class="hljs-keyword">int</span> number = <span class="hljs-number">0</span>;
        <span class="hljs-keyword">int</span> sum = <span class="hljs-number">0</span>;
        ArrayList valueList = <span class="hljs-keyword">new</span> ArrayList();
        
 
        <span class="hljs-keyword">for</span> (IntWritable value : values) {
            minValue = Math.min(minValue, value.get());
            maxValue = Math.max(maxValue, value.get());
            sum += value.get();
            valueList.add(value.get());
            number++;
        }
        
        <span class="hljs-keyword">double</span> value1 = <span class="hljs-number">0</span>;
        <span class="hljs-keyword">double</span> value2 = <span class="hljs-number">0</span>;
        <span class="hljs-keyword">for</span>(<span class="hljs-keyword">int</span> i = <span class="hljs-number">0</span>; i<number; i++){
            value1 = (<span class="hljs-keyword">double</span>)valueList.get(i)-sum/number;
            value2 = Math.pow(value1, <span class="hljs-number">2</span>)+value2;
        }
        
    
        context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(minValue));
        context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(maxValue));
        context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(sum / number));
        context.write(key, <span class="hljs-keyword">new</span> DoubleWritable(Math.sqrt(value2/number)));
    }
    
    
 
}
</code>

'빅데이터' 카테고리의 다른 글

[mapreduce]교통사고 발생건수 통계  (0) 2020.01.03
[mapreduce]ncdc 연도별 기온 통계  (0) 2020.01.03
[mapreduce] wordcount  (0) 2020.01.03