在eclipse使用map reduce編寫word count程序生成jar包並在虛擬機運行的步驟

---恢復內容開始---

1.首先準備一個須要統計的單詞文件 word.txt，咱們的單詞是以空格分開的，統計時按照空格分隔便可

hello hadoop
hello yarn
hello zookeeper
hdfs hadoop
select from hadoop
select from yarn
mapReduce
MapReduce

2.上傳word.txt到hdfs根目錄

$ bin/hdfs dfs -put test/word.txt /

3.準備工做完成後在eclipse編寫代碼，分別編寫Map、Reduce、Driver等Java文件

WordCountMap.java

map執行咱們的word.txt 文件是按行執行，每一行執行一個map

WordCountMap.java

map執行咱們的word.txt 文件是按行執行，每一行執行一個map

package com.ijeffrey.mapreduce.wordcount.client;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/**
* map 輸出的鍵值對必須和reducer輸入的鍵值對類型一致
* @author PXY
*
*/
public class WordCountMap extends Mapper<LongWritable, Text, Text, IntWritable> {

private Text keyout = new Text();
private IntWritable valueout = new IntWritable(1);

@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {

String line = value.toString();
// 個人文件記錄的單詞是以空格記錄單詞，因此這裏用空格來截取
String[] words = line.split(" ");

// 遍歷數組，並以k v 對的形式輸出
for (String word : words) {
keyout.set(word);
context.write(keyout, valueout);
}
}

}

WordCountReducer.java

package com.ijeffrey.mapreduce.wordcount.client;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

/**
* reducer 輸入的鍵值對必須和map輸出的鍵值對類型一致
* map <hello,1> <world,1> <hello,1> <apple,1> ....
* reduce 接收 <apple,[1]> <hello,[1,1]> <world,[1]>
* @author PXY
*
*/
public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable valueout = new IntWritable();

@Override
protected void reduce(Text key, Iterable<IntWritable> values,
Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
int count = 0; // 統計總數

// 遍歷數組，累加求和
for(IntWritable value : values){

// IntWritable類型不能和int類型相加，因此須要先使用get方法轉換成int類型
count += value.get();
}

// 將統計的結果轉成IntWritable
valueout.set(count);

// 最後reduce要輸出最終的 k v 對
context.write(key, valueout);

}
}

WordCountDriver.java

package com.ijeffrey.mapreduce.wordcount.client;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
* 運行主函數
* @author PXY
*
*/
public class WordCountDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();

// 得到一個job對象，用來完成一個mapreduce做業
Job job = Job.getInstance(conf);

// 讓程序找到主入口
job.setJarByClass(WordCountDriver.class);

// 指定輸入數據的目錄，指定數據計算完成後輸出的目錄
// sbin/yarn jar share/hadoop/xxxxxxx.jar wordcount /wordcount/input/ /wordcount/output/
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));

// 告訴我調用那個map方法和reduce方法
job.setMapperClass(WordCountMap.class);
job.setReducerClass(WordCountReducer.class);

// 指定map輸出鍵值對的類型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);

// 指定reduce輸出鍵值對的類型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);

// 提交job任務
boolean result = job.waitForCompletion(true);
System.exit(result ? 0 : 1);

}
}

}

4.將編寫完成的代碼打成jar包,並在集羣上運行

將jar上傳到到服務器，啓動服務後運行咱們本身編寫的MapReduce，統計根目錄下的word.txt並將運行結果寫入output

$ bin/yarn jar test/wordCount.jar com.ijeffrey.mapreduce.wordcount.client.WordCountDriver /word.txt /output

注意：運行jar的時候要添加Driver的徹底路徑

運行完成後查看output結果:

$ bin/hdfs dfs -text /output12/part-r-00000