【Hadoop] - windows開發環境搭建

文章說明：因Linux平臺再GUI頁面經過IDE進行Hadoop開發，會致使Linux在GUI上極度消耗資源，對於一些配置不是很高的PC，可能會出現卡頓的狀況，很是影響程序編寫，本文就詳細介紹如何在windows平臺進行hadoop開發，但願對各位學習Hadoop的同窗優異

---

工具：

• hadoop eclipse插件：hadoop-eclipse-plugin-2.7.3.jar
• hadoop windows平臺支持組件：winutils.exe
• hadoop 底層依賴庫：hadoop.dll

上述工具下載地址：下載地址

Hadoop版本 : hadoop-2.7.3

---

配置步驟：

1. 啓動hadoop : start-yarn.sh、start-dfs.sh

2. windows本地配置Linux的主機IP映射：（不配置直接使用IP也行）

3. 將hadoop-eclipse-plugin-2.7.3.jar放進eclipse的plugins目錄，啓動eclipse

4. eclipse配置Hadoop

5. 切換MapReduce視圖能夠看到HDFS文件系統的信息

---

運行MapReduce程序

1. 配置HADOOP環境變量：主要將bin、sbin放入PATH路徑
2. 將winutils.exe放在bin目錄，hadoop.dll放在windows System32目錄
3. 測試代碼

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * 統計文本詞頻信息
 * @author Zerone1993
 */
public class WordCount {

	static class WordMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
				throws IOException, InterruptedException {
			
			String str = value.toString();
			
			StringTokenizer st = new StringTokenizer(str);
			
			while(st.hasMoreTokens()){
				
				String temp = st.nextToken();
				
				context.write(new Text(temp), new IntWritable(1));
			}
		}
	}
	
	static class WordReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
	       
		    @Override
	        protected void reduce(Text arg0, Iterable<IntWritable> arg1,
	        		Reducer<Text, IntWritable, Text, IntWritable>.Context arg2) throws IOException, InterruptedException {

	                 int sum = 0;
	                 
	                 for(IntWritable temp : arg1){
	                	 
	                	 sum = sum + temp.get();
	                 
	                 }
	                 
	                 arg2.write(new Text(arg0), new IntWritable(sum));
	         }
	}
	
	public static void main(String[] args) {
		
		Configuration conf = new Configuration();
		
		conf.set("mapred.job.tracker", "master:50020");
		
		try{
		
		Job job = Job.getInstance(conf, "wordCount");
		job.setJarByClass(WordCount.class); //設置啓動做業類
		job.setMapperClass(WordMapper.class); //設置Map類
		job.setReducerClass(WordReducer.class);
		job.setMapOutputKeyClass(Text.class); //設置mapper輸出的key類型
		job.setMapOutputValueClass(IntWritable.class); //設置mapper輸出的value類型
		job.setNumReduceTasks(1); //設置Reduce Task的數量
		
		//設置mapreduce的輸入和輸出目錄
		FileInputFormat.addInputPath(job, new Path("hdfs://master:9090/user/squirrel/input/mapreduce/"));
		FileOutputFormat.setOutputPath(job, new Path("hdfs://master:9090/user/squirrel/output/mapreduce/") );
		
		//等待mapreduce整個過程完成
		System.exit(job.waitForCompletion(true)?0:1);
		
		}catch(Exception e){
			e.printStackTrace();
		}
	}
}

效果：