win7下idea遠程鏈接hadoop，運行wordCount org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.Acc

1.將hadoop-2.6.1.tar.gz解壓到本地

配置環境變量

HADOOP_HOME 

E:\kaifa\hadoop-2.6.1\hadoop-2.6.1

HADOOP_BIN_PATH

%HADOOP_HOME%\bin

HADOOP_PREFIX

%HADOOP_HOME%

配置path

E:\kaifa\jdk1.7.0_21\bin;%HADOOP_HOME%\bin;%HADOOP_HOME%\sbin;

 

2.用idea新建一個maven項目

導入hadoop依賴包

File>Project Structure>Project Settings>Libraries，點+號而後選擇Java，而後選擇解壓出來的hadoop-2.6.1文件夾下share\hadoop\下的jar包

 

 

 

 

pom.xml配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
 
    <groupId>com.hadoop261</groupId>
    <artifactId>myhadoop</artifactId>
    <version>1.0-SNAPSHOT</version>
 
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
            <version>2.6.1</version>
        </dependency>
        <dependency>
            <groupId>commons-cli</groupId>
            <artifactId>commons-cli</artifactId>
            <version>1.2</version>
        </dependency>
    </dependencies>
 
    <build>
        <finalName>${project.artifactId}</finalName>
    </build>

 

WcMapper.java

package hadoop.test;


import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
import java.util.StringTokenizer;

public class WcMapper extends Mapper<LongWritable,Text,Text,IntWritable>{
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
// IntWritable one=new IntWritable(1);
String line=value.toString();
StringTokenizer st=new StringTokenizer(line);
//StringTokenizer "kongge"
while (st.hasMoreTokens()){
String word= st.nextToken();
context.write(new Text(word),new IntWritable(1)); //output
}
}
}

 

McReducer.java

package hadoop.test;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;

/**
* Created by iespark on 2/26/16.
*/
public class McReducer extends Reducer<Text,IntWritable,Text,IntWritable> {
@Override
protected void reduce(Text key, Iterable<IntWritable> iterable, Context context) throws IOException, InterruptedException {
int sum=0;
for (IntWritable i:iterable){
sum=sum+i.get();
}
context.write(key,new IntWritable(sum));
}
}

 

JobRun.java

package hadoop.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
* Created by iespark on 2/26/16.
*/
public class JobRun {
public static void main(String[] args){
Configuration conf=new Configuration();
try{
Job job = Job.getInstance(conf, "word count");
onfiguration conf, String jobName
job.setJarByClass(JobRun.class);
job.setMapperClass(WcMapper.class);
job.setReducerClass(McReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//解決No job jar file set. User classes may not be found. See Job or Job#setJar(String)報錯的問題
job.setJar("E:\\idea2017workspace\\myhadoop\\out\\artifacts\\myhadoop_jar\\myhadoop.jar");

FileInputFormat.addInputPath(job,new Path(args[0]));
FileSystem fs= FileSystem.get(conf);
Path op1=new Path(args[1]);
if(fs.exists(op1)){
fs.delete(op1, true);
System.out.println("存在此輸出路徑，已刪除！！！");
}
FileOutputFormat.setOutputPath(job,op1);
System.exit(job.waitForCompletion(true)?0:1);
}catch (Exception e){
e.printStackTrace();
}
}
}

 

3.設置jar包的生成位置

 

 

注意最下面的Main Class別忘記選擇

而後把這個路徑放在JobRun.jar中的

job.setJar("E:\\idea2017workspace\\myhadoop\\out\\artifacts\\myhadoop_jar\\myhadoop.jar");

 

3.在sources文件夾中新增core-site.xml和log4j.properties文件

 

 

core-site.xml 配置內容和你的hadoop集羣的配置同樣

 

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<!--配置namenode的地址-->

<property>
<name>fs.defaultFS</name>
<value>hdfs://10.102.19.229:9000</value>
</property>
<!-- 指定hadoop運行時產生文件的存儲目錄 -->
<property>
<name>hadoop.tmp.dir</name>
<value>file:///data/hadoop/data/tmp</value>
</property>
</configuration>

 

log4j.properties文件能夠將hadoop-2.6.1下的hadoop-2.6.1\etc\hadoop中的log4j.properties複製過來，不用修改!

4.配置運行參數

咱們能夠直接在JobRun.java右鍵運行，可是還少兩個參數，一個是輸入路徑，一個是輸出路徑，下面Program arguments中的第一個是輸入路徑，第二個是輸出路徑。

注意：1.這兩個路徑中間要用空格隔開！

    2.這兩個路徑都是你hadoop上hdfs文件系統中的路徑，不是你win7本地的路徑！

    3.輸入路徑的a.txt是你要處理的文件，須要本身新建：hadoop fs -mkdir /input    hadoop fs -put ./a.txt /input

個人a.txt的內容是：

speak good cloud speek good good cloud speak english
EOF

運行結果就是統計每一個單詞出現了幾回

 

 

5.運行

看到下面的輸出，咱們就成功遠程到了linux上的hadoop並執行了wordCount程序！

 

 

咱們到linux控制檯查看運行結果：

能夠看到有了output文件夾（input是咱們本身建的）

 

 

 

 

咱們能夠用

hadoop fs -ls /output

繼續查看，看到有兩個文件，part-r-00000就是運行的結果！

 

 

 

咱們用

hadoop fs -cat /output/part-r-00000
查看執行結果

 

舒適提醒：爲了以防空指針等一些莫名其妙的的錯誤在此處須要把你的hadoop的配置文件裏面的core-site.xml、hdfs-site.xml和log4j.properties複製過來放在你的 src目錄下。而後在開始運行你的程序了。在此處咱們準備作測試的文件放在集羣根目錄下的data下。所需須要在你的hdfs文件系統的根目錄建立data文件夾，並上傳你要的測試文件。

org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException): Permission denied: user=Administrator, access=WRITE, inode="/":root:supergroup:drwxr-xr報錯