CentOS 6.5 編譯Eclipse plugin for hadoop 2.5及加載鏈接

1. rpm -ivh jdk-7u67-linux-x64.rpm

cd /opt
tar zxf /root/hadoop-2.2.0.tar.gz
tar zxf /root/apache-ant-1.9.4-bin.tar.gz
tar zxf /root/apache-maven-3.2.3-bin.tar.gz
tar zxf /root/eclipse-java-luna-SR1-linux-gtk-x86_64.tar.gz
unzip -x /root/hadoop2x-eclipse-plugin-master.zip (https://github.com/winghc/hadoop2x-eclipse-plugin)
ln -s apache-ant-1.9.4 ant
ln -s apache-maven-3.2.3 maven
ln -s hadoop-2.2.0 hadoop

2. echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
echo 'export HADOOP_HOME=/opt/hadoop' >> /etc/profile
echo 'export ANT_HOME=/opt/ant' >> /etc/profile
echo 'export MAVEN_HOME=/opt/maven' >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$MAVEN_HOME/bin:$HADOOP_HOME/bin:$PATH' >> /etc/profile
source /etc/profile

3. cp /root/ivy-2.1.0.jar /opt/hadoop2x-eclipse-plugin-master/ivy/
cd /opt/hadoop2x-eclipse-plugin-master/src/contrib/eclipse-plugin
ant jar -Dversion=2.5.0 -Declipse.home=/opt/eclipse -Dhadoop.home=/opt/hadoop

cp /opt/hadoop2x-eclipse-plugin-master/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.0.jar /opt/eclipse/plugins/

4. run eclipse: /opt/eclipse/eclipse (此目錄權限最好修改成：chmod -R 755 /opt/eclipse; chown -R hdfs:hadoop /opt/eclipse)

鏈接hadoop環境爲：CDH 5.2 on a Single Linux Node in Pseudo-distributed Mode（安裝參考附錄）

5. Windows 7同樣可以使用這個hadoop-eclipse-plugin-2.5.0.jar

附錄：

Installing CDH 5 with YARN on a Single Linux Node in Pseudo-distributed mode

分區考慮，不要使用LVM
root -- >50G
opt -- >50G
swap -- 2倍系統內存
RAM -- >8GB

1. disable selinux and iptables
service iptables stop
chkconfig iptables off; chkconfig ip6tables off

setenforce 0
sed -i 's,SELINUX=enforcing,SELINUX=disabled,g' /etc/selinux/config

2. disable ipv6 and kernel parameters tuning
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf

echo "vm.swappiness = 0" >> /etc/sysctl.conf

echo 'net.ipv4.tcp_retries2 = 2' >> /etc/sysctl.conf
echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf
echo "fs.file-max = 6815744" >> /etc/sysctl.conf
echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf
echo "net.core.rmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.wmem_default = 262144" >> /etc/sysctl.conf
echo "net.core.rmem_max = 16777216" >> /etc/sysctl.conf
echo "net.core.wmem_max = 16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_rmem = 4096 262144 16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_wmem = 4096 262144 16777216" >> /etc/sysctl.conf

only on ResourceManager and JobHistory Server
echo "net.core.somaxconn = 1000" >> /etc/sysctl.conf

sysctl -p

echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled" >> /etc/rc.local
echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag" >> /etc/rc.local
echo "echo no > /sys/kernel/mm/redhat_transparent_hugepage/khugepaged/defrag" >> /etc/rc.local

3. vi /etc/hosts to add all hosts FQDN, like below:
192.168.1.10    test.gkytech.com test

vi /etc/sysconfig/network to set FQDN

cat << EOF > /etc/yum.repos.d/iso.repo
[iso]
name=iso
baseurl=http://192.168.1.10/centos65
enable=1
gpgcheck=0
EOF

yum -y install ntp openssh-clients lzo

service ntpd start; chkconfig ntpd on

vi /etc/security/limits.conf

*         soft    core         unlimited
*         hard    core         unlimited
*         soft    nofile       65536
*         hard    nofile       65536
*         soft    nproc        unlimited
*         hard    nproc        unlimited
*         soft    memlock      unlimited
*         hard    memlock      unlimited

vi /etc/grub.conf
add "elevator=deadline"(no quotes) at the end of kernel line

reboot to take effect

4. cat << EOF > /etc/yum.repos.d/cdh520.repo
[cm520]
name=cm520
baseurl=http://192.168.1.10/cdh520
enable=1
gpgcheck=0
EOF

rpm -ivh jdk-7u67-linux-x64.rpm

echo 'export JAVA_HOME=/usr/java/default' >> /etc/profile
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
source /etc/profile

# To install Hadoop with YARN
yum -y install hadoop-conf-pseudo

# Step 1: Format the NameNode
sudo -u hdfs hdfs namenode -format

# Step 2: Start HDFS
for x in `cd /etc/init.d ; ls hadoop-hdfs-*` ; do service $x start ; done

# Step 3: Create the directories needed for Hadoop processes
/usr/lib/hadoop/libexec/init-hdfs.sh

# Step 4: Verify the HDFS File Structure
sudo -u hdfs hadoop fs -ls -R /

# Step 5: Start YARN

optional: to replace localhost with ip address, so you can access web ui

vi /etc/hadoop/conf/core-site.xml

vi /etc/hadoop/conf/mapred-site.xml

service hadoop-yarn-resourcemanager startservice hadoop-yarn-nodemanager startservice hadoop-mapreduce-historyserver start# Step 6: Create User Directoriesuseradd testsudo -u hdfs hadoop fs -mkdir /user/testsudo -u hdfs hadoop fs -chown test /user/test# Step 7: Testingsu - testhadoop jar /usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 10 100