hadoop 分佈式部署

一、節點：
192.168.100.40 node04.magedu.com node04 node04
192.168.100.30 node03.magedu.com node03 node03
192.168.100.20 node02.magedu.com node02 node02
192.168.100.10 node01.magedu.com node01 node01

二、配置java 環境
一、yum install java-1.7.0-openjdk-devel
二、vim /etc/profile.d/java.sh
export JAVA_HOME=/usr
#. /etc/profile.d/java.sh source /etc/profile.d/java.sh
####
pssh -ih iplist 'yum install java-1.7.0-openjdk-devel -y'
pssh -ih iplist 'echo export JAVA_HOME=/usr > /etc/profile.d/java.sh'
pssh -ih iplist '. /etc/profile.d/java.sh'
[root@node01 ~]# java -version

三、時間同步：
vim /etc/chrony.conf
server 192.168.100.10 iburst
allow 192.168.100.0/24

systemctl is-enabled chronyd
systecmtl enable chronyd
systemctl start chronyd
clock -w 軟時間同步應時間
clock -s 硬件時間同步軟時間

pssh -ih iplist 'date'

四、配置hadoop.sh
[root@node01~]#cat /etc/profile.d/hadoop.sh
export HADOOP_PREFIX=/bdapps/hadoop
export PATH=$PATH:${HADOOP_PREFIX}/bin:${HADOOP_PREFIX}/sbin
export HADOOP_YARN_HOME=${HADOOP_PREFIX}
export HADOOP_MAPPERD_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
. /etc/profile.d/hadoop.sh
for i in node0{2..4} ;do scp /etc/profile.d/java.sh /etc/profile.d/hadoop.sh $i:/etc/profile.d/;
pssh -ih iplist ". /etc/profile.d/java.sh ; . /etc/profile.d/hadoop.sh "

五、建立用戶：
groupadd hadoop
useradd hadoop(4個節點同一用戶)
echo 'magedu' |passwd --stdin hadoop
useradd -g hadoop hadoop
su - hadoop

pssh -ih iplist "groupadd hadoop"
pssh -ih iplist "useradd -g hadoop hadoop"
pssh -ih iplist "echo 'centos' |passwd --stdin hadoop "

六、hadoop 用戶免密：
su - hadoop
for i in 1 2 3 4 ; do ssh-copy-id -i .ssh/id_rsa.pub hadoop@node0${i}; done

七、visudo

useradd -g wheel hadoop
usermod -aG wheel hadoop
pssh -ih iplist 'usermod -aG wheel hadoop'
pssh -ih iplist 'id hadoop'

八、建立數據目錄及配置目錄
[root@node01 ~]#pssh -ih iplist 'mkdir -pv /bdapps /data/hadoop/hdfs/{nn,snn,dn}'
[root@node01 ~]# pssh -ih iplist 'chown -R hadoop:hadoop /data/hadoop/hdfs'
[root@node01 ~]# pssh -ih iplist 'tar xf hadoop-2.7.3.tar.gz -C /bdapps'
[root@node01 ~]# pssh -ih iplist ' ln -sv /bdapps/hadoop-2.7.3 /bdapps/hadoop'
[root@node01 ~]# pssh -ih iplist ' ls -l /bdapps/hadoop '
[root@node01 ~]# pssh -ih iplist ' cd /bdapps/hadoop ; mkdir logs;chmod g+w logs'
[root@node01 ~]# pssh -ih iplist ' chown -R hadoop:hadoop /bdapps'
[root@node01 ~]# pssh -ih iplist ' ls -ld /bdapps/hadoop/logs'
scp /bdapps/hadoop/etc/hadoop/* node3:/bdapps/hadoop/etc/hadoop

9 、hdfs 文件配置：
[root@node01 ~]# cd /bdapps/hadoop/etc/hadoop/
[root@node01 hadoop]# cat core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://node01:8020</value>
<final>true</final>
</property>
</configuration>
[root@node01 hadoop]# cat hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>

<property>
            <name>dfs.namenode.name.dir</name>
            <value>file:///data/hadoop/hdfs/nn</value>
    </property>

    <property>
            <name>dfs.datanode.data.dir</name>
            <value>file:///data/hadoop/hdfs/dn</value>
    </property>
    <property>
    <name>fs.checkpoint.dir</name>
            <value>file:///data/hadoop/hdfs/snn</value>
    </property>

    <property>
    <name>fs.checkpoint.edits.dir</name>
            <value>file:///data/hadoop/hdfs/snn</value>
    </property>

</configuration>

[root@node01 hadoop]# cat mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

</configuration>

[root@node01 hadoop]# cat yarn-site.xml
<configuration>
<property>
<name>yarn.resourcemanager.address</name>
<value>node01:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>node01:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>node01:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>node01:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>node01:8088</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.auxservices.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
</property>

</configuration>

十、拷貝配置文件：
[root@node01 hadoop]# su - hadoop
cd /bdapps/hadoop/etc/hadoop/
[hadoop@node01 hadoop]$ for i in node0{2..4}; do scp core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml $i:/bdapps/hadoop/etc/hadoop/; done
[root@node01 ~]# pssh -ih iplist 'ls /bdapps/hadoop/etc/hadoop/ -l '

十一、格式化：
su - hadoop

hdfs namenode -format

十二、啓動服務：
啓動hadoop：
namenode:hadoop-daemon.sh(start/stop) namenode
datanode:hadoop-daemon.sh(start|stop) datanode
secondary namenode: hadoop-daemon.sh(start|stop)secondarynamenode

Jps：查看java 進程； jps -v

resourcemanager:yarn-daemon.sh(start|stop)resourcemanager

nodemanager:yarn-daemon.sh(start|stop) nodemanager

###################
[hadoop@node01 mapreduce]$ cd /bdapps/hadoop/sbin/
[hadoop@node01 sbin]$ ls
distribute-exclude.sh kms.sh start-balancer.sh stop-all.cmd stop-yarn.cmd
hadoop-daemon.sh mr-jobhistory-daemon.sh start-dfs.cmd stop-all.sh stop-yarn.sh
hadoop-daemons.sh refresh-namenodes.sh start-dfs.sh stop-balancer.sh yarn-daemon.sh
hdfs-config.cmd slaves.sh start-secure-dns.sh stop-dfs.cmd yarn-daemons.sh
hdfs-config.sh start-all.cmd start-yarn.cmd stop-dfs.sh
httpfs.sh start-all.sh start-yarn.sh stop-secure-dns.sh
[hadoop@node01 sbin]$ stop-yarn.sh stop-dfs.sh start-dfs.sh start-yarn.sh
####################

1三、hadoop 測試
hdfs dfs -mkdir /test
hdfs dfs -put /etc/fstab /test/fstab

hdfs dfs -cat /test/fstab

hdfs dfs -put /etc/rc.d/init.d/functions /test/

hdfs dfs -put /etc/issue hdfs://master/test/

1四、排錯：
一、dn 與nn 一直刷嘗試鏈接：
[hadoop@node02 dn]$ tail /bdapps/hadoop/logs/hadoop-hadoop-datanode-node02.magedu.com.log
2020-11-08 19:34:49,420 INFO org.apache.hadoop.ipc.Client: Retrying connect to server: node01/192.168.100.10:8020. Already tried 2 time(s); retry policy is RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
沒有關閉防火牆：
sudo systemctl stop firewalld

二、nn節點 格式化：致使cluster id 與 dn clusterid 不一致：

[hadoop@node01 logs]$ tail hadoop-hadoop-datanode-node01.magedu.com.log -n 30
2020-11-08 19:14:12,537 WARN org.apache.hadoop.hdfs.server.common.Storage: Failed to add storage directory [DISK]file:/data/hadoop/hdfs/dn/
java.io.IOException: Incompatible clusterIDs in /data/hadoop/hdfs/dn: namenode clusterID = CID-98ce0fbd-2d2b-4fc4-a461-df4fbd2181de; datanode clusterID = CID-3148d552-2e5a-4a66-8a2f-5275bd1be8da
cat /data/hadoop/hdfs/dn/current/VERSION

三、nn 建立副本 數據流50010 鈍口禁止：因防火牆沒有關閉：

1五、webgui
web ui 概覽
hadoop: $hostname:50070
yarn:firefox localhost:8088

1六、
http://192.168.100.10:50070/dfshealth.html#tab-overview

[hadoop@node01 hadoop]$ hdfs dfsadmin -report

cd /data/hadoop/hdfs/dn/current/BP-28146739-192.168.100.10-1604886639374/current/finalized/subdir0/subdir0/

yarn:8088

yarn測試：
[hadoop@node01 mapreduce]$ pwd
/bdapps/hadoop/share/hadoop/mapreduce
[hadoop@node01 mapreduce]$ yarn jar /bdapps/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar

[hadoop@node01 mapreduce]$ hdfs dfs -ls /test/fstab.out/
Found 2 items
-rw-r--r-- 2 hadoop supergroup 0 2020-11-08 20:29 /test/fstab.out/_SUCCESS
-rw-r--r-- 2 hadoop supergroup 432 2020-11-08 20:29 /test/fstab.out/part-r-00000

[hadoop@node01 mapreduce]$ hdfs dfs -cat /test/fstab.out/part-r-00000
'/dev/disk' 1
/ 1
/boot 1
/etc/fstab 1
0 6
08:19:43 1
2020 1
5 1
Accessible 1
Created 1
Nov 1
See 1
Thu 1

yarn :

[hadoop@node01 sbin]$ yarn application -list -appStates=ALL
20/11/08 20:40:02 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Total number of applications (application-types: [] and states: [NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED]):1
Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL
application_1604890022094_0001 word count MAPREDUCE hadoop default FINISHED SUCCEEDED 100% http://node01.magedu.com:19888/jobhistory/job/job_1604890022094_0001
[hadoop@node01 sbin]$ yarn application -list
20/11/08 20:40:26 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Total number of applications (application-types: [] and states: [SUBMITTED, ACCEPTED, RUNNING]):0
Application-Id Application-Name Application-Type User Queue State Final-State Progress Tracking-URL
[hadoop@node01 sbin]$

[hadoop@node01 sbin]$ yarn application -status application_1604890022094_0001
20/11/08 20:44:49 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Application Report :
Application-Id : application_1604890022094_0001
Application-Name : word count
Application-Type : MAPREDUCE
User : hadoop
Queue : default
Start-Time : 1604896147557
Finish-Time : 1604896174398
Progress : 100%
State : FINISHED
Final-State : SUCCEEDED
Tracking-URL : http://node01.magedu.com:19888/jobhistory/job/job_1604890022094_0001
RPC Port : 39072
AM Host : node01.magedu.com
Aggregate Resource Allocation : 81943 MB-seconds, 46 vcore-seconds
Diagnostics :

-kill

[hadoop@node01 sbin]$ yarn application -kill application_1604890022094_0001
20/11/08 20:47:24 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Application application_1604890022094_0001 has already finished

node
#############
yarn node -list
yarn node -status $node-id
yarn logs -applicationId application_1444_01
yarn classpath
RMAdmin
yarn rmadmin -refreshNodes
DaemonLog；
####################

[hadoop@node01 sbin]$ yarn node -list
20/11/08 20:49:48 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Total Nodes:4
Node-Id Node-State Node-Http-Address Number-of-Running-Containers
node01.magedu.com:44801 RUNNING node01.magedu.com:8042 0
node02.magedu.com:35211 RUNNING node02.magedu.com:8042 0
node03.magedu.com:38156 RUNNING node03.magedu.com:8042 0
node04.magedu.com:38078 RUNNING node04.magedu.com:8042 0

[hadoop@node01 sbin]$ yarn node -status node01.magedu.com:44801
20/11/08 20:51:53 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
Node Report :
Node-Id : node01.magedu.com:44801
Rack : /default-rack
Node-State : RUNNING
Node-Http-Address : node01.magedu.com:8042
Last-Health-Update : Sun 08/Nov/20 08:50:39:428PST
Health-Report :
Containers : 0
Memory-Used : 0MB
Memory-Capacity : 8192MB
CPU-Used : 0 vcores
CPU-Capacity : 8 vcores
Node-Labels :

[hadoop@node01 sbin]$ yarn logs -applicationId application_1604890022094_0001
20/11/08 20:53:55 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8032
/tmp/logs/hadoop/logs/application_1604890022094_0001 does not exist.
Log aggregation has not completed or is not enabled.

[hadoop@node01 sbin]$ yarn rmadmin -refreshNodes
20/11/08 20:55:51 INFO client.RMProxy: Connecting to ResourceManager at node01/192.168.100.10:8033

運行yarn application