CentOS6非root用戶下安裝及配置CDH5.3.0

#install lsb package
sudo yum install -y redhat-lsb

#install net-tools package

sudo yum install -y net-tools
#sudo時不須要密碼(NOPASSWD前是\t)
sudo vim /etc/sudoers
nanjing ALL =(ALL)  NOPASSWD: ALL
#Set the hostname to a unique name (not localhost).
modify HOSTNAME
sudo vi /etc/sysconfig/network

sudo vi /etc/hosts

172.16.1.121 crawler1
172.16.1.122 crawler2
172.16.1.123 crawler3
172.16.1.124 controller1
172.16.1.125 master1
172.16.1.126 slave1
172.16.1.127 slave2
172.16.1.128 zombie1


#stop iptables and disable the service

sudo service iptables stop
sudo chkconfig --level 2345 iptables off


#download and install jdk 1.8.0_11
https://pan.baidu.com/s/1miHbW6k

sudo setenforce 0
sudo vi /etc/selinux/config 下的 SELINUX=disabled
sudo /usr/sbin/sestatus -v

#再controller1上從ftp服務器下載全部cdh5的包
#install CDH Manager5
Pre-requisites: multiple, Internet-connected Linux machines, with SSH access, and significant free space in /var and /opt.

wget http://archive.cloudera.com/cm5/installer/latest/cloudera-manager-installer.bin

chmod u+x cloudera-manager-installer.bin

#再controller1上
#Creating and Using a Package Repository for Cloudera Manager
#Installing Apache HTTPD。
sudo yum install -y httpd
sudo service httpd start
sudo chkconfig --level 2345 httpd on





cp cm5.3.0-centos6.tar /var/www/html/
cd /var/www/html/
sudo tar xzvf cm5.3.0-centos6.tar

#在master1上
sudo vi /etc/yum.repos.d/cm5.repo

[cm5]    
name=cm5
baseurl=http://172.16.1.124/cm/5
enabled=1
gpgcheck=0

#在controller1上
sudo yum install yum-utils createrepo
cd /var/www/html/cm/5.3.0
sudo createrepo .
#在master1上
sudo vi /etc/yum.repos.d/cloudera-cm.repo
[cloudera-cm]
# Packages for Cloudera's Distribution for cm, Version 5, on RedHat     or CentOS 7 x86_64
name=Cloudera's Distribution for cm, Version 5
#baseurl=http://repos.jenkins.cloudera.com/cm5.10.0-release/redhat/7/x86_64/cm/5/
#gpgkey = http://repos.jenkins.cloudera.com/cm5.10.0-release/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
#gpgcheck = 1
baseurl=http://172.16.1.124/cm/5.3.0
#gpgkey = http://repos.jenkins.cloudera.com/cm5.10.0-release/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera
gpgcheck = 0

sudo yum clean all


sudo ./cloudera-manager-installer.bin --skip_repo_package=1

#在controller1上
sudo mkdir -p /home/nanjing/cdh5/parcels
cd /home/nanjing/cdh5/parcels
wget --no-check-certificate https://archive.cloudera.com/cdh5/parcels/5.3.0/CDH-5.3.0-1.cdh5.3.0.p0.30-el6.parcel.sha1

chmod -R 777 /home/nanjing/cdh5/parcels/
chmod -R o+x ~/

sudo mkdir /var/www/html/cdh/

sudo ln -s /home/nanjing/cdh5/parcels /var/www/html/cdh/5

#設置master NTP，關閉Swap
sudo rm -f /etc/localtime
sudo cp  /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
sudo vim /etc/ntp.conf

server      ntp1.aliyun.com
server      time1.cloud.tencent.com

#設置默認策略爲容許任何主機進行時間同步

restrict default ignore

sudo hwclock -w


sudo service ntpd start
sudo chkconfig --level 2345 ntpd on

#其餘機器
sudo vim /etc/ntp.conf
server 172.16.1.125

sudo chkconfig --level 2345 ntpd on
sudo service ntpd start
sudo ntpq -p

#修改swappiness
# 臨時修改
sudo sysctl -w vm.swappiness=0
# 永久修改
sudo echo "vm.swappiness=0" >> /etc/sysctl.conf

 #修改打開句柄數
sudo vim /etc/security/limits.conf
*       soft    nofile          32768
*       hard    nofile          1048576
*       soft    nproc           65536
*       hard    nproc           unlimited
*       soft    memlock         unlimited
*       hard    memlock         unlimited

#將當前用戶添加到hdfs組：
sudo usermod -a -G hdfs nanjing
sudo usermod -a -G root nanjing
#打開瀏覽器
輸入http://172.16.1.125:7180/  用戶名密碼都是admin，登陸後繼續操做。
若是chrome登陸不成功，嘗試使用中文版Firefox。
登陸後選擇安裝express版本，搜索主機時填寫172.16.1.[125-127]
而後全選，在選擇安裝方式時選擇「使用 Parcel (建議)」，而後點擊後面的「更多選項」，
配置「遠程 Parcel 存儲庫 URL」 中的第一行爲：http://172.16.1.124/cdh/5/，
刪除其餘的「遠程 Parcel 存儲庫 URL」，點擊肯定，彈出層關閉後，頁面自動更新，
顯示「選擇 CDH 的版本」爲：「CDH-5.3.0-1.cdh5.3.0.p0.30 」
繼續安裝便可。

安裝的時候若是遇到下面的錯誤：
Command failed to run because service Hive has invalid configuration. Review and correct its configuration. First error: 'Hive Metastore Database Host' is required when using database type 'postgresql'
返回，選擇只安裝HDFS，OOzie，zookeeper，HBase，Yarn等基本組件便可，Spark、Hive等能夠後續再安裝。

若是是後續往集羣中添加機器，則能夠在添加遠程repo的時候，填寫：http://192.168.88.35/cm/5.3.0/

     已啓用「透明大頁面」，它可能會致使重大的性能問題。版本爲「CentOS release 6.4 (Final)」且發行版爲「2.6.32-358.el6.x86_64」的 Kernel 已將 enabled 設置爲「[always] never」，並將 defrag 設置爲「[always] never」。請運行「echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag」以禁用此設置，而後將同一命令添加到一個 init 腳本中，如 /etc/rc.local，這樣當系統重啓時就會予以設置。或者，升級到 RHEL 6.5 或更新版本，它們不存在此錯誤。將會影響到如下主機：