基於【CentOS-7+ Ambari 2.7.0 + HDP 3.0】搭建HAWQ數據倉庫03 —— 安裝HAWQ 2.3.0.0

1、 HAWQ2.3.0環境準備【所有主機節點】：

1， vim /etc/sysctl.conf,編輯以下內容:

kernel.shmmax= 1000000000
kernel.shmmni= 4096
kernel.shmall= 4000000000
kernel.sem= 250 512000 100 2048
kernel.sysrq= 1
kernel.core_uses_pid= 1
kernel.msgmnb= 65536
kernel.msgmax= 65536
kernel.msgmni= 2048
net.ipv4.tcp_syncookies= 0
net.ipv4.ip_forward= 0
net.ipv4.conf.default.accept_source_route= 0
net.ipv4.tcp_tw_recycle= 1
net.ipv4.tcp_max_syn_backlog= 200000
net.ipv4.conf.all.arp_filter= 1
net.ipv4.ip_local_port_range= 1281 65535
net.core.netdev_max_backlog= 200000
vm.overcommit_memory= 2
fs.nr_open= 3000000
kernel.threads-max= 798720
kernel.pid_max= 798720
#increase network
net.core.rmem_max=2097152
net.core.wmem_max=2097152

保存退出後，使用命令「sysctl -p」 使之生效：

sysctl -p

 

2， vim /etc/security/limits.conf 編輯以下內容:

* soft nofile 2900000
* hard nofile 2900000
* soft nproc 131072
* hard nproc 131072

從新登陸以使配置生效，或者使用命令臨時設置一下：

ulimit -n 10240

3， 添加gpadmin用戶（greeplum admin縮寫），使用/opt/gpadmin做爲主目錄

useradd --home=/opt/gpadmin/ --no-create-home --comment "HAWQ admin" gpadmin 
echo gpadmin | passwd --stdin gpadmin 
mkdir /opt/gpadmin
chown gpadmin:gpadmin /opt/gpadmin

2，添加gpadmin到/etc/sudoers，編輯該文件，添加以下行

gpadmin ALL=(ALL) NOPASSWD:ALL

3， 配置gpadmin用戶的ssh【免密互訪】
4， 準備軟件包依賴項：
libgsasl is needed by apache-hawq-2.3.0.0-el7.x86_64
protobuf >= 2.5.0 is needed by apache-hawq-2.3.0.0-el7.x86_64
net-snmp-libs is needed by apache-hawq-2.3.0.0-el7.x86_64
thrift >= 0.9.1 is needed by apache-hawq-2.3.0.0-el7.x86_64
boost >= 1.53.0 is needed by apache-hawq-2.3.0.0-el7.x86_64
其中gcc gcc-c++ protobuf net-snmp-libs boots可直接安裝。而安裝libgsasl須要添加一個包含libgsasl庫的源：新建/etc/yum.repos.d/fedora.repo文件，添加以下內容：

[epel-repo]
name=epel
baseurl=http://dl.fedoraproject.org/pub/epel/6/x86_64/
enabled=1
gpgcheck=0

執行安裝：

yum install gcc gcc-c++ -y
yum install protobuf -y
yum install net-snmp-libs -y
yum install boost -y 
yum install libgsasl-devel -y

 

C)下載安裝thrift 0.9.1

wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/t/thrift-0.9.1-15.el7.x86_64.rpm
rpm -ivh thrift-0.9.1-15.el7.x86_64.rpm

2、 下載安裝包apache-hawq-rpm-2.3.0.0-incubating.tar.gz，並解壓縮 

cd downloads
wget http://mirrors.tuna.tsinghua.edu.cn/apache/incubator/hawq/2.3.0.0-incubating/apache-hawq-rpm-2.3.0.0-incubating.tar.gz
tar -zxvf apache-hawq-rpm-2.3.0.0-incubating.tar.gz

 

3、 安裝服務端
1， 安裝
rpm -ivh --prefix=/opt/gpadmin apache-hawq-2.3.0.0-el7.x86_64.rpm

#在其餘主機中依次執行上面安裝

**移除安裝
yum remove apache-hawq -y

2 HAWQ的參數配置

0) 切換用戶gpadmin

[root@ep-bd01 apache-hawq]# su - gpadmin

編輯用戶gpadmin啓動文件.bash_proifile

source /opt/gpadmin/apache-hawq/greenplum_path.sh

執行命令

source /opt/gpadmin/apache-hawq/greenplum_path.sh

配置HAWQ的集羣及其它參數

1) vi   /opt/gpadmin/apache-hawq/etc/slaves

# 添加集羣主機列表

ep-bd01
ep-bd02
ep-bd03
ep-bd04
ep-bd05
ep-bd06

2) vi hawq-site.xml

# 修改以下參數
參數名 值

hadoop.security.authentication    simple
hawq_master_address_host    ep-bd01
hawq_dfs_url    ep-bd01:8020/hawq
hawq_rm_memory_limit_perseg    64GB
hawq_rm_yarn_address    ep-bd01:8032
hawq_dfs_url    ep-bd01:8020/hawq
hawq_global_rm_type    yarn
hawq_rm_yarn_scheduler_address    ep-bd01:8030
hawq_acl_type    standalone
hawq_master_directory    /opt/gpadmin/data/masterdd
hawq_segment_directory    /opt/gpadmin/data/segmentdd

vi yarn-site.xml

# 修改以下參數
參數名 值

hadoop.security.authentication    simple

2， 創建dfs/hawq

sudo -u hdfs hdfs dfs -mkdir /hawq
sudo -u hdfs hdfs dfs -chown gpadmin:gpadmin /hawq

3， 創建data目錄（所有節點）

sudo -u gpadmin mkdir -p /opt/gpadmin/data/masterdd /opt/gpadmin/data/segmentdd

4， 複製配置文件到其餘節點

su - gpadmin
gpscp -f slaves  slaves =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves  hawq-site.xml =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves  hdfs-client.xml =:/opt/gpadmin/apache-hawq/etc
gpscp -f slaves  yarn-client.xml =:/opt/gpadmin/apache-hawq/etc

 

5，初始化cluster【注意，這裏必須進入/opt/gpadmin/apache-hawq/bin】

su - gpadmin
cd /opt/gpadmin/apache-hawq/bin
source ../greenplum_path.sh
./hawq init cluster

# 初始化cluster失敗，調整以後，再次執行初始化以前須要

#刪除dfs目錄中內容：
sudo -u hdfs hdfs dfs -rm -r -f /hawq/*
#刪除數據目錄中內容

ssh ep-bd01 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd01 rm -rf /opt/gpadmin/data/segmentdd/*
ssh ep-bd02 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd02 rm -rf /opt/gpadmin/data/segmentdd/*
ssh ep-bd03 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd03 rm -rf /opt/gpadmin/data/segmentdd/*
ssh ep-bd04 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd04 rm -rf /opt/gpadmin/data/segmentdd/*
ssh ep-bd05 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd05 rm -rf /opt/gpadmin/data/segmentdd/*
ssh ep-bd06 rm -rf /opt/gpadmin/data/masterdd/*
ssh ep-bd06 rm -rf /opt/gpadmin/data/segmentdd/* 

 

# 啓動中止重啓hawqhawq stop/restart/start cluster