pof補充

[初始化系統]

sudo systemctl stop firewalld
sudo systemctl disable firewalld
sudo setenforce 0
sudo vi /etc/sysconfig/selinux
SELINUX=disabled

sudo tar -zxf docker-18.06.3-ce.tgz -C /usr/local/
cp /usr/local/docker/* /usr/bin/
mkdir -p /home/docker

[修正docker自啓動問題]

添加docker自啓動須要文件

sudo vi /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/bin/dockerd --graph /home/docker
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target

sudo groupadd docker

將您的用戶添加到該docker組

sudo usermod -aG docker $USER

將登錄用戶加入到docker用戶組中

sudo gpasswd -a $USER docker

更新用戶組

newgrp docker

配置docker以systemctl方式啓動

sudo systemctl start docker

配置docker開機啓動

sudo systemctl enable docker

---

數據庫是否換成maridb
[數據庫需確認]

copy sql腳本到數據庫容器
[完善docker cp sql腳本到容器]

使用docker cp方式拷貝dataexa-insight-microservice-poc.sql到容器根目錄下

docker cp /home/dataexa/insight-deploy/resources/mirrors/mysql/dataexa-insight-microservice-poc.sql mysql:/
docker exec -it mysql bash
mysql -uroot -pDataExa5528280
create database dataexa-insight-microservice-poc default character set utf8 default collate utf8_general_ci;
show databases;
use dataexa-insight-microservice-poc;

source生效 執行完整路徑下的sql腳本

source /dataexa-insight-microservice-poc.sql
show tables;

使用docker cp方式拷貝xxl-job-poc.sql到容器根目錄下

docker cp /home/dataexa/insight-deploy/resources/xxl/xxl-job-poc.sql mysql:/
source /xxl-job-poc.sql

平滑方式退出docker容器

鍵盤: curl+ p +q 三個鍵位一塊兒按出現 read escape sequence 便可平滑退出容器

---

[unzip /home/dataexa/insight-deploy/resources/html/platform.zip -d] /home/dataexa/insight-microservice/workspace/nginx_workspace/html/

須要先安裝unzip

unzip [缺unzip離線包]
sudo rpm -ivh /home/dataexa/insight-deploy/resources/tools_package/offline_installer/centos/unzip-6.0-20.el7.x86_64.rpm

查看unzip是否安裝成功
which unzip
/usr/bin/unzip

[粘貼複製的時候-v 沒有空格]
[修改完nginx.conf]
docker restart nginx

重啓前須要curl+p+q 平滑退出容器

重啓elasticsearch容器

docker restart elasticsearch

離線安裝bzip2

sudo rpm -ivh insight-deploy/resources/tools_package/offline_installer/centos/bzip2-1.0.6-13.el7.x86_64.rpm

檢查 bzip2 是否安裝

which bzip2
/usr/bin/bzip2

conda使用的python3.5.tar.gz包變成了zip包
[修正命令]
unzip /home/dataexa/insight-deploy/resources/conda/python/python3.5.zip -d /home/dataexa/anaconda3/envs

---

增長建立路徑[python監控路徑]

mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/container_monitor
mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5

---

vim 補充 [待修改] vim 不能用

sudo cp /home/dataexa/insight-deploy/resources/tools_package/offline_installer/ubuntu/vim.zip
sudo cd /usr/local
sudo unzip vim.zip
sudo ln -s /usr/local/vim/bin/vim /usr/local/bin/vim

查看vim是否安裝

sudo which vim

python [修正]
dlv5 修改[增長nohup python]
nohup python /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5/insight-v5service/tornado_service.py >> /home/dataexa/insight-microservice/workspace/python_service_workspace/dlv5/insight-v5service/nohup.out 2>&1 &

insight-tagging [建立目錄,cp以前沒目錄]
mkdir -p /home/dataexa/insight-microservice/workspace/python_service_workspace/data_labeling
cd /home/dataexa/insight-microservice/workspace/python_service_workspace/data_labeling/insight-tagging/

啓動以前，先把緩存清除

find . -name "pycache" |xargs rm -r

啓動命令

nohup python tornado_service.py >/dev/null 2>&1 &

解壓前建立spark⼯做⽬錄

mkdir -p /home/dataexa/insight-microservice/workspace/spark_workspace

解壓前建立hadoop⼯做⽬錄

mkdir -p /home/dataexa/insight-microservice/workspace/hadoop_workspace/

解壓前建立 yarn-conf⼯做⽬錄

mkdir -p /home/dataexa/insight-microservice/workspace/hadoopworkspace/yarn-conf/

Hadoop部署[修正環境變量] 修改jdk和hadoop環境變量都設置到dataexa用戶下
vi ~/.bashrc
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

vi etc/hadoop/hadoop-env.sh
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH

若是出現格式化文件系統 hdfs namenode -format 出錯

cd /tmp
將標識有hadoop文件名稱的hadoop相關文件 備份移動到另外一個目錄下
從新格式化文件系統

出現20/05/21 07:23:26 INFO ipc.Client: Retrying connect to server: 0.0.0.0/0.0.0.0:8032. Already tried 0 time(s); retry policy is RetryUpT

sbin/start-yarn.sh #啓動yarn便可

出現org.apache.hadoop.mapred.FileAlreadyExistsException: Output directory hdfs://localhost:9000/user/dataexa/output already exists

hdfs dfs -rm -r output
再從新執行便可

完整hadoop部署
[修改了jdk和hadoop的環境變量到dataexa用戶下]

解壓

tar xf /home/dataexa/insight-deploy/resources/jdk/jdk-8u231-linux-x64.tar.gz -C /home/dataexa/insight-deploy/resources/jdk

添加變量環境

vi ~/.bashrc
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH

生效dataexa下的用戶環境

source ~/.bashrc

進入hadoop環境

cd /home/dataexa/insight-deploy/resources/hadoop/hadoop-2.9.0
vi etc/hadoop/hadoop-env.sh
export JAVA_HOME=/home/dataexa/insight-deploy/resources/jdk/jdk1.8.0_231
export PATH=$JAVA_HOME/bin:$PATH

在Hadoop的安裝目錄下，建立input目錄

mkdir input

拷貝input文件到input目錄下

cp etc/hadoop/*.xml input

執行Hadoop job

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep input output 'dfs[a-z.]+'
上面的job是使用hadoop自帶的樣例，在input中統計含有dfs的字符串。

確認執行結果

cat output/*

添加主機地址和映射

sudo vi /etc/hosts
192.168.1.237 localhost

修改設定文件

vi etc/hadoop/core-site.xml


fs.defaultFS
hdfs://localhost:9000

vi etc/hadoop/hdfs-site.xml


dfs.replication
1

設置本機⽆密碼ssh登錄

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

測試

ssh dataexa@192.168.1.237

格式化文件系統

hdfs namenode -format

啓動名稱節點和數據節點後臺進程

sbin/start-dfs.sh

確認

jps

訪問NameNode的web頁面

http://192.168.1.237:50070/

建立HDFS

hdfs dfs -mkdir /user/test

拷貝input文件到HDFS目錄下

hdfs dfs -put etc/hadoop /user/test/input

確認，查看

hadoop fs -ls /user/test/input

執行Hadoop job

hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.0.jar grep /user/test/input output 'dfs[a-z.]+'

確認執行結果

hdfs dfs -cat output/*

或者從HDFS拷貝到本地查看

bin/hdfs dfs -get output output
cat output/*

中止daemon

sbin/stop-dfs.sh

執行YARN job

修改設定文件
cp etc/hadoop/mapred-site.xml.template etc/hadoop/mapred-site.xml
vi etc/hadoop/mapred-site.xml


mapreduce.framework.name
yarn

vi etc/hadoop/yarn-site.xml


yarn.nodemanager.aux-services
mapreduce_shuffle

啓動ResourceManger和NodeManager後臺進程

sbin/start-yarn.sh

訪問ResourceManger的web頁面

http://192.168.1.237:8088/

＃執行hadoop job
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.９.0.jar grep /user/test/input output 'dfs[a-z.]+'

若是這裏報錯,須要sbin/stop-yarn.sh再sbin/start-yarn.sh

確認執行結果

hdfs dfs -cat output/*

中止daemon

sbin/stop-yarn.sh

啓動jobhistory daemon

sbin/mr-jobhistory-daemon.sh start historyserver

訪問Job History Server的web頁面 出現數據便可

http://localhost:19888/