大數據篇：DolphinScheduler-1.2.0.release安裝部署

大數據篇：DolphinScheduler-1.2.0.release安裝部署

1 配置jdk

#查看命令
rpm -qa | grep java
#刪除命令
rpm -e --nodeps xxx

• 將oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm上傳至每一個節點安裝

  rpm -ivh oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm

• 修改配置文件

  vim /etc/profile
  #添加
  export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
  export PATH=$JAVA_HOME/bin:$PATH
  export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

• 刷新源

  source /etc/profile

• 檢驗

  java
  javac

2建立用戶

# 建立部署用戶
userdel -r dolphinscheduler 
useradd dolphinscheduler && echo dolphinscheduler | passwd --stdin dolphinscheduler
# 賦予 sudo 權限
chmod 640 /etc/sudoers
vim /etc/sudoers
# 大概在100行，在root下添加以下(注意更改smapp)
dolphinscheduler  ALL=(ALL)       NOPASSWD: NOPASSWD: ALL
# 而且須要註釋掉 Default requiretty 一行。若是有則註釋，沒有沒有跳過
#Default requiretty

3對部署用戶配置免密

全部節點

su dolphinscheduler
#生成密鑰對（公鑰和私鑰）三次回車生成密鑰
ssh-keygen -t rsa
#查看公鑰
cat ~/.ssh/id_rsa.pub
#將密匙輸出到/root/.ssh/authorized_keys
cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

主節點

#追加密鑰到主節點(須要操做及密碼驗證，追加完後查看一下該文件)--在主節點上操做,拷取從節點密匙
ssh 從節點機器IP cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

cat ~/.ssh/authorized_keys
#從主節點複製密鑰到從節點
scp ~/.ssh/authorized_keys dolphinscheduler@從節點機器IP:~/.ssh/authorized_keys

全部節點互相進行ssh鏈接

ssh dolphinscheduler@172.xx.xx.xxx
ssh dolphinscheduler@172.xx.xx.xxx

4pip、kazoo 安裝

curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
sudo python get-pip.py
pip --version
pip install kazoo
#使用python import kazoo,不報錯便可

5安裝包下載

# git 下載地址
https://github.com/apache/incubator-dolphinscheduler/releases

# 建立安裝目錄
sudo mkdir /opt/DolphinScheduler && sudo chown -R dolphinscheduler:dolphinscheduler /opt/DolphinScheduler

6解壓 dolphinscheduler安裝包(須要對應部署用戶權限)

#切換用戶上傳tar包
su dolphinscheduler

cd /opt/DolphinScheduler && mkdir dolphinScheduler-backend
tar -zxvf apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin.tar.gz -C dolphinScheduler-backend

cd /opt/DolphinScheduler && mkdir dolphinScheduler-ui
tar -zxvf apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-front-bin.tar.gz -C dolphinScheduler-ui

7部署mysql用戶

# 設置數據用戶 dolphinscheduler 的訪問密碼爲 dolphinscheduler，而且不對訪問的 ip 作限制
# 測試環境將訪問設置爲全部，若是是生產，能夠限制只能子網段的ip才能訪問（'192.168.1.%'）
CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';
flush privileges;

8建立表和導入基礎數據 修改 application-dao.properties中的下列屬性

vim  /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/application-dao.properties
#注意註釋postgre鏈接，打開mysql鏈接
>>>>

spring.datasource.url=jdbc:mysql://10.xx.xx.xx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
spring.datasource.username=dolphinscheduler
spring.datasource.password=dolphinscheduler

#執行建立表和導入基礎數據腳本(1.2缺乏mysql鏈接jar,先copy一個mysql-connector-java-5.1.34.jar到lib下)
cd /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin
sh script/create-dolphinscheduler.sh

9後端配置文件修改

9.1dolphinscheduler_env.sh

vim  /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/env/.dolphinscheduler_env.sh
>>>>
export HADOOP_HOME=/opt/cloudera/parcels/CDH/lib/hadoop
export HADOOP_CONF_DIR=/etc/hadoop/conf
export SPARK_HOME1=/opt/cloudera/parcels/CDH/lib/spark
#export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export HIVE_HOME=/opt/cloudera/parcels/CDH/lib/hive
#export FLINK_HOME=/opt/soft/flink
#export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH

9.2install.sh

vim  /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/install.sh
>>>>
# 1. mysql 配置
# 安裝完成後如下幾項配置位於 $installPath/conf/quartz.properties 中
# mysql 地址，端口；數據庫名稱；用戶名；密碼（注意：若是有特殊字符，請用 \ 轉移符進行轉移）
# for example postgresql or mysql ...
#dbtype="postgresql"
dbtype="mysql"

# db config
# db address and port
dbhost="10.xx.xx.xx:3306"

# db name
dbname="dolphinscheduler"

# db username
username="dolphinscheduler"

# db passwprd
# Note: if there are special characters, please use the \ transfer character to transfer
passowrd="dolphinscheduler"


# 2. 集羣架構配置
# 2.1 集羣部署環境配置
# 安裝完成後如下幾項配置位於 $installPath/conf/config/install_config.conf 中
# conf/config/install_config.conf config
# Note: the installation path is not the same as the current path (pwd)
# dolphinscheduler 集羣安裝目錄(不能與先有路徑相同)
installPath="/opt/DolphinScheduler/dolphinscheduler"

# deployment user
# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
# 部署用戶。注意：部署用戶須要有 sudo 權限及操做 hdfs 的權限，若是開啓 hdfs，根目錄須要自行建立
deployUser="dolphinscheduler"

# zk cluster
# zk 集羣
zkQuorum="172.xx.xx.xx:2181,172.xx.xx.xx:2181,172.xx.xx.xx:2181"

# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
# 安裝 DolphinScheduler 的機器 hostname 列表,若是是一臺ips="xx.xx.xx.xx"
ips="xx.xx.xx.232,xx.xx.xx.233"

# 2.2 各節點服務配置(必須是hostname，若是是單臺就只寫一個)
# 安裝完成後如下幾項配置位於 $installPath/conf/config/run_config.conf 中
# conf/config/run_config.conf config
# run master machine
# Note: list of hosts hostname for deploying master
# 運行 Master 的機器
masters="ds1.com,ds2.com"

# run worker machine
# note: list of machine hostnames for deploying workers
# 運行 Worker 的機器
workers="ds1.com,ds2.com"

# run alert machine
# note: list of machine hostnames for deploying alert server
# 運行 Alert 的機器（提供告警相關接口）
alertServer="ds1.com,ds2.com"

# run api machine
# note: list of machine hostnames for deploying api server
# 運行 Api 的機器（API 接口層，主要負責處理前端UI層的請求）
apiServers="ds1.com,ds2.com"

# 3. alert 配置
# 安裝完成後如下幾項配置位於 $installPath/conf/alert.properties 中
# alert config
# mail protocol
# 郵件協議
mailProtocol="SMTP"

# mail server host
# 郵件服務host
mailServerHost="smtp.qq.com"

# mail server port
# 郵件服務端口
mailServerPort="587"

# sender
# 發送人
mailSender="xxx@qq.com"

# user
mailUser="xxx@qq.com"

# sender password
# 發送人密碼(若是線下使用25端口則是郵箱密碼，不然是開啓SMTP服務的驗證碼)
mailPassword="xxxxx"

# TLS mail protocol support
starttlsEnable="true"

#認證
sslTrust="smtp.qq.com"


# SSL mail protocol support
# note: The SSL protocol is enabled by default. 
# only one of TLS and SSL can be in the true state.
# SSL郵件協議支持
# 注意：默認開啓的是SSL協議，TLS和SSL只能有一個處於true狀態
sslEnable="false"


# 下載Excel路徑
xlsFilePath="/opt/DolphinScheduler/xls"
# 企業微信企業ID配置
enterpriseWechatCorpId="xxxxxxxxxx"
# 企業微信應用Secret配置
enterpriseWechatSecret="xxxxxxxxxx"
# 企業微信應用AgentId配置
enterpriseWechatAgentId="xxxxxxxxxx"
# 企業微信用戶配置,多個用戶以,分割
enterpriseWechatUsers="xxxxx,xxxxx"


# 4. 開啓監控自啓動腳本
# 控制是否啓動自啓動腳本(監控master,worker狀態,若是掉線會自動啓動) 
#monitorServerState="false"
monitorServerState="true"

# 5. 資源中心配置
# 安裝完成後如下幾項配置位於 $installPath/conf/common/* 中
# 5.1 Hadoop 相關配置
# resource Center upload and select storage method：HDFS,S3,NONE
# 資源中心上傳選擇存儲方式：HDFS,S3,NONE
resUploadStartupType="HDFS"

# if resUploadStartupType is HDFS，defaultFS write namenode address，HA you need to put core-site.xml and hdfs-site.xml in the conf directory.
# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
# Note，s3 be sure to create the root directory /dolphinscheduler
# 若是存儲方式爲HDFS，defaultFS 寫 namenode 地址；支持 HA,須要將 core-site.xml 和 hdfs-site.xml 放到 conf 目錄下
defaultFS="hdfs://xx.xx.xx.xx:8020"

# if S3 is configured, the following configuration is required.
# 若是配置了S3，則須要有如下配置
#s3Endpoint="http://192.168.199.91:9010"
#s3AccessKey="A3DXS30FO22544RE"
#s3SecretKey="OloCLq3n+8+sdPHUhJ21XrSxTC+JK"


# 5.2 yarn 相關配置
# resourcemanager HA配置，若是是單 resourcemanager, 這裏爲 yarnHaIps="",填寫下面
#yarnHaIps=""
yarnHaIps="xx.xx.xx.xx,xx.xx.xx.xx"
# 若是是單 resourcemanager, 只須要配置一個主機名稱,若是是 resourcemanager HA,則默認配置就好
#singleYarnIp="xx.xx.xx.xx"

# 5.3 HDFS 根目錄及權限配置
# hdfs root path, the owner of the root path must be the deployment user. 
# versions prior to 1.1.0 do not automatically create the hdfs root directory, you need to create it yourself.
# hdfs 根路徑，根路徑的 owner 必須是部署用戶。1.1.0以前版本不會自動建立hdfs根目錄，須要自行建立
hdfsPath="/dolphinscheduler"
# have users who create directory permissions under hdfs root path /
# Note: if kerberos is enabled, hdfsRootUser="" can be used directly.
# 擁有在hdfs根路徑下建立目錄權限的用戶（此處不建議配置爲超級管理員用戶，根目錄需自行建立並修改權限）
# 注意：若是開啓了kerberos，則直接hdfsRootUser=""，就能夠
hdfsRootUser="dolphinscheduler"



# 6. common 配置(默認配置)
# 安裝完成後如下幾項配置位於 $installPath/conf/common/common.properties 中
# 程序路徑
programPath="/tmp/dolphinscheduler"
#下載路徑
downloadPath="/tmp/dolphinscheduler/download"
# 任務執行路徑
execPath="/tmp/dolphinscheduler/exec"
# SHELL環境變量路徑
shellEnvPath="$installPath/conf/env/.dolphinscheduler_env.sh"
# 資源文件的後綴
resSuffixs="txt,log,sh,conf,cfg,py,java,sql,hql,xml"
# 開發狀態,若是是true,對於SHELL腳本能夠在execPath目錄下查看封裝後的SHELL腳本,若是是false則執行完成直接刪除
devState="true"
# kerberos 配置
# kerberos 是否啓動
kerberosStartUp="false"
# kdc krb5 配置文件路徑
krb5ConfPath="$installPath/conf/krb5.conf"
# keytab 用戶名
keytabUserName="hdfs-mycluster@ESZ.COM"
# 用戶 keytab路徑
keytabPath="$installPath/conf/hdfs.headless.keytab"

# 7. zk 配置(默認配置)
# 安裝完成後如下幾項配置位於 $installPath/conf/zookeeper.properties 中
# zk根目錄
zkRoot="/dolphinscheduler"
# 用來記錄掛掉機器的zk目錄
zkDeadServers="$zkRoot/dead-servers"
# masters目錄
zkMasters="$zkRoot/masters"
# workers目錄
zkWorkers="$zkRoot/workers"
# zk master分佈式鎖
mastersLock="$zkRoot/lock/masters"
# zk worker分佈式鎖
workersLock="$zkRoot/lock/workers"
# zk master容錯分佈式鎖
mastersFailover="$zkRoot/lock/failover/masters"
# zk worker容錯分佈式鎖
workersFailover="$zkRoot/lock/failover/workers"
# zk master啓動容錯分佈式鎖
mastersStartupFailover="$zkRoot/lock/failover/startup-masters"
# zk session 超時
zkSessionTimeout="300"
# zk 鏈接超時
zkConnectionTimeout="300"
# zk 重試間隔
zkRetrySleep="100"
# zk重試最大次數
zkRetryMaxtime="5"

# 8. master 配置(默認配置)
# 安裝完成後如下幾項配置位於 $installPath/conf/master.properties 中
# master執行線程最大數,流程實例的最大並行度
masterExecThreads="100"
# master任務執行線程最大數,每個流程實例的最大並行度
masterExecTaskNum="20"
# master心跳間隔
masterHeartbeatInterval="10"
# master任務提交重試次數
masterTaskCommitRetryTimes="5"
# master任務提交重試時間間隔
masterTaskCommitInterval="100"
# master最大cpu平均負載,用來判斷master是否還有執行能力
masterMaxCpuLoadAvg="10"
# master預留內存,用來判斷master是否還有執行能力
masterReservedMemory="1"


# 9. worker 配置
# 安裝完成後如下幾項配置位於 $installPath/conf/worker.properties 中
# worker執行線程
workerExecThreads="100"
# worker心跳間隔
workerHeartbeatInterval="10"
# worker一次抓取任務數
workerFetchTaskNum="3"
# worker最大cpu平均負載,用來判斷worker是否還有執行能力,保持系統默認，默認爲cpu核數的2倍，當負載達到2倍時，
#workerMaxCupLoadAvg="10"
# worker預留內存,用來判斷master是否還有執行能力
workerReservedMemory="1"

# 10. api 配置(默認配置)
# 安裝完成後如下幾項配置位於 $installPath/conf/application.properties 中
# api 服務端口
apiServerPort="12345"
# api session 超時
apiServerSessionTimeout="7200"
# api 上下文路徑
apiServerContextPath="/dolphinscheduler/"
# spring 最大文件大小
springMaxFileSize="1024MB"
# spring 最大請求文件大小
springMaxRequestSize="1024MB"
# api 最大post請求大小
apiMaxHttpPostSize="5000000"

9.3添加 Hadoop 配置文件

# 若 install.sh 中，resUploadStartupType 爲 HDFS，且配置爲 HA，則需拷貝 hadoop 配置文件到 conf 目錄下
#cp /etc/hadoop/conf.cloudera.yarn/hdfs-site.xml /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/
#cp /etc/hadoop/conf.cloudera.yarn/core-site.xml /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/
scp /etc/hadoop/conf.cloudera.yarn/hdfs-site.xml smapp@xx.xx.xx.232:/opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/
scp /etc/hadoop/conf.cloudera.yarn/core-site.xml  smapp@xx.xx.xx.232:/opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/conf/

9.4建立 hdfs 根目錄

#sudo -u hdfs hadoop fs -rmr /dolphinscheduler
sudo -u hdfs hadoop fs -mkdir /dolphinscheduler
sudo -u hdfs hadoop fs -chown dolphinscheduler:dolphinscheduler /dolphinscheduler

9.5一鍵部署（須要執行第兩次，由於第一次有文件未被建立）

#在日誌中可能看到找不到java,修改其$JAVA_HOME
vim /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/bin/dolphinscheduler-daemon.sh
#進入腳本目錄啓動
cd /opt/DolphinScheduler/dolphinScheduler-backend/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-backend-bin/
sh install.sh

9.6查看Java進程ll

10前端部署

10.1須要niginx

vim /etc/nginx/conf.d/default.conf
》》》
upstream dolphinsch {
    server xx.xx.xx.232:12345;
    server xx.xx.xx.233:12345;
}
    server {
        listen       80;# 訪問端口
        server_name  localhost;
        #charset koi8-r;
        #access_log  /var/log/nginx/host.access.log  main;
    location / {
        root    /opt/DolphinScheduler/dolphinScheduler-ui/apache-dolphinscheduler-incubating-1.2.0-dolphinscheduler-front-bin/dist; # 靜態文件目錄，即前端解壓的 dist 目錄
        index  index.html index.html;
    }
    location /dolphinscheduler {
        proxy_pass http://dolphinsch; # 接口地址(自行修改)
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header x_real_ipP $remote_addr;
        proxy_set_header remote_addr $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_http_version 1.1;
        proxy_connect_timeout 4s;
        proxy_read_timeout 30s;
        proxy_send_timeout 12s;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";
    }
    #error_page  404              /404.html;
    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}


#修改上傳文件大小限制
sudo vim /etc/nginx/nginx.conf
# 在 http 內加入
>>>
client_max_body_size 1024m;

#重啓 nginx 服務
systemctl restart nginx

10.2訪問

admin
dophinscheduler123