搭建AirFlow—— 一段波折後的總結

聲明：本人借鑑了衆多同行的博客，因此總結了這篇博客，用來致敬他們，很是感謝他們，轉載請註明出處。

基礎環境：【CentOS 6.9】（cat /etc/redhat-release）
python2.七、pip、gcc、gcc-c++、Fernet、pandas(下載安裝包，手動編譯安裝)、numpy、MySQL-python、sqlite-devel、lxml、openssl、openssl-devel、mysql-devel
注：

   一、缺乏mysql_config  
      執行命令：ln -s /usr/local/mysql/bin/mysql_config /usr/local/bin/mysql_config  
   二、沒法使用sqlite
     find / -name _sqlite*.so  
     cp /usr/lib64/python2.6/lib-dynload/_sqlite3.so   /usr/local/lib/python2.7/lib-dynload/_sqlite3.so
   三、pip install 軟件==指定版本
   四、Installing build dependencies ... error...Double requirement given: numpy==1.12.1...     
      手動安裝pandas

 

# 環境準備好，開始

# airflow needs a home, ~/airflow is the default,
# but you can lay foundation somewhere else if you prefer
# (optional)
export AIRFLOW_HOME=~/airflow

# install from pypi using pip
pip install apache-airflow

# initialize the database 執行一遍，修改配置，再次執行
airflow initdb# start the web server, default port is 8080
airflow webserver -p 8080
--------------------------------------------------------------------------
cd ~/airflow
vim airflow.cfg

#修改airflow.cfg

[core]
# The home folder for airflow, default is ~/airflow
airflow_home = /root/airflow
dags_folder = /data/airflow/dags
base_log_folder = /root/airflow/logs


# The executor class that airflow should use. Choices include
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
executor = LocalExecutor

# mysqldb
sql_alchemy_conn = mysql://airflow:airflow@172.16.2.222:3306/airflow
sql_alchemy_pool_size = 10
# Secret key to save connection passwords in the db 手動生成
fernet_key = l5k-1nUD50nWXzTL9imndy6cQIVvIm_3efYIV4B1RiI=
[operators]
# The default owner assigned to each new operator, unless
# provided explicitly or passed via `default_args`

default_owner = Airflow
default_cpus = 5
default_ram = 8192
default_disk = 8192
default_gpus = 0

[webserver]
base_url = http://localhost:8080

# The ip specified when starting the web server
web_server_host = 0.0.0.0
web_server_port = 8080

# Expose the configuration file in the web server
expose_config = True

# Set to true to turn on authentication:
# http://pythonhosted.org/airflow/security.html#web-authentication
# pip install apache-airflow[password]
authenticate = True
auth_backend = airflow.contrib.auth.backends.password_auth

# Filter the list of dags by owner name (requires authentication to be enabled)
filter_by_owner = True

# Consistent page size across all listing views in the UI 優化性能
page_size = 15

#獲取FK

from cryptography.fernet import Fernet
fernet_key= Fernet.generate_key()
print(fernet_key)

#建立用戶

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import airflow
from airflow import models, settings
from airflow.contrib.auth.backends.password_auth import PasswordUser
from flask_bcrypt import generate_password_hash
user = PasswordUser(models.User())

user.username = '帳號'
user.email = '郵箱'
user._password = generate_password_hash('密碼', 12)

session = settings.Session()
session.add(user)
session.commit()
session.close()

#啓動Web服務

nohup airflow webserver -p 8080 &

#啓動調度

airflow scheduler

#修改時區

vim /usr/local/lib/python2.7/site-packages/airflow/www/templates/admin/master.html
//var UTCseconds = (x.getTime() + x.getTimezoneOffset()*60*1000);   
var UTCseconds = x.getTime();