【工具】Datax同步數據配置

1. dataX 說明

DataX 是阿里巴巴集團內被普遍使用的離線數據同步工具/平臺，實現包括 MySQL、Oracle、SqlServer、Postgre、HDFS、Hive、ADS、HBase、OTS、ODPS 等各類異構數據源之間高效的數據同步功能。

2. 下載

使用git

git clone git@github.com:alibaba/DataX.git

使用wget

wget http://datax-opensource.oss-c...

3. 安裝

> git 源碼編譯
    參考:https://github.com/alibaba/DataX/wiki/compile-datax
    
> wget 安裝

    參考  https://github.com/alibaba/DataX/wiki/Quick-Start

    下載後解壓至本地某個目錄，修改權限爲755，進入bin目錄，便可運行樣例同步做業：
       $ tar zxvf datax.tar.gz
       $ sudo chmod -R 755 {YOUR_DATAX_HOME}
       $ cd  {YOUR_DATAX_HOME}/bin
       $ python datax.py ../job/job.json

4. 配置

5 job json 文件配置

nysql數據同步到odps(mysq2odsp.json)：

{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mysqlreader",
                    "parameter": {
                        "username": "szy",
                        "password": "yP0tfUTHvP5q",
                        "column": ["uid","date_time"],
                        "connection": [
                            {
                                "table": [
                                    "datax_user"
                                ],
                                "jdbcUrl": [
                                    "jdbc:mysql://127.0.0.1:3306/ztjy"
                                ]
                            }
                        ]
                    }
                },
                "writer": {
                    "name": "odpswriter",
                    "parameter": {
                        "accessId": "06xxZ11yge6LekDx",
                        "accessKey": "PMbyrM2DZC3QIu586BTyGQCBS5l3nt",
                        "column": ["uid","date_time"],
                        "odpsServer": "http://service.odps.aliyun.com/api",
                        "partition": "",
                        "project": "odps_testproject001",
                        "table": "datax_test_user01",
                        "truncate": true
                    }
                }
            }
        ],
        "setting": {
            "speed": {
                "channel": 5
            }
        }
    }
}

odps同步到mysql(odsp2mysq.json)：

{
    "job": {
        "content": [
            {

            "reader": {
                    "name": "odpsreader",
                    "parameter": {
                        "accessId": "LTAIRFDnrN660HWe",
                        "accessKey": "T0mFczTQwziIItyySdZ9QMMvANEgc6",
                        "project": "xmszysample",
                        "table": "stat_user_tag",
                        "column": [
                            "user_id",
                            "province",
                            "city",
                            "county",
                            "birthday",
                            "baby_id",
                            "baby_birthday",
                            "model"
                        ],
                        "packageAuthorizedProject": "xmszysample",
                        "splitMode": "record",
                        "odpsServer": "http://service.odps.aliyun.com/api"
                    }
                },
                "writer": {
                    "name": "mysqlwriter",
                    "parameter": {
                        "writeMode": "insert",
                        "username": "szy",
                        "password": "yP0tfUTHvP5q",
                        "column": [
                            "user_id",
                            "province",
                            "city",
                            "county",
                            "birthday",
                            "baby_id",
                            "baby_birthday",
                            "model"
                        ],
                        "session": [
                            "set session sql_mode='ANSI'"
                        ],
                        "preSql": [
                            "truncate log_stat_user_tag"
                        ],
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:mysql://172.16.200.17:3306/ztjy_xm?useUnicode=true&characterEncoding=utf8",
                                "table": [
                                    "log_stat_user_tag"
                                ]
                            }
                        ]
                    }
                    
                }
            }
        ],
        "setting": {
            "speed": {
                "channel": 5
            }
        }
    }
}

5. 執行測試

mysql同步到odps  python datax.py /job/mysql2odps.json

odps同步到mysql  python datax.py /job/odps2mysql.json