數據分庫拆分測試---dble

時間 2019-12-09

標籤數據分庫拆分測試 dble 简体版

原文原文鏈接

數據庫表

server.xml配置

第一段 "< system >" 爲DBLE的服務端口（默認8066）和管理端口（默認9066）的配置html

管理端口只能接受DBLE的管理命令，這裏不作展開
服務端口即DBLE的業務訪問端口，能夠接受SQL語句

第二段「< user >」配置管理理用戶，默認爲man1，密碼爲654321node

便可以經過 mysql -P9066 -h 127.0.0.1 -u man1 -p654321來下發管理命令

第三段「< user >」配置業務用戶，配置了一個帳號test 密碼123456,針對數據庫testdb,讀寫權限都有，沒有針對表作任何特殊的權限，故把表配置作了註釋mysql

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!-- - - Licensed under the Apache License, Version 2.0 (the "License");
    - you may not use this file except in compliance with the License. - You 
    may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 
    - - Unless required by applicable law or agreed to in writing, software - 
    distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT 
    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the 
    License for the specific language governing permissions and - limitations 
    under the License. -->
<!DOCTYPE dble:server SYSTEM "server.dtd">
<dble:server xmlns:dble="http://dble.cloud/" version="2.19.03.0">
    <system>
        <!-- base config -->
        <property name="bindIp">0.0.0.0</property>
        < property name="serverPort">8066</property> 
        <property name="managerPort">9066</property> 
        <property name="maxCon">1024</property> 
        <property name="processors">4</property>
        <property name="backendProcessors">12</property>
        <property name="processorExecutor">4</property> -->
        <property name="backendProcessorExecutor">12</property>
        <property name="complexExecutor">8</property> -->
        <property name="writeToBackendExecutor">4</property>

        <!--<property name="fakeMySQLVersion">5.6.20</property>-->
        <property name="sequnceHandlerType">2</property>
        <!-- serverBacklog size,default 2048-->
        <property name="serverBacklog">2048</property>
        <!--<property name="serverNodeId">1</property>-->
        <!--<property name="showBinlogStatusTimeout">60000</property>-->

        <!--option-->
        <!--<property name="useCompression">1</property>-->
        <!--<property name="usingAIO">0</property>-->
        <!--<property name="useZKSwitch">true</property>-->
        <!--<property name="useThreadUsageStat">0</property>-->
        <!--<property name="usePerformanceMode">0</property>-->

        <!-- query time cost statistics-->
        <!--<property name="useCostTimeStat">0</property>-->
        <!--<property name="maxCostStatSize">100</property>-->
        <!--<property name="costSamplePercent">1</property>-->

        <!--connection -->
        <!--<property name="charset">utf8</property>-->
        <!--<property name="maxPacketSize">16777216</property>-->
        <!--<property name="txIsolation">3</property>-->

        <!--consistency-->
        <!-- check the consistency of table structure between nodes,default not -->
        <property name="checkTableConsistency">0</property>
        <!-- check periodt, he default period is 60000 milliseconds -->
        <property name="checkTableConsistencyPeriod">60000</property>
        <!-- 1 check the consistency of global table, 0 is not -->
        <property name="useGlobleTableCheck">0</property>
        <property name="glableTableCheckPeriod">86400000</property>

        <!-- heartbeat check period -->
        <property name="dataNodeIdleCheckPeriod">300000</property>
        <property name="dataNodeHeartbeatPeriod">10000</property>

        <!-- processor check conn-->
        <property name="processorCheckPeriod">1000</property><!-- unit millisecond -->
        <property name="sqlExecuteTimeout">300</property><!-- unit second -->
        <property name="idleTimeout">1800000</property><!-- unit millisecond -->

        <!-- transaction log -->
        <!-- 1 enable record the transaction log, 0 disable -->
        <property name="recordTxn">0</property>
        <!--<property name="transactionLogBaseDir">/txlogs</property>-->
        <!--<property name="transactionLogBaseName">server-tx</property>-->
        <!--<property name="transactionRatateSize">16</property>--><!-- unit M -->

        <!-- XA transaction -->
        <!-- use XA transaction ,if the mysql service crash,the unfinished XA commit/rollback will retry for several times
       it is the check period for ,default is 1000 milliseconds-->
        <property name="xaSessionCheckPeriod">1000</property>
        <!-- use XA transaction ,the finished XA log will removed. the default period is 1000 milliseconds-->
        <property name="xaLogCleanPeriod">1000</property>
        <!-- XA Recovery Log path -->
        <!--<property name="xaRecoveryLogBaseDir">/tmlogs/</property>-->
        <!-- XA Recovery Log name -->
        <!--<property name="xaRecoveryLogBaseName">tmlog</property>-->

        <!-- true is use JoinStrategy, default false-->
        <property name="useJoinStrategy">true</property>
        <property name="nestLoopConnSize">4</property>
        <property name="nestLoopRowsSize">2000</property>

        <!-- query memory used for per session,unit is M-->
        <property name="otherMemSize">4</property>
        <property name="orderMemSize">4</property>
        <property name="joinMemSize">4</property>

        <!-- off Heap unit:bytes-->
        <property name="bufferPoolChunkSize">4096</property>
        <property name="bufferPoolPageNumber">512</property>
        <property name="bufferPoolPageSize">2097152</property>
        <!--<property name="mappedFileSize">2097152</property>-->

        <!-- sql statistics-->
        <!-- 1 means use SQL statistics, 0 means not -->
        <property name="useSqlStat">0</property>
        <!--<property name="bufferUsagePercent">80</property>-->
        <!--<property name="clearBigSqLResultSetMapMs">600000</property>-->
        <!--<property name="sqlRecordCount">10</property>-->
        <!--<property name="maxResultSet">524288</property>-->

        <!-- backSocket unit:bytes-->
        <!--<property name="backSocketSoRcvbuf">4194304</property>-->
        <!--<property name="backSocketSoSndbuf">1048576</property>-->
        <!--<property name="backSocketNoDelay">1</property>-->

        <!-- frontSocket-->
        <!--<property name="frontSocketSoRcvbuf">1048576</property>-->
        <!--<property name="frontSocketSoSndbuf">4194304</property>-->
        <!--<property name="frontSocketNoDelay">1</property>-->

        <!-- for join tmp results -->
        <!--<property name="mergeQueueSize">1024</property>-->
        <!--<property name="orderByQueueSize">1024</property>-->
        <!--<property name="joinQueueSize">1024</property>-->

        <!-- if enable the slow query log -->
        <property name="enableSlowLog">0</property>
        <!-- the slow query log location -->
        <!--<property name="slowLogBaseDir">./slowlogs</property>-->
        <!--<property name="slowLogBaseName">slow-query</property>-->
        <!-- the max period for flushing the slow query log from memory to disk  after last time , unit is second -->
        <property name="flushSlowLogPeriod">1</property>
        <!-- the max records for flushing the slow query log from memory to disk after last time -->
        <property name="flushSlowLogSize">1000</property>
        <!-- the threshold for judging if the query is slow , unit is millisecond -->
        <property name="sqlSlowTime">100</property>


    </system>

    <!-- firewall config -->
    <!--
    <firewall>
    <whitehost>
          <host host="127.0.0.1" user="root"/>
          <host host="0:0:0:0:0:0:0:1" user="root"/>
       </whitehost>
       <blacklist check="true">
       <property name="selelctAllow">false</property>
       </blacklist>
    </firewall>
    -->
    <user name="man1">
        <property name="password">654321</property>
        <property name="manager">true</property>
        <!-- manager user can't set schema-->
    </user>

    <user name="test">
        <property name="password">123456</property>
        <property name="schemas">TESTDB</property>

        <!-- table's DML privileges  INSERT/UPDATE/SELECT/DELETE -->
        <!--
        <privileges check="false">
            <schema name="TESTDB" dml="0110" >
                <table name="tb01" dml="0000"></table>
                <table name="tb02" dml="1111"></table>
            </schema>
        </privileges>
         -->
    </user>


    <!--<alarm>
        <url>10.186.22.103</url>
        <port>5700</port>
        <level>warn</level>
        <serverId>dbleServer</serverId>
        <componentId>componentId</componentId>
        <componentType>componentType</componentType>
    </alarm>-->

</dble:server>

schema.xml

users用戶表按照取模的方式平均拆分到了MySQL A和MySQL B兩個數據數據庫實例上算法

schema 邏輯數據庫信息，此數據庫爲邏輯數據庫，name 與 server.xml 中 schema對應;
dataNode 分片信息，此爲分片節點的定義；分片名字和schema的dataNode對應；分片與下面的dataHost 物理實例進行關聯；
dataHost 物理實例組信息，dataHost下能夠掛載同組的讀寫物理實例節點，實現高可用或者讀寫分離；

每一個節點的屬性逐一說明：sql

schema:

屬性說明：數據庫

- name 邏輯數據庫名，與 server.xml 中的 schema 對應； - table:

子屬性說明：express

    -  name 表名，物理數據庫中表名
    -  dataNode 表存儲到哪些節點，多個節點用逗號分隔
    -  primaryKey 主鍵，用於主鍵緩存和自增識別，不做主鍵約束
    -  autoIncrement 是否自增 
    -  rule 分片規則名，具體規則下文 rule 詳細介紹

dataNode
屬性說明：apache

name 節點名，與 table 中 dataNode 對應
datahost 物理實例組名，與 datahost 中 name 對應
database 物理數據庫中數據庫名；

dataHost

屬性說明：segmentfault

name 物理數據庫名，與 dataNode 中 dataHost 對應
balance 均衡負載的方式
switchtype 寫節點的高可用切換方式；等於1時，心跳不健康發生切換
heartbeat 心跳檢測語句，注意語句結尾的分號要加
writehost 寫物理實例

```
子屬性說明 ：
```

-  host 物理實例名
-  url 物理庫IP+Port
-  user 物理庫用戶
-  password 物理庫密碼

<?xml version="1.0"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!DOCTYPE dble:schema SYSTEM "schema.dtd">
<dble:schema xmlns:dble="http://dble.cloud/" version="2.19.03.0">

    <schema name="testdb">
        <table name="users" primaryKey="ID"  dataNode="dn1,dn2" rule="sharding-by-mod2" />
    </schema>
    
    <dataNode name="dn1" dataHost="Group1" database="testdb"/>
    <dataNode name="dn2" dataHost="Group2" database="testdb"/>
    
    <!-- 物理數據庫配置 -->
    <dataHost name="Group1" maxCon="1000" minCon="10" balance="0" switchType="1" slaveThreshold="100">
        <heartbeat>show slave status</heartbeat>
        <writeHost host="MySQLA" url="192.168.1.53:3306" user="root" password="123456"/>
    </dataHost>

    <dataHost name="Group2" maxCon="1000" minCon="10" balance="0" switchType="1" slaveThreshold="100">
        <heartbeat>show slave status</heartbeat>
         <writeHost host="MySQLA" url="192.168.1.116:3306" user="root" password="123456"/>
    </dataHost>
    
</dble:schema>

rule.xml

主要關注rule屬性，rule屬性的內容來源於rule.xml這個文件，DBLE支持多種分表分庫的規則後端

這裏選擇的sharding-by-mod2，是hash算法的特例，就是將數據平均拆分。由於我後端是兩臺物理庫，因此rule.xml中hashmod2對應的partitionCountt爲2

<?xml version="1.0" encoding="UTF-8"?>
<!--
  ~ Copyright (C) 2016-2019 ActionTech.
  ~ License: http://www.gnu.org/licenses/gpl.html GPL version 2 or higher.
  -->

<!-- - - Licensed under the Apache License, Version 2.0 (the "License");
    - you may not use this file except in compliance with the License. - You 
    may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 
    - - Unless required by applicable law or agreed to in writing, software - 
    distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT 
    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the 
    License for the specific language governing permissions and - limitations 
    under the License. -->
<!DOCTYPE dble:rule SYSTEM "rule.dtd">
<dble:rule xmlns:dble="http://dble.cloud/" version="2.19.03.0">
    <tableRule name="sharding-by-mod2">
        <rule>
            <columns>id</columns>
            <algorithm>hashmod2</algorithm>
        </rule>
    </tableRule>
    <function name="hashmod2" class="Hash">
        <property name="partitionCount">2</property>
        <property name="partitionLength">1</property>
    </function>

</dble:rule>