corosync+pacemaker+crm實現drbd高可用

時間 2021-04-26

標籤 html node linux sql shell vim less ssh ide post 欄目 HTML 简体版

原文原文鏈接

系統：Centos 7.2 內核3.10.0-327.el7.x86_64 # 內核須要高於2.6.32html

Drbd : 192.168.8.111：node1/dev/drdb0 /mydetanode

192.168.8.112 : node2linux

Mysql_vip: 192.168.8.200 #下章實現
sql

# 須要的軟件包：shell

mariadb-5.5.53-linux-i686.tar.gzvim

drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpmless

kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpmssh

yum 須要安裝的軟件 pcs, ansible, crmsh, libxsltide

# drbd是基於主機名通訊的在/etc/hosts中增長 post

# 以本地ip地址如192.168.8.111 node1這樣的方式添加一個地址一行、 127.0.0.1 不要填uname -n的名稱

# 安裝crmsh yum源以下 [root@node1 ~]# vim /etc/yum.repos.d/ha.repo

[network_ha-clustering_Stable]

name=Stable High Availability/Clustering packages (CentOS_CentOS-7)

type=rpm-md

baseurl=http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7/

gpgcheck=1

gpgkey=http://download.opensuse.org/repositories/network:/ha-clustering:/Stable/CentOS_CentOS-7//repodata/repomd.xml.key

enabled=1

# 安裝crmsh 單臺主機安裝就行

yum -y install crmsh libxslt ansible pcs

# 主機互信

[root@node1 ~]# ssh-keygen

[root@node1 ~]# cat .ssh/id_rsa.pub > .ssh/authorized_keys

[root@node1 ~]# chmod go= .ssh/authorized_keys

[root@node1 ~]# scp -p .ssh/id_rsa.pub .ssh/authorized_keys 192.168.8.112:/root/.ssh/

# 出現如下錯誤

scp: /root/.ssh/: No such file or directory

# 解決方案

鏈接node2建立.ssh mkdir /root/.ssh

# drbd配置段

# 安裝drbd前先準備兩塊硬盤 node1 跟node2 一樣配置

rpm -ivh http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/drbd84-utils-8.9.1-1.el7.elrepo.x86_64.rpm

yum -y install http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/kmod-drbd84-8.4.6-1.el7.elrepo.x86_64.rpm

[root@node1 drbd-8.4.8-1]# modprobe drbd

[root@node1 drbd-8.4.8-1]# lsmod | grep drbd

drbd 405537 0

libcrc32c 12644 2 xfs,drbd

# 將新建的磁盤分區

[root@node1 sbin]# fdisk /dev/sdb# 本身新建就成、兩邊大小保持一致跟raid1同樣

# 配置drbd

[root@node1 sbin]#

# drbd配置: http://502245466.blog.51cto.com/7559397/1298945

# drbd常見錯誤: http://www.j3j5.com/post-128.html

[root@node1 drbd.d]# vim global_common.conf

global {

usage-count no;

}

common {

protocol C;

handlers {

pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";

pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";

local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";

}

startup {

}

options {

}

disk {

on-io-error detach;

}

net {

cram-hmac-alg "sha1";

shared-secret "xiong123";

}

syncer {

rate 1024M;

}

[root@node1 drbd.d]# vim mydrbd.res

resource mydrbd {

device /dev/drbd0;

disk /dev/sdb1;

meta-disk internal;

on node1{on爲開啓，node1爲節點名稱

address 192.168.8.111:7789;

}

on node2{

address 192.168.8.112:7789;

}

[root@node1 drbd.d]# scp mydrbd.res global_common.conf node2:/usr/local/drbd/etc/drbd.d/

############測試drbd是否成功######

[root@node1 drbd.d]# drbdadm create-md mydrbd

initializing activity log

NOT initializing bitmap

Writing meta data...

New drbd meta data block successfully created

# 配置啓動不然corosync ocf沒法直接定義drbd啓動

[root@node1 sbin]# systemctl enable drbd.service

# 此處爲測試drbd是否完整兩邊都得同時啓動

[root@node1 sbin]# systemctl start drbd.service

[root@node1 drbd.d]# drbdadm primary --force mydrbd # 將主機強制成爲主節點

[root@node73 drbd.d]# drbd-overview

0:mydrbd/0 Connected Primary/Secondary UpToDate/UpToDate# Primary/Secondary 主從，另外一端是Secondary/Primary

#格式磁盤

[root@node1 drbd.d]# mkfs.ext4 /dev/drbd0

# 掛載兩個節點都須要建立 /mydata/

[root@node1 drbd.d]# mount /dev/drbd0 /mydata/

[root@node1 drbd.d]# df -Th

/dev/drbd0 ext4 4.8G 20M 4.6G 1% /mydata

複製一個文件至/mydata目錄中，並將主drbd設置爲從

[root@node1 ~]# umount /mydata/

[root@node1 ~]# drbdadm secondary mydrbd

[root@node1 ~]# drbd-overview

0:mydrbd/0 Connected Secondary/Secondary UpToDate/Diskless

# 節點2操做配置無需格式化/dev/drbd0

[root@node2 drbd.d]# drbdadm primary mydrbd

[root@node2 drbd.d]# mkdir /mydata

[root@node2 drbd.d]# mount /dev/drbd0 /mydata/

[root@node2 drbd.d]# cd /mydata/

[root@node2 mydata]# ls

issue lost+found

# 最後將drbd服務都中止

[root@node1 /]# umount /mydata/

[root@node1 /]# systemctl stop drbd

[root@node2 /]# systemctl stop drbd

############################測試結束########################################

# pcs配置drbd高可用

# 安裝

# 安裝pcs軟件若是報/sbin/xxx命令的名字將crmsh.repo先改爲crmsh.repo.bak 再安裝

[root@node1 sbin]# ansible ha -m yum -a "name=pcs state=installed"

# pcs認證須要

[root@node1 sbin]# ansible ha -m shell -a 'echo "xiong" | passwd --stdin hacluster'

# 啓動pcs服務

[root@node1 sbin]# ansible ha -m service -a "name=pcsd state=started enabled=yes"

# 認證集羣服務

[root@node1 sbin]# pcs cluster auth node1 node2

Username: hacluster

Password:

node1: Authorized

node2: Authorized

# 添加一個集羣名爲myha 並設置組播地址爲235.35.32.24

[root@node1 sbin]# pcs cluster setup --name myha node1 node2 --mcast0 235.35.32.24

# 檢查集羣服務是否正常

[root@node1 sbin]# corosync-cmapctl | grep mem

runtime.totem.pg.mrp.srp.memb_commit_token_rx (u64) = 4

runtime.totem.pg.mrp.srp.memb_commit_token_tx (u64) = 4

runtime.totem.pg.mrp.srp.memb_join_rx (u64) = 5

runtime.totem.pg.mrp.srp.memb_join_tx (u64) = 3

runtime.totem.pg.mrp.srp.memb_merge_detect_rx (u64) = 332

runtime.totem.pg.mrp.srp.memb_merge_detect_tx (u64) = 332

runtime.totem.pg.mrp.srp.members.1.config_version (u64) = 0

runtime.totem.pg.mrp.srp.members.1.ip (str) = r(0) ip(192.168.8.111)

runtime.totem.pg.mrp.srp.members.1.join_count (u32) = 1

runtime.totem.pg.mrp.srp.members.1.status (str) = joined

runtime.totem.pg.mrp.srp.members.2.config_version (u64) = 0

runtime.totem.pg.mrp.srp.members.2.ip (str) = r(0) ip(192.168.8.112)

runtime.totem.pg.mrp.srp.members.2.join_count (u32) = 1

runtime.totem.pg.mrp.srp.members.2.status (str) = joined

# 檢查狀態

[root@node1 sbin]# pcs status

Cluster name: myha

WARNING: no stonith devices and stonith-enabled is not false

Stack: corosync

Current DC: node1 (version 1.1.13-10.el7-44eb2dd) - partition with quorum

2 nodes and 0 resources configured

Online: [ node1 node2 ]

# 配置高可用

crm(live)configure# primitive mydrbd ocf:linbit:drbd params drbd_resource="mystore" op monitor role="Master" interval=30s timeout=30s op monitor role="Slave" timeout=30s op start timeout=240s op stop timeout=100s

####################正確模板##########################################

primitive mydatas ocf:linbit:drbd \

params drbd_resource=mystore \

op monitor role=Master interval=10s timeout=20s \

op monitor role=Slave interval=20s timeout=20s \

op start timeout=240s interval=0 \

op stop timeout=100s interval=0

######################################################################

crm(live)configure# ms mydrbd_master mydrbd meta clone-max=2 clone-node-max=1 master-max=1 master-node-max=1

crm(live)configure# verify #檢查是否成功

crm(live)# status

Online: [ node73 node74 ]

Master/Slave Set: ms_mydatas [mydatas]

Masters: [ node73 ]

Slaves: [ node74 ]

# 配置文件掛載目錄

crm(live)configure# primitive mount ocf:heartbeat:Filesystem params device="/dev/drbd0" directory="/mydata" fstype="ext4" op start timeout=60s op stop timeout=60s

# 定義排序約束讓mount一直跟着ms_mydatas的主

crm(live)configure# colocation mount_with_ms_mydatas inf: mount ms_mydatas:Master

# 檢查是否成功

[root@node73 ~]# df -Th

/dev/drbd0 ext4 16G 45M 15G 1% /mydata

[root@node73 ~]# drbd-overview

0:mystore/0 Connected Primary/Secondary UpToDate/UpToDate /mydata ext4 16G 45M 15G 1%

####################### drbd主從配置文件###############################

primitive mount Filesystem \

params device="/dev/drbd0" directory="/mydata" fstype=ext4 \

op start timeout=60s interval=0 \

op stop timeout=60s interval=0

primitive mydatas ocf:linbit:drbd \

params drbd_resource=mystore \

op monitor role=Master interval=10s timeout=20s \

op monitor role=Slave interval=20s timeout=20s \

op start timeout=240s interval=0 \

op stop timeout=100s interval=0

ms ms_mydatas mydatas \

meta clone-max=2 clone-node-max=1 master-max=1 master-node-max=1 notify=true

colocation mount_with_ms_mydatas inf: mount ms_mydatas:Master

#########################################################################

# 錯誤整理

一、ERROR: error: unpack_resources:Resource start-up disabled since no STONITH resources have been defined

error: unpack_resources:Either configure some or disable STONITH with the stonith-enabled option

error: unpack_resources:NOTE: Clusters with shared data need STONITH to ensure data integrity

解決：crm(live)configure# property stonith-enabled=false

# 錯誤

[root@node1 drbd.d]# drbdadm primary mystore

0: State change failed: (-2) Need access to UpToDate data

Command 'drbdsetup-84 primary 0' terminated with exit code 17

# 解決方案

[root@node1 drbd.d]# drbdadm primary mystore --force

# 錯誤

[root@node1 drbd.d]# pcs status

Error: cluster is not currently running on this node

# 解決方案

[root@node1 drbd.d]# pcs cluster start --all

# 啓動主從失敗

Failed Actions:

* mydrbd_stop_0 on node74 'not configured' (6): call=8, status=complete, exitreason='none',

last-rc-change='Wed May 24 15:27:42 2017', queued=0ms, exec=138ms

* mydrbd_stop_0 on node73 'not configured' (6): call=8, status=complete, exitreason='none',

last-rc-change='Wed May 24 15:27:41 2017', queued=0ms, exec=134ms

# 解決方案

設置各類超時時長都加上"s" primitive mydrbd ocf:linbit:drbd params drbd_resource="mystore" op monitor role="Master" interval=30s timeout=30s op monitor role="Slave" timeout=30s op start timeout=240s op stop timeout=100s

drbd yum包下載地址：http://elrepo.reloumirrors.net/elrepo/el7/x86_64/RPMS/

配置參閱連接： http://blog.csdn.net/qq_21127313/article/details/68210977