OpenStack API部分高可用配置（二）

 

1、安裝與配置HAProxy

 

一、調整內核參數,容許綁定VIP:

vim /etc/sysctl.conf

【內容】

net.ipv4.ip_nonlocal_bind=1

 

sysctl -p

 

二、安裝HAProxy:

【源碼安裝】

wget -c http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.23.tar.gz

【備用連接：http://down1.chinaunix.net/distfiles/haproxy-1.4.21.tar.gz】

tar zxvf haproxy-1.4.23.tar.gz

cd haproxy-1.4.23

make TARGET=linux26      #若是是32位機器,則make TARGET=linux26 ARCH=i386

make install

mkdir -p /etc/haproxy

cp examples/haproxy.cfg /etc/haproxy/haproxy.cfg【能夠跳過】

 

【yum安裝】

yum install  haproxy

 

三、配置HAProxy:

vim /etc/haproxy/haproxy.cfg

【內容】

global

    daemon

 

defaults

    mode http

    maxconn 10000

    timeout connect 10s

    timeout client 10s

    timeout server 10s

 

frontend keystone-admin-vip

    bind 10.10.102.45:35357

    default_backend keystone-admin-api

 

frontend keystone-public-vip

    bind 10.10.102.45:5000

    default_backend keystone-public-api

 

frontend quantum-vip

    bind 10.10.102.45:9696

    default_backend quantum-api

 

frontend glance-vip

    bind 10.10.102.45:9191

    default_backend glance-api

 

frontend glance-registry-vip

    bind 10.10.102.45:9292

    default_backend glance-registry-api

 

frontend nova-ec2-vip

    bind 10.10.102.45:8773

    default_backend nova-ec2-api

 

frontend nova-compute-vip

    bind 10.10.102.45:8774

    default_backend nova-compute-api

 

frontend nova-metadata-vip

    bind 10.10.102.45:8775

    default_backend nova-metadata-api

 

frontend cinder-vip

    bind 10.10.102.45:8776

    default_backend cinder-api

 

backend keystone-admin-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:35357 check inter 10s

    server mesa-virt-02 10.10.102.7:35357 check inter 10s

   

 

backend keystone-public-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:5000 check inter 10s

    server mesa-virt-02 10.10.102.7:5000 check inter 10s

   

 

backend quantum-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:9696 check inter 10s

    server mesa-virt-02 10.10.102.7:9696 check inter 10s

   

 

backend glance-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:9191 check inter 10s

    server mesa-virt-02 10.10.102.7:9191 check inter 10s

   

 

backend glance-registry-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:9292 check inter 10s

    server mesa-virt-02 10.10.102.7:9292 check inter 10s

   

 

backend nova-ec2-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:8773 check inter 10s

    server mesa-virt-02 10.10.102.7:8773 check inter 10s

  

 

backend nova-compute-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:8774 check inter 10s

    server mesa-virt-02 10.10.102.7:8774 check inter 10s

   

 

backend nova-metadata-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:8775 check inter 10s

    server mesa-virt-02 10.10.102.7:8775 check inter 10s

   

 

backend cinder-api

    balance roundrobin

    server mesa-virt-01 10.10.102.6:8776 check inter 10s

    server mesa-virt-02 10.10.102.7:8776 check inter 10s

   

 

        

四、HAProxy的啓動管理腳本:

vim /etc/init.d/haproxy

【內容】

# cat /etc/init.d/haproxy

#!/bin/bash

#

# chkconfig: 2345 85 15

# description: HA-Proxy is a TCP/HTTP reverse proxy which is particularly suited \

#              for high availability environments.

# processname: haproxy

# config: /etc/haproxy.cfg

# pidfile: /var/run/haproxy.pid

 

# Source function library.

if [ -f /etc/init.d/functions ]; then

  . /etc/init.d/functions

elif [ -f /etc/rc.d/init.d/functions ] ; then

  . /etc/rc.d/init.d/functions

else

  exit 0

fi

 

CONF_FILE="/etc/haproxy/haproxy.cfg"

HAPROXY_BINARY="/usr/local/sbin/haproxy"

PID_FILE="/var/run/haproxy.pid"

 

# Source networking configuration.

. /etc/sysconfig/network

 

# Check that networking is up.

[ ${NETWORKING} = "no" ] && exit 0

 

[ -f ${CONF_FILE} ] || exit 1

 

RETVAL=0

 

start() {

  $HAPROXY_BINARY -c -q -f $CONF_FILE

  if [ $? -ne 0 ]; then

    echo "Errors found in configuration file."

    return 1

  fi

 

  echo -n "Starting HAproxy: "

  daemon $HAPROXY_BINARY -D -f $CONF_FILE -p $PID_FILE

  RETVAL=$?

  echo

  [ $RETVAL -eq 0 ] && touch /var/lock/subsys/haproxy

  return $RETVAL

}

 

stop() {

  echo -n "Shutting down HAproxy: "

  killproc haproxy -USR1

  RETVAL=$?

  echo

  [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/haproxy

  [ $RETVAL -eq 0 ] && rm -f $PID_FILE

  return $RETVAL

}

 

restart() {

  $HAPROXY_BINARY -c -q -f $CONF_FILE

  if [ $? -ne 0 ]; then

    echo "Errors found in configuration file, check it with 'haproxy check'."

    return 1

  fi

  stop

  start

}

 

check() {

  $HAPROXY_BINARY -c -q -V -f $CONF_FILE

}

 

rhstatus() {

  pid=$(pidof haproxy)

  if [ -z "$pid" ]; then

    echo "HAProxy is stopped."

    exit 3

  fi

  status haproxy

}

 

condrestart() {

  [ -e /var/lock/subsys/haproxy ] && restart || :

}

 

# See how we were called.

case "$1" in

  start)

    start

    ;;

  stop)

    stop

    ;;

  restart)

    restart

    ;;

  reload)

    restart

    ;;

  condrestart)

    condrestart

    ;;

  status)

    rhstatus

    ;;

  check)

    check

    ;;

  *)

    echo $"Usage: haproxy {start|stop|restart|reload|condrestart|status|check}"

    RETVAL=1

esac

 

exit $RETVAL

 

 

五、檢查HAProxy的配置是否正確:

# /etc/init.d/haproxy check

Configuration file is valid

 

【可能要通過以下處理：】

1). 【問題】

etc/init.d/haproxy check

-bash: /etc/init.d/haproxy: Permission denied

【解決】

cd /etc/init.d

chmod a+x haproxy

2). 【問題】

/etc/init.d/haproxy check

/etc/init.d/haproxy: line 70: /usr/local/bin/haproxy: No such file or directory

【解決】

cp /usr/local/sbin/haproxy /usr/local/bin/

 

 

 

 

 

2、Pacemaker + CoroSync配置

一、先定義一些資源屬性約束(包括禁止STONITH錯誤,忽略Quorum,防止資源在恢復以後移動等):

# crm configure

property stonith-enabled=false

property no-quorum-policy=ignore

rsc_defaults resource-stickiness=100

rsc_defaults failure-timeout=0

rsc_defaults migration-threshold=10

 

二、配置VIP資源:

crm(live)configure#

primitive api-vip ocf:heartbeat:IPaddr2 params ip=10.10.102.45 cidr_netmask=24 op monitor interval=5s

 

三、配置HAProxy資源:

crm(live)configure#

primitive haproxy lsb:haproxy op monitor interval="5s"

 

四、定義運行的HAProxy和VIP必須在同一節點上:

crm(live)configure#

colocation haproxy-with-vip INFINITY: haproxy api-vip

 

五、定義先接管VIP以後才啓動HAProxy:

crm(live)configure#

order haproxy-after-IP mandatory: api-vip haproxy

 

六、驗證並提交配置:

crm(live)configure# verify

crm(live)configure# commit

crm(live)configure# quit

 

 

七、資源狀態查看

 

查看資源狀態:

crm_mon -1 【可能有問題】

或

crm status

 

在機器1上查看:

/etc/init.d/haproxy status

haproxy (pid  1629) is running...

ip addr show eth0

2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000

    link/ether 08:00:27:67:ab:7e brd ff:ff:ff:ff:ff:ff

    inet 10.10.102.6/24 brd 192.168.1.255 scope global eth0

    inet 10.10.102.45/24 brd 192.168.1.255 scope global secondary eth0【此處會顯示虛擬ip】

    inet6 fe80::a00:27ff:fe67:ab7e/64 scope link

       valid_lft forever preferred_lft forever

 

 

在機器2上查看:

/etc/init.d/haproxy status

HAProxy is stopped.

ip addr show eth0

2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000

    link/ether 08:00:27:67:ab:7e brd ff:ff:ff:ff:ff:ff

    inet 10.10.102.7/24 brd 192.168.1.255 scope global eth0

    inet6 fe80::a00:27ff:fe67:ab7e/64 scope link tentative dadfailed

       valid_lft forever preferred_lft forever

從上面的這些信息,能夠知道VIP綁定在機器1上,同時只有機器1上的HAProxy已經啓動.

 

 

 

八、Failover測試

在pacemaker中standby正在運行資源的機器，觀察集羣資源在兩臺機器中間的轉移。

【用到的命令】

crm node standby 機器名

crm status

crm node online 機器名

 

 

 

 

 

 

3、以keystone爲例配置API服務

 

一、/etc/keystone/keystone.conf配置【若是是其餘API服務應該修改認證地址，不修改綁定地址】

【貌似無需修改，都綁定監聽本身的物理ip】

 

二、service endpoint配置【在一臺機器上修改就好了，因爲此時mysql已經實現HA，因此在任意一臺機器上修改endpoint對兩臺機器都是全局有效的】

1) 以當前admin身份登陸keystone

2）建立綁定vip的endpoint

keystone service-list

【記住service-list中keystone的service-id】

 

keystone endpoint-create --region RegionOne \

--service-id  $(keystone service-list | awk '/ keystone / {print $2}') \

  --publicurl "http://10.10.102.45:5000/v2.0" \

  --internalurl "http://10.10.102.45:5000/v2.0" \

  --adminurl http://10.10.102.45:35357/v2.0

【10.10.102.45爲vip】

 

3）刪除舊的endpoint

keystone endpoint-list

【記住舊的endpoint的id號】

 

keystone endpoint-delete 21a7b25a08d74882a711f09f0c313170

 

4）重啓服務

service openstack-keystone restart

 

5）驗證

export OS_AUTH_URL=http://10.10.102.45:35357/v2.0/

【10.10.102.45爲vip】

 

keystone user-list（正常）

 

 

4、測試與驗證：

 

因爲API服務沒有做爲pacemaker的資源，因此必須在兩臺機器中的一臺上手動停掉API服務，而後在其餘機器上用VIP登陸API服務

【例如，用到的命令以下】

service openstack-keystone stop

service openstack-keystone status

 

 

 

 

 

參考網站：

http://openstack.redhat.com/Load_Balance_OpenStack_API#HAProxy

http://www.zrwm.com/?p=6983

http://docwiki.cisco.com/wiki/COE_Grizzly_Release:_High-Availability_Manual_Installation_Guide#Glance_Installation

http://openstack.redhat.com/RDO_HighlyAvailable_and_LoadBalanced_Control_Services 【附帶Mysql和Qpid的HA配置】