java jar包項目雙機冷備 高可用方案-實例

需求描述

定時跑批的（batch，process）jar包服務必須單機運行，又要保證高可用

• 192.168.0.47 （running） 192.168.0.48（backup-HA）
• 須要額外安裝的應用 ： python3 ，pip3（fastapi ,uvicorn）

（sorry 沒有寫註釋的習慣）

pip3 install fastapi

pip3 install uvicorn

解決方案

在47上提供應用信息的接口，48上調用47的接口判斷java服務狀態，當47上的服務丟失48上的冷備項目馬上啓動，當47上的項目恢復48上的冷備項目中止。

192.168.0.47 上的操做：

[root@skpay-java-47l shell]# cat kelan_monitor.py   
#!/usr/bin/python  
# -*- coding:utf-8 -*-

import subprocess  
import os, sys, json, datetime, time  
  
import locale  
  
import urllib.request

from fastapi import FastAPI

  
def check_project_sum(project):  
    project_num = subprocess.getoutput(  
        'ps -ef | grep {tproject}-service | grep -v grep | wc -l'.format(tproject=project))  
    project_num_int = int(project_num)  
    return project_num_int

  
def check_project_startedtime(project):  
    project_startedtime = subprocess.getoutput(  
        'ps -ef | grep {tproject}-service | grep -v grep | awk {pri}'.format(tproject=project, pri="'{print $5}'"))  
    return project_startedtime

  
def check_project_pid(project):  
    project_pid = subprocess.getoutput(  
        'ps -ef | grep {tproject}-service | grep -v grep | awk {pri}'.format(tproject=project, pri="'{print $2}'"))  
    #project_pid_int = int(project_pid)  
    return project_pid

  
def check_project_mem(project):  
    project_mem = subprocess.getoutput(  
        'ps -aux | grep {tproject}-service | grep -v grep | awk {pri}'.format(tproject=project, pri="'{sum+=$4}; END{print sum}'"))

    return project_mem

  
app = FastAPI()

  
@app.get("/")  
def read_root():  
    return {"messages": "hellokugou"}

  
@app.get("/items/{item_id}")  
def read_item(item_id: int, q: str = None):  
    return {"item_id": item_id, "q": q}

  
@app.get("/project/{project}")  
def read_project(project: str):  
    project_pid = check_project_pid(project)  
    print(project, '#######################project')  
    print(project_pid, '@@@@@@@@@@@@@@pid')

    return {"project_name": project, "project_pid": project_pid, "project_running-sum": check_project_sum(project),  
            "project_started-time": check_project_startedtime(project),'project_mem': check_project_mem(project)}

  
if __name__ == '__main__':  
    import uvicorn

    uvicorn.run(app=app,  
                host="192.168.0.47",  
                port=9999,  
                workers=1)  
[root@skpay-java-47l shell]# pwd  
/data/shell

測試fastapi 腳本

[root@skpay-java-48l shell]# curl -s http://192.168.0.47:9999/project/process  
{"project_name":"process","project_pid":"20770","project_running-sum":1,"project_started-time":"18:55","project_mem":"2.3"}[root@skpay-java-48l shell]#   
[root@skpay-java-48l shell]#   
[root@skpay-java-48l shell]# curl -s http://192.168.0.47:9999  
{"messages":"hellokugou"}

nohup python kelan_monitor.py & //後臺啓動

192.168.0.48 上的操做：

[root@skpay-java-48l shell]# cat backup_monitor.py   
#!/usr/bin/python  
# -*- coding:utf-8 -*-

import subprocess  
import os, sys, json, datetime, time  
import locale  
import urllib.request  
import socket,requests

  
def dingding(messages):  
    content = {  
        "msgtype": "text",  
        "text": {  
            "content": "{tmessages}".format(tmessages=messages)  
        },  
        "at": {  
    #        "atMobiles": [  
    #            # 單獨 @ 某我的  
    #            "131xxxxxx81"  
    #        ],  
    #        "isAtAll": Falsethe monitor kelan kelan_monitor.py is not running'

            # @ 全部人  
            "isAtAll": True  
        }  
    }

    headers = {"Content-Type": "application/json;charset=utf-8"}

    url = "https://oapi.dingtalk.com/robot/send?access_token=5c88d672cxxxxxxxxxbxa"

    r = requests.post(url=url,headers=headers,json=content)  
    return r.content

  
app_file_path = '/data/app/'

  
def start_project(project):  
    start_cmd = 'sh +x /data/app/{tproject}/bin/{tproject}_std.sh start'.format(tproject=project)  
    os.system(start_cmd)

  
def check_project_sum(project):  
    project_num = subprocess.getoutput(  
        'ps -ef | grep {tproject}-service | grep -v grep | wc -l'.format(tproject=project))  
    project_num_int = int(project_num)  
    return project_num_int

  
def check_project_pid(project):  
    project_pid = subprocess.getoutput(  
        'ps -ef | grep {tproject}-service | grep -v grep | awk {pri}'.format(tproject=project, pri="'{print $2}'"))  
    # project_pid_int = int(project_pid)  
    return project_pid

  
def kill_project(project):  
    kill_cmd = 'kill -9 {tpid}'.format(tpid=check_project_pid(project))  
    os.system(kill_cmd)

project_list = ['batch','process']

#check_monitor = subprocess.getoutput('curl http://192.168.0.47:9999/project/{tproject}'.format(tproject=app_file_path))  
check_monitor = subprocess.getoutput('curl -s http://192.168.0.47:9999')  
#print(check_monitor,type(check_monitor))  
dict_check_monitor = json.loads(check_monitor)  
#print(dict_check_monitor,type(dict_check_monitor),dict_check_monitor['messages'])

localtime = time.asctime( time.localtime(time.time()) )  
#print(localtime)  
sock22 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  
sock22result = sock22.connect_ex(('192.168.0.47',22))  
#print(sock22result,type(sock22result),'port is open')

  
if dict_check_monitor['messages'] == 'hellokugou':  
    print(localtime,'the monitor kelan kelan_monitor.py is running')  
    for i in project_list:  
        if check_project_sum(i) == 0:  
            check_projectstr = subprocess.getoutput('curl -s http://192.168.0.47:9999/project/{tproject}'.format(tproject=i))  
            #print(check_projectstr,type(check_projectstr))  
            check_projectdict = json.loads(check_projectstr)  
            #print(check_projectdict,type(check_projectdict),check_projectdict['project_running-sum'],type(check_projectdict['project_running-sum']))  
            if check_projectdict['project_running-sum'] == 1:  
                print(i,'is running')  
            else:  
                start_project(i)  
        else:  
            check_projectstr = subprocess.getoutput('curl -s http://192.168.0.47:9999/project/{tproject}'.format(tproject=i))  
            check_projectdict = json.loads(check_projectstr)  
            if check_projectdict['project_running-sum'] == 1:  
                print(i,'is running')  
                kill_project(i)  
            else:  
                print(i,'in backup is running')  
else:  
    if sock22result == 0:  
        print(localtime,'the monitor kelan kelan_monitor.py is not running')  
        messages = '{tlocaltime} the monitor kelan kelan_monitor.py is not running'.format(tlocaltime=localtime)  
        dingding(messages)  
    else:  
        print(localtime,'the 192.168.0.47 network is wrong or ECS not star power')  
        for i in project_list:  
            start_project(i)  
[root@skpay-java-48l shell]# pwd  
/data/shell  
[root@skpay-java-48l shell]# ls  
backup_monitor.py  monitor_backup.log  
[root@skpay-java-48l shell]#

"30 /usr/bin/python /data/shell/backup_monitor.py >> monitor_backup.log" //定時每半小時出發一回，根據需求自行修改

手動測試

#####47停process服務

  
process #######################project  
20770 @@@@@@@@@@@@@@pid  
INFO:     192.168.0.48:42574 - "GET /project/process HTTP/1.1" 200 OK  
INFO:     192.168.120.20:55351 - "GET / HTTP/1.1" 200 OK  
INFO:     192.168.120.20:55351 - "GET /favicon.ico HTTP/1.1" 404 Not Found  
process #######################project  
20770 @@@@@@@@@@@@@@pid  
INFO:     192.168.120.20:55368 - "GET /project/process HTTP/1.1" 200 OK  
 

[root@skpay-java-48l data]# python backup_monitor.py   
Thu Jun 11 18:55:06 2020 the monitor kelan kelan_monitor.py is running  
batch is running  
process in backup is running  
  
[root@skpay-java-48l data]# python backup_monitor.py   
Thu Jun 11 18:55:15 2020 the monitor kelan kelan_monitor.py is running  
batch is running  
process in backup is running  
 

[root@skpay-java-48l data]# python backup_monitor.py   
Thu Jun 11 18:55:31 2020 the monitor kelan kelan_monitor.py is running  
batch is running  
process in backup is running

####47 process服務開啓  
[root@skpay-java-48l data]#   
[root@skpay-java-48l data]#   
[root@skpay-java-48l data]#   
[root@skpay-java-48l data]# python backup_monitor.py   
Thu Jun 11 18:56:40 2020 the monitor kelan kelan_monitor.py is running  
batch is running  
process is running