本程序主要實現網頁掛馬關鍵字檢測,原理是這樣的,經過定時分析用戶訪問的IIS web日誌,對日誌的裏的URL進行訪問,而後檢索是否包含相關的關鍵字,有就入庫,這只是go實現自動檢索及入庫,能夠加個前端,實現加關鍵及報警功能html
package main前端
import (
"bufio"
"code.google.com/p/mahonia"
"fmt"
"io"
"io/ioutil"
"labix.org/v2/mgo"
"labix.org/v2/mgo/bson"
"log"
"net"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
//qqwry爲IP庫
//ftp "github.com/jlaffaye/ftp"
//"github.com/yinheli/qqwry"
)
const Version = "CDN1.0"
//var ServerList []ServerMember
//PageKeyWord用來保存可疑Url及這個頁面包含有哪些關鍵字,以及日誌記錄於哪臺服務器及日誌文件,並最終插入數據庫
type PageHasKeyWord struct {
Url string
KeyWords []string
// UserIp string
LogFile string
ServerIp string
Audit bool
Auditor string //記錄誰審覈
AuditTime string //記錄審覈時間
Level int //可疑級別
}
type DoubtfulKeyWord struct {
Id bson.ObjectId "_id"
KeyWord string
//Level string
}
type UrlsToAnalysit struct {
Urls []string
LogFile string
ServerIp string
Domain string
}
//定義MonthTOstr映射,用來將月份換爲數字string,go裏的月份用英文單詞表示,要注意的go賦值要在函數體裏賦值
var MonthTOstr = make(map[string]string)
func main() {
//fmt.Printf("%s", getLogFileName())
if len(os.Args) != 4 {
log.Fatal("Usage:" + filepath.Base(os.Args[0]) + " log_dir " + "db_server_ip_address test")
os.Exit(1)
}
logFileName, _, _ := getLogFileName()
fmt.Println(time.Now())
logDbServer := os.Args[2]
dir := os.Args[1]
dbname := "webmonitordb"
//doubtfulKW爲string數組,從數據庫中獲取可疑檢索的URL,注意的是go是區分大小寫的,所以要注意mongodb裏的庫名及collection名大小寫
//doubtfulKW := []string{}
keyWords := getdoubtfulkeyword(logDbServer, dbname, "DangeroursWords")
//wordlist := []string{"百家樂", "博彩網", "網上賭場", "娛樂城", "賭博網站", "皇冠投注", "皇冠開戶", "真龍娛樂城", "三亞娛樂城"}
/*for n, item := range keyWords {
fmt.Println(n, item.KeyWord)
}*/
if os.Args[3] == "test" {
fmt.Println("wait!")
time.Sleep(time.Duration(90) * time.Second)
}
fmt.Println("start!")
filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
if f == nil {
return err
}
if f.IsDir() {
return nil
}
if f.Name() == logFileName {
//fmt.Println(path)
//fmt.Println(time.Now())
urls := logFileAnalysit(path)
//fmt.Println(urls)
for _, v := range urls.Urls {
//fmt.Println(n, ":", v)
url := "http://" + urls.Domain + v
//fmt.Println(n, url)
pagestr := getPage(url)
findWord(pagestr, url, urls.LogFile, urls.ServerIp, keyWords)
//fmt.Println(n)
}
}
return nil
})
}
func logFileAnalysit(logFile string) UrlsToAnalysit {
readLogFile, err := os.Open(logFile)
if err != nil {
log.Fatal(err)
}
defer readLogFile.Close()
//pathFields的做用是將日誌path解析成一個數據,從而能夠獲得日誌的域名
pathFields := strings.Split(logFile, "\\")
var domainName string
if len(pathFields) > 3 {
domainName = pathFields[len(pathFields)-3]
}
var Urls UrlsToAnalysit
Urls.Domain = domainName
Urls.LogFile = logFile
Urls.ServerIp = getLocalIpAddr()
Urls.Urls = append(Urls.Urls, "/") //監控站點首頁
// analysitTime := time.Now()
bfLogFile := bufio.NewReader(readLogFile)
//定義一個gbk轉utf8解碼器
//enc := mahonia.NewDecoder("gbk")
for {
logRecordStr, err := bfLogFile.ReadString('\n')
if err == io.EOF {
//注意,這裏要使用break,而不是return,return會使整個個程序退出了,break只是中斷當前的for循環
break
}
//以"#"開頭的要跳過,iiS 日誌前幾行是作了註釋的
if strings.HasPrefix(logRecordStr, "#") {
continue
}
//recordItems 是個string數組,用來臨時保存每行記錄裏的字段
//#Fields: date time s-sitename s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) sc-status sc-substatus sc-win32-status time-taken
fields := strings.Split(logRecordStr, " ")
//scstatus爲服務器狀態碼的index
if len(fields) < 13 { //不滿16個字段的跳過
continue
}
url := strings.ToLower(fields[5])
//fields[9] != Urls.ServerIp 過濾掉本身訪問的IP記錄
if (strings.HasSuffix(url, "htm") || strings.HasSuffix(url, "html")) && fields[12] == "200" && fields[9] != Urls.ServerIp {
flag := true
for _, v := range Urls.Urls {
if v == url {
flag = false
break
}
}
if flag {
Urls.Urls = append(Urls.Urls, url)
}
}
}
return Urls
}
//getLogFileName()根據當前的時間,生成一個將要被分析日誌文件名,由於不一樣IIS站點每小時時生成一個日誌文件,命名格式不exyymmddhh.log
func getLogFileName() (string, string, string) {
MonthTOstr := map[string]string{"January": "01",
"February": "02",
"March": "03",
"April": "04",
"May": "05",
"June": "06",
"July": "07",
"August": "08",
"September": "09",
"October": "10",
"November": "11",
"December": "12"}
timenow := time.Now()
year, month, day := timenow.Date()
//monthStr := month.String()
hour, _, _ := timenow.Clock()
yearStr := strings.TrimLeft(strconv.Itoa(year), "20") //去掉前面的四位數年份如"2014"年的「20」
dayStr, hourStr := strconv.Itoa(day), strconv.Itoa(hour)
if day < 10 {
dayStr = "0" + dayStr
}
if hour < 10 {
hourStr = "0" + hourStr
}
fileName := "ex" + yearStr + MonthTOstr[month.String()] + dayStr + hourStr + ".log"
logDay := yearStr + MonthTOstr[month.String()] + dayStr
logMonth := yearStr + MonthTOstr[month.String()]
//monthSrt := strconv.Itoa(timenow.Month())
//fmt.Println(fileName, logDay)
return fileName, logDay, logMonth
//fmt.Println(fileName)
}
func getPage(page string) string {
resp, err := http.Get(page)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
enc := mahonia.NewDecoder("gb2312")
pageData, _ := ioutil.ReadAll(resp.Body)
//return string(pageData)
PageStr := enc.ConvertString(string(pageData))
return PageStr
}
func findWord(str, url, logfile, serverip string, wordlist []DoubtfulKeyWord) {
var phkw PageHasKeyWord
flag := false
for _, item := range wordlist {
if strings.Contains(str, item.KeyWord) {
//fmt.Println("the page contains the word:", item.KeyWord, url)
phkw.KeyWords = append(phkw.KeyWords, item.KeyWord)
flag = true
}
}
if flag {
phkw.Url = url
phkw.LogFile = logfile
phkw.ServerIp = serverip
phkw.Audit = false
phkw.Level = len(phkw.KeyWords)
interdatadb("192.168.111.28", "webmonitordb", "dangerPage", phkw)
}
}
func interdatadb(dbserver, dbname, celection string, items PageHasKeyWord) {
session, err := mgo.Dial(dbserver)
if err != nil {
panic(err)
}
defer session.Close()
// Optional. Switch the session to a monotonic behavior.
session.SetMode(mgo.Monotonic, true)
c := session.DB(dbname).C(celection)
//fmt.Println(items)
//time.Sleep(time.Duration(90) * time.Second)
err = c.Insert(&PageHasKeyWord{items.Url, items.KeyWords, items.LogFile, items.ServerIp, items.Audit, items.Auditor, items.AuditTime, items.Level})
//err = c.Insert(&LogItems(logItem))
if err != nil {
panic(err)
}
}
func getdoubtfulkeyword(dbserver, dbname, collection string) []DoubtfulKeyWord {
// 鏈接數據庫
session, err := mgo.Dial(dbserver)
if err != nil {
panic(err)
}
defer session.Close()
// 獲取數據庫,獲取集合
c := session.DB(dbname).C(collection)
kws := []DoubtfulKeyWord{}
//kw := []string{}
//var ta task
err = c.Find(bson.M{}).All(&kws)
if err != nil {
panic(err)
}
/*for n, item := range kws {
fmt.Println(n, item.Id, item.KeyWord)
}*/
return kws
}
func getLocalIpAddr() string {
//這裏使用一個合法的IP就好了,端口隨便,即便沒有打開也行,也許由於使用UDP,若是用TCP的話,對端不打開就會有問題
conn, err := net.Dial("udp", "192.168.18.51:80")
if err != nil {
//fmt.Println(err.Error())
return "127.0.0.1"
}
defer conn.Close()
//fmt.Println(conn.LocalAddr().String())
//conn.
//fmt.Println(strings.Split(conn.LocalAddr().String(), ":")[0])
return strings.Split(conn.LocalAddr().String(), ":")[0]
}