Python實現目錄文件的全量和增量備份

目標：

　　1.傳入3個參數:源文件路徑，目標文件路徑，md5文件

　　2.每週一實現全量備份，其他時間增量備份

 

1.經過傳入的路徑，獲取該路徑下面的全部目錄和文件(遞歸)

方法一：使用os.listdir

代碼以下：

#!/usr/bin/env python
#coding:utf8

import os,sys

def lsdir(folder):
    contents = os.listdir(folder)
    print "%s\n%s\n" % (folder, contents)
    for path in contents:
        full_path = os.path.join(folder, path)
        if os.path.isdir(full_path):
            lsdir(full_path)

if __name__ == "__main__":
    lsdir(sys.argv[1])

•運行代碼，效果以下：

[root@localhost python]# python listdir.py /a
/a
['b', 'a.txt']

/a/b
['c', 'b.txt']

/a/b/c
['c.txt']

方法二：使用os.walk

代碼以下：

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os,sys

def lsdir(folder):
   contents = os.walk(folder)
   for path, folder, file in contents:
       print "%s\n%s\n" %(path, folder + file)

if __name__ == "__main__":
    lsdir(sys.argv[1])

•運行代碼，測試效果

[root@localhost python]# python listdir1.py /a
/a
['b', 'a.txt']

/a/b
['c', 'b.txt']

/a/b/c
['c.txt']

 

2.如何計算文件的md5值(每次讀取4K，直到讀取完文件全部內容，返回一個16進制的md5值)

代碼以下：

[root@localhost python]# cat md5.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import hashlib
import sys

def md5(fname):
    m = hashlib.md5()
    with open(fname) as fobj:
        while True:
            data = fobj.read(4096)
            if not data:
                break
            m.update(data)
    return m.hexdigest()

if __name__ == "__main__":
    print md5(sys.argv[1])

•運行代碼，測試效果

[root@localhost python]# python md5.py a.txt
c33da92372e700f98b006dfa5325cf0d
[root@localhost python]# md5sum a.txt
c33da92372e700f98b006dfa5325cf0d  a.txt

*提示：使用linux自帶的md5sum和本身編寫的Python計算的md5值相通

 

3.編寫全量和增量備份腳本

代碼以下：

#!/usr/bin/env python
#coding:utf8

import time
import os
import tarfile
import cPickle as p
import hashlib


def md5check(fname):
    m = hashlib.md5()
    with open(fname) as fobj:
        while True:
            data = fobj.read(4096)
            if not data:
                break
            m.update(data)
    return m.hexdigest()


def full_backup(src_dir, dst_dir, md5file):
    par_dir, base_dir = os.path.split(src_dir.rstrip('/'))
    back_name = '%s_full_%s.tar.gz' % (base_dir, time.strftime('%Y%m%d'))
    full_name = os.path.join(dst_dir, back_name)
    md5dict = {}

    tar = tarfile.open(full_name, 'w:gz')
    tar.add(src_dir)
    tar.close()
    for path, folders, files in os.walk(src_dir):
        for fname in files:
            full_path = os.path.join(path, fname)
            md5dict[full_path] = md5check(full_path)

    with open(md5file, 'w') as fobj:
        p.dump(md5dict, fobj)



def incr_backup(src_dir, dst_dir, md5file):
    par_dir, base_dir = os.path.split(src_dir.rstrip('/'))
    back_name = '%s_incr_%s.tar.gz' % (base_dir, time.strftime('%Y%m%d'))
    full_name = os.path.join(dst_dir, back_name)
    md5new = {}

    for path, folders, files in os.walk(src_dir):
        for fname in files:
            full_path = os.path.join(path, fname)
            md5new[full_path] = md5check(full_path)

    with open(md5file) as fobj:
        md5old = p.load(fobj)
    
    with open(md5file, 'w') as fobj:
        p.dump(md5new, fobj)

    tar = tarfile.open(full_name, 'w:gz')
    for key in md5new:
        if md5old.get(key) != md5new[key]:
            tar.add(key)
    tar.close()


if __name__ == '__main__':
    src_dir = '/Users/xkops/gxb/'
    dst_dir = '/tmp/'
    md5file = '/Users/xkops/md5.data'
    if time.strftime('%a') == 'Mon':
        full_backup(src_dir, dst_dir, md5file)
    else:
        incr_backup(src_dir, dst_dir, md5file)

•運行代碼，測試效果(執行前，修改須要備份的文件和路徑)，運行以後檢查/tmp下是否生成了當天的備份文件。