【原創】OpenStack Swift源碼分析（一）builder文件的生成

最近開始Swift源碼分析，無非就是不想紙上談兵，到底是用什麼算法，和方法實現的功能，因此決定開始寫源碼分析。

    當你安裝完swift,你須要作的第一件事情就是建立ring文件，而你用的第一個命令就是swift-ring-builder。swift-ring-builder文件位於源碼的bin目錄下，是swift最基本的命令，它與swift/common/ring/下的文件一塊兒實現ring文件建立，添加，平衡，等操做。

    對於初學者（好比說我）重寫源碼片斷，能夠更加深刻的瞭解源碼的原理，同時還能對python語言以及相關的庫有更深的瞭解，swift-ring-builder中主要的功能實現就是在Commands類中，好比default(),create(),add(),rebalance()等， 而後main方法會根據你提供的相應參數，來提供執行相應的方法，而後其中的方法會調用/swift/common/ring/下的builder.py中相應的方法最終實現相應的操做，

當咱們經過create建立account.builder文件的時候，commod == argv[2] 也就是create 而後執行create來建立account.builder。以後的操做，只要是存在account.builder文件，就會打開這個文件，生產builder實例，來進行相應的操做。其中的 default方法是顯示當前的builder信息，能夠用來在rebalance以前 檢查add的device 。

其中reblance是最重要的方法，當中會涉及到/swift/common/ring下的ring.py builder.py utils.py文件，涉及到了一致性哈希算法和策略的實現，下個博客會具體分析。

代碼片斷：基本就是把源碼抽出來，沒寫什麼註釋和異常處理，生成的builder文件，能夠經過diff命令，比較是否跟使用swift-ring-builder命令建立的builder文件同樣（答案固然是確定的）。

 

#! /usr/bin/env python

from sys import argv, exit, modules
import cPickle as pickle
from os.path import basename, dirname, exists, join as pathjoin
from itertools import islice, izip

from builder import RingBuilder


MAJOR_VERSION = 1
MINOR_VERSION = 3
EXIT_SUCCESS = 0
EXIT_WARNING = 1
EXIT_ERROR = 2


class Commands:

    def unknown():
        print 'Unknown command: %s' % argv[2]
        exit(EXIT_ERROR)
    
    def create():
        """
        my-ring-builder <builder_file> create <part_power> <replicas>
                                                <min_part_hours>
        """

        if len(argv) < 6:
            print Commands.create.__doc__.strip()
            exit(EXIT_ERROR)
        builder = RingBuilder(int(argv[3]), int(argv[4]), int(argv[5]))
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(EXIT_SUCCESS)

    def default():
        """
        Shows information about the ring and the devices within.
        """
        print '%s, build version %d ' % (argv[1], builder.version)
        zones = 0
        balance = 0
        if builder.devs:
            zones = len(set(d['zone'] for d in builder.devs if d is not None))
            #blance = builder.get_balance()
        print '%d partitions, %d replicas, %d zones, %d devices' \
                % (builder.parts, builder.replicas, zones,
                    len([d for d in builder.devs if d]))
        print 'The minimun number of hours before a partition can be' \
                'reassigned is %s' % builder.min_part_hours
        if builder.devs:
            print 'Devices:    id  zone      ip  address  port    name' \
                    'weight partition'
            weighted_parts = builder.parts * builder.replicas / \
                    sum(d['weight'] for d in builder.devs if d is not None)
            for dev in builder.devs:
                if dev is None:
                    continue
                if not dev['weight']:
                    if dev['parts']:
                        blance = 999.99
                    else:
                        blance = 0
                else:
                    blance = 100.0 * dev['parts'] / \
                            (dev['weight'] * weighted_parts) - 100.0
                print '        %5d %5d %15s %5d %9s %6.02f %10s' % \
                        (dev['id'], dev['zone'], dev['ip'], dev['port'],
                         dev['device'], dev['weight'], dev['parts'])
        exit(EXIT_SUCCESS)

    def add():
        """
        .......
        """

        dev_and_weights = izip(islice(argv, 3, len(argv), 2),
                               islice(argv, 4, len(argv), 2))
        for devstr, weightstr in dev_and_weights:
            if not devstr.startswith('z'):
                print 'Invalid add value: %s' % devstr
                exit(EXIT_ERROR)
            i = 1
            while i < len(devstr) and devstr[i].isdigit():
                i += 1
            zone = int(devstr[1:i])
            rest = devstr[i:]

            i = 1
            while i < len(rest) and rest[i] in '0123456789.':
                i += 1
            ip = rest[1:i]
            rest = rest[i:]
            print ip
            i = 1
            while i < len(rest) and rest[i].isdigit():
                i += 1
            port = int(rest[1:i])
            rest = rest[i:]

            i = 1
            while i < len(rest) and rest[i] != '_':
                i += 1
            device_name = rest[1:i]
            rest = rest[i:]

            meta = ''
            if rest.startswith('_'):
                meta = rest[1:]
            weight = float(weightstr)
            for dev in builder.devs:
                if dev is None:
                    continue
                if dev['ip'] == ip and dev['port'] == port and \
                        dev['device'] == device_name:
                    print 'already uses %s'
                    exit(EXIT_ERROR)

            next_dev_id = 0
            if builder.devs:
                next_dev_id = max(d['id'] for d in builder.devs if d) + 1
            builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ip,
                             'port': port, 'device': device_name,
                             'weight': weight, 'meta': meta})
            print 'Device z%s-[%s]:%s/%s_"%s" with %s weight got id %s' % \
                    (zone, ip, port, device_name, meta, weight, next_dev_id)
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(EXIT_SUCCESS)

    def rebalance():
        """
        Attempts to rebalance the ring by reassigning partitions
        """

        devs_changed = builder.devs_changed
            
        last_balance = builder.get_balance()
        parts, balance = builder.rebalance()
        
        builder.validate()
        print 'Reassigned %d (%.02f%%) partitions. Balance is now %.02f.' % \
                (parts, 100.0*parts / builder.parts, balance)
        status = EXIT_SUCCESS
        if balance > 5:
            print'NOTE: Balance of %.02f indicates you should push the' % \
                    balance
            status = EXIT_WARNING
        builder.get_ring().save(ring_file)
        pickle.dump(builder.to_dict(), open(argv[1], 'wb'), protocol=2)
        exit(status)
if __name__ == '__main__':
    
    if exists(argv[1]):
        builder = pickle.load(open(argv[1], 'rb'))
        if not hasattr(builder, 'devs'):
            builder_dict = builder
            builder = RingBuilder(1, 1, 1)
            builder.copy_from(builder_dict)
    ring_file = argv[1]
    if ring_file.endswith('.builder'):
        ring_file = ring_file[:-len('.builder')]
    ring_file += '.ring.gz'
    if len(argv) == 2:
        command = "default"
    else:
        command = argv[2]
    Commands.__dict__.get(command, Commands.unknown)()