網上有一具用於把excel表格數據導出爲google protobuf格式,及生成protobuf協議的工具:https://blog.csdn.net/linshuhe1/article/details/52062969 但作過大型遊戲的人都知道,當有大量數據須要配置時,同一個excel表格的數據,可能客戶端只須要其中的一些列,服務器須要的又是另外一些列。分紅兩個表來維護的話又增長工做量及出錯風險。因此我在前人的基礎上,稍微把這個工具改了一下,在excel表增長第一行,用於指定該列客戶端(填C)用到仍是服務器端用(填S)或者是客戶端,服務器都用(填CS)。而後在啓動導表腳本xls_deploy_tool.py時經過命令行的第四個參數(可選值爲:C,S,CS)來指定導出那些列,例如:python
python xls_deploy_tool.py GOODS_INFO xls/goods_info.xls C #只導出客戶端列git
python xls_deploy_tool.py GOODS_INFO xls/goods_info.xls S #只導出服務器端列數組
-----------工具源碼-------------------------------------------------------------------------------------------------------------------------------------------------服務器
#! /usr/bin/env python #coding=utf-8 ## # @file: xls_deploy_tool.py # @author: jameyli <lgy AT live DOT com> # @brief: xls 配置導表工具 # 主要功能: # 1 配置定義生成,根據excel 自動生成配置的PB定義 # 2 配置數據導入,將配置數據生成PB的序列化後的二進制數據或者文本數據 # # 說明: # 1 excel 的前五行用於結構定義, 其他則爲數據,第一行取值C-S-CS,分別表示該列數僅是客戶端或服務器端,或者二者都有,按第二行區分, 分別解釋: # required 必有屬性 # optional 可選屬性 # 第三行: 屬性類型 # 第四行:屬性名 # 第五行:註釋 # 數據行:屬性值 # repeated 代表下一個屬性是repeated,即數組 # 第三行: repeat的最大次數, excel中會重複列出該屬性 # 2011-11-29 作了修改 第二行若是是類型定義的話,則代表該列是repeated # 可是目前只支持整形 # 第四行:無用 # 第五行:註釋 # 數據行:實際的重複次數 # required_struct 必選結構屬性 # optional_struct 可選結構屬性 # 第三行:結構元素個數 # 第四行:結構名 # 第五行:在上層結構中的屬性名 # 數據行:不用填 # 1 | required/optional | repeated | required_struct/optional_struct | # | ------------------| ---------:| ---------------------------------:| # 2 | 屬性類型 | | 結構元素個數 | # 3 | 屬性名 | | 結構類型名 | # 4 | 註釋說明 | | 在上層結構中的屬性名 | # 5 | 屬性值 | | | # # # 開始設計的很理想,但願配置定義和配置數據保持一致,使用同一個excel # 不知道可否實現 # # 功能基本實現,並驗證過能夠經過CPP解析 ohye # # 2011-06-17 修改: # 表名sheet_name 使用大寫 # 結構定義使用大寫加下劃線 # 2011-06-20 修改bug: # excel命名中存在空格 # repeated_num = 0 時的狀況 # 2011-11-24 添加功能 # 默認值 # 2011-11-29 添加功能 # repeated 第二行若是是類型定義的話,則代表該列是repeated # 可是目前只支持整形 # TODO:: # 1 時間配置人性化 # 2 區分server/client 配置 # 3 repeated 優化 # 4 struct 優化 # 依賴: # 1 protobuf # 2 xlrd ## import xlrd # for read excel import sys import os # TAP的空格數 TAP_BLANK_NUM = 4 FIELD_CS_ROW = 0 FIELD_RULE_ROW = 1 # 這一行還表示重複的最大個數,或結構體元素數 FIELD_TYPE_ROW = 2 FIELD_NAME_ROW = 3 FIELD_COMMENT_ROW = 4 class LogHelp : """日誌輔助類""" _logger = None _close_imme = True @staticmethod def set_close_flag(flag): LogHelp._close_imme = flag @staticmethod def _initlog(): import logging LogHelp._logger = logging.getLogger() logfile = 'tnt_comm_deploy_tool.log' hdlr = logging.FileHandler(logfile) formatter = logging.Formatter('%(asctime)s|%(levelname)s|%(lineno)d|%(funcName)s|%(message)s') hdlr.setFormatter(formatter) LogHelp._logger.addHandler(hdlr) LogHelp._logger.setLevel(logging.NOTSET) # LogHelp._logger.setLevel(logging.WARNING) LogHelp._logger.info("\n\n\n") LogHelp._logger.info("logger is inited!") @staticmethod def get_logger() : if LogHelp._logger is None : LogHelp._initlog() return LogHelp._logger @staticmethod def close() : if LogHelp._close_imme: import logging if LogHelp._logger is None : return logging.shutdown() # log macro LOG_DEBUG=LogHelp.get_logger().debug LOG_INFO=LogHelp.get_logger().info LOG_WARN=LogHelp.get_logger().warn LOG_ERROR=LogHelp.get_logger().error class SheetInterpreter: """經過excel配置生成配置的protobuf定義文件""" def __init__(self, xls_file_path, sheet_name,cs="S"): self._xls_file_path = xls_file_path self._sheet_name = sheet_name self._cs = cs try : self._workbook = xlrd.open_workbook(self._xls_file_path) except BaseException, e : print "open xls file(%s) failed!"%(self._xls_file_path) raise try : self._sheet =self._workbook.sheet_by_name(self._sheet_name) except BaseException, e : print "open sheet(%s) failed!"%(self._sheet_name) # 行數和列數 self._row_count = len(self._sheet.col_values(0)) self._col_count = len(self._sheet.row_values(0)) self._row = 0 self._col = 0 # 將全部的輸出先寫到一個list, 最後統一寫到文件 self._output = [] # 排版縮進空格數 self._indentation = 0 # field number 結構嵌套時使用列表 # 新增一個結構,行增一個元素,結構定義完成後彈出 self._field_index_list = [1] # 當前行是否輸出,避免相同結構重複定義 self._is_layout = True # 保存全部結構的名字 self._struct_name_list = [] self._pb_file_name = "tnt_deploy_" + sheet_name.lower() + ".proto" def Interpreter(self) : """對外的接口""" LOG_INFO("begin Interpreter, row_count = %d, col_count = %d", self._row_count, self._col_count) self._LayoutFileHeader() self._output.append("package tnt_deploy;\n") self._LayoutStructHead(self._sheet_name) self._IncreaseIndentation() while self._col < self._col_count : self._FieldDefine(0) self._DecreaseIndentation() self._LayoutStructTail() self._LayoutArray() self._Write2File() LogHelp.close() # 將PB轉換成py格式 try : command = "protoc --python_out=./ " + self._pb_file_name os.system(command) except BaseException, e : print "protoc failed!" raise def _FieldDefine(self, repeated_num) : LOG_INFO("row=%d, col=%d, repeated_num=%d", self._row, self._col, repeated_num) field_cs = str(self._sheet.cell_value(FIELD_CS_ROW, self._col)) if field_cs.find(self._cs) < 0: self._col +=1 return field_rule = str(self._sheet.cell_value(FIELD_RULE_ROW, self._col)) if field_rule == "required" or field_rule == "optional" : field_type = str(self._sheet.cell_value(FIELD_TYPE_ROW, self._col)).strip() field_name = str(self._sheet.cell_value(FIELD_NAME_ROW, self._col)).strip() field_comment = unicode(self._sheet.cell_value(FIELD_COMMENT_ROW, self._col)) LOG_INFO("%s|%s|%s|%s", field_rule, field_type, field_name, field_comment) comment = field_comment.encode("utf-8") self._LayoutComment(comment) if repeated_num >= 1: field_rule = "repeated" self._LayoutOneField(field_rule, field_type, field_name) actual_repeated_num = 1 if (repeated_num == 0) else repeated_num self._col += actual_repeated_num elif field_rule == "repeated" : # 2011-11-29 修改 # 若repeated第二行是類型定義,則表示當前字段是repeated,而且數據在單列用分好相隔 second_row = str(self._sheet.cell_value(FIELD_TYPE_ROW, self._col)).strip() LOG_DEBUG("repeated|%s", second_row); # exel有可能有小數點 if second_row.isdigit() or second_row.find(".") != -1 : # 這裏後面通常會是一個結構體 repeated_num = int(float(second_row)) LOG_INFO("%s|%d", field_rule, repeated_num) self._col += 1 self._FieldDefine(repeated_num) else : # 通常是簡單的單字段,數值用分號相隔 field_type = second_row field_name = str(self._sheet.cell_value(FIELD_NAME_ROW, self._col)).strip() field_comment = unicode(self._sheet.cell_value(FIELD_COMMENT_ROW, self._col)) LOG_INFO("%s|%s|%s|%s", field_rule, field_type, field_name, field_comment) comment = field_comment.encode("utf-8") self._LayoutComment(comment) self._LayoutOneField(field_rule, field_type, field_name) self._col += 1 elif field_rule == "required_struct" or field_rule == "optional_struct": field_num = int(self._sheet.cell_value(FIELD_TYPE_ROW, self._col)) struct_name = str(self._sheet.cell_value(FIELD_NAME_ROW, self._col)).strip() field_name = str(self._sheet.cell_value(FIELD_COMMENT_ROW, self._col)).strip() LOG_INFO("%s|%d|%s|%s", field_rule, field_num, struct_name, field_name) if (self._IsStructDefined(struct_name)) : self._is_layout = False else : self._struct_name_list.append(struct_name) self._is_layout = True col_begin = self._col self._StructDefine(struct_name, field_num) col_end = self._col self._is_layout = True if repeated_num >= 1: field_rule = "repeated" elif field_rule == "required_struct": field_rule = "required" else: field_rule = "optional" self._LayoutOneField(field_rule, struct_name, field_name) actual_repeated_num = 1 if (repeated_num == 0) else repeated_num self._col += (actual_repeated_num-1) * (col_end-col_begin) else : self._col += 1 return def _IsStructDefined(self, struct_name) : for name in self._struct_name_list : if name == struct_name : return True return False def _StructDefine(self, struct_name, field_num) : """嵌套結構定義""" self._col += 1 self._LayoutStructHead(struct_name) self._IncreaseIndentation() self._field_index_list.append(1) while field_num > 0 : self._FieldDefine(0) field_num -= 1 self._field_index_list.pop() self._DecreaseIndentation() self._LayoutStructTail() def _LayoutFileHeader(self) : """生成PB文件的描述信息""" self._output.append("/**\n") self._output.append("* @file: " + self._pb_file_name + "\n") self._output.append("* @author: jameyli <jameyli AT tencent DOT com>\n") self._output.append("* @brief: 這個文件是經過工具自動生成的,建議不要手動修改\n") self._output.append("*/\n") self._output.append("\n") def _LayoutStructHead(self, struct_name) : """生成結構頭""" if not self._is_layout : return self._output.append("\n") self._output.append(" "*self._indentation + "message " + struct_name + "{\n") def _LayoutStructTail(self) : """生成結構尾""" if not self._is_layout : return self._output.append(" "*self._indentation + "}\n") self._output.append("\n") def _LayoutComment(self, comment) : # 改用C風格的註釋,防止會有分行 if not self._is_layout : return if comment.count("\n") > 1 : if comment[-1] != '\n': comment = comment + "\n" comment = comment.replace("\n", "\n" + " " * (self._indentation + TAP_BLANK_NUM), comment.count("\n")-1 ) self._output.append(" "*self._indentation + "/** " + comment + " "*self._indentation + "*/\n") else : self._output.append(" "*self._indentation + "/** " + comment + " */\n") def _LayoutOneField(self, field_rule, field_type, field_name) : """輸出一行定義""" if not self._is_layout : return if field_name.find('=') > 0 : name_and_value = field_name.split('=') self._output.append(" "*self._indentation + field_rule + " " + field_type \ + " " + str(name_and_value[0]).strip() + " = " + self._GetAndAddFieldIndex()\ + " [default = " + str(name_and_value[1]).strip() + "]" + ";\n") return if (field_rule != "required" and field_rule != "optional") : self._output.append(" "*self._indentation + field_rule + " " + field_type \ + " " + field_name + " = " + self._GetAndAddFieldIndex() + ";\n") return if field_type == "int32" or field_type == "int64"\ or field_type == "uint32" or field_type == "uint64"\ or field_type == "sint32" or field_type == "sint64"\ or field_type == "fixed32" or field_type == "fixed64"\ or field_type == "sfixed32" or field_type == "sfixed64" \ or field_type == "double" or field_type == "float" : self._output.append(" "*self._indentation + field_rule + " " + field_type \ + " " + field_name + " = " + self._GetAndAddFieldIndex()\ + " [default = 0]" + ";\n") elif field_type == "string" or field_type == "bytes" : self._output.append(" "*self._indentation + field_rule + " " + field_type \ + " " + field_name + " = " + self._GetAndAddFieldIndex()\ + " [default = \"\"]" + ";\n") else : self._output.append(" "*self._indentation + field_rule + " " + field_type \ + " " + field_name + " = " + self._GetAndAddFieldIndex() + ";\n") return def _IncreaseIndentation(self) : """增長縮進""" self._indentation += TAP_BLANK_NUM def _DecreaseIndentation(self) : """減小縮進""" self._indentation -= TAP_BLANK_NUM def _GetAndAddFieldIndex(self) : """得到字段的序號, 並將序號增長""" index = str(self._field_index_list[- 1]) self._field_index_list[-1] += 1 return index def _LayoutArray(self) : """輸出數組定義""" self._output.append("message " + self._sheet_name + "_ARRAY {\n") self._output.append(" repeated " + self._sheet_name + " items = 1;\n}\n") def _Write2File(self) : """輸出到文件""" pb_file = open(self._pb_file_name, "w+") pb_file.writelines(self._output) pb_file.close() class DataParser: """解析excel的數據""" def __init__(self, xls_file_path, sheet_name,cs="S"): self._xls_file_path = xls_file_path self._sheet_name = sheet_name self._cs = cs try : self._workbook = xlrd.open_workbook(self._xls_file_path) except BaseException, e : print "open xls file(%s) failed!"%(self._xls_file_path) raise try : self._sheet =self._workbook.sheet_by_name(self._sheet_name) except BaseException, e : print "open sheet(%s) failed!"%(self._sheet_name) raise self._row_count = len(self._sheet.col_values(0)) self._col_count = len(self._sheet.row_values(0)) self._row = 0 self._col = 0 try: self._module_name = "tnt_deploy_" + self._sheet_name.lower() + "_pb2" sys.path.append(os.getcwd()) exec('from '+self._module_name + ' import *'); self._module = sys.modules[self._module_name] except BaseException, e : print "load module(%s) failed"%(self._module_name) raise def Parse(self) : """對外的接口:解析數據""" LOG_INFO("begin parse, row_count = %d, col_count = %d", self._row_count, self._col_count) item_array = getattr(self._module, self._sheet_name+'_ARRAY')() # 先找到定義ID的列 id_col = 0 for id_col in range(0, self._col_count) : info_id = str(self._sheet.cell_value(self._row, id_col)).strip() if info_id == "" : continue else : break for self._row in range(4, self._row_count) : # 若是 id 是 空 直接跳過改行 info_id = str(self._sheet.cell_value(self._row, id_col)).strip() if info_id == "" : LOG_WARN("%d is None", self._row) continue item = item_array.items.add() self._ParseLine(item) LOG_INFO("parse result:\n%s", item_array) self._WriteReadableData2File(str(item_array)) data = item_array.SerializeToString() self._WriteData2File(data) #comment this line for test .by kevin at 2013年1月12日 17:23:35 LogHelp.close() def _ParseLine(self, item) : LOG_INFO("%d", self._row) self._col = 0 while self._col < self._col_count : self._ParseField(0, 0, item) def _ParseField(self, max_repeated_num, repeated_num, item) : field_cs = str(self._sheet.cell_value(FIELD_CS_ROW, self._col)) if field_cs.find(self._cs) < 0: self._col +=1 return field_rule = str(self._sheet.cell_value(0, self._col)).strip() if field_rule == "required" or field_rule == "optional" : field_name = str(self._sheet.cell_value(2, self._col)).strip() if field_name.find('=') > 0 : name_and_value = field_name.split('=') field_name = str(name_and_value[0]).strip() field_type = str(self._sheet.cell_value(1, self._col)).strip() LOG_INFO("%d|%d", self._row, self._col) LOG_INFO("%s|%s|%s", field_rule, field_type, field_name) if max_repeated_num == 0 : field_value = self._GetFieldValue(field_type, self._row, self._col) # 有value才設值 if field_value != None : item.__setattr__(field_name, field_value) self._col += 1 else : if repeated_num == 0 : if field_rule == "required" : print "required but repeated_num = 0" raise else : for col in range(self._col, self._col + repeated_num): field_value = self._GetFieldValue(field_type, self._row, col) # 有value才設值 if field_value != None : item.__getattribute__(field_name).append(field_value) self._col += max_repeated_num elif field_rule == "repeated" : # 2011-11-29 修改 # 若repeated第二行是類型定義,則表示當前字段是repeated,而且數據在單列用分好相隔 second_row = str(self._sheet.cell_value(FIELD_TYPE_ROW, self._col)).strip() LOG_DEBUG("repeated|%s", second_row); # exel有可能有小數點 if second_row.isdigit() or second_row.find(".") != -1 : # 這裏後面通常會是一個結構體 max_repeated_num = int(float(second_row)) read = self._sheet.cell_value(self._row, self._col) repeated_num = 0 if read == "" else int(self._sheet.cell_value(self._row, self._col)) LOG_INFO("%s|%d|%d", field_rule, max_repeated_num, repeated_num) if max_repeated_num == 0 : print "max repeated num shouldn't be 0" raise if repeated_num > max_repeated_num : repeated_num = max_repeated_num self._col += 1 self._ParseField(max_repeated_num, repeated_num, item) else : # 通常是簡單的單字段,數值用分號相隔 # 通常也只能是數字類型 field_type = second_row field_name = str(self._sheet.cell_value(FIELD_NAME_ROW, self._col)).strip() field_value_str = unicode(self._sheet.cell_value(self._row, self._col)) #field_value_str = unicode(self._sheet.cell_value(self._row, self._col)).strip() # LOG_INFO("%d|%d|%s|%s|%s", # self._row, self._col, field_rule, field_type, field_name, field_value_str) #2013-01-24 jamey #增長長度判斷 if len(field_value_str) > 0: if field_value_str.find(";\n") > 0 : field_value_list = field_value_str.split(";\n") else : field_value_list = field_value_str.split(";") for field_value in field_value_list : if field_type == "bytes": item.__getattribute__(field_name).append(field_value.encode("utf8")) else: item.__getattribute__(field_name).append(int(float(field_value))) self._col += 1 elif field_rule == "required_struct" or field_rule == "optional_struct": field_num = int(self._sheet.cell_value(FIELD_TYPE_ROW, self._col)) struct_name = str(self._sheet.cell_value(FIELD_NAME_ROW, self._col)).strip() field_name = str(self._sheet.cell_value(FIELD_COMMENT_ROW, self._col)).strip() LOG_INFO("%s|%d|%s|%s", field_rule, field_num, struct_name, field_name) col_begin = self._col # 至少循環一次 if max_repeated_num == 0 : struct_item = item.__getattribute__(field_name) self._ParseStruct(field_num, struct_item) else : if repeated_num == 0 : if field_rule == "required_struct" : print "required but repeated_num = 0" raise # 先讀取再刪除掉 struct_item = item.__getattribute__(field_name).add() self._ParseStruct(field_num, struct_item) item.__getattribute__(field_name).__delitem__(-1) else : for num in range(0, repeated_num): struct_item = item.__getattribute__(field_name).add() self._ParseStruct(field_num, struct_item) col_end = self._col max_repeated_num = 1 if (max_repeated_num == 0) else max_repeated_num actual_repeated_num = 1 if (repeated_num==0) else repeated_num self._col += (max_repeated_num - actual_repeated_num) * ((col_end-col_begin)/actual_repeated_num) else : self._col += 1 return def _ParseStruct(self, field_num, struct_item) : """嵌套結構數據讀取""" # 跳過結構體定義 self._col += 1 while field_num > 0 : self._ParseField(0, 0, struct_item) field_num -= 1 def _GetFieldValue(self, field_type, row, col) : """將pb類型轉換爲python類型""" field_value = self._sheet.cell_value(row, col) LOG_INFO("%d|%d|%s", row, col, field_value) try: if field_type == "int32" or field_type == "int64"\ or field_type == "uint32" or field_type == "uint64"\ or field_type == "sint32" or field_type == "sint64"\ or field_type == "fixed32" or field_type == "fixed64"\ or field_type == "sfixed32" or field_type == "sfixed64" : if len(str(field_value).strip()) <=0 : return None else : return int(field_value) elif field_type == "double" or field_type == "float" : if len(str(field_value).strip()) <=0 : return None else : return float(field_value) elif field_type == "string" : field_value = unicode(field_value) if len(field_value) <= 0 : return None else : return field_value elif field_type == "bytes" : field_value = unicode(field_value).encode('utf-8') if len(field_value) <= 0 : return None else : return field_value else : return None except BaseException, e : print "parse cell(%u, %u) error, please check it, maybe type is wrong."%(row, col) raise def _WriteData2File(self, data) : file_name = "tnt_deploy_" + self._sheet_name.lower() + ".data" file = open(file_name, 'wb+') file.write(data) file.close() def _WriteReadableData2File(self, data) : file_name = "tnt_deploy_" + self._sheet_name.lower() + ".txt" file = open(file_name, 'wb+') file.write(data) file.close() if __name__ == '__main__' : """入口""" if len(sys.argv) < 3 : print "Usage: %s sheet_name(should be upper) xls_file" %(sys.argv[0]) sys.exit(-1) # option 0 生成proto和data 1 只生成proto 2 只生成data, op = 0 if len(sys.argv) > 3 : op = int(sys.argv[3]) sheet_name = sys.argv[1] if (not sheet_name.isupper()): print "sheet_name should be upper" sys.exit(-2) xls_file_path = sys.argv[2] CS = "CS" if len(sys.argv) > 4: CS = sys.argv[4] #可指定須要導出的列C,S,CS if op == 0 or op == 1: try : tool = SheetInterpreter(xls_file_path, sheet_name,CS) tool.Interpreter() except BaseException, e : print "Interpreter Failed!!!" print e sys.exit(-3) print "Interpreter Success!!!" if op == 0 or op == 2: try : parser = DataParser(xls_file_path, sheet_name,CS) parser.Parse() except BaseException, e : print "Parse Failed!!!" print e sys.exit(-4) print "Parse Success!!!"