def generate_csv_source(data_file):
#判斷文件是否存在
if not os.path.exists(data_file):
# 拉取hive表數據
cmd_sql = 'hive -e "set hive.cli.print.header=true; \
select * from dw.full_d_usr_channel_sum_v3 where ds<>\'\' \
and type in (\'super\',\'9k9\',\'taobao\',\'sc\',\'zy\',\'ad\',\'licai\') \
" >%s' % (data_file)
print cmd_sql
subprocess.call(cmd_sql, shell=True)
# 替換其中的字段分隔符/t爲,
cmd_sed = 'sed -i "s/\t/,/g" %s' % (data_file)
subprocess.call(cmd_sed, shell=True)
print "文件已生成："+data_file
else:
print "最新文件已存在："+data_file
-- 寫人hive 表sql
def insert_to_table(data_cur, c_path,tab_name, ds,freq_type,c_type):
# data_cur.to_csv('./user_value_auto_compute_result.csv', index=False, header=None, encoding="utf8")
path_result = '%s/%s_%s_%s.csv' % (c_path,tab_name,freq_type,c_type)
print '生成的結果csv文件:',path_result
data_cur.to_csv(path_result, index=False,header=None, float_format = '%.4f', encoding="utf8")
# cmd_hive = '''hive -e "load data local inpath \'''' + c_path + '''/'''+tab_name+'''.csv\' OVERWRITE into table '''+tab_name+''' partition(c_type='%s',freq_type='%s',ds='%s')"''' % (c_type,freq_type,ds)
cmd_hive = '''hive -e "load data local inpath \'''' + path_result+'''\' OVERWRITE into table '''+tab_name+''' partition(c_type='%s',freq_type='%s',ds='%s')"''' % (c_type,freq_type,ds)
# 導入hive庫
print cmd_hive
subprocess.call(cmd_hive, shell=True)
print '生成文件:%s.csv' % (tab_name)shell