vertica merge 優化

-- 查看RDS的訂單數(MySQL)

select count(*) from  (   SELECT   tid,    IF(LOCATE('pay_time', jdp_response)=0,null,SUBSTR(jdp_response,LOCATE('pay_time', jdp_response)+11,19)) pay_time   FROM jdp_tb_trade   where jdp_modified>='2015-11-11'  ) t0  where t0.pay_time is not null and t0.pay_time>='2015-11-11 00:00:00' and t0.pay_time<'2015-11-12 00:00:00' ;

-- 402277

-- 查看同步並解析的訂單數(Vertica) select count(*) from trade where jdp_modified >='2015-11-11 00:00:00'   and pay_time >= '2015-11-11 00:00:00' and pay_time <'2015-11-12 00:00:00'

-- 402277

-- 查看數據倉庫的訂單數 select count(*) from rds_shop_order_detail where  pay_time >= '2015-11-11 00:00:00' and pay_time <'2015-11-12 00:00:00';

-- 402253 數據倉庫漏單了....

問題記錄：

1\雙11晚上，同步數據丟失5W多數據。 緣由:未明 解決方法:從新指定同步數據時間區間。

2\修改增量同步判讀字段modified爲jdp_modified.

3\加快crontab頻率致使錯誤。 緣由:因爲數據還在入庫時,新的JOB清空臨時表,致使表被鎖。 解決方法:加快入庫速度,優化SQL秒出,慢SQL須要30分鐘以上

如下語句知足優化的merge的條件 3.1\目標表的關聯列有主鍵 3.2\全部目標表的列都包括在更新插入語句中 3.3\插入、更新列的屬性相等 特別注意2.2，這種寫法在oracle語法會報錯，而在vertica若是不這樣寫，merge是全表掃描！

未優化的SQL： MERGE INTO jdp_tb_trade a USING jdp_tb_trade_tmp b ON a.tid = b.tid WHEN MATCHED THEN UPDATE SET status = b.status , type = b.type , seller_nick = b.seller_nick , buyer_nick = b.buyer_nick , created = b.created , modified = b.modified , jdp_hashcode = b.jdp_hashcode , jdp_response = b.jdp_response , jdp_created = b.jdp_created , jdp_modified = b.jdp_modified , load_time = substring(to_char(sysdate),0,20) WHEN NOT MATCHED THEN INSERT         ( tid , status , type , seller_nick , buyer_nick , created , modified , jdp_hashcode , jdp_response , jdp_created , jdp_modified , load_time )  VALUES ( b.tid , b.status , b.type , b.seller_nick , b.buyer_nick , b.created , b.modified , b.jdp_hashcode , b.jdp_response , b.jdp_created , b.jdp_modified , substring(to_char(sysdate),0,20) )

優化的SQL: MERGE INTO jdp_tb_trade a USING jdp_tb_trade_tmp b ON a.tid = b.tid WHEN MATCHED THEN UPDATE SET tid = b.tid , status = b.status , type = b.type , seller_nick = b.seller_nick , buyer_nick = b.buyer_nick , created = b.created , modified = b.modified , jdp_hashcode = b.jdp_hashcode , jdp_response = b.jdp_response , jdp_created = b.jdp_created , jdp_modified = b.jdp_modified , load_time = substring(to_char(sysdate),0,20) WHEN NOT MATCHED THEN INSERT         ( tid , status , type , seller_nick , buyer_nick , created , modified , jdp_hashcode , jdp_response , jdp_created , jdp_modified , load_time )  VALUES ( b.tid , b.status , b.type , b.seller_nick , b.buyer_nick , b.created , b.modified , b.jdp_hashcode , b.jdp_response , b.jdp_created , b.jdp_modified , substring(to_char(sysdate),0,20) )  

一些有用SQL:

-- 查看session

SELECT  t.* FROM SESSIONS t; 

-- 查看lock

SELECT * FROM locks;

-- 查看正在鎖的session

SELECT s.session_id, s.transaction_id,l.object_name, l.transaction_description, l.lock_mode FROM SESSIONS s inner join locks l on s.transaction_id = l.transaction_id; -- 查看transactions

SELECT * FROM transactions ;

-- 表分析

SELECT ANALYZE_STATISTICS('jdp_tb_trade');

-- 關閉session  

SELECT CLOSE_SESSION('inman-10893:0x1d464');

--關閉所有session

SELECT CLOSE_ALL_SESSIONS();