CIB Training Scripts For TPC-H Benchmark

http://52.11.56.155:7180/
http://52.11.56.155:8888/

impala-shell -i 172.31.25.244

sudo -u hdfs hdfs dfs -du -h /

http://54.149.20.119:25000/

第一步：
準備工做
生成tpc-h測試數據
在TPC-H的官網http://www.tpc.org/tpch/上下載dbgen工具http://www.tpc.org/tpch/spec/tpch_2_17_0.zip
[root@ip-172-31-25-243 training]# wget http://www.tpc.org/tpch/spec/tpch_2_17_0.zip

確認下載後的結果：
[root@ip-172-31-25-243 training]# ls
tpch_2_17_0.zip

若是沒有安裝解壓工具，請安裝解壓工具
[root@ip-172-31-25-243 training]# yum install unzip
解壓tpc-h數據生成工具，確認加壓結果:
[root@ip-172-31-25-243 training]# unzip tpch_2_17_0.zip
[root@ip-172-31-25-243 training]# ls
__MACOSX tpch_2_17_0 tpch_2_17_0.zip

切換到相應目錄並修改參數
[root@ip-172-31-25-243 training]# cd tpch_2_17_0
[root@ip-172-31-25-243 tpch_2_17_0]# ls
dbgen dev-tools ref_data
[root@ip-172-31-25-243 tpch_2_17_0]# cd dbgen
[root@ip-172-31-25-243 dbgen]# ls
answers bm_utils.c check_answers dbgen.dsp dss.ddl dsstypes.h makefile.suite PORTING.NOTES qgen.vcproj reference rnd.h shared.h text.c tpch.sln variants
bcd2.c BUGS column_split.sh dists.dss dss.h HISTORY permute.c print.c queries release.h rng64.c speed_seed.c tpcd.h tpch.vcproj varsub.c
bcd2.h build.c config.h driver.c dss.ri load_stub.c permute.h qgen.c README rnd.c rng64.h tests tpch.dsw update_release.sh

[root@ip-172-31-25-243 dbgen]# cp makefile.suite makefile
[root@ip-172-31-25-243 dbgen]# vi makefile
將其中的以下幾行：
CC =
DATABASE=
MACHINE =
WORKLOAD =
修改爲:
CC = gcc
DATABASE= ORACLE
MACHINE = LINUX
WORKLOAD = TPCH

構建生成工具
[root@ip-172-31-25-243 dbgen]# make
查看結果
[root@ip-172-31-25-243 dbgen]# ls
answers bm_utils.c build.o dbgen driver.o dsstypes.h makefile permute.o qgen queries rnd.c rng64.h speed_seed.o tpcd.h update_release.sh
bcd2.c bm_utils.o check_answers dbgen.dsp dss.ddl HISTORY makefile.suite PORTING.NOTES qgen.c README rnd.h rng64.o tests tpch.dsw variants
bcd2.h BUGS column_split.sh dists.dss dss.h load_stub.c permute.c print.c qgen.o reference rnd.o shared.h text.c tpch.sln varsub.c
bcd2.o build.c config.h driver.c dss.ri load_stub.o permute.h print.o qgen.vcproj release.h rng64.c speed_seed.c text.o tpch.vcproj varsub.o
發現多了兩個兩個可執行文件dbgen和ggen

切換到dbgen目錄下並運行生生數據程序
[root@ip-172-31-25-243 dbgen]# ./dbgen -s 10

查看運行結果：
[root@ip-172-31-25-243 dbgen]# ls *.tbl
customer.tbl lineitem.tbl nation.tbl orders.tbl partsupp.tbl part.tbl region.tbl supplier.tbl

第二步：
將數據寫入hdfs
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/customer
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/lineitem
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/nation
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/orders
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/part
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/partsupp
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/region
[root@ip-172-31-25-243 dbgen]# hdfs dfs -mkdir /user/training/tpch/supplier
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put customer.tbl /user/training/tpch/customer
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put lineitem.tbl /user/training/tpch/lineitem
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put nation.tbl /user/training/tpch/nation
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put orders.tbl /user/training/tpch/orders
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put part.tbl /user/training/tpch/part
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put partsupp.tbl /user/training/tpch/partsupp
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put region.tbl /user/training/tpch/region
[root@ip-172-31-25-243 dbgen]# hdfs dfs -put supplier.tbl /user/training/tpch/supplier
你們能夠看一下數據大小以及數據的分佈狀況,每一個表的大小

第三步：
進入hue界面
http://52.11.56.155:8888/
用戶名admin,密碼admin
進入Query Editors —> Hive
同時打開另外一個標籤頁，進入進入Query Editors —> Impala

建立數據庫和表
—-建立數據庫
在hive標籤頁中輸入
create database training

在界面左邊的DataBase部分，點擊刷新以後就能夠看到新建的數據庫training
選中該數據庫，而後再右邊的編輯窗口中輸入如下建表語句:

在hive標籤頁執行
-- create tables and load data
create external table customer (C_CUSTKEY INT, C_NAME STRING, C_ADDRESS STRING, C_NATIONKEY INT, C_PHONE STRING, C_ACCTBAL DOUBLE, C_MKTSEGMENT STRING, C_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/customer';
create external table lineitem (L_ORDERKEY INT, L_PARTKEY INT, L_SUPPKEY INT, L_LINENUMBER INT, L_QUANTITY DOUBLE, L_EXTENDEDPRICE DOUBLE, L_DISCOUNT DOUBLE, L_TAX DOUBLE, L_RETURNFLAG STRING, L_LINESTATUS STRING, L_SHIPDATE STRING, L_COMMITDATE STRING, L_RECEIPTDATE STRING, L_SHIPINSTRUCT STRING, L_SHIPMODE STRING, L_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/lineitem';
create external table nation (N_NATIONKEY INT, N_NAME STRING, N_REGIONKEY INT, N_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/nation';
create external table orders (O_ORDERKEY INT, O_CUSTKEY INT, O_ORDERSTATUS STRING, O_TOTALPRICE DOUBLE, O_ORDERDATE STRING, O_ORDERPRIORITY STRING, O_CLERK STRING, O_SHIPPRIORITY INT, O_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/orders';
create external table part (P_PARTKEY INT, P_NAME STRING, P_MFGR STRING, P_BRAND STRING, P_TYPE STRING, P_SIZE INT, P_CONTAINER STRING, P_RETAILPRICE DOUBLE, P_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/part';
create external table partsupp (PS_PARTKEY INT, PS_SUPPKEY INT, PS_AVAILQTY INT, PS_SUPPLYCOST DOUBLE, PS_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION'/user/training/tpch/partsupp';
create external table region (R_REGIONKEY INT, R_NAME STRING, R_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/region';
create external table supplier (S_SUPPKEY INT, S_NAME STRING, S_ADDRESS STRING, S_NATIONKEY INT, S_PHONE STRING, S_ACCTBAL DOUBLE, S_COMMENT STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' STORED AS TEXTFILE LOCATION '/user/training/tpch/supplier';

-- create orc tables;
create table customer_orc (C_CUSTKEY INT, C_NAME STRING, C_ADDRESS STRING, C_NATIONKEY INT, C_PHONE STRING, C_ACCTBAL DOUBLE, C_MKTSEGMENT STRING, C_COMMENT STRING) STORED AS orc;
create table lineitem_orc (L_ORDERKEY INT, L_PARTKEY INT, L_SUPPKEY INT, L_LINENUMBER INT, L_QUANTITY DOUBLE, L_EXTENDEDPRICE DOUBLE, L_DISCOUNT DOUBLE, L_TAX DOUBLE, L_RETURNFLAG STRING, L_LINESTATUS STRING, L_SHIPDATE STRING, L_COMMITDATE STRING, L_RECEIPTDATE STRING, L_SHIPINSTRUCT STRING, L_SHIPMODE STRING, L_COMMENT STRING) STORED AS orc;
create table nation_orc (N_NATIONKEY INT, N_NAME STRING, N_REGIONKEY INT, N_COMMENT STRING) STORED AS orc;
create table orders_orc (O_ORDERKEY INT, O_CUSTKEY INT, O_ORDERSTATUS STRING, O_TOTALPRICE DOUBLE, O_ORDERDATE STRING, O_ORDERPRIORITY STRING, O_CLERK STRING, O_SHIPPRIORITY INT, O_COMMENT STRING) STORED AS orc;
create table part_orc (P_PARTKEY INT, P_NAME STRING, P_MFGR STRING, P_BRAND STRING, P_TYPE STRING, P_SIZE INT, P_CONTAINER STRING, P_RETAILPRICE DOUBLE, P_COMMENT STRING) STORED AS orc;
create table partsupp_orc (PS_PARTKEY INT, PS_SUPPKEY INT, PS_AVAILQTY INT, PS_SUPPLYCOST DOUBLE, PS_COMMENT STRING) STORED AS orc;
create table region_orc (R_REGIONKEY INT, R_NAME STRING, R_COMMENT STRING) STORED AS orc;
create table supplier_orc (S_SUPPKEY INT, S_NAME STRING, S_ADDRESS STRING, S_NATIONKEY INT, S_PHONE STRING, S_ACCTBAL DOUBLE, S_COMMENT STRING) STORED AS orc;

在Impala標籤頁執行

invalidate metadata

而後點擊屏幕左邊的刷新按鈕，能夠看到剛纔在hive中創建的database training.
-- create parquet tables
create table customer_parquet (C_CUSTKEY INT, C_NAME STRING, C_ADDRESS STRING, C_NATIONKEY INT, C_PHONE STRING, C_ACCTBAL DOUBLE, C_MKTSEGMENT STRING, C_COMMENT STRING) STORED AS parquet;
create table lineitem_parquet (L_ORDERKEY INT, L_PARTKEY INT, L_SUPPKEY INT, L_LINENUMBER INT, L_QUANTITY DOUBLE, L_EXTENDEDPRICE DOUBLE, L_DISCOUNT DOUBLE, L_TAX DOUBLE, L_RETURNFLAG STRING, L_LINESTATUS STRING, L_SHIPDATE STRING, L_COMMITDATE STRING, L_RECEIPTDATE STRING, L_SHIPINSTRUCT STRING, L_SHIPMODE STRING, L_COMMENT STRING) STORED AS parquet;
create table nation_parquet (N_NATIONKEY INT, N_NAME STRING, N_REGIONKEY INT, N_COMMENT STRING) STORED AS parquet;
create table orders_parquet (O_ORDERKEY INT, O_CUSTKEY INT, O_ORDERSTATUS STRING, O_TOTALPRICE DOUBLE, O_ORDERDATE STRING, O_ORDERPRIORITY STRING, O_CLERK STRING, O_SHIPPRIORITY INT, O_COMMENT STRING) STORED AS parquet;
create table part_parquet (P_PARTKEY INT, P_NAME STRING, P_MFGR STRING, P_BRAND STRING, P_TYPE STRING, P_SIZE INT, P_CONTAINER STRING, P_RETAILPRICE DOUBLE, P_COMMENT STRING) STORED AS parquet;
create table partsupp_parquet (PS_PARTKEY INT, PS_SUPPKEY INT, PS_AVAILQTY INT, PS_SUPPLYCOST DOUBLE, PS_COMMENT STRING) STORED AS parquet;
create table region_parquet (R_REGIONKEY INT, R_NAME STRING, R_COMMENT STRING) STORED AS parquet;
create table supplier_parquet (S_SUPPKEY INT, S_NAME STRING, S_ADDRESS STRING, S_NATIONKEY INT, S_PHONE STRING, S_ACCTBAL DOUBLE, S_COMMENT STRING) STORED AS parquet;

—hive: load data to orc format
insert overwrite table lineitem_orc select * from lineitem;
insert overwrite table customer_orc select * from customer;
insert overwrite table nation_orc select * from nation;
insert overwrite table orders_orc select * from orders;
insert overwrite table part_orc select * from part;
insert overwrite table partsupp_orc select * from partsupp;
insert overwrite table region_orc select * from region;
insert overwrite table supplier_orc select * from supplier;

—impala: load data to parquet format
insert overwrite table lineitem_parquet select * from lineitem;
insert overwrite table customer_parquet select * from customer;
insert overwrite table nation_parquet select * from nation;
insert overwrite table orders_parquet select * from orders;
insert overwrite table part_parquet select * from part;
insert overwrite table partsupp_parquet select * from partsupp;
insert overwrite table region_parquet select * from region;
insert overwrite table supplier_parquet select * from supplier;

—impala compute stats
compute stats customer;
compute stats lineitem;
compute stats nation;
compute stats orders;
compute stats part;
compute stats partsupp;
compute stats region;
compute stats supplier;

compute stats customer_parquet;
compute stats lineitem_parquet;
compute stats nation_parquet;
compute stats orders_parquet;
compute stats part_parquet;
compute stats partsupp_parquet;
compute stats region_parquet;
compute stats supplier_parquet;

第四步：
--Query1:單表統計分析

--hive

-- hive+text
SELECT
L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY), SUM(L_EXTENDEDPRICE), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)), AVG(L_QUANTITY), AVG(L_EXTENDEDPRICE), AVG(L_DISCOUNT), COUNT(1)
FROM
lineitem
WHERE
L_SHIPDATE<='1998-09-02'
GROUP BY L_RETURNFLAG, L_LINESTATUS
ORDER BY L_RETURNFLAG, L_LINESTATUS;

-- hive+orc
SELECT
L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY), SUM(L_EXTENDEDPRICE), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)), AVG(L_QUANTITY), AVG(L_EXTENDEDPRICE), AVG(L_DISCOUNT), COUNT(1)
FROM
lineitem_orc
WHERE
L_SHIPDATE<='1998-09-02'
GROUP BY L_RETURNFLAG, L_LINESTATUS
ORDER BY L_RETURNFLAG, L_LINESTATUS;

-- impala+text
SELECT
L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY), SUM(L_EXTENDEDPRICE), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)), AVG(L_QUANTITY), AVG(L_EXTENDEDPRICE), AVG(L_DISCOUNT), COUNT(1)
FROM
lineitem
WHERE
L_SHIPDATE<='1998-09-02'
GROUP BY L_RETURNFLAG, L_LINESTATUS
ORDER BY L_RETURNFLAG, L_LINESTATUS;

--impala+parquet
SELECT
L_RETURNFLAG, L_LINESTATUS, SUM(L_QUANTITY), SUM(L_EXTENDEDPRICE), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)), SUM(L_EXTENDEDPRICE*(1-L_DISCOUNT)*(1+L_TAX)), AVG(L_QUANTITY), AVG(L_EXTENDEDPRICE), AVG(L_DISCOUNT), cast(COUNT(1) as int)
FROM
lineitem_parquet
WHERE
L_SHIPDATE<='1998-09-02'
GROUP BY L_RETURNFLAG, L_LINESTATUS
ORDER BY L_RETURNFLAG, L_LINESTATUS;

第五步:
--Query2:多表聯合查詢
--hive+text
select
l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority
from
customer c join orders o
on c.c_mktsegment = 'BUILDING' and c.c_custkey = o.o_custkey
join lineitem l
on l.l_orderkey = o.o_orderkey
where
o_orderdate < '1995-03-15' and l_shipdate > '1995-03-15'
group by l_orderkey, o_orderdate, o_shippriority
order by revenue desc, o_orderdate
limit 10;

--hive+orc
select
l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority
from
customer_orc c join orders_orc o
on c.c_mktsegment = 'BUILDING' and c.c_custkey = o.o_custkey
join lineitem_orc l
on l.l_orderkey = o.o_orderkey
where
o_orderdate < '1995-03-15' and l_shipdate > '1995-03-15'
group by l_orderkey, o_orderdate, o_shippriority
order by revenue desc, o_orderdate
limit 10;

--impala+text
select
l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority
from
customer c join orders o
on c.c_mktsegment = 'BUILDING' and c.c_custkey = o.o_custkey
join lineitem l
on l.l_orderkey = o.o_orderkey
where
o_orderdate < '1995-03-15' and l_shipdate > '1995-03-15'
group by l_orderkey, o_orderdate, o_shippriority
order by revenue desc, o_orderdate
limit 10;

--impala+parquet
select
l_orderkey, sum(l_extendedprice*(1-l_discount)) as revenue, o_orderdate, o_shippriority
from
customer_parquet c join orders_parquet o
on c.c_mktsegment = 'BUILDING' and c.c_custkey = o.o_custkey
join lineitem_parquet l
on l.l_orderkey = o.o_orderkey
where
o_orderdate < '1995-03-15' and l_shipdate > '1995-03-15'
group by l_orderkey, o_orderdate, o_shippriority
order by revenue desc, o_orderdate
limit 10;

—-hdfs
數據的塊分佈

--hive演示要點：
1. Hive的Shell執行方式和Hue界面對比
2. Hive建表語句，Hive查詢，統計分析，多表關聯查詢和統計
3. Hive的運行原理
4. Hive的SQL執行速度問題
5. 系統資源使用狀況
6. 併發訪問的問題，資源使用

--impala演示1. impala shell與Hue2. impala與hive執行速度的對比3. orc,parquet的壓縮比4. parquet的運行效率提高5. 系統資源使用狀況6. impala的運行原理7. impala運行時統計信息,impala的診斷和調優