hive

一些概念

安裝 mysql

yum安裝mysql-server沒有可用包問題解決方法：

step 1: wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm

step 2: rpm -ivh mysql-community-release-el7-5.noarch.rpm

通過以上兩個步驟後再次執行：yum install mysql-server 命令就能夠成功安裝了。

修改 密碼

# 停掉 mysql 服務
systemctl stop mysqld
#修改 mysql 配置
vim /etc/my.cnf
# Disabling symbolic-links is recommended to prevent assorted security risks
skip-grant-tables #添加這句話，這時候登入mysql就不須要密碼
symbolic-links=0

# 從新啓動 mysql 服務
systemctl start mysqld

mysql -u root -p    #敲回車
set password for root@localhost = password('root123'); # 這個會報錯誤
flush privileges;
set password for root@localhost = password('root123');
flush privileges;
quit;

systemctl stop mysqld.service
修改 配置文件 註釋掉 skip-grant 語句

hive 安裝步驟

# 去官網拷貝下載地址
wget xxx
tar -xzvf xxx.tar.gz
mv apache-hive-2.3.5-bin /usr/local/
cd /usr/local/apache-hive-2.3.5-bin/conf
須要修改兩個 配置文件

cp hive-env.sh.template hive-env.sh
cp hive-default.xml.template hive-site.xml

vim hive-env.sh ,添加 JAVA_HOME , HADOOP_HOME

vim hive-site.xml    # 修改 metastore 存儲在哪裏 (localhost 的 mysql 等)

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?><!--
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->
<configuration>

<property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://localhost:3306/metastore?createDatabaseIfNotExist=true</value>
</property>
<property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
</property>
<property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
</property>
<property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>root123</value>
</property>

<property>
 <name>hive.metastore.warehouse.dir</name>
<value>/hive/warhouse</value>
</property>

<property>
 <name>hive.exec.scratchdir</name>
 <value>/hive/tmp</value>
</property>

<property>
 <name>hive.querylog.location</name>
 <value>/hive/log</value>
</property>

</configuration>

接下來，咱們須要去 建立咱們的 mysql 數據庫服務器 以及數據庫實例
而且，須要在 hdfs 文件系統上爲 hive 建立 對應的文件目錄 如 下圖所示

hdfs dfs -ls /
hdfs dfs -mkdir /hive
hdfs dfs -mkdir /hive/warehouse
hdfs dfs -mkdir /hive/tmp
hdfs dfs -mkdir /hive/log

hdfs dfs -chmod -R 777 /hive

./schematool -dbType mysql -initSchema

# 提示 缺乏 myql 驅動 jar 包
cd /usr/local/apache-hivexxxx/lib
wget http://central.maven.org/maven2/mysql/mysql-connector-java/6.0.6/mysql-connector-java-6.0.6.jar


./schematool -dbType mysql -initSchema

# 以 metastore 方式 啓動 hive 
nohup ./hive --services metastore &

中間遇到 一些坑 主要是  jdbc 鏈接字符串的問題  &amp 進行轉義 與符號 
<property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://localhost:3306/metastore?createDatabaseIfNotExsit=true&amp;characterEncoding=UTF-8&amp;serverTimezone=UTC</value>
</property>


./hive  # 進入 交互式的 hive shell
![](https://img2018.cnblogs.com/blog/636379/201906/636379-20190616221558709-994510626.png)

./hive

進入 交互

show databases;

vim testdata.txt

1,tom,music-runing-code,c++:98.0-java:76.0-php:65.0
2,jerry,music-code,c++:93.0-java:70.0-php:55.0
3,john,code,go:87.0-python:93.0

使用 hive 進行導入

create table table1(id int,name string,interest array ,score map<string,string>) row format delimited fields terminated by ','
collection items terminated by '-' map keys terminated by ':' stored as textfile;

導入本地文件到 內部表
load data local inpath '/root/testdata.txt' overwrite into table table1;

hdfs dfs -ls /hive/warehouse/table1 # 能夠查看到 數據其實是保存在 了 hdfs 上，元數據信息保存在 mysql 上

外部表

hdfs dfs -mkdir /outertables/
hdfs dfs -copyFromLocal /root/testdata.txt /outertables/

create external table table2(id int,name string,interest array ,score map<string,string>) row format delimited fields terminated by ',' collection items terminated by '-' map keys terminated by ':' location '/outertables';
select * from table2;
desc formatted table2;

加載本地文件到 內部表

加載文件到外部表

內部表在 drop 時候 hdfs 上 /hive/warhouse/ 下的數據 也被幹掉了
而 外部表的 則不會被幹掉

external 關鍵字

分區表 ，能夠提升 查找效率

create table table3(id int, name string, interest array , score map<string,string>) partitioned by (year int) row format delimited fields terminated by ',' collection items terminated by '-' map keys terminated by ':' stored as textfile;

刪除分區

列式存儲

sqoop 能夠認爲 是 ETL 工具， 數據採集，溝通 傳統關係型數據庫與hdfs的橋樑 （轉化爲 MR 操做）

wget https://mirrors.tuna.tsinghua.edu.cn/apache/sqoop/1.4.7/sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz
tar xzvf xxx
mv xxx /usr/local

cp ../apache-hive-2.3.5-bin/lib/mysql-connector-java-6.0.6.jar ./lib/

./sqoop list-databases --connect jdbc:mysql://localhost:3306?serverTimezone=UTC --username root -password root123

presto 安裝

wget https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/0.212/presto-cli-0.212-executable.jar

wget https://repo1.maven.org/maven2/com/facebook/presto/presto-server/0.212/presto-server-0.212.tar.gz

總結

hive sql  映射成爲 map reduce 
hive bin 目錄下經常使用命令

sqoop  命令詳細  導入導出， list 等

presto