大數據第八課-HBase JAVA API 開發

時間 2020-06-03

標籤數據第八 hbase java api 開發欄目 Hadoop 简体版

原文原文鏈接

多虧一個老哥幫忙，否則就搞不下去了java

1、前期碰到的問題

IDEA 導入依賴包，不少次編譯不成功，檢查如下的方面node

一、maven裏面配置的本地倉庫位置。

二、settings的內容

     <mirror>
      <id>alimaven</id>
      <name>aliyun maven</name>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
      <mirrorOf>central</mirrorOf>        
    </mirror>

三、本地倉庫裏面的內容（看紅框裏面的文件是否在）

四、換本地倉庫

連接：https://pan.baidu.com/s/1VDB1QLPnWH-CKpgSUUXgZA 提取碼：z7qj 數據庫

2、HBase JAVA API的開發

一、依賴pom.xml文件內容

    <repositories>
        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>
    </repositories>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.0-mr1-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.testng</groupId>
            <artifactId>testng</artifactId>
            <version>6.14.3</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>1.2.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-server</artifactId>
            <version>1.2.0-cdh5.14.2</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.0</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>2.2</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*/RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

二、JAVA test的內容（在test裏面建立）

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class HBaseOperate {

    /**
     * 建立一張myuser表
     * @throws IOException
     */
    @Test
    public void createTable() throws IOException {

        Configuration configuration = HBaseConfiguration.create();
        //鏈接HBase集羣不須要指定HBase主節點的ip地址和端口號
        configuration.set("hbase.zookeeper.quorum","node01:2181,node02:2181,node03:2181");
        //建立鏈接對象
        Connection connection = ConnectionFactory.createConnection(configuration);
        //獲取鏈接對象，建立一張表
        //獲取管理員對象，來對手數據庫進行DDL的操做
        Admin admin = connection.getAdmin();
        //指定咱們的表名
        TableName myuser = TableName.valueOf("myuser");
        HTableDescriptor hTableDescriptor = new HTableDescriptor(myuser);
        //指定兩個列族
        HColumnDescriptor f1 = new HColumnDescriptor("f1");
        HColumnDescriptor f2 = new HColumnDescriptor("f2");
        hTableDescriptor.addFamily(f1);
        hTableDescriptor.addFamily(f2);
        admin.createTable(hTableDescriptor);
        admin.close();
        connection.close();

    }

    private Connection connection ;
    private final String TABLE_NAME = "myuser";
    private Table table ;

    @BeforeTest
    public void initTable () throws IOException {
        Configuration configuration = HBaseConfiguration.create();
        configuration.set("hbase.zookeeper.quorum","node01:2181,node02:2181,node03:2181");
        connection= ConnectionFactory.createConnection(configuration);
        table = connection.getTable(TableName.valueOf(TABLE_NAME));
    }

    @AfterTest
    public void close() throws IOException {
        table.close();
        connection.close();
    }


    /**
     *  向myuser表當中添加數據
     */
    @Test
    public void addData() throws IOException {
        //獲取表
        Table table = connection.getTable(TableName.valueOf(TABLE_NAME));
        Put put = new Put("0001".getBytes());//建立put對象，並指定rowkey值
        put.addColumn("f1".getBytes(),"name".getBytes(),"zhangsan".getBytes());
        put.addColumn("f1".getBytes(),"age".getBytes(), Bytes.toBytes(18));
        put.addColumn("f1".getBytes(),"id".getBytes(), Bytes.toBytes(25));
        put.addColumn("f1".getBytes(),"address".getBytes(), Bytes.toBytes("地球人"));
        table.put(put);
        table.close();
    }

    /**
     * hbase的批量插入數據
     */
    @Test
    public void batchInsert() throws IOException {
        //建立put對象，並指定rowkey
        Put put = new Put("0002".getBytes());
        //f1
        put.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(1));
        put.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("曹操"));
        put.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(30));
        //f2
        put.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("沛國譙縣"));
        put.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("16888888888"));
        put.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("helloworld"));

        Put put2 = new Put("0003".getBytes());
        put2.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(2));
        put2.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("劉備"));
        put2.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(32));
        put2.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put2.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("幽州涿郡涿縣"));
        put2.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("17888888888"));
        put2.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("talk is cheap , show me the code"));

        Put put3 = new Put("0004".getBytes());
        put3.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(3));
        put3.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("孫權"));
        put3.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(35));
        put3.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put3.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("下邳"));
        put3.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("12888888888"));
        put3.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("what are you 弄啥嘞！"));

        Put put4 = new Put("0005".getBytes());
        put4.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(4));
        put4.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("諸葛亮"));
        put4.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(28));
        put4.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put4.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("四川隆中"));
        put4.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("14888888888"));
        put4.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("出師表你背了嘛"));

        Put put5 = new Put("0006".getBytes());
        put5.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(5));
        put5.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("司馬懿"));
        put5.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(27));
        put5.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put5.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("哪裏人有待考究"));
        put5.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("15888888888"));
        put5.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("跟諸葛亮死掐"));


        Put put6 = new Put("0007".getBytes());
        put6.addColumn("f1".getBytes(),"id".getBytes(),Bytes.toBytes(5));
        put6.addColumn("f1".getBytes(),"name".getBytes(),Bytes.toBytes("xiaobubu—呂布"));
        put6.addColumn("f1".getBytes(),"age".getBytes(),Bytes.toBytes(28));
        put6.addColumn("f2".getBytes(),"sex".getBytes(),Bytes.toBytes("1"));
        put6.addColumn("f2".getBytes(),"address".getBytes(),Bytes.toBytes("內蒙人"));
        put6.addColumn("f2".getBytes(),"phone".getBytes(),Bytes.toBytes("15788888888"));
        put6.addColumn("f2".getBytes(),"say".getBytes(),Bytes.toBytes("貂蟬去哪了"));

        List<Put> listPut = new ArrayList<Put>();
        listPut.add(put);
        listPut.add(put2);
        listPut.add(put3);
        listPut.add(put4);
        listPut.add(put5);
        listPut.add(put6);

        table.put(listPut);
    }

    /**
     * 查詢rowkey爲0003的人
     */
    @Test
    public void getData() throws IOException {
        Table table = connection.getTable(TableName.valueOf(TABLE_NAME));
        //經過get對象，指定rowkey
        Get get = new Get(Bytes.toBytes("0003"));

        get.addFamily("f1".getBytes());//限制只查詢f1列族下面全部列的值
        //查詢f2  列族 phone  這個字段
        get.addColumn("f2".getBytes(),"phone".getBytes());
        //經過get查詢，返回一個result對象，全部的字段的數據都是封裝在result裏面了
        Result result = table.get(get);
        List<Cell> cells = result.listCells();  //獲取一條數據全部的cell，全部數據值都是在cell裏面 的
        for (Cell cell : cells) {
            byte[] family_name = CellUtil.cloneFamily(cell);//獲取列族名
            byte[] column_name = CellUtil.cloneQualifier(cell);//獲取列名
            byte[] rowkey = CellUtil.cloneRow(cell);//獲取rowkey
            byte[] cell_value = CellUtil.cloneValue(cell);//獲取cell值
            //須要判斷字段的數據類型，使用對應的轉換的方法，纔可以獲取到值
            if("age".equals(Bytes.toString(column_name))  || "id".equals(Bytes.toString(column_name))){
                System.out.println(Bytes.toString(family_name));
                System.out.println(Bytes.toString(column_name));
                System.out.println(Bytes.toString(rowkey));
                System.out.println(Bytes.toInt(cell_value));
            }else{
                System.out.println(Bytes.toString(family_name));
                System.out.println(Bytes.toString(column_name));
                System.out.println(Bytes.toString(rowkey));
                System.out.println(Bytes.toString(cell_value));
            }
        }
        table.close();
    }

    /**
     * 不知道rowkey的具體值，我想查詢rowkey範圍值是0003  到0006
     * select * from myuser  where age > 30  and id < 8  and name like 'zhangsan'
     *
     */
    @Test
    public void scanData() throws IOException {
        //獲取table
        Table table = connection.getTable(TableName.valueOf(TABLE_NAME));
        Scan scan = new Scan();//沒有指定startRow以及stopRow  全表掃描
        //只掃描f1列族
        scan.addFamily("f1".getBytes());
        //掃描 f2列族 phone  這個字段
        scan.addColumn("f2".getBytes(),"phone".getBytes());
        scan.setStartRow("0003".getBytes());
        scan.setStopRow("0007".getBytes());
        //經過getScanner查詢獲取到了表裏面全部的數據，是多條數據
        ResultScanner scanner = table.getScanner(scan);
        //遍歷ResultScanner 獲得每一條數據，每一條數據都是封裝在result對象裏面了
        for (Result result : scanner) {
            List<Cell> cells = result.listCells();
            for (Cell cell : cells) {
                byte[] family_name = CellUtil.cloneFamily(cell);
                byte[] qualifier_name = CellUtil.cloneQualifier(cell);
                byte[] rowkey = CellUtil.cloneRow(cell);
                byte[] value = CellUtil.cloneValue(cell);
                //判斷id和age字段，這兩個字段是整形值
                if("age".equals(Bytes.toString(qualifier_name))  || "id".equals(Bytes.toString(qualifier_name))){
                    System.out.println("數據的rowkey爲" +  Bytes.toString(rowkey)   +"======數據的列族爲" +  Bytes.toString(family_name)+"======數據的列名爲" +  Bytes.toString(qualifier_name) + "==========數據的值爲" +Bytes.toInt(value));
                }else{
                    System.out.println("數據的rowkey爲" +  Bytes.toString(rowkey)   +"======數據的列族爲" +  Bytes.toString(family_name)+"======數據的列名爲" +  Bytes.toString(qualifier_name) + "==========數據的值爲" +Bytes.toString(value));
                }
            }
        }
        table.close();
    }

}