Spark讀取mysql數據

添加依賴：

<dependency>

            <groupId>mysql</groupId>

            <artifactId>mysql-connector-java</artifactId>

            <version>5.1.6</version>

        </dependency>

讀取mysql數據：

SparkSession spark = SparkSession

                  .builder()

                  .appName("Java Spark MYSQL basic example")             

                  .getOrCreate();

       

    String url = "jdbc:mysql://10.37.154.65:3306/cmallpre2";

    String table = "t_es_order";

    Properties connectionProperties = new Properties();

    connectionProperties.setProperty("dbtable", table);// 設置表

    connectionProperties.setProperty("user", "CMALLusr");// 設置用戶名

    connectionProperties.setProperty("password", "PCy5VJYd1Ysn");// 設置密碼

 

    // 一個條件表示一個分區

    String[] predicates = new String[] {

            "1=1 order by TABLE_ID limit 1,10",

            "1=1 order by TABLE_ID limit 20,30" };

    // 讀取數據

    Dataset<Row> rows =spark.read().jdbc(url, table, predicates

                connectionProperties);

    //添加篩選條件

    Dataset<Row> filter = rows.filter(col("TABLE_ID").gt("10"));

    System.out.println("mysql count:" + filter.count());

    spark.close();