Spark操做mysql

1. 1. 示例代碼

val spark = SparkSession

      .builder()

      .appName("Spark SQL basic example")

      .config("spark.some.config.option", "some-value")

      .getOrCreate()

 

    val connectionProperties = new Properties()

    connectionProperties.put("user", "CMALLusr")

    connectionProperties.put("password", "PCy5VJYd1Ysn")

    val df = spark.read.jdbc("jdbc:mysql://10.37.154.65:3306/cmallpre2", "t_order_1", connectionProperties).toDF

   

    //df.select("SKU_NAME").show()

    val group_df=  df.groupBy("SKU_NAME").count().orderBy("count")

    group_df.show()

    group_df.printSchema

    //group_df.toJavaRDD.saveAsTextFile("/home/spark/group.txt")

spark.close()

 

1. 1. Maven的pom配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>sp1</groupId>

    <artifactId>sp1</artifactId>

    <version>0.0.1-SNAPSHOT</version>

 

    <dependencies>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-core_2.11</artifactId>

            <version>2.2.0</version>

        </dependency>

       

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-sql_2.11</artifactId>

            <version>2.2.0</version>

        </dependency>

       

        <dependency>

        <groupId>mysql</groupId>

        <artifactId>mysql-connector-java</artifactId>

        <version>5.1.6</version>

    </dependency>

   

 

        <!--spark scala -->

        <!-- <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId>

            <version>${scala.version}</version> </dependency> -->

 

    </dependencies>

 

    <build>

        <sourceDirectory>src</sourceDirectory>

        <plugins>

            <plugin>

                <artifactId>maven-compiler-plugin</artifactId>

                <version>3.1</version>

                <configuration>

                    <source />

                    <target />

                </configuration>

            </plugin>

 

            <plugin>

                <groupId>net.alchim31.maven</groupId>

                <artifactId>scala-maven-plugin</artifactId>

                <version>3.2.2</version>

                <executions>

                    <execution>

                        <goals>

                            <goal>compile</goal>

                            <goal>testCompile</goal>

                        </goals>

 

                    </execution>

                </executions>

                <configuration>

                    <scalaVersion>2.11.8</scalaVersion>

                </configuration>

            </plugin>

 

        </plugins>

    </build>

</project>