spark-sql性能優化之——多線程實現多Job併發執行

直接上代碼

val spark = SparkSession.builder() .appName("name") .master("local[2]") .getOrCreate() val df = spark.read.json("src\\main\\resources\\json.txt") df.show() //沒有多線程處理的狀況，連續執行兩個Action操做，生成兩個Job
  df.rdd.saveAsTextFile("") df.rdd.saveAsTextFile("") //用Executor實現多線程方式處理Job
  val dfList = Array(df,df) val executorService = Executors.newFixedThreadPool(2) for(df <- dfList) { executorService.submit(new Callable[Boolean]() { def call() : Boolean = { df.show() true } }) } executorService.shutdown() spark.stop()