最近項目準備把hadoop的MR轉換爲Spark,之前的MR是能夠直接提交java文件到集羣服務器中,但Spark我沒有找到相應的方式(有大神知道如何處理但願能夠告之下),我這邊使用了SparkAppHandle的方式來進行處理.java
CountDownLatch cdl= new CountDownLatch(1); SparkAppHandle handle = new SparkLauncher().setSparkHome("/usr/local/spark-2.2.0") .setAppResource("/usr/local/spark-2.2.0/lib/spark.jar") .setMainClass("run.aaa.spark.SimpleApp") .setMaster("yarn").setDeployMode("client") .setAppName("test yarn client") .setConf("spark.yarn.jars", "hdfs://master:9000/tmp/spark-jars/*") .setConf("spark.driver.allowMultipleContexts", "true") .setConf("spark.executor.cores", "2") .setConf("spark.executor.instances", "2") .addAppArgs("/README.md") .setVerbose(true) .startApplication(new SparkAppHandle.Listener() { // 這裏監放任務狀態,當任務結束時(不論是什麼緣由結束),isFinal方法會返回true,不然返回false @Override public void stateChanged(SparkAppHandle sparkAppHandle) { if (sparkAppHandle.getState().isFinal()) { cdl.countDown(); } System.out.println("state:" + sparkAppHandle.getState().toString()); } @Override public void infoChanged(SparkAppHandle sparkAppHandle) { System.out.println("Info:" + sparkAppHandle.getState().toString()); } }); System.out.println("The task is executing, please wait ...."); // 線程等待任務結束 cdl.await(); System.out.println("The task is finished!");