spark部署之yarn模式

spark部署之yarn模式

• hadoop-3.0.0集羣搭建
• 配置相應環境
  • java環境
  • scala（可配可不配）
  • hadoop環境
• 從官網下載spark
  • 解壓
  • 配置 /conf/spark-env.sh

    export JAVA_HOME=/usr/java/jdk1.8.0_45
    export HADOOP_CONF_DIR=/usr/java/hadoop-3.0.0/etc/hadoop
    export SPARK_MASTER_HOST=master
    export SPARK_WORKER_MEMORY=1g

  • 啓動
    • 啓動hdfs

    start-dfs.sh

    • 啓動yarn

    start-yarn.sh

    • 啓動spark-shell

    ./spark-shell --master yarn --deploy-mode client

    • 報錯

    2019-04-22 11:15:44,640 ERROR spark.SparkContext: Error initializing SparkContext.
    org.apache.spark.SparkException: Yarn application has already ended! It might have been killed or unable to launch application master.
    	at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.waitForApplication(YarnClientSchedulerBackend.scala:85)
    	at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:62)
    	at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:173)
    	at org.apache.spark.SparkContext.<init>(SparkContext.scala:509)
    	at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2516)
    	at org.apache.spark.sql.SparkSession$Builder$$anonfun$6.apply(SparkSession.scala:918)
    	at org.apache.spark.sql.SparkSession$Builder$$anonfun$6.apply(SparkSession.scala:910)
    	at scala.Option.getOrElse(Option.scala:121)
    	at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:910)
    	at org.apache.spark.repl.Main$.createSparkSession(Main.scala:101)
    	at $line3.$read$$iw$$iw.<init>(<console>:15)
    	at $line3.$read$$iw.<init>(<console>:42)
    	at $line3.$read.<init>(<console>:44)
    	at $line3.$read$.<init>(<console>:48)
    	at $line3.$read$.<clinit>(<console>)
    	at $line3.$eval$.$print$lzycompute(<console>:7)
    	at $line3.$eval$.$print(<console>:6)
    	at $line3.$eval.$print(<console>)
    	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    	at java.lang.reflect.Method.invoke(Method.java:497)
    	at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
    	at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
    	at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
    	at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
    	at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
    	at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
    	at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
    	at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
    	at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
    	at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
    	at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
    	at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
    	at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply$mcV$sp(SparkILoop.scala:38)
    	at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:37)
    	at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:37)
    	at scala.tools.nsc.interpreter.IMain.beQuietDuring(IMain.scala:214)
    	at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:37)
    	at org.apache.spark.repl.SparkILoop.loadFiles(SparkILoop.scala:98)
    	at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:920)
    	at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
    	at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
    	at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
    	at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
    	at org.apache.spark.repl.Main$.doMain(Main.scala:74)
    	at org.apache.spark.repl.Main$.main(Main.scala:54)
    	at org.apache.spark.repl.Main.main(Main.scala)
    	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    	at java.lang.reflect.Method.invoke(Method.java:497)
    	at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
    	at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
    	at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
    	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
    	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
    2019-04-22 11:15:44,796 WARN cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!
    2019-04-22 11:15:45,030 WARN metrics.MetricsSystem: Stopping a MetricsSystem that is not running
    org.apache.spark.SparkException: Yarn application has already ended! It might have been killed or unable to launch application master.
      at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.waitForApplication(YarnClientSchedulerBackend.scala:85)
      at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:62)
      at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:173)
      at org.apache.spark.SparkContext.<init>(SparkContext.scala:509)
      at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2516)
      at org.apache.spark.sql.SparkSession$Builder$$anonfun$6.apply(SparkSession.scala:918)
      at org.apache.spark.sql.SparkSession$Builder$$anonfun$6.apply(SparkSession.scala:910)
      at scala.Option.getOrElse(Option.scala:121)
      at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:910)
      at org.apache.spark.repl.Main$.createSparkSession(Main.scala:101)
      ... 47 elided

    • 解決
      • 停掉yarn

      stop-yarn.sh

      • 新增配置hadoop下的 /hadoop-3.0.0/etc/hadoop/yarn-site.xml

      <property>
        <name>yarn.nodemanager.vmem-check-enabled</name>
        <value>false</value>
        <description>Whether virtual memory limits will be enforced for containers</description>
      </property>
      <property>
         <name>yarn.nodemanager.vmem-pmem-ratio</name>
         <value>4</value>
         <description>Ratio between virtual memory to physical memory when setting memory limits for containers</description>
      </property>

      • 啓動yarn
      • 啓動spark-shell
      • 訪問web ui

      http://master:4040