解決辦法:
換了kerbores用戶,原來用的super_hdfs,後來換成了bdmp_testjava
報錯信息以下:apache
org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:376) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:453) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:435) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.hadoop.hive.thrift.TFilterTransport.readAll(TFilterTransport.java:62) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_functions(ThriftHiveMetastore.java:3425) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_functions(ThriftHiveMetastore.java:3413) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllFunctions(HiveMetaStoreClient.java:2146) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:105) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2084) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3568) at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:230) at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:214) at org.apache.hadoop.hive.ql.metadata.Hive.<init>(Hive.java:337) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:298) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:273) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:245) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anon$1.run(YarnSparkHadoopUtil.scala:323) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.doAsRealUser(YarnSparkHadoopUtil.scala:322) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastoreInner(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:199) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:146) at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:348) at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:721) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143) at org.apache.spark.deploy.yarn.Client.run(Client.scala:1012) at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072) at org.apache.spark.deploy.yarn.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 18/01/30 15:07:11 INFO hive.metastore: Closed a connection to metastore, current connections: 0 18/01/30 15:07:11 INFO hive.metastore: Trying to connect to metastore with URI thrift://redhat208.life.com:9083 18/01/30 15:07:11 INFO hive.metastore: Opened a connection to metastore, current connections: 1 18/01/30 15:07:11 INFO hive.metastore: Connected to metastore. 18/01/30 15:07:11 WARN metadata.Hive: Failed to register all functions. org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.thrift.transport.TTransportException at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3571) at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:230) at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:214) at org.apache.hadoop.hive.ql.metadata.Hive.<init>(Hive.java:337) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:298) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:273) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:245) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anon$1.run(YarnSparkHadoopUtil.scala:323) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.doAsRealUser(YarnSparkHadoopUtil.scala:322) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastoreInner(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:199) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:146) at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:348) at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:721) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143) at org.apache.spark.deploy.yarn.Client.run(Client.scala:1012) at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072) at org.apache.spark.deploy.yarn.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:376) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:453) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:435) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.hadoop.hive.thrift.TFilterTransport.readAll(TFilterTransport.java:62) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_functions(ThriftHiveMetastore.java:3425) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_functions(ThriftHiveMetastore.java:3413) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllFunctions(HiveMetaStoreClient.java:2146) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:105) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2084) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3568) ... 34 more 18/01/30 15:07:11 WARN security.UserGroupInformation: PriviledgedActionException as:superhdfs@HADOOP.TEST (auth:KERBEROS) cause:java.lang.reflect.InvocationTargetException Exception in thread "main" org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.thrift.transport.TTransportException at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:219) at org.apache.hadoop.hive.ql.metadata.Hive.<init>(Hive.java:337) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:298) at org.apache.hadoop.hive.ql.metadata.Hive.get(Hive.java:273) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:245) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anonfun$obtainTokenForHiveMetastoreInner$4.apply(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$$anon$1.run(YarnSparkHadoopUtil.scala:323) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.doAsRealUser(YarnSparkHadoopUtil.scala:322) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastoreInner(YarnSparkHadoopUtil.scala:244) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:199) at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil.obtainTokenForHiveMetastore(YarnSparkHadoopUtil.scala:146) at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:348) at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scala:721) at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143) at org.apache.spark.deploy.yarn.Client.run(Client.scala:1012) at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072) at org.apache.spark.deploy.yarn.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: org.apache.thrift.transport.TTransportException at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3571) at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:230) at org.apache.hadoop.hive.ql.metadata.Hive.registerAllFunctionsOnce(Hive.java:214) ... 32 more Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.thrift.transport.TSaslTransport.readLength(TSaslTransport.java:376) at org.apache.thrift.transport.TSaslTransport.readFrame(TSaslTransport.java:453) at org.apache.thrift.transport.TSaslTransport.read(TSaslTransport.java:435) at org.apache.thrift.transport.TSaslClientTransport.read(TSaslClientTransport.java:37) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.hadoop.hive.thrift.TFilterTransport.readAll(TFilterTransport.java:62) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:77) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.recv_get_all_functions(ThriftHiveMetastore.java:3425) at org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore$Client.get_all_functions(ThriftHiveMetastore.java:3413) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.getAllFunctions(HiveMetaStoreClient.java:2146) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.invoke(RetryingMetaStoreClient.java:105) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient$SynchronizedHandler.invoke(HiveMetaStoreClient.java:2084) at com.sun.proxy.$Proxy30.getAllFunctions(Unknown Source) at org.apache.hadoop.hive.ql.metadata.Hive.getAllFunctions(Hive.java:3568)