無法連接到配置單元通過斯卡拉火花sql

Question

我試圖連接到配置單元並通過scala選擇一些數據在火花sql測試，但我沒有這樣做。我使用的代碼如下：

object LoadHive { 
def main(args: Array[String]) { 
if (args.length < 2) { 
    println("Usage: [sparkmaster] [tablename]") 
    exit(1) 
} 
    val master = args(0) 
    val tableName = args(1) 
    val sc = new SparkContext(master, "LoadHive", System.getenv("SPARK_HOME")) 
    val hiveCtx = new HiveContext(sc) 
    val input = hiveCtx.sql("FROM src SELECT key, value") 
    val data = input.map(_.getInt(0)) 
    println(data.collect().toList) 
} 
} 

和異常日誌如下：

16/11/28 07:25:34 WARN Hive: Failed to access metastore. This class should not accessed in runtime. 
org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient 
    at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1236) 
    at org.apache.hadoop.hive.ql.metadata.Hive.reloadFunctions(Hive.java:174) 
    at org.apache.hadoop.hive.ql.metadata.Hive.<clinit>(Hive.java:166) 
    at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:503) 
    at org.apache.spark.sql.hive.client.ClientWrapper.<init>(ClientWrapper.scala:204) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:526) 
    at org.apache.spark.sql.hive.client.IsolatedClientLoader.createClient(IsolatedClientLoader.scala:249) 
    at org.apache.spark.sql.hive.HiveContext.metadataHive$lzycompute(HiveContext.scala:327) 
    at org.apache.spark.sql.hive.HiveContext.metadataHive(HiveContext.scala:237) 
    at org.apache.spark.sql.hive.HiveContext.setConf(HiveContext.scala:441) 
    at org.apache.spark.sql.hive.HiveContext.defaultOverrides(HiveContext.scala:226) 
    at org.apache.spark.sql.hive.HiveContext.<init>(HiveContext.scala:229) 
    at org.apache.spark.sql.hive.HiveContext.<init>(HiveContext.scala:101) 
    at com.oreilly.learningsparkexamples.scala.LoadHive$.main(LoadHive.scala:19) 
    at com.oreilly.learningsparkexamples.scala.LoadHive.main(LoadHive.scala) 
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
    at java.lang.reflect.Method.invoke(Method.java:606) 
    at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731) 
    at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) 
    at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) 
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) 
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 
Caused by: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient 
    at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1523) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:86) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:132) 
    at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104) 
    at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3005) 
    at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3024) 
    at org.apache.hadoop.hive.ql.metadata.Hive.getAllDatabases(Hive.java:1234) 
    ... 26 more 
Caused by: java.lang.reflect.InvocationTargetException 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) 
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) 
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) 
    at java.lang.reflect.Constructor.newInstance(Constructor.java:526) 
    at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1521) 
    ... 32 more 
Caused by: java.lang.NoSuchMethodError: org.apache.thrift.EncodingUtils.setBit(BIZ)B 
    at org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo.setCreateTimeIsSet(PrivilegeGrantInfo.java:245) 
    at org.apache.hadoop.hive.metastore.api.PrivilegeGrantInfo.<init>(PrivilegeGrantInfo.java:163) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultRoles_core(HiveMetaStore.java:675) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultRoles(HiveMetaStore.java:645) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:462) 
    at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:66) 
    at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:72) 
    at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:5762) 
    at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:199) 
    at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:74) 
    ... 37 more 

我覺得這個很困惑，有沒有人遇到這個問題？我是否需要用蜂巢來構建火花來解決這個問題？我認爲火花已經支持配置單元。我使用的火花版本是1.6.1。任何幫助表示讚賞！

Answer 1

要驗證星火正在與蜂巢運行

./bin/spark-sql --master local[*] 

，並執行任意SQL查詢（從SRC如SELECT COUNT（1）;）

注意星火必須與蜂巢相關支持在類路徑上存在hive-site.xml。