wangyum commented on a change in pull request #23788: [SPARK-23710][SQL] Only
upgrade hadoop-3.1's built-in Hive to 2.3.4
URL: https://github.com/apache/spark/pull/23788#discussion_r258386063
##########
File path: pom.xml
##########
@@ -2697,7 +2830,69 @@
<hadoop.version>3.1.0</hadoop.version>
<curator.version>2.12.0</curator.version>
<zookeeper.version>3.4.9</zookeeper.version>
+ <hive.group>org.apache.hive</hive.group>
+ <hive.classifier>core</hive.classifier>
+ <hive.version>2.3.4</hive.version>
+ <hive.version.short>${hive.version}</hive.version.short>
+ <hive.parquet.version>${parquet.version}</hive.parquet.version>
+ <orc.classifier></orc.classifier>
+ <calcite-avatica.artifactId>calcite-druid</calcite-avatica.artifactId>
+ <calcite.version>1.10.0</calcite.version>
+ <hive.parquet.group>org.apache.parquet</hive.parquet.group>
+ <datanucleus-core.version>4.1.17</datanucleus-core.version>
</properties>
+ <dependencies>
+ <dependency>
+ <groupId>${hive.group}</groupId>
+ <artifactId>hive-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${hive.group}</groupId>
+ <artifactId>hive-serde</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${hive.group}</groupId>
+ <artifactId>hive-shims</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${hive.group}</groupId>
+ <artifactId>hive-llap-client</artifactId>
Review comment:
Need this dependency, otherwise:
```java
build/sbt "hive/testOnly *.StatisticsSuite" -Phadoop-3.1
sbt.ForkMain$ForkError: java.lang.NoClassDefFoundError:
org/apache/hadoop/hive/llap/security/LlapSigner$Signable
at java.lang.Class.getDeclaredConstructors0(Native Method)
at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671)
at java.lang.Class.getConstructor0(Class.java:3075)
at java.lang.Class.getDeclaredConstructor(Class.java:2178)
at
org.apache.hive.common.util.ReflectionUtil.newInstance(ReflectionUtil.java:79)
at
org.apache.hadoop.hive.ql.exec.Registry.registerGenericUDTF(Registry.java:208)
at
org.apache.hadoop.hive.ql.exec.Registry.registerGenericUDTF(Registry.java:201)
at
org.apache.hadoop.hive.ql.exec.FunctionRegistry.<clinit>(FunctionRegistry.java:500)
at
org.apache.spark.sql.hive.test.TestHiveSparkSession.<init>(TestHive.scala:521)
at
org.apache.spark.sql.hive.test.TestHiveSparkSession.<init>(TestHive.scala:181)
at
org.apache.spark.sql.hive.test.TestHiveContext.<init>(TestHive.scala:129)
at org.apache.spark.sql.hive.test.TestHive$.<init>(TestHive.scala:53)
at org.apache.spark.sql.hive.test.TestHive$.<clinit>(TestHive.scala)
at
org.apache.spark.sql.hive.test.TestHiveSingleton.$init$(TestHiveSingleton.scala:30)
at
org.apache.spark.sql.hive.StatisticsSuite.<init>(StatisticsSuite.scala:45)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at
org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:435)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: sbt.ForkMain$ForkError: java.lang.ClassNotFoundException:
org.apache.hadoop.hive.llap.security.LlapSigner$Signable
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.getDeclaredConstructors0(Native Method)
at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671)
at java.lang.Class.getConstructor0(Class.java:3075)
at java.lang.Class.getDeclaredConstructor(Class.java:2178)
at
org.apache.hive.common.util.ReflectionUtil.newInstance(ReflectionUtil.java:79)
at
org.apache.hadoop.hive.ql.exec.Registry.registerGenericUDTF(Registry.java:208)
at
org.apache.hadoop.hive.ql.exec.Registry.registerGenericUDTF(Registry.java:201)
at
org.apache.hadoop.hive.ql.exec.FunctionRegistry.<clinit>(FunctionRegistry.java:500)
at
org.apache.spark.sql.hive.test.TestHiveSparkSession.<init>(TestHive.scala:521)
at
org.apache.spark.sql.hive.test.TestHiveSparkSession.<init>(TestHive.scala:181)
at
org.apache.spark.sql.hive.test.TestHiveContext.<init>(TestHive.scala:129)
at org.apache.spark.sql.hive.test.TestHive$.<init>(TestHive.scala:53)
at org.apache.spark.sql.hive.test.TestHive$.<clinit>(TestHive.scala)
at
org.apache.spark.sql.hive.test.TestHiveSingleton.$init$(TestHiveSingleton.scala:30)
at
org.apache.spark.sql.hive.StatisticsSuite.<init>(StatisticsSuite.scala:45)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native
Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at
org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:435)
at sbt.ForkMain$Run$2.call(ForkMain.java:296)
at sbt.ForkMain$Run$2.call(ForkMain.java:286)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
