nsivabalan commented on a change in pull request #3880:
URL: https://github.com/apache/hudi/pull/3880#discussion_r741526371
##########
File path: packaging/hudi-hive-sync-bundle/pom.xml
##########
@@ -69,12 +69,18 @@
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hive-sync</include>
+ <include>org.apache.hbase:hbase-client</include>
Review comment:
Integ test failure stacktrace
```
2021-10-27T17:37:02.0772040Z
#################################################################################################
2021-10-27T17:37:02.0774104Z 76786 [main] INFO
org.apache.hudi.integ.ITTestBase - Container : /adhoc-1, Running command
:spark-submit --class org.apache.hudi.hive.HiveSyncTool
/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-sync-bundle.jar
--database default --table stock_ticks_cow --base-path
/user/hive/warehouse/stock_ticks_cow --base-file-format PARQUET --user hive
--pass hive --jdbc-url jdbc:hive2://hiveserver:10000 --partitioned-by dt
2021-10-27T17:37:02.0775664Z 76786 [main] INFO
org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:02.0776313Z
#################################################################################################
2021-10-27T17:37:08.2649560Z 82970 [dockerjava-jaxrs-async-6] INFO
org.apache.hudi.integ.ITTestBase - onComplete called
2021-10-27T17:37:08.2650772Z 82973 [main] INFO
org.apache.hudi.integ.ITTestBase - Exit code for command : 1
2021-10-27T17:37:08.2651626Z 82973 [main] ERROR
org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2651987Z
2021-10-27T17:37:08.2652302Z ###### Stdout #######
2021-10-27T17:37:08.2652511Z
2021-10-27T17:37:08.2653143Z 82973 [main] ERROR
org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2653492Z
2021-10-27T17:37:08.2653794Z ###### Stderr #######
2021-10-27T17:37:08.2654735Z 21/10/27 17:37:03 WARN NativeCodeLoader: Unable
to load native-hadoop library for your platform... using builtin-java classes
where applicable
2021-10-27T17:37:08.2655593Z Exception in thread "main"
java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
2021-10-27T17:37:08.2656511Z at
org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex.<clinit>(HFileBootstrapIndex.java:92)
2021-10-27T17:37:08.2657147Z at java.lang.Class.forName0(Native
Method)
2021-10-27T17:37:08.2657684Z at
java.lang.Class.forName(Class.java:264)
2021-10-27T17:37:08.2658313Z at
org.apache.hudi.common.util.ReflectionUtils.getClass(ReflectionUtils.java:54)
2021-10-27T17:37:08.2659027Z at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:89)
2021-10-27T17:37:08.2659750Z at
org.apache.hudi.common.bootstrap.index.BootstrapIndex.getBootstrapIndex(BootstrapIndex.java:163)
2021-10-27T17:37:08.2660565Z at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:108)
2021-10-27T17:37:08.2661393Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:106)
2021-10-27T17:37:08.2662210Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:100)
2021-10-27T17:37:08.2662997Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:91)
2021-10-27T17:37:08.2663883Z at
org.apache.hudi.metadata.HoodieTableMetadataUtil.loadPartitionFileGroupsWithLatestFileSlices(HoodieTableMetadataUtil.java:352)
2021-10-27T17:37:08.2664793Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$openReadersIfNeeded$2(HoodieBackedTableMetadata.java:185)
2021-10-27T17:37:08.2665914Z at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
2021-10-27T17:37:08.2666733Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.openReadersIfNeeded(HoodieBackedTableMetadata.java:176)
2021-10-27T17:37:08.2667568Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKeyFromMetadata(HoodieBackedTableMetadata.java:124)
2021-10-27T17:37:08.2668390Z at
org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:153)
2021-10-27T17:37:08.2669162Z at
org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:94)
2021-10-27T17:37:08.2669904Z at
org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:281)
2021-10-27T17:37:08.2670666Z at
org.apache.hudi.sync.common.AbstractSyncHoodieClient.getPartitionsWrittenToSince(AbstractSyncHoodieClient.java:157)
2021-10-27T17:37:08.2671447Z at
org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:191)
2021-10-27T17:37:08.2672111Z at
org.apache.hudi.hive.HiveSyncTool.doSync(HiveSyncTool.java:131)
2021-10-27T17:37:08.2672783Z at
org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:117)
2021-10-27T17:37:08.2673420Z at
org.apache.hudi.hive.HiveSyncTool.main(HiveSyncTool.java:359)
2021-10-27T17:37:08.2679252Z at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2021-10-27T17:37:08.2679805Z at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2021-10-27T17:37:08.2680439Z at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2021-10-27T17:37:08.2680986Z at
java.lang.reflect.Method.invoke(Method.java:498)
2021-10-27T17:37:08.2681730Z at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
2021-10-27T17:37:08.2682398Z at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
2021-10-27T17:37:08.2683025Z at
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
2021-10-27T17:37:08.2683600Z at
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
2021-10-27T17:37:08.2684159Z at
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
2021-10-27T17:37:08.2684738Z at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
2021-10-27T17:37:08.2685438Z at
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
2021-10-27T17:37:08.2685953Z at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2021-10-27T17:37:08.2686470Z Caused by: java.lang.ClassNotFoundException:
org.apache.hadoop.hbase.util.Bytes
2021-10-27T17:37:08.2687071Z at
java.net.URLClassLoader.findClass(URLClassLoader.java:382)
2021-10-27T17:37:08.2687764Z at
java.lang.ClassLoader.loadClass(ClassLoader.java:424)
2021-10-27T17:37:08.2688262Z at
java.lang.ClassLoader.loadClass(ClassLoader.java:357)
2021-10-27T17:37:08.2688635Z ... 35 more
2021-10-27T17:37:08.2688791Z
2021-10-27T17:37:08.2689569Z 82974 [main] WARN
org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2689775Z
2021-10-27T17:37:08.2691310Z :::::: Original Command : [spark-submit,
--class, org.apache.hudi.hive.HiveSyncTool,
/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-sync-bundle.jar,
--database, default, --table, stock_ticks_cow, --base-path,
/user/hive/warehouse/stock_ticks_cow, --base-file-format, PARQUET, --user,
hive, --pass, hive, --jdbc-url, jdbc:hive2://hiveserver:10000,
--partitioned-by, dt]
2021-10-27T17:37:08.2692540Z 82974 [main] WARN
org.apache.hudi.integ.ITTestBase -
2021-10-27T17:37:08.2692747Z
2021-10-27T17:37:08.2693272Z :::::: Stderr of timed-out command ::::::
2021-10-27T17:37:08.2694154Z 21/10/27 17:37:03 WARN NativeCodeLoader: Unable
to load native-hadoop library for your platform... using builtin-java classes
where applicable
2021-10-27T17:37:08.2694870Z Exception in thread "main"
java.lang.NoClassDefFoundError: org/apache/hadoop/hbase/util/Bytes
2021-10-27T17:37:08.2695520Z at
org.apache.hudi.common.bootstrap.index.HFileBootstrapIndex.<clinit>(HFileBootstrapIndex.java:92)
2021-10-27T17:37:08.2696177Z at java.lang.Class.forName0(Native
Method)
2021-10-27T17:37:08.2696599Z at
java.lang.Class.forName(Class.java:264)
2021-10-27T17:37:08.2697097Z at
org.apache.hudi.common.util.ReflectionUtils.getClass(ReflectionUtils.java:54)
2021-10-27T17:37:08.2697706Z at
org.apache.hudi.common.util.ReflectionUtils.loadClass(ReflectionUtils.java:89)
2021-10-27T17:37:08.2698350Z at
org.apache.hudi.common.bootstrap.index.BootstrapIndex.getBootstrapIndex(BootstrapIndex.java:163)
2021-10-27T17:37:08.2699033Z at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:108)
2021-10-27T17:37:08.2699759Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:106)
2021-10-27T17:37:08.2700537Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:100)
2021-10-27T17:37:08.2701199Z at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.<init>(HoodieTableFileSystemView.java:91)
2021-10-27T17:37:08.2701889Z at
org.apache.hudi.metadata.HoodieTableMetadataUtil.loadPartitionFileGroupsWithLatestFileSlices(HoodieTableMetadataUtil.java:352)
2021-10-27T17:37:08.2702634Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$openReadersIfNeeded$2(HoodieBackedTableMetadata.java:185)
2021-10-27T17:37:08.2703280Z at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
2021-10-27T17:37:08.2704079Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.openReadersIfNeeded(HoodieBackedTableMetadata.java:176)
2021-10-27T17:37:08.2704827Z at
org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKeyFromMetadata(HoodieBackedTableMetadata.java:124)
2021-10-27T17:37:08.2705610Z at
org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:153)
2021-10-27T17:37:08.2706276Z at
org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:94)
2021-10-27T17:37:08.2707029Z at
org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:281)
2021-10-27T17:37:08.2707709Z at
org.apache.hudi.sync.common.AbstractSyncHoodieClient.getPartitionsWrittenToSince(AbstractSyncHoodieClient.java:157)
2021-10-27T17:37:08.2708388Z at
org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:191)
2021-10-27T17:37:08.2708930Z at
org.apache.hudi.hive.HiveSyncTool.doSync(HiveSyncTool.java:131)
2021-10-27T17:37:08.2709491Z at
org.apache.hudi.hive.HiveSyncTool.syncHoodieTable(HiveSyncTool.java:117)
2021-10-27T17:37:08.2710045Z at
org.apache.hudi.hive.HiveSyncTool.main(HiveSyncTool.java:359)
2021-10-27T17:37:08.2710527Z at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2021-10-27T17:37:08.2711056Z at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2021-10-27T17:37:08.2711675Z at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2021-10-27T17:37:08.2712214Z at
java.lang.reflect.Method.invoke(Method.java:498)
2021-10-27T17:37:08.2712756Z at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
2021-10-27T17:37:08.2713418Z at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
2021-10-27T17:37:08.2714150Z at
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
2021-10-27T17:37:08.2714662Z at
org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
2021-10-27T17:37:08.2715187Z at
org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
2021-10-27T17:37:08.2715751Z at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
2021-10-27T17:37:08.2716310Z at
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
2021-10-27T17:37:08.2716807Z at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2021-10-27T17:37:08.2717318Z Caused by: java.lang.ClassNotFoundException:
org.apache.hadoop.hbase.util.Bytes
2021-10-27T17:37:08.2717836Z at
java.net.URLClassLoader.findClass(URLClassLoader.java:382)
2021-10-27T17:37:08.2718421Z at
java.lang.ClassLoader.loadClass(ClassLoader.java:424)
2021-10-27T17:37:08.2718923Z at
java.lang.ClassLoader.loadClass(ClassLoader.java:357)
2021-10-27T17:37:08.2719294Z ... 35 more
2021-10-27T17:37:08.2719474Z
```
##########
File path: packaging/hudi-hive-sync-bundle/pom.xml
##########
@@ -69,12 +69,18 @@
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hive-sync</include>
+ <include>org.apache.hbase:hbase-client</include>
Review comment:
with the patch, CI is green now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
