[
https://issues.apache.org/jira/browse/HUDI-7394?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
voon updated HUDI-7394:
-----------------------
Description:
Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
Avro version.
hadoop-hdfs might be shipped with avro versions of an older version (depending
on the hadoop-hdfs version).
For example:
{code:java}
Running command : java -cp
...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
Class conflict occurs with the following error:
{code:java}
java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
at java.lang.Class.getDeclaredFields0(Native Method)
at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
at java.lang.Class.getDeclaredField(Class.java:2068)
at org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
at
org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
at
org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
at
org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
at
org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
at
org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
at
org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
at
org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
at
org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
at
io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
at
org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException:
org.apache.avro.message.BinaryMessageEncoder
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 56 more {code}
Firing a fix here to:
# Override the avro scope in hudi-timeline-server-bundle to compile
# Shade hudi-timeline-server-bundle's avro
This in theory will not break anything in the other bundles, i.e. Spark, which
uses the provided Avro in Spark's runtime.
Note that the changes here are only done in the hudi-timeline-server-bundle,
other modules are not modified.
was:
Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
Avro version.
hadoop-hdfs might be shipped with avro versions of an older version (depending
on the hadoop-hdfs version).
For example:
{code:java}
Running command : java -cp
...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
Class conflict occurs with the following error:
{code:java}
java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
at java.lang.Class.getDeclaredFields0(Native Method)
at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
at java.lang.Class.getDeclaredField(Class.java:2068)
at org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
at
org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
at
org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
at
org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
at
org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
at
org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
at
org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
at
org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
at
org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
at
io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
at
org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException:
org.apache.avro.message.BinaryMessageEncoder
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 56 more {code}
Firing a fix here to:
# Override the avro scope in hudi-timeline-server-bundle to compile
# Shade hudi-timeline-server-bundle's avro
This in theory will not break anything in the other bundles, i.e. Spark, which
uses the provided Avro in Spark's runtime.
Note that the changes here are only done in the hudi-timeline-server-bundle,
other modules are not modified.
> Fix run script of hudi-timeline-server-bundle
> ---------------------------------------------
>
> Key: HUDI-7394
> URL: https://issues.apache.org/jira/browse/HUDI-7394
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: voon
> Assignee: voon
> Priority: Major
>
> Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
> Avro version.
>
> hadoop-hdfs might be shipped with avro versions of an older version
> (depending on the hadoop-hdfs version).
>
> For example:
> {code:java}
> Running command : java -cp
> ...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
>
> Class conflict occurs with the following error:
> {code:java}
> java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
> at java.lang.Class.getDeclaredFields0(Native Method)
> at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
> at java.lang.Class.getDeclaredField(Class.java:2068)
> at
> org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
> at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
> at
> org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
> at
> org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
> at
> org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
> at
> org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
> at
> org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
> at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> at
> java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
> at
> org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
> at
> org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
> at
> org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
> at
> org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
> at
> java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
> at
> org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
> at
> org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:502)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.ClassNotFoundException:
> org.apache.avro.message.BinaryMessageEncoder
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
> ... 56 more {code}
>
>
> Firing a fix here to:
> # Override the avro scope in hudi-timeline-server-bundle to compile
> # Shade hudi-timeline-server-bundle's avro
>
> This in theory will not break anything in the other bundles, i.e. Spark,
> which uses the provided Avro in Spark's runtime.
>
> Note that the changes here are only done in the hudi-timeline-server-bundle,
> other modules are not modified.
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
