[
https://issues.apache.org/jira/browse/HUDI-7394?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
voon updated HUDI-7394:
-----------------------
Description:
h1. Issue Description
Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
Avro version.
hadoop-hdfs might be shipped with avro versions of an older version (depending
on the hadoop-hdfs version).
For example:
{code:java}
Running command : java -cp
...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
Class conflict occurs with the following error:
{code:java}
java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
at java.lang.Class.getDeclaredFields0(Native Method)
at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
at java.lang.Class.getDeclaredField(Class.java:2068)
at org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
at
org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
at
org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
at
org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
at
org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
at
org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
at
org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
at
org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
at
org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
at
io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
at
org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException:
org.apache.avro.message.BinaryMessageEncoder
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 56 more {code}
Firing a fix here to:
# Override the avro scope in hudi-timeline-server-bundle to compile
# Shade hudi-timeline-server-bundle's avro
This in theory will not break anything in the other bundles, i.e. Spark, which
uses the provided Avro in Spark's runtime.
Note that the changes here are only done in the hudi-timeline-server-bundle,
other modules are not modified.
h1. Steps to reproduce
Start a standalone hudi-timeline-server using the
hudi-timeline-server-bundle.jar
{code:java}
sh run.sh
{code}
In another shell console, send a request to it:
{code:java}
curl
'http://localhost:26754/v1/hoodie/view/timeline/instants/all?basepath=hdfs%3A%2F%2Fdev%2Fhudi_table'
{code}
was:
Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
Avro version.
hadoop-hdfs might be shipped with avro versions of an older version (depending
on the hadoop-hdfs version).
For example:
{code:java}
Running command : java -cp
...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
Class conflict occurs with the following error:
{code:java}
java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
at java.lang.Class.getDeclaredFields0(Native Method)
at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
at java.lang.Class.getDeclaredField(Class.java:2068)
at org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
at
org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
at
org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
at
org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
at
org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
at
org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
at
org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
at
org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
at
org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
at
org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
at
org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
at
java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
at
org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
at
org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
at
org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
at
org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
at
io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at
org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
at
org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at
org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at
org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
at
org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
at
org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:502)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at
org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at
org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at
org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
at
org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException:
org.apache.avro.message.BinaryMessageEncoder
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 56 more {code}
Firing a fix here to:
# Override the avro scope in hudi-timeline-server-bundle to compile
# Shade hudi-timeline-server-bundle's avro
This in theory will not break anything in the other bundles, i.e. Spark, which
uses the provided Avro in Spark's runtime.
Note that the changes here are only done in the hudi-timeline-server-bundle,
other modules are not modified.
h1. Steps to reproduce
Start a standalone hudi-timeline-server using the
hudi-timeline-server-bundle.jar
{code:java}
sh run.sh
{code}
In another shell console, send a request to it:
{code:java}
curl
'http://localhost:26754/v1/hoodie/view/timeline/instants/all?basepath=hdfs%3A%2F%2Fdev%2Fhudi_table'
{code}
> Fix run script of hudi-timeline-server-bundle
> ---------------------------------------------
>
> Key: HUDI-7394
> URL: https://issues.apache.org/jira/browse/HUDI-7394
> Project: Apache Hudi
> Issue Type: Bug
> Reporter: voon
> Assignee: voon
> Priority: Major
> Labels: pull-request-available
>
> h1. Issue Description
> Hudi's timeline server bundle relies/is using hadoop-hdfs's common-lib-jars
> Avro version.
>
> hadoop-hdfs might be shipped with avro versions of an older version
> (depending on the hadoop-hdfs version).
>
> For example:
> {code:java}
> Running command : java -cp
> ...:/usr/share/hadoop-client/share/hadoop/common/lib/avro-1.7.7.jar:... {code}
>
> Class conflict occurs with the following error:
> {code:java}
> java.lang.NoClassDefFoundError: Lorg/apache/avro/message/BinaryMessageEncoder;
> at java.lang.Class.getDeclaredFields0(Native Method)
> at java.lang.Class.privateGetDeclaredFields(Class.java:2583)
> at java.lang.Class.getDeclaredField(Class.java:2068)
> at
> org.apache.avro.specific.SpecificData.createSchema(SpecificData.java:240)
> at org.apache.avro.specific.SpecificData.getSchema(SpecificData.java:189)
> at
> org.apache.avro.specific.SpecificDatumReader.<init>(SpecificDatumReader.java:34)
> at
> org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeAvroMetadata(TimelineMetadataUtils.java:203)
> at
> org.apache.hudi.common.table.timeline.TimelineMetadataUtils.deserializeCompactionPlan(TimelineMetadataUtils.java:166)
> at
> org.apache.hudi.common.util.CompactionUtils.getCompactionPlan(CompactionUtils.java:149)
> at
> org.apache.hudi.common.util.CompactionUtils.lambda$getAllPendingCompactionPlans$2(CompactionUtils.java:139)
> at
> java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
> at
> java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
> at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
> at
> java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
> at
> java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
> at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
> at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:566)
> at
> org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionPlans(CompactionUtils.java:143)
> at
> org.apache.hudi.common.util.CompactionUtils.getAllPendingCompactionOperations(CompactionUtils.java:163)
> at
> org.apache.hudi.common.table.view.AbstractTableFileSystemView.init(AbstractTableFileSystemView.java:113)
> at
> org.apache.hudi.common.table.view.HoodieTableFileSystemView.init(HoodieTableFileSystemView.java:108)
> at
> org.apache.hudi.common.table.view.SpillableMapBasedFileSystemView.<init>(SpillableMapBasedFileSystemView.java:72)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.createSpillableMapBasedFileSystemView(FileSystemViewManager.java:156)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.lambda$createViewManager$31512a51$1(FileSystemViewManager.java:255)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.lambda$getFileSystemView$0(FileSystemViewManager.java:103)
> at
> java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660)
> at
> org.apache.hudi.common.table.view.FileSystemViewManager.getFileSystemView(FileSystemViewManager.java:101)
> at
> org.apache.hudi.timeline.service.handlers.TimelineHandler.getTimeline(TimelineHandler.java:50)
> at
> org.apache.hudi.timeline.service.RequestHandler.lambda$registerTimelineAPI$2(RequestHandler.java:237)
> at
> org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:518)
> at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22)
> at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46)
> at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17)
> at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143)
> at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41)
> at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107)
> at
> io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
> at
> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
> at
> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
> at
> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
> at
> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
> at
> org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61)
> at
> org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174)
> at
> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
> at org.eclipse.jetty.server.Server.handle(Server.java:502)
> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
> at
> org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
> at
> org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765)
> at
> org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.lang.ClassNotFoundException:
> org.apache.avro.message.BinaryMessageEncoder
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
> ... 56 more {code}
>
>
> Firing a fix here to:
> # Override the avro scope in hudi-timeline-server-bundle to compile
> # Shade hudi-timeline-server-bundle's avro
>
> This in theory will not break anything in the other bundles, i.e. Spark,
> which uses the provided Avro in Spark's runtime.
>
> Note that the changes here are only done in the hudi-timeline-server-bundle,
> other modules are not modified.
>
> h1. Steps to reproduce
> Start a standalone hudi-timeline-server using the
> hudi-timeline-server-bundle.jar
> {code:java}
> sh run.sh
> {code}
> In another shell console, send a request to it:
> {code:java}
> curl
> 'http://localhost:26754/v1/hoodie/view/timeline/instants/all?basepath=hdfs%3A%2F%2Fdev%2Fhudi_table'
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
