vamsikarnika commented on PR #11681:
URL: https://github.com/apache/hudi/pull/11681#issuecomment-2257504848
> Hi @vamsikarnika, could you paste the stacktrace here and provide a bit
more context about the issue?
Hi @CTTY, While running the metasync for Glue, we realised that
`last_commit_time_sync` is not getting populated for Snapshot table, which is
causing the meta sync to sync all the partitions everytime and it's taking a
lot of time.
While debugging we realized that the issue is AWS Glue Client is getting
closed before the async call to update last synced time is completed.
Stacktrace:
java.util.concurrent.RejectedExecutionException: event executor terminated
at
io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:923)
at
io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:350)
at
io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:343)
at
io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:825)
at
io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:815)
at
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
at
io.netty.util.concurrent.AbstractEventExecutor.submit(AbstractEventExecutor.java:115)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.utils.NettyUtils.doInEventLoop(NettyUtils.java:254)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.http2.HttpOrHttp2ChannelPool.close(HttpOrHttp2ChannelPool.java:215)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.ListenerInvokingChannelPool.close(ListenerInvokingChannelPool.java:132)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.ReleaseOnceChannelPool.close(ReleaseOnceChannelPool.java:95)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.HealthCheckedChannelPool.close(HealthCheckedChannelPool.java:150)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.CancellableAcquireChannelPool.close(CancellableAcquireChannelPool.java:76)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.SimpleChannelPoolAwareChannelPool.close(SimpleChannelPoolAwareChannelPool.java:57)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.SdkChannelPoolMap.remove(SdkChannelPoolMap.java:56)
at
java.util.concurrent.ConcurrentHashMap$KeySetView.forEach(ConcurrentHashMap.java:4649)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.SdkChannelPoolMap.close(SdkChannelPoolMap.java:93)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.AwaitCloseChannelPoolMap.close(AwaitCloseChannelPoolMap.java:166)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.internal.utils.NettyUtils.runAndLogError(NettyUtils.java:386)
at
org.apache.hudi.software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient.close(NettyNioAsyncHttpClient.java:198)
at
org.apache.hudi.software.amazon.awssdk.utils.IoUtils.closeQuietly(IoUtils.java:70)
at
org.apache.hudi.software.amazon.awssdk.utils.IoUtils.closeIfCloseable(IoUtils.java:87)
at
org.apache.hudi.software.amazon.awssdk.utils.AttributeMap.closeIfPossible(AttributeMap.java:674)
at
org.apache.hudi.software.amazon.awssdk.utils.AttributeMap.access$1600(AttributeMap.java:49)
at
org.apache.hudi.software.amazon.awssdk.utils.AttributeMap$DerivedValue.close(AttributeMap.java:628)
at java.util.HashMap$Values.forEach(HashMap.java:982)
at
org.apache.hudi.software.amazon.awssdk.utils.AttributeMap.close(AttributeMap.java:107)
at
org.apache.hudi.software.amazon.awssdk.core.client.config.SdkClientConfiguration.close(SdkClientConfiguration.java:126)
at
org.apache.hudi.software.amazon.awssdk.core.internal.http.HttpClientDependencies.close(HttpClientDependencies.java:82)
at
org.apache.hudi.software.amazon.awssdk.core.internal.http.AmazonAsyncHttpClient.close(AmazonAsyncHttpClient.java:75)
at
org.apache.hudi.software.amazon.awssdk.core.internal.handler.BaseAsyncClientHandler.close(BaseAsyncClientHandler.java:254)
at
org.apache.hudi.software.amazon.awssdk.services.glue.DefaultGlueAsyncClient.close(DefaultGlueAsyncClient.java:15110)
at
org.apache.hudi.aws.sync.AWSGlueCatalogSyncClient.close(AWSGlueCatalogSyncClient.java:578)
at org.apache.hudi.hive.HiveSyncTool.close(HiveSyncTool.java:216)
at
org.apache.hudi.sync.common.util.SyncUtilHelpers.runHoodieMetaSync(SyncUtilHelpers.java:80)
at
org.apache.hudi.utilities.streamer.StreamSync.runMetaSync(StreamSync.java:1067)
at
org.apache.hudi.utilities.streamer.StreamSync.writeToSinkAndDoMetaSync(StreamSync.java:907)
at
org.apache.hudi.utilities.streamer.StreamSync.syncOnce(StreamSync.java:471)
at
com.onehouse.hudi.OnehouseDeltaStreamer$MultiTableSyncService$StreamSyncService.ingestOnce(OnehouseDeltaStreamer.java:1379)
at org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
at
com.onehouse.hudi.OnehouseDeltaStreamer$MultiTableSyncService.lambda$scheduleStreamInfo$6(OnehouseDeltaStreamer.java:461)
at
java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
