xxubai opened a new issue, #3817: URL: https://github.com/apache/amoro/issues/3817
### What happened? As title ### Affects Versions master/0.8.x ### What table formats are you seeing the problem on? Iceberg ### What engines are you seeing the problem on? AMS ### How to reproduce _No response_ ### Relevant log output ```shell 2025-10-13 06:58:02,918 ERROR [async-orphan-files-cleaning-executor-0] [org.apache.amoro.server.table.executor.OrphanFilesCleaningExecutor] [] - catalog.db.table(tableId=512791) failed to clean orphan file java.lang.IllegalStateException: Connection pool shut down at org.apache.http.util.Asserts.check(Asserts.java:34) ~[httpcore-4.4.13.jar:4.4.13] at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:269) ~[httpclient-4.5.13.jar:4.5.13] at software.amazon.awssdk.http.apache.internal.conn.ClientConnectionManagerFactory<span>DelegatingHttpClientConnectionManager.requestConnection(ClientConnectionManagerFactory.java:75) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.http.apache.internal.conn.ClientConnectionManagerFactory</span>InstrumentedHttpClientConnectionManager.requestConnection(ClientConnectionManagerFactory.java:57) ~[apache-client-2.24.12.jar:?] at org.apache.http.impl.execchain.MainClientExec.exec ute(MainClientExec.java:176) ~[httpclient-4.5.13.jar:4.5.13] at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[httpclient-4.5.13.jar:4.5.13] at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[httpclient-4.5.13.jar:4.5.13] at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.13.jar:4.5.13] at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.13.jar:4.5.13] at software.amazon.awssdk.http.apache.internal.impl.ApacheSdkHttpClient.execute(ApacheSdkHttpClient.java:72) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.http.apache.ApacheHttpClient.execute(ApacheHttpClient.java:254) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.http.apache.ApacheHttpClient.access<span>500(ApacheHttpClient.java:104) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.http.apache.ApacheHttpClient</span>1.call(ApacheH ttpClient.java:231) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.http.apache.ApacheHttpClient<span>1.call(ApacheHttpClient.java:228) ~[apache-client-2.24.12.jar:?] at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:99) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:79) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:57) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:40) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder</span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipel ineBuilder<span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder</span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder<span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:72) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78) ~[sdk-core-2.24.12.jar:?] at software.a mazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:55) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:39) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder</span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.S treamManagingStage.execute(StreamManagingStage.java:56) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:50) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:32) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder<span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder</span>ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient<span>RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:224) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(Bas eSyncClientHandler.java:103) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:173) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda</span>execute<span>1(BaseSyncClientHandler.java:80) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:182) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:74) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45) ~[sdk-core-2.24.12.jar:?] at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:53) ~[aws-core-2.24.12.jar:?] at software.amazon.awssdk.services.s3.DefaultS3Client.headObject(DefaultS3Client.java:6319) ~[s3-2. 24.12.jar:?] at org.apache.iceberg.aws.s3.BaseS3File.getObjectMetadata(BaseS3File.java:85) ~[iceberg-aws-1.7.2.jar:?] at org.apache.iceberg.aws.s3.S3InputFile.getLength(S3InputFile.java:77) ~[iceberg-aws-1.7.2.jar:?] at org.apache.iceberg.avro.AvroIterable.newFileReader(AvroIterable.java:102) ~[iceberg-core-1.7.2.jar:?] at org.apache.iceberg.avro.AvroIterable.iterator(AvroIterable.java:77) ~[iceberg-core-1.7.2.jar:?] at org.apache.iceberg.io.CloseableIterable</span>7<span>1.<init>(CloseableIterable.java:188) ~[iceberg-api-1.7.2.jar:?] at org.apache.iceberg.io.CloseableIterable</init></span>7.iterator(CloseableIterable.java:187) ~[iceberg-api-1.7.2.jar:?] at org.apache.iceberg.io.CloseableIterable<span>7</span>1.<init>(CloseableIterable.java:188) ~[iceberg-api-1.7.2.jar:?] at org.apache.iceberg.io.CloseableIterable<span>7.iterator(CloseableIterable.java:187) ~[iceberg-api-1.7.2.jar:?] at org.apache.iceberg.io.CloseableIterable</span>ConcatCloseableIterable<span>ConcatCloseableIterato r.hasNext(CloseableIterable.java:257) ~[iceberg-api-1.7.2.jar:?] at java.lang.Iterable.forEach(Iterable.java:74) ~[?:?] at org.apache.amoro.server.utils.IcebergTableUtil.getAllManifestFiles(IcebergTableUtil.java:213) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintainer.getValidMetadataFiles(IcebergTableMaintainer.java:599) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintainer.clearInternalTableMetadata(IcebergTableMaintainer.java:411) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintainer.cleanMetadata(IcebergTableMaintainer.java:307) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.optimizing.maintainer.IcebergTableMaintainer.cleanOrphanFiles(IcebergTableMaintainer.java:142) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.serve r.table.executor.OrphanFilesCleaningExecutor.execute(OrphanFilesCleaningExecutor.java:61) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.table.executor.BaseTableExecutor.executeTask(BaseTableExecutor.java:82) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at org.apache.amoro.server.table.executor.BaseTableExecutor.lambda</span>initHandler<span>0(BaseTableExecutor.java:72) ~[amoro-ams-0.8.0-incubating.jar:0.8.0-incubating] at java.util.concurrent.Executors</span>RunnableAdapter.call(Executors.java:515) [?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?] at java.util.concurrent.ScheduledThreadPoolExecutor<span>ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) [?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?] at java.util.concurrent.ThreadPoolExecutor</span>Worker.run(ThreadPoolExecutor.java:628) [?:?] at java.lang.Thread.run(Thread.java:829) [?:?]</init> ``` ### Anything else 1. Register Iceberg tables which are stored in S3 2. Enable orphan cleaning for some Iceberg tables 3. Clean metadata orphan files ### Are you willing to submit a PR? - [x] Yes I am willing to submit a PR! ### Code of Conduct - [x] I agree to follow this project's Code of Conduct -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
