[
https://issues.apache.org/jira/browse/FLINK-35754?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17863110#comment-17863110
]
Feng Jin edited comment on FLINK-35754 at 7/5/24 3:17 AM:
----------------------------------------------------------
{color:#000000}I tried to reproduce the issue locally, and the detailed error
message is as follows. When querying the test-filesystem source, the file was
overwritten by the refresh job.{color}
{code:java}
2024-07-05 09:40:02,600 WARN org.apache.flink.runtime.taskmanager.Task
[] - Source: my_materialized_table_in_full_mode[65] (1/1)#0
(990457fb8e9757dde310ff68388501f4_bc764cd8ddf7a0cff126f51c16239658_0_0)
switched from RUNNING to FAILED with failure cause:
java.lang.RuntimeException: One or more fetchers have encountered exception
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherManager.checkErrors(SplitFetcherManager.java:333)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.SourceReaderBase.getNextFetch(SourceReaderBase.java:228)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.SourceReaderBase.pollNext(SourceReaderBase.java:190)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.api.operators.SourceOperator.emitNext(SourceOperator.java:422)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.io.StreamTaskSourceInput.emitNext(StreamTaskSourceInput.java:68)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:65)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:638)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:231)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:973)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:917)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:970)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:949)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:763)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:575)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_392]
Caused by: java.lang.RuntimeException: SplitFetcher thread 0 received
unexpected exception while polling the records
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:168)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:117)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_392]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_392]
... 1 more
Caused by: java.io.FileNotFoundException: File
file:/var/folders/bn/63j9v_yx2_187ws6f7r2x8vr0000gn/T/junit5290522536490830518/filesystem_catalog_7/test_db/my_materialized_table_in_full_mode/data/ds=2024-07-05/part-9295afcb-1ce5-48bc-a6f8-1194e3fae586-task-0-file-0
does not exist or the user running Flink ('feng') has insufficient permissions
to access it.
at
org.apache.flink.core.fs.local.LocalFileSystem.getFileStatus(LocalFileSystem.java:106)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.api.common.io.FileInputFormat.open(FileInputFormat.java:849)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.api.common.io.DelimitedInputFormat.open(DelimitedInputFormat.java:499)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$LineBytesInputFormat.open(DeserializationSchemaAdapter.java:175)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$Reader.<init>(DeserializationSchemaAdapter.java:112)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$Reader.<init>(DeserializationSchemaAdapter.java:105)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter.createReader(DeserializationSchemaAdapter.java:85)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter.createReader(DeserializationSchemaAdapter.java:49)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.FileInfoExtractorBulkFormat.createReader(FileInfoExtractorBulkFormat.java:109)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.src.impl.FileSourceSplitReader.checkSplitOrStartNext(FileSourceSplitReader.java:112)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.src.impl.FileSourceSplitReader.fetch(FileSourceSplitReader.java:65)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.FetchTask.run(FetchTask.java:58)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:165)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:117)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_392]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_392]
... 1 more{code}
{color:#000000}{color:#000000}It can be observed that the jobs for "query" and
"fresh" are basically submitted at the same time, which leads to this
issue.{color}{color}
!image-2024-07-05-11-15-52-731.png!
was (Author: hackergin):
{color:#000000}I tried to reproduce the issue locally, and the detailed error
message is as follows. {color:#000000}{color:#000000}When querying the
test-filesystem source, the file was overwritten by the refresh
job.{color}{color}
{color}
{code:java}
2024-07-05 09:40:02,600 WARN org.apache.flink.runtime.taskmanager.Task
[] - Source: my_materialized_table_in_full_mode[65] (1/1)#0
(990457fb8e9757dde310ff68388501f4_bc764cd8ddf7a0cff126f51c16239658_0_0)
switched from RUNNING to FAILED with failure cause:
java.lang.RuntimeException: One or more fetchers have encountered exception
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherManager.checkErrors(SplitFetcherManager.java:333)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.SourceReaderBase.getNextFetch(SourceReaderBase.java:228)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.SourceReaderBase.pollNext(SourceReaderBase.java:190)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.api.operators.SourceOperator.emitNext(SourceOperator.java:422)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.io.StreamTaskSourceInput.emitNext(StreamTaskSourceInput.java:68)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:65)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:638)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:231)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:973)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:917)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:970)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:949)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:763)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:575)
[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_392]
Caused by: java.lang.RuntimeException: SplitFetcher thread 0 received
unexpected exception while polling the records
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:168)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:117)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_392]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_392]
... 1 more
Caused by: java.io.FileNotFoundException: File
file:/var/folders/bn/63j9v_yx2_187ws6f7r2x8vr0000gn/T/junit5290522536490830518/filesystem_catalog_7/test_db/my_materialized_table_in_full_mode/data/ds=2024-07-05/part-9295afcb-1ce5-48bc-a6f8-1194e3fae586-task-0-file-0
does not exist or the user running Flink ('feng') has insufficient permissions
to access it.
at
org.apache.flink.core.fs.local.LocalFileSystem.getFileStatus(LocalFileSystem.java:106)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.api.common.io.FileInputFormat.open(FileInputFormat.java:849)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.api.common.io.DelimitedInputFormat.open(DelimitedInputFormat.java:499)
~[flink-dist-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$LineBytesInputFormat.open(DeserializationSchemaAdapter.java:175)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$Reader.<init>(DeserializationSchemaAdapter.java:112)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter$Reader.<init>(DeserializationSchemaAdapter.java:105)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter.createReader(DeserializationSchemaAdapter.java:85)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.DeserializationSchemaAdapter.createReader(DeserializationSchemaAdapter.java:49)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.table.FileInfoExtractorBulkFormat.createReader(FileInfoExtractorBulkFormat.java:109)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.src.impl.FileSourceSplitReader.checkSplitOrStartNext(FileSourceSplitReader.java:112)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.file.src.impl.FileSourceSplitReader.fetch(FileSourceSplitReader.java:65)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.FetchTask.run(FetchTask.java:58)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.runOnce(SplitFetcher.java:165)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher.run(SplitFetcher.java:117)
~[flink-connector-files-2.0-SNAPSHOT.jar:2.0-SNAPSHOT]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
~[?:1.8.0_392]
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
~[?:1.8.0_392]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
~[?:1.8.0_392]
... 1 more{code}
> SqlGatewayE2ECase.testMaterializedTableInFullMode failed due to Internal
> Server Error
> -------------------------------------------------------------------------------------
>
> Key: FLINK-35754
> URL: https://issues.apache.org/jira/browse/FLINK-35754
> Project: Flink
> Issue Type: Bug
> Affects Versions: 1.20.0
> Reporter: Weijie Guo
> Priority: Major
> Attachments: image-2024-07-05-11-15-52-731.png
>
>
> {code:java}
> Jul 03 03:14:31 java.lang.RuntimeException: Execute statement failed
> Jul 03 03:14:31 at
> org.apache.flink.tests.util.flink.FlinkDistribution$TestSqlGatewayRestClient.executeStatementWithResult(FlinkDistribution.java:592)
> Jul 03 03:14:31 at
> org.apache.flink.table.gateway.SqlGatewayE2ECase.lambda$testMaterializedTableInFullMode$3(SqlGatewayE2ECase.java:353)
> Jul 03 03:14:31 at
> org.apache.flink.core.testutils.CommonTestUtils.waitUtil(CommonTestUtils.java:210)
> Jul 03 03:14:31 at
> org.apache.flink.table.gateway.SqlGatewayE2ECase.testMaterializedTableInFullMode(SqlGatewayE2ECase.java:350)
> Jul 03 03:14:31 at
> java.base/java.lang.reflect.Method.invoke(Method.java:568)
> Jul 03 03:14:31 at
> org.apache.flink.util.ExternalResource$1.evaluate(ExternalResource.java:48)
> Jul 03 03:14:31 at
> org.apache.flink.util.TestNameProvider$1.evaluate(TestNameProvider.java:45)
> Jul 03 03:14:31 at
> org.testcontainers.containers.FailureDetectingExternalResource$1.evaluate(FailureDetectingExternalResource.java:29)
> Jul 03 03:14:31 Caused by: java.lang.RuntimeException: The rest request is
> not successful: Internal Server Error
> Jul 03 03:14:31 at
> org.apache.flink.tests.util.flink.FlinkDistribution$TestSqlGatewayRestClient.sendRequest(FlinkDistribution.java:601)
> Jul 03 03:14:31 at
> org.apache.flink.tests.util.flink.FlinkDistribution$TestSqlGatewayRestClient.getOperationResult(FlinkDistribution.java:570)
> Jul 03 03:14:31 at
> org.apache.flink.tests.util.flink.FlinkDistribution$TestSqlGatewayRestClient.executeStatementWithResult(FlinkDistribution.java:590)
> Jul 03 03:14:31 ... 7 more
> Jul 03 03:14:31
> {code}
> https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=60620&view=logs&j=e8e46ef5-75cc-564f-c2bd-1797c35cbebe&t=60c49903-2505-5c25-7e46-de91b1737bea&l=14602
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
