This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 30ebc1d325 [core] Improve the error when listing a Format Table custom
partition location fails (#9748)
30ebc1d325 is described below
commit 30ebc1d325303336a0c919db7b6d4e6fb242b962
Author: Dapeng Sun(孙大鹏) <[email protected]>
AuthorDate: Sat Sep 12 21:39:48 2026 +0800
[core] Improve the error when listing a Format Table custom partition
location fails (#9748)
---
.../table/format/CatalogSplitEnumerator.java | 44 ++++++++++++++++----
.../format/CatalogManagedPartitionScanTest.java | 48 ++++++++++++++++++++++
2 files changed, 84 insertions(+), 8 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java
b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java
index 1425cd538e..a65d224de3 100644
---
a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java
+++
b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogSplitEnumerator.java
@@ -116,19 +116,26 @@ final class CatalogSplitEnumerator extends
SplitEnumerator {
for (Pair<LinkedHashMap<String, String>, Path> partition : partitions)
{
boolean useCatalogContextFileIO =
fileIOResolver.useCatalogContextFileIO(partition.getValue());
- if (useCatalogContextFileIO) {
- fileIOResolver.prepare(partition.getValue(), true);
- } else if (!tableFileIOPrepared) {
- fileIOResolver.prepare(partition.getValue(), false);
- tableFileIOPrepared = true;
+ try {
+ if (useCatalogContextFileIO) {
+ fileIOResolver.prepare(partition.getValue(), true);
+ } else if (!tableFileIOPrepared) {
+ fileIOResolver.prepare(partition.getValue(), false);
+ tableFileIOPrepared = true;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(
+ listFailureMessage(
+ partition.getKey(), partition.getValue(),
useCatalogContextFileIO),
+ e);
}
}
Function<Pair<LinkedHashMap<String, String>, Path>, List<Split>>
lister =
pair -> {
BinaryRow partitionRow = toPartitionRow(pair.getKey());
+ boolean useCatalogContextFileIO =
+
fileIOResolver.useCatalogContextFileIO(pair.getValue());
try {
- boolean useCatalogContextFileIO =
-
fileIOResolver.useCatalogContextFileIO(pair.getValue());
return createSplits(
fileIOResolver.fileIO(useCatalogContextFileIO),
pair.getValue(),
@@ -139,7 +146,9 @@ final class CatalogSplitEnumerator extends SplitEnumerator {
return Collections.emptyList();
} catch (IOException e) {
throw new RuntimeException(
- "Failed to list files for partition " +
pair.getValue(), e);
+ listFailureMessage(
+ pair.getKey(), pair.getValue(),
useCatalogContextFileIO),
+ e);
}
};
int parallelism =
@@ -362,6 +371,25 @@ final class CatalogSplitEnumerator extends SplitEnumerator
{
spec, table.fullName(), table.partitionKeys()));
}
+ /** Says which credentials listed the partition, so a permission failure
is actionable. */
+ private String listFailureMessage(
+ LinkedHashMap<String, String> spec, Path path, boolean
useCatalogContextFileIO) {
+ String message =
+ String.format(
+ "Failed to list files for partition '%s' of format
table %s at '%s'.",
+ PartitionPathUtils.generatePartitionName(spec, false),
+ table.fullName(),
+ path);
+ if (!useCatalogContextFileIO) {
+ return message;
+ }
+ return message
+ + " The partition is registered at a custom location outside
the table"
+ + " directory, so it is listed with the filesystem credentials
of the catalog"
+ + " context (the engine's own configuration, for example
fs.oss.* or the Hadoop"
+ + " configuration), not with the table's data token.";
+ }
+
private void warnMissingPartition(LinkedHashMap<String, String> spec, Path
path) {
LOG.warn(
"Partition '{}' of format table {} is registered in the
catalog but its directory "
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
index 5b6b2f226c..8e070dd7da 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
@@ -545,6 +545,52 @@ class CatalogManagedPartitionScanTest {
assertThat(tableFileIO.listedPaths).isEmpty();
}
+ @Test
+ void testCustomLocationListingFailureNamesTheCatalogContextCredentials()
throws Exception {
+ InjectingLocalFileIO clientFileIO = new InjectingLocalFileIO();
+ Path externalPath = new Path(tempDir.resolve("external").toUri());
+ writeDataFile(clientFileIO, externalPath, "files");
+ clientFileIO.failListingContaining("external", new IOException("not
found login secrets"));
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(),
isNull()))
+ .thenReturn(
+ new PagedList<>(
+ Collections.singletonList(
+ partition("2025", "11",
externalPath.toString())),
+ null));
+ Path tablePath = new Path(tempDir.resolve("table").toUri());
+ TableRootOnlyLocalFileIO tableFileIO = new
TableRootOnlyLocalFileIO(tablePath);
+ FileIOLoader clientLoader =
+ new FileIOLoader() {
+ @Override
+ public String getScheme() {
+ return "file";
+ }
+
+ @Override
+ public LocalFileIO load(Path path) {
+ return clientFileIO;
+ }
+ };
+ CatalogContext catalogContext = CatalogContext.create(new Options(),
clientLoader, null);
+ FormatTable table =
+ createTable(
+ tableFileIO, tablePath, partitionManager(catalog),
false, catalogContext);
+
+ // A permission failure on an archive bucket must say that the
engine's own credentials
+ // listed it, or the user keeps looking at the table token.
+ assertThatThrownBy(() -> new FormatTableScan(table, null,
null).plan().splits())
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("partition 'year=2025/month=11'")
+ .hasMessageContaining(table.fullName())
+ .hasMessageContaining("'" + externalPath + "'")
+ .hasMessageContaining("custom location outside the table
directory")
+ .hasMessageContaining("credentials of the catalog context")
+ .hasMessageContaining("not with the table's data token")
+ .hasRootCauseMessage("not found login secrets");
+ assertThat(tableFileIO.listedPaths).isEmpty();
+ }
+
@Test
void testWhitespacePartitionValueIsVisible() throws Exception {
Catalog catalog = mock(Catalog.class);
@@ -919,6 +965,8 @@ class CatalogManagedPartitionScanTest {
stringPartitionTable(fileIO, tablePath,
recordingCatalog(partitions), 4);
assertThatThrownBy(() -> new FormatTableScan(table, null,
null).plan().splits())
.isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("partition 'year=2025/month=11'")
+ .hasMessageNotContaining("custom location")
.hasRootCauseInstanceOf(IOException.class)
.hasRootCauseMessage("boom");
}
