This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new cabcf2549d7 branch-4.1:[fix](iceberg) Skip concurrently dropped nested
namespaces (#69628)
cabcf2549d7 is described below
commit cabcf2549d7658bd40c5088a0eaf8558bd787287
Author: daidai <[email protected]>
AuthorDate: Tue Oct 27 09:51:53 2026 +0800
branch-4.1:[fix](iceberg) Skip concurrently dropped nested namespaces
(#69628)
### What problem does this PR solve?
Issue Number: N/A
Related PR: #68248
Problem Summary:
When `iceberg.rest.nested-namespace-enabled=true`, Iceberg REST
namespace discovery recursively lists child namespaces. If another
session drops a child between listing the parent and listing its
children, `NoSuchNamespaceException` currently aborts the entire `SHOW
DATABASES` result.
This change catches `NoSuchNamespaceException` only at the child
recursion boundary. The stale child and its descendants are skipped,
while failures to list the root and other authorization or service
failures are still propagated. The behavior matches the existing master
implementation from #68248.
### Release note
Fix Iceberg REST nested namespace listing when another session drops a
namespace concurrently.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [ ] No.
- [x] Yes. A concurrently dropped child namespace no longer aborts the
catalog listing.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/iceberg/IcebergMetadataOps.java | 15 +++--
.../datasource/iceberg/IcebergMetadataOpTest.java | 78 ++++++++++++++++++++++
2 files changed, 89 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 2d131f31f12..3af3e027054 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -196,10 +196,17 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
if (nestedNamespaceEnabled) {
return nsCatalog.listNamespaces(parentNs)
.stream()
- .flatMap(childNs -> Stream.concat(
- Stream.of(childNs.toString()),
- listNestedNamespaces(childNs).stream()
- )).collect(Collectors.toList());
+ .flatMap(childNs -> {
+ try {
+ List<String> descendants =
listNestedNamespaces(childNs);
+ return
Stream.concat(Stream.of(childNs.toString()), descendants.stream());
+ } catch (NoSuchNamespaceException e) {
+ // A child can be dropped after its parent was
listed. Skip that branch,
+ // including the stale child name, without hiding
failures to list the root.
+ LOG.debug("Namespace {} was dropped during
listing", childNs, e);
+ return Stream.empty();
+ }
+ }).collect(Collectors.toList());
}
return nsCatalog.listNamespaces(parentNs)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
index 5cff492e473..15236f8354f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpTest.java
@@ -24,6 +24,7 @@ import
org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
+import org.apache.doris.datasource.property.metastore.IcebergRestProperties;
import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
import org.apache.iceberg.CatalogProperties;
@@ -35,6 +36,9 @@ import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.exceptions.ForbiddenException;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.exceptions.ServiceFailureException;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
@@ -57,6 +61,80 @@ import java.util.concurrent.atomic.AtomicInteger;
public class IcebergMetadataOpTest {
+ @Test
+ public void testListDatabaseNamesSkipsNamespacesDeletedDuringTraversal() {
+ Catalog icebergCatalog = Mockito.mock(Catalog.class,
+
Mockito.withSettings().extraInterfaces(SupportsNamespaces.class));
+ SupportsNamespaces nsCatalog = (SupportsNamespaces) icebergCatalog;
+ Namespace deleted = Namespace.of("deleted");
+ Namespace parent = Namespace.of("parent");
+ Namespace deletedChild = Namespace.of("parent", "deleted");
+ Namespace emptyChild = Namespace.of("parent", "empty");
+ Namespace sibling = Namespace.of("sibling");
+
+
Mockito.when(nsCatalog.listNamespaces(Namespace.empty())).thenReturn(Arrays.asList(deleted,
parent, sibling));
+
Mockito.when(nsCatalog.listNamespaces(parent)).thenReturn(Arrays.asList(deletedChild,
emptyChild));
+
Mockito.when(nsCatalog.listNamespaces(emptyChild)).thenReturn(Collections.emptyList());
+ Mockito.when(nsCatalog.listNamespaces(deleted))
+ .thenThrow(new NoSuchNamespaceException("Namespace does not
exist: %s", deleted));
+ Mockito.when(nsCatalog.listNamespaces(deletedChild))
+ .thenThrow(new NoSuchNamespaceException("Namespace does not
exist: %s", deletedChild));
+
+ List<String> result =
nestedNamespaceOps(icebergCatalog).listDatabaseNames();
+
+ Assert.assertEquals(Arrays.asList("parent", "parent.empty",
"sibling"), result);
+ }
+
+ @Test
+ public void testListDatabaseNamesPropagatesMissingRootNamespace() {
+ Catalog icebergCatalog = Mockito.mock(Catalog.class,
+
Mockito.withSettings().extraInterfaces(SupportsNamespaces.class));
+ SupportsNamespaces nsCatalog = (SupportsNamespaces) icebergCatalog;
+ NoSuchNamespaceException failure = new
NoSuchNamespaceException("Missing root");
+
Mockito.when(nsCatalog.listNamespaces(Namespace.empty())).thenThrow(failure);
+
+ RuntimeException exception =
Assert.assertThrows(RuntimeException.class,
+ () -> nestedNamespaceOps(icebergCatalog).listDatabaseNames());
+
+ Assert.assertSame(failure, exception.getCause());
+ }
+
+ @Test
+ public void testListDatabaseNamesPropagatesOtherTraversalFailures() {
+ for (RuntimeException failure : Arrays.asList(
+ new ForbiddenException("Access denied"), new
ServiceFailureException("Service unavailable"))) {
+ Catalog icebergCatalog = Mockito.mock(Catalog.class,
+
Mockito.withSettings().extraInterfaces(SupportsNamespaces.class));
+ SupportsNamespaces nsCatalog = (SupportsNamespaces) icebergCatalog;
+ Namespace child = Namespace.of("child");
+ Mockito.when(nsCatalog.listNamespaces(Namespace.empty()))
+ .thenReturn(Collections.singletonList(child));
+ Mockito.when(nsCatalog.listNamespaces(child)).thenThrow(failure);
+
+ RuntimeException exception =
Assert.assertThrows(RuntimeException.class,
+ () ->
nestedNamespaceOps(icebergCatalog).listDatabaseNames());
+
+ Assert.assertSame(failure, exception.getCause());
+ }
+ }
+
+ private IcebergMetadataOps nestedNamespaceOps(Catalog icebergCatalog) {
+ IcebergRestExternalCatalog dorisCatalog =
Mockito.mock(IcebergRestExternalCatalog.class);
+ CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class);
+ IcebergRestProperties restProperties =
Mockito.mock(IcebergRestProperties.class);
+ IcebergCatalogResourceTracker tracker = new
IcebergCatalogResourceTracker();
+
+
Mockito.when(restProperties.isIcebergRestNestedNamespaceEnabled()).thenReturn(true);
+
Mockito.when(catalogProperty.getMetastoreProperties()).thenReturn(restProperties);
+
Mockito.when(dorisCatalog.getCatalogProperty()).thenReturn(catalogProperty);
+ Mockito.when(dorisCatalog.getExecutionAuthenticator()).thenReturn(new
ExecutionAuthenticator() {
+ });
+
Mockito.when(dorisCatalog.getProperties()).thenReturn(Collections.emptyMap());
+ Mockito.when(dorisCatalog.beginCatalogOperation(Mockito.any()))
+ .thenAnswer(invocation -> tracker.beginOperation());
+ return new IcebergMetadataOps(dorisCatalog, icebergCatalog);
+ }
+
@Test
public void testCatalogOperationDelaysGenerationRetirement() throws
Exception {
IcebergExternalCatalog dorisCatalog =
Mockito.mock(IcebergExternalCatalog.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
