flyrain commented on code in PR #4408:
URL: https://github.com/apache/polaris/pull/4408#discussion_r3238415065
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java:
##########
@@ -1242,70 +1241,87 @@ private <T extends PolarisEntity & LocationBasedEntity>
void validateNoLocationO
return Namespace.of(newLevels);
})
.toList();
- LOGGER.debug(
- "Resolving {} sibling entities to validate location",
- siblingTables.size() + siblingNamespaces.size());
- PolarisResolutionManifest resolutionManifest =
- new PolarisResolutionManifest(
- diagnostics,
- callContext.getRealmContext(),
- resolverFactory,
- principal,
- parentPath.getFirst().getName());
+ List<ResolvedPathKey> pathsToResolve =
+ new ArrayList<>(siblingTables.size() + siblingNamespaces.size());
siblingTables.forEach(
- tbl ->
- resolutionManifest.addPath(
- new ResolverPath(
- PolarisCatalogHelpers.tableIdentifierToList(tbl),
- PolarisEntityType.TABLE_LIKE)));
+ tbl -> {
+ if (!tbl.name().equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofTableLike(tbl));
+ }
+ });
siblingNamespaces.forEach(
- ns ->
- resolutionManifest.addPath(
- new ResolverPath(Arrays.asList(ns.levels()),
PolarisEntityType.NAMESPACE)));
+ ns -> {
+ if (!ns.level(ns.length() - 1).equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofNamespace(ns));
+ }
+ });
+
+ StorageLocation targetLocation = StorageLocation.of(location);
+ for (PolarisEntity entityToCheck :
+ resolveOptionalPaths(pathsToResolve, parentPath.getFirst().getName()))
{
+ String loc =
+
entityToCheck.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION);
+ if (loc == null) {
+ continue;
+ }
+
+ StorageLocation siblingLocation = StorageLocation.of(loc);
+
+ if (targetLocation.isChildOf(siblingLocation) ||
siblingLocation.isChildOf(targetLocation)) {
+ throw new ForbiddenException(
+ "Unable to create table at location '%s' because it conflicts with
existing table or namespace at "
Review Comment:
This method also runs for namespaces — message reads weird in that case.
"Unable to create entity" or similar.
##########
runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogTest.java:
##########
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.PolarisDiagnostics;
+import org.apache.polaris.core.auth.PolarisPrincipal;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.entity.CatalogEntity;
+import org.apache.polaris.core.entity.PolarisEntity;
+import org.apache.polaris.core.entity.PolarisEntityType;
+import org.apache.polaris.core.exceptions.CommitConflictException;
+import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
+import
org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
+import org.apache.polaris.core.persistence.resolver.ResolvedPathKey;
+import org.apache.polaris.core.persistence.resolver.Resolver;
+import org.apache.polaris.core.persistence.resolver.ResolverFactory;
+import org.apache.polaris.core.persistence.resolver.ResolverPath;
+import org.apache.polaris.core.persistence.resolver.ResolverStatus;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class IcebergCatalogTest {
+
+ @Mock ResolverFactory resolverFactory;
+ @Mock CallContext callContext;
+ @Mock PolarisResolutionManifestCatalogView resolvedEntityView;
+ @Mock CatalogEntity catalogEntity;
+ @Mock PolarisDiagnostics diagnostics;
+ @Mock Resolver resolver;
+
+ @Mock PolarisEntity ns1;
+ @Mock PolarisEntity ns2;
+ @Mock PolarisEntity table3;
+
+ private final PolarisPrincipal principal = PolarisPrincipal.of("test",
Map.of(), Set.of());
+
+ private IcebergCatalog catalog;
+ private ResolvedPolarisEntity rns1 = new ResolvedPolarisEntity(ns1,
List.of(), List.of());
+ private ResolvedPolarisEntity rns2 = new ResolvedPolarisEntity(ns2,
List.of(), List.of());
+ private ResolvedPolarisEntity rt3 = new ResolvedPolarisEntity(table3,
List.of(), List.of());
+
+ @BeforeEach
+ void initMocks() {
+
when(resolvedEntityView.getResolvedCatalogEntity()).thenReturn(catalogEntity);
+ when(resolverFactory.createResolver(any(), any())).thenReturn(resolver);
+ when(resolver.resolveAll()).thenReturn(new
ResolverStatus(ResolverStatus.StatusEnum.SUCCESS));
+
+ rns1 = new ResolvedPolarisEntity(ns1, List.of(), List.of());
+ rns2 = new ResolvedPolarisEntity(ns2, List.of(), List.of());
+ rt3 = new ResolvedPolarisEntity(table3, List.of(), List.of());
+
+ catalog =
+ new IcebergCatalog(
+ diagnostics,
+ resolverFactory,
+ null,
+ callContext,
+ resolvedEntityView,
+ principal,
+ null,
+ null,
+ null,
+ null,
+ null);
+ }
+
+ @Test
+ void testFailedLocationSiblingResolutionException() {
+ when(resolver.resolveAll())
+ .thenReturn(new ResolverStatus(PolarisEntityType.TABLE_LIKE,
"test-name"));
+ Assertions.assertThatThrownBy(() ->
catalog.resolveOptionalPaths(List.of(), "test"))
+ .isInstanceOf(CommitConflictException.class)
+ .hasMessageContaining("Unable to resolve sibling entities to validate
location")
+ .hasMessageContaining("test-name")
+
.hasMessageContaining(ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED.name());
+
+ when(resolver.resolveAll())
+ .thenReturn(
+ new ResolverStatus(
+ new ResolverPath(
+ List.of("ns1", "ns2", "table3"),
PolarisEntityType.TABLE_LIKE, true),
+ 123));
+ Assertions.assertThatThrownBy(() ->
catalog.resolveOptionalPaths(List.of(), "test"))
+ .isInstanceOf(CommitConflictException.class)
+ .hasMessageContaining("Unable to resolve sibling entities to validate
location")
+ .hasMessageContaining("ns1.ns2.table3")
+ .hasMessageContaining("failed index: 123")
+
.hasMessageContaining(ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED.name());
+ }
+
+ @Test
+ void testLocationSiblingResolution() {
+ ResolvedPathKey k1 = ResolvedPathKey.ofTableLike(TableIdentifier.of("ns1",
"ns2", "t3"));
+ ResolvedPathKey k2 =
ResolvedPathKey.ofTableLike(TableIdentifier.of("missing"));
+ ResolvedPathKey k3 = ResolvedPathKey.ofNamespace(Namespace.of("ns1",
"ns2"));
+ ResolvedPathKey k4 = ResolvedPathKey.ofNamespace(Namespace.of("ns1",
"ns2", "partial"));
+
+ List<List<ResolvedPolarisEntity>> resolved = new ArrayList<>();
+ resolved.add(List.of(rns1, rns2, rt3));
+ resolved.add(List.of());
+ resolved.add(List.of(rns1, rns2));
+ resolved.add(List.of(rns1, rns2));
+
+ when(resolver.getResolvedPaths()).thenReturn(resolved);
+ List<PolarisEntity> entities = catalog.resolveOptionalPaths(List.of(k1,
k2, k3, k4), "test");
+ assertThat(entities).hasSize(2);
+ assertThat(entities.get(0)).isSameAs(table3);
+ assertThat(entities.get(1)).isSameAs(ns2);
+ }
+}
Review Comment:
Worth adding a case that drives `validateNoLocationOverlap` end-to-end with
a generic-table sibling — would have surfaced the `getPropertiesAsMap` vs
`getInternalPropertiesAsMap` issue.
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java:
##########
@@ -1242,70 +1241,87 @@ private <T extends PolarisEntity & LocationBasedEntity>
void validateNoLocationO
return Namespace.of(newLevels);
})
.toList();
- LOGGER.debug(
- "Resolving {} sibling entities to validate location",
- siblingTables.size() + siblingNamespaces.size());
- PolarisResolutionManifest resolutionManifest =
- new PolarisResolutionManifest(
- diagnostics,
- callContext.getRealmContext(),
- resolverFactory,
- principal,
- parentPath.getFirst().getName());
+ List<ResolvedPathKey> pathsToResolve =
+ new ArrayList<>(siblingTables.size() + siblingNamespaces.size());
siblingTables.forEach(
- tbl ->
- resolutionManifest.addPath(
- new ResolverPath(
- PolarisCatalogHelpers.tableIdentifierToList(tbl),
- PolarisEntityType.TABLE_LIKE)));
+ tbl -> {
+ if (!tbl.name().equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofTableLike(tbl));
+ }
+ });
siblingNamespaces.forEach(
- ns ->
- resolutionManifest.addPath(
- new ResolverPath(Arrays.asList(ns.levels()),
PolarisEntityType.NAMESPACE)));
+ ns -> {
+ if (!ns.level(ns.length() - 1).equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofNamespace(ns));
+ }
+ });
+
+ StorageLocation targetLocation = StorageLocation.of(location);
+ for (PolarisEntity entityToCheck :
+ resolveOptionalPaths(pathsToResolve, parentPath.getFirst().getName()))
{
+ String loc =
+
entityToCheck.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION);
+ if (loc == null) {
+ continue;
+ }
+
+ StorageLocation siblingLocation = StorageLocation.of(loc);
+
+ if (targetLocation.isChildOf(siblingLocation) ||
siblingLocation.isChildOf(targetLocation)) {
+ throw new ForbiddenException(
+ "Unable to create table at location '%s' because it conflicts with
existing table or namespace at "
+ + "location '%s'",
+ targetLocation, siblingLocation);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ List<PolarisEntity> resolveOptionalPaths(List<ResolvedPathKey> keys, String
catalogName) {
+ LOGGER.debug("Resolving {} sibling entities to validate location",
keys.size());
+
+ PolarisResolutionManifest resolutionManifest =
+ new PolarisResolutionManifest(
+ diagnostics, callContext.getRealmContext(), resolverFactory,
principal, catalogName);
+
+ keys.forEach(
+ k -> {
+ resolutionManifest.addPath(new ResolverPath(k, true)); // optional
path
+ });
+
ResolverStatus status = resolutionManifest.resolveAll();
- if (!status.getStatus().equals(ResolverStatus.StatusEnum.SUCCESS)) {
+
+ if (status.getStatus() != ResolverStatus.StatusEnum.SUCCESS) {
String message =
"Unable to resolve sibling entities to validate location - " +
status.getStatus();
if
(status.getStatus().equals(ResolverStatus.StatusEnum.ENTITY_COULD_NOT_BE_RESOLVED))
{
message += ". Could not resolve entity: " +
status.getFailedToResolvedEntityName();
}
- throw new IllegalStateException(message);
+
+ if
(status.getStatus().equals(ResolverStatus.StatusEnum.PATH_COULD_NOT_BE_FULLY_RESOLVED))
{
+ ResolverPath path = status.getFailedToResolvePath();
+ if (path != null) {
+ message += ". path: " + String.join(".", path.entityNames());
Review Comment:
Two sequential `if (status.getStatus().equals(...))` on the same enum —
switch or `else if` reads cleaner.
##########
runtime/service/src/test/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalogTest.java:
##########
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.catalog.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.PolarisDiagnostics;
+import org.apache.polaris.core.auth.PolarisPrincipal;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.entity.CatalogEntity;
+import org.apache.polaris.core.entity.PolarisEntity;
+import org.apache.polaris.core.entity.PolarisEntityType;
+import org.apache.polaris.core.exceptions.CommitConflictException;
+import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
+import
org.apache.polaris.core.persistence.resolver.PolarisResolutionManifestCatalogView;
+import org.apache.polaris.core.persistence.resolver.ResolvedPathKey;
+import org.apache.polaris.core.persistence.resolver.Resolver;
+import org.apache.polaris.core.persistence.resolver.ResolverFactory;
+import org.apache.polaris.core.persistence.resolver.ResolverPath;
+import org.apache.polaris.core.persistence.resolver.ResolverStatus;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+class IcebergCatalogTest {
Review Comment:
Can we rename it to something specific like
`IcebergCatalogSiblingResolutionTest`. The current name IcebergCatalogTest
reads like the canonical test for the whole class and collides with the
`AbstractIcebergCatalogTest` hierarchy.
##########
runtime/service/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java:
##########
@@ -1242,70 +1241,87 @@ private <T extends PolarisEntity & LocationBasedEntity>
void validateNoLocationO
return Namespace.of(newLevels);
})
.toList();
- LOGGER.debug(
- "Resolving {} sibling entities to validate location",
- siblingTables.size() + siblingNamespaces.size());
- PolarisResolutionManifest resolutionManifest =
- new PolarisResolutionManifest(
- diagnostics,
- callContext.getRealmContext(),
- resolverFactory,
- principal,
- parentPath.getFirst().getName());
+ List<ResolvedPathKey> pathsToResolve =
+ new ArrayList<>(siblingTables.size() + siblingNamespaces.size());
siblingTables.forEach(
- tbl ->
- resolutionManifest.addPath(
- new ResolverPath(
- PolarisCatalogHelpers.tableIdentifierToList(tbl),
- PolarisEntityType.TABLE_LIKE)));
+ tbl -> {
+ if (!tbl.name().equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofTableLike(tbl));
+ }
+ });
siblingNamespaces.forEach(
- ns ->
- resolutionManifest.addPath(
- new ResolverPath(Arrays.asList(ns.levels()),
PolarisEntityType.NAMESPACE)));
+ ns -> {
+ if (!ns.level(ns.length() - 1).equals(name)) {
+ pathsToResolve.add(ResolvedPathKey.ofNamespace(ns));
+ }
+ });
+
+ StorageLocation targetLocation = StorageLocation.of(location);
+ for (PolarisEntity entityToCheck :
+ resolveOptionalPaths(pathsToResolve, parentPath.getFirst().getName()))
{
+ String loc =
+
entityToCheck.getPropertiesAsMap().get(PolarisEntityConstants.ENTITY_BASE_LOCATION);
Review Comment:
Regression for generic tables: `GenericTableEntity.getBaseLocation()` reads
from `getInternalPropertiesAsMap()`, not `getPropertiesAsMap()`. So this lookup
returns null for any generic-table sibling, the `continue` below skips it, and
overlap is silently no longer enforced for generic tables. The pre-PR code had
a `subType == GENERIC_TABLE` branch that handled this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
