>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20866?usp=email )
Change subject: [NO ISSUE][EXT]: ensure no dependent collections on catalog drop
......................................................................
[NO ISSUE][EXT]: ensure no dependent collections on catalog drop
Ext-ref: MB-70463
Change-Id: I52bafb87fbe5fb856a6220ee11b2bf985bc3b70c
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/catalog-already-exists/test.001.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.000.ddl.sqlpp
A
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.001.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
7 files changed, 144 insertions(+), 4 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/66/20866/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
index 8c33407..39382bd 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/handlers/CatalogStatementHandler.java
@@ -19,12 +19,16 @@
package org.apache.asterix.app.translator.handlers;
import static org.apache.asterix.app.translator.QueryTranslator.abort;
+import static
org.apache.asterix.external.util.iceberg.IcebergConstants.ICEBERG_CATALOG_NAME;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.common.api.IMetadataLockManager;
import org.apache.asterix.common.config.CatalogConfig;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.metadata.IMetadataLockUtil;
@@ -42,7 +46,9 @@
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Catalog;
+import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.EntityDetails;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.IcebergCatalog;
import org.apache.asterix.metadata.entities.IcebergCatalogDetails;
import org.apache.asterix.metadata.utils.Creator;
@@ -160,10 +166,14 @@
}
}
- // if we have any collections on catalog without cascade passed,
we fail the drop, so check one only
- // otherwise, list all so we drop them all
- // TODO(htowaileb): list collections on catalog to drop
- boolean firstOnly = !statement.isCascade();
+ List<Dataset> collections =
getCollectionsCreatedOnToCatalog(mdTxnCtx, catalogName);
+ if (!collections.isEmpty()) {
+ SourceLocation srcLoc = statement.getSourceLocation();
+ String names =
collections.stream().map(Dataset::getDatasetName)
+ .collect(java.util.stream.Collectors.joining(", ",
"[", "]"));
+ throw new
CompilationException(ErrorCode.CANNOT_DROP_CATALOG_HAS_COLLECTIONS, srcLoc,
catalogName,
+ names);
+ }
beforeDropTxnCommit(metadataProvider, creator,
EntityDetails.newCatalog(catalogName));
MetadataManager.INSTANCE.dropCatalog(mdTxnCtx, catalogName);
@@ -175,6 +185,27 @@
}
}
+ private List<Dataset>
getCollectionsCreatedOnToCatalog(MetadataTransactionContext mdTxnCtx, String
catalogName)
+ throws AlgebricksException {
+ List<Dataset> collections =
MetadataManager.INSTANCE.getAllDatasets(mdTxnCtx);
+ List<Dataset> attachedToCatalog = new ArrayList<>();
+ for (Dataset collection : collections) {
+ if (collection.getDatasetType() !=
DatasetConfig.DatasetType.EXTERNAL) {
+ continue;
+ }
+ ExternalDatasetDetails details = (ExternalDatasetDetails)
collection.getDatasetDetails();
+ if (!details.getProperties().containsKey(ICEBERG_CATALOG_NAME)) {
+ continue;
+ }
+ String collectionCatalogName =
details.getProperties().get(ICEBERG_CATALOG_NAME);
+ if (!catalogName.equalsIgnoreCase(collectionCatalogName)) {
+ continue;
+ }
+ attachedToCatalog.add(collection);
+ }
+ return attachedToCatalog;
+ }
+
private boolean isCompileOnly() {
return !sessionConfig.isExecuteQuery();
}
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/catalog-already-exists/test.001.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/catalog-already-exists/test.001.ddl.sqlpp
new file mode 100644
index 0000000..75d7436
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/catalog-already-exists/test.001.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP CATALOG myGlueCatalog;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.000.ddl.sqlpp
new file mode 100644
index 0000000..7532b5e
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.000.ddl.sqlpp
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+CREATE CATALOG myGlueCatalog
+TYPE Iceberg
+SOURCE AWS_GLUE
+WITH {
+ "warehouse": "s3://cbas-iceberg-playground/my-glue-catalog/warehouse/",
+ "accessKeyId" : "myAccessKeyId",
+ "secretAccessKey" : "mySecretAccessKey",
+ "region" : "us-west-2"
+};
+
+CREATE TYPE test AS OPEN {};
+CREATE EXTERNAL COLLECTION test(test) USING S3
+(
+ ("table-format"="iceberg"),
+ ("namespace"="glue_namespace"),
+ ("tableName"="users"),
+ ("accessKeyId"="fakeAccessKeyId"),
+ ("secretAccessKey"="fakeSecretAccessKey"),
+ ("region"="eu-central-1"),
+ ("catalogName"="myGlueCatalog"),
+ ("format"="parquet"),
+ ("decimal-to-double"="true")
+
+);
+
+CREATE EXTERNAL COLLECTION test2(test) USING S3
+(
+ ("table-format"="iceberg"),
+ ("namespace"="glue_namespace"),
+ ("tableName"="users"),
+ ("accessKeyId"="fakeAccessKeyId"),
+ ("secretAccessKey"="fakeSecretAccessKey"),
+ ("region"="eu-central-1"),
+ ("catalogName"="myGlueCatalog"),
+ ("format"="parquet"),
+ ("decimal-to-double"="true")
+
+);
+
+DROP CATALOG myGlueCatalog;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.001.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.001.ddl.sqlpp
new file mode 100644
index 0000000..6ee0d09
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/iceberg/catalog/negative/drop-has-dependent-collections/test.001.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP COLLECTION test IF EXISTS;
+DROP COLLECTION test2 IF EXISTS;
+DROP CATALOG myGlueCatalog IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
index e314d27..d61ad43 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_iceberg.xml
@@ -59,5 +59,11 @@
<expected-error>ASX1227: Cannot find catalog with name
myDoesNotExistGlueCatalog</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="iceberg/catalog/negative">
+ <compilation-unit name="drop-has-dependent-collections">
+ <output-dir compare="Text">drop-has-dependent-collections</output-dir>
+ <expected-error>ASX1243: Cannot drop catalog 'myGlueCatalog' because
it has the following dependent collections: '[test, test2]'</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index dd8d358..9a86583 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -348,6 +348,7 @@
NO_VALID_CREDENTIALS_PROVIDED_FOR_BIGLAKE_METASTORE_CATALOG(1240),
INVALID_FRAME_BASED_MEMORY_BUDGET(1241),
COLLECTION_IS_NOT_AN_ICEBERG_TABLE_COLLECTION(1242),
+ CANNOT_DROP_CATALOG_HAS_COLLECTIONS(1243),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index fe1e4ab..19769e8 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -350,6 +350,7 @@
1240 = No valid credentials provided to access Biglake Metastore catalog.
1241 = Invalid `%1$s` "%2$s" for frame size=%3$s. value should be >= %4$s *
frame size: `%1$s` "%5$s" in %6$s
1242 = Collection '%1$s' is not an Iceberg table external collection.
+1243 = Cannot drop catalog '%1$s' because it has the following dependent
collections: '%2$s'.
# Feed Errors
3001 = Illegal state.
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20866?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I52bafb87fbe5fb856a6220ee11b2bf985bc3b70c
Gerrit-Change-Number: 20866
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
