This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 8c5e5efba40a7a1cfec21564711acfd7dacd0efc Author: ayush.tripathi <[email protected]> AuthorDate: Thu Dec 12 12:21:38 2024 +0530 [ASTERIXDB-3537][COMP] Support truncate Dataset Statements - user model changes: no - storage format changes: no - interface changes: yes Ext-ref: MB-63067 Change-Id: Ia476bd12832cac4a958de67d75cde03d17efa405 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19206 Integration-Tests: Jenkins <[email protected]> Tested-by: Ali Alsuliman <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19533 Tested-by: Jenkins <[email protected]> --- .../asterix/translator/AbstractLangTranslator.java | 9 ++ .../asterix/app/translator/QueryTranslator.java | 73 +++++++++ .../truncate-dataset.01.ddl.sqlpp | 28 ++++ .../truncate-dataset.02.update.sqlpp | 30 ++++ .../truncate-dataset.03.query.sqlpp | 22 +++ .../truncate-dataset.04.query.sqlpp | 22 +++ .../truncate-dataset.05.query.sqlpp | 22 +++ .../truncate-dataset.06.ddl.sqlpp | 21 +++ .../truncate-dataset.07.query.sqlpp | 22 +++ .../truncate-dataset.08.query.sqlpp | 22 +++ .../truncate-dataset.09.query.sqlpp | 22 +++ .../truncate-dataset.10.ddl.sqlpp | 25 +++ .../truncate-dataset-1/truncate-dataset-1.03.adm | 3 + .../truncate-dataset-1/truncate-dataset-1.04.adm | 3 + .../truncate-dataset-1/truncate-dataset-1.05.adm | 1 + .../truncate-dataset-1/truncate-dataset-1.07.adm | 1 + .../truncate-dataset-1/truncate-dataset-1.08.adm | 0 .../truncate-dataset-1/truncate-dataset-1.09.adm | 0 .../src/test/resources/runtimets/sqlpp_queries.xml | 5 + .../asterix/common/metadata/IMetadataLockUtil.java | 3 + asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf | 3 + .../apache/asterix/lang/common/base/Statement.java | 1 + .../common/statement/TruncateDatasetStatement.java | 71 +++++++++ .../lang/common/visitor/FormatPrintVisitor.java | 9 ++ .../base/AbstractQueryExpressionVisitor.java | 6 + .../lang/common/visitor/base/ILangVisitor.java | 3 + .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 40 +++++ .../asterix/metadata/utils/DatasetPartitions.java | 63 ++++++++ .../apache/asterix/metadata/utils/DatasetUtil.java | 103 +++++++++++++ .../asterix/metadata/utils/MetadataLockUtil.java | 8 + .../metadata/utils/TruncateOperatorDescriptor.java | 168 +++++++++++++++++++++ 31 files changed, 809 insertions(+) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java index 6536c8617f..755d4f67c2 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java @@ -63,6 +63,7 @@ import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.statement.IndexDropStatement; import org.apache.asterix.lang.common.statement.InsertStatement; import org.apache.asterix.lang.common.statement.LoadStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.UpsertStatement; @@ -266,6 +267,14 @@ public abstract class AbstractLangTranslator { } break; + case TRUNCATE: + namespace = getStatementNamespace(((TruncateDatasetStatement) stmt).getNamespace(), activeNamespace); + invalidOperation = isSystemNamespace(namespace); + if (invalidOperation) { + message = formatObjectDdlMessage("truncate", dataset(), namespace, usingDb); + } + break; + case DATASET_DROP: namespace = getStatementNamespace(((DropDatasetStatement) stmt).getNamespace(), activeNamespace); invalidOperation = isSystemNamespace(namespace); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 1b2ec7309e..c4c16abe2b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -166,6 +166,7 @@ import org.apache.asterix.lang.common.statement.SetStatement; import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.statement.StopFeedStatement; import org.apache.asterix.lang.common.statement.SynonymDropStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.UpsertStatement; @@ -421,6 +422,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen case DATAVERSE_DROP: handleDataverseDropStatement(metadataProvider, stmt, hcc, requestParameters); break; + case TRUNCATE: + handleDatasetTruncateStatement(metadataProvider, stmt, requestParameters); + break; case DATASET_DROP: handleDatasetDropStatement(metadataProvider, stmt, hcc, requestParameters); break; @@ -2378,6 +2382,28 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen // may be overridden by product extensions for additional checks after dropping a database/dataverse } + public void handleDatasetTruncateStatement(MetadataProvider metadataProvider, Statement stmt, + IRequestParameters requestParameters) throws Exception { + TruncateDatasetStatement truncateStmt = (TruncateDatasetStatement) stmt; + SourceLocation sourceLoc = truncateStmt.getSourceLocation(); + String datasetName = truncateStmt.getDatasetName().getValue(); + metadataProvider.validateDatabaseObjectName(truncateStmt.getNamespace(), datasetName, sourceLoc); + Namespace stmtActiveNamespace = getActiveNamespace(truncateStmt.getNamespace()); + DataverseName dataverseName = stmtActiveNamespace.getDataverseName(); + String databaseName = stmtActiveNamespace.getDatabaseName(); + if (isCompileOnly()) { + return; + } + lockUtil.truncateDatasetBegin(lockManager, metadataProvider.getLocks(), databaseName, dataverseName, + datasetName); + try { + doTruncateDataset(databaseName, dataverseName, datasetName, metadataProvider, truncateStmt.getIfExists(), + sourceLoc); + } finally { + metadataProvider.getLocks().unlock(); + } + } + public void handleDatasetDropStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception { DropDatasetStatement stmtDelete = (DropDatasetStatement) stmt; @@ -2399,6 +2425,53 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } } + protected void doTruncateDataset(String databaseName, DataverseName dataverseName, String datasetName, + MetadataProvider metadataProvider, boolean ifExists, SourceLocation sourceLoc) throws Exception { + MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); + metadataProvider.setMetadataTxnContext(mdTxnCtx); + Dataset ds = null; + try { + //TODO(DB): also check for database existence? + + // Check if the dataverse exists + Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, databaseName, dataverseName); + if (dv == null) { + if (ifExists) { + if (warningCollector.shouldWarn()) { + warningCollector.warn(Warning.of(sourceLoc, ErrorCode.UNKNOWN_DATAVERSE, MetadataUtil + .dataverseName(databaseName, dataverseName, metadataProvider.isUsingDatabase()))); + } + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); + return; + } else { + throw new CompilationException(ErrorCode.UNKNOWN_DATAVERSE, sourceLoc, MetadataUtil + .dataverseName(databaseName, dataverseName, metadataProvider.isUsingDatabase())); + } + } + ds = metadataProvider.findDataset(databaseName, dataverseName, datasetName, true); + if (ds == null) { + if (ifExists) { + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); + return; + } else { + throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, sourceLoc, datasetName, + MetadataUtil.dataverseName(databaseName, dataverseName, + metadataProvider.isUsingDatabase())); + } + } + validateDatasetState(metadataProvider, ds, sourceLoc); + + DatasetUtil.truncate(metadataProvider, ds); + + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); + } catch (Exception e) { + LOGGER.error("failed to truncate collection {}", + new DatasetFullyQualifiedName(databaseName, dataverseName, datasetName), e); + abort(e, e, mdTxnCtx); + throw e; + } + } + protected boolean doDropDataset(String databaseName, DataverseName dataverseName, String datasetName, MetadataProvider metadataProvider, boolean ifExists, IHyracksClientConnection hcc, IRequestParameters requestParameters, boolean dropCorrespondingNodeGroup, SourceLocation sourceLoc) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.01.ddl.sqlpp new file mode 100644 index 0000000000..18178ed69a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.01.ddl.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse test if exists; +create dataverse test; +use test; + +CREATE COLLECTION orders if not exists primary key (my_id: string); + +CREATE TYPE type1 as {my_id: int}; +CREATE COLLECTION users(type1) primary key my_id; +CREATE INDEX users_first_name ON users(name.first: string) TYPE BTREE; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.02.update.sqlpp new file mode 100644 index 0000000000..54bd0639a1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.02.update.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +INSERT INTO test.orders([ +{"my_id": "a", "f": null }, +{"my_id": "b"}, +{"my_id": "c", "f": {"inner_f": "foo", "inner_f2": {"f3": "bar"} } } +]); + +INSERT INTO test.users([ +{"my_id": 1, "address":{"city": "C1"}, "name":{"first": "F1", "last": "L1"}}, +{"my_id": 2, "address":{"city": "C2"}, "name":{"first": "F2", "last": "L1"}}, +{"my_id": 3, "address":{"city": "C2"}, "name":{"first": "F1", "last": "L2"}} +]); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.03.query.sqlpp new file mode 100644 index 0000000000..6af3c4d006 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.03.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select * +from test.orders +order by my_id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.04.query.sqlpp new file mode 100644 index 0000000000..cb96fc6aed --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.04.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select * +from test.users +order by my_id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.05.query.sqlpp new file mode 100644 index 0000000000..c96ae04906 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.05.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +SET `import-private-functions` `true`; +FROM DUMP_INDEX("test", "users", "users_first_name") AS v +SELECT COUNT(*); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.06.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.06.ddl.sqlpp new file mode 100644 index 0000000000..96970d34d1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.06.ddl.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +TRUNCATE DATASET test.users; +TRUNCATE DATASET test.orders; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.07.query.sqlpp new file mode 100644 index 0000000000..625ed3f66c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.07.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + SET `import-private-functions` `true`; + FROM DUMP_INDEX("test", "users", "users_first_name") AS v + SELECT COUNT(*); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.08.query.sqlpp new file mode 100644 index 0000000000..6af3c4d006 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.08.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select * +from test.orders +order by my_id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.09.query.sqlpp new file mode 100644 index 0000000000..cb96fc6aed --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.09.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select * +from test.users +order by my_id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.10.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.10.ddl.sqlpp new file mode 100644 index 0000000000..3b957ba352 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/truncate-dataset-1/truncate-dataset.10.ddl.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use test; + +drop dataset orders; +drop dataset users; + +drop dataverse test; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.03.adm new file mode 100644 index 0000000000..84966a8e66 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.03.adm @@ -0,0 +1,3 @@ +{ "orders": { "my_id": "a", "f": null } } +{ "orders": { "my_id": "b" } } +{ "orders": { "my_id": "c", "f": { "inner_f": "foo", "inner_f2": { "f3": "bar" } } } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.04.adm new file mode 100644 index 0000000000..92ad35d4b2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.04.adm @@ -0,0 +1,3 @@ +{ "users": { "my_id": 1, "address": { "city": "C1" }, "name": { "first": "F1", "last": "L1" } } } +{ "users": { "my_id": 2, "address": { "city": "C2" }, "name": { "first": "F2", "last": "L1" } } } +{ "users": { "my_id": 3, "address": { "city": "C2" }, "name": { "first": "F1", "last": "L2" } } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.05.adm new file mode 100644 index 0000000000..2b5dd693ee --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.05.adm @@ -0,0 +1 @@ +{ "$1": 3 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.07.adm new file mode 100644 index 0000000000..3ff59f66ac --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.07.adm @@ -0,0 +1 @@ +{ "$1": 0 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.08.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.08.adm new file mode 100644 index 0000000000..e69de29bb2 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/dml/truncate-dataset-1/truncate-dataset-1.09.adm new file mode 100644 index 0000000000..e69de29bb2 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 801e9314a1..ae41564940 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -4532,6 +4532,11 @@ </test-case> </test-group> <test-group name="dml"> + <test-case FilePath="dml"> + <compilation-unit name="truncate-dataset-1"> + <output-dir compare="Clean-JSON">truncate-dataset-1</output-dir> + </compilation-unit> + </test-case> <test-case FilePath="dml"> <compilation-unit name="insert-with-autogenerated-pk_adm-with-sec-primary-index"> <output-dir compare="Text">insert-with-autogenerated-pk_adm-with-sec-primary-index</output-dir> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java index 4cf938e1ff..6ba483970e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/IMetadataLockUtil.java @@ -57,6 +57,9 @@ public interface IMetadataLockUtil { void dropDatasetBegin(IMetadataLockManager lockManager, LockList locks, String database, DataverseName dataverseName, String datasetName) throws AlgebricksException; + void truncateDatasetBegin(IMetadataLockManager lockManager, LockList locks, String database, + DataverseName dataverseName, String datasetName) throws AlgebricksException; + void modifyDatasetBegin(IMetadataLockManager lockManager, LockList locks, String database, DataverseName dataverseName, String datasetName) throws AlgebricksException; diff --git a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf index cbfa92e5a5..be1bb15759 100644 --- a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf +++ b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf @@ -139,6 +139,7 @@ SingleStmnt ::= UseStmnt |InsertStmnt |UpsertStmnt |DeleteStmnt + |TruncateStmnt UseStmnt ::= "USE" DataverseName @@ -250,6 +251,8 @@ DropStmnt ::= "DROP" ("DATAVERSE" DataverseName | "INDEX" DoubleQualifiedName | "FUNCTION" FunctionSignature ) ("IF" "EXISTS")? +TruncateStmnt ::= "TRUNCATE" "DATASET" QualifiedName ("IF" "EXISTS")? + FunctionSignature ::= QualifiedName ( ( "(" ( FunctionParameters? | IntegerLiteral ) ")" ) | ("@" IntegerLiteral) ) LoadStmnt ::= "LOAD" "DATASET" QualifiedName "USING" AdapterName Configuration ("PRE-SORTED")? diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java index 42c7eb6f7e..8e845b8cdf 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java @@ -72,6 +72,7 @@ public interface Statement extends ILangExpression { } enum Kind { + TRUNCATE, DATASET_DECL, DATAVERSE_DECL, DATABASE_DROP, diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TruncateDatasetStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TruncateDatasetStatement.java new file mode 100644 index 0000000000..7367f1d161 --- /dev/null +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TruncateDatasetStatement.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.lang.common.statement; + +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.common.metadata.Namespace; +import org.apache.asterix.lang.common.base.AbstractStatement; +import org.apache.asterix.lang.common.base.Statement; +import org.apache.asterix.lang.common.struct.Identifier; +import org.apache.asterix.lang.common.visitor.base.ILangVisitor; + +public class TruncateDatasetStatement extends AbstractStatement { + private final Namespace namespace; + private final Identifier datasetName; + private final boolean ifExists; + + public TruncateDatasetStatement(Namespace namespace, Identifier datasetName, boolean ifExists) { + this.namespace = namespace; + this.datasetName = datasetName; + this.ifExists = ifExists; + } + + @Override + public Statement.Kind getKind() { + return Kind.TRUNCATE; + } + + public Namespace getNamespace() { + return namespace; + } + + public DataverseName getDataverseName() { + return namespace == null ? null : namespace.getDataverseName(); + } + + public Identifier getDatasetName() { + return datasetName; + } + + public boolean getIfExists() { + return ifExists; + } + + @Override + public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws CompilationException { + return visitor.visit(this, arg); + } + + @Override + public byte getCategory() { + return Category.UPDATE; + } + +} diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java index 6151b024d7..6ff28e77c4 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java @@ -110,6 +110,7 @@ import org.apache.asterix.lang.common.statement.SetStatement; import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.statement.StopFeedStatement; import org.apache.asterix.lang.common.statement.SynonymDropStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.UpdateStatement; @@ -638,6 +639,14 @@ public abstract class FormatPrintVisitor implements ILangVisitor<Void, Integer> return null; } + @Override + public Void visit(TruncateDatasetStatement del, Integer step) throws CompilationException { + out.println(skip(step) + "truncate " + datasetSymbol + + generateFullName(del.getDataverseName(), del.getDatasetName()) + generateIfExists(del.getIfExists()) + + SEMICOLON); + return null; + } + @Override public Void visit(InsertStatement insert, Integer step) throws CompilationException { out.print(skip(step) + "insert into " + datasetSymbol diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java index 0118f4cd72..9c202860eb 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java @@ -67,6 +67,7 @@ import org.apache.asterix.lang.common.statement.SetStatement; import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.statement.StopFeedStatement; import org.apache.asterix.lang.common.statement.SynonymDropStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.UpdateStatement; @@ -105,6 +106,11 @@ public abstract class AbstractQueryExpressionVisitor<R, T> implements ILangVisit return null; } + @Override + public R visit(TruncateDatasetStatement del, T arg) throws CompilationException { + return null; + } + @Override public R visit(DatasetDecl dd, T arg) throws CompilationException { return null; diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java index 1ed9b3cab0..c74911831a 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java @@ -85,6 +85,7 @@ import org.apache.asterix.lang.common.statement.SetStatement; import org.apache.asterix.lang.common.statement.StartFeedStatement; import org.apache.asterix.lang.common.statement.StopFeedStatement; import org.apache.asterix.lang.common.statement.SynonymDropStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.TypeDecl; import org.apache.asterix.lang.common.statement.TypeDropStatement; import org.apache.asterix.lang.common.statement.UpdateStatement; @@ -111,6 +112,8 @@ public interface ILangVisitor<R, T> { R visit(DropDatasetStatement del, T arg) throws CompilationException; + R visit(TruncateDatasetStatement del, T arg) throws CompilationException; + R visit(InsertStatement insert, T arg) throws CompilationException; R visit(DeleteStatement del, T arg) throws CompilationException; diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index 6b3aa54380..31641a7f4d 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -155,6 +155,7 @@ import org.apache.asterix.lang.common.statement.DataverseDropStatement; import org.apache.asterix.lang.common.statement.DeleteStatement; import org.apache.asterix.lang.common.statement.DisconnectFeedStatement; import org.apache.asterix.lang.common.statement.DropDatasetStatement; +import org.apache.asterix.lang.common.statement.TruncateDatasetStatement; import org.apache.asterix.lang.common.statement.ExternalDetailsDecl; import org.apache.asterix.lang.common.statement.FeedDropStatement; import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement; @@ -1015,6 +1016,7 @@ Statement SingleStatement() throws ParseException: | stmt = LoadStatement() | stmt = CopyStatement() | stmt = DropStatement() + | stmt = TruncateStatement() | stmt = SetStatement() | stmt = InsertStatement() | stmt = DeleteStatement() @@ -2367,6 +2369,43 @@ Pair<List<Integer>, List<List<String>>> PrimaryKeyFields() throws ParseException return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields); } } +Statement TruncateStatement() throws ParseException: +{ + Token startToken = null; + Statement stmt = null; +} +{ + <TRUNCATE> { startToken = token; } + ( + stmt = TruncateDatasetStatement(startToken) + ) + { + return stmt; + } +} +TruncateDatasetStatement TruncateDatasetStatement(Token startStmtToken) throws ParseException: +{ + TruncateDatasetStatement stmt = null; +} +{ + Dataset() stmt = TruncateDatasetSpecification(startStmtToken) + { + return stmt; + } +} + +TruncateDatasetStatement TruncateDatasetSpecification(Token startStmtToken) throws ParseException: +{ + Pair<Namespace,Identifier> pairId = null; + boolean ifExists = false; +} +{ + pairId = QualifiedName() ifExists = IfExists() + { + TruncateDatasetStatement stmt = new TruncateDatasetStatement(pairId.first, pairId.second, ifExists); + return addSourceLocation(stmt, startStmtToken); + } +} Statement DropStatement() throws ParseException: { @@ -5969,6 +6008,7 @@ TOKEN [IGNORE_CASE]: | <DISTINCT : "distinct"> | <DIV : "div"> | <DROP : "drop"> + | <TRUNCATE : "truncate"> | <ELEMENT : "element"> | <EXPLAIN : "explain"> | <ELSE : "else"> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetPartitions.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetPartitions.java new file mode 100644 index 0000000000..09e0730099 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetPartitions.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.utils; + +import java.io.Serializable; +import java.util.List; + +import org.apache.asterix.metadata.entities.Dataset; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; + +public class DatasetPartitions implements Serializable { + private static final long serialVersionUID = 1L; + private final Dataset dataset; + private final List<Integer> partitions; + private final IIndexDataflowHelperFactory primaryIndexDataflowHelperFactory; + private final List<IIndexDataflowHelperFactory> secondaryIndexDataflowHelperFactories; + + public DatasetPartitions(Dataset dataset, List<Integer> partitions, + IIndexDataflowHelperFactory primaryIndexDataflowHelperFactory, + List<IIndexDataflowHelperFactory> secondaryIndexDataflowHelperFactories) { + this.dataset = dataset; + this.partitions = partitions; + this.primaryIndexDataflowHelperFactory = primaryIndexDataflowHelperFactory; + this.secondaryIndexDataflowHelperFactories = secondaryIndexDataflowHelperFactories; + } + + public Dataset getDataset() { + return dataset; + } + + public List<Integer> getPartitions() { + return partitions; + } + + public IIndexDataflowHelperFactory getPrimaryIndexDataflowHelperFactory() { + return primaryIndexDataflowHelperFactory; + } + + public List<IIndexDataflowHelperFactory> getSecondaryIndexDataflowHelperFactories() { + return secondaryIndexDataflowHelperFactories; + } + + @Override + public String toString() { + return "{ \"dataset\" : " + dataset + ", \"partitions\" : " + partitions + " }"; + } +} diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index 7df9b477b2..8bcf7a2f69 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@ -24,10 +24,12 @@ import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.EMPTY_TYP import java.io.DataOutput; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.builders.RecordBuilder; @@ -35,9 +37,12 @@ import org.apache.asterix.common.cluster.PartitioningProperties; import org.apache.asterix.common.config.DatasetConfig; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory; +import org.apache.asterix.common.context.DatasetLifecycleManager; +import org.apache.asterix.common.context.DatasetResource; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.context.ITransactionSubsystemProvider; import org.apache.asterix.common.context.TransactionSubsystemProvider; +import org.apache.asterix.common.dataflow.DatasetLocalResource; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; @@ -46,6 +51,7 @@ import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.common.metadata.MetadataConstants; import org.apache.asterix.common.metadata.MetadataUtil; import org.apache.asterix.common.transactions.IRecoveryManager; +import org.apache.asterix.common.utils.JobUtils; import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.formats.base.IDataFormat; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; @@ -71,11 +77,13 @@ import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil; import org.apache.asterix.runtime.operators.LSMPrimaryUpsertOperatorDescriptor; import org.apache.asterix.runtime.utils.RuntimeUtils; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider; +import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory; @@ -99,6 +107,7 @@ import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.build.IndexBuilderFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexCreateOperatorDescriptor; +import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor; import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory; @@ -107,6 +116,8 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMTupleFilterCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor; import org.apache.hyracks.storage.common.IResourceFactory; +import org.apache.hyracks.storage.common.IStorageManager; +import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory; import org.apache.hyracks.util.LogRedactionUtil; import org.apache.logging.log4j.LogManager; @@ -745,6 +756,98 @@ public class DatasetUtil { && dataset.getDatasetFormatInfo().getFormat() == DatasetConfig.DatasetFormat.COLUMN; } + public static void truncate(MetadataProvider metadataProvider, Dataset ds) throws Exception { + if (ds.getDatasetType() == DatasetType.INTERNAL) { + IHyracksClientConnection hcc; + Map<String, List<DatasetPartitions>> nc2Resources = getNodeResources(metadataProvider, ds); + AlgebricksAbsolutePartitionConstraint nodeSet = + new AlgebricksAbsolutePartitionConstraint(nc2Resources.keySet().toArray(new String[0])); + JobSpecification job = new JobSpecification(); + IOperatorDescriptor truncateOp = new TruncateOperatorDescriptor(job, nc2Resources); + AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(job, truncateOp, nodeSet); + hcc = metadataProvider.getApplicationContext().getHcc(); + JobUtils.runJobIfActive(hcc, job, true); + } else { + throw new IllegalArgumentException("Cannot truncate a non-internal dataset."); + } + } + + public static DatasetPartitions getPartitionsAndDataflowHelperFactory(Dataset dataset, + List<DatasetPartitions> ncResources, IIndexDataflowHelperFactory primary, + List<IIndexDataflowHelperFactory> secondaries) { + DatasetPartitions partitionsAndDataflowHelperFactory; + if (ncResources.isEmpty()) { + partitionsAndDataflowHelperFactory = + new DatasetPartitions(dataset, new ArrayList<>(), primary, secondaries); + ncResources.add(partitionsAndDataflowHelperFactory); + } else { + DatasetPartitions last = ncResources.get(ncResources.size() - 1); + if (last.getPrimaryIndexDataflowHelperFactory() == primary) { + partitionsAndDataflowHelperFactory = last; + } else { + partitionsAndDataflowHelperFactory = + new DatasetPartitions(dataset, new ArrayList<>(), primary, secondaries); + ncResources.add(partitionsAndDataflowHelperFactory); + } + } + return partitionsAndDataflowHelperFactory; + } + + public static Map<String, List<DatasetPartitions>> getNodeResources(MetadataProvider metadataProvider, + Dataset dataset) throws AlgebricksException { + Map<String, List<DatasetPartitions>> nc2Resources = new HashMap<>(); + IStorageManager storageManager = metadataProvider.getStorageComponentProvider().getStorageManager(); + + // get secondary indexes + List<Index> secondaryIndexes = + metadataProvider + .getDatasetIndexes(dataset.getDatabaseName(), dataset.getDataverseName(), + dataset.getDatasetName()) + .stream().filter(Index::isSecondaryIndex).collect(Collectors.toList()); + PartitioningProperties partitioningProperties = metadataProvider.getPartitioningProperties(dataset); + IIndexDataflowHelperFactory primary = + new IndexDataflowHelperFactory(storageManager, partitioningProperties.getSplitsProvider()); + List<IIndexDataflowHelperFactory> secondaries = + secondaryIndexes.isEmpty() ? Collections.emptyList() : new ArrayList<>(); + for (Index index : secondaryIndexes) { + PartitioningProperties idxPartitioningProperties = + metadataProvider.getPartitioningProperties(dataset, index.getIndexName()); + secondaries + .add(new IndexDataflowHelperFactory(storageManager, idxPartitioningProperties.getSplitsProvider())); + } + AlgebricksAbsolutePartitionConstraint computeLocations = + (AlgebricksAbsolutePartitionConstraint) partitioningProperties.getConstraints(); + int[][] computeStorageMap = partitioningProperties.getComputeStorageMap(); + for (int j = 0; j < computeLocations.getLocations().length; j++) { + String loc = computeLocations.getLocations()[j]; + DatasetPartitions partitionsAndDataflowHelperFactories = getPartitionsAndDataflowHelperFactory(dataset, + nc2Resources.computeIfAbsent(loc, key -> new ArrayList<>()), primary, secondaries); + List<Integer> dsPartitions = partitionsAndDataflowHelperFactories.getPartitions(); + int[] computeStoragePartitions = computeStorageMap[j]; + for (int storagePartition : computeStoragePartitions) { + dsPartitions.add(storagePartition); + } + } + return nc2Resources; + } + + public static DatasetResource getDatasetResource(DatasetLifecycleManager dslMgr, Integer partition, + IndexDataflowHelper indexHelper) throws HyracksDataException { + LocalResource lr = indexHelper.getResource(); + DatasetLocalResource dslr = (DatasetLocalResource) lr.getResource(); + int datasetId = dslr.getDatasetId(); + DatasetResource dsr = dslMgr.getDatasetLifecycle(datasetId); + // Ensure that no active operations exists + int numActiveOperations = + dslMgr.getOperationTracker(datasetId, partition, lr.getPath()).getNumActiveOperations(); + if (numActiveOperations > 0) { + throw new IllegalStateException("Can't truncate the collection " + dsr.getDatasetInfo().getDatasetID() + + " because the number of active operations = " + numActiveOperations + " in the partition " + + partition); + } + return dsr; + } + public static boolean isDeltaTable(Dataset dataset) { return dataset.getDatasetType() == DatasetType.EXTERNAL && ExternalDataUtils .isDeltaTable(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java index 0b649c4fe9..130cc0f7e4 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java @@ -151,6 +151,14 @@ public class MetadataLockUtil implements IMetadataLockUtil { lockMgr.acquireDatasetWriteLock(locks, database, dataverseName, datasetName); } + @Override + public void truncateDatasetBegin(IMetadataLockManager lockMgr, LockList locks, String database, + DataverseName dataverseName, String datasetName) throws AlgebricksException { + lockMgr.acquireDatabaseReadLock(locks, database); + lockMgr.acquireDataverseReadLock(locks, database, dataverseName); + lockMgr.acquireDatasetWriteLock(locks, database, dataverseName, datasetName); + } + @Override public void dropTypeBegin(IMetadataLockManager lockMgr, LockList locks, String database, DataverseName dataverseName, String typeName) throws AlgebricksException { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TruncateOperatorDescriptor.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TruncateOperatorDescriptor.java new file mode 100644 index 0000000000..24ed7b24c8 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/TruncateOperatorDescriptor.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.utils; + +import static org.apache.asterix.metadata.utils.DatasetUtil.getDatasetResource; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import org.apache.asterix.common.api.INcApplicationContext; +import org.apache.asterix.common.context.DatasetLifecycleManager; +import org.apache.asterix.common.context.DatasetResource; +import org.apache.asterix.common.ioopcallbacks.LSMIOOperationCallback; +import org.apache.asterix.common.storage.DatasetResourceReference; +import org.apache.hyracks.api.application.INCServiceContext; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.dataflow.IOperatorNodePushable; +import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; +import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; +import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable; +import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; +import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters; +import org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentIdGenerator; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor; +import org.apache.hyracks.storage.common.LocalResource; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +@SuppressWarnings("squid:S1181") +public class TruncateOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { + + private static final Logger LOGGER = LogManager.getLogger(); + private static final long serialVersionUID = 1L; + private static final long TIMEOUT = 5; + private static final TimeUnit TIMEOUT_UNIT = TimeUnit.MINUTES; + private final Map<String, List<DatasetPartitions>> allDatasets; + + public TruncateOperatorDescriptor(IOperatorDescriptorRegistry spec, + Map<String, List<DatasetPartitions>> allDatasets) { + super(spec, 0, 0); + this.allDatasets = allDatasets; + } + + @Override + public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, + IRecordDescriptorProvider recordDescProvider, int part, int nPartitions) throws HyracksDataException { + return new AbstractUnaryOutputSourceOperatorNodePushable() { + @Override + public void initialize() throws HyracksDataException { + INcApplicationContext appCtx = + (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext(); + INCServiceContext ctx = appCtx.getServiceContext(); + DatasetLifecycleManager dslMgr = (DatasetLifecycleManager) appCtx.getDatasetLifecycleManager(); + List<DatasetPartitions> nodeDatasets = new ArrayList<>(); + try { + List<DatasetPartitions> datasets = allDatasets.get(ctx.getNodeId()); + for (DatasetPartitions dataset : datasets) { + nodeDatasets.add(dataset); + for (Integer partition : dataset.getPartitions()) { + IIndexDataflowHelper indexDataflowHelper = + dataset.getPrimaryIndexDataflowHelperFactory().create(ctx, partition); + indexDataflowHelper.open(); + try { + ILSMIndex index = (ILSMIndex) indexDataflowHelper.getIndexInstance(); + // Partial Rollback + final LocalResource resource = indexDataflowHelper.getResource(); + final int indexStoragePartition = + DatasetResourceReference.of(resource).getPartitionNum(); + DatasetResource dsr = getDatasetResource(dslMgr, indexStoragePartition, + (IndexDataflowHelper) indexDataflowHelper); + dsr.getDatasetInfo().waitForIO(); + ILSMComponentIdGenerator idGenerator = dslMgr.getComponentIdGenerator( + dsr.getDatasetID(), indexStoragePartition, resource.getPath()); + idGenerator.refresh(); + truncate(ctx, index, dataset.getSecondaryIndexDataflowHelperFactories(), partition, + idGenerator.getId()); + } finally { + indexDataflowHelper.close(); + } + } + LOGGER.info("Truncated collection {} partitions {}", dataset.getDataset(), + dataset.getPartitions()); + } + } catch (Throwable e) { + LOGGER.log(Level.ERROR, "Exception while truncating {}", nodeDatasets, e); + throw HyracksDataException.create(e); + } + } + }; + } + + private static void truncate(INCServiceContext ctx, ILSMIndex primaryIndex, + List<IIndexDataflowHelperFactory> secondaries, Integer partition, ILSMComponentId nextComponentId) + throws HyracksDataException { + Future<Void> truncateFuture = ctx.getControllerService().getExecutor().submit(() -> { + INcApplicationContext appCtx = (INcApplicationContext) ctx.getApplicationContext(); + long flushLsn = appCtx.getTransactionSubsystem().getLogManager().getAppendLSN(); + Map<String, Object> flushMap = new HashMap<>(); + flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn); + flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, nextComponentId); + ILSMIndexAccessor lsmAccessor = primaryIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); + lsmAccessor.getOpContext().setParameters(flushMap); + Predicate<ILSMComponent> predicate = c -> true; + List<IIndexDataflowHelper> openedSecondaries = new ArrayList<>(); + Throwable hde = null; + try { + for (int j = 0; j < secondaries.size(); j++) { + IIndexDataflowHelper sIndexDataflowHelper = secondaries.get(j).create(ctx, partition); + sIndexDataflowHelper.open(); + openedSecondaries.add(sIndexDataflowHelper); + } + // truncate primary + lsmAccessor.deleteComponents(predicate); + // truncate secondaries + for (int j = 0; j < openedSecondaries.size(); j++) { + ILSMIndex sIndex = (ILSMIndex) openedSecondaries.get(j).getIndexInstance(); + ILSMIndexAccessor sLsmAccessor = sIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); + sLsmAccessor.getOpContext().setParameters(flushMap); + sLsmAccessor.deleteComponents(predicate); + } + } catch (Throwable th) { + hde = HyracksDataException.create(th); + } finally { + hde = ResourceReleaseUtils.close(openedSecondaries, hde); + } + if (hde != null) { + throw HyracksDataException.create(hde); + } + return null; + }); + try { + truncateFuture.get(TIMEOUT, TIMEOUT_UNIT); + } catch (Exception e) { + LOGGER.fatal("halting due to a failure to truncate", e); + } + } + +}
