Repository: asterixdb Updated Branches: refs/heads/master 878051979 -> 6e7ff4670
Support rebalancing datasets with indexes. - Remove type arguments from the methods in IndexUtils that generate index operation (e.g., create, load, compact) jobs. Do type extraction inside SecondaryIndexOperationsHelper. Change-Id: I9c0720382440ae44441a8f8847e75649a3822fa2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1790 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> BAD: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/6e7ff467 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/6e7ff467 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/6e7ff467 Branch: refs/heads/master Commit: 6e7ff4670a136e716954e6acd0cdaf0195e6ad75 Parents: 8780519 Author: Yingyi Bu <[email protected]> Authored: Tue May 30 14:55:36 2017 -0700 Committer: Yingyi Bu <[email protected]> Committed: Tue May 30 17:18:17 2017 -0700 ---------------------------------------------------------------------- .../asterix/app/translator/QueryTranslator.java | 66 ++++++-------------- .../org/apache/asterix/utils/RebalanceUtil.java | 29 +++++++-- .../single_dataset_with_index.1.ddl.sqlpp | 49 +++++++++++++++ .../single_dataset_with_index.10.get.http | 20 ++++++ .../single_dataset_with_index.11.query.sqlpp | 25 ++++++++ .../single_dataset_with_index.12.query.sqlpp | 22 +++++++ .../single_dataset_with_index.13.query.sqlpp | 25 ++++++++ .../single_dataset_with_index.2.update.sqlpp | 23 +++++++ .../single_dataset_with_index.3.query.sqlpp | 25 ++++++++ .../single_dataset_with_index.4.post.http | 20 ++++++ .../single_dataset_with_index.5.get.http | 20 ++++++ .../single_dataset_with_index.6.query.sqlpp | 25 ++++++++ .../single_dataset_with_index.7.query.sqlpp | 22 +++++++ .../single_dataset_with_index.8.query.sqlpp | 25 ++++++++ .../single_dataset_with_index.9.post.http | 20 ++++++ .../src/test/resources/runtimets/rebalance.xml | 5 ++ .../single_dataset_with_index.10.adm | 1 + .../single_dataset_with_index.11.adm | 3 + .../single_dataset_with_index.12.adm | 1 + .../single_dataset_with_index.13.adm | 32 ++++++++++ .../single_dataset_with_index.3.adm | 3 + .../single_dataset_with_index.4.adm | 1 + .../single_dataset_with_index.5.adm | 1 + .../single_dataset_with_index.6.adm | 3 + .../single_dataset_with_index.7.adm | 1 + .../single_dataset_with_index.8.adm | 32 ++++++++++ .../single_dataset_with_index.9.adm | 1 + asterixdb/asterix-metadata/pom.xml | 7 ++- .../asterix/metadata/MetadataManager.java | 2 +- .../utils/ExternalIndexingOperations.java | 2 +- .../asterix/metadata/utils/IndexUtil.java | 38 ++++------- .../utils/SecondaryBTreeOperationsHelper.java | 5 +- .../utils/SecondaryIndexOperationsHelper.java | 33 ++++++---- .../SecondaryInvertedIndexOperationsHelper.java | 6 +- .../utils/SecondaryRTreeOperationsHelper.java | 6 +- .../SecondaryTreeIndexOperationsHelper.java | 8 +-- 36 files changed, 497 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 8ea34fd..90a07af 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -31,9 +31,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Properties; import java.util.Random; -import java.util.Map.Entry; import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; @@ -52,12 +52,12 @@ import org.apache.asterix.api.http.server.ResultUtil; import org.apache.asterix.app.result.ResultHandle; import org.apache.asterix.app.result.ResultReader; import org.apache.asterix.common.config.ClusterProperties; -import org.apache.asterix.common.config.ExternalProperties; -import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp; import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.common.config.DatasetConfig.TransactionState; +import org.apache.asterix.common.config.ExternalProperties; +import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.exceptions.ACIDException; @@ -147,23 +147,22 @@ import org.apache.asterix.metadata.utils.IndexUtil; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.asterix.metadata.utils.MetadataUtil; -import org.apache.asterix.metadata.utils.TypeUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.TypeSignature; import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory; import org.apache.asterix.translator.AbstractLangTranslator; -import org.apache.asterix.translator.IStatementExecutor; -import org.apache.asterix.translator.IStatementExecutorContext; -import org.apache.asterix.translator.SessionConfig; -import org.apache.asterix.translator.SessionOutput; -import org.apache.asterix.translator.TypeTranslator; import org.apache.asterix.translator.CompiledStatements.CompiledDeleteStatement; import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement; import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement; import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement; import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement; +import org.apache.asterix.translator.IStatementExecutor; +import org.apache.asterix.translator.IStatementExecutorContext; +import org.apache.asterix.translator.SessionConfig; +import org.apache.asterix.translator.SessionOutput; +import org.apache.asterix.translator.TypeTranslator; import org.apache.asterix.translator.util.ValidateUtil; import org.apache.asterix.utils.DataverseUtil; import org.apache.asterix.utils.FeedOperations; @@ -193,8 +192,6 @@ import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; -import com.google.common.collect.Lists; - /* * Provides functionality for executing a batch of Query statements (queries included) * sequentially. @@ -962,18 +959,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen false, MetadataUtil.PENDING_ADD_OP); MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index); - ARecordType enforcedType = null; - ARecordType enforcedMetaType = null; - if (stmtCreateIndex.isEnforced()) { - Pair<ARecordType, ARecordType> enforcedTypes = - TypeUtil.createEnforcedType(aRecordType, metaRecordType, Lists.newArrayList(index)); - enforcedType = enforcedTypes.first; - enforcedMetaType = enforcedTypes.second; - } - // #. prepare to create the index artifact in NC. - spec = IndexUtil.buildSecondaryIndexCreationJobSpec(ds, index, aRecordType, metaRecordType, enforcedType, - enforcedMetaType, metadataProvider); + spec = IndexUtil.buildSecondaryIndexCreationJobSpec(ds, index, metadataProvider); if (spec == null) { throw new CompilationException("Failed to create job spec for creating index '" + stmtCreateIndex.getDatasetName() + "." + stmtCreateIndex.getIndexName() + "'"); @@ -991,8 +978,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen metadataProvider.setMetadataTxnContext(mdTxnCtx); // #. load data into the index in NC. - spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, aRecordType, metaRecordType, enforcedType, - enforcedMetaType, metadataProvider); + spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; @@ -2228,15 +2214,6 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new AlgebricksException( "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName + "."); } - String itemTypeName = ds.getItemTypeName(); - Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - ds.getItemTypeDataverseName(), itemTypeName); - ARecordType metaRecordType = null; - if (ds.hasMetaPart()) { - metaRecordType = - (ARecordType) MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName()).getDatatype(); - } // Prepare jobs to compact the datatset and its indexes List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName); if (indexes.isEmpty()) { @@ -2246,21 +2223,16 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName); jobsToExecute.add(DatasetUtil.compactDatasetJobSpec(dataverse, datasetName, metadataProvider)); - ARecordType aRecordType = (ARecordType) dt.getDatatype(); - Pair<ARecordType, ARecordType> enforcedTypes = - TypeUtil.createEnforcedType(aRecordType, metaRecordType, indexes); - ARecordType enforcedType = enforcedTypes.first; - ARecordType enforcedMeta = enforcedTypes.second; + if (ds.getDatasetType() == DatasetType.INTERNAL) { - for (int j = 0; j < indexes.size(); j++) { - if (indexes.get(j).isSecondaryIndex()) { - jobsToExecute.add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, indexes.get(j), aRecordType, - metaRecordType, enforcedType, enforcedMeta, metadataProvider)); + for (Index index : indexes) { + if (index.isSecondaryIndex()) { + jobsToExecute + .add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, index, metadataProvider)); } } } else { - prepareCompactJobsForExternalDataset(indexes, ds, jobsToExecute, aRecordType, metaRecordType, - metadataProvider, enforcedType, enforcedMeta); + prepareCompactJobsForExternalDataset(indexes, ds, jobsToExecute, metadataProvider); } MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false; @@ -2281,12 +2253,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } protected void prepareCompactJobsForExternalDataset(List<Index> indexes, Dataset ds, - List<JobSpecification> jobsToExecute, ARecordType aRecordType, ARecordType metaRecordType, - MetadataProvider metadataProvider, ARecordType enforcedType, ARecordType enforcedMeta) + List<JobSpecification> jobsToExecute, MetadataProvider metadataProvider) throws AlgebricksException { for (int j = 0; j < indexes.size(); j++) { - jobsToExecute.add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, indexes.get(j), aRecordType, - metaRecordType, enforcedType, enforcedMeta, metadataProvider)); + jobsToExecute.add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, indexes.get(j), metadataProvider)); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java index 514a837..d715410 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java @@ -168,6 +168,9 @@ public class RebalanceUtil { // Populates the data from the rebalance source to the rebalance target. populateDataToRebalanceTarget(source, target, metadataProvider, hcc); + + // Creates and loads indexes for the rebalance target. + createAndLoadSecondaryIndexesForTarget(source, target, metadataProvider, hcc); } private static void rebalanceSwitch(Dataset source, Dataset target, MetadataProvider metadataProvider, @@ -189,17 +192,14 @@ public class RebalanceUtil { // Drops the source dataset files. dropDatasetFiles(source, metadataProvider, hcc); - // Drops the source dataset's metadata entry. - MetadataManager.INSTANCE.dropDataset(mdTxnCtx, source.getDataverseName(), source.getDatasetName()); + // Updates the dataset entry in the metadata storage + MetadataManager.INSTANCE.updateDataset(mdTxnCtx, target); // Drops the metadata entry of source dataset's node group. String sourceNodeGroup = source.getNodeGroupName(); if (!sourceNodeGroup.equals(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) { MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, sourceNodeGroup); } - - // Adds a new rebalanced dataset entry in the metadata storage - MetadataManager.INSTANCE.addDataset(mdTxnCtx, target); } // Creates the files for the rebalance target dataset. @@ -291,4 +291,23 @@ public class RebalanceUtil { MetadataLockManager.INSTANCE.acquireDatasetWriteLock(locks, dataset.getDataverseName() + "." + dataset.getDatasetName()); } + + // Creates and loads all secondary indexes for the rebalance target dataset. + private static void createAndLoadSecondaryIndexesForTarget(Dataset source, Dataset target, + MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { + for (Index index : metadataProvider.getDatasetIndexes(source.getDataverseName(), source.getDatasetName())) { + if (!index.isSecondaryIndex()) { + continue; + } + // Creates the secondary index. + JobSpecification indexCreationJobSpec = IndexUtil.buildSecondaryIndexCreationJobSpec(target, index, + metadataProvider); + JobUtils.runJob(hcc, indexCreationJobSpec, true); + + // Loads the secondary index. + JobSpecification indexLoadingJobSpec = IndexUtil.buildSecondaryIndexLoadingJobSpec(target, index, + metadataProvider); + JobUtils.runJob(hcc, indexLoadingJobSpec, true); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.1.ddl.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.1.ddl.sqlpp new file mode 100644 index 0000000..93efc34 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.1.ddl.sqlpp @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +drop dataverse tpch if exists; +create dataverse tpch; + +use tpch; + + +create type tpch.LineItemType as + closed { + l_orderkey : bigint, + l_partkey : bigint, + l_suppkey : bigint, + l_linenumber : bigint, + l_quantity : double, + l_extendedprice : double, + l_discount : double, + l_tax : double, + l_returnflag : string, + l_linestatus : string, + l_shipdate : string, + l_commitdate : string, + l_receiptdate : string, + l_shipinstruct : string, + l_shipmode : string, + l_comment : string +} + +create dataset LineItem(LineItemType) primary key l_orderkey,l_linenumber; + +create index lineitem_shipdateIx on LineItem (l_shipdate); +create index lineitem_receiptdateIx on LineItem (l_receiptdate); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.10.get.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.10.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.10.get.http new file mode 100644 index 0000000..360a01f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.10.get.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/connector?dataverseName=tpch&datasetName=LineItem http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.11.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.11.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.11.query.sqlpp new file mode 100644 index 0000000..a82d537 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.11.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +select value l +from LineItem l +where l_shipdate="1994-01-20" +order by l_orderkey, l_linenumber; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.12.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.12.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.12.query.sqlpp new file mode 100644 index 0000000..9b19e48 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.12.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select DatasetName, GroupName, rebalanceCount +from Metadata.`Dataset` +where DatasetName = "LineItem"; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.13.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.13.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.13.query.sqlpp new file mode 100644 index 0000000..d8fda37 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.13.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +explain select value l +from LineItem l +where l_shipdate="1994-01-20" +order by l_orderkey, l_linenumber; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.2.update.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.2.update.sqlpp new file mode 100644 index 0000000..8a59946 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.2.update.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +load dataset LineItem using localfs ((`path`=`asterix_nc1://data/tpch0.001/lineitem.tbl`), + (`format`=`delimited-text`),(`delimiter`=`|`)); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.3.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.3.query.sqlpp new file mode 100644 index 0000000..a82d537 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.3.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +select value l +from LineItem l +where l_shipdate="1994-01-20" +order by l_orderkey, l_linenumber; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.4.post.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.4.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.4.post.http new file mode 100644 index 0000000..737c1fe --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.4.post.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/admin/rebalance?dataverseName=tpch&datasetName=LineItem&nodes="asterix_nc1" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.5.get.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.5.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.5.get.http new file mode 100644 index 0000000..360a01f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.5.get.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/connector?dataverseName=tpch&datasetName=LineItem http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.6.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.6.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.6.query.sqlpp new file mode 100644 index 0000000..a82d537 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.6.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +select value l +from LineItem l +where l_shipdate="1994-01-20" +order by l_orderkey, l_linenumber; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.7.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.7.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.7.query.sqlpp new file mode 100644 index 0000000..9b19e48 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.7.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select DatasetName, GroupName, rebalanceCount +from Metadata.`Dataset` +where DatasetName = "LineItem"; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.8.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.8.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.8.query.sqlpp new file mode 100644 index 0000000..d8fda37 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.8.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use tpch; + +explain select value l +from LineItem l +where l_shipdate="1994-01-20" +order by l_orderkey, l_linenumber; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.9.post.http ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.9.post.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.9.post.http new file mode 100644 index 0000000..1a677fc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/rebalance/single_dataset_with_index/single_dataset_with_index.9.post.http @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/admin/rebalance?dataverseName=tpch&datasetName=LineItem&nodes="asterix_nc1,asterix_nc2" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml b/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml index aa51361..1b3d63c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/rebalance.xml @@ -46,5 +46,10 @@ ERROR_BODY: {"results":"target nodes should not be empty"}</expected-error> <output-dir compare="Text">single_dataset</output-dir> </compilation-unit> </test-case> + <test-case FilePath="rebalance"> + <compilation-unit name="single_dataset_with_index"> + <output-dir compare="Text">single_dataset_with_index</output-dir> + </compilation-unit> + </test-case> </test-group> </test-suite> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm new file mode 100644 index 0000000..44c244c --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm @@ -0,0 +1 @@ +{"temp":false,"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/2/ LineItem_idx_LineItem"}]} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.11.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.11.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.11.adm new file mode 100644 index 0000000..3b97084 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.11.adm @@ -0,0 +1,3 @@ +{ "l_orderkey": 258, "l_partkey": 107, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 8.0, "l_extendedprice": 8056.8, "l_discount": 0.0, "l_tax": 0.07, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-21", "l_receiptdate": "1994-02-09", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ully about the fluffily silent dependencies" } +{ "l_orderkey": 2304, "l_partkey": 200, "l_suppkey": 2, "l_linenumber": 1, "l_quantity": 42.0, "l_extendedprice": 46208.4, "l_discount": 0.0, "l_tax": 0.01, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-04", "l_receiptdate": "1994-02-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "quests are blithely alongside of" } +{ "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41.0, "l_extendedprice": 43958.97, "l_discount": 0.08, "l_tax": 0.03, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.12.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.12.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.12.adm new file mode 100644 index 0000000..3c6303a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.12.adm @@ -0,0 +1 @@ +{ "DatasetName": "LineItem", "GroupName": "LineItem_2", "rebalanceCount": 2 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.adm new file mode 100644 index 0000000..c7b09c6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.13.adm @@ -0,0 +1,32 @@ +distribute result [$$l] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$l]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- SORT_MERGE_EXCHANGE [$$7(ASC), $$8(ASC) ] |PARTITIONED| + select (eq($$l.getField(10), "1994-01-20")) + -- STREAM_SELECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + unnest-map [$$7, $$8, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", FALSE, FALSE, 2, $$15, $$16, 2, $$15, $$16, TRUE, TRUE, TRUE) + -- BTREE_SEARCH |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + order (ASC, $$15) (ASC, $$16) + -- STABLE_SORT [$$15(ASC), $$16(ASC)] |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$16, $$15]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + unnest-map [$$14, $$15, $$16] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", FALSE, FALSE, 1, $$12, 1, $$13, TRUE, TRUE, TRUE) + -- BTREE_SEARCH |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + assign [$$12, $$13] <- ["1994-01-20", "1994-01-20"] + -- ASSIGN |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.3.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.3.adm new file mode 100644 index 0000000..3b97084 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.3.adm @@ -0,0 +1,3 @@ +{ "l_orderkey": 258, "l_partkey": 107, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 8.0, "l_extendedprice": 8056.8, "l_discount": 0.0, "l_tax": 0.07, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-21", "l_receiptdate": "1994-02-09", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ully about the fluffily silent dependencies" } +{ "l_orderkey": 2304, "l_partkey": 200, "l_suppkey": 2, "l_linenumber": 1, "l_quantity": 42.0, "l_extendedprice": 46208.4, "l_discount": 0.0, "l_tax": 0.01, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-04", "l_receiptdate": "1994-02-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "quests are blithely alongside of" } +{ "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41.0, "l_extendedprice": 43958.97, "l_discount": 0.08, "l_tax": 0.03, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.4.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.4.adm new file mode 100644 index 0000000..4b9eb7d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.4.adm @@ -0,0 +1 @@ +{"results":"successful"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm new file mode 100644 index 0000000..4f0990e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm @@ -0,0 +1 @@ +{"temp":false,"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/1/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/1/LineItem_idx_LineItem"}]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.6.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.6.adm new file mode 100644 index 0000000..3b97084 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.6.adm @@ -0,0 +1,3 @@ +{ "l_orderkey": 258, "l_partkey": 107, "l_suppkey": 4, "l_linenumber": 1, "l_quantity": 8.0, "l_extendedprice": 8056.8, "l_discount": 0.0, "l_tax": 0.07, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-21", "l_receiptdate": "1994-02-09", "l_shipinstruct": "NONE", "l_shipmode": "REG AIR", "l_comment": "ully about the fluffily silent dependencies" } +{ "l_orderkey": 2304, "l_partkey": 200, "l_suppkey": 2, "l_linenumber": 1, "l_quantity": 42.0, "l_extendedprice": 46208.4, "l_discount": 0.0, "l_tax": 0.01, "l_returnflag": "A", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-03-04", "l_receiptdate": "1994-02-05", "l_shipinstruct": "COLLECT COD", "l_shipmode": "RAIL", "l_comment": "quests are blithely alongside of" } +{ "l_orderkey": 5988, "l_partkey": 172, "l_suppkey": 1, "l_linenumber": 1, "l_quantity": 41.0, "l_extendedprice": 43958.97, "l_discount": 0.08, "l_tax": 0.03, "l_returnflag": "R", "l_linestatus": "F", "l_shipdate": "1994-01-20", "l_commitdate": "1994-02-06", "l_receiptdate": "1994-02-10", "l_shipinstruct": "COLLECT COD", "l_shipmode": "AIR", "l_comment": "the pending, express reque" } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.7.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.7.adm new file mode 100644 index 0000000..2760db0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.7.adm @@ -0,0 +1 @@ +{ "DatasetName": "LineItem", "GroupName": "LineItem_1", "rebalanceCount": 1 } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.adm new file mode 100644 index 0000000..c7b09c6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.8.adm @@ -0,0 +1,32 @@ +distribute result [$$l] +-- DISTRIBUTE_RESULT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$l]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- SORT_MERGE_EXCHANGE [$$7(ASC), $$8(ASC) ] |PARTITIONED| + select (eq($$l.getField(10), "1994-01-20")) + -- STREAM_SELECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + unnest-map [$$7, $$8, $$l] <- index-search("LineItem", 0, "tpch", "LineItem", FALSE, FALSE, 2, $$15, $$16, 2, $$15, $$16, TRUE, TRUE, TRUE) + -- BTREE_SEARCH |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + order (ASC, $$15) (ASC, $$16) + -- STABLE_SORT [$$15(ASC), $$16(ASC)] |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + project ([$$16, $$15]) + -- STREAM_PROJECT |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + unnest-map [$$14, $$15, $$16] <- index-search("lineitem_shipdateIx", 0, "tpch", "LineItem", FALSE, FALSE, 1, $$12, 1, $$13, TRUE, TRUE, TRUE) + -- BTREE_SEARCH |PARTITIONED| + exchange + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + assign [$$12, $$13] <- ["1994-01-20", "1994-01-20"] + -- ASSIGN |PARTITIONED| + empty-tuple-source + -- EMPTY_TUPLE_SOURCE |PARTITIONED| http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.9.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.9.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.9.adm new file mode 100644 index 0000000..4b9eb7d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.9.adm @@ -0,0 +1 @@ +{"results":"successful"} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/pom.xml b/asterixdb/asterix-metadata/pom.xml index 59131dd..fb66291 100644 --- a/asterixdb/asterix-metadata/pom.xml +++ b/asterixdb/asterix-metadata/pom.xml @@ -1,4 +1,4 @@ -<!-- +<!-- ! Licensed to the Apache Software Foundation (ASF) under one ! or more contributor license agreements.See the NOTICE file ! distributed with this work for additional information @@ -187,5 +187,10 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-collections4</artifactId> + <version>4.1</version> + </dependency> </dependencies> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java index 05bf08e..0cc1958 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java @@ -957,7 +957,7 @@ public class MetadataManager implements IMetadataManager { throw new MetadataException(e); } // reflect the dataset into the cache - ctx.dropDataset(dataset); + ctx.dropDataset(dataset.getDataverseName(), dataset.getDatasetName()); ctx.addDataset(dataset); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java index 9cee9cb..5bb0aa9 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/ExternalIndexingOperations.java @@ -441,7 +441,7 @@ public class ExternalIndexingOperations { for (ExternalFile file : appendedFiles) { files.add(file); } - return IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, null, null, null, null, metadataProvider, files); + return IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, files); } public static JobSpecification buildCommitJob(Dataset ds, List<Index> indexes, MetadataProvider metadataProvider) http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java index 96ca8d7..8be3307 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.metadata.utils; -import java.util.Collections; import java.util.List; import org.apache.asterix.common.config.DatasetConfig; @@ -31,11 +30,9 @@ import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.transaction.management.service.transaction.JobIdFactory; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.job.IJobletEventListenerFactory; @@ -104,57 +101,43 @@ public class IndexUtil { public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException { - ARecordType recordType = - (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); - ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset); - ARecordType enforcedType = null; - ARecordType enforcedMetaType = null; - if (index.isEnforcingKeyFileds()) { - Pair<ARecordType, ARecordType> enforcedTypes = - TypeUtil.createEnforcedType(recordType, metaType, Collections.singletonList(index)); - enforcedType = enforcedTypes.first; - enforcedMetaType = enforcedTypes.second; - } + SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, - physicalOptimizationConfig, recordType, metaType, enforcedType, enforcedMetaType); + physicalOptimizationConfig); return secondaryIndexHelper.buildDropJobSpec(); } - public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index index, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType, + public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, - physicalOptimizationConfig, recType, metaType, enforcedType, enforcedMetaType); + physicalOptimizationConfig); return secondaryIndexHelper.buildCreationJobSpec(); } - public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType, + public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, - physicalOptimizationConfig, recType, metaType, enforcedType, enforcedMetaType); + physicalOptimizationConfig); return secondaryIndexHelper.buildLoadingJobSpec(); } - public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType, + public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, List<ExternalFile> files) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, - physicalOptimizationConfig, recType, metaType, enforcedType, enforcedMetaType); + physicalOptimizationConfig); secondaryIndexHelper.setExternalFiles(files); return secondaryIndexHelper.buildLoadingJobSpec(); } - public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index index, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType, + public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, - physicalOptimizationConfig, recType, metaType, enforcedType, enforcedMetaType); + physicalOptimizationConfig); return secondaryIndexHelper.buildCompactJobSpec(); } @@ -175,4 +158,5 @@ public class IndexUtil { spec.setJobletEventListenerFactory(jobEventListenerFactory); return jobId; } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java index b31bd47..617536f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java @@ -59,9 +59,8 @@ import org.apache.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDe public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperationsHelper { protected SecondaryBTreeOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig physOptConf, - MetadataProvider metadataProvider, ARecordType recType, ARecordType metaType, ARecordType enforcedType, - ARecordType enforcedMetaType) { - super(dataset, index, physOptConf, metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + MetadataProvider metadataProvider) throws AlgebricksException { + super(dataset, index, physOptConf, metadataProvider); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java index b437798..1f2c2fd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java @@ -46,6 +46,7 @@ import org.apache.asterix.runtime.evaluators.functions.AndDescriptor; import org.apache.asterix.runtime.evaluators.functions.CastTypeDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor; import org.apache.asterix.runtime.evaluators.functions.NotDescriptor; +import org.apache.commons.collections4.IteratorUtils; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -114,37 +115,47 @@ public abstract class SecondaryIndexOperationsHelper { // Prevent public construction. Should be created via createIndexCreator(). protected SecondaryIndexOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig physOptConf, - MetadataProvider metadataProvider, ARecordType recType, ARecordType metaType, ARecordType enforcedType, - ARecordType enforcedMetaType) { + MetadataProvider metadataProvider) throws AlgebricksException { this.dataset = dataset; this.index = index; this.physOptConf = physOptConf; this.metadataProvider = metadataProvider; - this.itemType = recType; - this.metaType = metaType; - this.enforcedItemType = enforcedType; - this.enforcedMetaType = enforcedMetaType; + this.itemType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), + dataset.getItemTypeName()); + this.metaType = DatasetUtil.getMetaType(metadataProvider, dataset); + Pair<ARecordType, ARecordType> enforcedTypes = getEnforcedType(index, itemType, metaType); + this.enforcedItemType = enforcedTypes.first; + this.enforcedMetaType = enforcedTypes.second; + } + + private static Pair<ARecordType, ARecordType> getEnforcedType(Index index, ARecordType aRecordType, + ARecordType metaRecordType) throws AlgebricksException { + return index.isEnforcingKeyFileds() + ? TypeUtil.createEnforcedType(aRecordType, metaRecordType, + IteratorUtils.toList(IteratorUtils.singletonIterator(index))) + : new Pair<>(null, null); + } public static SecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index, - MetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType) throws AlgebricksException { + MetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AlgebricksException { + SecondaryIndexOperationsHelper indexOperationsHelper; switch (index.getIndexType()) { case BTREE: indexOperationsHelper = new SecondaryBTreeOperationsHelper(dataset, index, physOptConf, - metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + metadataProvider); break; case RTREE: indexOperationsHelper = new SecondaryRTreeOperationsHelper(dataset, index, physOptConf, - metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + metadataProvider); break; case SINGLE_PARTITION_WORD_INVIX: case SINGLE_PARTITION_NGRAM_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: indexOperationsHelper = new SecondaryInvertedIndexOperationsHelper(dataset, index, physOptConf, - metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + metadataProvider); break; default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java index a38d9b9..97bc00f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java @@ -26,7 +26,6 @@ import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.om.utils.RecordUtil; @@ -78,9 +77,8 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp private int[] secondaryFilterFieldsForNonBulkLoadOps; protected SecondaryInvertedIndexOperationsHelper(Dataset dataset, Index index, - PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider, ARecordType recType, - ARecordType metaType, ARecordType enforcedType, ARecordType enforcedMetaType) { - super(dataset, index, physOptConf, metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + PhysicalOptimizationConfig physOptConf, MetadataProvider metadataProvider) throws AlgebricksException { + super(dataset, index, physOptConf, metadataProvider); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java index d9fba59..8db43d3 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java @@ -32,7 +32,6 @@ import org.apache.asterix.formats.nontagged.TypeTraitProvider; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; @@ -71,9 +70,8 @@ public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperations protected RecordDescriptor secondaryRecDescForPointMBR = null; protected SecondaryRTreeOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig physOptConf, - MetadataProvider metadataProvider, ARecordType recType, ARecordType metaType, ARecordType enforcedType, - ARecordType enforcedMetaType) { - super(dataset, index, physOptConf, metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + MetadataProvider metadataProvider) throws AlgebricksException { + super(dataset, index, physOptConf, metadataProvider); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/6e7ff467/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java index a8b3deb..907192c 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java @@ -23,7 +23,6 @@ import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.runtime.utils.RuntimeUtils; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; @@ -37,17 +36,16 @@ import org.apache.hyracks.storage.am.common.api.IIndexBuilderFactory; import org.apache.hyracks.storage.am.common.build.IndexBuilderFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexCreateOperatorDescriptor; -import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor; import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor; import org.apache.hyracks.storage.common.IResourceFactory; public abstract class SecondaryTreeIndexOperationsHelper extends SecondaryIndexOperationsHelper { protected SecondaryTreeIndexOperationsHelper(Dataset dataset, Index index, PhysicalOptimizationConfig physOptConf, - MetadataProvider metadataProvider, ARecordType recType, ARecordType metaType, ARecordType enforcedType, - ARecordType enforcedMetaType) { - super(dataset, index, physOptConf, metadataProvider, recType, metaType, enforcedType, enforcedMetaType); + MetadataProvider metadataProvider) throws AlgebricksException { + super(dataset, index, physOptConf, metadataProvider); } @Override
