Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1600
Change subject: ASTERIXDB-1841: Fix Inverted Index Drop Operator
......................................................................
ASTERIXDB-1841: Fix Inverted Index Drop Operator
Change-Id: I2c0f52770fd791ab9293c661de810c0b3b9ba8fb
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
A
asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
A
asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
A
asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
A
asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
A
asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
M asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
A
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
A
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
15 files changed, 313 insertions(+), 107 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/00/1600/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index e64cf14..c4868fc 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -1052,7 +1052,7 @@
bActiveTxn = true;
metadataProvider.setMetadataTxnContext(mdTxnCtx);
try {
- JobSpecification jobSpec =
IndexUtil.buildDropSecondaryIndexJobSpec(index, metadataProvider, ds);
+ JobSpecification jobSpec =
IndexUtil.buildDropIndexJobSpec(index, metadataProvider, ds);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
bActiveTxn = false;
JobUtils.runJob(hcc, jobSpec, true);
@@ -1201,7 +1201,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
for (int k = 0; k < indexes.size(); k++) {
if (indexes.get(k).isSecondaryIndex()) {
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k),
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k),
metadataProvider, datasets.get(j)));
}
}
@@ -1217,7 +1217,7 @@
jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider,
datasets.get(j)));
} else {
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(k),
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(k),
metadataProvider, datasets.get(j)));
}
}
@@ -1390,7 +1390,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (indexes.get(j).isSecondaryIndex()) {
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(j),
metadataProvider, ds));
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j),
metadataProvider, ds));
}
}
Index primaryIndex =
@@ -1429,7 +1429,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(indexes.get(j),
metadataProvider, ds));
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j),
metadataProvider, ds));
} else {
jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(metadataProvider,
ds));
}
@@ -1519,7 +1519,7 @@
}
}
// #. prepare a job to drop the index in NC.
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index,
metadataProvider, ds));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index,
metadataProvider, ds));
// #. mark PendingDropOp on the existing index
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName,
datasetName, indexName);
@@ -1560,7 +1560,7 @@
throw new AlgebricksException("Dropping a dataset's files
index is not allowed.");
}
// #. prepare a job to drop the index in NC.
-
jobsToExecute.add(IndexUtil.buildDropSecondaryIndexJobSpec(index,
metadataProvider, ds));
+ jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(index,
metadataProvider, ds));
List<Index> datasetIndexes =
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
dataverseName, datasetName);
if (datasetIndexes.size() == 2) {
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
new file mode 100644
index 0000000..60deb2d
--- /dev/null
+++
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.2.ddl.aql
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Create different types of secondary indexes then drop them
after restart.
+* Expected Res : Success
+* Date : 20th March 2017
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type Name as open {
+first : string,
+last : string
+}
+
+create type Person as open {
+name : Name
+}
+
+create type TestType as open {
+id : int32,
+name : string,
+locn : point,
+zip : string,
+person : Person
+}
+
+create dataset t1(TestType) primary key id;
+create index rtree_index_point on t1(locn) type rtree;
+create index rtree_index_point_open on t1(open_locn:point?) type rtree
enforced;
+create index keyWD_indx on t1(name) type keyword;
+create index keyWD_indx_open on t1(nickname:string?) type keyword enforced;
+create index secndIndx on t1(zip);
+create index nested on t1(person.name.first);
+create index secndIndx_open on t1(address:string?) enforced;
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.3.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
new file mode 100644
index 0000000..42a361e
--- /dev/null
+++
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.4.ddl.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Create different types of secondary indexes then drop them
after restart.
+* Expected Res : Success
+* Date : 20th March 2017
+*/
+use dataverse test;
+drop index t1.rtree_index_point;
+drop index t1.rtree_index_point_open;
+drop index t1.keyWD_indx;
+drop index t1.keyWD_indx_open;
+drop index t1.secndIndx;
+drop index t1.nested;
+drop index t1.secndIndx_open;
\ No newline at end of file
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++
b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/dml_after_restart/multiple_secondary_indices/multiple_secondary_indices.5.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
diff --git
a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
index a9644b0..07e2d6d 100644
--- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml
@@ -34,6 +34,13 @@
</compilation-unit>
</test-case>
</test-group>
+ <test-group name="dml_after_restart">
+ <test-case FilePath="dml_after_restart">
+ <compilation-unit name="multiple_secondary_indices">
+ <output-dir compare="Text">multiple_secondary_indices</output-dir>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="recover_after_abort">
<test-case FilePath="recover_after_abort">
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index ad75b6b..bf2749d 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -295,7 +295,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (indexes.get(j).isSecondaryIndex()) {
- jobsToExecute.add(IndexUtil.dropJob(indexes.get(j),
metadataProvider, this));
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j),
metadataProvider, this));
}
}
Index primaryIndex =
@@ -334,7 +334,7 @@
MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
datasetName);
for (int j = 0; j < indexes.size(); j++) {
if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
- jobsToExecute.add(IndexUtil.dropJob(indexes.get(j),
metadataProvider, this));
+
jobsToExecute.add(IndexUtil.buildDropIndexJobSpec(indexes.get(j),
metadataProvider, this));
} else {
jobsToExecute.add(DatasetUtil.buildDropFilesIndexJobSpec(metadataProvider,
this));
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
index 701d0d6..a5df766 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/IndexUtil.java
@@ -18,11 +18,8 @@
*/
package org.apache.asterix.metadata.utils;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.collect.Lists;
import org.apache.asterix.common.config.OptimizationConfUtil;
-import org.apache.asterix.common.context.IStorageComponentProvider;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.indexing.ExternalFile;
@@ -30,18 +27,13 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.runtime.utils.RuntimeUtils;
-import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import
org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
-import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import
org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+
+import java.util.List;
public class IndexUtil {
@@ -97,27 +89,24 @@
return empty;
}
- public static JobSpecification dropJob(Index index, MetadataProvider
metadataProvider, Dataset dataset)
+ public static JobSpecification buildDropIndexJobSpec(Index index,
MetadataProvider metadataProvider,
+ Dataset dataset)
throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IStorageComponentProvider storageComponentProvider =
metadataProvider.getStorageComponentProvider();
- Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint =
- metadataProvider.getSplitProviderAndConstraints(dataset,
index.getIndexName());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtil.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
ARecordType recordType =
(ARecordType)
metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
ARecordType metaType = DatasetUtil.getMetaType(metadataProvider,
dataset);
- IIndexDataflowHelperFactory dataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, recordType, metaType, compactionInfo.first,
compactionInfo.second);
- IndexDropOperatorDescriptor btreeDrop =
- new IndexDropOperatorDescriptor(spec,
storageComponentProvider.getStorageManager(),
-
storageComponentProvider.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first,
- dataflowHelperFactory,
storageComponentProvider.getMetadataPageManagerFactory());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
btreeDrop,
- splitsAndConstraint.second);
- spec.addRoot(btreeDrop);
- return spec;
+ ARecordType enforcedType = null;
+ ARecordType enforcedMetaType = null;
+ if (index.isEnforcingKeyFileds()) {
+ Pair<ARecordType, ARecordType> enforcedTypes =
+ TypeUtil.createEnforcedType(recordType, metaType,
Lists.newArrayList(index));
+ enforcedType = enforcedTypes.first;
+ enforcedMetaType = enforcedTypes.second;
+ }
+ SecondaryIndexOperationsHelper secondaryIndexHelper =
+
SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index,
metadataProvider,
+ physicalOptimizationConfig, recordType, metaType,
enforcedType, enforcedMetaType);
+ return secondaryIndexHelper.buildDropJobSpec();
}
public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset
dataset, Index index,
@@ -146,31 +135,6 @@
physicalOptimizationConfig, recType, metaType,
enforcedType, enforcedMetaType);
secondaryIndexHelper.setExternalFiles(files);
return secondaryIndexHelper.buildLoadingJobSpec();
- }
-
- public static JobSpecification buildDropSecondaryIndexJobSpec(Index index,
MetadataProvider metadataProvider,
- Dataset dataset) throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IStorageComponentProvider storageComponentProvider =
metadataProvider.getStorageComponentProvider();
- Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint =
- metadataProvider.getSplitProviderAndConstraints(dataset,
index.getIndexName());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtil.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
- ARecordType recordType =
- (ARecordType)
metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
- ARecordType metaType = DatasetUtil.getMetaType(metadataProvider,
dataset);
- IIndexDataflowHelperFactory dataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, recordType, metaType, compactionInfo.first,
compactionInfo.second);
- // The index drop operation should be persistent regardless of temp
datasets or permanent dataset.
- IndexDropOperatorDescriptor btreeDrop =
- new IndexDropOperatorDescriptor(spec,
storageComponentProvider.getStorageManager(),
-
storageComponentProvider.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first,
- dataflowHelperFactory,
storageComponentProvider.getMetadataPageManagerFactory());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
btreeDrop,
- splitsAndConstraint.second);
- spec.addRoot(btreeDrop);
-
- return spec;
}
public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset
dataset, Index index, ARecordType recType,
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 63368c7..59068c8 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -66,7 +66,7 @@
import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import org.apache.hyracks.storage.common.file.LocalResource;
-public class SecondaryBTreeOperationsHelper extends
SecondaryIndexOperationsHelper {
+public class SecondaryBTreeOperationsHelper extends
SecondaryTreeIndexOperationsHelper {
protected SecondaryBTreeOperationsHelper(Dataset dataset, Index index,
PhysicalOptimizationConfig physOptConf,
IPropertiesProvider propertiesProvider, MetadataProvider
metadataProvider, ARecordType recType,
@@ -234,28 +234,6 @@
@Override
protected int getNumSecondaryKeys() {
return index.getKeyFieldNames().size();
- }
-
- @Override
- public JobSpecification buildCompactJobSpec() throws AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- LSMTreeIndexCompactOperatorDescriptor compactOp;
- IIndexDataflowHelperFactory dataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(metadataProvider,
- index, itemType, metaType, mergePolicyFactory,
mergePolicyFactoryProperties);
- IStorageComponentProvider storageComponentProvider =
metadataProvider.getStorageComponentProvider();
- compactOp =
- new LSMTreeIndexCompactOperatorDescriptor(spec,
storageComponentProvider.getStorageManager(),
-
storageComponentProvider.getIndexLifecycleManagerProvider(),
secondaryFileSplitProvider,
- secondaryTypeTraits, secondaryComparatorFactories,
secondaryBloomFilterKeyFields,
- dataflowHelperFactory,
-
dataset.getModificationCallbackFactory(storageComponentProvider, index, null,
- IndexOperation.FULL_MERGE, null),
-
storageComponentProvider.getMetadataPageManagerFactory());
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
compactOp,
- secondaryPartitionConstraint);
- spec.addRoot(compactOp);
- spec.setConnectorPolicyAssignmentPolicy(new
ConnectorPolicyAssignmentPolicy());
- return spec;
}
@Override
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index d731603..efb3315 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -185,6 +185,8 @@
public abstract JobSpecification buildCompactJobSpec() throws
AlgebricksException;
+ public abstract JobSpecification buildDropJobSpec() throws
AlgebricksException;
+
protected void init() throws AlgebricksException {
payloadSerde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
metaSerde =
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index b86004a..5ab36c1 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -36,6 +36,7 @@
import org.apache.asterix.runtime.utils.RuntimeUtils;
import
org.apache.asterix.transaction.management.resource.LSMInvertedIndexLocalResourceMetadataFactory;
import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
+import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -57,17 +58,22 @@
import
org.apache.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer;
import org.apache.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import
org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import
org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexBulkLoadOperatorDescriptor;
import
org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCompactOperator;
import
org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexCreateOperatorDescriptor;
+import
org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDropOperatorDescriptor;
import
org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider;
import org.apache.hyracks.storage.common.file.LocalResource;
+
+import java.util.Map;
public class SecondaryInvertedIndexOperationsHelper extends
SecondaryIndexOperationsHelper {
@@ -376,4 +382,27 @@
spec.setConnectorPolicyAssignmentPolicy(new
ConnectorPolicyAssignmentPolicy());
return spec;
}
+
+ @Override
+ public JobSpecification buildDropJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IStorageComponentProvider storageComponentProvider =
metadataProvider.getStorageComponentProvider();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint =
+ metadataProvider.getSplitProviderAndConstraints(dataset,
index.getIndexName());
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+ DatasetUtil.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+ ARecordType recordType =
+ (ARecordType)
metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
+ ARecordType metaType = DatasetUtil.getMetaType(metadataProvider,
dataset);
+ IIndexDataflowHelperFactory dataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(metadataProvider,
+ index, recordType, metaType, compactionInfo.first,
compactionInfo.second);
+ LSMInvertedIndexDropOperatorDescriptor invIdxDrop =
+ new LSMInvertedIndexDropOperatorDescriptor(spec,
storageComponentProvider.getStorageManager(),
+
storageComponentProvider.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first,
+ dataflowHelperFactory,
storageComponentProvider.getMetadataPageManagerFactory());
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
invIdxDrop,
+ splitsAndConstraint.second);
+ spec.addRoot(invIdxDrop);
+ return spec;
+ }
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index 460b635..93a88e2 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -72,7 +72,7 @@
import org.apache.hyracks.storage.common.file.LocalResource;
@SuppressWarnings("rawtypes")
-public class SecondaryRTreeOperationsHelper extends
SecondaryIndexOperationsHelper {
+public class SecondaryRTreeOperationsHelper extends
SecondaryTreeIndexOperationsHelper {
protected IPrimitiveValueProviderFactory[] valueProviderFactories;
protected int numNestedSecondaryKeyFields;
@@ -429,26 +429,5 @@
}
}
return fieldPermutation;
- }
-
- @Override
- public JobSpecification buildCompactJobSpec() throws AsterixException,
AlgebricksException {
- JobSpecification spec = RuntimeUtils.createJobSpecification();
- IIndexDataflowHelperFactory indexDataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(
- metadataProvider, index, itemType, metaType,
mergePolicyFactory, mergePolicyFactoryProperties);
- LSMTreeIndexCompactOperatorDescriptor compactOp = new
LSMTreeIndexCompactOperatorDescriptor(spec,
-
metadataProvider.getStorageComponentProvider().getStorageManager(),
-
metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
- secondaryFileSplitProvider, secondaryTypeTraits,
secondaryComparatorFactories,
- secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
-
dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(),
index, null,
- IndexOperation.FULL_MERGE, null),
-
metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
-
-
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
compactOp,
- secondaryPartitionConstraint);
- spec.addRoot(compactOp);
- spec.setConnectorPolicyAssignmentPolicy(new
ConnectorPolicyAssignmentPolicy());
- return spec;
}
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
new file mode 100644
index 0000000..2b025c9
--- /dev/null
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryTreeIndexOperationsHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.utils;
+
+import org.apache.asterix.common.config.IPropertiesProvider;
+import org.apache.asterix.common.context.IStorageComponentProvider;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
+import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import
org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import
org.apache.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import
org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import
org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
+import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import
org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+
+import java.util.Map;
+
+public abstract class SecondaryTreeIndexOperationsHelper extends
SecondaryIndexOperationsHelper {
+
+ protected SecondaryTreeIndexOperationsHelper(Dataset dataset,
+ Index index,
+ PhysicalOptimizationConfig physOptConf,
+ IPropertiesProvider propertiesProvider,
+ MetadataProvider metadataProvider,
+ ARecordType recType, ARecordType metaType,
+ ARecordType enforcedType,
+ ARecordType enforcedMetaType) {
+ super(dataset, index, physOptConf, propertiesProvider,
metadataProvider, recType, metaType, enforcedType,
+ enforcedMetaType);
+ }
+
+ @Override
+ public JobSpecification buildDropJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IStorageComponentProvider storageComponentProvider =
metadataProvider.getStorageComponentProvider();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
splitsAndConstraint =
+ metadataProvider.getSplitProviderAndConstraints(dataset,
index.getIndexName());
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
+ DatasetUtil.getMergePolicyFactory(dataset,
metadataProvider.getMetadataTxnContext());
+ ARecordType recordType =
+ (ARecordType)
metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
+ ARecordType metaType = DatasetUtil.getMetaType(metadataProvider,
dataset);
+ IIndexDataflowHelperFactory dataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(metadataProvider,
+ index, recordType, metaType, compactionInfo.first,
compactionInfo.second);
+ // The index drop operation should be persistent regardless of temp
datasets or permanent dataset.
+ IndexDropOperatorDescriptor btreeDrop =
+ new IndexDropOperatorDescriptor(spec,
storageComponentProvider.getStorageManager(),
+
storageComponentProvider.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first,
+ dataflowHelperFactory,
storageComponentProvider.getMetadataPageManagerFactory());
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
btreeDrop,
+ splitsAndConstraint.second);
+ spec.addRoot(btreeDrop);
+ return spec;
+ }
+
+ @Override
+ public JobSpecification buildCompactJobSpec() throws AlgebricksException {
+ JobSpecification spec = RuntimeUtils.createJobSpecification();
+ IIndexDataflowHelperFactory indexDataflowHelperFactory =
dataset.getIndexDataflowHelperFactory(
+ metadataProvider, index, itemType, metaType,
mergePolicyFactory, mergePolicyFactoryProperties);
+ LSMTreeIndexCompactOperatorDescriptor compactOp = new
LSMTreeIndexCompactOperatorDescriptor(spec,
+
metadataProvider.getStorageComponentProvider().getStorageManager(),
+
metadataProvider.getStorageComponentProvider().getIndexLifecycleManagerProvider(),
+ secondaryFileSplitProvider, secondaryTypeTraits,
secondaryComparatorFactories,
+ secondaryBloomFilterKeyFields, indexDataflowHelperFactory,
+
dataset.getModificationCallbackFactory(metadataProvider.getStorageComponentProvider(),
index, null,
+ IndexOperation.FULL_MERGE, null),
+
metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory());
+
+
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec,
compactOp,
+ secondaryPartitionConstraint);
+ spec.addRoot(compactOp);
+ spec.setConnectorPolicyAssignmentPolicy(new
ConnectorPolicyAssignmentPolicy());
+ return spec;
+ }
+}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
new file mode 100644
index 0000000..fbbd088
--- /dev/null
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDropOperatorDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
+
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
+import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
+import
org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import
org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
+import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import org.apache.hyracks.storage.common.IStorageManager;
+import org.apache.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider;
+
+public class LSMInvertedIndexDropOperatorDescriptor extends
AbstractLSMInvertedIndexOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public LSMInvertedIndexDropOperatorDescriptor(IOperatorDescriptorRegistry
spec, IStorageManager storageManager,
+ IIndexLifecycleManagerProvider lifecycleManagerProvider,
IFileSplitProvider fileSplitProvider,
+ IIndexDataflowHelperFactory dataflowHelperFactory,
IPageManagerFactory pageManagerFactory) {
+ super(spec, 0, 0, null, storageManager, fileSplitProvider,
lifecycleManagerProvider, new ITypeTraits[] {
+ null },
+ new IBinaryComparatorFactory[] { null }, new ITypeTraits[] {
+ null }, new IBinaryComparatorFactory[] { null }, null,
+ dataflowHelperFactory, null, false, false, null,
NoOpLocalResourceFactoryProvider.INSTANCE,
+ NoOpOperationCallbackFactory.INSTANCE,
NoOpOperationCallbackFactory.INSTANCE, pageManagerFactory);
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int
nPartitions) throws HyracksDataException {
+ return new IndexDropOperatorNodePushable(this, ctx, partition);
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1600
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2c0f52770fd791ab9293c661de810c0b3b9ba8fb
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
