Ali Alsuliman has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2245
Change subject: [ASTERIXDB-2203][IDX] Eliminate sort operator for secondary
primary index - user model changes: no - storage format changes: no - interface
changes: no
......................................................................
[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
- user model changes: no
- storage format changes: no
- interface changes: no
details:
Unit test case to compare the job spec for a normal secondary BTree vs
secondary primary index creation and verify that sort operator is removed
for the latter.
Change-Id: Ifc4f90928250aafd90d965028beb07561222086c
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
A
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/ddl/SecondaryBTreeOperationsHelperTest.java
2 files changed, 148 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/45/2245/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
index e24ef2d..6cf1940 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
@@ -21,6 +21,7 @@
import java.io.InputStream;
import java.util.Arrays;
import java.util.LinkedHashSet;
+import java.util.Map;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.app.active.ActiveNotificationHandler;
@@ -59,6 +60,42 @@
}
/**
+ * Creates a dataset with multiple fields
+ * @param dataset The name of the dataset
+ * @param fields The fields of the dataset
+ * @param PKName The primary key field name
+ * @throws Exception
+ */
+ public static void createDataset(String dataset, Map<String, String>
fields, String PKName) throws Exception {
+ StringBuilder stringBuilder = new StringBuilder("");
+ fields.forEach((fName, fType) ->
stringBuilder.append(fName).append(":").append(fType).append(","));
+ stringBuilder.deleteCharAt(stringBuilder.length() - 1);
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE TYPE dsType AS {" +
stringBuilder + "};", OUTPUT_FORMAT);
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE DATASET " + dataset +
"(dsType) PRIMARY KEY " + PKName + ";", OUTPUT_FORMAT);
+ }
+
+ /**
+ * Creates a secondary primary index
+ * @param dataset the name of the dataset
+ * @param indexName the name of the index
+ * @throws Exception
+ */
+ public static void createPrimaryIndex(String dataset, String indexName)
throws Exception {
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE PRIMARY INDEX " +
indexName + " ON " + dataset + ";", OUTPUT_FORMAT);
+ }
+
+ /**
+ * Creates a secondary BTree index
+ * @param dataset the name of the dataset
+ * @param indexName the name of the index
+ * @param SKName the name of the field
+ * @throws Exception
+ */
+ public static void createSecondaryBTreeIndex(String dataset, String
indexName, String SKName) throws Exception {
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE INDEX " + indexName + "
ON " + dataset + "(" + SKName + ");", OUTPUT_FORMAT);
+ }
+
+ /**
* Upserts {@code count} ids into {@code dataset}
*
* @param dataset
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/ddl/SecondaryBTreeOperationsHelperTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/ddl/SecondaryBTreeOperationsHelperTest.java
new file mode 100644
index 0000000..36cee31
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/ddl/SecondaryBTreeOperationsHelperTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.ddl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.utils.IndexUtil;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.sort.AbstractSorterOperatorDescriptor;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class SecondaryBTreeOperationsHelperTest {
+ protected static final String TEST_CONFIG_FILE_NAME =
"src/main/resources/cc.conf";
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new
AsterixHyracksIntegrationUtil();
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
TEST_CONFIG_FILE_NAME);
+ integrationUtil.init(true, TEST_CONFIG_FILE_NAME);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ /**
+ * Tests eliminating the sort operator from the job spec when the index to
be created is secondary primary index
+ *
+ * @throws Exception
+ */
+ @Test
+ public void createPrimaryIndex() throws Exception {
+ ICcApplicationContext appCtx =
+ (ICcApplicationContext)
integrationUtil.getClusterControllerService().getApplicationContext();
+ final MetadataProvider metadataProvider = new MetadataProvider(appCtx,
null);
+ MetadataTransactionContext mdTxn =
MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxn);
+ try {
+ final String datasetName = "DS";
+ final String primaryIndexName = "PIxd";
+ final String secondaryIndexName = "SIdx";
+ final String PKFieldName = "id";
+ final String SKFieldName = "name";
+ final Map<String, String> fields = new HashMap<>(2);
+ fields.put(PKFieldName, "int");
+ fields.put(SKFieldName, "string");
+
+ // create the dataset
+ TestDataUtil.createDataset(datasetName, fields, PKFieldName);
+ final Dataset dataset = metadataProvider.findDataset(null,
datasetName);
+ Assert.assertNotNull(dataset);
+
+ Index index;
+ JobSpecification jobSpecification;
+
+ // create a secondary primary index
+ TestDataUtil.createPrimaryIndex(datasetName, primaryIndexName);
+ index = metadataProvider
+ .getIndex(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME,
datasetName, primaryIndexName);
+ Assert.assertNotNull(index);
+ jobSpecification =
IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider);
+
jobSpecification.getOperatorMap().values().forEach(iOperatorDescriptor -> {
+ Assert.assertFalse(iOperatorDescriptor instanceof
AbstractSorterOperatorDescriptor);
+ });
+
+ // create a normal BTree index
+ TestDataUtil.createSecondaryBTreeIndex(datasetName,
secondaryIndexName, SKFieldName);
+ index = metadataProvider
+ .getIndex(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME,
datasetName, secondaryIndexName);
+ Assert.assertNotNull(index);
+ jobSpecification =
IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider);
+ final long count =
jobSpecification.getOperatorMap().values().stream()
+ .filter(op -> op instanceof
AbstractSorterOperatorDescriptor).count();
+ Assert.assertTrue(count != 0);
+ } finally {
+ MetadataManager.INSTANCE.commitTransaction(mdTxn);
+ metadataProvider.getLocks().unlock();
+ }
+ }
+}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2245
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ifc4f90928250aafd90d965028beb07561222086c
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ali Alsuliman <[email protected]>
