http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java new file mode 100644 index 0000000..61c1dfe --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ResourceUtils.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.utils; + +import org.apache.asterix.app.resource.RequiredCapacityVisitor; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; +import org.apache.hyracks.api.job.resource.ClusterCapacity; +import org.apache.hyracks.api.job.resource.IClusterCapacity; + +public class ResourceUtils { + + private ResourceUtils() { + } + + /** + * Calculates the required cluster capacity from a given query plan, the computation locations, + * the operator memory budgets, and frame size. + * + * @param plan, + * a given query plan. + * @param computationLocations, + * the partitions for computation. + * @param sortFrameLimit, + * the frame limit for one sorter partition. + * @param groupFrameLimit, + * the frame limit for one group-by partition. + * @param joinFrameLimit + * the frame limit for one joiner partition. + * @param frameSize + * the frame size used in query execution. + * @return the required cluster capacity for executing the query. + * @throws AlgebricksException + * if the query plan is malformed. + */ + public static IClusterCapacity getRequiredCompacity(ILogicalPlan plan, + AlgebricksAbsolutePartitionConstraint computationLocations, int sortFrameLimit, int groupFrameLimit, + int joinFrameLimit, int frameSize) + throws AlgebricksException { + // Creates a cluster capacity visitor. + IClusterCapacity clusterCapacity = new ClusterCapacity(); + RequiredCapacityVisitor visitor = new RequiredCapacityVisitor(computationLocations.getLocations().length, + sortFrameLimit, groupFrameLimit, joinFrameLimit, frameSize, clusterCapacity); + + // There could be only one root operator for a top-level query plan. + ILogicalOperator rootOp = plan.getRoots().get(0).getValue(); + rootOp.accept(visitor, null); + return clusterCapacity; + } + +}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java index d34a9cf..d9a0a79 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiLetTest.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.asterix.api.http.server.ConnectorApiServlet; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.MetadataProvider; @@ -36,7 +37,7 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.JSONDeserializerForTypes; +import org.apache.asterix.om.utils.JSONDeserializerForTypes; import org.apache.asterix.test.runtime.SqlppExecutionTest; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.client.NodeControllerInfo; @@ -103,8 +104,7 @@ public class ConnectorApiLetTest { Assert.assertFalse(temp); String primaryKey = actualResponse.get("keys").asText(); Assert.assertEquals("DataverseName,DatasetName", primaryKey); - ARecordType recordType = (ARecordType) JSONDeserializerForTypes - .convertFromJSON(actualResponse.get("type")); + ARecordType recordType = (ARecordType) JSONDeserializerForTypes.convertFromJSON(actualResponse.get("type")); Assert.assertEquals(getMetadataRecordType("Metadata", "Dataset"), recordType); // Checks the correctness of results. @@ -140,9 +140,11 @@ public class ConnectorApiLetTest { // Calls ConnectorAPIServlet.formResponseObject. nodeMap.put("asterix_nc1", mockInfo1); nodeMap.put("asterix_nc2", mockInfo2); - PA.invokeMethod(let, "formResponseObject(" + ObjectNode.class.getName() + ", " + FileSplit.class.getName() - + "[], " + ARecordType.class.getName() + ", " + String.class.getName() + ", boolean, " + Map.class - .getName() + ")", actualResponse, splits, recordType, primaryKey, true, nodeMap); + PA.invokeMethod(let, + "formResponseObject(" + ObjectNode.class.getName() + ", " + FileSplit.class.getName() + "[], " + + ARecordType.class.getName() + ", " + String.class.getName() + ", boolean, " + + Map.class.getName() + ")", + actualResponse, splits, recordType, primaryKey, true, nodeMap); // Constructs expected response. ObjectNode expectedResponse = om.createObjectNode(); expectedResponse.put("temp", true); @@ -166,11 +168,11 @@ public class ConnectorApiLetTest { private ARecordType getMetadataRecordType(String dataverseName, String datasetName) throws Exception { MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction(); // Retrieves file splits of the dataset. - MetadataProvider metadataProvider = new MetadataProvider(null); + MetadataProvider metadataProvider = new MetadataProvider(null, new StorageComponentProvider()); metadataProvider.setMetadataTxnContext(mdTxnCtx); Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); - ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), - dataset.getItemTypeName()); + ARecordType recordType = + (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); // Metadata transaction commits. MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); return recordType; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceLetTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceLetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceLetTest.java deleted file mode 100644 index 7b7c0d2..0000000 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceLetTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.api.http.servlet; - -import org.apache.asterix.api.http.server.QueryServiceServlet; -import org.junit.Assert; -import org.junit.Test; - -public class QueryServiceLetTest { - - @Test - public void testTimeUnitFormatNanos() throws Exception { - Assert.assertEquals("123.456789012s", QueryServiceServlet.TimeUnit.formatNanos(123456789012l)); - Assert.assertEquals("12.345678901s", QueryServiceServlet.TimeUnit.formatNanos(12345678901l)); - Assert.assertEquals("1.234567890s", QueryServiceServlet.TimeUnit.formatNanos(1234567890l)); - Assert.assertEquals("123.456789ms", QueryServiceServlet.TimeUnit.formatNanos(123456789l)); - Assert.assertEquals("12.345678ms", QueryServiceServlet.TimeUnit.formatNanos(12345678l)); - Assert.assertEquals("1.234567ms", QueryServiceServlet.TimeUnit.formatNanos(1234567l)); - Assert.assertEquals("123.456µs", QueryServiceServlet.TimeUnit.formatNanos(123456l)); - Assert.assertEquals("12.345µs", QueryServiceServlet.TimeUnit.formatNanos(12345l)); - Assert.assertEquals("1.234µs", QueryServiceServlet.TimeUnit.formatNanos(1234l)); - Assert.assertEquals("123ns", QueryServiceServlet.TimeUnit.formatNanos(123l)); - Assert.assertEquals("12ns", QueryServiceServlet.TimeUnit.formatNanos(12l)); - Assert.assertEquals("1ns", QueryServiceServlet.TimeUnit.formatNanos(1l)); - Assert.assertEquals("-123.456789012s", QueryServiceServlet.TimeUnit.formatNanos(-123456789012l)); - Assert.assertEquals("120.000000000s", QueryServiceServlet.TimeUnit.formatNanos(120000000000l)); - Assert.assertEquals("-12ns", QueryServiceServlet.TimeUnit.formatNanos(-12l)); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceServletTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceServletTest.java new file mode 100644 index 0000000..e0539ac --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryServiceServletTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.api.http.servlet; + +import org.apache.asterix.api.http.server.QueryServiceServlet; +import org.junit.Assert; +import org.junit.Test; + +public class QueryServiceServletTest { + + @Test + public void testTimeUnitFormatNanos() throws Exception { + Assert.assertEquals("123.456789012s", QueryServiceServlet.TimeUnit.formatNanos(123456789012l)); + Assert.assertEquals("12.345678901s", QueryServiceServlet.TimeUnit.formatNanos(12345678901l)); + Assert.assertEquals("1.234567890s", QueryServiceServlet.TimeUnit.formatNanos(1234567890l)); + Assert.assertEquals("123.456789ms", QueryServiceServlet.TimeUnit.formatNanos(123456789l)); + Assert.assertEquals("12.345678ms", QueryServiceServlet.TimeUnit.formatNanos(12345678l)); + Assert.assertEquals("1.234567ms", QueryServiceServlet.TimeUnit.formatNanos(1234567l)); + Assert.assertEquals("123.456µs", QueryServiceServlet.TimeUnit.formatNanos(123456l)); + Assert.assertEquals("12.345µs", QueryServiceServlet.TimeUnit.formatNanos(12345l)); + Assert.assertEquals("1.234µs", QueryServiceServlet.TimeUnit.formatNanos(1234l)); + Assert.assertEquals("123ns", QueryServiceServlet.TimeUnit.formatNanos(123l)); + Assert.assertEquals("12ns", QueryServiceServlet.TimeUnit.formatNanos(12l)); + Assert.assertEquals("1ns", QueryServiceServlet.TimeUnit.formatNanos(1l)); + Assert.assertEquals("-123.456789012s", QueryServiceServlet.TimeUnit.formatNanos(-123456789012l)); + Assert.assertEquals("120.000000000s", QueryServiceServlet.TimeUnit.formatNanos(120000000000l)); + Assert.assertEquals("-12ns", QueryServiceServlet.TimeUnit.formatNanos(-12l)); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java index b482948..619e7a5 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionApiLetTest.java @@ -32,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.asterix.api.http.server.VersionApiServlet; import org.apache.asterix.common.config.BuildProperties; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.asterix.test.runtime.SqlppExecutionTest; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.http.api.IServletRequest; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java index beadc66..bc18045 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java @@ -19,39 +19,45 @@ package org.apache.asterix.app.bootstrap; import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.logging.Logger; import org.apache.asterix.algebra.operators.physical.CommitRuntime; import org.apache.asterix.app.external.TestLibrarian; import org.apache.asterix.app.nc.NCAppRuntimeContext; +import org.apache.asterix.app.nc.TransactionSubsystem; +import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.common.config.TransactionProperties; -import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider; import org.apache.asterix.common.context.DatasetLifecycleManager; +import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.context.TransactionSubsystemProvider; -import org.apache.asterix.common.dataflow.LSMIndexUtil; import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable; import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor; -import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory; import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType; import org.apache.asterix.common.transactions.IResourceFactory; import org.apache.asterix.common.transactions.ITransactionManager; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; +import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.entities.Dataverse; +import org.apache.asterix.metadata.entities.Index; +import org.apache.asterix.metadata.utils.DatasetUtil; +import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; -import org.apache.asterix.runtime.util.RuntimeComponentsProvider; +import org.apache.asterix.runtime.utils.RuntimeComponentsProvider; import org.apache.asterix.test.runtime.ExecutionTestUtil; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory; -import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerProvider; import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadataFactory; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider; import org.apache.asterix.transaction.management.service.logging.LogReader; -import org.apache.asterix.transaction.management.service.transaction.TransactionSubsystem; import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.runtime.base.IPushRuntime; @@ -71,18 +77,18 @@ import org.apache.hyracks.api.io.ManagedFileSplit; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.job.JobSpecification; import org.apache.hyracks.api.util.HyracksConstants; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorNodePushable; +import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory; -import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelper; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory; import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory; import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider; import org.apache.hyracks.storage.common.file.LocalResource; @@ -95,8 +101,8 @@ public class TestNodeController { protected static final Logger LOGGER = Logger.getLogger(TestNodeController.class.getName()); protected static final String PATH_ACTUAL = "unittest" + File.separator; - protected static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources", "nodetests" }, - File.separator); + protected static final String PATH_BASE = + StringUtils.join(new String[] { "src", "test", "resources", "nodetests" }, File.separator); protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml"; protected static TransactionProperties txnProperties; @@ -109,7 +115,7 @@ public class TestNodeController { public static final int KB32 = 32768; public static final int PARTITION = 0; public static final double BLOOM_FILTER_FALSE_POSITIVE_RATE = 0.01; - public static final TransactionSubsystemProvider TXN_SUBSYSTEM_PROVIDER = new TransactionSubsystemProvider(); + public static final TransactionSubsystemProvider TXN_SUBSYSTEM_PROVIDER = TransactionSubsystemProvider.INSTANCE; // Mutables private JobId jobId; private long jobCounter = 0L; @@ -157,23 +163,24 @@ public class TestNodeController { public LSMInsertDeleteOperatorNodePushable getInsertPipeline(IHyracksTaskContext ctx, Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, - ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields) - throws AlgebricksException, HyracksDataException { + ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields, + int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, + StorageComponentProvider storageComponentProvider) throws AlgebricksException, HyracksDataException { PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, - mergePolicyFactory, mergePolicyProperties, filterFields); + mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, + storageComponentProvider); IndexOperation op = IndexOperation.INSERT; - IModificationOperationCallbackFactory modOpCallbackFactory = new PrimaryIndexModificationOperationCallbackFactory( - getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, op, - ResourceType.LSM_BTREE, true); - LSMTreeInsertDeleteOperatorDescriptor indexOpDesc = getInsertOpratorDesc(primaryIndexInfo, - modOpCallbackFactory); - LSMBTreeDataflowHelperFactory dataflowHelperFactory = getPrimaryIndexDataflowHelperFactory(ctx, - primaryIndexInfo); + IModificationOperationCallbackFactory modOpCallbackFactory = + new PrimaryIndexModificationOperationCallbackFactory(getTxnJobId(), dataset.getDatasetId(), + primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, op, ResourceType.LSM_BTREE, true); + LSMTreeInsertDeleteOperatorDescriptor indexOpDesc = + getInsertOpratorDesc(primaryIndexInfo, modOpCallbackFactory); + IIndexDataflowHelperFactory dataflowHelperFactory = + getPrimaryIndexDataflowHelperFactory(ctx, primaryIndexInfo, storageComponentProvider, dataset); Mockito.when(indexOpDesc.getIndexDataflowHelperFactory()).thenReturn(dataflowHelperFactory); IRecordDescriptorProvider recordDescProvider = primaryIndexInfo.getInsertRecordDescriptorProvider(); - LSMInsertDeleteOperatorNodePushable insertOp = new LSMInsertDeleteOperatorNodePushable( - indexOpDesc, ctx, PARTITION, primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider, - op, true); + LSMInsertDeleteOperatorNodePushable insertOp = new LSMInsertDeleteOperatorNodePushable(indexOpDesc, ctx, + PARTITION, primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider, op, true); CommitRuntime commitOp = new CommitRuntime(ctx, getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, false, true, PARTITION, true); insertOp.setOutputFrameWriter(0, commitOp, primaryIndexInfo.rDesc); @@ -183,21 +190,23 @@ public class TestNodeController { public IPushRuntime getFullScanPipeline(IFrameWriter countOp, IHyracksTaskContext ctx, Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, - NoMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields) - throws HyracksDataException, AlgebricksException { + NoMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] filterFields, + int[] primaryKeyIndexes, List<Integer> primaryKeyIndicators, + StorageComponentProvider storageComponentProvider) throws HyracksDataException, AlgebricksException { IPushRuntime emptyTupleOp = new EmptyTupleSourceRuntimeFactory().createPushRuntime(ctx); JobSpecification spec = new JobSpecification(); PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, - mergePolicyFactory, mergePolicyProperties, filterFields); - LSMBTreeDataflowHelperFactory indexDataflowHelperFactory = getPrimaryIndexDataflowHelperFactory(ctx, - primaryIndexInfo); + mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, + storageComponentProvider); + IIndexDataflowHelperFactory indexDataflowHelperFactory = + getPrimaryIndexDataflowHelperFactory(ctx, primaryIndexInfo, storageComponentProvider, dataset); BTreeSearchOperatorDescriptor searchOpDesc = new BTreeSearchOperatorDescriptor(spec, primaryIndexInfo.rDesc, RuntimeComponentsProvider.RUNTIME_PROVIDER, RuntimeComponentsProvider.RUNTIME_PROVIDER, primaryIndexInfo.fileSplitProvider, primaryIndexInfo.primaryIndexTypeTraits, primaryIndexInfo.primaryIndexComparatorFactories, primaryIndexInfo.primaryIndexBloomFilterKeyFields, primaryIndexInfo.primaryKeyIndexes, primaryIndexInfo.primaryKeyIndexes, true, true, indexDataflowHelperFactory, false, false, null, NoOpOperationCallbackFactory.INSTANCE, filterFields, - filterFields, LSMIndexUtil.getMetadataPageManagerFactory()); + filterFields, storageComponentProvider.getMetadataPageManagerFactory()); BTreeSearchOperatorNodePushable searchOp = new BTreeSearchOperatorNodePushable(searchOpDesc, ctx, 0, primaryIndexInfo.getSearchRecordDescriptorProvider(), /*primaryIndexInfo.primaryKeyIndexes*/null, /*primaryIndexInfo.primaryKeyIndexes*/null, true, true, filterFields, filterFields); @@ -218,10 +227,8 @@ public class TestNodeController { public LSMTreeInsertDeleteOperatorDescriptor getInsertOpratorDesc(PrimaryIndexInfo primaryIndexInfo, IModificationOperationCallbackFactory modOpCallbackFactory) { - LSMTreeInsertDeleteOperatorDescriptor indexOpDesc = Mockito - .mock(LSMTreeInsertDeleteOperatorDescriptor.class); - Mockito.when(indexOpDesc.getLifecycleManagerProvider()) - .thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); + LSMTreeInsertDeleteOperatorDescriptor indexOpDesc = Mockito.mock(LSMTreeInsertDeleteOperatorDescriptor.class); + Mockito.when(indexOpDesc.getLifecycleManagerProvider()).thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); Mockito.when(indexOpDesc.getStorageManager()).thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); Mockito.when(indexOpDesc.getFileSplitProvider()).thenReturn(primaryIndexInfo.fileSplitProvider); Mockito.when(indexOpDesc.getLocalResourceFactoryProvider()) @@ -232,15 +239,14 @@ public class TestNodeController { Mockito.when(indexOpDesc.getTreeIndexBloomFilterKeyFields()) .thenReturn(primaryIndexInfo.primaryIndexBloomFilterKeyFields); Mockito.when(indexOpDesc.getModificationOpCallbackFactory()).thenReturn(modOpCallbackFactory); - Mockito.when(indexOpDesc.getPageManagerFactory()).thenReturn(LSMIndexUtil - .getMetadataPageManagerFactory()); + Mockito.when(indexOpDesc.getPageManagerFactory()) + .thenReturn(primaryIndexInfo.storageComponentProvider.getMetadataPageManagerFactory()); return indexOpDesc; } public TreeIndexCreateOperatorDescriptor getIndexCreateOpDesc(PrimaryIndexInfo primaryIndexInfo) { TreeIndexCreateOperatorDescriptor indexOpDesc = Mockito.mock(TreeIndexCreateOperatorDescriptor.class); - Mockito.when(indexOpDesc.getLifecycleManagerProvider()) - .thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); + Mockito.when(indexOpDesc.getLifecycleManagerProvider()).thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); Mockito.when(indexOpDesc.getStorageManager()).thenReturn(RuntimeComponentsProvider.RUNTIME_PROVIDER); Mockito.when(indexOpDesc.getFileSplitProvider()).thenReturn(primaryIndexInfo.fileSplitProvider); Mockito.when(indexOpDesc.getLocalResourceFactoryProvider()) @@ -250,8 +256,8 @@ public class TestNodeController { .thenReturn(primaryIndexInfo.primaryIndexComparatorFactories); Mockito.when(indexOpDesc.getTreeIndexBloomFilterKeyFields()) .thenReturn(primaryIndexInfo.primaryIndexBloomFilterKeyFields); - Mockito.when(indexOpDesc.getPageManagerFactory()).thenReturn(LSMIndexUtil - .getMetadataPageManagerFactory()); + Mockito.when(indexOpDesc.getPageManagerFactory()) + .thenReturn(primaryIndexInfo.storageComponentProvider.getMetadataPageManagerFactory()); return indexOpDesc; } @@ -261,64 +267,66 @@ public class TestNodeController { return new ConstantFileSplitProvider(new FileSplit[] { fileSplit }); } - public ILocalResourceFactoryProvider getPrimaryIndexLocalResourceMetadataProvider(Dataset dataset, + public ILocalResourceFactoryProvider getPrimaryIndexLocalResourceMetadataProvider( + IStorageComponentProvider storageComponentProvider, Index index, Dataset dataset, ITypeTraits[] primaryIndexTypeTraits, IBinaryComparatorFactory[] primaryIndexComparatorFactories, int[] primaryIndexBloomFilterKeyFields, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) { + IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, + ILSMOperationTrackerFactory opTrackerProvider) throws AlgebricksException { IResourceFactory localResourceMetadata = new LSMBTreeLocalResourceMetadataFactory(primaryIndexTypeTraits, primaryIndexComparatorFactories, primaryIndexBloomFilterKeyFields, true, dataset.getDatasetId(), mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields, - filterFields); - ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider( - localResourceMetadata, LocalResource.LSMBTreeResource); + filterFields, opTrackerProvider, dataset.getIoOperationCallbackFactory(index), + storageComponentProvider.getMetadataPageManagerFactory()); + ILocalResourceFactoryProvider localResourceFactoryProvider = + new PersistentLocalResourceFactoryProvider(localResourceMetadata, LocalResource.LSMBTreeResource); return localResourceFactoryProvider; } - public LSMBTreeDataflowHelper getPrimaryIndexDataflowHelper(IHyracksTaskContext ctx, - PrimaryIndexInfo primaryIndexInfo, TreeIndexCreateOperatorDescriptor indexOpDesc) + public IIndexDataflowHelper getPrimaryIndexDataflowHelper(IHyracksTaskContext ctx, + PrimaryIndexInfo primaryIndexInfo, TreeIndexCreateOperatorDescriptor indexOpDesc, + IStorageComponentProvider storageComponentProvider, Dataset dataset) throws AlgebricksException, HyracksDataException { - LSMBTreeDataflowHelperFactory dataflowHelperFactory = new LSMBTreeDataflowHelperFactory( - new AsterixVirtualBufferCacheProvider(primaryIndexInfo.dataset.getDatasetId()), - primaryIndexInfo.mergePolicyFactory, primaryIndexInfo.mergePolicyProperties, - new PrimaryIndexOperationTrackerProvider(primaryIndexInfo.dataset.getDatasetId()), - RuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE, - BLOOM_FILTER_FALSE_POSITIVE_RATE, true, primaryIndexInfo.filterTypeTraits, - primaryIndexInfo.filterCmpFactories, primaryIndexInfo.btreeFields, primaryIndexInfo.filterFields, true); - IndexDataflowHelper dataflowHelper = dataflowHelperFactory.createIndexDataflowHelper(indexOpDesc, ctx, - PARTITION); - return (LSMBTreeDataflowHelper) dataflowHelper; + return getPrimaryIndexDataflowHelperFactory(ctx, primaryIndexInfo, storageComponentProvider, dataset) + .createIndexDataflowHelper(indexOpDesc, ctx, PARTITION); } - public LSMBTreeDataflowHelperFactory getPrimaryIndexDataflowHelperFactory(IHyracksTaskContext ctx, - PrimaryIndexInfo primaryIndexInfo) throws AlgebricksException { - return new LSMBTreeDataflowHelperFactory( - new AsterixVirtualBufferCacheProvider(primaryIndexInfo.dataset.getDatasetId()), - primaryIndexInfo.mergePolicyFactory, primaryIndexInfo.mergePolicyProperties, - new PrimaryIndexOperationTrackerProvider(primaryIndexInfo.dataset.getDatasetId()), - RuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE, - BLOOM_FILTER_FALSE_POSITIVE_RATE, true, primaryIndexInfo.filterTypeTraits, - primaryIndexInfo.filterCmpFactories, primaryIndexInfo.btreeFields, primaryIndexInfo.filterFields, true); + public IIndexDataflowHelperFactory getPrimaryIndexDataflowHelperFactory(IHyracksTaskContext ctx, + PrimaryIndexInfo primaryIndexInfo, IStorageComponentProvider storageComponentProvider, Dataset dataset) + throws AlgebricksException { + Dataverse dataverse = new Dataverse(dataset.getDataverseName(), NonTaggedDataFormat.class.getName(), + MetadataUtil.PENDING_NO_OP); + Index index = primaryIndexInfo.getIndex(); + MetadataProvider mdProvider = new MetadataProvider(dataverse, storageComponentProvider); + return dataset.getIndexDataflowHelperFactory(mdProvider, index, primaryIndexInfo.recordType, + primaryIndexInfo.metaType, primaryIndexInfo.mergePolicyFactory, + primaryIndexInfo.mergePolicyProperties); } - public LSMBTreeDataflowHelper getPrimaryIndexDataflowHelper(Dataset dataset, IAType[] primaryKeyTypes, + public IIndexDataflowHelper getPrimaryIndexDataflowHelper(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, - Map<String, String> mergePolicyProperties, int[] filterFields) - throws AlgebricksException, HyracksDataException { + Map<String, String> mergePolicyProperties, int[] filterFields, + IStorageComponentProvider storageComponentProvider, int[] primaryKeyIndexes, + List<Integer> primaryKeyIndicators) throws AlgebricksException, HyracksDataException { PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, - mergePolicyFactory, mergePolicyProperties, filterFields); + mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, + storageComponentProvider); TreeIndexCreateOperatorDescriptor indexOpDesc = getIndexCreateOpDesc(primaryIndexInfo); - return getPrimaryIndexDataflowHelper(createTestContext(true), primaryIndexInfo, indexOpDesc); + return getPrimaryIndexDataflowHelper(createTestContext(true), primaryIndexInfo, indexOpDesc, + storageComponentProvider, dataset); } public void createPrimaryIndex(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, - int[] filterFields) throws AlgebricksException, HyracksDataException { + int[] filterFields, IStorageComponentProvider storageComponentProvider, int[] primaryKeyIndexes, + List<Integer> primaryKeyIndicators) throws AlgebricksException, HyracksDataException { PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType, - mergePolicyFactory, mergePolicyProperties, filterFields); + mergePolicyFactory, mergePolicyProperties, filterFields, primaryKeyIndexes, primaryKeyIndicators, + storageComponentProvider); TreeIndexCreateOperatorDescriptor indexOpDesc = getIndexCreateOpDesc(primaryIndexInfo); - LSMBTreeDataflowHelper dataflowHelper = getPrimaryIndexDataflowHelper(createTestContext(true), primaryIndexInfo, - indexOpDesc); + IIndexDataflowHelper dataflowHelper = getPrimaryIndexDataflowHelper(createTestContext(true), primaryIndexInfo, + indexOpDesc, storageComponentProvider, dataset); dataflowHelper.create(); } @@ -331,10 +339,11 @@ public class TestNodeController { } private IBinaryComparatorFactory[] createPrimaryIndexComparatorFactories(IAType[] primaryKeyTypes) { - IBinaryComparatorFactory[] primaryIndexComparatorFactories = new IBinaryComparatorFactory[primaryKeyTypes.length]; + IBinaryComparatorFactory[] primaryIndexComparatorFactories = + new IBinaryComparatorFactory[primaryKeyTypes.length]; for (int j = 0; j < primaryKeyTypes.length; ++j) { - primaryIndexComparatorFactories[j] = BinaryComparatorFactoryProvider.INSTANCE - .getBinaryComparatorFactory(primaryKeyTypes[j], true); + primaryIndexComparatorFactories[j] = + BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(primaryKeyTypes[j], true); } return primaryIndexComparatorFactories; } @@ -344,8 +353,8 @@ public class TestNodeController { int i = 0; ISerializerDeserializer<?>[] primaryIndexSerdes = new ISerializerDeserializer<?>[primaryIndexNumOfTupleFields]; for (; i < primaryKeyTypes.length; i++) { - primaryIndexSerdes[i] = SerializerDeserializerProvider.INSTANCE - .getSerializerDeserializer(primaryKeyTypes[i]); + primaryIndexSerdes[i] = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(primaryKeyTypes[i]); } primaryIndexSerdes[i++] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType); if (metaType != null) { @@ -371,12 +380,11 @@ public class TestNodeController { public IHyracksTaskContext createTestContext(boolean withMessaging) throws HyracksDataException { IHyracksTaskContext ctx = TestUtils.create(KB32); if (withMessaging) { - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx); } ctx = Mockito.spy(ctx); Mockito.when(ctx.getJobletContext()).thenReturn(jobletCtx); - Mockito.when(ctx.getIOManager()) - .thenReturn(ExecutionTestUtil.integrationUtil.ncs[0].getIoManager()); + Mockito.when(ctx.getIOManager()).thenReturn(ExecutionTestUtil.integrationUtil.ncs[0].getIoManager()); return ctx; } @@ -420,10 +428,18 @@ public class TestNodeController { private RecordDescriptor rDesc; private int[] primaryIndexInsertFieldsPermutations; private int[] primaryKeyIndexes; - - public PrimaryIndexInfo(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, ARecordType metaType, - ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, - int[] filterFields) throws AlgebricksException { + private List<List<String>> keyFieldNames; + private List<Integer> keyFieldSourceIndicators; + private List<IAType> keyFieldTypes; + private Index index; + private IStorageComponentProvider storageComponentProvider; + + public PrimaryIndexInfo(Dataset dataset, IAType[] primaryKeyTypes, ARecordType recordType, + ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, + Map<String, String> mergePolicyProperties, int[] filterFields, int[] primaryKeyIndexes, + List<Integer> primaryKeyIndicators, IStorageComponentProvider storageComponentProvider) + throws AlgebricksException { + this.storageComponentProvider = storageComponentProvider; this.dataset = dataset; this.primaryKeyTypes = primaryKeyTypes; this.recordType = recordType; @@ -431,30 +447,44 @@ public class TestNodeController { this.mergePolicyFactory = mergePolicyFactory; this.mergePolicyProperties = mergePolicyProperties; this.filterFields = filterFields; + this.primaryKeyIndexes = primaryKeyIndexes; primaryIndexNumOfTupleFields = primaryKeyTypes.length + (1 + ((metaType == null) ? 0 : 1)); - primaryIndexTypeTraits = createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, primaryKeyTypes, - recordType, metaType); + primaryIndexTypeTraits = + createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, metaType); primaryIndexComparatorFactories = createPrimaryIndexComparatorFactories(primaryKeyTypes); primaryIndexBloomFilterKeyFields = createPrimaryIndexBloomFilterFields(primaryKeyTypes.length); - filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recordType); - filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset, recordType, + filterTypeTraits = DatasetUtil.computeFilterTypeTraits(dataset, recordType); + filterCmpFactories = DatasetUtil.computeFilterBinaryComparatorFactories(dataset, recordType, NonTaggedDataFormat.INSTANCE.getBinaryComparatorFactoryProvider()); - btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset); - localResourceFactoryProvider = getPrimaryIndexLocalResourceMetadataProvider(dataset, primaryIndexTypeTraits, - primaryIndexComparatorFactories, primaryIndexBloomFilterKeyFields, mergePolicyFactory, - mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields, filterFields); + btreeFields = DatasetUtil.createBTreeFieldsWhenThereisAFilter(dataset); fileSplitProvider = getFileSplitProvider(dataset); - primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, - metaType); + primaryIndexSerdes = + createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, metaType); rDesc = new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits); primaryIndexInsertFieldsPermutations = new int[primaryIndexNumOfTupleFields]; for (int i = 0; i < primaryIndexNumOfTupleFields; i++) { primaryIndexInsertFieldsPermutations[i] = i; } - primaryKeyIndexes = new int[primaryKeyTypes.length]; - for (int i = 0; i < primaryKeyIndexes.length; i++) { - primaryKeyIndexes[i] = i; + keyFieldSourceIndicators = primaryKeyIndicators; + keyFieldNames = new ArrayList<>(); + keyFieldTypes = Arrays.asList(primaryKeyTypes); + for (int i = 0; i < keyFieldSourceIndicators.size(); i++) { + Integer indicator = keyFieldSourceIndicators.get(i); + String[] fieldNames = + indicator == Index.RECORD_INDICATOR ? recordType.getFieldNames() : metaType.getFieldNames(); + keyFieldNames.add(Arrays.asList(fieldNames[primaryKeyIndexes[i]])); } + index = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), + IndexType.BTREE, keyFieldNames, keyFieldSourceIndicators, keyFieldTypes, false, true, + MetadataUtil.PENDING_NO_OP); + localResourceFactoryProvider = getPrimaryIndexLocalResourceMetadataProvider(storageComponentProvider, + index, dataset, primaryIndexTypeTraits, primaryIndexComparatorFactories, + primaryIndexBloomFilterKeyFields, mergePolicyFactory, mergePolicyProperties, filterTypeTraits, + filterCmpFactories, btreeFields, filterFields, dataset.getIndexOperationTrackerFactory(index)); + } + + public Index getIndex() { + return index; } public IRecordDescriptorProvider getInsertRecordDescriptorProvider() { @@ -468,8 +498,8 @@ public class TestNodeController { ISerializerDeserializer<?>[] primaryKeySerdes = new ISerializerDeserializer<?>[primaryKeyTypes.length]; for (int i = 0; i < primaryKeyTypes.length; i++) { primaryKeyTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(primaryKeyTypes[i]); - primaryKeySerdes[i] = SerializerDeserializerProvider.INSTANCE - .getSerializerDeserializer(primaryKeyTypes[i]); + primaryKeySerdes[i] = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(primaryKeyTypes[i]); } RecordDescriptor searcgRecDesc = new RecordDescriptor(primaryKeySerdes, primaryKeyTypeTraits); IRecordDescriptorProvider rDescProvider = Mockito.mock(IRecordDescriptorProvider.class); @@ -481,10 +511,10 @@ public class TestNodeController { public RecordDescriptor getSearchOutputDesc(IAType[] keyTypes, ARecordType recordType, ARecordType metaType) { int primaryIndexNumOfTupleFields = keyTypes.length + (1 + ((metaType == null) ? 0 : 1)); - ITypeTraits[] primaryIndexTypeTraits = createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, keyTypes, - recordType, metaType); - ISerializerDeserializer<?>[] primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, - keyTypes, recordType, metaType); + ITypeTraits[] primaryIndexTypeTraits = + createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType); + ISerializerDeserializer<?>[] primaryIndexSerdes = + createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType); return new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java index b305e27..5cd8a63 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java @@ -29,14 +29,15 @@ import java.util.Arrays; import java.util.List; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; -import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.common.config.ClusterProperties; +import org.apache.asterix.common.config.ExternalProperties; import org.apache.asterix.compiler.provider.AqlCompilationProvider; import org.apache.asterix.event.schema.cluster.Cluster; import org.apache.asterix.event.schema.cluster.MasterNode; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.statement.RunStatement; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.asterix.translator.IStatementExecutor; import org.apache.asterix.translator.SessionConfig; import org.junit.Assert; @@ -49,7 +50,7 @@ public class QueryTranslatorTest { @Test public void test() throws Exception { - List<Statement> statements = new ArrayList<Statement>(); + List<Statement> statements = new ArrayList<>(); SessionConfig mockSessionConfig = mock(SessionConfig.class); RunStatement mockRunStatement = mock(RunStatement.class); @@ -70,8 +71,8 @@ public class QueryTranslatorTest { when(mockMasterNode.getClientIp()).thenReturn("127.0.0.1"); IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory().create(statements, mockSessionConfig, - new AqlCompilationProvider()); - List<String> parameters = new ArrayList<String>(); + new AqlCompilationProvider(), new StorageComponentProvider()); + List<String> parameters = new ArrayList<>(); parameters.add("examples/pregelix-example-jar-with-dependencies.jar"); parameters.add("org.apache.pregelix.example.PageRankVertex"); parameters.add("-ip 10.0.2.15 -port 3199"); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java index a253ac0..a3d6102 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ConnectorDescriptorWithMessagingTest.java @@ -45,7 +45,7 @@ import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeser import org.apache.hyracks.dataflow.common.data.marshalling.Integer64SerializerDeserializer; import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer; import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningWithMessageConnectorDescriptor; import org.apache.hyracks.dataflow.std.connectors.PartitionWithMessageDataWriter; import org.apache.hyracks.test.support.TestUtils; @@ -72,7 +72,7 @@ public class ConnectorDescriptorWithMessagingTest { IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); @@ -81,8 +81,8 @@ public class ConnectorDescriptorWithMessagingTest { BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() }; RecordDescriptor rDesc = new RecordDescriptor(serdes); TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory(); - IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, - NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS); + IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, + CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS); partitioner.open(); FrameTupleAccessor fta = new FrameTupleAccessor(rDesc); List<TestFrameWriter> recipients = new ArrayList<>(); @@ -146,15 +146,15 @@ public class ConnectorDescriptorWithMessagingTest { IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); writeRandomMessage(message, MessagingFrameTupleAppender.MARKER_MESSAGE, DEFAULT_FRAME_SIZE + 1); ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, BooleanSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() }; RecordDescriptor rDesc = new RecordDescriptor(serdes); TestPartitionWriterFactory partitionWriterFactory = new TestPartitionWriterFactory(); - IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, CURRENT_PRODUCER, - NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS); + IFrameWriter partitioner = connector.createPartitioner(ctx, rDesc, partitionWriterFactory, + CURRENT_PRODUCER, NUMBER_OF_CONSUMERS, NUMBER_OF_CONSUMERS); partitioner.open(); FrameTupleAccessor fta = new FrameTupleAccessor(rDesc); List<TestFrameWriter> recipients = new ArrayList<>(); @@ -230,7 +230,7 @@ public class ConnectorDescriptorWithMessagingTest { IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); message.getBuffer().clear(); writeRandomMessage(message, MessagingFrameTupleAppender.MARKER_MESSAGE, DEFAULT_FRAME_SIZE); ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { @@ -288,7 +288,7 @@ public class ConnectorDescriptorWithMessagingTest { IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE); message.getBuffer().flip(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java index 00d725a..edd1848 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/LogMarkerTest.java @@ -18,8 +18,10 @@ */ package org.apache.asterix.test.dataflow; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import org.apache.asterix.app.bootstrap.TestNodeController; import org.apache.asterix.app.data.gen.TestTupleCounterFrameWriter; @@ -31,7 +33,9 @@ import org.apache.asterix.common.transactions.DatasetId; import org.apache.asterix.common.transactions.ILogRecord; import org.apache.asterix.common.transactions.ITransactionContext; import org.apache.asterix.external.util.DataflowUtils; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy; import org.apache.asterix.om.types.ARecordType; @@ -50,8 +54,8 @@ import org.apache.hyracks.api.util.HyracksConstants; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender; -import org.apache.hyracks.dataflow.common.util.TaskUtils; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelper; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; +import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory; import org.junit.After; @@ -71,7 +75,8 @@ public class LogMarkerTest { private static final GenerationFunction[] META_GEN_FUNCTION = null; private static final boolean[] UNIQUE_META_FIELDS = null; private static final int[] KEY_INDEXES = { 0 }; - private static final int[] KEY_INDICATORS = { 0 }; + private static final int[] KEY_INDICATORS = { Index.RECORD_INDICATOR }; + private static final List<Integer> KEY_INDICATORS_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR }); private static final int NUM_OF_RECORDS = 100000; private static final int SNAPSHOT_SIZE = 1000; private static final int DATASET_ID = 101; @@ -97,18 +102,20 @@ public class LogMarkerTest { try { TestNodeController nc = new TestNodeController(null, false); nc.init(); + StorageComponentProvider storageManager = new StorageComponentProvider(); Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, Collections.emptyList(), null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0); try { nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, - null); + null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST); IHyracksTaskContext ctx = nc.createTestContext(true); nc.newJobId(); ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true); LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, - RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null); + RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, + KEY_INDICATORS_LIST, storageManager); insertOp.open(); TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS); @@ -123,7 +130,7 @@ public class LogMarkerTest { marker.getBuffer().putLong(markerId); marker.getBuffer().flip(); markerId++; - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx); tupleAppender.flush(insertOp); } ITupleReference tuple = tupleGenerator.next(); @@ -134,8 +141,9 @@ public class LogMarkerTest { } insertOp.close(); nc.getTransactionManager().completedTransaction(txnCtx, new DatasetId(-1), -1, true); - LSMBTreeDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, - RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null); + IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, + RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, + KEY_INDICATORS_LIST); dataflowHelper.open(); LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance(); long lsn = btree.getMostRecentMarkerLSN(); @@ -157,7 +165,8 @@ public class LogMarkerTest { TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false); IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, - META_TYPE, new NoMergePolicyFactory(), null, null); + META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, + storageManager); emptyTupleOp.open(); emptyTupleOp.close(); Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java index d5501c3..1158611 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java @@ -30,6 +30,7 @@ import org.apache.asterix.api.java.AsterixJavaClient; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.compiler.provider.AqlCompilationProvider; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.test.aql.TestExecutor; import org.apache.asterix.test.base.AsterixTestHelper; import org.junit.Test; @@ -61,7 +62,7 @@ public class DmlTest { Reader loadReader = new BufferedReader( new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8")); AsterixJavaClient asterixLoad = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), loadReader, - ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory()); + ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory(), new StorageComponentProvider()); try { asterixLoad.compile(true, false, false, false, false, true, false); } catch (AsterixException e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java index 9fbf850..10e8658 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/logging/CheckpointingTest.java @@ -19,13 +19,15 @@ package org.apache.asterix.test.logging; import java.io.File; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import org.apache.asterix.app.bootstrap.TestNodeController; import org.apache.asterix.app.data.gen.TupleGenerator; import org.apache.asterix.app.data.gen.TupleGenerator.GenerationFunction; -import org.apache.asterix.common.config.TransactionProperties; import org.apache.asterix.common.config.DatasetConfig.DatasetType; +import org.apache.asterix.common.config.TransactionProperties; import org.apache.asterix.common.configuration.AsterixConfiguration; import org.apache.asterix.common.configuration.Property; import org.apache.asterix.common.dataflow.LSMInsertDeleteOperatorNodePushable; @@ -34,7 +36,9 @@ import org.apache.asterix.common.transactions.ICheckpointManager; import org.apache.asterix.common.transactions.IRecoveryManager; import org.apache.asterix.common.transactions.ITransactionContext; import org.apache.asterix.external.util.DataflowUtils; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy; import org.apache.asterix.om.types.ARecordType; @@ -71,7 +75,8 @@ public class CheckpointingTest { private static final GenerationFunction[] META_GEN_FUNCTION = null; private static final boolean[] UNIQUE_META_FIELDS = null; private static final int[] KEY_INDEXES = { 0 }; - private static final int[] KEY_INDICATORS = { 0 }; + private static final int[] KEY_INDICATOR = { Index.RECORD_INDICATOR }; + private static final List<Integer> KEY_INDICATOR_LIST = Arrays.asList(new Integer[] { Index.RECORD_INDICATOR }); private static final int DATASET_ID = 101; private static final String DATAVERSE_NAME = "TestDV"; private static final String DATASET_NAME = "TestDS"; @@ -105,6 +110,7 @@ public class CheckpointingTest { public void testDeleteOldLogFiles() { try { TestNodeController nc = new TestNodeController(new File(TEST_CONFIG_FILE_PATH).getAbsolutePath(), false); + StorageComponentProvider storageManager = new StorageComponentProvider(); nc.init(); Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, @@ -112,15 +118,16 @@ public class CheckpointingTest { null, DatasetType.INTERNAL, DATASET_ID, 0); try { nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, - null); + null, storageManager, KEY_INDEXES, KEY_INDICATOR_LIST); IHyracksTaskContext ctx = nc.createTestContext(false); nc.newJobId(); ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true); // Prepare insert operation LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, - RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null); + RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATOR_LIST, + storageManager); insertOp.open(); - TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, + TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATOR, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS); VSizeFrame frame = new VSizeFrame(ctx); FrameTupleAppender tupleAppender = new FrameTupleAppender(frame); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java index 8db2bc0..01fe46a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java @@ -32,12 +32,14 @@ import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; import org.apache.asterix.api.java.AsterixJavaClient; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; import org.apache.asterix.common.config.GlobalConfig; +import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.compiler.provider.AqlCompilationProvider; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.IdentitiyResolverFactory; +import org.apache.asterix.file.StorageComponentProvider; import org.apache.asterix.test.base.AsterixTestHelper; import org.apache.asterix.test.common.TestHelper; import org.apache.asterix.test.runtime.HDFSCluster; @@ -76,6 +78,7 @@ public class OptimizerTest { private static final ILangCompilationProvider sqlppCompilationProvider = new SqlppCompilationProvider(); protected static ILangCompilationProvider extensionLangCompilationProvider = null; protected static IStatementExecutorFactory statementExecutorFactory = new DefaultStatementExecutorFactory(); + protected static IStorageComponentProvider storageComponentProvider = new StorageComponentProvider(); protected static AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); @@ -124,7 +127,7 @@ public class OptimizerTest { @Parameters(name = "OptimizerTest {index}: {0}") public static Collection<Object[]> tests() { - Collection<Object[]> testArgs = new ArrayList<Object[]>(); + Collection<Object[]> testArgs = new ArrayList<>(); if (only.isEmpty()) { suiteBuildPerFile(new File(PATH_QUERIES), testArgs, ""); } else { @@ -178,7 +181,8 @@ public class OptimizerTest { provider = extensionLangCompilationProvider; } IHyracksClientConnection hcc = integrationUtil.getHyracksClientConnection(); - AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider, statementExecutorFactory); + AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider, statementExecutorFactory, + storageComponentProvider); try { asterix.compile(true, false, false, true, true, false, false); } catch (AsterixException e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java index f488244..90eb441 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java @@ -31,7 +31,7 @@ import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.library.ILibraryManager; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.IdentitiyResolverFactory; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.asterix.testframework.xml.TestSuite; import org.apache.hyracks.control.nc.NodeControllerService; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml index a3864e4..0b93b96 100644 --- a/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/metadata/testsuite.xml @@ -417,13 +417,13 @@ <test-case FilePath="exception"> <compilation-unit name="issue_255_create_dataset_error_1"> <output-dir compare="Text">none</output-dir> - <expected-error>Type not found for partitioning key [name]</expected-error> + <expected-error>Field "name" is not found</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_255_create_dataset_error_2"> <output-dir compare="Text">none</output-dir> - <expected-error>Error: The partitioning key [open-type] cannot be of type record.</expected-error> + <expected-error>Field of type RECORD cannot be used as a primary key field</expected-error> </compilation-unit> </test-case> <!-- Feed datasets are not supported anymore @@ -437,13 +437,13 @@ <test-case FilePath="exception"> <compilation-unit name="issue_266_create_dataset_error_1"> <output-dir compare="Text">none</output-dir> - <expected-error>Type not found for partitioning key [point]</expected-error> + <expected-error>Field "point" is not found</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> <compilation-unit name="issue_266_create_dataset_error_2"> <output-dir compare="Text">none</output-dir> - <expected-error>Error: The partitioning key [id] cannot be nullable</expected-error> + <expected-error>The primary key field "id" cannot be nullable</expected-error> </compilation-unit> </test-case> <test-case FilePath="exception"> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml index ab55c89..5be8639 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml @@ -3317,7 +3317,7 @@ <test-case FilePath="misc"> <compilation-unit name="partition-by-nonexistent-field"> <!-- Seriously?? 3 expected errors --> <output-dir compare="Text">partition-by-nonexistent-field</output-dir> - <expected-error>Type not found for partitioning key [id]</expected-error> + <expected-error>Field "id" is not found</expected-error> <expected-error>Cannot find dataset</expected-error> <expected-error>Could not find dataset</expected-error> </compilation-unit> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index e5c5bdd..a76ecbc 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -3119,7 +3119,7 @@ <test-case FilePath="misc"> <compilation-unit name="partition-by-nonexistent-field"> <output-dir compare="Text">partition-by-nonexistent-field</output-dir> - <expected-error>Type not found for partitioning key [id]</expected-error> + <expected-error>Field "id" is not found</expected-error> <expected-error>Cannot find dataset testds in dataverse test</expected-error> <expected-error>Cannot find dataset testds in dataverse test nor an alias with name testds!</expected-error> </compilation-unit> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/pom.xml b/asterixdb/asterix-common/pom.xml index 277f94a..731ac14 100644 --- a/asterixdb/asterix-common/pom.xml +++ b/asterixdb/asterix-common/pom.xml @@ -24,7 +24,6 @@ <version>0.9.1-SNAPSHOT</version> </parent> <artifactId>asterix-common</artifactId> - <licenses> <license> <name>Apache License, Version 2.0</name> @@ -33,11 +32,9 @@ <comments>A business-friendly OSS license</comments> </license> </licenses> - <properties> <appendedResourcesDirectory>${basedir}/../src/main/appended-resources</appendedResourcesDirectory> </properties> - <build> <plugins> <plugin> @@ -173,7 +170,6 @@ </plugin> </plugins> </build> - <dependencies> <dependency> <groupId>commons-io</groupId> @@ -201,6 +197,10 @@ </dependency> <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>algebricks-data</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-util</artifactId> </dependency> <dependency> @@ -316,5 +316,4 @@ <artifactId>jackson-annotations</artifactId> </dependency> </dependencies> - -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAppRuntimeContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAppRuntimeContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAppRuntimeContext.java index 7bc9421..548d714 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAppRuntimeContext.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAppRuntimeContext.java @@ -22,6 +22,8 @@ import java.io.IOException; import java.rmi.RemoteException; import java.util.concurrent.Executor; +import org.apache.asterix.common.config.IPropertiesProvider; +import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.library.ILibraryManager; @@ -40,58 +42,58 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; import org.apache.hyracks.storage.common.file.ILocalResourceRepository; import org.apache.hyracks.storage.common.file.IResourceIdFactory; -public interface IAppRuntimeContext { +public interface IAppRuntimeContext extends IPropertiesProvider { - public IIOManager getIOManager(); + IIOManager getIOManager(); - public Executor getThreadExecutor(); + Executor getThreadExecutor(); - public ITransactionSubsystem getTransactionSubsystem(); + ITransactionSubsystem getTransactionSubsystem(); - public boolean isShuttingdown(); + boolean isShuttingdown(); - public ILSMIOOperationScheduler getLSMIOScheduler(); + ILSMIOOperationScheduler getLSMIOScheduler(); - public ILSMMergePolicyFactory getMetadataMergePolicyFactory(); + ILSMMergePolicyFactory getMetadataMergePolicyFactory(); - public IBufferCache getBufferCache(); + IBufferCache getBufferCache(); - public IFileMapProvider getFileMapManager(); + IFileMapProvider getFileMapManager(); - public ILocalResourceRepository getLocalResourceRepository(); + ILocalResourceRepository getLocalResourceRepository(); - public IDatasetLifecycleManager getDatasetLifecycleManager(); + IDatasetLifecycleManager getDatasetLifecycleManager(); - public IResourceIdFactory getResourceIdFactory(); + IResourceIdFactory getResourceIdFactory(); - public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID); + ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID); - public void initialize(boolean initialRun) throws IOException, ACIDException, AsterixException; + void initialize(boolean initialRun) throws IOException, ACIDException, AsterixException; - public void setShuttingdown(boolean b); + void setShuttingdown(boolean b); - public void deinitialize() throws HyracksDataException; + void deinitialize() throws HyracksDataException; - public double getBloomFilterFalsePositiveRate(); + double getBloomFilterFalsePositiveRate(); - public Object getActiveManager(); + Object getActiveManager(); - public IRemoteRecoveryManager getRemoteRecoveryManager(); + IRemoteRecoveryManager getRemoteRecoveryManager(); - public IReplicaResourcesManager getReplicaResourcesManager(); + IReplicaResourcesManager getReplicaResourcesManager(); - public IReplicationManager getReplicationManager(); + IReplicationManager getReplicationManager(); - public IReplicationChannel getReplicationChannel(); + IReplicationChannel getReplicationChannel(); - public ILibraryManager getLibraryManager(); + ILibraryManager getLibraryManager(); /** * Exports the metadata node to the metadata RMI port. * * @throws RemoteException */ - public void exportMetadataNodeStub() throws RemoteException; + void exportMetadataNodeStub() throws RemoteException; /** * Initializes the metadata node and bootstraps the metadata. @@ -99,12 +101,17 @@ public interface IAppRuntimeContext { * @param newUniverse * @throws Exception */ - public void initializeMetadata(boolean newUniverse) throws Exception; + void initializeMetadata(boolean newUniverse) throws Exception; /** * Unexports the metadata node from the RMI registry * * @throws RemoteException */ - public void unexportMetadataNodeStub() throws RemoteException; + void unexportMetadataNodeStub() throws RemoteException; + + /** + * @return instance of {@link org.apache.asterix.common.context.IStorageComponentProvider} + */ + IStorageComponentProvider getStorageComponentProvider(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryMaanger.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryMaanger.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryMaanger.java deleted file mode 100644 index 48b1e73..0000000 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryMaanger.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.common.cluster; - -import org.apache.asterix.common.api.IClusterEventsSubscriber; - -public interface IGlobalRecoveryMaanger extends IClusterEventsSubscriber { - - /** - * Starts the global recovery process if the cluster state changed to ACTIVE. - */ - public void startGlobalRecovery(); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryManager.java new file mode 100644 index 0000000..b54bb39 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IGlobalRecoveryManager.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.cluster; + +import org.apache.asterix.common.api.IClusterEventsSubscriber; + +public interface IGlobalRecoveryManager extends IClusterEventsSubscriber { + + /** + * Starts the global recovery process if the cluster state changed to ACTIVE. + */ + public void startGlobalRecovery(); +}
