http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java new file mode 100644 index 0000000..0560bd0 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/BTreeDataflowHelperFactoryProvider.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.declared; + +import java.util.Map; + +import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider; +import org.apache.asterix.common.context.IStorageComponentProvider; +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.external.indexing.IndexingConstants; +import org.apache.asterix.metadata.api.IIndexDataflowHelperFactoryProvider; +import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.metadata.entities.Index; +import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry; +import org.apache.asterix.metadata.utils.IndexUtil; +import org.apache.asterix.om.types.ARecordType; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; +import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory; +import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; + +public class BTreeDataflowHelperFactoryProvider implements IIndexDataflowHelperFactoryProvider { + + public static final BTreeDataflowHelperFactoryProvider INSTANCE = new BTreeDataflowHelperFactoryProvider(); + + private BTreeDataflowHelperFactoryProvider() { + } + + public static String externalFileIndexName(Dataset dataset) { + return dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX); + } + + @Override + public IIndexDataflowHelperFactory getIndexDataflowHelperFactory(MetadataProvider mdProvider, Dataset dataset, + Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, + Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, + IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException { + int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits); + int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index); + IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider(); + switch (dataset.getDatasetType()) { + case EXTERNAL: + return index.getIndexName().equals(externalFileIndexName(dataset)) + ? new ExternalBTreeDataflowHelperFactory(mergePolicyFactory, mergePolicyProperties, + dataset.getIndexOperationTrackerFactory(index), + storageComponentProvider.getIoOperationSchedulerProvider(), + dataset.getIoOperationCallbackFactory(index), + mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(), + ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, mdProvider), + !dataset.getDatasetDetails().isTemp()) + : new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyProperties, + dataset.getIndexOperationTrackerFactory(index), + storageComponentProvider.getIoOperationSchedulerProvider(), + dataset.getIoOperationCallbackFactory(index), + mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(), + new int[] { index.getKeyFieldNames().size() }, + ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, mdProvider), + !dataset.getDatasetDetails().isTemp()); + case INTERNAL: + return new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), + mergePolicyFactory, mergePolicyProperties, dataset.getIndexOperationTrackerFactory(index), + storageComponentProvider.getIoOperationSchedulerProvider(), + dataset.getIoOperationCallbackFactory(index), + mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate(), index.isPrimaryIndex(), + filterTypeTraits, filterCmpFactories, btreeFields, filterFields, + !dataset.getDatasetDetails().isTemp()); + default: + throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, + dataset.getDatasetType().toString()); + } + } +}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index 973651f..68c7e22 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@ -28,7 +28,7 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.ExternalDatasetDetails; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.metadata.utils.KeyFieldTypeUtils; +import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; @@ -73,7 +73,7 @@ public class DatasetDataSource extends DataSource { ARecordType recordType = (ARecordType) itemType; ARecordType metaRecordType = (ARecordType) metaItemType; List<IAType> partitioningKeyTypes = - KeyFieldTypeUtils.getPartitioningKeyTypes(internalDatasetDetails, recordType, metaRecordType); + KeyFieldTypeUtil.getPartitioningKeyTypes(internalDatasetDetails, recordType, metaRecordType); int n = partitioningKeyTypes.size(); schemaTypes = metaItemType == null ? new IAType[n + 1] : new IAType[n + 2]; for (int keyIndex = 0; keyIndex < n; ++keyIndex) { @@ -106,7 +106,7 @@ public class DatasetDataSource extends DataSource { ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails(); IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset, - edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, false, null, null); + edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null); return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory, NonTaggedDataFormat.INSTANCE); case INTERNAL: @@ -118,9 +118,9 @@ public class DatasetDataSource extends DataSource { int[] minFilterFieldIndexes = createFilterIndexes(minFilterVars, opSchema); int[] maxFilterFieldIndexes = createFilterIndexes(maxFilterVars, opSchema); - return metadataProvider.buildBtreeRuntime(jobSpec, scanVariables, opSchema, typeEnv, context, true, + return metadataProvider.buildBtreeRuntime(jobSpec, opSchema, typeEnv, context, true, false, ((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null, - true, true, implConfig, minFilterFieldIndexes, maxFilterFieldIndexes); + true, true, minFilterFieldIndexes, maxFilterFieldIndexes); default: throw new AlgebricksException("Unknown datasource type"); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java index 703f8c8..0d3d06d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java @@ -33,7 +33,7 @@ import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java index 5b5844c..cdf699d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java @@ -27,7 +27,7 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.external.api.IAdapterFactory; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; @@ -60,14 +60,14 @@ public class LoadableDataSource extends DataSource { this.targetDataset = targetDataset; this.adapter = adapter; this.adapterProperties = properties; - partitioningKeys = DatasetUtils.getPartitioningKeys(targetDataset); + partitioningKeys = DatasetUtil.getPartitioningKeys(targetDataset); ARecordType recType = (ARecordType) itemType; isPKAutoGenerated = ((InternalDatasetDetails) targetDataset.getDatasetDetails()).isAutogenerated(); if (isPKAutoGenerated) { // Since the key is auto-generated, we need to use another // record type (possibly nested) which has all fields except the PK try { - recType = getStrippedPKType(new LinkedList<String>(partitioningKeys.get(0)), recType); + recType = getStrippedPKType(new LinkedList<>(partitioningKeys.get(0)), recType); } catch (AsterixException e) { throw new AlgebricksException(e); } @@ -81,7 +81,7 @@ public class LoadableDataSource extends DataSource { List<IAType> fieldTypes = new LinkedList<>(); int j = 0; for (int i = 0; i < recType.getFieldNames().length; i++) { - IAType fieldType = null; + IAType fieldType; if (partitioningKeys.get(0).equals(recType.getFieldNames()[j])) { if (recType.getFieldTypes()[j].getTypeTag() == ATypeTag.RECORD) { if (j != 0) { @@ -132,15 +132,10 @@ public class LoadableDataSource extends DataSource { List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException { - // This is a load into dataset operation LoadableDataSource alds = (LoadableDataSource) dataSource; - List<List<String>> aldsPartitioningKeys = alds.getPartitioningKeys(); - boolean isAldsPKAutoGenerated = ((InternalDatasetDetails) alds.getTargetDataset().getDatasetDetails()) - .isAutogenerated(); ARecordType itemType = (ARecordType) alds.getLoadedType(); IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(), - alds.getAdapter(), alds.getAdapterProperties(), itemType, isAldsPKAutoGenerated, aldsPartitioningKeys, - null); + alds.getAdapter(), alds.getAdapterProperties(), itemType, null); RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); return metadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java index 536a688..f1a90c7 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java @@ -35,7 +35,7 @@ import org.apache.asterix.metadata.entities.NodeGroup; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.properties.DefaultNodeGroupDomain; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
