http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesCommitOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesCommitOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesCommitOperatorDescriptor.java index fb6e0f9..2ddb646 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesCommitOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesCommitOperatorDescriptor.java @@ -29,39 +29,32 @@ import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.common.api.ITwoPCIndex; -import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory; import org.apache.log4j.Logger; public class ExternalDatasetIndexesCommitOperatorDescriptor extends AbstractExternalDatasetIndexesOperatorDescriptor { - private static final Logger LOGGER = Logger - .getLogger(ExternalDatasetIndexesCommitOperatorDescriptor.class.getName()); + private static final long serialVersionUID = 1L; + private static final Logger LOGGER = + Logger.getLogger(ExternalDatasetIndexesCommitOperatorDescriptor.class.getName()); public ExternalDatasetIndexesCommitOperatorDescriptor(IOperatorDescriptorRegistry spec, - ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory, - IndexInfoOperatorDescriptor fileIndexesInfo, - List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> bTreeIndexesInfos, - List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) { - super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, bTreeIndexesDataflowHelperFactories, - bTreeIndexesInfos, rTreeIndexesDataflowHelperFactories, rTreeIndexesInfos); + IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo, + List<IIndexDataflowHelperFactory> indexesDataflowHelperFactories, + List<IndexInfoOperatorDescriptor> indexesInfos) { + super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, indexesDataflowHelperFactories, + indexesInfos); } - private static final long serialVersionUID = 1L; - @Override protected void performOpOnIndex(IIndexDataflowHelperFactory indexDataflowHelperFactory, IHyracksTaskContext ctx, IndexInfoOperatorDescriptor fileIndexInfo, int partition) { try { - FileReference resourecePath = IndexFileNameUtil.getIndexAbsoluteFileRef(fileIndexInfo, partition, - ctx.getIOManager()); + FileReference resourecePath = + IndexFileNameUtil.getIndexAbsoluteFileRef(fileIndexInfo, partition, ctx.getIOManager()); LOGGER.warn("performing the operation on " + resourecePath.getFile().getAbsolutePath()); // Get DataflowHelper - IIndexDataflowHelper indexHelper = indexDataflowHelperFactory.createIndexDataflowHelper(fileIndexInfo, ctx, - partition); + IIndexDataflowHelper indexHelper = + indexDataflowHelperFactory.createIndexDataflowHelper(fileIndexInfo, ctx, partition); // Get index IIndex index = indexHelper.getIndexInstance(); // commit transaction
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java index 0312eac..f74f4e9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesRecoverOperatorDescriptor.java @@ -25,24 +25,18 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.common.impls.AbortRecoverLSMIndexFileManager; -import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory; public class ExternalDatasetIndexesRecoverOperatorDescriptor extends AbstractExternalDatasetIndexesOperatorDescriptor { private static final long serialVersionUID = 1L; public ExternalDatasetIndexesRecoverOperatorDescriptor(IOperatorDescriptorRegistry spec, - ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory, - IndexInfoOperatorDescriptor fileIndexesInfo, - List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> bTreeIndexesInfos, - List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) { - super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, bTreeIndexesDataflowHelperFactories, - bTreeIndexesInfos, rTreeIndexesDataflowHelperFactories, rTreeIndexesInfos); + IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo, + List<IIndexDataflowHelperFactory> indexesDataflowHelperFactories, + List<IndexInfoOperatorDescriptor> indexesInfos) { + super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, indexesDataflowHelperFactories, + indexesInfos); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexOperatorDescriptor.java index ec82ede..2bb986d 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalFilesIndexOperatorDescriptor.java @@ -21,7 +21,6 @@ package org.apache.asterix.external.operators; import java.io.IOException; import java.util.List; -import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.external.indexing.FileIndexTupleTranslator; import org.apache.asterix.external.indexing.FilesIndexDescription; @@ -46,7 +45,7 @@ import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree; import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree.LSMTwoPCBTreeBulkLoader; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider; /** @@ -63,13 +62,13 @@ public class ExternalFilesIndexOperatorDescriptor extends AbstractTreeIndexOpera private List<ExternalFile> files; public ExternalFilesIndexOperatorDescriptor(IOperatorDescriptorRegistry spec, - IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, + IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider, IIndexDataflowHelperFactory dataflowHelperFactory, ILocalResourceFactoryProvider localResourceFactoryProvider, List<ExternalFile> files, boolean createNewIndex, IMetadataPageManagerFactory metadataPageManagerFactory) { super(spec, 0, 0, null, storageManager, lifecycleManagerProvider, fileSplitProvider, new FilesIndexDescription().EXTERNAL_FILE_INDEX_TYPE_TRAITS, - new FilesIndexDescription().FILES_INDEX_COMP_FACTORIES, FilesIndexDescription.BLOOM_FILTER_FIELDS, + FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, FilesIndexDescription.BLOOM_FILTER_FIELDS, dataflowHelperFactory, null, false, false, null, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE, NoOpOperationCallbackFactory.INSTANCE, metadataPageManagerFactory); @@ -121,11 +120,11 @@ public class ExternalFilesIndexOperatorDescriptor extends AbstractTreeIndexOpera // The files must be ordered according to their numbers for (ExternalFile file : files) { switch (file.getPendingOp()) { - case PENDING_ADD_OP: - case PENDING_APPEND_OP: + case ADD_OP: + case APPEND_OP: bulkLoader.add(filesTupleTranslator.getTupleFromFile(file)); break; - case PENDING_DROP_OP: + case DROP_OP: bulkLoader.delete(filesTupleTranslator.getTupleFromFile(file)); break; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java index 59352a0..a994c1f 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalIndexBulkModifyOperatorDescriptor.java @@ -32,7 +32,7 @@ import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFa import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.file.NoOpLocalResourceFactoryProvider; public class ExternalIndexBulkModifyOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor { @@ -44,7 +44,7 @@ public class ExternalIndexBulkModifyOperatorDescriptor extends AbstractTreeIndex private final long numElementsHint; public ExternalIndexBulkModifyOperatorDescriptor(IOperatorDescriptorRegistry spec, - IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, + IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, IIndexDataflowHelperFactory dataflowHelperFactory, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java index 24e9cbe..d4718a9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalLookupOperatorDescriptor.java @@ -34,12 +34,12 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelper; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; /* * This operator is intended for using record ids to access data in external sources @@ -49,17 +49,17 @@ public class ExternalLookupOperatorDescriptor extends AbstractTreeIndexOperatorD private final LookupAdapterFactory<?> adapterFactory; public ExternalLookupOperatorDescriptor(IOperatorDescriptorRegistry spec, LookupAdapterFactory<?> adapterFactory, - RecordDescriptor outRecDesc, ExternalBTreeDataflowHelperFactory externalFilesIndexDataFlowHelperFactory, + RecordDescriptor outRecDesc, IIndexDataflowHelperFactory externalFilesIndexDataFlowHelperFactory, boolean propagateInput, IIndexLifecycleManagerProvider lcManagerProvider, - IStorageManagerInterface storageManager, IFileSplitProvider fileSplitProvider, int datasetId, - double bloomFilterFalsePositiveRate, ISearchOperationCallbackFactory searchOpCallbackFactory, + IStorageManager storageManager, IFileSplitProvider fileSplitProvider, + ISearchOperationCallbackFactory searchOpCallbackFactory, boolean retainMissing, IMissingWriterFactory missingWriterFactory, - IMetadataPageManagerFactory metadataPageManagerFactory) { + IPageManagerFactory pageManagerFactory) { super(spec, 1, 1, outRecDesc, storageManager, lcManagerProvider, fileSplitProvider, new FilesIndexDescription().EXTERNAL_FILE_INDEX_TYPE_TRAITS, - new FilesIndexDescription().FILES_INDEX_COMP_FACTORIES, FilesIndexDescription.BLOOM_FILTER_FIELDS, + FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, FilesIndexDescription.BLOOM_FILTER_FIELDS, externalFilesIndexDataFlowHelperFactory, null, propagateInput, retainMissing, missingWriterFactory, - null, searchOpCallbackFactory, null, metadataPageManagerFactory); + null, searchOpCallbackFactory, null, pageManagerFactory); this.adapterFactory = adapterFactory; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java index 1a5eeef..22960e4 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalRTreeSearchOperatorDescriptor.java @@ -29,26 +29,26 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; -import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; public class ExternalRTreeSearchOperatorDescriptor extends RTreeSearchOperatorDescriptor { private static final long serialVersionUID = 1L; public ExternalRTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc, - IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, + IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] keyFields, - ExternalRTreeDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainMissing, + IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainMissing, IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchOpCallbackFactory, - IMetadataPageManagerFactory metadataPageManagerFactory) { + IPageManagerFactory pageManagerFactory) { super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, keyFields, dataflowHelperFactory, retainInput, retainMissing, missingWriterFactory, - searchOpCallbackFactory, null, null, metadataPageManagerFactory); + searchOpCallbackFactory, null, null, pageManagerFactory); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java new file mode 100644 index 0000000..96a6848 --- /dev/null +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalScanOperatorDescriptor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.external.operators; + +import org.apache.asterix.external.api.IAdapterFactory; +import org.apache.asterix.external.api.IDataSourceAdapter; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.dataflow.IOperatorNodePushable; +import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; +import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.job.JobSpecification; +import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; +import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable; + +/* + * A single activity operator that provides the functionality of scanning data using an + * instance of the configured adapter. + */ +public class ExternalScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { + + private static final long serialVersionUID = 1L; + + private IAdapterFactory adapterFactory; + + public ExternalScanOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc, + IAdapterFactory dataSourceAdapterFactory) { + super(spec, 0, 1); + recordDescriptors[0] = rDesc; + this.adapterFactory = dataSourceAdapterFactory; + } + + @Override + public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, + IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) + throws HyracksDataException { + + return new AbstractUnaryOutputSourceOperatorNodePushable() { + + @Override + public void initialize() throws HyracksDataException { + IDataSourceAdapter adapter = null; + try { + writer.open(); + adapter = adapterFactory.createAdapter(ctx, partition); + adapter.start(partition, writer); + } catch (Exception e) { + writer.fail(); + throw new HyracksDataException(e); + } finally { + writer.close(); + } + } + }; + + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java index 66a6c42..ed017a3 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java @@ -41,7 +41,7 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.HyracksConstants; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; /* @@ -107,7 +107,7 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp this.feedManager = (ActiveManager) ((IAppRuntimeContext) ctx.getJobletContext().getApplicationContext() .getApplicationObject()).getActiveManager(); this.message = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); this.opDesc = feedMetaOperatorDescriptor; this.recordDescProvider = recordDescProvider; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java index af54245..48276d5 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java @@ -41,7 +41,7 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.HyracksConstants; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable { @@ -101,7 +101,7 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper .getApplicationObject()).getActiveManager(); this.targetId = targetId; this.message = new VSizeFrame(ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx); this.recordDescProvider = recordDescProvider; this.opDesc = feedMetaOperatorDescriptor; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/IndexInfoOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/IndexInfoOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/IndexInfoOperatorDescriptor.java index 7aa4b25..050cbcf 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/IndexInfoOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/IndexInfoOperatorDescriptor.java @@ -33,7 +33,7 @@ import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.file.ILocalResourceFactoryProvider; /* @@ -43,9 +43,9 @@ public class IndexInfoOperatorDescriptor implements IIndexOperatorDescriptor{ private static final long serialVersionUID = 1L; private final IFileSplitProvider fileSplitProvider; - private final IStorageManagerInterface storageManager; + private final IStorageManager storageManager; private final IIndexLifecycleManagerProvider lifecycleManagerProvider; - public IndexInfoOperatorDescriptor(IFileSplitProvider fileSplitProvider,IStorageManagerInterface storageManager, + public IndexInfoOperatorDescriptor(IFileSplitProvider fileSplitProvider,IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider){ this.fileSplitProvider = fileSplitProvider; this.lifecycleManagerProvider = lifecycleManagerProvider; @@ -69,7 +69,7 @@ public class IndexInfoOperatorDescriptor implements IIndexOperatorDescriptor{ } @Override - public IStorageManagerInterface getStorageManager() { + public IStorageManager getStorageManager() { return storageManager; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java index b14a2c4..0df1412 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DelimitedDataParser.java @@ -34,7 +34,7 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.om.base.AMutableString; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.dataflow.common.data.parsers.IValueParser; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/HiveRecordParser.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/HiveRecordParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/HiveRecordParser.java index fdcd4ff..5e6e288 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/HiveRecordParser.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/HiveRecordParser.java @@ -38,7 +38,7 @@ import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.AUnorderedListType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.SerDe; import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java index 2ca9f82..5ec0399 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedUtils.java @@ -28,7 +28,7 @@ import org.apache.asterix.common.cluster.ClusterPartition; import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.utils.StoragePathUtil; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint.PartitionConstraintType; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java index 9556054..53a46ab 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/HDFSUtils.java @@ -31,13 +31,13 @@ import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.external.indexing.IndexingScheduler; import org.apache.asterix.external.indexing.RecordId.RecordIdType; import org.apache.asterix.external.input.stream.HDFSInputStream; -import org.apache.asterix.runtime.util.AppContextInfo; -import org.apache.asterix.runtime.util.ClusterStateManager; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.asterix.hivecompat.io.RCFileInputFormat; +import org.apache.asterix.runtime.utils.AppContextInfo; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -102,7 +102,7 @@ public class HDFSUtils { // file was deleted at some point, skip to next file continue; } - if (file.getPendingOp() == ExternalFilePendingOp.PENDING_ADD_OP + if (file.getPendingOp() == ExternalFilePendingOp.ADD_OP && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) { // Get its information from HDFS name node BlockLocation[] fileBlocks = fs.getFileBlockLocations(fileStatus, 0, file.getSize()); @@ -117,7 +117,7 @@ public class HDFSUtils { orderedExternalFiles.add(file); } } - } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP + } else if (file.getPendingOp() == ExternalFilePendingOp.NO_OP && fileStatus.getModificationTime() == file.getLastModefiedTime().getTime()) { long oldSize = 0L; long newSize = file.getSize(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolver.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolver.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolver.java index 222d49d..9970f27 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolver.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/NodeResolver.java @@ -30,7 +30,7 @@ import java.util.Set; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.external.api.INodeResolver; -import org.apache.asterix.runtime.util.RuntimeUtils; +import org.apache.asterix.runtime.utils.RuntimeUtils; /** * Resolves a value (DNS/IP Address) or a (Node Controller Id) to the id of a Node Controller running at the location. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java index 6841e6b..98105b2 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReaderFactory.java @@ -22,7 +22,7 @@ import java.util.Map; import org.apache.asterix.external.api.IRecordReader; import org.apache.asterix.external.api.IRecordReaderFactory; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.api.context.IHyracksTaskContext; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java index 77b8df1..c417aea 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/stream/TwitterFirehoseStreamFactory.java @@ -26,7 +26,7 @@ import java.util.Map; import org.apache.asterix.external.api.AsterixInputStream; import org.apache.asterix.external.api.IInputStreamFactory; import org.apache.asterix.external.input.stream.TwitterFirehoseInputStream; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java index 40523ab..93b01f1 100644 --- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java +++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java @@ -70,9 +70,9 @@ import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.AUnorderedListType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; -import org.apache.asterix.om.util.NonTaggedFormatUtil; import org.apache.asterix.om.util.container.IObjectPool; import org.apache.asterix.om.util.container.ListObjectPool; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IMutableValueStorage; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java index b12195f..06fbf33 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java @@ -26,7 +26,7 @@ public interface Statement extends ILangExpression { public static final List<Byte> KINDS = Collections.unmodifiableList(Kind.range(Kind.DATASET_DECL, Kind.RUN)); /** - * get a byte representing the statement kind + * get a byte representing the statement kind. * Note: bytes 0x00 - 0x7f are reserved for core asterix statements * Use negative bytes for extension statements * http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java index cf57dfe..8907a72 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java @@ -44,8 +44,8 @@ import org.apache.asterix.om.base.ARecord; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.base.IACursor; import org.apache.asterix.om.base.IAObject; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.BuiltinType; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; @@ -96,7 +96,7 @@ public class LangRecordParseUtil { ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage(); ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage(); RecordBuilder recordBuilder = new RecordBuilder(); - recordBuilder.reset(ARecordType.FULLY_OPEN_RECORD_TYPE); + recordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE); recordBuilder.init(); List<FieldBinding> fbList = recordValue.getFbList(); HashSet<String> fieldNames = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/pom.xml b/asterixdb/asterix-metadata/pom.xml index c73b1a4..a122189 100644 --- a/asterixdb/asterix-metadata/pom.xml +++ b/asterixdb/asterix-metadata/pom.xml @@ -171,5 +171,21 @@ <version>2.0.2-beta</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> </dependencies> -</project> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java index 588f847..8ecc0ed 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java @@ -101,7 +101,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.dataflow.common.util.TupleUtils; +import org.apache.hyracks.dataflow.common.utils.TupleUtils; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; import org.apache.hyracks.storage.am.common.api.IIndex; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java index a8c3e44..87c1c473 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java @@ -26,7 +26,6 @@ import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; import org.apache.asterix.external.feed.api.IFeed; import org.apache.asterix.external.feed.api.IFeed.FeedType; -import org.apache.asterix.metadata.api.IMetadataEntity; import org.apache.asterix.metadata.entities.CompactionPolicy; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.DatasourceAdapter; @@ -38,6 +37,7 @@ import org.apache.asterix.metadata.entities.Function; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.Library; import org.apache.asterix.metadata.entities.NodeGroup; +import org.apache.asterix.metadata.utils.MetadataUtil; /** * Used to implement serializable transactions against the MetadataCache. @@ -68,7 +68,7 @@ public class MetadataTransactionContext extends MetadataCache { // The APIs in this class make sure that these two caches are kept in sync. protected MetadataCache droppedCache = new MetadataCache(); - protected ArrayList<MetadataLogicalOperation> opLog = new ArrayList<MetadataLogicalOperation>(); + protected ArrayList<MetadataLogicalOperation> opLog = new ArrayList<>(); private final JobId jobId; public MetadataTransactionContext(JobId jobId) { @@ -122,20 +122,20 @@ public class MetadataTransactionContext extends MetadataCache { public void dropDataset(String dataverseName, String datasetName) { Dataset dataset = new Dataset(dataverseName, datasetName, null, null, null, null, null, null, null, null, -1, - IMetadataEntity.PENDING_NO_OP); + MetadataUtil.PENDING_NO_OP); droppedCache.addDatasetIfNotExists(dataset); logAndApply(new MetadataLogicalOperation(dataset, false)); } public void dropIndex(String dataverseName, String datasetName, String indexName) { Index index = new Index(dataverseName, datasetName, indexName, null, null, null, null, false, false, - IMetadataEntity.PENDING_NO_OP); + MetadataUtil.PENDING_NO_OP); droppedCache.addIndexIfNotExists(index); logAndApply(new MetadataLogicalOperation(index, false)); } public void dropDataverse(String dataverseName) { - Dataverse dataverse = new Dataverse(dataverseName, null, IMetadataEntity.PENDING_NO_OP); + Dataverse dataverse = new Dataverse(dataverseName, null, MetadataUtil.PENDING_NO_OP); droppedCache.addDataverseIfNotExists(dataverse); logAndApply(new MetadataLogicalOperation(dataverse, false)); } @@ -236,8 +236,8 @@ public class MetadataTransactionContext extends MetadataCache { public void dropFeed(String dataverseName, String feedName, IFeed.FeedType feedType) { Feed feed = null; - feed = new Feed(dataverseName, feedName, null, feedType, (feedType == FeedType.PRIMARY) ? feedName : null, null, - null); + feed = new Feed(dataverseName, feedName, null, feedType, (feedType == FeedType.PRIMARY) ? feedName : null, + null, null); droppedCache.addFeedIfNotExists(feed); logAndApply(new MetadataLogicalOperation(feed, false)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IIndexDataflowHelperFactoryProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IIndexDataflowHelperFactoryProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IIndexDataflowHelperFactoryProvider.java new file mode 100644 index 0000000..679d6eb --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IIndexDataflowHelperFactoryProvider.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.api; + +import java.util.Map; + +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.metadata.entities.Index; +import org.apache.asterix.om.types.ARecordType; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; + +@FunctionalInterface +public interface IIndexDataflowHelperFactoryProvider { + + /** + * Get the index dataflow helper factory + * + * @param mdProvider + * the system's metadata provider + * @param dataset + * the index dataset + * @param index + * the index + * @param recordType + * the dataset's record type + * @param metaType + * the detaset's meta type + * @param mergePolicyFactory + * the index's merge policy factory + * @param mergePolicyProperties + * the index's merge policy properties + * @param filterTypeTraits + * the dataset's filter type traits + * @param filterCmpFactories + * the dataset's filter comparator factories + * @return the index dataflow helper factory + * @throws AlgebricksException + * if the dataflow helper factory couldn't be created for the index + */ + IIndexDataflowHelperFactory getIndexDataflowHelperFactory(MetadataProvider mdProvider, Dataset dataset, Index index, + ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, + Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, + IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException; +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java index 1759c0f..482cc3b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java @@ -23,12 +23,8 @@ import java.io.Serializable; import org.apache.asterix.metadata.MetadataCache; public interface IMetadataEntity<T> extends Serializable { - public static final int PENDING_NO_OP = 0; - public static final int PENDING_ADD_OP = 1; - public static final int PENDING_DROP_OP = 2; public T addToCache(MetadataCache cache); public T dropFromCache(MetadataCache cache); - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java index 9b55151..5626a83 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.asterix.common.api.IExtension; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider; +import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.exceptions.HyracksDataException; /** @@ -56,6 +57,6 @@ public interface IMetadataExtension extends IExtension { * @throws RemoteException * @throws ACIDException */ - void initializeMetadata() throws HyracksDataException, RemoteException, ACIDException; + void initializeMetadata(INCApplicationContext appCtx) throws HyracksDataException, RemoteException, ACIDException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/INCExtensionManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/INCExtensionManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/INCExtensionManager.java new file mode 100644 index 0000000..62c4273 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/INCExtensionManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.api; + +import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider; + +@FunctionalInterface +public interface INCExtensionManager { + + /** + * @return the metadata tuple translator provider + */ + MetadataTupleTranslatorProvider getMetadataTupleTranslatorProvider(); +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index b64d2ad..15f0a35 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -29,14 +29,11 @@ import java.util.logging.Logger; import org.apache.asterix.common.api.IAppRuntimeContext; import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.asterix.common.cluster.ClusterPartition; -import org.apache.asterix.common.config.MetadataProperties; import org.apache.asterix.common.config.ClusterProperties; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.GlobalConfig; -import org.apache.asterix.common.config.IPropertiesProvider; -import org.apache.asterix.common.context.BaseOperationTracker; +import org.apache.asterix.common.config.MetadataProperties; import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory; -import org.apache.asterix.common.dataflow.LSMIndexUtil; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory; import org.apache.asterix.common.transactions.Resource; @@ -50,7 +47,6 @@ import org.apache.asterix.metadata.IDatasetDetails; import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; -import org.apache.asterix.metadata.api.IMetadataEntity; import org.apache.asterix.metadata.api.IMetadataIndex; import org.apache.asterix.metadata.entities.BuiltinTypeMap; import org.apache.asterix.metadata.entities.CompactionPolicy; @@ -67,8 +63,11 @@ import org.apache.asterix.metadata.entities.Node; import org.apache.asterix.metadata.entities.NodeGroup; import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; import org.apache.asterix.metadata.utils.MetadataConstants; +import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; +import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerFactory; +import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerFactory; import org.apache.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata; import org.apache.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider; import org.apache.asterix.transaction.management.service.transaction.TransactionManagementConstants.LockManagerConstants.LockMode; @@ -80,9 +79,10 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree; -import org.apache.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils; +import org.apache.hyracks.storage.am.lsm.btree.utils.LSMBTreeUtil; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory; @@ -125,8 +125,6 @@ public class MetadataBootstrap { MetadataPrimaryIndexes.LIBRARY_DATASET, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET }; - private static IPropertiesProvider propertiesProvider; - private MetadataBootstrap() { } @@ -141,14 +139,12 @@ public class MetadataBootstrap { * @throws MetadataException * @throws Exception */ - public static void startUniverse(IPropertiesProvider asterixPropertiesProvider, - INCApplicationContext ncApplicationContext, boolean isNewUniverse) + public static void startUniverse(INCApplicationContext ncApplicationContext, boolean isNewUniverse) throws RemoteException, ACIDException, MetadataException { MetadataBootstrap.setNewUniverse(isNewUniverse); runtimeContext = (IAppRuntimeContext) ncApplicationContext.getApplicationObject(); - propertiesProvider = asterixPropertiesProvider; - MetadataProperties metadataProperties = propertiesProvider.getMetadataProperties(); + MetadataProperties metadataProperties = runtimeContext.getMetadataProperties(); metadataNodeName = metadataProperties.getMetadataNodeName(); nodeNames = metadataProperties.getNodeNames(); dataLifecycleManager = runtimeContext.getDatasetLifecycleManager(); @@ -164,7 +160,7 @@ public class MetadataBootstrap { MetadataManager.INSTANCE.lock(mdTxnCtx, LockMode.X); for (int i = 0; i < PRIMARY_INDEXES.length; i++) { - enlistMetadataDataset(PRIMARY_INDEXES[i]); + enlistMetadataDataset(ncApplicationContext, PRIMARY_INDEXES[i]); } if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info( @@ -204,7 +200,7 @@ public class MetadataBootstrap { private static void insertInitialDataverses(MetadataTransactionContext mdTxnCtx) throws MetadataException { String dataFormat = NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT; MetadataManager.INSTANCE.addDataverse(mdTxnCtx, - new Dataverse(MetadataConstants.METADATA_DATAVERSE_NAME, dataFormat, IMetadataEntity.PENDING_NO_OP)); + new Dataverse(MetadataConstants.METADATA_DATAVERSE_NAME, dataFormat, MetadataUtil.PENDING_NO_OP)); MetadataManager.INSTANCE.addDataverse(mdTxnCtx, MetadataBuiltinEntities.DEFAULT_DATAVERSE); } @@ -227,8 +223,7 @@ public class MetadataBootstrap { indexes[i].getDataverseName(), indexes[i].getPayloadRecordType().getTypeName(), indexes[i].getNodeGroupName(), GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME, GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, id, new HashMap<String, String>(), - DatasetType.INTERNAL, indexes[i].getDatasetId().getId(), - IMetadataEntity.PENDING_NO_OP)); + DatasetType.INTERNAL, indexes[i].getDatasetId().getId(), MetadataUtil.PENDING_NO_OP)); } if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Finished inserting initial datasets."); @@ -246,12 +241,10 @@ public class MetadataBootstrap { types.addAll(BuiltinTypeMap.getAllBuiltinTypes()); getMetadataTypes(types); for (int i = 0; i < types.size(); i++) { - MetadataManager.INSTANCE.addDatatype(mdTxnCtx, - new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME, types.get(i).getTypeName(), types.get(i), - false)); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME, + types.get(i).getTypeName(), types.get(i), false)); } - MetadataManager.INSTANCE.addDatatype(mdTxnCtx, - MetadataBuiltinEntities.ANY_OBJECT_DATATYPE); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, MetadataBuiltinEntities.ANY_OBJECT_DATATYPE); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Finished inserting initial datatypes."); } @@ -274,8 +267,7 @@ public class MetadataBootstrap { MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, defaultGroup); } - private static void insertInitialAdapters(MetadataTransactionContext mdTxnCtx) - throws MetadataException { + private static void insertInitialAdapters(MetadataTransactionContext mdTxnCtx) throws MetadataException { String[] builtInAdapterClassNames = new String[] { GenericAdapterFactory.class.getName() }; DatasourceAdapter adapter; for (String adapterClassName : builtInAdapterClassNames) { @@ -306,8 +298,7 @@ public class MetadataBootstrap { } } - private static DatasourceAdapter getAdapter(String adapterFactoryClassName) - throws MetadataException { + private static DatasourceAdapter getAdapter(String adapterFactoryClassName) throws MetadataException { try { String adapterName = ((IAdapterFactory) (Class.forName(adapterFactoryClassName).newInstance())).getAlias(); return new DatasourceAdapter(new AdapterIdentifier(MetadataConstants.METADATA_DATAVERSE_NAME, adapterName), @@ -336,8 +327,9 @@ public class MetadataBootstrap { * @param index * @throws HyracksDataException */ - public static void enlistMetadataDataset(IMetadataIndex index) throws HyracksDataException { - ClusterPartition metadataPartition = propertiesProvider.getMetadataProperties().getMetadataPartition(); + public static void enlistMetadataDataset(INCApplicationContext appCtx, IMetadataIndex index) + throws HyracksDataException { + ClusterPartition metadataPartition = runtimeContext.getMetadataProperties().getMetadataPartition(); int metadataDeviceId = metadataPartition.getIODeviceNum(); String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath( ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId()); @@ -353,29 +345,33 @@ public class MetadataBootstrap { int[] bloomFilterKeyFields = index.getBloomFilterKeyFields(); LSMBTree lsmBtree; long resourceID; - ILSMOperationTracker opTracker = - index.isPrimaryIndex() ? runtimeContext.getLSMBTreeOperationTracker(index.getDatasetId().getId()) - : new BaseOperationTracker(index.getDatasetId().getId(), - dataLifecycleManager.getDatasetInfo(index.getDatasetId().getId())); + // opTrackerProvider and ioOpCallbackFactory should both be acquired through IStorageManager + // We are unable to do this since IStorageManager needs a dataset to determine the appropriate + // objects + ILSMOperationTrackerFactory opTrackerProvider = + index.isPrimaryIndex() ? new PrimaryIndexOperationTrackerFactory(index.getDatasetId().getId()) + : new SecondaryIndexOperationTrackerFactory(index.getDatasetId().getId()); + ILSMIOOperationCallbackFactory ioOpCallbackFactory = LSMBTreeIOOperationCallbackFactory.INSTANCE; if (isNewUniverse()) { - lsmBtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, fileMapProvider, - typeTraits, - comparatorFactories, bloomFilterKeyFields, runtimeContext.getBloomFilterFalsePositiveRate(), - runtimeContext.getMetadataMergePolicyFactory() - .createMergePolicy(GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager), - opTracker, runtimeContext.getLSMIOScheduler(), - LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), index.isPrimaryIndex(), - null, null, null, null, true, LSMIndexUtil.getMetadataPageManagerFactory()); + // TODO(amoudi) Creating the index should be done through the same code path as other indexes + // This is to be done by having a metadata dataset associated with each index + lsmBtree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, fileMapProvider, + typeTraits, comparatorFactories, bloomFilterKeyFields, + runtimeContext.getBloomFilterFalsePositiveRate(), + runtimeContext.getMetadataMergePolicyFactory().createMergePolicy( + GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager), + opTrackerProvider.getOperationTracker(appCtx), runtimeContext.getLSMIOScheduler(), + ioOpCallbackFactory.createIOOperationCallback(), index.isPrimaryIndex(), null, null, null, null, + true, runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory()); lsmBtree.create(); resourceID = index.getResourceID(); - Resource localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits, - comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(), index.getDatasetId().getId(), - metadataPartition.getPartitionId(), - runtimeContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, - null, null, null, null); - ILocalResourceFactoryProvider localResourceFactoryProvider = - new PersistentLocalResourceFactoryProvider(partition -> localResourceMetadata, - LocalResource.LSMBTreeResource); + Resource localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits, comparatorFactories, + bloomFilterKeyFields, index.isPrimaryIndex(), index.getDatasetId().getId(), + metadataPartition.getPartitionId(), runtimeContext.getMetadataMergePolicyFactory(), + GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, null, null, null, null, opTrackerProvider, + ioOpCallbackFactory, runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory()); + ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider( + partition -> localResourceMetadata, LocalResource.LSMBTreeResource); ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory(); localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, resourceName, ITreeIndexFrame.Constants.VERSION, metadataPartition.getPartitionId())); @@ -384,7 +380,7 @@ public class MetadataBootstrap { final LocalResource resource = localResourceRepository.get(file.getRelativePath()); if (resource == null) { throw new HyracksDataException("Could not find required metadata indexes. Please delete " - + propertiesProvider.getMetadataProperties().getTransactionLogDirs() + + runtimeContext.getMetadataProperties().getTransactionLogDirs() .get(runtimeContext.getTransactionSubsystem().getId()) + " to intialize as a new instance. (WARNING: all data will be lost.)"); } @@ -394,15 +390,15 @@ public class MetadataBootstrap { } lsmBtree = (LSMBTree) dataLifecycleManager.get(file.getRelativePath()); if (lsmBtree == null) { - lsmBtree = LSMBTreeUtils.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, - fileMapProvider, - typeTraits, comparatorFactories, bloomFilterKeyFields, + lsmBtree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, bufferCache, + fileMapProvider, typeTraits, comparatorFactories, bloomFilterKeyFields, runtimeContext.getBloomFilterFalsePositiveRate(), runtimeContext.getMetadataMergePolicyFactory().createMergePolicy( GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, dataLifecycleManager), - opTracker, runtimeContext.getLSMIOScheduler(), - LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), index.isPrimaryIndex(), - null, null, null, null, true, LSMIndexUtil.getMetadataPageManagerFactory()); + opTrackerProvider.getOperationTracker(appCtx), runtimeContext.getLSMIOScheduler(), + LSMBTreeIOOperationCallbackFactory.INSTANCE.createIOOperationCallback(), + index.isPrimaryIndex(), null, null, null, null, true, + runtimeContext.getStorageComponentProvider().getMetadataPageManagerFactory()); dataLifecycleManager.register(file.getRelativePath(), lsmBtree); } } @@ -458,7 +454,7 @@ public class MetadataBootstrap { private static void recoverDataverse(MetadataTransactionContext mdTxnCtx, Dataverse dataverse) throws MetadataException { - if (dataverse.getPendingOp() != IMetadataEntity.PENDING_NO_OP) { + if (dataverse.getPendingOp() != MetadataUtil.PENDING_NO_OP) { // drop pending dataverse MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverse.getDataverseName()); if (LOGGER.isLoggable(Level.INFO)) { @@ -474,7 +470,7 @@ public class MetadataBootstrap { } private static void recoverDataset(MetadataTransactionContext mdTxnCtx, Dataset dataset) throws MetadataException { - if (dataset.getPendingOp() != IMetadataEntity.PENDING_NO_OP) { + if (dataset.getPendingOp() != MetadataUtil.PENDING_NO_OP) { // drop pending dataset MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName()); if (LOGGER.isLoggable(Level.INFO)) { @@ -482,36 +478,31 @@ public class MetadataBootstrap { "Dropped a pending dataset: " + dataset.getDataverseName() + "." + dataset.getDatasetName()); } } else { - List<Index> indexes = - MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataset.getDataverseName(), - dataset.getDatasetName()); + List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataset.getDataverseName(), + dataset.getDatasetName()); for (Index index : indexes) { - if (index.getPendingOp() != IMetadataEntity.PENDING_NO_OP) { + if (index.getPendingOp() != MetadataUtil.PENDING_NO_OP) { // drop pending index MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(), index.getIndexName()); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Dropped a pending index: " + dataset.getDataverseName() + "." - + dataset.getDatasetName() - + "." + index.getIndexName()); + + dataset.getDatasetName() + "." + index.getIndexName()); } } } } if (dataset.getDatasetType() == DatasetType.EXTERNAL) { // if the dataset has no indexes, delete all its files - List<Index> indexes = - MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataset.getDataverseName(), - dataset.getDatasetName()); + List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataset.getDataverseName(), + dataset.getDatasetName()); if (indexes.isEmpty()) { - List<ExternalFile> files = - MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); + List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset); for (ExternalFile file : files) { MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Dropped an external file: " + dataset.getDataverseName() + "." - + dataset.getDatasetName() - + "." + file.getFileNumber()); + + dataset.getDatasetName() + "." + file.getFileNumber()); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java index edbbc4a..d796fed 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBuiltinEntities.java @@ -18,26 +18,25 @@ */ package org.apache.asterix.metadata.bootstrap; -import org.apache.asterix.metadata.api.IMetadataEntity; import org.apache.asterix.metadata.entities.Datatype; import org.apache.asterix.metadata.entities.Dataverse; import org.apache.asterix.metadata.utils.MetadataConstants; +import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.utils.RecordUtil; import org.apache.asterix.runtime.formats.NonTaggedDataFormat; public class MetadataBuiltinEntities { //--------------------------------------- Dataverses ----------------------------------------// public static final String DEFAULT_DATAVERSE_NAME = "Default"; public static final Dataverse DEFAULT_DATAVERSE = - new Dataverse(DEFAULT_DATAVERSE_NAME, NonTaggedDataFormat.class.getName(), - IMetadataEntity.PENDING_NO_OP); + new Dataverse(DEFAULT_DATAVERSE_NAME, NonTaggedDataFormat.class.getName(), MetadataUtil.PENDING_NO_OP); //--------------------------------------- Datatypes -----------------------------------------// public static final ARecordType ANY_OBJECT_RECORD_TYPE = new ARecordType("AnyObject", new String[0], new IAType[0], true); - public static final Datatype ANY_OBJECT_DATATYPE = - new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME, ANY_OBJECT_RECORD_TYPE.getTypeName(), - ARecordType.FULLY_OPEN_RECORD_TYPE, false); + public static final Datatype ANY_OBJECT_DATATYPE = new Datatype(MetadataConstants.METADATA_DATAVERSE_NAME, + ANY_OBJECT_RECORD_TYPE.getTypeName(), RecordUtil.FULLY_OPEN_RECORD_TYPE, false); private MetadataBuiltinEntities() { } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/cluster/ClusterManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/cluster/ClusterManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/cluster/ClusterManager.java index fa826de..5bfe876 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/cluster/ClusterManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/cluster/ClusterManager.java @@ -47,7 +47,7 @@ import org.apache.asterix.event.service.ServiceProvider; import org.apache.asterix.event.util.PatternCreator; import org.apache.asterix.installer.schema.conf.Configuration; import org.apache.asterix.metadata.api.IClusterManager; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; public class ClusterManager implements IClusterManager { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java index 0a8e9b1..90c8b1e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/dataset/hints/DatasetHints.java @@ -21,7 +21,7 @@ package org.apache.asterix.metadata.dataset.hints; import java.util.HashSet; import java.util.Set; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.hyracks.algebricks.common.utils.Pair; /**
