Introduce IStorageComponentProvider Change-Id: If86750cdb2436c713f6598e54d4aaaf23d9f7bbf Reviewed-on: https://asterix-gerrit.ics.uci.edu/1451 Tested-by: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> Integration-Tests: Jenkins <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ef9be0f9 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ef9be0f9 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ef9be0f9 Branch: refs/heads/master Commit: ef9be0f9804502205c0bc60a9a07e95f3b849fd9 Parents: 5e17af2 Author: Abdullah Alamoudi <[email protected]> Authored: Wed Feb 1 22:19:54 2017 -0800 Committer: Yingyi Bu <[email protected]> Committed: Thu Feb 2 10:22:33 2017 -0800 ---------------------------------------------------------------------- asterixdb/asterix-algebra/pom.xml | 5 - .../physical/BTreeSearchPOperator.java | 19 +- .../operators/physical/CommitRuntime.java | 12 +- .../physical/ExternalDataLookupPOperator.java | 25 +- .../physical/InvertedIndexPOperator.java | 113 +-- .../rules/ByNameToByIndexFieldAccessRule.java | 2 +- .../rules/CheckFilterExpressionTypeRule.java | 2 +- .../optimizer/rules/ConstantFoldingRule.java | 6 +- .../FullTextContainsParameterCheckRule.java | 2 +- ...eDynamicTypeCastForExternalFunctionRule.java | 2 +- .../rules/IntroduceDynamicTypeCastRule.java | 2 +- ...IntroduceSecondaryIndexInsertDeleteRule.java | 2 +- .../optimizer/rules/LoadRecordFieldsRule.java | 2 +- .../optimizer/rules/PushFieldAccessRule.java | 6 +- .../rules/RemoveUnusedOneToOneEquiJoinRule.java | 4 +- .../optimizer/rules/ResolveVariableRule.java | 2 +- .../optimizer/rules/UnnestToDataScanRule.java | 6 +- .../am/AbstractIntroduceAccessMethodRule.java | 97 +-- .../rules/am/AccessMethodJobGenParams.java | 27 +- .../optimizer/rules/am/AccessMethodUtils.java | 75 +- .../optimizer/rules/am/BTreeAccessMethod.java | 26 +- .../am/IntroduceLSMComponentFilterRule.java | 14 +- .../rules/am/InvertedIndexAccessMethod.java | 2 +- .../rules/am/InvertedIndexJobGenParams.java | 2 +- .../rules/am/OptimizableOperatorSubTree.java | 4 +- .../optimizer/rules/am/RTreeAccessMethod.java | 8 +- .../rules/typecast/StaticTypeCastUtil.java | 2 +- .../translator/AbstractLangTranslator.java | 4 +- .../AqlPlusExpressionToPlanTranslator.java | 4 +- .../asterix/translator/CompiledStatements.java | 107 +-- .../translator/IStatementExecutorFactory.java | 8 +- .../LangExpressionToPlanTranslator.java | 149 ++-- .../asterix/translator/util/ValidateUtil.java | 95 +- asterixdb/asterix-app/pom.xml | 13 - .../apache/asterix/api/common/APIFramework.java | 46 +- .../java/org/apache/asterix/api/common/Job.java | 51 -- .../asterix/api/http/server/ApiServlet.java | 9 +- .../api/http/server/ClusterApiServlet.java | 2 +- .../api/http/server/ConnectorApiServlet.java | 11 +- .../asterix/api/http/server/DdlApiServlet.java | 6 +- .../api/http/server/DiagnosticsApiServlet.java | 2 +- .../asterix/api/http/server/FullApiServlet.java | 6 +- .../server/NodeControllerDetailsApiServlet.java | 2 +- .../api/http/server/QueryApiServlet.java | 6 +- .../api/http/server/QueryServiceServlet.java | 27 +- .../http/server/QueryWebInterfaceServlet.java | 2 +- .../asterix/api/http/server/RestApiServlet.java | 10 +- .../api/http/server/ShutdownApiServlet.java | 2 +- .../api/http/server/UpdateApiServlet.java | 6 +- .../api/http/server/VersionApiServlet.java | 2 +- .../asterix/api/java/AsterixJavaClient.java | 21 +- .../asterix/app/cc/CCExtensionManager.java | 109 +++ .../app/cc/CompilerExtensionManager.java | 128 --- .../app/cc/IStatementExecutorExtension.java | 1 + .../asterix/app/cc/ResourceIdManager.java | 2 +- .../external/ExternalIndexingOperations.java | 771 ---------------- .../app/external/ExternalLibraryUtils.java | 18 +- .../asterix/app/external/FeedOperations.java | 179 ---- .../app/external/FeedWorkCollection.java | 8 +- .../asterix/app/nc/NCAppRuntimeContext.java | 51 +- .../asterix/app/nc/NCExtensionManager.java | 39 +- .../apache/asterix/app/nc/RecoveryManager.java | 868 +++++++++++++++++++ .../asterix/app/nc/TransactionSubsystem.java | 201 +++++ .../asterix/app/result/ResultPrinter.java | 1 + .../apache/asterix/app/result/ResultReader.java | 2 +- .../DefaultStatementExecutorFactory.java | 9 +- .../asterix/app/translator/QueryTranslator.java | 658 ++++++-------- .../org/apache/asterix/drivers/AsterixCLI.java | 3 +- .../asterix/drivers/AsterixClientDriver.java | 3 +- .../apache/asterix/file/DatasetOperations.java | 259 ------ .../asterix/file/DataverseOperations.java | 40 - .../apache/asterix/file/IndexOperations.java | 139 --- .../asterix/file/JobSpecificationUtils.java | 32 - .../file/SecondaryBTreeOperationsHelper.java | 400 --------- .../file/SecondaryIndexOperationsHelper.java | 585 ------------- .../SecondaryInvertedIndexOperationsHelper.java | 383 -------- .../file/SecondaryRTreeOperationsHelper.java | 513 ----------- .../asterix/file/StorageComponentProvider.java | 78 ++ .../bootstrap/CCApplicationEntryPoint.java | 67 +- .../bootstrap/ClusterLifecycleListener.java | 2 +- .../hyracks/bootstrap/ClusterWorkExecutor.java | 2 +- .../bootstrap/GlobalRecoveryManager.java | 52 +- .../apache/asterix/util/FlushDatasetUtils.java | 76 -- .../java/org/apache/asterix/util/JobUtils.java | 50 -- .../org/apache/asterix/util/ResourceUtils.java | 70 -- .../org/apache/asterix/utils/DataverseUtil.java | 46 + .../org/apache/asterix/utils/ExtensionUtil.java | 102 +++ .../apache/asterix/utils/FlushDatasetUtil.java | 77 ++ .../org/apache/asterix/utils/ResourceUtils.java | 70 ++ .../api/http/servlet/ConnectorApiLetTest.java | 20 +- .../api/http/servlet/QueryServiceLetTest.java | 46 - .../http/servlet/QueryServiceServletTest.java | 46 + .../api/http/servlet/VersionApiLetTest.java | 2 +- .../app/bootstrap/TestNodeController.java | 248 +++--- .../aql/translator/QueryTranslatorTest.java | 11 +- .../ConnectorDescriptorWithMessagingTest.java | 18 +- .../asterix/test/dataflow/LogMarkerTest.java | 27 +- .../org/apache/asterix/test/dml/DmlTest.java | 3 +- .../asterix/test/logging/CheckpointingTest.java | 17 +- .../asterix/test/optimizer/OptimizerTest.java | 8 +- .../asterix/test/runtime/ExecutionTestUtil.java | 2 +- .../src/test/resources/metadata/testsuite.xml | 8 +- .../src/test/resources/runtimets/testsuite.xml | 2 +- .../resources/runtimets/testsuite_sqlpp.xml | 2 +- asterixdb/asterix-common/pom.xml | 11 +- .../asterix/common/api/IAppRuntimeContext.java | 59 +- .../common/cluster/IGlobalRecoveryMaanger.java | 29 - .../common/cluster/IGlobalRecoveryManager.java | 29 + .../asterix/common/config/DatasetConfig.java | 42 +- .../context/IStorageComponentProvider.java | 74 ++ .../context/ITransactionSubsystemProvider.java | 6 +- .../context/TransactionSubsystemProvider.java | 4 + .../dataflow/IApplicationContextInfo.java | 30 +- .../asterix/common/dataflow/LSMIndexUtil.java | 6 - .../LSMInsertDeleteOperatorNodePushable.java | 4 +- ...rtedIndexInsertDeleteOperatorDescriptor.java | 16 +- .../LSMTreeInsertDeleteOperatorDescriptor.java | 15 +- .../asterix/common/exceptions/ErrorCode.java | 13 + ...TreeWithBuddyIOOperationCallbackFactory.java | 3 +- ...InvertedIndexIOOperationCallbackFactory.java | 3 +- .../LSMRTreeIOOperationCallbackFactory.java | 2 +- .../transactions/ITransactionManager.java | 2 +- .../transactions/ITransactionSubsystem.java | 24 +- .../asterix/common/transactions/Resource.java | 19 +- .../common/transactions/ResourceFactory.java | 13 +- .../asterix/common/utils/ANameSchema.java | 29 - .../org/apache/asterix/common/utils/Job.java | 51 ++ .../apache/asterix/common/utils/JobUtils.java | 54 ++ .../asterix/common/utils/StoragePathUtil.java | 5 +- .../asterix/common/utils/TransactionUtil.java | 5 + .../main/resources/asx_errormsg/en.properties | 13 + .../api/IExternalDataSourceFactory.java | 4 +- .../dataflow/FeedRecordDataFlowController.java | 4 +- .../external/dataflow/FeedTupleForwarder.java | 4 +- .../feed/management/FeedEventsListener.java | 2 +- .../external/feed/runtime/IngestionRuntime.java | 8 +- .../asterix/external/indexing/ExternalFile.java | 2 +- .../indexing/FilesIndexDescription.java | 15 +- .../external/indexing/IndexingConstants.java | 37 +- .../input/record/RecordWithMetadataAndPK.java | 2 +- .../converter/DCPMessageToRecordConverter.java | 4 +- .../factory/SocketServerInputStreamFactory.java | 2 +- .../external/library/ExternalFunction.java | 2 +- .../external/library/java/JObjectUtil.java | 2 +- ...xternalDatasetIndexesOperatorDescriptor.java | 35 +- .../ExternalBTreeSearchOperatorDescriptor.java | 10 +- .../ExternalDataScanOperatorDescriptor.java | 74 -- ...alDatasetIndexesAbortOperatorDescriptor.java | 16 +- ...lDatasetIndexesCommitOperatorDescriptor.java | 31 +- ...DatasetIndexesRecoverOperatorDescriptor.java | 16 +- .../ExternalFilesIndexOperatorDescriptor.java | 13 +- ...ternalIndexBulkModifyOperatorDescriptor.java | 4 +- .../ExternalLookupOperatorDescriptor.java | 18 +- .../ExternalRTreeSearchOperatorDescriptor.java | 14 +- .../ExternalScanOperatorDescriptor.java | 74 ++ .../operators/FeedMetaComputeNodePushable.java | 4 +- .../operators/FeedMetaStoreNodePushable.java | 4 +- .../operators/IndexInfoOperatorDescriptor.java | 8 +- .../external/parser/DelimitedDataParser.java | 2 +- .../external/parser/HiveRecordParser.java | 2 +- .../apache/asterix/external/util/FeedUtils.java | 2 +- .../apache/asterix/external/util/HDFSUtils.java | 8 +- .../asterix/external/util/NodeResolver.java | 2 +- .../record/reader/kv/KVTestReaderFactory.java | 2 +- .../stream/TwitterFirehoseStreamFactory.java | 2 +- .../asterix/external/library/ClassAdParser.java | 2 +- .../asterix/lang/common/base/Statement.java | 2 +- .../lang/common/util/LangRecordParseUtil.java | 4 +- asterixdb/asterix-metadata/pom.xml | 18 +- .../apache/asterix/metadata/MetadataNode.java | 2 +- .../metadata/MetadataTransactionContext.java | 14 +- .../IIndexDataflowHelperFactoryProvider.java | 65 ++ .../asterix/metadata/api/IMetadataEntity.java | 4 - .../metadata/api/IMetadataExtension.java | 3 +- .../metadata/api/INCExtensionManager.java | 30 + .../metadata/bootstrap/MetadataBootstrap.java | 131 ++- .../bootstrap/MetadataBuiltinEntities.java | 11 +- .../metadata/cluster/ClusterManager.java | 2 +- .../metadata/dataset/hints/DatasetHints.java | 2 +- .../BTreeDataflowHelperFactoryProvider.java | 93 ++ .../metadata/declared/DatasetDataSource.java | 10 +- .../metadata/declared/FeedDataSource.java | 2 +- .../metadata/declared/LoadableDataSource.java | 15 +- .../metadata/declared/MetadataManagerUtil.java | 2 +- .../metadata/declared/MetadataProvider.java | 814 +++++++---------- .../asterix/metadata/entities/Dataset.java | 459 +++++++++- .../entities/ExternalDatasetDetails.java | 41 +- .../apache/asterix/metadata/entities/Index.java | 31 +- .../DatasetTupleTranslator.java | 8 +- .../DatatypeTupleTranslator.java | 2 +- .../IndexTupleTranslator.java | 52 +- .../asterix/metadata/feeds/FeedOperations.java | 182 ++++ .../functions/MetadataBuiltinFunctions.java | 2 +- .../asterix/metadata/utils/DatasetUtil.java | 528 +++++++++++ .../asterix/metadata/utils/DatasetUtils.java | 285 ------ .../utils/ExternalDatasetsRegistry.java | 15 +- .../utils/ExternalIndexingOperations.java | 621 +++++++++++++ .../asterix/metadata/utils/IndexUtil.java | 188 ++++ ...ertedIndexDataflowHelperFactoryProvider.java | 117 +++ .../metadata/utils/KeyFieldTypeUtil.java | 211 +++++ .../metadata/utils/KeyFieldTypeUtils.java | 209 ----- .../metadata/utils/MetadataLockManager.java | 22 +- .../asterix/metadata/utils/MetadataUtil.java | 41 + .../RTreeDataflowHelperFactoryProvider.java | 171 ++++ .../utils/SecondaryBTreeOperationsHelper.java | 355 ++++++++ .../utils/SecondaryIndexOperationsHelper.java | 543 ++++++++++++ .../SecondaryInvertedIndexOperationsHelper.java | 379 ++++++++ .../utils/SecondaryRTreeOperationsHelper.java | 454 ++++++++++ .../utils/SplitsAndConstraintsUtil.java | 22 +- .../apache/asterix/metadata/utils/TypeUtil.java | 156 ++++ .../asterix/builders/AbstractListBuilder.java | 2 +- .../apache/asterix/builders/RecordBuilder.java | 2 +- .../common/AOrderedListBinaryTokenizer.java | 2 +- .../AOrderedListSerializerDeserializer.java | 2 +- .../serde/ARecordSerializerDeserializer.java | 7 +- .../AUnorderedListSerializerDeserializer.java | 2 +- .../org/apache/asterix/om/base/ARecord.java | 3 +- .../om/pointables/AListVisitablePointable.java | 4 +- .../pointables/ARecordVisitablePointable.java | 4 +- .../asterix/om/pointables/cast/AListCaster.java | 2 +- .../om/pointables/cast/ARecordCaster.java | 4 +- .../pointables/nonvisitor/AListPointable.java | 2 +- .../pointables/nonvisitor/ARecordPointable.java | 5 +- .../impl/ClosedRecordConstructorResultType.java | 2 +- .../impl/FieldAccessByIndexResultType.java | 2 +- .../impl/FieldAccessByNameResultType.java | 2 +- .../impl/OpenRecordConstructorResultType.java | 2 +- .../impl/RecordAddFieldsTypeComputer.java | 2 +- .../om/typecomputer/impl/TypeComputeUtils.java | 3 +- .../apache/asterix/om/types/ARecordType.java | 25 +- .../apache/asterix/om/types/TypeTagUtil.java | 3 +- .../asterix/om/util/ConstantExpressionUtil.java | 114 --- .../om/util/JSONDeserializerForTypes.java | 95 -- .../asterix/om/util/NonTaggedFormatUtil.java | 272 ------ .../util/ResettableByteArrayOutputStream.java | 32 - .../om/utils/ConstantExpressionUtil.java | 114 +++ .../om/utils/JSONDeserializerForTypes.java | 95 ++ .../asterix/om/utils/NonTaggedFormatUtil.java | 272 ++++++ .../org/apache/asterix/om/utils/RecordUtil.java | 78 ++ .../utils/ResettableByteArrayOutputStream.java | 32 + .../om/util/JSONDeserializerForTypesTest.java | 1 + .../runtime/evaluators/common/ListAccessor.java | 2 +- .../AnyCollectionMemberDescriptor.java | 2 +- .../evaluators/functions/GetItemDescriptor.java | 2 +- .../records/FieldAccessByIndexEvalFactory.java | 5 +- .../records/FieldAccessByNameEvalFactory.java | 2 +- .../records/FieldAccessNestedEvalFactory.java | 5 +- .../records/GetRecordFieldValueEvalFactory.java | 5 +- .../records/RecordPairsDescriptor.java | 3 +- .../runtime/formats/NonTaggedDataFormat.java | 11 +- .../CompleteFailbackResponseMessage.java | 2 +- ...reparePartitionsFailbackResponseMessage.java | 2 +- .../message/ReportMaxResourceIdMessage.java | 2 +- .../message/ResourceIdRequestMessage.java | 4 +- .../TakeoverMetadataNodeResponseMessage.java | 2 +- .../TakeoverPartitionsResponseMessage.java | 2 +- ...SMInvertedIndexUpsertOperatorDescriptor.java | 10 +- .../LSMPrimaryUpsertOperatorNodePushable.java | 4 +- .../LSMTreeUpsertOperatorDescriptor.java | 10 +- .../std/SubsetCollectionDescriptor.java | 2 +- .../asterix/runtime/util/AppContextInfo.java | 210 ----- .../runtime/util/ClusterStateManager.java | 679 --------------- .../runtime/util/RuntimeComponentsProvider.java | 79 -- .../asterix/runtime/util/RuntimeUtils.java | 68 -- .../asterix/runtime/utils/AppContextInfo.java | 208 +++++ .../runtime/utils/ClusterStateManager.java | 679 +++++++++++++++ .../utils/RuntimeComponentsProvider.java | 79 ++ .../asterix/runtime/utils/RuntimeUtils.java | 72 ++ .../asterix/tools/datagen/AdmDataGen.java | 2 +- .../PrimaryIndexOperationTrackerFactory.java | 45 + .../PrimaryIndexOperationTrackerProvider.java | 45 - .../SecondaryIndexOperationTrackerFactory.java | 45 + .../SecondaryIndexOperationTrackerProvider.java | 45 - .../ExternalBTreeLocalResourceMetadata.java | 32 +- ...ternalBTreeLocalResourceMetadataFactory.java | 13 +- ...rnalBTreeWithBuddyLocalResourceMetadata.java | 46 +- ...eeWithBuddyLocalResourceMetadataFactory.java | 13 +- .../ExternalRTreeLocalResourceMetadata.java | 25 +- ...ternalRTreeLocalResourceMetadataFactory.java | 12 +- .../resource/LSMBTreeLocalResourceMetadata.java | 51 +- .../LSMBTreeLocalResourceMetadataFactory.java | 13 +- .../LSMInvertedIndexLocalResourceMetadata.java | 47 +- ...vertedIndexLocalResourceMetadataFactory.java | 14 +- .../resource/LSMRTreeLocalResourceMetadata.java | 34 +- .../LSMRTreeLocalResourceMetadataFactory.java | 12 +- .../PersistentLocalResourceRepository.java | 9 +- .../management/service/logging/LogBuffer.java | 9 +- .../management/service/logging/LogManager.java | 8 +- .../logging/LogManagerWithReplication.java | 4 +- .../service/recovery/RecoveryManager.java | 863 ------------------ .../AsterixRuntimeComponentsProvider.java | 79 -- .../service/transaction/TransactionContext.java | 4 +- .../service/transaction/TransactionManager.java | 14 +- .../transaction/TransactionSubsystem.java | 200 ----- .../expressions/ExpressionRuntimeProvider.java | 71 ++ ...obGenToExpressionRuntimeProviderAdapter.java | 71 -- .../data/IBinaryComparatorFactoryProvider.java | 16 +- .../algebricks/data/ITypeTraitProvider.java | 6 +- .../piglet/compiler/PigletCompiler.java | 4 +- .../common/io/MessagingFrameTupleAppender.java | 4 +- .../dataflow/common/util/FrameDebugUtils.java | 298 ------- .../dataflow/common/util/SerdeUtils.java | 147 ---- .../hyracks/dataflow/common/util/TaskUtils.java | 77 -- .../dataflow/common/util/TupleUtils.java | 140 --- .../dataflow/common/utils/FrameDebugUtils.java | 298 +++++++ .../dataflow/common/utils/SerdeUtils.java | 147 ++++ .../hyracks/dataflow/common/utils/TaskUtil.java | 77 ++ .../dataflow/common/utils/TupleUtils.java | 140 +++ .../btree/client/InsertPipelineExample.java | 6 +- .../client/PrimaryIndexBulkLoadExample.java | 6 +- .../btree/client/PrimaryIndexSearchExample.java | 6 +- .../client/SecondaryIndexBulkLoadExample.java | 6 +- .../client/SecondaryIndexSearchExample.java | 6 +- .../btree/helper/BTreeHelperStorageManager.java | 56 ++ .../btree/helper/StorageManagerInterface.java | 56 -- .../am/btree/AbstractBTreeOperatorTest.java | 6 +- .../lsm/btree/LSMBTreeOperatorTestHelper.java | 4 +- .../lsm/rtree/LSMRTreeOperatorTestHelper.java | 4 +- ...eWithAntiMatterTuplesOperatorTestHelper.java | 4 +- .../am/rtree/AbstractRTreeOperatorTest.java | 6 +- .../dataflow/BTreeSearchOperatorDescriptor.java | 4 +- .../BTreeUpdateSearchOperatorDescriptor.java | 4 +- .../hyracks/storage/am/btree/impls/BTree.java | 2 +- .../storage/am/btree/impls/BTreeOpContext.java | 2 +- .../am/btree/impls/BTreeRangeSearchCursor.java | 2 +- .../api/IIndexLifecycleManagerProvider.java | 6 +- .../common/api/IMetadataPageManagerFactory.java | 4 + .../am/common/api/IPrimitiveValueProvider.java | 13 +- .../api/IPrimitiveValueProviderFactory.java | 9 +- .../AbstractIndexOperatorDescriptor.java | 8 +- .../AbstractTreeIndexOperatorDescriptor.java | 11 +- .../dataflow/IIndexOperatorDescriptor.java | 4 +- .../dataflow/IndexDropOperatorDescriptor.java | 4 +- .../TreeIndexBulkLoadOperatorDescriptor.java | 4 +- .../TreeIndexCreateOperatorDescriptor.java | 4 +- ...reeIndexDiskOrderScanOperatorDescriptor.java | 4 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 4 +- .../TreeIndexStatsOperatorDescriptor.java | 4 +- ...endOnlyLinkedMetadataPageManagerFactory.java | 5 + .../am/common/ophelpers/IndexOperation.java | 1 + .../storage/am/common/util/TreeIndexUtils.java | 2 +- .../dataflow/ExternalBTreeDataflowHelper.java | 13 +- .../ExternalBTreeDataflowHelperFactory.java | 4 +- .../ExternalBTreeWithBuddyDataflowHelper.java | 16 +- ...rnalBTreeWithBuddyDataflowHelperFactory.java | 4 +- .../btree/dataflow/LSMBTreeDataflowHelper.java | 15 +- .../dataflow/LSMBTreeDataflowHelperFactory.java | 4 +- .../btree/impls/LSMBTreeRangeSearchCursor.java | 2 +- .../am/lsm/btree/util/LSMBTreeUtils.java | 216 ----- .../am/lsm/btree/utils/LSMBTreeUtil.java | 218 +++++ .../api/ILSMIOOperationCallbackFactory.java | 3 +- .../common/api/ILSMIOOperationScheduler.java | 6 +- .../api/ILSMIOOperationSchedulerProvider.java | 6 +- .../common/api/ILSMOperationTrackerFactory.java | 28 + .../api/ILSMOperationTrackerProvider.java | 27 - .../AbstractLSMIndexDataflowHelper.java | 8 +- .../AbstractLSMIndexDataflowHelperFactory.java | 6 +- .../LSMTreeIndexCompactOperatorDescriptor.java | 4 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 4 +- .../impls/NoOpOperationTrackerFactory.java | 70 ++ .../impls/NoOpOperationTrackerProvider.java | 71 -- .../ThreadCountingOperationTrackerFactory.java | 38 + .../ThreadCountingOperationTrackerProvider.java | 39 - ...tractLSMInvertedIndexOperatorDescriptor.java | 4 +- ...InvertedIndexBulkLoadOperatorDescriptor.java | 4 +- .../LSMInvertedIndexCompactOperator.java | 4 +- ...SMInvertedIndexCreateOperatorDescriptor.java | 4 +- .../LSMInvertedIndexDataflowHelper.java | 8 +- .../LSMInvertedIndexDataflowHelperFactory.java | 4 +- ...InvertedIndexInsertUpdateDeleteOperator.java | 4 +- ...SMInvertedIndexSearchOperatorDescriptor.java | 4 +- ...rtitionedLSMInvertedIndexDataflowHelper.java | 8 +- ...edLSMInvertedIndexDataflowHelperFactory.java | 4 +- .../inmemory/InMemoryInvertedListCursor.java | 2 +- .../ondisk/OnDiskInvertedIndex.java | 2 +- .../AbstractLSMRTreeDataflowHelper.java | 9 +- .../dataflow/ExternalRTreeDataflowHelper.java | 6 +- .../ExternalRTreeDataflowHelperFactory.java | 4 +- .../rtree/dataflow/LSMRTreeDataflowHelper.java | 6 +- .../dataflow/LSMRTreeDataflowHelperFactory.java | 4 +- ...RTreeWithAntiMatterTuplesDataflowHelper.java | 4 +- ...thAntiMatterTuplesDataflowHelperFactory.java | 4 +- .../dataflow/RTreeSearchOperatorDescriptor.java | 4 +- .../hyracks/storage/common/IStorageManager.java | 60 ++ .../common/IStorageManagerInterface.java | 37 - ...stractModificationOperationCallbackTest.java | 2 +- .../am/btree/AbstractOperationCallbackTest.java | 2 +- .../AbstractSearchOperationCallbackTest.java | 2 +- .../am/btree/OrderedIndexExamplesTest.java | 2 +- .../am/btree/OrderedIndexMultiThreadTest.java | 2 +- .../am/btree/OrderedIndexTestDriver.java | 2 +- .../storage/am/btree/OrderedIndexTestUtils.java | 2 +- .../storage/am/common/TreeIndexTestUtils.java | 2 +- .../am/rtree/AbstractRTreeExamplesTest.java | 2 +- .../am/rtree/AbstractRTreeMultiThreadTest.java | 2 +- .../am/rtree/AbstractRTreeTestDriver.java | 2 +- .../storage/am/rtree/RTreeTestUtils.java | 2 +- .../test/support/TestStorageManager.java | 50 ++ .../TestStorageManagerComponentHolder.java | 2 +- .../support/TestStorageManagerInterface.java | 50 -- .../storage/am/bloomfilter/BloomFilterTest.java | 2 +- .../MurmurHashForITupleReferenceTest.java | 2 +- .../BTreeModificationOperationCallbackTest.java | 2 +- .../storage/am/btree/BTreeSearchCursorTest.java | 2 +- .../btree/BTreeSearchOperationCallbackTest.java | 2 +- .../storage/am/btree/BTreeUpdateSearchTest.java | 2 +- .../storage/am/btree/StorageFileAccessTest.java | 271 ++++++ .../storage/am/btree/StorageManagerTest.java | 271 ------ .../storage/am/btree/util/BTreeTestContext.java | 2 +- .../am/lsm/btree/LSMBTreeExamplesTest.java | 6 +- ...MBTreeModificationOperationCallbackTest.java | 12 +- .../LSMBTreeSearchOperationCallbackTest.java | 12 +- .../multithread/LSMBTreeMultiThreadTest.java | 4 +- .../am/lsm/btree/perf/BTreePageSizePerf.java | 2 +- .../am/lsm/btree/perf/LSMTreeRunner.java | 4 +- .../am/lsm/btree/perf/PerfExperiment.java | 2 +- .../am/lsm/btree/tuples/LSMBTreeTuplesTest.java | 4 +- .../am/lsm/btree/util/LSMBTreeTestContext.java | 5 +- .../am/lsm/btree/util/LSMBTreeTestHarness.java | 2 +- .../common/LSMInvertedIndexTestHarness.java | 2 +- .../multithread/LSMInvertedIndexTestWorker.java | 2 +- .../util/LSMInvertedIndexTestContext.java | 4 +- .../lsm/rtree/AbstractLSMRTreeExamplesTest.java | 2 +- .../am/lsm/rtree/util/LSMRTreeTestContext.java | 2 +- .../am/lsm/rtree/util/LSMRTreeTestHarness.java | 2 +- ...LSMRTreeWithAntiMatterTuplesTestContext.java | 2 +- .../storage/am/rtree/RTreeSearchCursorTest.java | 2 +- .../am/rtree/utils/RTreeTestContext.java | 2 +- 428 files changed, 12187 insertions(+), 11269 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml index 1209f5a..8588381 100644 --- a/asterixdb/asterix-algebra/pom.xml +++ b/asterixdb/asterix-algebra/pom.xml @@ -153,11 +153,6 @@ </exclusions> </dependency> <dependency> - <groupId>org.apache.asterix</groupId> - <artifactId>asterix-transactions</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.apache.hyracks</groupId> <artifactId>algebricks-compiler</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java index 4db5430..9dd57d5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java @@ -47,7 +47,6 @@ import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty; import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; @@ -98,7 +97,7 @@ public class BTreeSearchPOperator extends IndexSearchPOperator { @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) - throws AlgebricksException { + throws AlgebricksException { AbstractUnnestMapOperator unnestMap = (AbstractUnnestMapOperator) op; ILogicalExpression unnestExpr = unnestMap.getExpressionRef().getValue(); if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { @@ -120,20 +119,12 @@ public class BTreeSearchPOperator extends IndexSearchPOperator { MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); Dataset dataset = metadataProvider.findDataset(jobGenParams.getDataverseName(), jobGenParams.getDatasetName()); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); - List<LogicalVariable> outputVars = unnestMap.getVariables(); - if (jobGenParams.getRetainInput()) { - outputVars = new ArrayList<LogicalVariable>(); - VariableUtilities.getLiveVariables(unnestMap, outputVars); - } - boolean retainNull = false; - if (op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP) { - // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples. - retainNull = true; - } + // By nature, LEFT_OUTER_UNNEST_MAP should generate null values for non-matching tuples. + boolean retainMissing = op.getOperatorTag() == LogicalOperatorTag.LEFT_OUTER_UNNEST_MAP; Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> btreeSearch = metadataProvider.buildBtreeRuntime( - builder.getJobSpec(), outputVars, opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainNull, + builder.getJobSpec(), opSchema, typeEnv, context, jobGenParams.getRetainInput(), retainMissing, dataset, jobGenParams.getIndexName(), lowKeyIndexes, highKeyIndexes, jobGenParams.isLowKeyInclusive(), - jobGenParams.isHighKeyInclusive(), implConfig, minFilterFieldIndexes, maxFilterFieldIndexes); + jobGenParams.isHighKeyInclusive(), minFilterFieldIndexes, maxFilterFieldIndexes); builder.contributeHyracksOperator(unnestMap, btreeSearch.first); builder.contributeAlgebricksPartitionConstraint(btreeSearch.first, btreeSearch.second); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitRuntime.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitRuntime.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitRuntime.java index 8c61300..63a91ac 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitRuntime.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/CommitRuntime.java @@ -41,7 +41,7 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.storage.am.bloomfilter.impls.MurmurHash128Bit; public class CommitRuntime extends AbstractOneInputOneOutputOneFramePushRuntime { @@ -65,8 +65,8 @@ public class CommitRuntime extends AbstractOneInputOneOutputOneFramePushRuntime public CommitRuntime(IHyracksTaskContext ctx, JobId jobId, int datasetId, int[] primaryKeyFields, boolean isTemporaryDatasetWriteJob, boolean isWriteTransaction, int resourcePartition, boolean isSink) { this.ctx = ctx; - IAppRuntimeContext runtimeCtx = (IAppRuntimeContext) ctx.getJobletContext() - .getApplicationContext().getApplicationObject(); + IAppRuntimeContext runtimeCtx = + (IAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject(); this.transactionManager = runtimeCtx.getTransactionSubsystem().getTransactionManager(); this.logMgr = runtimeCtx.getTransactionSubsystem().getLogManager(); this.jobId = jobId; @@ -85,8 +85,8 @@ public class CommitRuntime extends AbstractOneInputOneOutputOneFramePushRuntime try { transactionContext = transactionManager.getTransactionContext(jobId, false); transactionContext.setWriteTxn(isWriteTransaction); - ILogMarkerCallback callback = TaskUtils.<ILogMarkerCallback> get(ILogMarkerCallback.KEY_MARKER_CALLBACK, - ctx); + ILogMarkerCallback callback = + TaskUtil.<ILogMarkerCallback> get(ILogMarkerCallback.KEY_MARKER_CALLBACK, ctx); logRecord = new LogRecord(callback); if (isSink) { return; @@ -126,7 +126,7 @@ public class CommitRuntime extends AbstractOneInputOneOutputOneFramePushRuntime } } } - VSizeFrame message = TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx); + VSizeFrame message = TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx); if (message != null && MessagingFrameTupleAppender.getMessageType(message) == MessagingFrameTupleAppender.MARKER_MESSAGE) { try { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java index 98bf45f..1d79455 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java @@ -18,7 +18,6 @@ */ package org.apache.asterix.algebra.operators.physical; -import java.util.ArrayList; import java.util.List; import org.apache.asterix.metadata.declared.DataSource; @@ -27,7 +26,6 @@ import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.DatasetDataSource; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; @@ -47,7 +45,6 @@ import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceProperties import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractScanPOperator; import org.apache.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty; import org.apache.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator; @@ -63,22 +60,19 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator { private DataSourceId datasetId; private Dataset dataset; private ARecordType recordType; - private Index secondaryIndex; private boolean requiresBroadcast; private boolean retainInput; - private boolean retainNull; + private boolean retainMissing; public ExternalDataLookupPOperator(DataSourceId datasetId, Dataset dataset, ARecordType recordType, - Index secondaryIndex, List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput, - boolean retainNull) { + List<LogicalVariable> ridVarList, boolean requiresBroadcast, boolean retainInput, boolean retainMissing) { this.datasetId = datasetId; this.dataset = dataset; this.recordType = recordType; - this.secondaryIndex = secondaryIndex; this.ridVarList = ridVarList; this.requiresBroadcast = requiresBroadcast; this.retainInput = retainInput; - this.retainNull = retainNull; + this.retainMissing = retainMissing; } public Dataset getDataset() { @@ -137,17 +131,10 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator { } int[] ridIndexes = getKeyIndexes(ridVarList, inputSchemas); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); - List<LogicalVariable> outputVars = new ArrayList<LogicalVariable>(); - if (retainInput) { - VariableUtilities.getLiveVariables(unnestMap, outputVars); - } else { - VariableUtilities.getProducedVariables(unnestMap, outputVars); - } - MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); - Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = metadataProvider - .buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex, ridIndexes, retainInput, - typeEnv, outputVars, opSchema, context, metadataProvider, retainNull); + Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = + metadataProvider.buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, ridIndexes, retainInput, + typeEnv, opSchema, context, metadataProvider, retainMissing); builder.contributeHyracksOperator(unnestMap, externalLoopup.first); builder.contributeAlgebricksPartitionConstraint(externalLoopup.first, externalLoopup.second); ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java index 7d8dcd1..86faa6c 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java @@ -22,31 +22,25 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import org.apache.asterix.common.config.StorageProperties; -import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider; import org.apache.asterix.common.dataflow.IApplicationContextInfo; -import org.apache.asterix.common.dataflow.LSMIndexUtil; -import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory; import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.SearchModifierType; import org.apache.asterix.optimizer.rules.am.InvertedIndexJobGenParams; -import org.apache.asterix.runtime.util.AppContextInfo; -import org.apache.asterix.runtime.util.RuntimeComponentsProvider; -import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider; +import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -76,12 +70,10 @@ import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory; import org.apache.hyracks.data.std.primitive.ShortPointable; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory; +import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory; -import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexSearchOperatorDescriptor; -import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.PartitionedLSMInvertedIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; /** @@ -140,12 +132,12 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { retainNull = true; } // Build runtime. - Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch = buildInvertedIndexRuntime( - metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema, jobGenParams.getRetainInput(), - retainNull, jobGenParams.getDatasetName(), dataset, jobGenParams.getIndexName(), - jobGenParams.getSearchKeyType(), keyIndexes, jobGenParams.getSearchModifierType(), - jobGenParams.getSimilarityThreshold(), minFilterFieldIndexes, maxFilterFieldIndexes, - jobGenParams.getIsFullTextSearch()); + Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> invIndexSearch = + buildInvertedIndexRuntime(metadataProvider, context, builder.getJobSpec(), unnestMapOp, opSchema, + jobGenParams.getRetainInput(), retainNull, jobGenParams.getDatasetName(), dataset, + jobGenParams.getIndexName(), jobGenParams.getSearchKeyType(), keyIndexes, + jobGenParams.getSearchModifierType(), jobGenParams.getSimilarityThreshold(), + minFilterFieldIndexes, maxFilterFieldIndexes, jobGenParams.getIsFullTextSearch()); // Contribute operator in hyracks job. builder.contributeHyracksOperator(unnestMapOp, invIndexSearch.first); @@ -161,12 +153,11 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { SearchModifierType searchModifierType, IAlgebricksConstantValue similarityThreshold, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes, boolean isFullTextSearchQuery) throws AlgebricksException { - try { IAObject simThresh = ((AsterixConstantValue) similarityThreshold).getObject(); IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), dataset.getItemTypeDataverseName(), dataset.getItemTypeName()).getDatatype(); - int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); Index secondaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(), dataset.getDatasetName(), indexName); if (secondaryIndex == null) { @@ -177,8 +168,9 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { List<IAType> secondaryKeyTypeEntries = secondaryIndex.getKeyFieldTypes(); int numSecondaryKeys = secondaryKeyFieldEntries.size(); if (numSecondaryKeys != 1) { - throw new AlgebricksException("Cannot use " + numSecondaryKeys - + " fields as a key for an inverted index. There can be only one field as a key for the inverted index index."); + throw new AlgebricksException( + "Cannot use " + numSecondaryKeys + " fields as a key for an inverted index. " + + "There can be only one field as a key for the inverted index index."); } if (itemType.getTypeTag() != ATypeTag.RECORD) { throw new AlgebricksException("Only record types can be indexed."); @@ -205,8 +197,8 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { } if (isPartitioned) { // The partitioning field is hardcoded to be a short *without* an Asterix type tag. - tokenComparatorFactories[numSecondaryKeys] = PointableBinaryComparatorFactory - .of(ShortPointable.FACTORY); + tokenComparatorFactories[numSecondaryKeys] = + PointableBinaryComparatorFactory.of(ShortPointable.FACTORY); tokenTypeTraits[numSecondaryKeys] = ShortPointable.TYPE_TRAITS; } @@ -221,17 +213,13 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { int start = outputRecDesc.getFieldCount() - numPrimaryKeys; IBinaryComparatorFactory[] invListsComparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(outputVars, start, numPrimaryKeys, typeEnv, context); - ITypeTraits[] invListsTypeTraits = JobGenHelper.variablesToTypeTraits(outputVars, start, numPrimaryKeys, - typeEnv, context); - - ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recordType); - IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset, - recordType, context.getBinaryComparatorFactoryProvider()); - - int[] filterFields = null; - int[] invertedIndexFields = null; - int[] filterFieldsForNonBulkLoadOps = null; - int[] invertedIndexFieldsForNonBulkLoadOps = null; + ITypeTraits[] invListsTypeTraits = + JobGenHelper.variablesToTypeTraits(outputVars, start, numPrimaryKeys, typeEnv, context); + ITypeTraits[] filterTypeTraits = DatasetUtil.computeFilterTypeTraits(dataset, recordType); + int[] filterFields; + int[] invertedIndexFields; + int[] filterFieldsForNonBulkLoadOps; + int[] invertedIndexFieldsForNonBulkLoadOps; if (filterTypeTraits != null) { filterFields = new int[1]; filterFields[0] = numTokenKeys + numPrimaryKeys; @@ -247,50 +235,37 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { invertedIndexFieldsForNonBulkLoadOps[k] = k; } } - IApplicationContextInfo appContext = (IApplicationContextInfo) context.getAppContext(); - Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider - .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName, indexName, + Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = + metadataProvider.getSplitProviderAndConstraints(dataset.getDataverseName(), datasetName, indexName, dataset.getDatasetDetails().isTemp()); // TODO: Here we assume there is only one search key field. int queryField = keyFields[0]; // Get tokenizer and search modifier factories. - IInvertedIndexSearchModifierFactory searchModifierFactory = InvertedIndexAccessMethod - .getSearchModifierFactory(searchModifierType, simThresh, secondaryIndex); + IInvertedIndexSearchModifierFactory searchModifierFactory = + InvertedIndexAccessMethod.getSearchModifierFactory(searchModifierType, simThresh, secondaryIndex); IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod .getBinaryTokenizerFactory(searchModifierType, searchKeyType, secondaryIndex); - IIndexDataflowHelperFactory dataflowHelperFactory; - - StorageProperties storageProperties = AppContextInfo.INSTANCE.getStorageProperties(); - Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils - .getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext()); - boolean temp = dataset.getDatasetDetails().isTemp(); - if (!isPartitioned) { - dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory( - new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first, - compactionInfo.second, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()), - RuntimeComponentsProvider.RUNTIME_PROVIDER, - LSMInvertedIndexIOOperationCallbackFactory.INSTANCE, - storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits, - filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, !temp); - } else { - dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory( - new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first, - compactionInfo.second, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()), - RuntimeComponentsProvider.RUNTIME_PROVIDER, - LSMInvertedIndexIOOperationCallbackFactory.INSTANCE, - storageProperties.getBloomFilterFalsePositiveRate(), invertedIndexFields, filterTypeTraits, - filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, - invertedIndexFieldsForNonBulkLoadOps, !temp); - } + ARecordType metaType = dataset.hasMetaPart() + ? (ARecordType) metadataProvider + .findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()).getType() + : null; + Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = + DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext()); + IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider, + secondaryIndex, recordType, metaType, compactionInfo.first, compactionInfo.second); LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor( - jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first, + jobSpec, queryField, appContext.getStorageManager(), secondarySplitsAndConstraint.first, appContext.getIndexLifecycleManagerProvider(), tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories, dataflowHelperFactory, queryTokenizerFactory, - searchModifierFactory, outputRecDesc, retainInput, retainMissing, context.getMissingWriterFactory(), - NoOpOperationCallbackFactory.INSTANCE, minFilterFieldIndexes, maxFilterFieldIndexes, - LSMIndexUtil.getMetadataPageManagerFactory(), isFullTextSearchQuery); + searchModifierFactory, outputRecDesc, retainInput, retainMissing, + context.getMissingWriterFactory(), + dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(), secondaryIndex, + ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId(), + IndexOperation.SEARCH, null), + minFilterFieldIndexes, maxFilterFieldIndexes, + metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory(), + isFullTextSearchQuery); return new Pair<>(invIndexSearchOp, secondarySplitsAndConstraint.second); } catch (MetadataException e) { throw new AlgebricksException(e); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java index be8a14b..c7d5ad2 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ByNameToByIndexFieldAccessRule.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.Collections; import org.apache.asterix.algebra.base.OperatorAnnotation; -import org.apache.asterix.om.util.ConstantExpressionUtil; import org.apache.asterix.lang.common.util.FunctionUtil; import org.apache.asterix.om.base.AInt32; import org.apache.asterix.om.constants.AsterixConstantValue; @@ -32,6 +31,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java index 1400950..ab359bf 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java @@ -22,7 +22,7 @@ package org.apache.asterix.optimizer.rules; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index 3ef5f2b..98e9f9d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -42,7 +42,7 @@ import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AbstractCollectionType; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -56,7 +56,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExp import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; -import org.apache.hyracks.algebricks.core.algebra.expressions.LogicalExpressionJobGenToExpressionRuntimeProviderAdapter; +import org.apache.hyracks.algebricks.core.algebra.expressions.ExpressionRuntimeProvider; import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression; @@ -126,7 +126,7 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { BinaryHashFunctionFamilyProvider.INSTANCE, BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY, BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null, - new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(QueryLogicalExpressionJobGen.INSTANCE), + new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE), ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null); private static final IOperatorSchema[] _emptySchemas = new IOperatorSchema[] {}; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FullTextContainsParameterCheckRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FullTextContainsParameterCheckRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FullTextContainsParameterCheckRule.java index e589065..20c043a 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FullTextContainsParameterCheckRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/FullTextContainsParameterCheckRule.java @@ -26,7 +26,7 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescriptor; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java index 1b3acea..a813285 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java @@ -27,7 +27,7 @@ import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java index e9eb6e8..535edbe 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java @@ -31,7 +31,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java index 38e5984..949786a 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java @@ -47,7 +47,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java index a0fe4de..7336181 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java @@ -30,7 +30,7 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.asterix.optimizer.base.AnalysisUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java index 73057ae..6ca822a 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java @@ -30,7 +30,7 @@ import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.AInt32; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.base.IAObject; @@ -38,7 +38,7 @@ import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.asterix.optimizer.base.AnalysisUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; @@ -332,7 +332,7 @@ public class PushFieldAccessRule implements IAlgebraicRewriteRule { } fldName = rt.getFieldNames()[pos]; } - int p = DatasetUtils.getPositionOfPartitioningKeyField(dataset, fldName); + int p = DatasetUtil.getPositionOfPartitioningKeyField(dataset, fldName); if (p < 0) { // not one of the partitioning fields setAsFinal(access, context, finalAnnot); return false; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java index 03c7663..a20a33d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java @@ -26,7 +26,7 @@ import java.util.Set; import org.apache.asterix.metadata.declared.DatasetDataSource; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -231,7 +231,7 @@ public class RemoveUnusedOneToOneEquiJoinRule implements IAlgebraicRewriteRule { DatasetDataSource datasetDataSource = (DatasetDataSource) dataScan.getDataSource(); pkVars.clear(); if (datasetDataSource.getDataset().getDatasetDetails() instanceof InternalDatasetDetails) { - int numPKs = DatasetUtils.getPartitioningKeys(datasetDataSource.getDataset()).size(); + int numPKs = DatasetUtil.getPartitioningKeys(datasetDataSource.getDataset()).size(); for (int i = 0; i < numPKs; i++) { pkVars.add(dataScan.getVariables().get(i)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java index f8e57a1..4ae26fc 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ResolveVariableRule.java @@ -34,7 +34,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableObject; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java index cdabeaa..74e6ec5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java @@ -36,14 +36,14 @@ import org.apache.asterix.metadata.entities.Feed; import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; -import org.apache.asterix.metadata.utils.DatasetUtils; +import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.ConstantExpressionUtil; +import org.apache.asterix.om.utils.ConstantExpressionUtil; import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils; import org.apache.asterix.translator.util.PlanTranslationUtil; import org.apache.commons.lang3.mutable.Mutable; @@ -123,7 +123,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { DataSourceId asid = new DataSourceId(dataverseName, datasetName); List<LogicalVariable> variables = new ArrayList<>(); if (dataset.getDatasetType() == DatasetType.INTERNAL) { - int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); for (int i = 0; i < numPrimaryKeys; i++) { variables.add(context.newVar()); }
