http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java index 49ffa28..3c9acf4 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java @@ -42,16 +42,38 @@ public class DatasetConfig { LENGTH_PARTITIONED_NGRAM_INVIX } - public enum ExternalDatasetTransactionState { - COMMIT, // The committed state <- nothing is required-> - BEGIN, // The state after starting the refresh transaction <- will either abort moving to committed state or move to ready to commit-> - READY_TO_COMMIT // The transaction is ready to commit <- can only move forward to committed state-> - }; + public enum TransactionState { + /** + * The committed state <- nothing is required-> + */ + COMMIT, + /** + * The state after starting the refresh transaction + * <- will either abort moving to committed state or move to ready to commit-> + */ + BEGIN, + /** + * The transaction is ready to commit <- can only move forward to committed state-> + */ + READY_TO_COMMIT + } public enum ExternalFilePendingOp { - PENDING_NO_OP, // the stored file is part of a committed transaction nothing is required - PENDING_ADD_OP, // the stored file is part of an ongoing transaction (will be added if transaction succeed) - PENDING_DROP_OP, // the stored file is part of an ongoing transaction (will be dropped if transaction succeed) - PENDING_APPEND_OP // the stored file is part of an ongoing transaction (will be updated if transaction succeed) - }; + /** + * the stored file is part of a committed transaction nothing is required + */ + NO_OP, + /** + * the stored file is part of an ongoing transaction (will be added if transaction succeed) + */ + ADD_OP, + /** + * the stored file is part of an ongoing transaction (will be dropped if transaction succeed) + */ + DROP_OP, + /** + * the stored file is part of an ongoing transaction (will be updated if transaction succeed) + */ + APPEND_OP + } }
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java new file mode 100644 index 0000000..d454349 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.context; + +import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider; +import org.apache.hyracks.algebricks.data.ITypeTraitProvider; +import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider; +import org.apache.hyracks.storage.common.IStorageManager; + +/** + * Responsible for storage components + */ +public interface IStorageComponentProvider { + + /** + * @return {@link org.apache.asterix.common.context.ITransactionSubsystemProvider} instance + */ + ITransactionSubsystemProvider getTransactionSubsystemProvider(); + + /** + * @return {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider} instance + */ + ILSMIOOperationSchedulerProvider getIoOperationSchedulerProvider(); + + /** + * @return the application's root + * {@link org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider} instance + */ + IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider(); + + /** + * @return {@link org.apache.hyracks.storage.common.IStorageManager} instance + */ + IStorageManager getStorageManager(); + + /** + * @return {@link org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory} instance + */ + IMetadataPageManagerFactory getMetadataPageManagerFactory(); + + /** + * @return {@link org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory} instance + */ + IPrimitiveValueProviderFactory getPrimitiveValueProviderFactory(); + + /** + * @return {@link org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider} instance + */ + IBinaryComparatorFactoryProvider getComparatorFactoryProvider(); + + /** + * @return {@link org.apache.hyracks.algebricks.data.ITypeTraitProvider} instance + */ + ITypeTraitProvider getTypeTraitProvider(); +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java index 586fb4d..1dd1845 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java @@ -24,6 +24,10 @@ import java.io.Serializable; import org.apache.asterix.common.transactions.ITransactionSubsystem; import org.apache.hyracks.api.context.IHyracksTaskContext; +/** + * Provides the system's implementation of {@link ITransactionSubsystem} + */ +@FunctionalInterface public interface ITransactionSubsystemProvider extends Serializable { - public ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx); + ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java index 706c303..cfbad43 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java @@ -30,6 +30,10 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; */ public class TransactionSubsystemProvider implements ITransactionSubsystemProvider { private static final long serialVersionUID = 1L; + public static final TransactionSubsystemProvider INSTANCE = new TransactionSubsystemProvider(); + + private TransactionSubsystemProvider() { + } @Override public ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java index b9d547f..eb944a2 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java @@ -18,15 +18,20 @@ */ package org.apache.asterix.common.dataflow; -import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger; +import org.apache.asterix.common.cluster.IGlobalRecoveryManager; import org.apache.asterix.common.library.ILibraryManager; import org.apache.hyracks.api.application.ICCApplicationContext; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; /** - * Provides methods for obtaining the IIndexLifecycleManagerProvider, IStorageManagerInterface and - * ICCApplicationContext implementation. + * Provides methods for obtaining + * {@link org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider}, + * {@link org.apache.hyracks.storage.common.IStorageManager}, + * {@link org.apache.hyracks.api.application.ICCApplicationContext}, + * {@link org.apache.asterix.common.cluster.IGlobalRecoveryManager}, + * and {@link org.apache.asterix.common.library.ILibraryManager} + * at the cluster controller side. */ public interface IApplicationContextInfo { @@ -38,26 +43,23 @@ public interface IApplicationContextInfo { public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider(); /** - * Returns an instance of the implementation for IStorageManagerInterface. - * - * @return IStorageManagerInterface implementation instance + * @return an instance which implements {@link org.apache.hyracks.storage.common.IStorageManager} */ - public IStorageManagerInterface getStorageManagerInterface(); + public IStorageManager getStorageManager(); /** - * Returns an instance of the implementation for ICCApplicationContext. - * - * @return ICCApplicationContext implementation instance + * @return an instance which implements {@link org.apache.hyracks.api.application.ICCApplicationContext} */ public ICCApplicationContext getCCApplicationContext(); /** - * @return the global recovery manager. + * @return the global recovery manager which implements + * {@link org.apache.asterix.common.cluster.IGlobalRecoveryManager} */ - public IGlobalRecoveryMaanger getGlobalRecoveryManager(); + public IGlobalRecoveryManager getGlobalRecoveryManager(); /** - * @return the library manager (at CC side). + * @return the library manager which implements {@link org.apache.asterix.common.library.ILibraryManager} */ public ILibraryManager getLibraryManager(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java index 27bb14f..cd40179 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java @@ -21,8 +21,6 @@ package org.apache.asterix.common.dataflow; import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback; import org.apache.asterix.common.transactions.ILogManager; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; -import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex; @@ -49,8 +47,4 @@ public class LSMIndexUtil { .getIOOperationCallback(); return ioOpCallback.getComponentFileLSNOffset(lsmComponent, componentFilePath); } - - public static IMetadataPageManagerFactory getMetadataPageManagerFactory() { - return new AppendOnlyLinkedMetadataPageManagerFactory(); - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java index 20912ef..ac878ec 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java @@ -33,7 +33,7 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender; import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; @@ -82,7 +82,7 @@ public class LSMInsertDeleteOperatorNodePushable extends LSMIndexInsertUpdateDel try { if (isPrimary && ctx.getSharedObject() != null) { PrimaryIndexLogMarkerCallback callback = new PrimaryIndexLogMarkerCallback(lsmIndex); - TaskUtils.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx); + TaskUtil.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx); } writer.open(); modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback( http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java index 4a4502a..2ff0617 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java @@ -28,14 +28,14 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexInsertUpdateDeleteOperator; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; public class LSMInvertedIndexInsertDeleteOperatorDescriptor extends LSMInvertedIndexInsertUpdateDeleteOperator { @@ -43,27 +43,27 @@ public class LSMInvertedIndexInsertDeleteOperatorDescriptor extends LSMInvertedI private final String indexName; - public LSMInvertedIndexInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, - RecordDescriptor recDesc, IStorageManagerInterface storageManager, IFileSplitProvider fileSplitProvider, + public LSMInvertedIndexInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc, + IStorageManager storageManager, IFileSplitProvider fileSplitProvider, IIndexLifecycleManagerProvider lifecycleManagerProvider, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory, int[] fieldPermutation, IndexOperation op, IIndexDataflowHelperFactory dataflowHelperFactory, ITupleFilterFactory tupleFilterFactory, IModificationOperationCallbackFactory modificationOpCallbackFactory, String indexName, - IMetadataPageManagerFactory metadataPageManagerFactory) { + IPageManagerFactory pageManagerFactory) { super(spec, recDesc, storageManager, fileSplitProvider, lifecycleManagerProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListComparatorFactories, tokenizerFactory, fieldPermutation, op, dataflowHelperFactory, tupleFilterFactory, modificationOpCallbackFactory, - metadataPageManagerFactory); + pageManagerFactory); this.indexName = indexName; } @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { - return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, - recordDescProvider, op, false); + return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider, op, + false); } public String getIndexName() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java index 0f83dd3..b9e7c23 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java @@ -29,14 +29,14 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexInsertUpdateDeleteOperatorDescriptor; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpdateDeleteOperatorDescriptor { @@ -48,19 +48,18 @@ public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpd private final String indexName; public LSMTreeInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc, - IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, + IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, int[] fieldPermutation, IndexOperation op, IIndexDataflowHelperFactory dataflowHelperFactory, ITupleFilterFactory tupleFilterFactory, boolean isPrimary, String indexName, IMissingWriterFactory nullWriterFactory, IModificationOperationCallbackFactory modificationOpCallbackProvider, - ISearchOperationCallbackFactory searchOpCallbackProvider, - IMetadataPageManagerFactory metadataPageManagerFactory) { + ISearchOperationCallbackFactory searchOpCallbackProvider, IPageManagerFactory pageManagerFactory) { super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, bloomFilterKeyFields, fieldPermutation, op, dataflowHelperFactory, tupleFilterFactory, nullWriterFactory, modificationOpCallbackProvider, searchOpCallbackProvider, - metadataPageManagerFactory); + pageManagerFactory); this.isPrimary = isPrimary; this.indexName = indexName; } @@ -68,8 +67,8 @@ public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpd @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { - return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, - recordDescProvider, op, isPrimary); + return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider, op, + isPrimary); } public boolean isPrimary() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index fad00a7..22c080a 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -64,6 +64,19 @@ public class ErrorCode { public static final int COMPILATION_INVALID_PARAMETER_NUMBER = 1008; public static final int COMPILATION_INVALID_RETURNING_EXPRESSION = 1009; public static final int COMPILATION_FULLTEXT_PHRASE_FOUND = 1010; + public static final int COMPILATION_UNKNOWN_DATASET_TYPE = 1011; + public static final int COMPILATION_UNKNOWN_INDEX_TYPE = 1012; + public static final int COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD = 1013; + public static final int COMPILATION_FIELD_NOT_FOUND = 1014; + public static final int COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX = 1015; + public static final int COMPILATION_INDEX_TYPE_NOT_SUPPORTED_FOR_DATASET_TYPE = 1016; + public static final int COMPILATION_FILTER_CANNOT_BE_NULLABLE = 1017; + public static final int COMPILATION_ILLEGAL_FILTER_TYPE = 1018; + public static final int COMPILATION_CANNOT_AUTOGENERATE_COMPOSITE_PRIMARY_KEY = 1019; + public static final int COMPILATION_ILLEGAL_AUTOGENERATED_TYPE = 1020; + public static final int COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE = 1021; + public static final int COMPILATION_ILLEGAL_PRIMARY_KEY_TYPE = 1022; + public static final int COMPILATION_CANT_DROP_ACTIVE_DATASET = 1023; // Feed errors public static final int DATAFLOW_ILLEGAL_STATE = 3001; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java index 7d33340..848de29 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java @@ -25,7 +25,8 @@ public class LSMBTreeWithBuddyIOOperationCallbackFactory implements ILSMIOOperat private static final long serialVersionUID = 1L; - public static LSMBTreeWithBuddyIOOperationCallbackFactory INSTANCE = new LSMBTreeWithBuddyIOOperationCallbackFactory(); + public static final LSMBTreeWithBuddyIOOperationCallbackFactory INSTANCE = + new LSMBTreeWithBuddyIOOperationCallbackFactory(); private LSMBTreeWithBuddyIOOperationCallbackFactory() { } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java index 3b71027..8951cb4 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java @@ -26,7 +26,8 @@ public class LSMInvertedIndexIOOperationCallbackFactory implements ILSMIOOperati private static final long serialVersionUID = 1L; - public static LSMInvertedIndexIOOperationCallbackFactory INSTANCE = new LSMInvertedIndexIOOperationCallbackFactory(); + public static final LSMInvertedIndexIOOperationCallbackFactory INSTANCE = + new LSMInvertedIndexIOOperationCallbackFactory(); private LSMInvertedIndexIOOperationCallbackFactory() { } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java index 6ae833e..954c6e1 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java @@ -26,7 +26,7 @@ public class LSMRTreeIOOperationCallbackFactory implements ILSMIOOperationCallba private static final long serialVersionUID = 1L; - public static LSMRTreeIOOperationCallbackFactory INSTANCE = new LSMRTreeIOOperationCallbackFactory(); + public static final LSMRTreeIOOperationCallbackFactory INSTANCE = new LSMRTreeIOOperationCallbackFactory(); private LSMRTreeIOOperationCallbackFactory() { } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java index af056ae..ebad94d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java @@ -122,7 +122,7 @@ public interface ITransactionManager { * @see ITransactionSubsystem * @return TransactionProvider */ - public ITransactionSubsystem getTransactionProvider(); + public ITransactionSubsystem getTransactionSubsystem(); /** * @return The current max job id. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java index b3a3eba..7bd55e8 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java @@ -18,19 +18,29 @@ */ package org.apache.asterix.common.transactions; +import org.apache.asterix.common.config.TransactionProperties; + +/** + * A transaction subsystem is responsible of maintaining Atomicity, Consistency, Isolation, and Durability + * on a record level. + */ public interface ITransactionSubsystem { - public ILogManager getLogManager(); + ILogManager getLogManager(); + + ILockManager getLockManager(); + + ITransactionManager getTransactionManager(); - public ILockManager getLockManager(); + IRecoveryManager getRecoveryManager(); - public ITransactionManager getTransactionManager(); + IAppRuntimeContextProvider getAsterixAppRuntimeContextProvider(); - public IRecoveryManager getRecoveryManager(); + String getId(); - public IAppRuntimeContextProvider getAsterixAppRuntimeContextProvider(); + ICheckpointManager getCheckpointManager(); - public String getId(); + TransactionProperties getTransactionProperties(); - public ICheckpointManager getCheckpointManager(); + void incrementEntityCommitCount(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java index 0032a70..e53ca7f 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java @@ -21,15 +21,20 @@ package org.apache.asterix.common.transactions; import java.io.Serializable; import java.util.List; +import org.apache.hyracks.api.application.INCApplicationContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.io.IODeviceHandle; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory; import org.apache.hyracks.storage.common.file.LocalResource; /** + * TODO(amoudi): Change this class and its subclasses to use json serialization instead of Java serialization * The base resource that will be written to disk. it will go in the serializable resource * member in {@link LocalResource} */ @@ -41,14 +46,22 @@ public abstract class Resource implements Serializable { protected final ITypeTraits[] filterTypeTraits; protected final IBinaryComparatorFactory[] filterCmpFactories; protected final int[] filterFields; + protected final ILSMOperationTrackerFactory opTrackerProvider; + protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory; + protected final IMetadataPageManagerFactory metadataPageManagerFactory; public Resource(int datasetId, int partition, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) { + IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, + ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory, + IMetadataPageManagerFactory metadataPageManagerFactory) { this.datasetId = datasetId; this.partition = partition; this.filterTypeTraits = filterTypeTraits; this.filterCmpFactories = filterCmpFactories; this.filterFields = filterFields; + this.opTrackerProvider = opTrackerProvider; + this.ioOpCallbackFactory = ioOpCallbackFactory; + this.metadataPageManagerFactory = metadataPageManagerFactory; } public int partition() { @@ -59,8 +72,8 @@ public abstract class Resource implements Serializable { return datasetId; } - public abstract ILSMIndex createIndexInstance(IAppRuntimeContextProvider runtimeContextProvider, - LocalResource resource) throws HyracksDataException; + public abstract ILSMIndex createIndexInstance(INCApplicationContext appCtx, LocalResource resource) + throws HyracksDataException; public static int getIoDeviceNum(IIOManager ioManager, IODeviceHandle deviceHandle) { List<IODeviceHandle> ioDevices = ioManager.getIODevices(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java index f7f7a71..891edd8 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java @@ -20,6 +20,9 @@ package org.apache.asterix.common.transactions; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; +import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory; public abstract class ResourceFactory implements IResourceFactory { @@ -28,12 +31,20 @@ public abstract class ResourceFactory implements IResourceFactory { protected final ITypeTraits[] filterTypeTraits; protected final IBinaryComparatorFactory[] filterCmpFactories; protected final int[] filterFields; + protected final ILSMOperationTrackerFactory opTrackerProvider; + protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory; + protected final IMetadataPageManagerFactory metadataPageManagerFactory; public ResourceFactory(int datasetId, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) { + IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, + ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory, + IMetadataPageManagerFactory metadataPageManagerFactory) { this.datasetId = datasetId; this.filterTypeTraits = filterTypeTraits; this.filterCmpFactories = filterCmpFactories; this.filterFields = filterFields; + this.opTrackerProvider = opTrackerProvider; + this.ioOpCallbackFactory = ioOpCallbackFactory; + this.metadataPageManagerFactory = metadataPageManagerFactory; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java deleted file mode 100644 index 2ce3cca..0000000 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.common.utils; - -/* - * Author: Guangqiang Li - * Created on Nov 30, 2009 - */ -public class ANameSchema { - public static final int LOCALNAME_PROPERTY = 0; - public static final int URI = 1; - public static final int NODE_TYPE = 2; -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java new file mode 100644 index 0000000..88de295 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.utils; + +import org.apache.hyracks.api.job.JobSpecification; + +public class Job { + + public enum SubmissionMode { + SYNCHRONOUS, + ASYNCHRONOUS + } + + private final JobSpecification jobSpec; + private final SubmissionMode submissionMode; + + public Job(JobSpecification jobSpecification, SubmissionMode submissionMode) { + this.jobSpec = jobSpecification; + this.submissionMode = submissionMode; + } + + public Job(JobSpecification jobSpec) { + this.jobSpec = jobSpec; + this.submissionMode = SubmissionMode.SYNCHRONOUS; + } + + public JobSpecification getJobSpec() { + return jobSpec; + } + + public SubmissionMode getSubmissionMode() { + return submissionMode; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java new file mode 100644 index 0000000..9fb6a31 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.common.utils; + +import org.apache.hyracks.api.client.IHyracksClientConnection; +import org.apache.hyracks.api.job.JobId; +import org.apache.hyracks.api.job.JobSpecification; + +public class JobUtils { + + public enum ProgressState { + NO_PROGRESS, + ADDED_PENDINGOP_RECORD_TO_METADATA + } + + public static JobId runJob(IHyracksClientConnection hcc, JobSpecification spec, boolean waitForCompletion) + throws Exception { + JobId[] jobIds = executeJobArray(hcc, new Job[] { new Job(spec) }, waitForCompletion); + return jobIds[0]; + } + + public static JobId[] executeJobArray(IHyracksClientConnection hcc, Job[] jobs, boolean waitForCompletion) + throws Exception { + JobId[] startedJobIds = new JobId[jobs.length]; + for (int i = 0; i < jobs.length; i++) { + JobSpecification spec = jobs[i].getJobSpec(); + spec.setMaxReattempts(0); + JobId jobId = hcc.startJob(spec); + startedJobIds[i] = jobId; + if (waitForCompletion) { + hcc.waitForCompletion(jobId); + } + } + return startedJobIds; + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java index 34af5c3..9f2e3e7 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java @@ -39,6 +39,9 @@ public class StoragePathUtil { public static final String DATASET_INDEX_NAME_SEPARATOR = "_idx_"; public static final String ADAPTER_INSTANCE_PREFIX = "adapter_"; + private StoragePathUtil() { + } + public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints( FileSplit[] splits) { IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits); @@ -67,7 +70,7 @@ public class StoragePathUtil { } private static String prepareFullIndexName(String datasetName, String idxName) { - return (datasetName + DATASET_INDEX_NAME_SEPARATOR + idxName); + return datasetName + DATASET_INDEX_NAME_SEPARATOR + idxName; } public static int getPartitionNumFromName(String name) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java index 2878d5a..4ba35ae 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java @@ -29,6 +29,11 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; public class TransactionUtil { + public static final boolean PROFILE_MODE = false; + + private TransactionUtil() { + } + public static void formJobTerminateLogRecord(ITransactionContext txnCtx, LogRecord logRecord, boolean isCommit) { logRecord.setTxnCtx(txnCtx); TransactionUtil.formJobTerminateLogRecord(logRecord, txnCtx.getJobId().getId(), isCommit); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index f0e98c9..f1e9836 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -49,6 +49,19 @@ 1007 = Invalid expression: function %1$s expects its %2$s input parameter to be a %3$s expression, but the actual expression is %4$s 1008 = Invalid parameter number: function %1$s cannot take %2$s parameters 1010 = Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted +1011 = Unknown dataset type %1$s +1012 = Unknown index type %1$s +1013 = Cannot use %1$s fields as a key for the %2$s index. The index can only support keys of size %3$s +1014 = Field \"%1$s\" is not found +1015 = Index of type %1$s is not supported for dataset \"%2$s\" since it has composite primary keys +1016 = Index of type %1$s is not supported for dataset of type %2$s +1017 = The filter field \"%1$s\" cannot be an optional field +1018 = Field of type %1$s cannot be used as a filter field +1019 = Cannot autogenerate a composite primary key +1020 = Cannot autogenerate a primary key for primary key of type %1$s. Autogenerated primary keys must be of type %2$s +1021 = The primary key field \"%1$s\" cannot be nullable +1022 = Field of type %1$s cannot be used as a primary key field +1023 = Can't drop dataset %1$s since it is connected to active entity: %2$s # Feed Errors 3001 = Illegal state. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java index b35d9c6..e2274b9 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java @@ -25,8 +25,8 @@ import java.util.Map; import java.util.Set; import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.asterix.runtime.util.AppContextInfo; -import org.apache.asterix.runtime.util.ClusterStateManager; +import org.apache.asterix.runtime.utils.AppContextInfo; +import org.apache.asterix.runtime.utils.ClusterStateManager; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.HyracksDataException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java index 859a011..2e687ba 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java @@ -39,7 +39,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.HyracksConstants; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; import org.apache.log4j.Logger; public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowController { @@ -144,7 +144,7 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl ExecutorService executorService = sendMarker ? Executors.newSingleThreadExecutor() : null; if (sendMarker && dataflowMarker == null) { dataflowMarker = new DataflowMarker(recordReader.getProgressReporter(), - TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx)); + TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx)); dataflowMarkerResult = executorService.submit(dataflowMarker); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java index b8d0532..d31e074 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java @@ -33,7 +33,7 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender; import org.apache.hyracks.dataflow.common.comm.util.FrameUtils; import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; public class FeedTupleForwarder implements ITupleForwarder { @@ -59,7 +59,7 @@ public class FeedTupleForwarder implements ITupleForwarder { this.writer = writer; this.appender = new FrameTupleAppender(frame); // Set null feed message - VSizeFrame message = TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx); + VSizeFrame message = TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx); // a null message message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java index 64f0b8d..fa48b58 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java @@ -45,7 +45,7 @@ import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor; import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor; import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor; import org.apache.asterix.external.util.FeedUtils.JobType; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory; import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java index 34237c4..d32a604 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java @@ -29,7 +29,7 @@ import org.apache.hyracks.api.comm.VSizeFrame; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.util.HyracksConstants; -import org.apache.hyracks.dataflow.common.util.TaskUtils; +import org.apache.hyracks.dataflow.common.utils.TaskUtil; public class IngestionRuntime extends SubscribableRuntime { @@ -50,9 +50,9 @@ public class IngestionRuntime extends SubscribableRuntime { dWriter.subscribe(collector); subscribers.add(collectionRuntime); if (numSubscribers == 0) { - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx); - TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, - TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx), collectionRuntime.getCtx()); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx); + TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, + TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx), collectionRuntime.getCtx()); start(); } numSubscribers++; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java index 7acb1f8..2aac497 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java @@ -48,7 +48,7 @@ public class ExternalFile implements Serializable, Comparable<ExternalFile> { this.fileName = ""; this.lastModefiedTime = new Date(); this.size = 0; - this.pendingOp = ExternalFilePendingOp.PENDING_NO_OP; + this.pendingOp = ExternalFilePendingOp.NO_OP; } public ExternalFile(String dataverseName, String datasetName, int fileNumber, String fileName, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java index 3ac8da8..152fa8b 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java @@ -37,12 +37,13 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; @SuppressWarnings("rawtypes") public class FilesIndexDescription { - public final static int FILE_INDEX_TUPLE_SIZE = 2; - public final static int FILE_KEY_INDEX = 0; - public final static int FILE_KEY_SIZE = 1; - public final static int FILE_PAYLOAD_INDEX = 1; - public final static String[] payloadFieldNames = { "FileName", "FileSize", "FileModDate" }; - public final static IAType[] payloadFieldTypes = { BuiltinType.ASTRING, BuiltinType.AINT64, BuiltinType.ADATETIME }; + public static final int FILE_INDEX_TUPLE_SIZE = 2; + public static final int FILE_KEY_INDEX = 0; + public static final int FILE_KEY_SIZE = 1; + public static final int FILE_PAYLOAD_INDEX = 1; + private static final String[] payloadFieldNames = { "FileName", "FileSize", "FileModDate" }; + private static final IAType[] payloadFieldTypes = { BuiltinType.ASTRING, BuiltinType.AINT64, + BuiltinType.ADATETIME }; public static final int[] BLOOM_FILTER_FIELDS = { 0 }; public static final int EXTERNAL_FILE_NAME_FIELD_INDEX = 0; @@ -58,7 +59,7 @@ public class FilesIndexDescription { public final RecordDescriptor FILE_BUDDY_BTREE_RECORD_DESCRIPTOR; public final ISerializerDeserializer[] EXTERNAL_FILE_BUDDY_BTREE_FIELDS = new ISerializerDeserializer[1]; public final ISerializerDeserializer[] EXTERNAL_FILE_TUPLE_FIELDS = new ISerializerDeserializer[FILE_INDEX_TUPLE_SIZE]; - public final IBinaryComparatorFactory[] FILES_INDEX_COMP_FACTORIES = new IBinaryComparatorFactory[] { + public static final IBinaryComparatorFactory[] FILES_INDEX_COMP_FACTORIES = new IBinaryComparatorFactory[] { BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(BuiltinType.AINT32, true) }; public FilesIndexDescription() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java index a269144..86f5aa5 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java @@ -20,6 +20,7 @@ package org.apache.asterix.external.indexing; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -81,7 +82,7 @@ public class IndexingConstants { public static final int RECORD_OFFSET_FIELD_INDEX = 1; public static final int ROW_NUMBER_FIELD_INDEX = 2; - public static final ArrayList<List<String>> RecordIDFields = new ArrayList<List<String>>(); + public static final List<List<String>> RECORD_ID_FIELDS; static { @@ -101,25 +102,32 @@ public class IndexingConstants { rowNumberEvalFactory = new TupleFieldEvaluatorFactory(3); // Add field names - RecordIDFields.add(new ArrayList<String>(Arrays.asList("FileNumber"))); - RecordIDFields.add(new ArrayList<String>(Arrays.asList("RecordOffset"))); - RecordIDFields.add(new ArrayList<String>(Arrays.asList("RowNumber"))); + List<List<String>> ridFields = new ArrayList<>(); + ridFields.add(new ArrayList<>(Arrays.asList("FileNumber"))); + ridFields.add(new ArrayList<>(Arrays.asList("RecordOffset"))); + ridFields.add(new ArrayList<>(Arrays.asList("RowNumber"))); + RECORD_ID_FIELDS = Collections.unmodifiableList(ridFields); + } + + private IndexingConstants() { } // This function returns the size of the RID for the passed file input format public static int getRIDSize(String fileInputFormat) { - if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) + if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) { return 3; - else + } else { return 2; + } } // This function returns the size of the RID for the passed file input format public static IBinaryComparatorFactory[] getComparatorFactories(String fileInputFormat) { - if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) + if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) { return rCFileRIDComparatorFactories; - else + } else { return txtSeqFileRIDComparatorFactories; + } } public static IAType getFieldType(int fieldNumber) throws AsterixException { @@ -197,9 +205,14 @@ public class IndexingConstants { public static List<List<String>> getRIDKeys(Map<String, String> properties) { String fileInputFormat = properties.get(KEY_INPUT_FORMAT); - if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) - return RecordIDFields; - else - return RecordIDFields.subList(0, ROW_NUMBER_FIELD_INDEX); + if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) { + return RECORD_ID_FIELDS; + } else { + return RECORD_ID_FIELDS.subList(0, ROW_NUMBER_FIELD_INDEX); + } + } + + public static String getFilesIndexName(String datasetName) { + return datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java index c2f74e4..ebf1d17 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java @@ -40,7 +40,7 @@ import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java index 38e5fa3..f948c1c 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java @@ -30,9 +30,9 @@ import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.input.record.CharArrayRecord; import org.apache.asterix.external.input.record.RecordWithMetadataAndPK; import org.apache.asterix.external.util.ExternalDataConstants; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.om.utils.RecordUtil; import com.couchbase.client.core.message.dcp.DCPRequest; import com.couchbase.client.core.message.dcp.MutationMessage; @@ -57,7 +57,7 @@ public class DCPMessageToRecordConverter implements IRecordToRecordWithMetadataA public DCPMessageToRecordConverter() { this.value = new CharArrayRecord(); this.recordWithMetadata = new RecordWithMetadataAndPK<>(value, CB_META_TYPES, - ARecordType.FULLY_OPEN_RECORD_TYPE, PK_INDICATOR, PK_INDEXES, PK_TYPES); + RecordUtil.FULLY_OPEN_RECORD_TYPE, PK_INDICATOR, PK_INDEXES, PK_TYPES); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java index aafee8e..5337be1 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java @@ -32,7 +32,7 @@ import org.apache.asterix.external.api.AsterixInputStream; import org.apache.asterix.external.api.IInputStreamFactory; import org.apache.asterix.external.input.stream.SocketServerInputStream; import org.apache.asterix.external.util.ExternalDataConstants; -import org.apache.asterix.runtime.util.RuntimeUtils; +import org.apache.asterix.runtime.utils.RuntimeUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.utils.Pair; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java index 1b89b0c..c80aebc 100755 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java @@ -32,7 +32,7 @@ import org.apache.asterix.om.functions.IExternalFunctionInfo; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.hierachy.ATypeHierarchy; -import org.apache.asterix.runtime.util.AppContextInfo; +import org.apache.asterix.runtime.utils.AppContextInfo; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java index 95e566b..a107421 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java @@ -56,8 +56,8 @@ import org.apache.asterix.om.types.AUnorderedListType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.EnumDeserializer; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.util.NonTaggedFormatUtil; import org.apache.asterix.om.util.container.IObjectPool; +import org.apache.asterix.om.utils.NonTaggedFormatUtil; public class JObjectUtil { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java index 8080f81..d9471d1 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java @@ -32,9 +32,6 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable; import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory; // This is an operator that takes a single file index and an array of secondary indexes // it is intended to be used for @@ -45,27 +42,20 @@ public abstract class AbstractExternalDatasetIndexesOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { private static final long serialVersionUID = 1L; - private ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory; + private IIndexDataflowHelperFactory filesIndexDataflowHelperFactory; private IndexInfoOperatorDescriptor fileIndexInfo; - private List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories; - private List<IndexInfoOperatorDescriptor> bTreeIndexesInfos; - private List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories; - private List<IndexInfoOperatorDescriptor> rTreeIndexesInfos; + private List<IIndexDataflowHelperFactory> treeIndexesDataflowHelperFactories; + private List<IndexInfoOperatorDescriptor> treeIndexesInfos; public AbstractExternalDatasetIndexesOperatorDescriptor(IOperatorDescriptorRegistry spec, - ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory, - IndexInfoOperatorDescriptor fileIndexesInfo, - List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> bTreeIndexesInfos, - List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) { + IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo, + List<IIndexDataflowHelperFactory> treeIndexesDataflowHelperFactories, + List<IndexInfoOperatorDescriptor> indexesInfos) { super(spec, 0, 0); this.filesIndexDataflowHelperFactory = filesIndexDataflowHelperFactory; this.fileIndexInfo = fileIndexesInfo; - this.bTreeIndexesDataflowHelperFactories = bTreeIndexesDataflowHelperFactories; - this.bTreeIndexesInfos = bTreeIndexesInfos; - this.rTreeIndexesDataflowHelperFactories = rTreeIndexesDataflowHelperFactories; - this.rTreeIndexesInfos = rTreeIndexesInfos; + this.treeIndexesDataflowHelperFactories = treeIndexesDataflowHelperFactories; + this.treeIndexesInfos = indexesInfos; } // opening and closing the index is done inside these methods since we don't always need open indexes @@ -87,13 +77,8 @@ public abstract class AbstractExternalDatasetIndexesOperatorDescriptor performOpOnIndex(filesIndexDataflowHelperFactory, ctx, fileIndexInfo, partition); } // perform operation on btrees - for (int i = 0; i < bTreeIndexesDataflowHelperFactories.size(); i++) { - performOpOnIndex(bTreeIndexesDataflowHelperFactories.get(i), ctx, bTreeIndexesInfos.get(i), - partition); - } - // perform operation on rtrees - for (int i = 0; i < rTreeIndexesDataflowHelperFactories.size(); i++) { - performOpOnIndex(rTreeIndexesDataflowHelperFactories.get(i), ctx, rTreeIndexesInfos.get(i), + for (int i = 0; i < treeIndexesDataflowHelperFactories.size(); i++) { + performOpOnIndex(treeIndexesDataflowHelperFactories.get(i), ctx, treeIndexesInfos.get(i), partition); } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java index 5f4e159..ac715a0 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java @@ -30,27 +30,27 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; -import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory; +import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; -import org.apache.hyracks.storage.common.IStorageManagerInterface; +import org.apache.hyracks.storage.common.IStorageManager; public class ExternalBTreeSearchOperatorDescriptor extends BTreeSearchOperatorDescriptor { private static final long serialVersionUID = 1L; public ExternalBTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc, - IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, + IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider, IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainMissing, IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchOpCallbackProvider, - IMetadataPageManagerFactory metadataPageManagerFactory) { + IPageManagerFactory pageManagerFactory) { super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, bloomFilterKeyFields, lowKeyFields, highKeyFields, lowKeyInclusive, highKeyInclusive, dataflowHelperFactory, retainInput, retainMissing, missingWriterFactory, - searchOpCallbackProvider, null, null, metadataPageManagerFactory); + searchOpCallbackProvider, null, null, pageManagerFactory); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java deleted file mode 100644 index aed8bb9..0000000 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.external.operators; - -import org.apache.asterix.external.api.IAdapterFactory; -import org.apache.asterix.external.api.IDataSourceAdapter; -import org.apache.hyracks.api.context.IHyracksTaskContext; -import org.apache.hyracks.api.dataflow.IOperatorNodePushable; -import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; -import org.apache.hyracks.api.dataflow.value.RecordDescriptor; -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor; -import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable; - -/* - * A single activity operator that provides the functionality of scanning data using an - * instance of the configured adapter. - */ -public class ExternalDataScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor { - - private static final long serialVersionUID = 1L; - - private IAdapterFactory adapterFactory; - - public ExternalDataScanOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc, - IAdapterFactory dataSourceAdapterFactory) { - super(spec, 0, 1); - recordDescriptors[0] = rDesc; - this.adapterFactory = dataSourceAdapterFactory; - } - - @Override - public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) - throws HyracksDataException { - - return new AbstractUnaryOutputSourceOperatorNodePushable() { - - @Override - public void initialize() throws HyracksDataException { - IDataSourceAdapter adapter = null; - try { - writer.open(); - adapter = adapterFactory.createAdapter(ctx, partition); - adapter.start(partition, writer); - } catch (Throwable th) { - writer.fail(); - throw new HyracksDataException(th); - } finally { - writer.close(); - } - } - }; - - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java index 8321074..3ce0da8 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java @@ -25,24 +25,18 @@ import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory; -import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory; import org.apache.hyracks.storage.am.lsm.common.impls.AbortRecoverLSMIndexFileManager; -import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory; public class ExternalDatasetIndexesAbortOperatorDescriptor extends AbstractExternalDatasetIndexesOperatorDescriptor { private static final long serialVersionUID = 1L; public ExternalDatasetIndexesAbortOperatorDescriptor(IOperatorDescriptorRegistry spec, - ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory, - IndexInfoOperatorDescriptor fileIndexesInfo, - List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> bTreeIndexesInfos, - List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories, - List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) { - super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, bTreeIndexesDataflowHelperFactories, - bTreeIndexesInfos, rTreeIndexesDataflowHelperFactories, rTreeIndexesInfos); + IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo, + List<IIndexDataflowHelperFactory> indexesDataflowHelperFactories, + List<IndexInfoOperatorDescriptor> indexesInfos) { + super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, indexesDataflowHelperFactories, + indexesInfos); } @Override
