http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IActiveEntityController.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IActiveEntityController.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IActiveEntityController.java new file mode 100644 index 0000000..c73a433 --- /dev/null +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IActiveEntityController.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.metadata.api; + +import java.util.List; + +import org.apache.asterix.active.IActiveEntityEventsListener; +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Dataset; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +public interface IActiveEntityController extends IActiveEntityEventsListener { + + /** + * Start the active entity + * + * @param metadataProvider + * @throws HyracksDataException + * @throws InterruptedException + */ + void start(MetadataProvider metadataProvider) throws HyracksDataException, InterruptedException; + + /** + * Stop the active entity + * + * @param metadataProvider + * @throws HyracksDataException + * @throws InterruptedException + */ + void stop(MetadataProvider metadataProvider) throws HyracksDataException, InterruptedException; + + /** + * Suspend the active entity + * This call stops and freezes the active entity. The calling thread must call resume to release + * locks on the entity + * + * @param metadataProvider + * @throws HyracksDataException + * @throws InterruptedException + */ + void suspend(MetadataProvider metadataProvider) throws HyracksDataException, InterruptedException; + + /** + * Resumes the active entity activity prior to the suspend call + * + * @param metadataProvider + * @throws HyracksDataException + * @throws InterruptedException + */ + void resume(MetadataProvider metadataProvider) throws HyracksDataException, InterruptedException; + + /** + * Start failure recovery + * + * @throws HyracksDataException + */ + void recover() throws HyracksDataException; + + /** + * Add dataset to the list of associated datasets + * + * @param dataset + * the dataset to add + * @throws HyracksDataException + * if the entity is active + */ + void add(Dataset dataset) throws HyracksDataException; + + /** + * Remove dataset to the list of associated datasets + * + * @param dataset + * the dataset to add + * @throws HyracksDataException + * if the entity is active + */ + void remove(Dataset dataset) throws HyracksDataException; + + /** + * @return the list of associated datasets + */ + List<Dataset> getDatasets(); + + /** + * replace the dataset object with the passed updated object + * + * @param target + */ + void replace(Dataset dataset); + +}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java index 9a9f18d..fbeb353 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java @@ -23,7 +23,7 @@ import java.io.IOException; import java.io.Serializable; import java.rmi.RemoteException; -import org.apache.asterix.metadata.MetadataException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java index 57fcc5e..6ad1a20 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java @@ -23,10 +23,10 @@ import java.rmi.RemoteException; import java.util.List; import org.apache.asterix.common.exceptions.ACIDException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.common.metadata.IMetadataBootstrap; import org.apache.asterix.external.indexing.ExternalFile; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.entities.CompactionPolicy; import org.apache.asterix.metadata.entities.Dataset; @@ -671,6 +671,16 @@ public interface IMetadataManager extends IMetadataBootstrap { throws MetadataException; /** + * Upsert an extension entity to its extension dataset under the ongoing metadata transaction + * + * @param mdTxnCtx + * @param entity + * @throws MetadataException + */ + <T extends IExtensionMetadataEntity> void upsertEntity(MetadataTransactionContext mdTxnCtx, T entity) + throws MetadataException; + + /** * Deletes an extension entity from its extension dataset under the ongoing metadata transaction * * @param mdTxnCtx http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java index cfbbda3..78360fd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java @@ -25,10 +25,10 @@ import java.rmi.RemoteException; import java.util.List; import org.apache.asterix.common.exceptions.ACIDException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.external.indexing.ExternalFile; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.entities.CompactionPolicy; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.DatasourceAdapter; @@ -749,6 +749,17 @@ public interface IMetadataNode extends Remote, Serializable { throws MetadataException, RemoteException; /** + * Upserts an extension entity under the ongoing transaction job id + * + * @param jobId + * @param entity + * @throws MetadataException + * @throws RemoteException + */ + <T extends IExtensionMetadataEntity> void upsertEntity(JobId jobId, T entity) + throws MetadataException, RemoteException; + + /** * Deletes an extension entity under the ongoing transaction job id * * @param jobId @@ -781,4 +792,5 @@ public interface IMetadataNode extends Remote, Serializable { List<FeedConnection> getFeedConnections(JobId jobId, String dataverseName, String feedName) throws MetadataException, RemoteException; + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IValueExtractor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IValueExtractor.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IValueExtractor.java index c8db613..c2b61ee 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IValueExtractor.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IValueExtractor.java @@ -22,8 +22,8 @@ package org.apache.asterix.metadata.api; import java.io.IOException; import java.rmi.RemoteException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.transactions.JobId; -import org.apache.asterix.metadata.MetadataException; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java index 93b19f1..dc38749 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java @@ -36,6 +36,7 @@ import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider; import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.exceptions.ACIDException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.external.adapter.factory.GenericAdapterFactory; @@ -44,7 +45,6 @@ import org.apache.asterix.external.api.IDataSourceAdapter; import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.metadata.IDatasetDetails; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.api.IMetadataIndex; @@ -54,7 +54,6 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.DatasourceAdapter; import org.apache.asterix.metadata.entities.Datatype; import org.apache.asterix.metadata.entities.Dataverse; -import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure; @@ -160,7 +159,7 @@ public class MetadataBootstrap { insertNodes(mdTxnCtx); insertInitialGroups(mdTxnCtx); insertInitialAdapters(mdTxnCtx); - insertInitialFeedPolicies(mdTxnCtx); + BuiltinFeedPolicies.insertInitialFeedPolicies(mdTxnCtx); insertInitialCompactionPolicies(mdTxnCtx); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Finished creating metadata B-trees."); @@ -262,15 +261,6 @@ public class MetadataBootstrap { } } - private static void insertInitialFeedPolicies(MetadataTransactionContext mdTxnCtx) throws MetadataException { - for (FeedPolicyEntity feedPolicy : BuiltinFeedPolicies.policies) { - MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy); - } - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("Finished adding built-in feed policies."); - } - } - private static void insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx) throws MetadataException { String[] builtInCompactionPolicyClassNames = new String[] { ConstantMergePolicyFactory.class.getName(), PrefixMergePolicyFactory.class.getName(), @@ -358,7 +348,8 @@ public class MetadataBootstrap { if (index.getResourceId() != resource.getId()) { throw new HyracksDataException("Resource Id doesn't match expected metadata index resource id"); } - IndexDataflowHelper indexHelper = new IndexDataflowHelper(ncServiceCtx, storageComponentProvider.getStorageManager(), file); + IndexDataflowHelper indexHelper = + new IndexDataflowHelper(ncServiceCtx, storageComponentProvider.getStorageManager(), file); indexHelper.open(); // Opening the index through the helper will ensure it gets instantiated indexHelper.close(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java index 49b32c0..c23755d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.asterix.common.config.DatasetConfig.DatasetType; -import org.apache.asterix.metadata.MetadataException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.entities.Dataset; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 68a2448..7d28a06 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -35,6 +35,8 @@ import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.dataflow.LSMTreeInsertDeleteOperatorDescriptor; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.exceptions.MetadataException; +import org.apache.asterix.common.metadata.LockList; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory; @@ -56,7 +58,6 @@ import org.apache.asterix.formats.base.IDataFormat; import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider; import org.apache.asterix.formats.nontagged.LinearizeComparatorFactoryProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetCardinalityHint; @@ -70,8 +71,6 @@ import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.feeds.FeedMetadataUtil; import org.apache.asterix.metadata.lock.ExternalDatasetsRegistry; -import org.apache.asterix.metadata.lock.LockList; -import org.apache.asterix.metadata.lock.MetadataLockManager; import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil; @@ -140,7 +139,7 @@ import org.apache.hyracks.storage.common.IStorageManager; public class MetadataProvider implements IMetadataProvider<DataSourceId, String> { private final ICcApplicationContext appCtx; - private final IStorageComponentProvider storaegComponentProvider; + private final IStorageComponentProvider storageComponentProvider; private final StorageProperties storageProperties; private final Dataverse defaultDataverse; private final LockList locks; @@ -158,11 +157,10 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> private boolean isTemporaryDatasetWriteJob = true; private boolean blockingOperatorDisabled = false; - public MetadataProvider(ICcApplicationContext appCtx, Dataverse defaultDataverse, - IStorageComponentProvider componentProvider) { + public MetadataProvider(ICcApplicationContext appCtx, Dataverse defaultDataverse) { this.appCtx = appCtx; this.defaultDataverse = defaultDataverse; - this.storaegComponentProvider = componentProvider; + this.storageComponentProvider = appCtx.getStorageComponentProvider(); storageProperties = appCtx.getStorageProperties(); locks = new LockList(); } @@ -293,8 +291,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> return null; } String fqName = dv + '.' + dataset; - MetadataLockManager.INSTANCE.acquireDataverseReadLock(locks, dv); - MetadataLockManager.INSTANCE.acquireDatasetReadLock(locks, fqName); + appCtx.getMetadataLockManager().acquireDataverseReadLock(locks, dv); + appCtx.getMetadataLockManager().acquireDatasetReadLock(locks, fqName); return MetadataManagerUtil.findDataset(mdTxnCtx, dv, dataset); } @@ -398,23 +396,23 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime( - JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception { + JobSpecification jobSpec, Feed feed, FeedPolicyAccessor policyAccessor) throws Exception { Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> factoryOutput; - factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx, + factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(feed, policyAccessor, mdTxnCtx, getApplicationContext()); - ARecordType recordType = FeedMetadataUtil.getOutputType(primaryFeed, primaryFeed.getAdapterConfiguration(), + ARecordType recordType = FeedMetadataUtil.getOutputType(feed, feed.getAdapterConfiguration(), ExternalDataConstants.KEY_TYPE_NAME); IAdapterFactory adapterFactory = factoryOutput.first; FeedIntakeOperatorDescriptor feedIngestor = null; switch (factoryOutput.third) { case INTERNAL: - feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, primaryFeed, adapterFactory, recordType, + feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed, adapterFactory, recordType, policyAccessor, factoryOutput.second); break; case EXTERNAL: - String libraryName = primaryFeed.getAdapterName().trim() - .split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[0]; - feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, primaryFeed, libraryName, + String libraryName = + feed.getAdapterName().trim().split(FeedConstants.NamingConstants.LIBRARY_NAME_SEPARATOR)[0]; + feedIngestor = new FeedIntakeOperatorDescriptor(jobSpec, feed, libraryName, adapterFactory.getClass().getName(), recordType, policyAccessor, factoryOutput.second); break; default: @@ -449,7 +447,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } ISearchOperationCallbackFactory searchCallbackFactory = dataset.getSearchCallbackFactory( - storaegComponentProvider, theIndex, jobId, IndexOperation.SEARCH, primaryKeyFields); + storageComponentProvider, theIndex, jobId, IndexOperation.SEARCH, primaryKeyFields); IStorageManager storageManager = getStorageComponentProvider().getStorageManager(); IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory(storageManager, spPc.first); BTreeSearchOperatorDescriptor btreeSearchOp; @@ -493,10 +491,10 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } ISearchOperationCallbackFactory searchCallbackFactory = dataset.getSearchCallbackFactory( - storaegComponentProvider, secondaryIndex, jobId, IndexOperation.SEARCH, primaryKeyFields); + storageComponentProvider, secondaryIndex, jobId, IndexOperation.SEARCH, primaryKeyFields); RTreeSearchOperatorDescriptor rtreeSearchOp; IIndexDataflowHelperFactory indexDataflowHelperFactory = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), spPc.first); + new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), spPc.first); if (dataset.getDatasetType() == DatasetType.INTERNAL) { rtreeSearchOp = new RTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, keyFields, true, true, indexDataflowHelperFactory, retainInput, retainMissing, context.getMissingWriterFactory(), @@ -585,8 +583,8 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> // right callback // (ex. what's the expected behavior when there is an error during // bulkload?) - IIndexDataflowHelperFactory indexHelperFactory = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), splitsAndConstraint.first); + IIndexDataflowHelperFactory indexHelperFactory = new IndexDataflowHelperFactory( + storageComponentProvider.getStorageManager(), splitsAndConstraint.first); TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, null, fieldPermutation, GlobalConfig.DEFAULT_TREE_FILL_FACTOR, false, numElementsHint, true, indexHelperFactory); @@ -804,11 +802,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> dataset.getDatasetName(), fileIndexName); // Create the file index data flow helper IIndexDataflowHelperFactory indexDataflowHelperFactory = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), spPc.first); + new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), spPc.first); // Create the out record descriptor, appContext and fileSplitProvider for the files index RecordDescriptor outRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); ISearchOperationCallbackFactory searchOpCallbackFactory = dataset - .getSearchCallbackFactory(storaegComponentProvider, fileIndex, jobId, IndexOperation.SEARCH, null); + .getSearchCallbackFactory(storageComponentProvider, fileIndex, jobId, IndexOperation.SEARCH, null); // Create the operator ExternalLookupOperatorDescriptor op = new ExternalLookupOperatorDescriptor(jobSpec, adapterFactory, outRecDesc, indexDataflowHelperFactory, searchOpCallbackFactory, @@ -865,7 +863,6 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> context.getMissingWriterFactory()); } - public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime( JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory, IDataFormat format) throws AlgebricksException { @@ -985,9 +982,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> primaryKeyFields[i] = i; } IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory( - storaegComponentProvider, primaryIndex, jobId, indexOp, primaryKeyFields); - IIndexDataflowHelperFactory idfh = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), splitsAndConstraint.first); + storageComponentProvider, primaryIndex, jobId, indexOp, primaryKeyFields); + IIndexDataflowHelperFactory idfh = new IndexDataflowHelperFactory( + storageComponentProvider.getStorageManager(), splitsAndConstraint.first); IOperatorDescriptor op; if (bulkload) { long numElementsHint = getCardinalityPerPartitionHint(dataset); @@ -1119,9 +1116,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> // prepare callback JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId(); IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory( - storaegComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); - IIndexDataflowHelperFactory idfh = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), splitsAndConstraint.first); + storageComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); + IIndexDataflowHelperFactory idfh = new IndexDataflowHelperFactory( + storageComponentProvider.getStorageManager(), splitsAndConstraint.first); IOperatorDescriptor op; if (bulkload) { long numElementsHint = getCardinalityPerPartitionHint(dataset); @@ -1221,9 +1218,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> // prepare callback JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId(); IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory( - storaegComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); - IIndexDataflowHelperFactory indexDataflowHelperFactory = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), splitsAndConstraint.first); + storageComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); + IIndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory( + storageComponentProvider.getStorageManager(), splitsAndConstraint.first); IOperatorDescriptor op; if (bulkload) { long numElementsHint = getCardinalityPerPartitionHint(dataset); @@ -1337,9 +1334,9 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> // prepare callback JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId(); IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory( - storaegComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); - IIndexDataflowHelperFactory indexDataFlowFactory = - new IndexDataflowHelperFactory(storaegComponentProvider.getStorageManager(), splitsAndConstraint.first); + storageComponentProvider, secondaryIndex, jobId, indexOp, modificationCallbackPrimaryKeyFields); + IIndexDataflowHelperFactory indexDataFlowFactory = new IndexDataflowHelperFactory( + storageComponentProvider.getStorageManager(), splitsAndConstraint.first); IOperatorDescriptor op; if (bulkload) { long numElementsHint = getCardinalityPerPartitionHint(dataset); @@ -1562,7 +1559,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } public IStorageComponentProvider getStorageComponentProvider() { - return storaegComponentProvider; + return storageComponentProvider; } public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> getSplitProviderAndConstraints(Dataset ds) http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/BuiltinTypeMap.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/BuiltinTypeMap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/BuiltinTypeMap.java index fd55815..1f0a08e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/BuiltinTypeMap.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/BuiltinTypeMap.java @@ -25,8 +25,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.transactions.JobId; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataNode; import org.apache.asterix.om.types.AUnionType; import org.apache.asterix.om.types.BuiltinType; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java index 020ff6c..fa60bba 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java @@ -27,8 +27,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.IntStream; -import org.apache.asterix.active.ActiveLifecycleListener; import org.apache.asterix.active.IActiveEntityEventsListener; +import org.apache.asterix.active.IActiveNotificationHandler; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory; import org.apache.asterix.common.context.IStorageComponentProvider; @@ -40,8 +40,8 @@ import org.apache.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallb import org.apache.asterix.common.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory; import org.apache.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory; import org.apache.asterix.common.metadata.IDataset; -import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.common.transactions.IRecoveryManager.ResourceType; +import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.common.utils.JobUtils; import org.apache.asterix.common.utils.JobUtils.ProgressState; import org.apache.asterix.external.feed.management.FeedConnectionId; @@ -57,7 +57,6 @@ import org.apache.asterix.metadata.api.IMetadataEntity; import org.apache.asterix.metadata.declared.BTreeResourceFactoryProvider; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.lock.ExternalDatasetsRegistry; -import org.apache.asterix.metadata.lock.MetadataLockManager; import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.ExternalIndexingOperations; import org.apache.asterix.metadata.utils.IndexUtil; @@ -67,6 +66,7 @@ import org.apache.asterix.metadata.utils.RTreeResourceFactoryProvider; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.RecordUtil; +import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback.Operation; import org.apache.asterix.transaction.management.opcallbacks.LockThenSearchOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexModificationOperationCallbackFactory; @@ -77,7 +77,6 @@ import org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexSearc import org.apache.asterix.transaction.management.opcallbacks.TempDatasetPrimaryIndexModificationOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.TempDatasetSecondaryIndexModificationOperationCallbackFactory; import org.apache.asterix.transaction.management.opcallbacks.UpsertOperationCallbackFactory; -import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback.Operation; import org.apache.asterix.transaction.management.resource.DatasetLocalResourceFactory; import org.apache.asterix.transaction.management.runtime.CommitRuntimeFactory; import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory; @@ -197,10 +196,12 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { this.rebalanceCount = rebalanceCount; } + @Override public String getDataverseName() { return dataverseName; } + @Override public String getDatasetName() { return datasetName; } @@ -316,9 +317,9 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>(); if (getDatasetType() == DatasetType.INTERNAL) { // prepare job spec(s) that would disconnect any active feeds involving the dataset. - ActiveLifecycleListener activeListener = - (ActiveLifecycleListener) metadataProvider.getApplicationContext().getActiveLifecycleListener(); - IActiveEntityEventsListener[] activeListeners = activeListener.getNotificationHandler().getEventListeners(); + IActiveNotificationHandler activeListener = (IActiveNotificationHandler) metadataProvider + .getApplicationContext().getActiveNotificationHandler(); + IActiveEntityEventsListener[] activeListeners = activeListener.getEventListeners(); for (IActiveEntityEventsListener listener : activeListeners) { if (listener.isEntityUsingDataset(this)) { throw new CompilationException(ErrorCode.COMPILATION_CANT_DROP_ACTIVE_DATASET, @@ -403,7 +404,8 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { // Drops the associated nodegroup if it is no longer used by any other dataset. if (dropCorrespondingNodeGroup) { - MetadataLockManager.INSTANCE.acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroupName); + metadataProvider.getApplicationContext().getMetadataLockManager() + .acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroupName); MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx.getValue(), nodeGroupName, true); } } @@ -697,8 +699,8 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { public RecordDescriptor getPrimaryRecordDescriptor(MetadataProvider metadataProvider) throws AlgebricksException { List<List<String>> partitioningKeys = getPrimaryKeys(); int numPrimaryKeys = partitioningKeys.size(); - ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 - + (hasMetaPart() ? 1 : 0)]; + ISerializerDeserializer[] primaryRecFields = + new ISerializerDeserializer[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider(); List<Integer> indicators = null; @@ -710,9 +712,9 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { // Set the serde/traits for primary keys for (int i = 0; i < numPrimaryKeys; i++) { - IAType keyType = (indicators == null || indicators.get(i) == 0) - ? itemType.getSubFieldType(partitioningKeys.get(i)) - : metaType.getSubFieldType(partitioningKeys.get(i)); + IAType keyType = + (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) + : metaType.getSubFieldType(partitioningKeys.get(i)); primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType); primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType); } @@ -722,8 +724,8 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); if (hasMetaPart()) { // Set the serde and traits for the meta record field - primaryRecFields[numPrimaryKeys + 1] = SerializerDeserializerProvider.INSTANCE - .getSerializerDeserializer(metaType); + primaryRecFields[numPrimaryKeys + 1] = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType); primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType); } return new RecordDescriptor(primaryRecFields, primaryTypeTraits); @@ -749,9 +751,9 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); } for (int i = 0; i < numPrimaryKeys; i++) { - IAType keyType = (indicators == null || indicators.get(i) == 0) - ? recordType.getSubFieldType(partitioningKeys.get(i)) - : metaType.getSubFieldType(partitioningKeys.get(i)); + IAType keyType = + (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) + : metaType.getSubFieldType(partitioningKeys.get(i)); cmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true); } return cmpFactories; @@ -802,8 +804,8 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { // Gets an array of partition numbers for this dataset. protected int[] getDatasetPartitions(MetadataProvider metadataProvider) throws AlgebricksException { - FileSplit[] splitsForDataset = metadataProvider.splitsForIndex(metadataProvider.getMetadataTxnContext(), this, - getDatasetName()); + FileSplit[] splitsForDataset = + metadataProvider.splitsForIndex(metadataProvider.getMetadataTxnContext(), this, getDatasetName()); return IntStream.range(0, splitsForDataset.length).toArray(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java index b05e61e..151c9ca 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/FeedConnection.java @@ -24,9 +24,7 @@ import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.metadata.MetadataCache; import org.apache.asterix.metadata.api.IMetadataEntity; -import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; -import java.util.ArrayList; import java.util.List; /** http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java index 57f0a07..bf19b03 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java @@ -275,4 +275,9 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index> { throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, indexType); } } + + @Override + public String toString() { + return dataverseName + '.' + datasetName + '.' + indexName; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java index 2ca7215..3c409da 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java @@ -23,8 +23,8 @@ import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.CompactionPolicy; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java index b9464ed..8da34c0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java @@ -37,9 +37,9 @@ import org.apache.asterix.builders.RecordBuilder; import org.apache.asterix.builders.UnorderedListBuilder; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.common.config.DatasetConfig.TransactionState; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.metadata.IDatasetDetails; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Dataset; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java index 7f4e28d..9cd73d7 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java @@ -24,10 +24,10 @@ import java.io.DataInput; import java.io.DataInputStream; import java.util.Calendar; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.external.api.IDataSourceAdapter; import org.apache.asterix.external.dataset.adapter.AdapterIdentifier; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.DatasourceAdapter; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java index 44b5548..fe0b40d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java @@ -29,9 +29,9 @@ import java.util.Calendar; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.builders.OrderedListBuilder; import org.apache.asterix.builders.RecordBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataNode; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java index 89aaad3..4b20a5b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java @@ -24,8 +24,8 @@ import java.io.DataInput; import java.io.DataInputStream; import java.util.Calendar; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Dataverse; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java index ea04f1d..a29048e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java @@ -24,9 +24,9 @@ import java.io.DataInputStream; import java.util.Date; import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.external.indexing.ExternalFile; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.om.base.ADateTime; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java index 420e4fc..2a2387d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedConnectionTupleTranslator.java @@ -27,9 +27,9 @@ import java.util.List; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.builders.UnorderedListBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.FeedConnection; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java index 7b631b6..8ddac7a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java @@ -29,8 +29,8 @@ import java.util.Map; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.builders.RecordBuilder; import org.apache.asterix.builders.UnorderedListBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.FeedPolicyEntity; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java index 4737d79..2541e97 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java @@ -30,8 +30,8 @@ import java.util.Map; import org.apache.asterix.builders.IARecordBuilder; import org.apache.asterix.builders.RecordBuilder; import org.apache.asterix.builders.UnorderedListBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Feed; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java index 7cbc6ac..eed082c 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java @@ -26,8 +26,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.asterix.builders.OrderedListBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Function; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java index 7b1d9a6..fec4ddc 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java @@ -29,9 +29,9 @@ import java.util.List; import org.apache.asterix.builders.OrderedListBuilder; import org.apache.asterix.common.config.DatasetConfig.IndexType; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataNode; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java index 1d8cff8..9c2b824 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java @@ -24,8 +24,8 @@ import java.io.DataInput; import java.io.DataInputStream; import java.util.Calendar; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Library; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java index c2cdf0d..5329df4 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java @@ -27,8 +27,8 @@ import java.util.Calendar; import java.util.List; import org.apache.asterix.builders.UnorderedListBuilder; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.NodeGroup; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java index dae11bc..b1847fd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java @@ -19,8 +19,8 @@ package org.apache.asterix.metadata.entitytupletranslators; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; import org.apache.asterix.metadata.entities.Node; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java index dfa00ab..41b356a 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java @@ -20,13 +20,20 @@ package org.apache.asterix.metadata.feeds; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.external.feed.policy.FeedPolicyAccessor; +import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.utils.MetadataConstants; public class BuiltinFeedPolicies { + private static final Logger LOGGER = Logger.getLogger(BuiltinFeedPolicies.class.getName()); + public static final FeedPolicyEntity BASIC = initializeBasicPolicy(); public static final FeedPolicyEntity ADVANCED_FT_DISCARD = initializeAdvancedFTDiscardPolicy(); @@ -35,15 +42,18 @@ public class BuiltinFeedPolicies { public static final FeedPolicyEntity ELASTIC = initializeAdvancedFTElasticPolicy(); - public static final FeedPolicyEntity[] policies = new FeedPolicyEntity[] { BASIC, ADVANCED_FT_DISCARD, - ADVANCED_FT_SPILL, ELASTIC }; + private static final FeedPolicyEntity[] POLICIES = + new FeedPolicyEntity[] { BASIC, ADVANCED_FT_DISCARD, ADVANCED_FT_SPILL, ELASTIC }; public static final FeedPolicyEntity DEFAULT_POLICY = BASIC; public static final String CONFIG_FEED_POLICY_KEY = "policy"; + private BuiltinFeedPolicies() { + } + public static FeedPolicyEntity getFeedPolicy(String policyName) { - for (FeedPolicyEntity policy : policies) { + for (FeedPolicyEntity policy : POLICIES) { if (policy.getPolicyName().equalsIgnoreCase(policyName)) { return policy; } @@ -53,8 +63,8 @@ public class BuiltinFeedPolicies { //Basic private static FeedPolicyEntity initializeBasicPolicy() { - Map<String, String> policyParams = new HashMap<String, String>(); - policyParams.put(FeedPolicyAccessor.ELASTIC, "false"); + Map<String, String> policyParams = new HashMap<>(); + policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); String description = "Basic"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Basic", description, policyParams); @@ -62,40 +72,47 @@ public class BuiltinFeedPolicies { // Discard private static FeedPolicyEntity initializeAdvancedFTDiscardPolicy() { - Map<String, String> policyParams = new HashMap<String, String>(); - policyParams.put(FeedPolicyAccessor.ELASTIC, "false"); - policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, "true"); - policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, "false"); + Map<String, String> policyParams = new HashMap<>(); + policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); + policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); + policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, Boolean.toString(false)); policyParams.put(FeedPolicyAccessor.MAX_FRACTION_DISCARD, "100"); - policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, "true"); + policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, Boolean.toString(true)); String description = "FlowControl 100% Discard during congestion"; - return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Discard", description, - policyParams); + return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Discard", description, policyParams); } // Spill private static FeedPolicyEntity initializeAdvancedFTSpillPolicy() { - Map<String, String> policyParams = new HashMap<String, String>(); - policyParams.put(FeedPolicyAccessor.ELASTIC, "false"); - policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, "true"); + Map<String, String> policyParams = new HashMap<>(); + policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(false)); + policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); policyParams.put(FeedPolicyAccessor.SPILL_TO_DISK_ON_CONGESTION, "" + Boolean.TRUE); - policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, "" + FeedPolicyAccessor.NO_LIMIT); + policyParams.put(FeedPolicyAccessor.MAX_SPILL_SIZE_ON_DISK, "" + Long.toString(FeedPolicyAccessor.NO_LIMIT)); String description = "FlowControl 100% Spill during congestion"; - return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Spill", description, - policyParams); + return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "Spill", description, policyParams); } // AdvancedFT_Elastic private static FeedPolicyEntity initializeAdvancedFTElasticPolicy() { - Map<String, String> policyParams = new HashMap<String, String>(); - policyParams.put(FeedPolicyAccessor.ELASTIC, "true"); - policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, "true"); - policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, "true"); + Map<String, String> policyParams = new HashMap<>(); + policyParams.put(FeedPolicyAccessor.ELASTIC, Boolean.toString(true)); + policyParams.put(FeedPolicyAccessor.FLOWCONTROL_ENABLED, Boolean.toString(true)); + policyParams.put(FeedPolicyAccessor.LOGGING_STATISTICS, Boolean.toString(true)); String description = "Basic Monitored Fault-Tolerant Elastic"; return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Elastic", description, policyParams); } + public static void insertInitialFeedPolicies(MetadataTransactionContext mdTxnCtx) throws MetadataException { + for (FeedPolicyEntity feedPolicy : BuiltinFeedPolicies.POLICIES) { + MetadataManager.INSTANCE.addFeedPolicy(mdTxnCtx, feedPolicy); + } + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info("Finished adding built-in feed policies."); + } + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java index 94b36ed..3f9c002 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java @@ -27,6 +27,7 @@ import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.external.api.IAdapterFactory; import org.apache.asterix.external.api.IDataSourceAdapter; import org.apache.asterix.external.api.IDataSourceAdapter.AdapterType; @@ -36,7 +37,6 @@ import org.apache.asterix.external.provider.AdapterFactoryProvider; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; -import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.MetadataProvider; @@ -59,8 +59,11 @@ import org.apache.hyracks.api.dataflow.value.RecordDescriptor; */ public class FeedMetadataUtil { + private FeedMetadataUtil() { + } + public static Dataset validateIfDatasetExists(MetadataProvider metadataProvider, String dataverse, - String datasetName, MetadataTransactionContext ctx) throws AlgebricksException { + String datasetName) throws AlgebricksException { Dataset dataset = metadataProvider.findDataset(dataverse, datasetName); if (dataset == null) { throw new CompilationException("Unknown target dataset :" + datasetName); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java index 9cc9d4a..c0d9bda 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.asterix.metadata.MetadataException; +import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.entities.Datatype; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java index 542dd72..31f2089 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/DatasetLock.java @@ -21,65 +21,83 @@ package org.apache.asterix.metadata.lock; import java.util.Objects; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.asterix.om.base.AMutableInt32; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.exceptions.MetadataException; +import org.apache.asterix.common.metadata.IMetadataLock; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; public class DatasetLock implements IMetadataLock { private final String key; - private final ReentrantReadWriteLock dsLock; + private final ReentrantReadWriteLock lock; + private final ReentrantReadWriteLock dsReadLock; private final ReentrantReadWriteLock dsModifyLock; - private final AMutableInt32 indexBuildCounter; + private final MutableInt indexBuildCounter; public DatasetLock(String key) { this.key = key; - dsLock = new ReentrantReadWriteLock(true); + lock = new ReentrantReadWriteLock(true); + dsReadLock = new ReentrantReadWriteLock(true); dsModifyLock = new ReentrantReadWriteLock(true); - indexBuildCounter = new AMutableInt32(0); + indexBuildCounter = new MutableInt(0); } - private void acquireReadLock() { + private void readLock() { // query // build index // insert - dsLock.readLock().lock(); + lock.readLock().lock(); } - private void releaseReadLock() { + private void readUnlock() { // query // build index // insert - dsLock.readLock().unlock(); + lock.readLock().unlock(); } - private void acquireWriteLock() { + private void writeLock() { // create ds // delete ds // drop index - dsLock.writeLock().lock(); + lock.writeLock().lock(); } - private void releaseWriteLock() { + private void writeUnlock() { // create ds // delete ds // drop index - dsLock.writeLock().unlock(); + lock.writeLock().unlock(); } - private void acquireReadModifyLock() { + private void readReadLock() { + dsReadLock.readLock().lock(); + } + + private void modifyReadLock() { // insert dsModifyLock.readLock().lock(); } - private void releaseReadModifyLock() { + private void modifyReadUnlock() { // insert dsModifyLock.readLock().unlock(); } - private void acquireWriteModifyLock() { + private void readReadUnlock() { + dsReadLock.readLock().unlock(); + } + + private void readWriteUnlock() { + dsReadLock.writeLock().unlock(); + } + + private void modifySharedWriteLock() { // Build index statement synchronized (indexBuildCounter) { - if (indexBuildCounter.getIntegerValue() > 0) { - indexBuildCounter.setValue(indexBuildCounter.getIntegerValue() + 1); + if (indexBuildCounter.getValue() > 0) { + indexBuildCounter.setValue(indexBuildCounter.getValue() + 1); } else { dsModifyLock.writeLock().lock(); indexBuildCounter.setValue(1); @@ -87,73 +105,104 @@ public class DatasetLock implements IMetadataLock { } } - private void releaseWriteModifyLock() { + private void modifySharedWriteUnlock() { // Build index statement synchronized (indexBuildCounter) { - if (indexBuildCounter.getIntegerValue() == 1) { + if (indexBuildCounter.getValue() == 1) { dsModifyLock.writeLock().unlock(); } - indexBuildCounter.setValue(indexBuildCounter.getIntegerValue() - 1); + indexBuildCounter.setValue(indexBuildCounter.getValue() - 1); } } - private void acquireRefreshLock() { - // Refresh External Dataset statement + private void modifyExclusiveWriteLock() { dsModifyLock.writeLock().lock(); } - private void releaseRefreshLock() { - // Refresh External Dataset statement + private void modifyExclusiveWriteUnlock() { dsModifyLock.writeLock().unlock(); } @Override - public void acquire(IMetadataLock.Mode mode) { + public void upgrade(IMetadataLock.Mode from, IMetadataLock.Mode to) throws AlgebricksException { + if (from == IMetadataLock.Mode.EXCLUSIVE_MODIFY && to == IMetadataLock.Mode.UPGRADED_WRITE) { + dsReadLock.readLock().unlock(); + dsReadLock.writeLock().lock(); + } else { + throw new MetadataException(ErrorCode.ILLEGAL_LOCK_UPGRADE_OPERATION, from, to); + } + } + + @Override + public void downgrade(IMetadataLock.Mode from, IMetadataLock.Mode to) throws AlgebricksException { + if (from == IMetadataLock.Mode.UPGRADED_WRITE && to == IMetadataLock.Mode.EXCLUSIVE_MODIFY) { + dsReadLock.writeLock().unlock(); + dsReadLock.readLock().lock(); + } else { + throw new MetadataException(ErrorCode.ILLEGAL_LOCK_DOWNGRADE_OPERATION, from, to); + } + } + + @Override + public void lock(IMetadataLock.Mode mode) { switch (mode) { case INDEX_BUILD: - acquireReadLock(); - acquireWriteModifyLock(); + readLock(); + modifySharedWriteLock(); break; case MODIFY: - acquireReadLock(); - acquireReadModifyLock(); + readLock(); + readReadLock(); + modifyReadLock(); break; - case REFRESH: - acquireReadLock(); - acquireRefreshLock(); + case EXCLUSIVE_MODIFY: + readLock(); + readReadLock(); + modifyExclusiveWriteLock(); break; - case INDEX_DROP: case WRITE: - acquireWriteLock(); + writeLock(); break; - default: - acquireReadLock(); + case READ: + readLock(); + readReadLock(); break; + default: + throw new IllegalStateException("locking mode " + mode + " is not supported"); } } @Override - public void release(IMetadataLock.Mode mode) { + public void unlock(IMetadataLock.Mode mode) { switch (mode) { case INDEX_BUILD: - releaseWriteModifyLock(); - releaseReadLock(); + modifySharedWriteUnlock(); + readUnlock(); break; case MODIFY: - releaseReadModifyLock(); - releaseReadLock(); + modifyReadUnlock(); + readReadUnlock(); + readUnlock(); break; - case REFRESH: - releaseRefreshLock(); - releaseReadLock(); + case EXCLUSIVE_MODIFY: + modifyExclusiveWriteUnlock(); + readReadUnlock(); + readUnlock(); break; - case INDEX_DROP: case WRITE: - releaseWriteLock(); + writeUnlock(); break; - default: - releaseReadLock(); + case READ: + readReadUnlock(); + readUnlock(); + break; + case UPGRADED_WRITE: + readWriteUnlock(); + modifyExclusiveWriteUnlock(); + readUnlock(); break; + default: + throw new IllegalStateException("unlocking mode " + mode + " is not supported"); } } @@ -177,4 +226,9 @@ public class DatasetLock implements IMetadataLock { } return Objects.equals(key, ((DatasetLock) o).key); } + + @Override + public String toString() { + return key; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/IMetadataLock.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/IMetadataLock.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/IMetadataLock.java deleted file mode 100644 index 4d7fa93..0000000 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/IMetadataLock.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.metadata.lock; - -/** - * A Metadata lock local to compilation node - */ -public interface IMetadataLock { - - enum Mode { - READ, - MODIFY, - REFRESH, - INDEX_BUILD, - INDEX_DROP, - WRITE - } - - /** - * Acquire a lock - * - * @param mode - * lock mode - */ - void acquire(IMetadataLock.Mode mode); - - /** - * Release a lock - * - * @param mode - * lock mode - */ - void release(IMetadataLock.Mode mode); - - /** - * Get the lock's key - * - * @return the key identiying the lock - */ - String getKey(); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a85f4121/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/LockList.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/LockList.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/LockList.java deleted file mode 100644 index c5c9fe9..0000000 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/lock/LockList.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.metadata.lock; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.asterix.common.exceptions.ErrorCode; -import org.apache.commons.lang3.tuple.Pair; - -/** - * The LockList is used for two phase locking. - */ -public class LockList { - private final List<Pair<IMetadataLock.Mode, IMetadataLock>> locks = new ArrayList<>(); - private final Set<String> lockSet = new HashSet<>(); - private boolean lockPhase = true; - - /** - * Acquires a lock. - * - * @param mode - * the lock mode. - * @param lock - * the lock object. - */ - public void add(IMetadataLock.Mode mode, IMetadataLock lock) throws AsterixException { - if (!lockPhase) { - throw new AsterixException(ErrorCode.COMPILATION_TWO_PHASE_LOCKING_VIOLATION); - } - if (lockSet.contains(lock.getKey())) { - return; - } - lock.acquire(mode); - locks.add(Pair.of(mode, lock)); - lockSet.add(lock.getKey()); - } - - /** - * Once unlock() is called, no caller can call add(IMetadataLock.Mode mode, IMetadataLock lock), - * except that reset() is called. - */ - public void unlock() { - for (int i = locks.size() - 1; i >= 0; i--) { - Pair<IMetadataLock.Mode, IMetadataLock> pair = locks.get(i); - pair.getRight().release(pair.getLeft()); - } - locks.clear(); - lockSet.clear(); - lockPhase = false; - } - - /** - * Clears the state and starts another pass of two phase locking again. - */ - public void reset() { - unlock(); - lockPhase = true; - } -}
