Repository: asterixdb Updated Branches: refs/heads/master 7c53fcf40 -> 7f00136dc
[ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Completion - user model changes: no - storage format changes: no - interface changes: yes Added resource id to ITransactionOperationTracker before/after txn operations. Details: - Currently, an index could be evicted/dropped while a transaction waiting for its completion (commit/rollback). This change prevents that by incrementing the reference counter of the indexes registered in the transaction. - Add test case. Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223 Reviewed-on: https://asterix-gerrit.ics.uci.edu/2132 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/7f00136d Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/7f00136d Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/7f00136d Branch: refs/heads/master Commit: 7f00136dc098c6e7956de6843323c5affd887723 Parents: 7c53fcf Author: Murtadha Hubail <[email protected]> Authored: Tue Nov 7 18:35:58 2017 +0300 Committer: Murtadha Hubail <[email protected]> Committed: Thu Nov 9 10:06:39 2017 -0800 ---------------------------------------------------------------------- .../test/dataflow/ComponentRollbackTest.java | 1 + .../IndexDropOperatorNodePushableTest.java | 59 ++++++++++++++++++++ .../common/context/BaseOperationTracker.java | 10 +++- .../context/ITransactionOperationTracker.java | 10 +++- .../service/transaction/TransactionContext.java | 4 +- 5 files changed, 78 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java index ac5e606..6bff50d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/ComponentRollbackTest.java @@ -264,6 +264,7 @@ public class ComponentRollbackTest { tupleAppender.write(insertOp, true); } insertOp.close(); + nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true); searchAndAssertCount(nc, ctx, dataset, storageManager, TOTAL_NUM_OF_RECORDS); // rollback the last disk component lsmAccessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java index 2bac49e..7ced87d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java @@ -30,17 +30,28 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.asterix.app.bootstrap.TestNodeController; import org.apache.asterix.common.config.DatasetConfig; +import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.file.StorageComponentProvider; +import org.apache.asterix.metadata.MetadataManager; +import org.apache.asterix.metadata.MetadataTransactionContext; +import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities; +import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.metadata.entities.InternalDatasetDetails; +import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; +import org.apache.asterix.test.common.TestExecutor; import org.apache.asterix.test.common.TestHelper; +import org.apache.asterix.test.runtime.ExecutionTestUtil; +import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileSplit; +import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable; @@ -64,6 +75,7 @@ public class IndexDropOperatorNodePushableTest { private static final String DATA_TYPE_NAME = "DUMMY"; private static final String NODE_GROUP_NAME = "DEFAULT"; private final AtomicBoolean dropFailed = new AtomicBoolean(false); + private final TestExecutor testExecutor = new TestExecutor(); @Before public void setUp() throws Exception { @@ -77,6 +89,12 @@ public class IndexDropOperatorNodePushableTest { TestHelper.deleteExistingInstanceFiles(); } + /** + * Tests dropping a dataset using different + * drop options + * + * @throws Exception + */ @Test public void dropOptionsTest() throws Exception { TestNodeController nc = new TestNodeController(null, false); @@ -107,6 +125,47 @@ public class IndexDropOperatorNodePushableTest { } } + /** + * Tests dropping an index whose dataset has no active + * operations + * + * @throws Exception + */ + @Test + public void dropIndexInUseTest() throws Exception { + TestNodeController nc = new TestNodeController(null, false); + try { + nc.init(); + String datasetName = "ds"; + String indexName = "fooIdx"; + // create dataset and index + final TestCaseContext.OutputFormat format = TestCaseContext.OutputFormat.CLEAN_JSON; + testExecutor.executeSqlppUpdateOrDdl("CREATE TYPE KeyType AS { id: int, foo: int };", format); + testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " + datasetName + "(KeyType) PRIMARY KEY id;", format); + testExecutor.executeSqlppUpdateOrDdl("CREATE INDEX " + indexName + " on " + datasetName + "(foo)", format); + final MetadataTransactionContext mdTxn = MetadataManager.INSTANCE.beginTransaction(); + ICcApplicationContext appCtx = + (ICcApplicationContext) ExecutionTestUtil.integrationUtil.getClusterControllerService() + .getApplicationContext(); + MetadataProvider metadataProver = new MetadataProvider(appCtx, null); + metadataProver.setMetadataTxnContext(mdTxn); + final String defaultDv = MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName(); + final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn, defaultDv, datasetName); + MetadataManager.INSTANCE.commitTransaction(mdTxn); + FileSplit[] splits = SplitsAndConstraintsUtil + .getIndexSplits(appCtx.getClusterStateManager(), dataset, indexName, Arrays.asList("asterix_nc1")); + final ConstantFileSplitProvider constantFileSplitProvider = + new ConstantFileSplitProvider(Arrays.copyOfRange(splits, 0, 1)); + IndexDataflowHelperFactory helperFactory = + new IndexDataflowHelperFactory(nc.getStorageManager(), constantFileSplitProvider); + IHyracksTaskContext ctx = nc.createTestContext(true); + IIndexDataflowHelper dataflowHelper = helperFactory.create(ctx.getJobletContext().getServiceContext(), 0); + dropInUse(ctx, helperFactory, dataflowHelper); + } finally { + nc.deInit(); + } + } + private void dropInUse(IHyracksTaskContext ctx, IndexDataflowHelperFactory helperFactory, IIndexDataflowHelper dataflowHelper) throws Exception { dropFailed.set(false); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java index 2068192..9f57981 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java @@ -63,12 +63,18 @@ public class BaseOperationTracker implements ITransactionOperationTracker { } @Override - public void beforeTransaction() { + public void beforeTransaction(long resourceId) { + /* + * Increment dataset and index ref count to prevent them + * from being evicted/dropped until the transaction completes + */ dsInfo.touch(); + dsInfo.getIndexes().get(resourceId).touch(); } @Override - public void afterTransaction() { + public void afterTransaction(long resourceId) { dsInfo.untouch(); + dsInfo.getIndexes().get(resourceId).untouch(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java index e6aeec5..301801b 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java @@ -25,12 +25,18 @@ public interface ITransactionOperationTracker extends ILSMOperationTracker { /** * Called before a transaction performs any operations on * {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex} + * with resource id {@code resourceId} + * + * @param resourceId */ - void beforeTransaction(); + void beforeTransaction(long resourceId); /** * Called after a transaction completes its operations on * {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex} + * with resource id {@code resourceId} + * + * @param resourceId */ - void afterTransaction(); + void afterTransaction(long resourceId); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/7f00136d/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java index 4846b6e..3159e6b 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java @@ -122,7 +122,7 @@ public class TransactionContext implements ITransactionContext { final ITransactionOperationTracker txnOpTracker = (ITransactionOperationTracker) index.getOperationTracker(); indexMap.put(resourceId, txnOpTracker); - txnOpTracker.beforeTransaction(); + txnOpTracker.beforeTransaction(resourceId); } } } @@ -258,7 +258,7 @@ public class TransactionContext implements ITransactionContext { } } finally { synchronized (indexMap) { - indexMap.values().forEach(ITransactionOperationTracker::afterTransaction); + indexMap.forEach((resource, opTracker) -> opTracker.afterTransaction(resource)); } } }
