Murtadha Hubail has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2132
Change subject: [ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn
Completion
......................................................................
[ASTERIXDB-2130][TX] Prevent Dropping Index Pending Txn Completion
- user model changes: no
- storage format changes: no
- interface changes: yes
Added resource id to ITransactionOperationTracker
before/after txn operations.
Details:
- Currently, an index could be evicted/dropped while a transaction
waiting for its completion (commit/rollback). This change prevents
that by incrementing the reference counter of the indexes registered
in the transaction.
- Add test case.
Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223
---
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
4 files changed, 73 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/32/2132/1
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
index 2bac49e..37b35bb 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/IndexDropOperatorNodePushableTest.java
@@ -30,17 +30,28 @@
import org.apache.asterix.app.bootstrap.TestNodeController;
import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.file.StorageComponentProvider;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
+import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.common.TestHelper;
+import org.apache.asterix.test.runtime.ExecutionTestUtil;
+import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileSplit;
+import org.apache.hyracks.dataflow.std.file.ConstantFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexDataflowHelper;
import
org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
import
org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorNodePushable;
@@ -64,6 +75,7 @@
private static final String DATA_TYPE_NAME = "DUMMY";
private static final String NODE_GROUP_NAME = "DEFAULT";
private final AtomicBoolean dropFailed = new AtomicBoolean(false);
+ private final TestExecutor testExecutor = new TestExecutor();
@Before
public void setUp() throws Exception {
@@ -77,6 +89,12 @@
TestHelper.deleteExistingInstanceFiles();
}
+ /**
+ * Tests dropping a dataset with using different
+ * drop options
+ *
+ * @throws Exception
+ */
@Test
public void dropOptionsTest() throws Exception {
TestNodeController nc = new TestNodeController(null, false);
@@ -107,6 +125,47 @@
}
}
+ /**
+ * Tests dropping an index whose dataset has no active
+ * operations
+ *
+ * @throws Exception
+ */
+ @Test
+ public void dropIndexInUseTest() throws Exception {
+ TestNodeController nc = new TestNodeController(null, false);
+ try {
+ nc.init();
+ String datasetName = "ds";
+ String indexName = "valueIdx";
+ // create dataset and index
+ final TestCaseContext.OutputFormat format =
TestCaseContext.OutputFormat.CLEAN_JSON;
+ testExecutor.executeSqlppUpdateOrDdl("CREATE TYPE KeyType AS { id:
int, foo: int };", format);
+ testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " +
datasetName + "(KeyType) PRIMARY KEY id;", format);
+ testExecutor.executeSqlppUpdateOrDdl("CREATE INDEX " + indexName +
" on " + datasetName + "(foo)", format);
+ final MetadataTransactionContext mdTxn =
MetadataManager.INSTANCE.beginTransaction();
+ ICcApplicationContext appCtx =
+ (ICcApplicationContext)
ExecutionTestUtil.integrationUtil.getClusterControllerService()
+ .getApplicationContext();
+ MetadataProvider metadataProver = new MetadataProvider(appCtx,
null);
+ metadataProver.setMetadataTxnContext(mdTxn);
+ final String defaultDv =
MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName();
+ final Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxn,
defaultDv, datasetName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxn);
+ FileSplit[] splits = SplitsAndConstraintsUtil
+ .getIndexSplits(appCtx.getClusterStateManager(), dataset,
indexName, Arrays.asList("asterix_nc1"));
+ final ConstantFileSplitProvider constantFileSplitProvider =
+ new ConstantFileSplitProvider(Arrays.copyOfRange(splits,
0, 1));
+ IndexDataflowHelperFactory helperFactory =
+ new IndexDataflowHelperFactory(nc.getStorageManager(),
constantFileSplitProvider);
+ IHyracksTaskContext ctx = nc.createTestContext(true);
+ IIndexDataflowHelper dataflowHelper =
helperFactory.create(ctx.getJobletContext().getServiceContext(), 0);
+ dropInUse(ctx, helperFactory, dataflowHelper);
+ } finally {
+ nc.deInit();
+ }
+ }
+
private void dropInUse(IHyracksTaskContext ctx, IndexDataflowHelperFactory
helperFactory,
IIndexDataflowHelper dataflowHelper) throws Exception {
dropFailed.set(false);
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
index 2068192..4f91b25 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/BaseOperationTracker.java
@@ -63,12 +63,12 @@
}
@Override
- public void beforeTransaction() {
- dsInfo.touch();
+ public void beforeTransaction(long resourceId) {
+ dsInfo.getIndexes().get(resourceId).touch();
}
@Override
- public void afterTransaction() {
- dsInfo.untouch();
+ public void afterTransaction(long resourceId) {
+ dsInfo.getIndexes().get(resourceId).untouch();
}
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
index e6aeec5..301801b 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionOperationTracker.java
@@ -25,12 +25,18 @@
/**
* Called before a transaction performs any operations on
* {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+ * with resource id {@code resourceId}
+ *
+ * @param resourceId
*/
- void beforeTransaction();
+ void beforeTransaction(long resourceId);
/**
* Called after a transaction completes its operations on
* {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex}
+ * with resource id {@code resourceId}
+ *
+ * @param resourceId
*/
- void afterTransaction();
+ void afterTransaction(long resourceId);
}
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
index 4846b6e..3159e6b 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/service/transaction/TransactionContext.java
@@ -122,7 +122,7 @@
final ITransactionOperationTracker txnOpTracker =
(ITransactionOperationTracker)
index.getOperationTracker();
indexMap.put(resourceId, txnOpTracker);
- txnOpTracker.beforeTransaction();
+ txnOpTracker.beforeTransaction(resourceId);
}
}
}
@@ -258,7 +258,7 @@
}
} finally {
synchronized (indexMap) {
-
indexMap.values().forEach(ITransactionOperationTracker::afterTransaction);
+ indexMap.forEach((resource, opTracker) ->
opTracker.afterTransaction(resource));
}
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/2132
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If6a938cbb5a9c3b7f5cc59505c07ae45b3425223
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <[email protected]>
