This is an automated email from the ASF dual-hosted git repository.
luochen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new d6f10a8 [ASTERIXDB-2730][STO] Do not wait on flushing metadata indexes
d6f10a8 is described below
commit d6f10a8c82ce6dc9bc37d6d708c64b3f90d8dc31
Author: luochen <[email protected]>
AuthorDate: Sun May 17 21:14:37 2020 -0700
[ASTERIXDB-2730][STO] Do not wait on flushing metadata indexes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Ensure GVBC does not wait on flushing metadata indexes because
metadata indexes support full ACID transactions and waiting on them
can lead to deadlocks.
- Add more log messages to GVBC.
Change-Id: Iba4020d8584768940176023ac4ab9135cff7b7e6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/6345
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
---
.../common/context/GlobalVirtualBufferCache.java | 29 +++++++++++++++++++++-
1 file changed, 28 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index c0197b0..e3bd13d 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.asterix.common.config.StorageProperties;
+import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
@@ -102,6 +103,10 @@ public class GlobalVirtualBufferCache implements
IVirtualBufferCache, ILifeCycle
if (!primaryIndexes.contains(index)) {
// make sure only add index once
primaryIndexes.add(index);
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Registered {} index {} to the global VBC",
+ isMetadataIndex(index) ? "metadata" : "primary",
index.toString());
+ }
}
if (index.getNumOfFilterFields() > 0) {
// handle filtered primary index
@@ -123,6 +128,10 @@ public class GlobalVirtualBufferCache implements
IVirtualBufferCache, ILifeCycle
int pos = primaryIndexes.indexOf(index);
if (pos >= 0) {
primaryIndexes.remove(index);
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Unregistered {} index {} to the global VBC",
+ isMetadataIndex(index) ? "metadata" : "primary",
index.toString());
+ }
if (primaryIndexes.isEmpty()) {
flushPtr = 0;
} else if (flushPtr > pos) {
@@ -160,6 +169,11 @@ public class GlobalVirtualBufferCache implements
IVirtualBufferCache, ILifeCycle
opTracker.notifyAll();
}
}
+
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Completed flushing {}. Resetting
flushIndex back to null.",
+ memoryComponent.getIndex().toString());
+ }
}
}
checkAndNotifyFlushThread();
@@ -421,6 +435,11 @@ public class GlobalVirtualBufferCache implements
IVirtualBufferCache, ILifeCycle
return vbc.getUsage();
}
+ private boolean isMetadataIndex(ILSMIndex index) {
+ BaseOperationTracker opTracker = (BaseOperationTracker)
index.getOperationTracker();
+ return
MetadataIndexImmutableProperties.isMetadataDataset(opTracker.getDatasetInfo().getDatasetID());
+ }
+
/**
* We use a dedicated thread to schedule flushes to avoid deadlock. We
cannot schedule flushes directly during
* page pins because page pins can be called while synchronized on op
trackers (e.g., when resetting a
@@ -476,9 +495,17 @@ public class GlobalVirtualBufferCache implements
IVirtualBufferCache, ILifeCycle
opTracker.flushIfNeeded();
// If the flush cannot be scheduled at this time,
then there must be active writers.
// The flush will be eventually scheduled when
writers exit
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Requested {} flushing primary
index {}",
+ isMetadataIndex(primaryIndex) ?
"metadata" : "primary",
+ primaryIndex.toString());
+ }
}
- if (flushable || opTracker.isFlushLogCreated()) {
+ if ((flushable || opTracker.isFlushLogCreated()) &&
!isMetadataIndex(primaryIndex)) {
+ // global vbc cannot wait on metadata indexes
because metadata indexes support full
+ // ACID transactions. Waiting on metadata indexes
can introduce deadlocks.
flushingIndex = primaryIndex;
+ LOGGER.debug("Waiting for flushing primary index
{} to complete...", primaryIndex);
break;
}
}
