>From Wail Alkowaileet <[email protected]>:
Wail Alkowaileet has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18400 )
Change subject: [NO ISSUE][CLUS] Avoid locking resources while waiting for IO
......................................................................
[NO ISSUE][CLUS] Avoid locking resources while waiting for IO
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Waiting for all I/O operations should not acquire a lock on
when cleaning up a partition.
Change-Id: I767474703e5519bf576db63fc988aeb67aebffe4
---
M asterixdb/LICENSE
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
5 files changed, 67 insertions(+), 6 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/00/18400/1
diff --git a/asterixdb/LICENSE b/asterixdb/LICENSE
index cc61b93..4c63d01 100644
--- a/asterixdb/LICENSE
+++ b/asterixdb/LICENSE
@@ -744,3 +744,35 @@
PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
---
+ Portions of the AsterixDB csv-spectrum tests
+ located at:
+ and
+ asterix-app/data/csv-spectrum/*
+
+ are available under BSD:
+---
+ Copyright (c) 2013, Max Ogden
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
modification,
+ are permitted provided that the following conditions are met:
+
+ Redistributions of source code must retain the above copyright notice, this
list
+ of conditions and the following disclaimer.
+ Redistributions in binary form must reproduce the above copyright notice,
this
+ list of conditions and the following disclaimer in the documentation and/or
+ other materials provided with the distribution.
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR
+ ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON
+ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+---
+
+
+
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
index 9064db5..c8db4cd 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetInfo.java
@@ -266,6 +266,8 @@
synchronized (this) {
while (partitionPendingIO.getOrDefault(partition, 0) > 0) {
try {
+ int numPendingIOOps =
partitionPendingIO.getOrDefault(partition, 0);
+ LOGGER.debug("Waiting for {} IO operations in {} partition
{}", numPendingIOOps, this, partition);
wait();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
@@ -273,6 +275,8 @@
}
}
+ LOGGER.debug("All IO operations for {} partition {} are finished",
this, partition);
+
Set<IndexInfo> indexes = partitionIndexes.get(partition);
if (indexes != null) {
// Perform the required operation
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
index abf2870..53c7aa5 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/hdfs/parquet/converter/AbstractComplexConverter.java
@@ -22,7 +22,6 @@
import java.io.IOException;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.AsterixTypeToParquetTypeVisitor;
-import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ArrayConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ObjectConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.ObjectRepeatedConverter;
import
org.apache.asterix.external.input.record.reader.hdfs.parquet.converter.nested.RepeatedConverter;
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
index 0790a22..6875ebd 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/CleanupBlockingIOOperation.java
@@ -62,7 +62,7 @@
}
/**
- * Clean all inactive indexes while the DatasetLifeCycleManager is
synchronized
+ * Clean all inactive indexes
*/
@Override
public void afterOperation() throws HyracksDataException {
diff --git
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index d0b4aa1..1b3487d 100644
---
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -559,13 +559,18 @@
}
public void cleanup(int partition) throws HyracksDataException {
- beforeReadAccess();
+ LOGGER.info("Cleaning up indexes in partition {}", partition);
+ CleanupBlockingIOOperation cleanupOp = new
CleanupBlockingIOOperation(partition, this, ioManager);
+ boolean failed = false;
try {
- CleanupBlockingIOOperation cleanupOp = new
CleanupBlockingIOOperation(partition, this, ioManager);
datasetLifecycleManager.waitForIOAndPerform(AllDatasetsReplicationStrategy.INSTANCE,
partition, cleanupOp);
+ } catch (HyracksDataException e) {
+ failed = true;
+ throw e;
} finally {
- clearResourcesCache();
- afterReadAccess();
+ if (!failed) {
+ LOGGER.info("Finished cleaning up indexes in partition {}
successfully", partition);
+ }
}
}
@@ -575,6 +580,7 @@
* @param index to clean
*/
void cleanupIndex(FileReference index) throws HyracksDataException {
+ beforeReadAccess();
try {
deleteIndexMaskedFiles(index);
if (isValidIndex(index)) {
@@ -582,6 +588,9 @@
}
} catch (IOException | ParseException e) {
throw HyracksDataException.create(e);
+ } finally {
+ clearResourcesCache();
+ afterReadAccess();
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18400
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I767474703e5519bf576db63fc988aeb67aebffe4
Gerrit-Change-Number: 18400
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <[email protected]>
Gerrit-MessageType: newchange
