>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21308?usp=email )
Change subject: [ASTERIXDB-3649][HYR] Ensure reportPartitionConsumed for all partitions ...................................................................... [ASTERIXDB-3649][HYR] Ensure reportPartitionConsumed for all partitions - user model changes: no - storage format changes: no - interface changes: no Details: The following applies to non-async requests. - Change result set reader to read all locations even empty ones. Otherwise, if empty locations were left unread, then the result state in both the NCs and CC will stay around. - Call reportPartitionConsumed for all partitions for the same reason. Otherwise, the job record in the CC will stay around. Ext-ref: MB-71997, MB-69763 Change-Id: I330d0668c8b41d5096d095a6067f8214ca60a4f1 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21308 Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Integration-Tests: Jenkins <[email protected]> --- M hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java M hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java M hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java 3 files changed, 11 insertions(+), 16 deletions(-) Approvals: Jenkins: Verified; Verified Ian Maxon: Looks good to me, approved Murtadha Hubail: Looks good to me, approved diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java index dd72e2e..e0a0ae8 100644 --- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java +++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java @@ -27,7 +27,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.JobId; import org.apache.hyracks.api.result.IResultDirectory; -import org.apache.hyracks.api.result.ResultDirectoryRecord; import org.apache.hyracks.api.result.ResultSetId; import org.apache.hyracks.client.net.ClientNetworkManager; @@ -96,12 +95,7 @@ // exhausted all known records return false; } - ResultDirectoryRecord record = getRecord(currentRecord); - // skip empty records - if (record.isEmpty()) { - return false; - } - requestRecordData(record); + requestRecordData(getRecord(currentRecord)); return true; } diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java index 9765805..df86178 100644 --- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java +++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java @@ -88,7 +88,11 @@ frame.reset(); int readSize = 0; try { - if (isFirstRead() && !hasNextRecord()) { + boolean firstRead = isFirstRead(); + if (firstRead) { + getResultRecords(); + } + if (firstRead && !hasNextRecord()) { return readSize; } // read until frame is full or all result records have been read @@ -164,16 +168,11 @@ protected boolean hasNextRecord() throws HyracksDataException { currentRecord++; - ResultDirectoryRecord record = getRecord(currentRecord); - // skip empty records - while (record.isEmpty() && ++currentRecord < knownRecords.length) { - record = getRecord(currentRecord); - } if (currentRecord == knownRecords.length) { // exhausted all known records return false; } - requestRecordData(record); + requestRecordData(getRecord(currentRecord)); return true; } diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java index b13988a..8203637 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java @@ -157,8 +157,10 @@ public synchronized void removePartition(JobId jobId, ResultSetId resultSetId, int partition) throws HyracksException { ResultSetMap rsIdMap = partitionResultStateMap.get(jobId); - if (rsIdMap != null && rsIdMap.removePartition(jobId, resultSetId, partition)) { - partitionResultStateMap.remove(jobId); + if (rsIdMap != null) { + if (rsIdMap.removePartition(jobId, resultSetId, partition)) { + partitionResultStateMap.remove(jobId); + } reportPartitionConsumed(jobId, resultSetId, partition); } } -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21308?usp=email To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: asterixdb Gerrit-Branch: lumina Gerrit-Change-Id: I330d0668c8b41d5096d095a6067f8214ca60a4f1 Gerrit-Change-Number: 21308 Gerrit-PatchSet: 3 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]>
