Murtadha Hubail has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/3432
Change subject: [NO ISSUE][OTH] Simplify ResultJobRecord APIs
......................................................................
[NO ISSUE][OTH] Simplify ResultJobRecord APIs
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Simplify ResultJobRecord APIs by allowing only a single ResultSetId
per ResultJobRecord (i.e. per job).
- Fail result partition registration when a job attempts to use
multiple ResultSetIds or inconsistent number of partitions and
log the inconsistency.
- Remove unused methods.
Change-Id: I37816efc92ee9f5e66f29ce74dec4c6c5bd07c6f
---
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
2 files changed, 37 insertions(+), 39 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/32/3432/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
index b3b0706..eebfd9d 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
@@ -21,12 +21,12 @@
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class ResultJobRecord implements IResultStateRecord {
@@ -77,15 +77,13 @@
}
private static final long serialVersionUID = 1L;
-
+ private static final Logger LOGGER = LogManager.getLogger();
private final long timestamp;
private long jobStartTime;
private long jobEndTime;
private Status status;
-
+ private ResultSetId rsId;
private ResultSetMetaData resultSetMetaData;
-
- private Map<ResultSetId, ResultSetMetaData> resultSetMetadataMap = new
HashMap<>();
public ResultJobRecord() {
this.timestamp = System.nanoTime();
@@ -116,10 +114,6 @@
updateState(State.SUCCESS);
}
- public void fail(ResultSetId rsId, int partition) {
- getOrCreateDirectoryRecord(rsId, partition).fail();
- }
-
public void fail(List<Exception> exceptions) {
updateState(State.FAILED);
status.setExceptions(exceptions);
@@ -139,47 +133,40 @@
StringBuilder sb = new StringBuilder();
sb.append("{ \"status\": ").append(status.toString()).append(", ");
sb.append("\"timestamp\": ").append(timestamp).append(", ");
- sb.append("\"resultsets\":
").append(Arrays.toString(resultSetMetadataMap.entrySet().toArray())).append("
}");
+ sb.append("\"resultset\": ").append(resultSetMetaData).append(" }");
return sb.toString();
}
public synchronized void setResultSetMetaData(ResultSetId rsId,
IResultMetadata metadata, int nPartitions)
throws HyracksDataException {
- ResultSetMetaData rsMd = resultSetMetadataMap.get(rsId);
- if (rsMd == null) {
- final ResultSetMetaData resultSetMetaData = new
ResultSetMetaData(nPartitions, metadata);
- resultSetMetadataMap.put(rsId, resultSetMetaData);
- this.resultSetMetaData = resultSetMetaData;
- } else if (rsMd.getRecords().length != nPartitions) {
- throw
HyracksDataException.create(ErrorCode.INCONSISTENT_RESULT_METADATA,
rsId.toString());
+ if (this.rsId == null) {
+ this.rsId = rsId;
+ this.resultSetMetaData = new ResultSetMetaData(nPartitions,
metadata);
+ } else if (!this.rsId.equals(rsId) ||
resultSetMetaData.getRecords().length != nPartitions) {
+ logInconsistentMetadata(rsId, nPartitions);
+ throw
HyracksDataException.create(ErrorCode.INCONSISTENT_RESULT_METADATA,
this.rsId.toString());
}
- //TODO(tillw) throwing a HyracksDataException here hangs the execution
tests
}
- public ResultSetMetaData getResultSetMetaData(ResultSetId rsId) {
- return resultSetMetadataMap.get(rsId);
- }
-
- public synchronized ResultDirectoryRecord
getOrCreateDirectoryRecord(ResultSetId rsId, int partition) {
- ResultDirectoryRecord[] records =
getResultSetMetaData(rsId).getRecords();
+ public synchronized ResultDirectoryRecord getOrCreateDirectoryRecord(int
partition) {
+ ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
if (records[partition] == null) {
records[partition] = new ResultDirectoryRecord();
}
return records[partition];
}
- public synchronized ResultDirectoryRecord getDirectoryRecord(ResultSetId
rsId, int partition)
- throws HyracksDataException {
- ResultDirectoryRecord[] records =
getResultSetMetaData(rsId).getRecords();
+ public synchronized ResultDirectoryRecord getDirectoryRecord(int
partition) throws HyracksDataException {
+ ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
if (records[partition] == null) {
throw HyracksDataException.create(ErrorCode.RESULT_NO_RECORD,
partition, rsId);
}
return records[partition];
}
- public synchronized void updateState(ResultSetId rsId) {
+ public synchronized void updateState() {
int successCount = 0;
- ResultDirectoryRecord[] records =
getResultSetMetaData(rsId).getRecords();
+ ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
for (ResultDirectoryRecord record : records) {
if ((record != null) && (record.getStatus() ==
ResultDirectoryRecord.Status.SUCCESS)) {
successCount++;
@@ -193,4 +180,18 @@
public synchronized ResultSetMetaData getResultSetMetaData() {
return resultSetMetaData;
}
+
+ private void logInconsistentMetadata(ResultSetId rsId, int nPartitions) {
+ if (LOGGER.isWarnEnabled()) {
+ LOGGER.warn("inconsistent result metadata for result set {}",
this.rsId);
+ if (this.rsId.equals(rsId)) {
+ LOGGER.warn("inconsistent result set id. Current {}, new {}",
this.rsId, rsId);
+ }
+ final int expectedPartitions =
resultSetMetaData.getRecords().length;
+ if (expectedPartitions != nPartitions) {
+ LOGGER.warn("inconsistent result set number of partitions.
Current {}, new {}", expectedPartitions,
+ nPartitions);
+ }
+ }
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
index a2218e2..bfecc48 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
@@ -121,7 +121,7 @@
throws HyracksDataException {
ResultJobRecord djr = getNonNullResultJobRecord(jobId);
djr.setResultSetMetaData(rsId, metadata, nPartitions);
- ResultDirectoryRecord record = djr.getOrCreateDirectoryRecord(rsId,
partition);
+ ResultDirectoryRecord record =
djr.getOrCreateDirectoryRecord(partition);
record.setNetworkAddress(networkAddress);
record.setEmpty(emptyResult);
@@ -147,8 +147,8 @@
public synchronized void reportResultPartitionWriteCompletion(JobId jobId,
ResultSetId rsId, int partition)
throws HyracksDataException {
ResultJobRecord djr = getNonNullResultJobRecord(jobId);
- djr.getDirectoryRecord(rsId, partition).writeEOS();
- djr.updateState(rsId);
+ djr.getDirectoryRecord(partition).writeEOS();
+ djr.updateState();
notifyAll();
}
@@ -178,7 +178,7 @@
@Override
public synchronized IResultMetadata getResultMetadata(JobId jobId,
ResultSetId rsId) throws HyracksDataException {
- return
getNonNullResultJobRecord(jobId).getResultSetMetaData(rsId).getMetadata();
+ return
getNonNullResultJobRecord(jobId).getResultSetMetaData().getMetadata();
}
@Override
@@ -228,7 +228,6 @@
private ResultDirectoryRecord[] updatedRecords(JobId jobId, ResultSetId
rsId, ResultDirectoryRecord[] knownRecords)
throws HyracksDataException {
ResultJobRecord djr = getNonNullResultJobRecord(jobId);
-
if (djr.getStatus().getState() == State.FAILED) {
List<Exception> caughtExceptions = djr.getStatus().getExceptions();
if (caughtExceptions != null && !caughtExceptions.isEmpty()) {
@@ -241,13 +240,11 @@
throw
HyracksDataException.create(ErrorCode.RESULT_FAILURE_NO_EXCEPTION, rsId, jobId);
}
}
-
- final ResultSetMetaData resultSetMetaData =
djr.getResultSetMetaData(rsId);
+ final ResultSetMetaData resultSetMetaData = djr.getResultSetMetaData();
if (resultSetMetaData == null) {
return null;
}
ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
-
return Arrays.equals(records, knownRecords) ? null : records;
}
@@ -255,7 +252,7 @@
for (JobId jId : getJobIds()) {
pw.print(jId.toString());
pw.print(" - ");
- pw.println(String.valueOf(getResultJobRecord(jId)));
+ pw.println(getResultJobRecord(jId));
}
pw.flush();
return pw;
--
To view, visit https://asterix-gerrit.ics.uci.edu/3432
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-MessageType: newchange
Gerrit-Change-Id: I37816efc92ee9f5e66f29ce74dec4c6c5bd07c6f
Gerrit-Change-Number: 3432
Gerrit-PatchSet: 1
Gerrit-Owner: Murtadha Hubail <[email protected]>
