abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/2047
Change subject: [NO ISSUE] Add performance traces for frame writes
......................................................................
[NO ISSUE] Add performance traces for frame writes
- user model changes: no
- storage format changes: no
- interface changes: no
details:
- Add traces for writes to end of local pipeline.
Change-Id: Ib32f4122fdddff1d0dce282a99829e0e0ad820e9
---
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
M
hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java
3 files changed, 62 insertions(+), 18 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/47/2047/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 4bb2869..34e778b 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -128,6 +128,7 @@
public static final int JOB_HAS_BEEN_CLEARED_FROM_HISTORY = 92;
public static final int JOB_HAS_NOT_BEEN_CREATED_YET = 93;
public static final int CANNOT_READ_CLOSED_FILE = 94;
+ public static final int TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME = 95;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index ede2f86..a6291d7 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -111,5 +111,6 @@
92 = Job %1$s has been cleared from job history
93 = Job %1$s has not been created yet
94 = Cannot read closed file (%1$s)
+95 = Tuple of size %1$s did not fit into an empty frame
10000 = The given rule collection %1$s is not an instance of the List class.
diff --git
a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java
b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java
index 063afe7..5d0aec1 100644
---
a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java
+++
b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/util/FrameUtils.java
@@ -24,9 +24,14 @@
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.comm.IFrameTupleAppender;
import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.trace.Tracer;
public class FrameUtils {
+
+ private FrameUtils() {
+ }
public static void copyWholeFrame(ByteBuffer srcFrame, ByteBuffer
destFrame) {
srcFrame.clear();
@@ -67,7 +72,7 @@
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.appendSkipEmptyField(fieldSlots, bytes,
offset, length)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
length);
}
}
return flushedBytes;
@@ -93,7 +98,7 @@
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.append(bytes, offset, length)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
length);
}
}
return flushedBytes;
@@ -109,14 +114,14 @@
* @throws HyracksDataException
*/
public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender
frameTupleAppender,
- IFrameTupleAccessor tupleAccessor, int tStartOffset, int
tEndOffset)
- throws HyracksDataException {
+ IFrameTupleAccessor tupleAccessor, int tStartOffset, int
tEndOffset) throws HyracksDataException {
int flushedBytes = 0;
if (!frameTupleAppender.append(tupleAccessor, tStartOffset,
tEndOffset)) {
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.append(tupleAccessor, tStartOffset,
tEndOffset)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
+ tEndOffset - tStartOffset);
}
}
return flushedBytes;
@@ -137,7 +142,30 @@
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.append(tupleAccessor, tIndex)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
+ tupleAccessor.getTupleLength(tIndex));
+ }
+ }
+ return flushedBytes;
+ }
+
+ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender
frameTupleAppender,
+ IFrameTupleAccessor tupleAccessor, int tIndex, Tracer tracer,
String cat, String nc, String args)
+ throws HyracksDataException {
+ int flushedBytes = 0;
+ if (!frameTupleAppender.append(tupleAccessor, tIndex)) {
+ flushedBytes = frameTupleAppender.getBuffer().capacity();
+ long tid = -1L;
+ if (tracer != null && tracer.isEnabled()) {
+ tid = tracer.durationB(nc, cat, args);
+ }
+ frameTupleAppender.write(writer, true);
+ if (tracer != null && tracer.isEnabled()) {
+ tracer.durationE(tid, args);
+ }
+ if (!frameTupleAppender.append(tupleAccessor, tIndex)) {
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
+ tupleAccessor.getTupleLength(tIndex));
}
}
return flushedBytes;
@@ -153,8 +181,8 @@
* @return the number of bytes that have been flushed, 0 if not get
flushed.
* @throws HyracksDataException
*/
- public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender
tupleAppender,
- int[] fieldEndOffsets, byte[] byteArray, int start, int size)
throws HyracksDataException {
+ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender
tupleAppender, int[] fieldEndOffsets,
+ byte[] byteArray, int start, int size) throws HyracksDataException
{
int flushedBytes = 0;
if (!tupleAppender.append(fieldEndOffsets, byteArray, start, size)) {
@@ -162,7 +190,7 @@
tupleAppender.write(writer, true);
if (!tupleAppender.append(fieldEndOffsets, byteArray, start,
size)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME, size);
}
}
return flushedBytes;
@@ -186,7 +214,8 @@
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.appendConcat(accessor0, tIndex0,
accessor1, tIndex1)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
+ accessor0.getTupleLength(tIndex0) +
accessor1.getTupleLength(tIndex1));
}
}
return flushedBytes;
@@ -205,14 +234,19 @@
* @throws HyracksDataException
*/
public static int appendConcatToWriter(IFrameWriter writer,
IFrameTupleAppender frameTupleAppender,
- IFrameTupleAccessor accessor0, int tIndex0, int[] fieldSlots1,
byte[] bytes1, int offset1,
- int dataLen1) throws HyracksDataException {
+ IFrameTupleAccessor accessor0, int tIndex0, int[] fieldSlots1,
byte[] bytes1, int offset1, int dataLen1)
+ throws HyracksDataException {
int flushedBytes = 0;
if (!frameTupleAppender.appendConcat(accessor0, tIndex0, fieldSlots1,
bytes1, offset1, dataLen1)) {
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.appendConcat(accessor0, tIndex0,
fieldSlots1, bytes1, offset1, dataLen1)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ int startOffset0 = accessor0.getTupleStartOffset(tIndex0);
+ int endOffset0 = accessor0.getTupleEndOffset(tIndex0);
+ int length0 = endOffset0 - startOffset0;
+ int slotsLen1 = fieldSlots1.length * 4;
+ int length1 = slotsLen1 + dataLen1;
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
length0 + length1);
}
}
return flushedBytes;
@@ -234,7 +268,13 @@
flushedBytes = frameTupleAppender.getBuffer().capacity();
frameTupleAppender.write(writer, true);
if (!frameTupleAppender.appendProjection(accessor, tIndex,
fields)) {
- throw new HyracksDataException("The output cannot be fit into
a frame.");
+ int fTargetSlotsLength = fields.length * 4;
+ int length = fTargetSlotsLength;
+ for (int i = 0; i < fields.length; ++i) {
+ length += (accessor.getFieldEndOffset(tIndex, fields[i])
+ - accessor.getFieldStartOffset(tIndex, fields[i]));
+ }
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
length);
}
}
return flushedBytes;
@@ -249,14 +289,14 @@
* @return the number of bytes that have been flushed, 0 if not get
flushed.
* @throws HyracksDataException
*/
- public static int appendFieldToWriter(IFrameWriter writer,
IFrameFieldAppender appender, byte[] array,
- int start, int length) throws HyracksDataException {
+ public static int appendFieldToWriter(IFrameWriter writer,
IFrameFieldAppender appender, byte[] array, int start,
+ int length) throws HyracksDataException {
int flushedBytes = 0;
if (!appender.appendField(array, start, length)) {
flushedBytes = appender.getBuffer().capacity();
appender.write(writer, true);
if (!appender.appendField(array, start, length)) {
- throw new HyracksDataException("Could not write frame: the
size of the tuple is too long");
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME,
length);
}
}
return flushedBytes;
@@ -278,7 +318,9 @@
flushedBytes = appender.getBuffer().capacity();
appender.write(writer, true);
if (!appender.appendField(accessor, tid, fid)) {
- throw new HyracksDataException("Could not write frame: the
size of the tuple is too long");
+ int fStartOffset = accessor.getFieldStartOffset(tid, fid);
+ int fLen = accessor.getFieldEndOffset(tid, fid) - fStartOffset;
+ throw
HyracksDataException.create(ErrorCode.TUPLE_CAN_NOT_FIT_INTO_EMPTY_FRAME, fLen);
}
}
return flushedBytes;
--
To view, visit https://asterix-gerrit.ics.uci.edu/2047
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib32f4122fdddff1d0dce282a99829e0e0ad820e9
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <[email protected]>
