[
https://issues.apache.org/jira/browse/DRILL-7403?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16956602#comment-16956602
]
ASF GitHub Bot commented on DRILL-7403:
---------------------------------------
paul-rogers commented on pull request #1871: DRILL-7403: Validate batch checks,
vector integretity in unit tests
URL: https://github.com/apache/drill/pull/1871#discussion_r337308769
##########
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/validate/BatchValidator.java
##########
@@ -150,57 +424,63 @@ private int validateOffsetVector(String name,
UInt4Vector offsetVector, int valu
error(name, offsetVector, "Offset (0) must be 0 but was " + prevOffset);
}
- // Note <= comparison: offset vectors have (n+1) entries.
-
- for (int i = 1; i <= valueCount; i++) {
- int offset = accessor.get(i);
+ for (int i = 1; i < offsetCount; i++) {
+ final int offset = accessor.get(i);
if (offset < prevOffset) {
- error(name, offsetVector, "Decreasing offsets at (" + (i-1) + ", " + i
+ ") = (" + prevOffset + ", " + offset + ")");
+ error(name, offsetVector, String.format(
+ "Offset vector [%d] contained %d, expected >= %d",
+ i, offset, prevOffset));
} else if (offset > maxOffset) {
- error(name, offsetVector, "Invalid offset at index " + i + " = " +
offset + " exceeds maximum of " + maxOffset);
+ error(name, offsetVector, String.format(
+ "Invalid offset at index %d: %d exceeds maximum of %d",
+ i, offset, maxOffset));
}
prevOffset = offset;
}
return prevOffset;
}
private void error(String name, ValueVector vector, String msg) {
- if (errorCount == 0) {
- logger.error("Found one or more vector errors from " +
batch.getClass().getSimpleName());
- }
- errorCount++;
- if (errorCount >= MAX_ERRORS) {
- return;
- }
- String fullMsg = "Column " + name + " of type " +
vector.getClass().getSimpleName( ) + ": " + msg;
- logger.error(fullMsg);
- if (errorList != null) {
- errorList.add(fullMsg);
- }
+ errorReporter.error(name, vector, msg);
}
- private void validateNullableVector(String name, NullableVector vector) {
- // Can't validate at this time because the bits vector is in each
- // generated subtype.
-
- // Validate a VarChar vector because it is common.
-
- if (vector instanceof NullableVarCharVector) {
- VarCharVector values = ((NullableVarCharVector)
vector).getValuesVector();
- validateVarCharVector(name + "-values", values, rowCount);
+ private void verifyIsSetVector(ValueVector parent, UInt1Vector bv) {
+ final String name = String.format("%s (%s)-bits",
+ parent.getField().getName(),
+ parent.getClass().getSimpleName());
+ final int rowCount = parent.getAccessor().getValueCount();
+ final int bitCount = bv.getAccessor().getValueCount();
+ if (bitCount != rowCount) {
+ error(name, bv, String.format(
+ "Value count = %d, but bit count = %d",
+ rowCount, bitCount));
+ }
+ final UInt1Vector.Accessor ba = bv.getAccessor();
+ for (int i = 0; i < bitCount; i++) {
+ final int value = ba.get(i);
+ if (value != 0 && value != 1) {
+ error(name, bv, String.format(
+ "%s %s: bit vector[%d] = %d, expected 0 or 1",
+ i, value));
+ }
}
- }
-
- private void validateFixedWidthVector(String name, FixedWidthVector vector) {
- // TODO Auto-generated method stub
-
}
/**
- * Obtain the list of errors. For use in unit-testing this class.
- * @return the list of errors found, or null if error capture was
- * not enabled
+ * Print a record batch. Uses code only available in a test build.
+ * Classes are not visible to the compiler; must load dynamically.
+ * Does nothing if the class is not available.
*/
- public List<String> errors() { return errorList; }
+ public static void print(RecordBatch batch) {
+ try {
+ final Class<?> helper =
Class.forName("org.apache.drill.test.rowSet.RowSetUtilities");
Review comment:
Removed for now, will add back later (and fix any issues) if/when needed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Validate batch checks, vector integretity in unit tests
> -------------------------------------------------------
>
> Key: DRILL-7403
> URL: https://issues.apache.org/jira/browse/DRILL-7403
> Project: Apache Drill
> Issue Type: Improvement
> Affects Versions: 1.16.0, 1.17.0
> Reporter: Paul Rogers
> Assignee: Paul Rogers
> Priority: Minor
> Labels: ready-to-commit
> Fix For: 1.17.0
>
>
> Drill provides a {{BatchValidator}} that checks vectors. It is disabled by
> default. This enhancement adds more checks, including checks for row counts
> (of which there are surprisingly many.)
> Since most operators will fail if the check is enabled, this enhancement also
> adds a table to keep track of which operators pass the checks (and for which
> checks should be enabled) and those that still need work. This allows the
> checks to exist in the code, and to be enabled incrementally as we fix the
> various problems.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
