Repository: incubator-gobblin Updated Branches: refs/heads/master 9402a9037 -> 1507b6055
[GOBBLIN-247] Fixed ArrayIndexOutOfBoundsException for avro-to-orc Validation Job Closes #2100 from aditya1105/avro-to-orc Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/1507b605 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/1507b605 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/1507b605 Branch: refs/heads/master Commit: 1507b6055d3f058cc17b1accb01c897a21b9513c Parents: 9402a90 Author: aditya1105 <[email protected]> Authored: Tue Sep 12 03:05:57 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Tue Sep 12 03:05:57 2017 -0700 ---------------------------------------------------------------------- .../conversion/hive/validation/ValidationJob.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1507b605/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java index 53805c8..1a61803 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java @@ -391,7 +391,7 @@ public class ValidationJob extends AbstractJob { log.debug(String.format("Going to execute queries: %s for format: %s", dataValidationQueries, format)); List<Long> rowDataValidatedCount = ValidationJob.this.getValidationOutputFromHive(dataValidationQueries); // Validate and populate report - validateAndPopulateReport(hiveDataset.getTable().getCompleteName(), updateTime, rowCounts, rowDataValidatedCount.get(0)); + validateAndPopulateReport(hiveDataset.getTable().getCompleteName(), updateTime, rowCounts, rowDataValidatedCount); return null; } @@ -460,7 +460,7 @@ public class ValidationJob extends AbstractJob { List<Long> rowDataValidatedCount = ValidationJob.this.getValidationOutputFromHive(dataValidationQueries); // Validate and populate report - validateAndPopulateReport(sourcePartition.getCompleteName(), updateTime, rowCounts, rowDataValidatedCount.get(0)); + validateAndPopulateReport(sourcePartition.getCompleteName(), updateTime, rowCounts, rowDataValidatedCount); return null; } @@ -606,12 +606,17 @@ public class ValidationJob extends AbstractJob { return rowCounts; } - private void validateAndPopulateReport(String datasetIdentifier, long conversionInstance, List<Long> rowCounts, Long rowDataValidatedCount) { + private void validateAndPopulateReport(String datasetIdentifier, long conversionInstance, List<Long> rowCounts, List<Long> rowDataValidatedCount) { if (null == rowCounts || rowCounts.size() == 0) { this.warnConversions.put(String.format("Dataset: %s Instance: %s", datasetIdentifier, conversionInstance), "No conversion details found"); this.eventSubmitter.submit(EventConstants.VALIDATION_NOOP_EVENT, ImmutableMap.of("datasetUrn", datasetIdentifier)); return; } + if (null == rowDataValidatedCount || rowDataValidatedCount.size() == 0) { + this.warnConversions.put(String.format("Dataset: %s Instance: %s", datasetIdentifier, conversionInstance), "No conversion details found"); + this.eventSubmitter.submit(EventConstants.VALIDATION_NOOP_EVENT, ImmutableMap.of("datasetUrn", datasetIdentifier)); + return; + } long rowCountCached = -1; boolean isFirst = true; for (Long rowCount : rowCounts) { @@ -642,7 +647,7 @@ public class ValidationJob extends AbstractJob { } // Data count validation - if (rowCountCached == rowDataValidatedCount) { + if (rowCountCached == rowDataValidatedCount.get(0)) { this.dataValidationSuccessful.put(String.format("Dataset: %s Instance: %s", datasetIdentifier, conversionInstance), "Common rows matched expected value. Expected: " + rowCountCached + " Found: " + rowDataValidatedCount); } else {
