LENS-795 : Look ahead is failing with partition not found if any process time partitions are absent
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/d7f09ede Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/d7f09ede Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/d7f09ede Branch: refs/heads/current-release-line Commit: d7f09ede37e1f2d75e3e3d62a2e3a0a2375575cc Parents: e5b793c Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Sep 30 19:17:26 2015 +0530 Committer: Raju Bairishetti <raju.bairishe...@im0933-x0.corp.inmobi.com> Committed: Wed Sep 30 19:17:26 2015 +0530 ---------------------------------------------------------------------- .../PartitionRangesForPartitionColumns.java | 6 ++++-- .../lens/cube/parse/StorageTableResolver.java | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/d7f09ede/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java index b389a8f..fa3f6ef 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/PartitionRangesForPartitionColumns.java @@ -41,10 +41,12 @@ public class PartitionRangesForPartitionColumns extends HashMap<String, RangesPa .rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0]))); } - public Set<String> toSet() { + public Set<String> toSet(Set<String> partColsQueried) { Set<String> ret = Sets.newHashSet(); for (Map.Entry<String, RangesPartitionTimeline> entry : entrySet()) { - ret.add(entry.getKey() + ":" + entry.getValue().getRanges()); + if (partColsQueried.contains(entry.getKey())) { + ret.add(entry.getKey() + ":" + entry.getValue().getRanges()); + } } return ret; } http://git-wip-us.apache.org/repos/asf/lens/blob/d7f09ede/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 68ab5ab..f67fc26 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -363,7 +363,9 @@ class StorageTableResolver implements ContextRewriter { PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns(); boolean noPartsForRange = false; Set<String> unsupportedTimeDims = Sets.newHashSet(); + Set<String> partColsQueried = Sets.newHashSet(); for (TimeRange range : cubeql.getTimeRanges()) { + partColsQueried.add(range.getPartitionColumn()); StringBuilder extraWhereClause = new StringBuilder(); Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts); // If no partitions were found, then we'll fallback. @@ -389,6 +391,7 @@ class StorageTableResolver implements ContextRewriter { if (fallBackRange == null) { break; } + partColsQueried.add(fallBackRange.getPartitionColumn()); rangeParts = getPartitions(cfact.fact, fallBackRange, skipStorageCauses, missingParts); extraWhereClause.append(sep) .append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim)); @@ -433,7 +436,7 @@ class StorageTableResolver implements ContextRewriter { i.remove(); continue; } - Set<String> nonExistingParts = missingParts.toSet(); + Set<String> nonExistingParts = missingParts.toSet(partColsQueried); if (!nonExistingParts.isEmpty()) { addNonExistingParts(cfact.fact.getName(), nonExistingParts); } @@ -637,8 +640,10 @@ class StorageTableResolver implements ContextRewriter { for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, partWhereClauseFormat); - updateFactPartitionStorageTablesFrom(fact, innerPart, pPart.getStorageTables()); - partitions.add(innerPart); + updateFactPartitionStorageTablesFrom(fact, innerPart, pPart); + if (innerPart.isFound()) { + partitions.add(innerPart); + } } log.debug("added all sub partitions blindly in pPart: {}", pPart); } @@ -678,7 +683,7 @@ class StorageTableResolver implements ContextRewriter { updatePeriods, addNonExistingParts, failOnPartialData, skipStorageCauses, missingPartitions); } - void updateFactPartitionStorageTablesFrom(CubeFactTable fact, + private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, FactPartition part, Set<String> storageTableNames) throws LensException, HiveException, ParseException { for (String storageTableName : storageTableNames) { if (client.factPartitionExists(fact, part, storageTableName)) { @@ -687,4 +692,10 @@ class StorageTableResolver implements ContextRewriter { } } } + + private void updateFactPartitionStorageTablesFrom(CubeFactTable fact, + FactPartition part, FactPartition pPart) throws LensException, HiveException, ParseException { + updateFactPartitionStorageTablesFrom(fact, part, pPart.getStorageTables()); + part.setFound(part.isFound() && pPart.isFound()); + } }