LENS-1443: Fallback ranges not working for virtual facts
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c174583f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c174583f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c174583f Branch: refs/heads/master Commit: c174583ff7946d0347e0a0a87272f42c0023aecf Parents: 9da5b40 Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Jun 21 13:20:09 2017 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Thu Jul 13 14:42:50 2017 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/metadata/FactTable.java | 38 ++++++++++---------- .../apache/lens/cube/parse/JoinCandidate.java | 7 ++-- .../lens/cube/parse/LeastPartitionResolver.java | 21 +++++------ .../lens/cube/parse/StorageCandidate.java | 2 +- .../lens/cube/parse/StorageTableResolver.java | 5 +-- 5 files changed, 37 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java index f87cf44..a463c47 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java @@ -35,132 +35,132 @@ public interface FactTable extends Named { * * @return Map of storage to set of update periods */ - public Map<String, Set<UpdatePeriod>> getUpdatePeriods(); + Map<String, Set<UpdatePeriod>> getUpdatePeriods(); /** * Cube to which this fact belongs to * * @return the cube string */ - public String getCubeName(); + String getCubeName(); /** * The set of Storage names * * @return set of strings */ - public Set<String> getStorages(); + Set<String> getStorages(); /** *The type of the fact * * @return table type {@link CubeTableType} */ - public CubeTableType getTableType(); + CubeTableType getTableType(); /** * Config properties * * @return map of string, string */ - public Map<String, String> getProperties(); + Map<String, String> getProperties(); /** * Valid columns of the fact * * @return list of column names */ - public Set<String> getValidColumns(); + Set<String> getValidColumns(); /** * Weight of the fact * * @return weight of the fact in double */ - public double weight(); + double weight(); /** * Set of all the columns names of the fact * * @return set of column names */ - public Set<String> getAllFieldNames(); + Set<String> getAllFieldNames(); /** *tag for checking data completeness * * @return Tag String */ - public String getDataCompletenessTag(); + String getDataCompletenessTag(); /** * List of columns of the fact * * @return set of {@link FieldSchema} */ - public List<FieldSchema> getColumns(); + List<FieldSchema> getColumns(); /** * Is Aggregated Fact * * @return true if fact is Aggregated , false otherwise */ - public boolean isAggregated(); + boolean isAggregated(); /** * Absolute start time of the fact * * @return Absolute Start time of the fact {@link Date} */ - public Date getAbsoluteStartTime(); + Date getAbsoluteStartTime(); /** * Relative start time of the fact * * @return Relative Start time of the fact {@link Date} */ - public Date getRelativeStartTime(); + Date getRelativeStartTime(); /** * Start time of the fact * * @return Start time of the fact {@link Date} */ - public Date getStartTime(); + Date getStartTime(); /** * Absolute end time of the fact * * @return Absolute End time of the fact {@link Date} */ - public Date getAbsoluteEndTime(); + Date getAbsoluteEndTime(); /** * Relative End time of the Fact * * @return Relative end time of the fact {@link Date} */ - public Date getRelativeEndTime(); + Date getRelativeEndTime(); /** * End time of the fact * * @return End time of the fact {@link Date} */ - public Date getEndTime(); + Date getEndTime(); /** * Is Virtual Fact * * @return true if fact is a virtual fact, false otherwise */ - public boolean isVirtualFact(); + boolean isVirtualFact(); /** * Storage name of the fact * * @return Storage name of the fact */ - public String getSourceFactName(); + String getSourceFactName(); } http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java index d9915f4..1c2b9f8 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinCandidate.java @@ -91,12 +91,11 @@ public class JoinCandidate implements Candidate { @Override public boolean evaluateCompleteness(TimeRange timeRange, TimeRange parentTimeRange, boolean failOnPartialData) throws LensException { + boolean complete = true; for (Candidate child : children) { - if (!child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData)) { - return false; - } + complete &= child.evaluateCompleteness(timeRange, parentTimeRange, failOnPartialData); } - return true; + return complete; } /** http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java index 22b1d03..a76e9b1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java @@ -48,17 +48,18 @@ class LeastPartitionResolver implements ContextRewriter { factPartCount.put(candidate, parts); } } + if (!factPartCount.isEmpty()) { + double minPartitions = Collections.min(factPartCount.values()); - double minPartitions = Collections.min(factPartCount.values()); - - for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) { - Candidate candidate = i.next(); - if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) { - log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}", - candidate, factPartCount.get(candidate), minPartitions); - i.remove(); - cubeql.addCandidatePruningMsg(candidate, - new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS)); + for (Iterator<Candidate> i = cubeql.getCandidates().iterator(); i.hasNext();) { + Candidate candidate = i.next(); + if (factPartCount.containsKey(candidate) && factPartCount.get(candidate) > minPartitions) { + log.info("Not considering Candidate:{} as it requires more partitions to be" + " queried:{} minimum:{}", + candidate, factPartCount.get(candidate), minPartitions); + i.remove(); + cubeql.addCandidatePruningMsg(candidate, + new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.MORE_PARTITIONS)); + } } } } http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index 6e5aa4c..c6ef6d2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -604,7 +604,7 @@ public class StorageCandidate implements Candidate, CandidateTable { break; } } - TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getName(), cubeQueryContext); + TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getSourceFactName(), cubeQueryContext); log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange); if (fallBackRange == null) { break; http://git-wip-us.apache.org/repos/asf/lens/blob/c174583f/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index d7da8cb..7f0f2d4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -20,6 +20,7 @@ package org.apache.lens.cube.parse; import static org.apache.lens.cube.parse.CandidateTablePruneCause.incompletePartitions; import static org.apache.lens.cube.parse.CandidateTablePruneCause.partitionColumnsMissing; +import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange; import java.util.*; @@ -324,13 +325,13 @@ class StorageTableResolver implements ContextRewriter { } else if (!sc.getValidUpdatePeriods().contains(UpdatePeriod.CONTINUOUS)) { if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); - TimeRange fallBackRange = StorageUtil.getFallbackRange(range, sc.getFact().getName(), cubeql); + TimeRange fallBackRange = getFallbackRange(range, sc.getFact().getSourceFactName(), cubeql); while (fallBackRange != null) { pruningCauseForThisTimeRange = null; if (!client.partColExists(sc.getFact(), sc.getStorageName(), fallBackRange.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); - fallBackRange = StorageUtil.getFallbackRange(fallBackRange, sc.getFact().getName(), cubeql); + fallBackRange = getFallbackRange(fallBackRange, sc.getFact().getSourceFactName(), cubeql); } else { if (!sc.isPartiallyValidForTimeRange(fallBackRange)) { pruningCauseForThisTimeRange =
