Repository: lens Updated Branches: refs/heads/master f8a387917 -> c4cd6d84e
LENS-1391 : Fix pending review comments on MaxCoveringFactResolver and LeastPartitionResolver Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4cd6d84 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4cd6d84 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4cd6d84 Branch: refs/heads/master Commit: c4cd6d84ec533d84c24717591c890972f5d858cb Parents: f8a3879 Author: Lavkesh Lahngir <lavk...@linux.com> Authored: Wed Apr 19 11:05:57 2017 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Apr 19 11:05:57 2017 +0530 ---------------------------------------------------------------------- .../lens/cli/commands/LensSchemaCommands.java | 12 +--------- .../cube/parse/CandidateTablePruneCause.java | 7 ++++++ .../lens/cube/parse/LeastPartitionResolver.java | 15 +++---------- .../cube/parse/MaxCoveringFactResolver.java | 23 ++++++-------------- .../lens/cube/parse/StorageCandidate.java | 5 +++-- .../server/query/QueryAPIErrorResponseTest.java | 2 +- 6 files changed, 22 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index befe4e6..d3f9142 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -20,19 +20,11 @@ package org.apache.lens.cli.commands; import java.io.File; import java.io.FilenameFilter; -import java.util.List; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.lens.api.metastore.SchemaTraverser; -import org.apache.lens.api.metastore.XBaseCube; -import org.apache.lens.api.metastore.XDerivedCube; -import org.apache.lens.api.metastore.XDimension; -import org.apache.lens.api.metastore.XDimensionTable; -import org.apache.lens.api.metastore.XFactTable; -import org.apache.lens.api.metastore.XSegmentation; -import org.apache.lens.api.metastore.XStorage; +import org.apache.lens.api.metastore.*; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; @@ -42,9 +34,7 @@ import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; import org.springframework.shell.support.logging.HandlerUtils; import org.springframework.stereotype.Component; -import org.springframework.util.Assert; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; @Component http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index 1c0d356..29af419 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -235,6 +235,7 @@ public class CandidateTablePruneCause { // if a time dim is not supported by the fact. Would be set if and only if // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute private Set<String> unsupportedTimeDims; + private MaxCoveringFactResolver.TimeCovered maxTimeCovered; // time covered // ranges in which fact is invalid private List<TimeRange> invalidRanges; @@ -295,6 +296,12 @@ public class CandidateTablePruneCause { return cause; } + public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA); + cause.setMaxTimeCovered(timeCovered); + return cause; + } + public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String> colSet) { CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH); http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java index a9bd164..5adbc23 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java @@ -28,7 +28,7 @@ import org.apache.lens.server.api.error.LensException; import lombok.extern.slf4j.Slf4j; /** - * Prune candidate fact sets which require more partitions than minimum parts. + * Prune candidates which require more partitions than minimum parts. */ @Slf4j class LeastPartitionResolver implements ContextRewriter { @@ -41,9 +41,9 @@ class LeastPartitionResolver implements ContextRewriter { //The number of partitions being calculated is not the actual number of partitions, // they are number of time values now instead of partitions. // This seems fine, as the less number of time values actually represent the rollups on time. And with - // MaxCoveringFactResolver facts with less partitions which are not covering the range would be removed. + // MaxCoveringFactResolver candidates with less partitions which are not covering the range would be removed. for (Candidate candidate : cubeql.getCandidates()) { - factPartCount.put(candidate, getPartCount(candidate)); + factPartCount.put(candidate, candidate.getParticipatingPartitions().size()); } double minPartitions = Collections.min(factPartCount.values()); @@ -60,13 +60,4 @@ class LeastPartitionResolver implements ContextRewriter { } } } - - private int getPartCount(Candidate candidate) { - int parts = 0; - for (StorageCandidate sc : CandidateUtil.getStorageCandidates(candidate)) { - parts += sc.getNumQueriedParts(); - } - return parts; - } - } http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java index 34180d1..86e5634 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java @@ -23,7 +23,6 @@ import java.util.Iterator; import java.util.Map; import org.apache.lens.cube.metadata.FactPartition; -import org.apache.lens.cube.metadata.UpdatePeriod; import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline; import org.apache.lens.server.api.error.LensException; @@ -47,7 +46,7 @@ class MaxCoveringFactResolver implements ContextRewriter { public void rewriteContext(CubeQueryContext cubeql) { if (failOnPartialData) { // if fail on partial data is true, by the time this resolver starts, - // all candidate fact sets are covering full time range. We can avoid + // all candidates are covering full time range. We can avoid // redundant computation. return; } @@ -62,7 +61,7 @@ class MaxCoveringFactResolver implements ContextRewriter { } private void resolveByTimeCovered(CubeQueryContext cubeql) { - // For each part column, which candidate fact sets are covering how much amount. + // For each part column, which candidates are covering how much amount. // Later, we'll maximize coverage for each queried part column. Map<String, Map<Candidate, Long>> partCountsPerPartCol = Maps.newHashMap(); for (Candidate cand : cubeql.getCandidates()) { @@ -73,7 +72,7 @@ class MaxCoveringFactResolver implements ContextRewriter { partCountsPerPartCol.get(entry.getKey()).put(cand, entry.getValue()); } } - // for each queried partition, prune fact sets that are covering less range than max + // for each queried partition, prune candidate that are covering less range than max for (String partColQueried : cubeql.getPartitionColumnsQueried()) { if (partCountsPerPartCol.get(partColQueried) != null) { long maxTimeCovered = Collections.max(partCountsPerPartCol.get(partColQueried).values()); @@ -89,8 +88,7 @@ class MaxCoveringFactResolver implements ContextRewriter { log.info("Not considering Candidate:{} from Candidate set as it covers less time than the max" + " for partition column: {} which is: {}", candidate, partColQueried, timeCovered); iter.remove(); - cubeql.addCandidatePruningMsg(candidate, - new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.LESS_DATA)); + cubeql.addCandidatePruningMsg(candidate, CandidateTablePruneCause.lessData(timeCovered)); } } } @@ -112,7 +110,7 @@ class MaxCoveringFactResolver implements ContextRewriter { return; } - // We prune those candidate fact set, whose dataCompletenessFactor is less than maxDataCompletenessFactor + // We prune those candidate whose dataCompletenessFactor is less than maxDataCompletenessFactor Iterator<Candidate> iter = cubeql.getCandidates().iterator(); while (iter.hasNext()) { Candidate cand = iter.next(); @@ -121,8 +119,7 @@ class MaxCoveringFactResolver implements ContextRewriter { log.info("Not considering Candidate :{} from the list as the dataCompletenessFactor for this:{} is " + "less than the max:{}", cand, dataCompletenessFactor, maxDataCompletenessFactor); iter.remove(); - cubeql.addCandidatePruningMsg(cand, - new CandidateTablePruneCause(CandidateTablePruneCause.CandidateTablePruneCode.INCOMPLETE_PARTITION)); + cubeql.addCandidatePruningMsg(cand, CandidateTablePruneCause.incompletePartitions(null)); } } } @@ -145,19 +142,13 @@ class MaxCoveringFactResolver implements ContextRewriter { } /** - * Returns time covered by fact set for each part column. + * Returns time covered by candidate for each part column. * * @param cand * @return */ private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) { Map<String, Long> ret = Maps.newHashMap(); - UpdatePeriod smallest = UpdatePeriod.values()[UpdatePeriod.values().length - 1]; - for (FactPartition part : cand.getParticipatingPartitions()) { - if (part.getPeriod().compareTo(smallest) < 0) { - smallest = part.getPeriod(); - } - } PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns(); for (FactPartition part : cand.getParticipatingPartitions()) { if (part.isFound()) { http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java index d95cf27..95e3c95 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java @@ -152,8 +152,6 @@ public class StorageCandidate implements Candidate, CandidateTable { */ @Getter private Set<String> nonExistingPartitions = new HashSet<>(); - @Getter - private int numQueriedParts = 0; /** * This will be true if this storage candidate has multiple storage tables (one per update period) @@ -162,6 +160,9 @@ public class StorageCandidate implements Candidate, CandidateTable { @Getter private boolean isStorageTblsAtUpdatePeriodLevel; + @Getter + private int numQueriedParts = 0; + public StorageCandidate(StorageCandidate sc) throws LensException { this(sc.getCube(), sc.getFact(), sc.getStorageName(), sc.getCubeql()); this.validUpdatePeriods.addAll(sc.getValidUpdatePeriods()); http://git-wip-us.apache.org/repos/asf/lens/blob/c4cd6d84/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java index 064da01..5409d21 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java @@ -283,7 +283,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest { //Create a StorageTable XStorageTables tables = new XStorageTables(); - tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY")); + tables.getStorageTable().add(createStorageTblElement(testStorage, "DAILY")); xFactTable.setStorageTables(tables); createFactFailFast(target, sessionId, xFactTable, mt);