LENS-974: Add cube-segmentation for base cube
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b58749e2 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b58749e2 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b58749e2 Branch: refs/heads/current-release-line Commit: b58749e2061e0e731fc1855e0bf4a3b37c601c38 Parents: d78766c Author: Rajat Khandelwal <[email protected]> Authored: Tue May 2 17:25:57 2017 +0530 Committer: Rajat Khandelwal <[email protected]> Committed: Tue May 2 17:25:58 2017 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/cube-0.1.xsd | 4 +- .../lens/cli/commands/LensSchemaCommands.java | 11 +- .../NoCandidateFactAvailableException.java | 7 +- .../org/apache/lens/cube/metadata/Cube.java | 20 +- .../lens/cube/metadata/CubeFactTable.java | 10 + .../org/apache/lens/cube/metadata/DateUtil.java | 15 +- .../lens/cube/metadata/MetastoreUtil.java | 41 +- .../lens/cube/metadata/TimePartitionRange.java | 6 + .../apache/lens/cube/metadata/TimeRange.java | 109 +-- .../lens/cube/parse/AggregateResolver.java | 13 +- .../apache/lens/cube/parse/AliasReplacer.java | 3 +- .../org/apache/lens/cube/parse/Candidate.java | 265 +++++- .../parse/CandidateCoveringSetsResolver.java | 135 +-- .../lens/cube/parse/CandidateExploder.java | 45 + .../cube/parse/CandidateTablePruneCause.java | 40 + .../lens/cube/parse/CandidateTableResolver.java | 105 +-- .../apache/lens/cube/parse/CandidateUtil.java | 216 +---- .../lens/cube/parse/CubeQueryConfUtil.java | 1 + .../lens/cube/parse/CubeQueryContext.java | 276 ++---- .../lens/cube/parse/CubeQueryRewriter.java | 31 +- .../apache/lens/cube/parse/DefaultQueryAST.java | 8 +- .../cube/parse/DenormalizationResolver.java | 17 +- .../apache/lens/cube/parse/DimHQLContext.java | 87 +- .../lens/cube/parse/DimOnlyHQLContext.java | 61 +- .../lens/cube/parse/ExpressionResolver.java | 57 +- .../apache/lens/cube/parse/GroupbyResolver.java | 2 +- .../lens/cube/parse/HQLContextInterface.java | 85 -- .../org/apache/lens/cube/parse/HQLParser.java | 81 +- .../apache/lens/cube/parse/JoinCandidate.java | 135 ++- .../apache/lens/cube/parse/JoinResolver.java | 30 +- .../lens/cube/parse/LeastPartitionResolver.java | 9 +- .../parse/MultiCandidateQueryWriterContext.java | 93 ++ .../org/apache/lens/cube/parse/PruneCauses.java | 25 +- .../lens/cube/parse/QueriedPhraseContext.java | 78 +- .../org/apache/lens/cube/parse/QueryAST.java | 24 +- .../org/apache/lens/cube/parse/QueryWriter.java | 31 + .../lens/cube/parse/QueryWriterContext.java | 34 + .../lens/cube/parse/SegmentationCandidate.java | 382 ++++++++ .../cube/parse/SegmentationInnerRewriter.java | 76 ++ .../lens/cube/parse/SimpleHQLContext.java | 85 +- .../lens/cube/parse/StorageCandidate.java | 440 ++++----- .../cube/parse/StorageCandidateHQLContext.java | 164 ++++ .../lens/cube/parse/StorageTableResolver.java | 212 ++--- .../org/apache/lens/cube/parse/StorageUtil.java | 21 +- .../lens/cube/parse/TimerangeResolver.java | 2 +- .../apache/lens/cube/parse/UnionCandidate.java | 163 ++-- .../lens/cube/parse/UnionQueryWriter.java | 173 ++-- .../lens/cube/parse/join/AutoJoinContext.java | 80 +- .../cube/parse/join/BridgeTableJoinContext.java | 8 +- .../apache/lens/cube/parse/join/JoinClause.java | 15 +- .../cube/metadata/TestCubeMetastoreClient.java | 1 - .../apache/lens/cube/metadata/TestDateUtil.java | 14 +- .../apache/lens/cube/parse/CubeTestSetup.java | 104 ++- .../lens/cube/parse/TestBaseCubeQueries.java | 36 +- .../lens/cube/parse/TestCubeRewriter.java | 34 +- .../parse/TestCubeSegmentationRewriter.java | 322 +++++++ .../cube/parse/TestDenormalizationResolver.java | 9 +- .../lens/cube/parse/TestExpressionResolver.java | 2 +- .../apache/lens/cube/parse/TestHQLParser.java | 66 +- .../lens/cube/parse/TestJoinResolver.java | 14 +- .../lens/cube/parse/TestQueryMetrics.java | 38 +- .../lens/cube/parse/TestQueryRewrite.java | 14 +- .../lens/cube/parse/TestTimeRangeResolver.java | 11 +- .../lens/cube/parse/TestUnionQueries.java | 13 +- .../resources/schema/cubes/base/b1c1cube.xml | 903 ++++++++++++++++++ .../test/resources/schema/cubes/base/b1cube.xml | 909 +++++++++++++++++++ .../resources/schema/cubes/base/b2c1cube.xml | 903 ++++++++++++++++++ .../test/resources/schema/cubes/base/b2cube.xml | 909 +++++++++++++++++++ .../resources/schema/cubes/base/basecube.xml | 4 + .../resources/schema/cubes/base/testcube.xml | 74 +- .../cubes/derived/union_join_ctx_der1.xml | 1 + .../test/resources/schema/facts/b1b2fact1.xml | 56 ++ .../src/test/resources/schema/facts/b1fact1.xml | 85 ++ .../src/test/resources/schema/facts/b2fact1.xml | 66 ++ .../test/resources/schema/facts/testfact2.xml | 3 + .../schema/facts/union_join_ctx_fact1.xml | 9 - .../resources/schema/segmentations/b1seg1.xml | 37 + .../resources/schema/segmentations/b2seg1.xml | 37 + .../resources/schema/segmentations/seg1.xml | 13 +- .../resources/schema/segmentations/seg2.xml | 37 + .../comparators/ChainedComparatorTest.java | 22 +- 81 files changed, 6947 insertions(+), 1830 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-api/src/main/resources/cube-0.1.xsd ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd index 68ccc13..8158e6d 100644 --- a/lens-api/src/main/resources/cube-0.1.xsd +++ b/lens-api/src/main/resources/cube-0.1.xsd @@ -1126,7 +1126,7 @@ The following properties can be specified at a segment level : 1. lens.metastore.cube.column.mapping : The column mapping for columns of segment if they are different in underlying cube. The value is speciified with comma separated map entries specified with - key-values separated by equalto. Example value: id=id1,name=name1 + key-values separated by equalto. Example value: id=id1,name=name1. Not yet supported. </xs:documentation> </xs:annotation> </xs:element> @@ -1141,7 +1141,7 @@ </xs:documentation> </xs:annotation> <xs:sequence> - <xs:element name="segment" minOccurs="2" maxOccurs="unbounded" type="x_segment"/> + <xs:element name="segment" minOccurs="1" maxOccurs="unbounded" type="x_segment"/> </xs:sequence> </xs:complexType> http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java index d3f9142..60bd9e0 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,7 +24,14 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.lens.api.metastore.*; +import org.apache.lens.api.metastore.SchemaTraverser; +import org.apache.lens.api.metastore.XBaseCube; +import org.apache.lens.api.metastore.XDerivedCube; +import org.apache.lens.api.metastore.XDimension; +import org.apache.lens.api.metastore.XDimensionTable; +import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XSegmentation; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.cli.commands.annotations.UserDocumentation; import org.springframework.beans.factory.annotation.Autowired; http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java index 21dda16..3188d1b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,9 +18,10 @@ */ package org.apache.lens.cube.error; + +import org.apache.lens.cube.parse.Candidate; import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.cube.parse.PruneCauses; -import org.apache.lens.cube.parse.StorageCandidate; import org.apache.lens.server.api.error.LensException; import lombok.Getter; @@ -33,8 +34,8 @@ import lombok.Getter; public class NoCandidateFactAvailableException extends LensException { @Getter - private final PruneCauses<StorageCandidate> briefAndDetailedError; + private final PruneCauses<Candidate> briefAndDetailedError; public NoCandidateFactAvailableException(CubeQueryContext cubeql) { super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), cubeql.getStoragePruningMsgs().getBriefCause()); http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java index b376aaf..7c3da2c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java @@ -24,6 +24,7 @@ import java.util.*; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.ql.metadata.Table; +import com.google.common.collect.Sets; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -96,6 +97,12 @@ public class Cube extends AbstractBaseTable implements CubeInterface { public Set<CubeMeasure> getMeasures() { return measures; } + public Optional<Date> getColumnStartTime(String column) { + return Optional.ofNullable(getColumnByName(column)).map(CubeColumn::getStartTime); + } + public Optional<Date> getColumnEndTime(String column) { + return Optional.ofNullable(getColumnByName(column)).map(CubeColumn::getEndTime); + } public Set<CubeDimAttribute> getDimAttributes() { return dimensions; @@ -363,10 +370,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface { @Override public boolean allFieldsQueriable() { String canBeQueried = getProperties().get(MetastoreConstants.CUBE_ALL_FIELDS_QUERIABLE); - if (canBeQueried != null) { - return Boolean.parseBoolean(canBeQueried); - } - return true; + return canBeQueried == null || Boolean.parseBoolean(canBeQueried.toLowerCase()); } @Override @@ -378,6 +382,14 @@ public class Cube extends AbstractBaseTable implements CubeInterface { return fieldNames; } + public Set<CubeColumn> getAllFields() { + Set<CubeColumn> columns = Sets.newHashSet(); + columns.addAll(getMeasures()); + columns.addAll(getDimAttributes()); + columns.addAll(getExpressions()); + return columns; + } + /** * @see org.apache.lens.cube.metadata.AbstractBaseTable */ http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index e00122d..88bc1fc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -19,6 +19,7 @@ package org.apache.lens.cube.metadata; import java.util.*; +import java.util.function.Predicate; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.UpdatePeriod.UpdatePeriodComparator; @@ -75,6 +76,15 @@ public class CubeFactTable extends AbstractCubeTable { addProperties(); } + public boolean hasColumn(String column) { + List<String> validColumns = getValidColumns(); + if (validColumns != null) { + return validColumns.contains(column); + } else { + return getColumns().stream().map(FieldSchema::getName).anyMatch(Predicate.isEqual(column)); + } + } + @Override protected void addProperties() { super.addProperties(); http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java index 99ad233..17e30a1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/DateUtil.java @@ -21,12 +21,10 @@ package org.apache.lens.cube.metadata; import static java.util.Calendar.MONTH; import java.text.DateFormat; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Set; -import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -149,23 +147,22 @@ public final class DateUtil { public static String relativeToAbsolute(String relative, Date now) throws LensException { if (RELDATE_VALIDATOR.matcher(relative).matches()) { - return ABSDATE_PARSER.get().format(resolveRelativeDate(relative, now)); + return formatAbsDate(resolveRelativeDate(relative, now)); } else { return relative; } } + public static String formatAbsDate(Date date) { + return ABSDATE_PARSER.get().format(date).replaceAll("([-:,]0+)+$", ""); + } + static Cache<String, Date> stringToDateCache = CacheBuilder.newBuilder() .expireAfterWrite(2, TimeUnit.HOURS).maximumSize(100).build(); public static Date resolveAbsoluteDate(final String str) throws LensException { try { - return stringToDateCache.get(str, new Callable<Date>() { - @Override - public Date call() throws ParseException { - return ABSDATE_PARSER.get().parse(getAbsDateFormatString(str)); - } - }); + return stringToDateCache.get(str, () -> ABSDATE_PARSER.get().parse(getAbsDateFormatString(str))); } catch (Exception e) { log.error("Invalid date format. expected only {} date provided:{}", ABSDATE_FMT, str, e); throw new LensException(LensCubeErrorCode.WRONG_TIME_RANGE_FORMAT.getLensErrorInfo(), ABSDATE_FMT, str); http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index 599027f..40f766b 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -23,17 +23,30 @@ import static org.apache.lens.cube.error.LensCubeErrorCode.EXPRESSION_NOT_PARSAB import static org.apache.lens.cube.metadata.MetastoreConstants.*; import java.text.ParseException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; import org.apache.hadoop.hive.ql.parse.ParseDriver; import org.apache.hadoop.hive.ql.parse.ParseUtils; +import org.antlr.runtime.CommonToken; + import com.google.common.collect.Sets; public class MetastoreUtil { @@ -428,7 +441,7 @@ public class MetastoreUtil { public static Set<Named> getNamedSetFromStringSet(Set<String> strings) { Set<Named> nameds = Sets.newHashSet(); - for(final String s: strings) { + for (final String s : strings) { nameds.add(new Named() { @Override public String getName() { @@ -438,6 +451,7 @@ public class MetastoreUtil { } return nameds; } + public static <E extends Named> void addNameStrings(Map<String, String> props, String key, Collection<E> set) { addNameStrings(props, key, set, MAX_PARAM_LENGTH); } @@ -579,13 +593,18 @@ public class MetastoreUtil { } public static ASTNode copyAST(ASTNode original) { + return copyAST(original, x -> Pair.of(new ASTNode(x), true)); + } - ASTNode copy = new ASTNode(original); // Leverage constructor - - if (original.getChildren() != null) { - for (Object o : original.getChildren()) { - ASTNode childCopy = copyAST((ASTNode) o); - copy.addChild(childCopy); + public static ASTNode copyAST(ASTNode original, + Function<ASTNode, Pair<ASTNode, Boolean>> overrideCopyFunction) { + Pair<ASTNode, Boolean> copy1 = overrideCopyFunction.apply(original); + ASTNode copy = copy1.getLeft(); + if (copy1.getRight()) { + if (original.getChildren() != null) { + for (Node o : original.getChildren()) { + copy.addChild(copyAST((ASTNode) o, overrideCopyFunction)); + } } } return copy; @@ -594,5 +613,7 @@ public class MetastoreUtil { public static String getUpdatePeriodStoragePrefixKey(String factTableName, String storageName, String updatePeriod) { return MetastoreUtil.getFactKeyPrefix(factTableName) + "." + storageName + "." + updatePeriod; } - + public static ASTNode getStringLiteralAST(String literal) { + return new ASTNode(new CommonToken(HiveParser.StringLiteral, literal)); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java index 2e85111..8e17f02 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java @@ -20,6 +20,8 @@ package org.apache.lens.cube.metadata; import java.util.Date; import java.util.Iterator; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.apache.lens.server.api.error.LensException; @@ -135,4 +137,8 @@ public class TimePartitionRange implements Iterable<TimePartition>, Named { public boolean isValidAndNonEmpty() { return begin.before(end); } + + Stream<TimePartition> stream() { + return StreamSupport.stream(spliterator(), false); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java index 8286894..3e58e63 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimeRange.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,11 +18,14 @@ */ package org.apache.lens.cube.metadata; +import static java.util.Comparator.naturalOrder; + import static org.apache.lens.cube.metadata.DateUtil.ABSDATE_PARSER; import java.util.Calendar; import java.util.Date; import java.util.Set; +import java.util.stream.Stream; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.server.api.error.LensException; @@ -32,21 +35,34 @@ import org.apache.hadoop.hive.ql.parse.ASTNode; import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NonNull; /** * Timerange data structure */ @JsonIgnoreProperties({"astNode", "parent"}) @Data +@EqualsAndHashCode(of = {"partitionColumn", "fromDate", "toDate"}) +@Builder public class TimeRange { - private String partitionColumn; - private Date toDate; - private Date fromDate; - private ASTNode astNode; - private ASTNode parent; - private int childIndex; + private final String partitionColumn; + private final Date toDate; + @NonNull + private final Date fromDate; + private final ASTNode astNode; + private final ASTNode parent; + private final int childIndex; + + public TimeRange truncate(Date candidateStartTime, Date candidateEndTime) { + return TimeRange.builder().partitionColumn(getPartitionColumn()) + .fromDate(Stream.of(getFromDate(), candidateStartTime).max(naturalOrder()).orElse(candidateStartTime)) + .toDate(Stream.of(getToDate(), candidateEndTime).min(naturalOrder()).orElse(candidateEndTime)) + .build(); + } public boolean isCoverableBy(Set<UpdatePeriod> updatePeriods) { return DateUtil.isCoverableBy(fromDate, toDate, updatePeriods); @@ -61,7 +77,7 @@ public class TimeRange { * @throws LensException If the truncated time range is invalid. */ public TimeRange truncate(UpdatePeriod updatePeriod) throws LensException { - TimeRange timeRange = new TimeRangeBuilder().partitionColumn(partitionColumn) + TimeRange timeRange = TimeRange.builder().partitionColumn(partitionColumn) .fromDate(updatePeriod.getCeilDate(fromDate)).toDate(updatePeriod.getFloorDate(toDate)).build(); timeRange.validate(); return timeRange; @@ -71,80 +87,37 @@ public class TimeRange { return toDate.getTime() - fromDate.getTime(); } - public static class TimeRangeBuilder { - private final TimeRange range; - - public TimeRangeBuilder() { - this.range = new TimeRange(); - } - - public TimeRangeBuilder partitionColumn(String col) { - range.partitionColumn = col; - return this; - } - - public TimeRangeBuilder toDate(Date to) { - range.toDate = to; - return this; - } - - public TimeRangeBuilder fromDate(Date from) { - range.fromDate = from; - return this; - } - - public TimeRangeBuilder astNode(ASTNode node) { - range.astNode = node; - return this; - } - public TimeRangeBuilder parent(ASTNode parent) { - range.parent = parent; - return this; - } - - public TimeRangeBuilder childIndex(int childIndex) { - range.childIndex = childIndex; - return this; - } - - public TimeRange build() { - return range; - } + public TimeRangeBuilder cloneAsBuilder() { + return builder(). + astNode(getAstNode()).childIndex(getChildIndex()).parent(getParent()).partitionColumn(getPartitionColumn()); } - - public static TimeRangeBuilder getBuilder() { - return new TimeRangeBuilder(); + private boolean fromEqualsTo() { + return fromDate.equals(toDate); } - - private TimeRange() { - + private boolean fromAfterTo() { + return fromDate.after(toDate); + } + public boolean isValid() { + return !(fromEqualsTo() || fromAfterTo()); } - public void validate() throws LensException { - if (partitionColumn == null || fromDate == null || toDate == null || fromDate.equals(toDate)) { + if (partitionColumn == null || fromDate == null || toDate == null || fromEqualsTo()) { throw new LensException(LensCubeErrorCode.INVALID_TIME_RANGE.getLensErrorInfo()); } - if (fromDate.after(toDate)) { + if (fromAfterTo()) { throw new LensException(LensCubeErrorCode.FROM_AFTER_TO.getLensErrorInfo(), fromDate.toString(), toDate.toString()); } } - public String toTimeDimWhereClause() { - return toTimeDimWhereClause(null, partitionColumn); - } - public String toTimeDimWhereClause(String prefix, String column) { if (StringUtils.isNotBlank(column)) { column = prefix + "." + column; } - return new StringBuilder() - .append(column).append(" >= '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(fromDate)).append("'") - .append(" AND ") - .append(column).append(" < '").append(DateUtil.HIVE_QUERY_DATE_PARSER.get().format(toDate)).append("'") - .toString(); + return column + " >= '" + DateUtil.HIVE_QUERY_DATE_PARSER.get().format(fromDate) + "'" + + " AND " + column + " < '" + DateUtil.HIVE_QUERY_DATE_PARSER.get().format(toDate) + "'"; } @Override @@ -155,12 +128,12 @@ public class TimeRange { /** iterable from fromDate(including) to toDate(excluding) incrementing increment units of updatePeriod */ public static Iterable iterable(Date fromDate, Date toDate, UpdatePeriod updatePeriod, int increment) { - return TimeRange.getBuilder().fromDate(fromDate).toDate(toDate).build().iterable(updatePeriod, increment); + return TimeRange.builder().fromDate(fromDate).toDate(toDate).build().iterable(updatePeriod, increment); } /** iterable from fromDate(including) incrementing increment units of updatePeriod. Do this numIters times */ public static Iterable iterable(Date fromDate, int numIters, UpdatePeriod updatePeriod, int increment) { - return TimeRange.getBuilder().fromDate(fromDate).build().iterable(updatePeriod, numIters, increment); + return TimeRange.builder().fromDate(fromDate).build().iterable(updatePeriod, numIters, increment); } private Iterable iterable(UpdatePeriod updatePeriod, int numIters, int increment) { @@ -181,7 +154,7 @@ public class TimeRange { private long numIters; private int increment; - public Iterable(UpdatePeriod updatePeriod, long numIters, int increment) { + Iterable(UpdatePeriod updatePeriod, long numIters, int increment) { this.updatePeriod = updatePeriod; this.numIters = numIters; if (this.numIters < 0) { http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java index 30b1a90..be7180e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java @@ -73,14 +73,12 @@ class AggregateResolver implements ContextRewriter { Iterator<Candidate> candItr = cubeql.getCandidates().iterator(); while (candItr.hasNext()) { Candidate candidate = candItr.next(); - if (candidate instanceof StorageCandidate) { + if (candidate instanceof StorageCandidate) { // only work on storage candidates StorageCandidate sc = (StorageCandidate) candidate; if (sc.getFact().isAggregated()) { cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.missingDefaultAggregate()); candItr.remove(); } - } else { - throw new LensException("Not a storage candidate!!"); } } nonDefaultAggregates = true; @@ -116,11 +114,10 @@ class AggregateResolver implements ContextRewriter { ASTNode child = (ASTNode) selectAST.getChild(i); String expr = HQLParser.getString((ASTNode) child.getChild(0).getChild(1)); if (cubeql.getQueriedExprs().contains(expr)) { - for (Iterator<ExpressionResolver.ExpressionContext> itrContext = - cubeql.getExprCtx().getAllExprsQueried().get(expr).iterator(); itrContext.hasNext();) { - for (Iterator<ExprColumn.ExprSpec> itrCol = - itrContext.next().getExprCol().getExpressionSpecs().iterator(); itrCol.hasNext();) { - ASTNode exprAST = HQLParser.parseExpr(itrCol.next().getExpr(), cubeql.getConf()); + for (ExpressionResolver.ExpressionContext expressionContext + : cubeql.getExprCtx().getAllExprsQueried().get(expr)) { + for (ExprColumn.ExprSpec exprSpec : expressionContext.getExprCol().getExpressionSpecs()) { + ASTNode exprAST = HQLParser.parseExpr(exprSpec.getExpr(), cubeql.getConf()); if (HQLParser.isAggregateAST(exprAST)) { return true; } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java index bbf8ab9..30e81cd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java @@ -154,7 +154,8 @@ class AliasReplacer implements ContextRewriter { } } - static void extractTabAliasForCol(Map<String, String> colToTableAlias, TrackQueriedColumns tqc) throws LensException { + private static void extractTabAliasForCol(Map<String, String> colToTableAlias, TrackQueriedColumns tqc) + throws LensException { Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE); if (columns == null) { return; http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java index f241cb3..9f07336 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/Candidate.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,13 +19,25 @@ package org.apache.lens.cube.parse; import java.util.Collection; +import java.util.Collections; import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; +import org.apache.lens.cube.metadata.CubeInterface; +import org.apache.lens.cube.metadata.CubeMetastoreClient; +import org.apache.lens.cube.metadata.Dimension; import org.apache.lens.cube.metadata.FactPartition; import org.apache.lens.cube.metadata.TimeRange; import org.apache.lens.server.api.error.LensException; +import org.apache.hadoop.conf.Configuration; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + /** * This interface represents candidates that are involved in different phases of query rewriting. * At the lowest level, Candidate is represented by a StorageCandidate that has a fact on a storage @@ -41,36 +53,71 @@ public interface Candidate { /** * Returns all the fact columns * - * @return + * @return collection of column names */ Collection<String> getColumns(); /** + * Returns whether this candidate has the asked column or not + * @param column column to check + * @return whether this candidate contains the column + */ + default boolean hasColumn(String column) { + return getColumns().contains(column); + } + + /** * Start Time for this candidate (calculated based on schema) * - * @return + * @return start time of this candidate */ Date getStartTime(); /** * End Time for this candidate (calculated based on schema) * - * @return + * @return end time of this candidate */ Date getEndTime(); /** - * Returns the cost of this candidate - * - * @return + * Returns true if the Candidate is valid for all the timeranges based on its start and end times. + * @param timeRanges time ranges to check + * @return whether this candidate is completely valid for all the time ranges + */ + default boolean isCompletelyValidForTimeRanges(List<TimeRange> timeRanges) { + return timeRanges.stream().allMatch(range -> range.truncate(getStartTime(), getEndTime()).equals(range)); + } + + /** + * Utility method to check whether this candidate is partially valid for any of the given time ranges + * @param timeRanges time ranges to check + * @return whether this candidate is partially valid for any of the ranges + */ + default boolean isPartiallyValidForTimeRanges(List<TimeRange> timeRanges) { + return timeRanges.stream().map(timeRange -> + timeRange.truncate(getStartTime(), getEndTime())).anyMatch(TimeRange::isValid); + } + + /** + * Utility method for checking whether this candidate is partially valid for the given time range + * @param timeRange time range to check + * @return whether this candidate is partially valid + */ + default boolean isPartiallyValidForTimeRange(TimeRange timeRange) { + return isPartiallyValidForTimeRanges(Collections.singletonList(timeRange)); + } + + /** + * @return the cost of this candidate */ double getCost(); /** * Returns true if this candidate contains the given candidate * - * @param candidate - * @return + * @param candidate candidate to check + * @return whether this contains the candidate in question */ boolean contains(Candidate candidate); @@ -78,9 +125,17 @@ public interface Candidate { * Returns child candidates of this candidate if any. * Note: StorageCandidate will return null * - * @return + * @return child candidates if this is a complex candidate, else null + */ + Collection<? extends Candidate> getChildren(); + + /** + * Count of children + * @return number of children it has. 0 if null. */ - Collection<Candidate> getChildren(); + default int getChildrenCount() { + return Optional.ofNullable(getChildren()).map(Collection::size).orElse(0); + } /** * Is time range coverable based on start and end times configured in schema for the composing storage candidates @@ -92,9 +147,9 @@ public interface Candidate { * registered partitions. So isTimeRangeCoverable = false implies evaluateCompleteness = false but vice versa is * not true. * - * @param timeRange - * @return - * @throws LensException + * @param timeRange The time range to check + * @return whether this time range is coverable by this candidate + * @throws LensException propagated exceptions */ boolean isTimeRangeCoverable(TimeRange timeRange) throws LensException; @@ -117,23 +172,193 @@ public interface Candidate { * Note: This method can be called only after call to * {@link #evaluateCompleteness(TimeRange, TimeRange, boolean)} * - * @return + * @return a set of participating partitions */ Set<FactPartition> getParticipatingPartitions(); /** - * Checks whether an expression is evaluable by a candidate + * Checks whether an expression is evaluable by this candidate * 1. For a JoinCandidate, atleast one of the child candidates should be able to answer the expression * 2. For a UnionCandidate, all child candidates should answer the expression * - * @param expr :Expression need to be evaluated for Candidate - * @return + * @param expressionContext Expression to be evaluated for Candidate + * @return Whether the given expression is evaluable or not + */ + boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expressionContext); + + /** + * Checks whether an expression is evaluable by this candidate + * 1. For a JoinCandidate, atleast one of the child candidates should be able to answer the expression + * 2. For a UnionCandidate, all child candidates should answer the expression + * + * @param expr Expression to be evaluated for Candidate + * @return Whether the given expression is evaluable or not + */ + boolean isExpressionEvaluable(String expr); + + /** + * Checks whether a dim attribute is evaluable by this candidate + * @param dim dim attribute + * @return whether the dim attribute is evaluable by this candidate + * @throws LensException propageted exception */ - boolean isExpressionEvaluable(ExpressionResolver.ExpressionContext expr); + boolean isDimAttributeEvaluable(String dim) throws LensException; /** * Gets the index positions of answerable measure phrases in CubeQueryContext#selectPhrases - * @return + * @return set of indices of answerable phrases */ Set<Integer> getAnswerableMeasurePhraseIndices(); + + /** + * Clones this candidate + * @return the clone + * @throws LensException propagated exception + */ + default Candidate copy() throws LensException { + throw new LensException("Candidate " + this + " doesn't support copy"); + } + + /** + * Checks whether the given queries phrase is evaluable by this candidate + * @param phrase Phrase to check + * @return whether the phrase is evaluable by this candidate + * @throws LensException propagated exception + */ + boolean isPhraseAnswerable(QueriedPhraseContext phrase) throws LensException; + + /** + * Add `index` as answerable index in a pre-decided list of queried phrases. + * @param index index to mark as answerable + */ + void addAnswerableMeasurePhraseIndices(int index); + /** + * Default method to update querieble phrase indices in candidate + * @param qpcList List of queries phrases + * @throws LensException propagated exception + */ + default void updateStorageCandidateQueriablePhraseIndices(List<QueriedPhraseContext> qpcList) throws LensException { + for (int index = 0; index < qpcList.size(); index++) { + if (isPhraseAnswerable(qpcList.get(index))) { + addAnswerableMeasurePhraseIndices(index); + } + } + } + + /** + * Utility method for clubbing column contains check and column range validity check. + * @param column column name to check + * @return true if this candidate can answer this column looking at existence and time validity. + */ + default boolean isColumnPresentAndValidForRange(String column) { + return getColumns().contains(column) && isColumnValidForRange(column); + } + + /** + * Utility method for checking column time range validity. + * @param column column to check + * @return true if this column is valid for all ranges queried + */ + default boolean isColumnValidForRange(String column) { + Optional<Date> start = getColumnStartTime(column); + Optional<Date> end = getColumnEndTime(column); + return (!start.isPresent() + || getCubeQueryContext().getTimeRanges().stream().noneMatch(range -> range.getFromDate().before(start.get()))) + && (!end.isPresent() + || getCubeQueryContext().getTimeRanges().stream().noneMatch(range -> range.getToDate().after(end.get()))); + } + + /** + * This method should give start time of a column, if there's any. Else, this should return Optional.absent + * @param column column name + * @return optional start time of this column + */ + Optional<Date> getColumnStartTime(String column); + /** + * This method should give end time of a column, if there's any. Else, this should return Optional.absent + * @param column column name + * @return optional end time of this column + */ + Optional<Date> getColumnEndTime(String column); + + /** + * A candidate always works along with its cube query context. So a top level method to retrieve that. + * @return cube query context for this candidate. + */ + CubeQueryContext getCubeQueryContext(); + + /** + * Utility method to return the configuration of its cube query context. + * @return getCubeQueryContext().getConf() + */ + default Configuration getConf() { + return getCubeQueryContext().getConf(); + } + + /** + * Utility method to return the metastore client of its cube query context + * @return getCubeQueryContext().getMetastoreClient() + */ + default CubeMetastoreClient getCubeMetastoreClient() { + return getCubeQueryContext().getMetastoreClient(); + } + + /** + * Utility method to return cube of its cube query context + * @param <T> a subclass of CubeInterface + * @return getCubeQueryContext().getCube() + */ + @SuppressWarnings("unchecked") + default <T extends CubeInterface> T getCube() { + return (T) getCubeQueryContext().getCube(); + } + + /** + * Filters phrases that are covered by this candidate + * @param phrases queried phrases to check + * @return a set of queried phrases belonging to the list `phrases` that are answerable by this candidate + * @throws LensException propagated exception + */ + default Set<QueriedPhraseContext> coveredPhrases(Set<QueriedPhraseContext> phrases) throws LensException { + Set<QueriedPhraseContext> covered = Sets.newHashSet(); + for (QueriedPhraseContext msr : phrases) { + if (isPhraseAnswerable(msr)) { + covered.add(msr); + } + } + return covered; + } + + /** + * Explode this candidate into another candidate. + * Generally candidates can return `this` in this method + * Special case is storage candidate that returns UnionCandidate if there are multiple update periods covered and + * update periods have different tables. + * @return converted candidate + */ + Candidate explode() throws LensException; + + /** + * Get query writer context from the candidate. Default implementation is for Union, Join and Segmentation candidates. + * In the default implementation, a MultiCandidateQueryWriterContext is returned, whose children are obtained by + * getting the query writer contexts from the children of this candidate. + * @param dimsToQuery Dimensions and corresponding picked CandidateDim for query + * @param rootCubeQueryContext Root query context. + * @return A Query Writer Context + * @throws LensException exception to be propagated + */ + default QueryWriterContext toQueryWriterContext(Map<Dimension, CandidateDim> dimsToQuery, + CubeQueryContext rootCubeQueryContext) throws LensException { + if (getChildren() != null) { + List<QueryWriterContext> writerContexts = Lists.newArrayList(); + for (Candidate candidate : getChildren()) { + writerContexts.add(candidate.toQueryWriterContext(dimsToQuery, rootCubeQueryContext)); + } + if (writerContexts.size() == 1) { + return writerContexts.iterator().next(); + } + return new MultiCandidateQueryWriterContext(writerContexts, rootCubeQueryContext); + } + throw new IllegalArgumentException("Candidate doesn't have children and no suitable implementation found"); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java index b22d972..d6f8ad1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateCoveringSetsResolver.java @@ -53,7 +53,7 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { List<Candidate> timeRangeCoveringSet = resolveTimeRangeCoveringFactSet(cubeql, queriedMsrs, qpcList); if (timeRangeCoveringSet.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_UNION_CANDIDATE_AVAILABLE.getLensErrorInfo(), - cubeql.getCube().getName(), cubeql.getTimeRanges().toString(), getColumns(queriedMsrs).toString()); + cubeql.getCube().getName(), cubeql.getTimeRanges().toString(), getColumns(queriedMsrs).toString()); } log.info("Time covering candidates :{}", timeRangeCoveringSet); @@ -61,10 +61,10 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { cubeql.getCandidates().clear(); cubeql.getCandidates().addAll(timeRangeCoveringSet); } else if (!timeRangeCoveringSet.isEmpty()) { - List<List<Candidate>> measureCoveringSets = resolveJoinCandidates(timeRangeCoveringSet, queriedMsrs, cubeql); + List<List<Candidate>> measureCoveringSets = resolveJoinCandidates(timeRangeCoveringSet, queriedMsrs); if (measureCoveringSets.isEmpty()) { throw new LensException(LensCubeErrorCode.NO_JOIN_CANDIDATE_AVAILABLE.getLensErrorInfo(), - cubeql.getCube().getName(), getColumns(queriedMsrs).toString()); + cubeql.getCube().getName(), getColumns(queriedMsrs).toString()); } updateFinalCandidates(measureCoveringSets, cubeql); } @@ -119,66 +119,43 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { private List<Candidate> resolveTimeRangeCoveringFactSet(CubeQueryContext cubeql, Set<QueriedPhraseContext> queriedMsrs, List<QueriedPhraseContext> qpcList) throws LensException { List<Candidate> candidateSet = new ArrayList<>(); - if (!cubeql.getCandidates().isEmpty()) { - // All Candidates - List<Candidate> allCandidates = new ArrayList<>(cubeql.getCandidates()); - // Partially valid candidates - List<Candidate> allCandidatesPartiallyValid = new ArrayList<>(); - for (Candidate cand : allCandidates) { - // Assuming initial list of candidates populated are StorageCandidate - assert (cand instanceof StorageCandidate); - StorageCandidate sc = (StorageCandidate) cand; - if (CandidateUtil.isValidForTimeRanges(sc, cubeql.getTimeRanges())) { - candidateSet.add(CandidateUtil.cloneStorageCandidate(sc)); - } else if (CandidateUtil.isPartiallyValidForTimeRanges(sc, cubeql.getTimeRanges())) { - allCandidatesPartiallyValid.add(CandidateUtil.cloneStorageCandidate(sc)); - } else { - cubeql.addCandidatePruningMsg(sc, CandidateTablePruneCause.storageNotAvailableInRange( - cubeql.getTimeRanges())); - } - + // All Candidates + List<Candidate> allCandidates = new ArrayList<>(cubeql.getCandidates()); + // Partially valid candidates + List<Candidate> allCandidatesPartiallyValid = new ArrayList<>(); + for (Candidate cand : allCandidates) { + if (cand.isCompletelyValidForTimeRanges(cubeql.getTimeRanges())) { + candidateSet.add(cand.copy()); + } else if (cand.isPartiallyValidForTimeRanges(cubeql.getTimeRanges())) { + allCandidatesPartiallyValid.add(cand.copy()); + } else { + cubeql.addCandidatePruningMsg(cand, CandidateTablePruneCause.storageNotAvailableInRange( + cubeql.getTimeRanges())); } - // Get all covering fact sets - List<UnionCandidate> unionCoveringSet = - getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql); - // Sort the Collection based on no of elements - unionCoveringSet.sort(new CandidateUtil.ChildrenSizeBasedCandidateComparator<UnionCandidate>()); - // prune non covering sets - pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql); - // prune candidate set which doesn't contain any common measure i - pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, queriedMsrs, cubeql); - // prune redundant covering sets - pruneRedundantUnionCoveringSets(unionCoveringSet); - // pruing done in the previous steps, now create union candidates - candidateSet.addAll(unionCoveringSet); - updateQueriableMeasures(candidateSet, qpcList, cubeql); } + // Get all covering fact sets + List<UnionCandidate> unionCoveringSet = + getCombinations(new ArrayList<>(allCandidatesPartiallyValid), cubeql); + // Sort the Collection based on no of elements + unionCoveringSet.sort(Comparator.comparing(Candidate::getChildrenCount)); + // prune non covering sets + pruneUnionCandidatesNotCoveringAllRanges(unionCoveringSet, cubeql); + // prune candidate set which doesn't contain any common measure i + pruneUnionCoveringSetWithoutAnyCommonMeasure(unionCoveringSet, queriedMsrs); + // prune redundant covering sets + pruneRedundantUnionCoveringSets(unionCoveringSet); + // pruing done in the previous steps, now create union candidates + candidateSet.addAll(unionCoveringSet); + updateQueriableMeasures(candidateSet, qpcList); return candidateSet; } - - private boolean isMeasureAnswerablebyUnionCandidate(QueriedPhraseContext msr, Candidate uc, - CubeQueryContext cubeql) throws LensException { - // Candidate is a single StorageCandidate - if ((uc instanceof StorageCandidate) && !msr.isEvaluable(cubeql, (StorageCandidate) uc)) { - return false; - } else if ((uc instanceof UnionCandidate)){ - for (Candidate cand : uc.getChildren()) { - if (!msr.isEvaluable(cubeql, (StorageCandidate) cand)) { - return false; - } - } - } - return true; - } - private void pruneUnionCoveringSetWithoutAnyCommonMeasure(List<UnionCandidate> ucs, - Set<QueriedPhraseContext> queriedMsrs, - CubeQueryContext cubeql) throws LensException { + Set<QueriedPhraseContext> queriedMsrs) throws LensException { for (ListIterator<UnionCandidate> itr = ucs.listIterator(); itr.hasNext();) { boolean toRemove = true; UnionCandidate uc = itr.next(); for (QueriedPhraseContext msr : queriedMsrs) { - if (isMeasureAnswerablebyUnionCandidate(msr, uc, cubeql)) { + if (uc.isPhraseAnswerable(msr)) { toRemove = false; break; } @@ -223,16 +200,16 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { return combinations; } - private List<List<Candidate>> resolveJoinCandidates(List<Candidate> unionCandidates, - Set<QueriedPhraseContext> msrs, CubeQueryContext cubeql) throws LensException { + private List<List<Candidate>> resolveJoinCandidates(List<Candidate> candidates, + Set<QueriedPhraseContext> msrs) throws LensException { List<List<Candidate>> msrCoveringSets = new ArrayList<>(); - List<Candidate> ucSet = new ArrayList<>(unionCandidates); + List<Candidate> ucSet = new ArrayList<>(candidates); // Check if a single set can answer all the measures and exprsWithMeasures for (Iterator<Candidate> i = ucSet.iterator(); i.hasNext();) { boolean evaluable = false; Candidate uc = i.next(); for (QueriedPhraseContext msr : msrs) { - evaluable = isMeasureAnswerablebyUnionCandidate(msr, uc, cubeql); + evaluable = uc.isPhraseAnswerable(msr); if (!evaluable) { break; } @@ -253,10 +230,10 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { // find the remaining measures in other facts if (i.hasNext()) { Set<QueriedPhraseContext> remainingMsrs = new HashSet<>(msrs); - Set<QueriedPhraseContext> coveredMsrs = CandidateUtil.coveredMeasures(candidate, msrs, cubeql); + Set<QueriedPhraseContext> coveredMsrs = candidate.coveredPhrases(msrs); remainingMsrs.removeAll(coveredMsrs); - List<List<Candidate>> coveringSets = resolveJoinCandidates(ucSet, remainingMsrs, cubeql); + List<List<Candidate>> coveringSets = resolveJoinCandidates(ucSet, remainingMsrs); if (!coveringSets.isEmpty()) { for (List<Candidate> candSet : coveringSets) { candSet.add(candidate); @@ -273,43 +250,9 @@ public class CandidateCoveringSetsResolver implements ContextRewriter { } private void updateQueriableMeasures(List<Candidate> cands, - List<QueriedPhraseContext> qpcList, CubeQueryContext cubeql) throws LensException { + List<QueriedPhraseContext> qpcList) throws LensException { for (Candidate cand : cands) { - updateStorageCandidateQueriableMeasures(cand, qpcList, cubeql); - } - } - - - private void updateStorageCandidateQueriableMeasures(Candidate unionCandidate, - List<QueriedPhraseContext> qpcList, CubeQueryContext cubeql) throws LensException { - QueriedPhraseContext msrPhrase; - boolean isEvaluable; - for (int index = 0; index < qpcList.size(); index++) { - - if (!qpcList.get(index).hasMeasures(cubeql)) { - //Not a measure phrase. Skip it - continue; - } - - msrPhrase = qpcList.get(index); - if (unionCandidate instanceof StorageCandidate && msrPhrase.isEvaluable(cubeql, - (StorageCandidate) unionCandidate)) { - ((StorageCandidate) unionCandidate).setAnswerableMeasurePhraseIndices(index); - } else if (unionCandidate instanceof UnionCandidate) { - isEvaluable = true; - for (Candidate childCandidate : unionCandidate.getChildren()) { - if (!msrPhrase.isEvaluable(cubeql, (StorageCandidate) childCandidate)) { - isEvaluable = false; - break; - } - } - if (isEvaluable) { - //Set the index for all the children in this case - for (Candidate childCandidate : unionCandidate.getChildren()) { - ((StorageCandidate) childCandidate).setAnswerableMeasurePhraseIndices(index); - } - } - } + cand.updateStorageCandidateQueriablePhraseIndices(qpcList); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateExploder.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateExploder.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateExploder.java new file mode 100644 index 0000000..7dea785 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateExploder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.Set; + +import org.apache.lens.server.api.error.LensException; + +import com.google.common.collect.Sets; + +/** + * Created on 21/04/17. + */ +public class CandidateExploder implements ContextRewriter { + /** + * Replacing all candidates with their exploded versions in cubeql.getCandidates() + * @param cubeql CubeQueryContext + * @throws LensException propagated exception + */ + @Override + public void rewriteContext(CubeQueryContext cubeql) throws LensException { + Set<Candidate> candidateSet = Sets.newHashSet(); + for (Candidate candidate : cubeql.getCandidates()) { + candidateSet.add(candidate.explode()); + } + cubeql.getCandidates().clear(); + cubeql.getCandidates().addAll(candidateSet); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index 29af419..8f154cc 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -18,6 +18,7 @@ */ package org.apache.lens.cube.parse; +import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toSet; import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; @@ -27,11 +28,13 @@ import static com.google.common.collect.Lists.newArrayList; import java.util.*; import org.apache.lens.cube.metadata.TimeRange; +import org.apache.lens.server.api.error.LensException; import org.codehaus.jackson.annotate.JsonWriteNullProperties; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.Data; import lombok.NoArgsConstructor; @@ -58,6 +61,15 @@ public class CandidateTablePruneCause { UNSUPPORTED_STORAGE("Unsupported Storage"), // invalid cube table INVALID("Invalid cube table provided in query"), + SEGMENTATION_PRUNED_WITH_ERROR ("%s") { + @Override + Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { + return new Object[]{ + causes.stream().map(cause->cause.innerException).map(LensException::getMessage).collect(joining(",")), + }; + } + }, + // expression is not evaluable in the candidate COLUMN_NOT_FOUND("%s are not %s") { Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { @@ -167,6 +179,22 @@ public class CandidateTablePruneCause { }; } }, + SEGMENTATION_PRUNED("%s") { + @Override + Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { + Map<String, String> briefCause = Maps.newHashMap(); + for (CandidateTablePruneCause cause : causes) { + briefCause.putAll(cause.getInnerCauses()); + } + if (briefCause.size() == 1) { + return new Object[]{briefCause.values().iterator().next(), }; + } + return new Object[]{ + "segmentation pruned: " + + briefCause.entrySet().stream().map(entry->entry.getKey()+": "+entry.getValue()).collect(joining(";")), + }; + } + }, // missing partitions for cube table MISSING_PARTITIONS("Missing partitions for the cube table: %s") { Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { @@ -244,6 +272,8 @@ public class CandidateTablePruneCause { private Map<String, SkipUpdatePeriodCode> updatePeriodRejectionCause; + private Map<String, String> innerCauses; + private LensException innerException; public CandidateTablePruneCause(CandidateTablePruneCode cause) { this.cause = cause; @@ -357,4 +387,14 @@ public class CandidateTablePruneCause { cause.updatePeriodRejectionCause = updatePeriodRejectionCause; return cause; } + public static CandidateTablePruneCause segmentationPruned(Map<String, String> inner) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(SEGMENTATION_PRUNED); + cause.innerCauses = inner; + return cause; + } + public static CandidateTablePruneCause segmentationPruned(LensException e) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(SEGMENTATION_PRUNED_WITH_ERROR); + cause.innerException = e; + return cause; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java index 6d61f1f..a8d6fbd 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java @@ -25,12 +25,12 @@ import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode; import org.apache.lens.cube.parse.CubeQueryContext.OptionalDimCtx; import org.apache.lens.cube.parse.CubeQueryContext.QueriedExprColumn; -import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext; import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -70,7 +70,7 @@ class CandidateTableResolver implements ContextRewriter { // Before checking for candidate table columns, prune join paths containing non existing columns // in populated candidate tables cubeql.getAutoJoinCtx().pruneAllPaths(cubeql.getCube(), - CandidateUtil.getStorageCandidates(cubeql.getCandidates()), null); + CandidateUtil.getColumnsFromCandidates(cubeql.getCandidates()), null); cubeql.getAutoJoinCtx().pruneAllPathsForCandidateDims(cubeql.getCandidateDimTables()); cubeql.getAutoJoinCtx().refreshJoinPathColumns(); } @@ -100,6 +100,11 @@ class CandidateTableResolver implements ContextRewriter { } } log.info("Populated storage candidates: {}", cubeql.getCandidates()); + List<SegmentationCandidate> segmentationCandidates = Lists.newArrayList(); + for (Segmentation segmentation : cubeql.getMetastoreClient().getAllSegmentations(cubeql.getCube())) { + segmentationCandidates.add(new SegmentationCandidate(cubeql, segmentation)); + } + cubeql.getCandidates().addAll(segmentationCandidates); } if (cubeql.getDimensions().size() != 0) { @@ -183,11 +188,11 @@ class CandidateTableResolver implements ContextRewriter { } private static boolean isColumnAvailableFrom(@NonNull final Date date, Date startTime) { - return (startTime == null) ? true : date.equals(startTime) || date.after(startTime); + return (startTime == null) || (date.equals(startTime) || date.after(startTime)); } private static boolean isColumnAvailableTill(@NonNull final Date date, Date endTime) { - return (endTime == null) ? true : date.equals(endTime) || date.before(endTime); + return (endTime == null) || (date.equals(endTime) || date.before(endTime)); } private static boolean isFactColumnValidForRange(CubeQueryContext cubeql, CandidateTable cfact, String col) { @@ -262,43 +267,44 @@ class CandidateTableResolver implements ContextRewriter { for (String expr : cubeql.getQueriedExprs()) { cubeql.getExprCtx().updateEvaluables(expr, sc); } - - // go over the columns accessed in the query and find out which tables - // can answer the query - // the candidate facts should have all the dimensions queried and - // atleast - // one measure - boolean toRemove = false; - for (QueriedPhraseContext qur : dimExprs) { - if (!qur.isEvaluable(cubeql, sc)) { - log.info("Not considering storage candidate:{} as columns {} are not available", sc, qur.getColumns()); - cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - qur.getColumns())); - toRemove = true; - break; - } - } - - // check if the candidate fact has atleast one measure queried - // if expression has measures, they should be considered along with other measures and see if the fact can be - // part of measure covering set - if (!checkForFactColumnExistsAndValidForRange(sc, queriedMsrs, cubeql)) { - Set<String> columns = getColumns(queriedMsrs); - log.info("Not considering storage candidate:{} as columns {} is not available", sc, columns); - cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( - columns)); + } + // go over the columns accessed in the query and find out which tables + // can answer the query + // the candidate facts should have all the dimensions queried and + // atleast + // one measure + boolean toRemove = false; + for (QueriedPhraseContext qur : dimExprs) { + if (!cand.isPhraseAnswerable(qur)) { + log.info("Not considering storage candidate:{} as columns {} are not available", cand, qur.getColumns()); + cubeql.addStoragePruningMsg(cand, CandidateTablePruneCause.columnNotFound( + qur.getColumns())); toRemove = true; + break; } + } - // go over join chains and prune facts that dont have any of the columns in each chain + // check if the candidate fact has atleast one measure queried + // if expression has measures, they should be considered along with other measures and see if the fact can be + // part of measure covering set + if (!checkForFactColumnExistsAndValidForRange(cand, queriedMsrs)) { + Set<String> columns = getColumns(queriedMsrs); + log.info("Not considering storage candidate:{} as columns {} is not available", cand, columns); + cubeql.addStoragePruningMsg(cand, CandidateTablePruneCause.columnNotFound(columns)); + toRemove = true; + } + + // go over join chains and prune facts that dont have any of the columns in each chain + if (cand instanceof StorageCandidate) { + StorageCandidate sc = (StorageCandidate) cand; for (JoinChain chain : cubeql.getJoinchains().values()) { OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(Aliased.create((Dimension) cubeql.getCubeTbls() - .get(chain.getName()), chain.getName())); + .get(chain.getName()), chain.getName())); if (!checkForFactColumnExistsAndValidForRange(sc, chain.getSourceColumns(), cubeql)) { // check if chain is optional or not if (optdim == null) { log.info("Not considering storage candidate:{} as columns {} are not available", sc, - chain.getSourceColumns()); + chain.getSourceColumns()); cubeql.addStoragePruningMsg(sc, CandidateTablePruneCause.columnNotFound( chain.getSourceColumns())); toRemove = true; @@ -306,17 +312,14 @@ class CandidateTableResolver implements ContextRewriter { } } } - - if (toRemove) { - i.remove(); - } - } else { - throw new LensException("Not a storage candidate!!"); + } + if (toRemove) { + i.remove(); } } if (cubeql.getCandidates().size() == 0) { throw new LensException(LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(), - getColumns(cubeql.getQueriedPhrases()).toString()); + getColumns(cubeql.getQueriedPhrases()).toString()); } } } @@ -355,8 +358,7 @@ class CandidateTableResolver implements ContextRewriter { Collection<String> colSet = joincolumnsEntry.getValue().get(dim); if (!checkForFactColumnExistsAndValidForRange(cdim, colSet, cubeql)) { - if (optdim == null || optdim.isRequiredInJoinChain - || (optdim != null && optdim.requiredForCandidates.contains(cdim))) { + if (optdim == null || optdim.isRequiredInJoinChain || optdim.requiredForCandidates.contains(cdim)) { i.remove(); removed = true; log.info("Not considering dimtable:{} as its columns are not part of any join paths. Join columns:{}", @@ -375,8 +377,7 @@ class CandidateTableResolver implements ContextRewriter { Collection<String> colSet = joincolumnsEntry.getValue().get(dim); if (!checkForFactColumnExistsAndValidForRange(cdim, colSet, cubeql)) { - if (optdim == null || optdim.isRequiredInJoinChain - || (optdim != null && optdim.requiredForCandidates.contains(cdim))) { + if (optdim == null || optdim.isRequiredInJoinChain || optdim.requiredForCandidates.contains(cdim)) { i.remove(); removed = true; log.info("Not considering dimtable:{} as its columns are not part of any join paths. Join columns:{}", @@ -435,8 +436,7 @@ class CandidateTableResolver implements ContextRewriter { colSet = joincolumnsEntry.getValue().get(cubeql.getCube()); if (!checkForFactColumnExistsAndValidForRange(sc, colSet, cubeql)) { - if (optdim == null || optdim.isRequiredInJoinChain - || (optdim != null && optdim.requiredForCandidates.contains(sc))) { + if (optdim == null || optdim.isRequiredInJoinChain || optdim.requiredForCandidates.contains(sc)) { i.remove(); log.info("Not considering storage candidate :{} as it does not have columns in any of the join paths." + " Join columns:{}", sc, colSet); @@ -568,13 +568,7 @@ class CandidateTableResolver implements ContextRewriter { // check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable // if no evaluable expressions exist, then remove the candidate if (ec.getEvaluableExpressions().get(candidate) != null) { - Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator(); - while (escIter.hasNext()) { - ExprSpecContext esc = escIter.next(); - if (esc.getExprDims().contains(dim.getObject())) { - escIter.remove(); - } - } + ec.getEvaluableExpressions().get(candidate).removeIf(esc -> esc.getExprDims().contains(dim.getObject())); } if (cubeql.getExprCtx().isEvaluable(col.getExprCol(), candidate)) { // candidate has other evaluable expressions @@ -672,14 +666,13 @@ class CandidateTableResolver implements ContextRewriter { } - private static boolean checkForFactColumnExistsAndValidForRange(StorageCandidate sc, - Collection<QueriedPhraseContext> colSet, - CubeQueryContext cubeql) throws LensException { + private static boolean checkForFactColumnExistsAndValidForRange(Candidate sc, + Collection<QueriedPhraseContext> colSet) throws LensException { if (colSet == null || colSet.isEmpty()) { return true; } for (QueriedPhraseContext qur : colSet) { - if (qur.isEvaluable(cubeql, sc)) { + if (sc.isPhraseAnswerable(qur)) { return true; } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java index b9ff0ef..467ca0a 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateUtil.java @@ -21,12 +21,8 @@ package org.apache.lens.cube.parse; import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier; import java.util.*; +import java.util.stream.Collectors; -import org.apache.lens.cube.metadata.*; -import org.apache.lens.server.api.error.LensException; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -47,102 +43,11 @@ public final class CandidateUtil { // (design) HideUtilityClassConstructor: Utility classes should not have a public or default constructor. } - /** - * Returns true if the Candidate is valid for all the timeranges based on its start and end times. - * @param candidate - * @param timeRanges - * @return - */ - public static boolean isValidForTimeRanges(Candidate candidate, List<TimeRange> timeRanges) { - for (TimeRange timeRange : timeRanges) { - if (!(timeRange.getFromDate().after(candidate.getStartTime()) - && timeRange.getToDate().before(candidate.getEndTime()))) { - return false; - } - } - return true; - } - - static boolean isCandidatePartiallyValidForTimeRange(Date candidateStartTime, Date candidateEndTime, - Date timeRangeStart, Date timeRangeEnd) { - Date start = candidateStartTime.after(timeRangeStart) ? candidateStartTime : timeRangeStart; - Date end = candidateEndTime.before(timeRangeEnd) ? candidateEndTime : timeRangeEnd; - if (end.after(start)) { - return true; - } - return false; - } - - - static boolean isPartiallyValidForTimeRange(Candidate cand, TimeRange timeRange) { - return isPartiallyValidForTimeRanges(cand, Arrays.asList(timeRange)); - } - - static boolean isPartiallyValidForTimeRanges(Candidate cand, List<TimeRange> timeRanges) { - return timeRanges.stream().anyMatch(timeRange -> - isCandidatePartiallyValidForTimeRange(cand.getStartTime(), cand.getEndTime(), - timeRange.getFromDate(), timeRange.getToDate())); - } - - /** - * Copy Query AST from sourceAst to targetAst - * - * @param sourceAst - * @param targetAst - * @throws LensException - */ - static void copyASTs(QueryAST sourceAst, QueryAST targetAst) throws LensException { - - targetAst.setSelectAST(MetastoreUtil.copyAST(sourceAst.getSelectAST())); - targetAst.setWhereAST(MetastoreUtil.copyAST(sourceAst.getWhereAST())); - if (sourceAst.getJoinAST() != null) { - targetAst.setJoinAST(MetastoreUtil.copyAST(sourceAst.getJoinAST())); - } - if (sourceAst.getGroupByAST() != null) { - targetAst.setGroupByAST(MetastoreUtil.copyAST(sourceAst.getGroupByAST())); - } - if (sourceAst.getHavingAST() != null) { - targetAst.setHavingAST(MetastoreUtil.copyAST(sourceAst.getHavingAST())); - } - if (sourceAst.getOrderByAST() != null) { - targetAst.setOrderByAST(MetastoreUtil.copyAST(sourceAst.getOrderByAST())); - } - - targetAst.setLimitValue(sourceAst.getLimitValue()); - targetAst.setFromString(sourceAst.getFromString()); - targetAst.setWhereString(sourceAst.getWhereString()); - } public static Set<StorageCandidate> getStorageCandidates(final Candidate candidate) { return getStorageCandidates(new HashSet<Candidate>(1) {{ add(candidate); }}); } - // this function should only be used for union candidates and never for join candidates. - // future scope of improvement: move the data model to use polymorphism - static Set<QueriedPhraseContext> coveredMeasures(Candidate candSet, Collection<QueriedPhraseContext> msrs, - CubeQueryContext cubeql) throws LensException { - Set<QueriedPhraseContext> coveringSet = new HashSet<>(); - for (QueriedPhraseContext msr : msrs) { - if (candSet.getChildren() == null) { - if (msr.isEvaluable(cubeql, (StorageCandidate) candSet)) { - coveringSet.add(msr); - } - } else { - boolean allCanAnswer = true; - for (Candidate cand : candSet.getChildren()) { - if (!msr.isEvaluable(cubeql, (StorageCandidate) cand)) { - allCanAnswer = false; - break; - } - } - if (allCanAnswer) { - coveringSet.add(msr); - } - } - } - return coveringSet; - } - /** * Returns true is the Candidates cover the entire time range. * @param candidates @@ -150,7 +55,7 @@ public final class CandidateUtil { * @param endTime * @return */ - public static boolean isTimeRangeCovered(Collection<Candidate> candidates, Date startTime, Date endTime) { + static boolean isTimeRangeCovered(Collection<Candidate> candidates, Date startTime, Date endTime) { RangeSet<Date> set = TreeRangeSet.create(); for (Candidate candidate : candidates) { set.add(Range.range(candidate.getStartTime(), BoundType.CLOSED, candidate.getEndTime(), BoundType.OPEN)); @@ -177,8 +82,9 @@ public final class CandidateUtil { List<Candidate> prunedCandidates = new ArrayList<>(); Iterator<Candidate> itr = candidates.iterator(); while (itr.hasNext()) { - if (itr.next().contains(filterCandidate)) { - prunedCandidates.add(itr.next()); + Candidate cur = itr.next(); + if (cur.contains(filterCandidate)) { + prunedCandidates.add(cur); itr.remove(); } } @@ -191,96 +97,35 @@ public final class CandidateUtil { * @param candidates * @return */ - public static Set<StorageCandidate> getStorageCandidates(Collection<Candidate> candidates) { + public static Set<StorageCandidate> getStorageCandidates(Collection<? extends Candidate> candidates) { Set<StorageCandidate> storageCandidateSet = new HashSet<>(); getStorageCandidates(candidates, storageCandidateSet); return storageCandidateSet; } - private static void getStorageCandidates(Collection<Candidate> candidates, - Set<StorageCandidate> storageCandidateSet) { + private static void getStorageCandidates(Collection<? extends Candidate> candidates, + Collection<StorageCandidate> storageCandidateSet) { for (Candidate candidate : candidates) { - if (candidate.getChildren() == null) { - //Expecting this to be a StorageCandidate as it has no children. - storageCandidateSet.add((StorageCandidate)candidate); - } else { - getStorageCandidates(candidate.getChildren(), storageCandidateSet); - } - } - } - - public static StorageCandidate cloneStorageCandidate(StorageCandidate sc) throws LensException{ - return new StorageCandidate(sc); - } - - public static boolean factHasColumn(CubeFactTable fact, String column) { - for (FieldSchema factField : fact.getColumns()) { - if (factField.getName().equals(column)) { - return true; + getStorageCandidates(candidate, storageCandidateSet); + } + } + static void getStorageCandidates(Candidate candidate, + Collection<StorageCandidate> storageCandidateSet) { + if (candidate.getChildren() == null) { + // Expecting this to be a StorageCandidate as it has no children. + if (candidate instanceof StorageCandidate) { + storageCandidateSet.add((StorageCandidate) candidate); + } else if (candidate instanceof SegmentationCandidate) { + SegmentationCandidate segC = (SegmentationCandidate) candidate; + for (CubeQueryContext cubeQueryContext : segC.cubeQueryContextMap.values()) { + if (cubeQueryContext.getPickedCandidate() != null) { + getStorageCandidates(cubeQueryContext.getPickedCandidate(), storageCandidateSet); + } + } } + } else { + getStorageCandidates(candidate.getChildren(), storageCandidateSet); } - return false; - } - - public static String getTimeRangeWhereClasue(TimeRangeWriter rangeWriter, - StorageCandidate sc, TimeRange range) throws LensException { - String rangeWhere = rangeWriter.getTimeRangeWhereClause(sc.getCubeql(), - sc.getCubeql().getAliasForTableName(sc.getCube().getName()), - sc.getRangeToPartitions().get(range)); - if (sc.getRangeToExtraWhereFallBack().containsKey(range)) { - rangeWhere = "((" + rangeWhere + ") and (" + sc.getRangeToExtraWhereFallBack().get(range) + "))"; - } - return rangeWhere; - } - - public static class ChildrenSizeBasedCandidateComparator<T> implements Comparator<Candidate> { - @Override - public int compare(Candidate o1, Candidate o2) { - return o1.getChildren().size() - o2.getChildren().size(); - } - } - - private static final String BASE_QUERY_FORMAT = "SELECT %s FROM %s"; - - public static String buildHQLString(String select, String from, String where, - String groupby, String orderby, String having, Integer limit) { - List<String> qstrs = new ArrayList<String>(); - qstrs.add(select); - qstrs.add(from); - if (!StringUtils.isBlank(where)) { - qstrs.add(where); - } - if (!StringUtils.isBlank(groupby)) { - qstrs.add(groupby); - } - if (!StringUtils.isBlank(having)) { - qstrs.add(having); - } - if (!StringUtils.isBlank(orderby)) { - qstrs.add(orderby); - } - if (limit != null) { - qstrs.add(String.valueOf(limit)); - } - - StringBuilder queryFormat = new StringBuilder(); - queryFormat.append(BASE_QUERY_FORMAT); - if (!StringUtils.isBlank(where)) { - queryFormat.append(" WHERE %s"); - } - if (!StringUtils.isBlank(groupby)) { - queryFormat.append(" GROUP BY %s"); - } - if (!StringUtils.isBlank(having)) { - queryFormat.append(" HAVING %s"); - } - if (!StringUtils.isBlank(orderby)) { - queryFormat.append(" ORDER BY %s"); - } - if (limit != null) { - queryFormat.append(" LIMIT %s"); - } - return String.format(queryFormat.toString(), qstrs.toArray(new String[qstrs.size()])); } /** @@ -292,7 +137,7 @@ public final class CandidateUtil { * 1. Replace queriedAlias with finalAlias if both are not same * 2. If queriedAlias is missing add finalAlias as alias */ - public static void updateFinalAlias(ASTNode selectAST, CubeQueryContext cubeql) { + static void updateFinalAlias(ASTNode selectAST, CubeQueryContext cubeql) { for (int i = 0; i < selectAST.getChildCount(); i++) { ASTNode selectExpr = (ASTNode) selectAST.getChild(i); ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier); @@ -312,8 +157,7 @@ public final class CandidateUtil { } } } - - - - + static Set<String> getColumnsFromCandidates(Collection<? extends Candidate> scSet) { + return scSet.stream().map(Candidate::getColumns).flatMap(Collection::stream).collect(Collectors.toSet()); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b58749e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java index 300d798..eeaa3af 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java @@ -42,6 +42,7 @@ public final class CubeQueryConfUtil { public static final String VALID_STORAGE_DIM_TABLES = "lens.cube.query.valid." + "dim.storgaetables"; public static final String DRIVER_SUPPORTED_STORAGES = "lens.cube.query.driver." + "supported.storages"; public static final String FAIL_QUERY_ON_PARTIAL_DATA = "lens.cube.query.fail.if.data.partial"; + public static final String RESOLVE_SEGMENTATIONS = "lens.cube.query.resolve.segmentations"; public static final String NON_EXISTING_PARTITIONS = "lens.cube.query.nonexisting.partitions"; public static final String QUERY_MAX_INTERVAL = "lens.cube.query.max.interval"; public static final String PROCESS_TIME_PART_COL = "lens.cube.query.process.time" + ".partition.column";
