LENS-604 : Add "time dimension not supported" as a fact prune cause
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/51cdd988 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/51cdd988 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/51cdd988 Branch: refs/heads/current-release-line Commit: 51cdd98831f2f392e24f16718b3cd4409d2250ed Parents: 2542874 Author: Rajat Khandelwal <[email protected]> Authored: Fri Jun 19 10:42:35 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Fri Jun 19 10:42:35 2015 +0530 ---------------------------------------------------------------------- .../cube/parse/CandidateTablePruneCause.java | 41 +++++++++---- .../lens/cube/parse/StorageTableResolver.java | 61 +++++++++++++------- .../lens/cube/parse/TestCubeRewriter.java | 8 +-- .../server/query/QueryAPIErrorResponseTest.java | 2 +- 4 files changed, 74 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java index 5a1f8f9..75d5581 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java @@ -18,6 +18,8 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; + import java.util.*; import org.codehaus.jackson.annotate.JsonWriteNullProperties; @@ -98,7 +100,19 @@ public class CandidateTablePruneCause { // no candidate storges for cube table, storage cause will have why each // storage is not a candidate NO_CANDIDATE_STORAGES("No candidate storages for any table"), - // cube table has more weight + // time dimension not supported. Either directly or indirectly. + TIMEDIM_NOT_SUPPORTED("Queried data not available for time dimensions: %s") { + @Override + Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { + Set<String> dims = Sets.newHashSet(); + for(CandidateTablePruneCause cause: causes){ + dims.addAll(cause.getUnsupportedTimeDims()); + } + return new Object[]{ + dims.toString(), + }; + } + }, NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE("No fact update periods for given range"), NO_COLUMN_PART_OF_A_JOIN_PATH("No column part of a join path. Join columns: [%s]") { Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) { @@ -222,7 +236,9 @@ public class CandidateTablePruneCause { private List<String> joinColumns; // the columns that are missing default aggregate. only set in case of MISSING_DEFAULT_AGGREGATE private List<String> columnsMissingDefaultAggregate; - + // if a time dim is not supported by the fact. Would be set if and only if + // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute + private Set<String> unsupportedTimeDims; // time covered private MaxCoveringFactResolver.TimeCovered maxTimeCovered; // ranges in which fact is invalid @@ -234,17 +250,22 @@ public class CandidateTablePruneCause { // Different static constructors for different causes. public static CandidateTablePruneCause factNotAvailableInRange(List<TimeRange> ranges) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE); + CandidateTablePruneCause cause = new CandidateTablePruneCause(FACT_NOT_AVAILABLE_IN_RANGE); cause.invalidRanges = ranges; return cause; } + public static CandidateTablePruneCause timeDimNotSupported(Set<String> unsupportedTimeDims) { + CandidateTablePruneCause cause = new CandidateTablePruneCause(TIMEDIM_NOT_SUPPORTED); + cause.unsupportedTimeDims = unsupportedTimeDims; + return cause; + } public static CandidateTablePruneCause columnNotFound(Collection<String>... missingColumns) { List<String> colList = new ArrayList<String>(); for (Collection<String> missing : missingColumns) { colList.addAll(missing); } - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.COLUMN_NOT_FOUND); + CandidateTablePruneCause cause = new CandidateTablePruneCause(COLUMN_NOT_FOUND); cause.setMissingColumns(colList); return cause; } @@ -262,27 +283,27 @@ public class CandidateTablePruneCause { for (String column : exprs) { colList.add(column); } - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE); + CandidateTablePruneCause cause = new CandidateTablePruneCause(EXPRESSION_NOT_EVALUABLE); cause.setMissingExpressions(colList); return cause; } public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) { CandidateTablePruneCause cause = - new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS); + new CandidateTablePruneCause(MISSING_PARTITIONS); cause.setMissingPartitions(nonExistingParts); return cause; } public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.LESS_DATA); + CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA); cause.setMaxTimeCovered(timeCovered); return cause; } public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String> colSet) { CandidateTablePruneCause cause = - new CandidateTablePruneCause(CandidateTablePruneCode.NO_COLUMN_PART_OF_A_JOIN_PATH); + new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH); cause.setJoinColumns(new ArrayList<String>() { { addAll(colSet); @@ -292,7 +313,7 @@ public class CandidateTablePruneCause { } public static CandidateTablePruneCause noCandidateStorages(Map<String, SkipStorageCause> storageCauses) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.NO_CANDIDATE_STORAGES); + CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_CANDIDATE_STORAGES); cause.setStorageCauses(new HashMap<String, SkipStorageCause>()); for (Map.Entry<String, SkipStorageCause> entry : storageCauses.entrySet()) { String key = entry.getKey(); @@ -303,7 +324,7 @@ public class CandidateTablePruneCause { } public static CandidateTablePruneCause missingDefaultAggregate(String... names) { - CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_DEFAULT_AGGREGATE); + CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_DEFAULT_AGGREGATE); cause.setColumnsMissingDefaultAggregate(Lists.newArrayList(names)); return cause; } http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java index 049826a..ce6f434 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java @@ -18,6 +18,9 @@ */ package org.apache.lens.cube.parse; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.*; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*; +import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST; import static org.apache.lens.cube.parse.DateUtil.WSPACE; import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd; @@ -44,6 +47,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.util.ReflectionUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; /** * Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or @@ -58,16 +62,15 @@ class StorageTableResolver implements ContextRewriter { CubeMetastoreClient client; private final boolean failOnPartialData; private final List<String> validDimTables; - private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap = - new HashMap<CubeFactTable, Map<UpdatePeriod, Set<String>>>(); + private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap = new HashMap<>(); private String processTimePartCol = null; private final UpdatePeriod maxInterval; - private final Map<String, Set<String>> nonExistingPartitions = new HashMap<String, Set<String>>(); + private final Map<String, Set<String>> nonExistingPartitions = new HashMap<>(); private TimeRangeWriter rangeWriter; private DateFormat partWhereClauseFormat = null; private PHASE phase; - static enum PHASE { + enum PHASE { FACT_TABLES, FACT_PARTITIONS, DIM_TABLE_AND_PARTITIONS; static PHASE first() { @@ -218,7 +221,7 @@ class StorageTableResolver implements ContextRewriter { } if (storageTables.isEmpty()) { LOG.info("Not considering dim table:" + dimtable + " as no candidate storage tables eixst"); - cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses)); + cubeql.addDimPruningMsgs(dim, dimtable, noCandidateStorages(skipStorageCauses)); i.remove(); continue; } @@ -291,7 +294,7 @@ class StorageTableResolver implements ContextRewriter { } if (storageTableMap.isEmpty()) { LOG.info("Not considering fact table:" + fact + " as it does not" + " have any storage tables"); - cubeql.addFactPruningMsgs(fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses)); + cubeql.addFactPruningMsgs(fact, noCandidateStorages(skipStorageCauses)); i.remove(); } } @@ -352,24 +355,33 @@ class StorageTableResolver implements ContextRewriter { private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws SemanticException { // Find candidate tables wrt supported storages Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); - Map<TimeRange, String> whereClasueForFallback = new LinkedHashMap<TimeRange, String>(); + Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>(); while (i.hasNext()) { CandidateFact cfact = i.next(); - List<FactPartition> answeringParts = new ArrayList<FactPartition>(); - HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>(); + List<FactPartition> answeringParts = new ArrayList<>(); + HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<>(); PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns(); boolean noPartsForRange = false; + Set<String> unsupportedTimeDims = Sets.newHashSet(); for (TimeRange range : cubeql.getTimeRanges()) { StringBuilder extraWhereClause = new StringBuilder(); Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts); // If no partitions were found, then we'll fallback. - String partcol = range.getPartitionColumn(); + String partCol = range.getPartitionColumn(); + boolean partColNotSupported = rangeParts.isEmpty(); + for(String storage: cfact.fact.getStorages()) { + String storageTableName = MetastoreUtil.getFactStorageTableName(cfact.fact.getName(), storage).toLowerCase(); + partColNotSupported &= skipStorageCauses.containsKey(storageTableName) + && skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST) + && skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol); + } TimeRange prevRange = range; String sep = ""; while (rangeParts.isEmpty()) { // TODO: should we add a condition whether on range's partcol any missing partitions are not there - String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partcol); - if (!cfact.getColumns().contains(timeDim)) { + String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partCol); + if (partColNotSupported && !cfact.getColumns().contains(timeDim)) { + unsupportedTimeDims.add(cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn())); break; } TimeRange fallBackRange = getFallbackRange(prevRange, cfact, cubeql); @@ -381,13 +393,13 @@ class StorageTableResolver implements ContextRewriter { extraWhereClause.append(sep) .append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim)); sep = " AND "; - partcol = fallBackRange.getPartitionColumn(); prevRange = fallBackRange; + partCol = prevRange.getPartitionColumn(); if (!rangeParts.isEmpty()) { break; } } - whereClasueForFallback.put(range, extraWhereClause.toString()); + whereClauseForFallback.put(range, extraWhereClause.toString()); if (rangeParts.isEmpty()) { LOG.info("No partitions for fallback range:" + range); noPartsForRange = true; @@ -399,7 +411,7 @@ class StorageTableResolver implements ContextRewriter { for (FactPartition factPart : rangeParts) { for (String table : factPart.getStorageTables()) { if (!tablePartMap.containsKey(table)) { - tablePartMap.put(table, new LinkedHashSet<FactPartition>(Arrays.asList(factPart))); + tablePartMap.put(table, new LinkedHashSet<>(Collections.singletonList(factPart))); } else { LinkedHashSet<FactPartition> storagePart = tablePartMap.get(table); storagePart.add(factPart); @@ -414,6 +426,13 @@ class StorageTableResolver implements ContextRewriter { cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts); cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString())); } + if (!unsupportedTimeDims.isEmpty()) { + LOG.info("Not considering fact table:" + cfact.fact + " as it doesn't support time dimensions: " + + unsupportedTimeDims); + cubeql.addFactPruningMsgs(cfact.fact, timeDimNotSupported(unsupportedTimeDims)); + i.remove(); + continue; + } Set<String> nonExistingParts = missingParts.toSet(); if (!nonExistingParts.isEmpty()) { addNonExistingParts(cfact.fact.getName(), nonExistingParts); @@ -428,13 +447,13 @@ class StorageTableResolver implements ContextRewriter { * 3. Storage tables do not have the update period for the timerange queried. */ if (failOnPartialData && !nonExistingParts.isEmpty()) { - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts)); + cubeql.addFactPruningMsgs(cfact.fact, missingPartitions(nonExistingParts)); } else if (!skipStorageCauses.isEmpty()) { - CandidateTablePruneCause cause = CandidateTablePruneCause.noCandidateStorages(skipStorageCauses); + CandidateTablePruneCause cause = noCandidateStorages(skipStorageCauses); cubeql.addFactPruningMsgs(cfact.fact, cause); } else { CandidateTablePruneCause cause = - new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); + new CandidateTablePruneCause(NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE); cubeql.addFactPruningMsgs(cfact.fact, cause); } i.remove(); @@ -445,7 +464,7 @@ class StorageTableResolver implements ContextRewriter { StorageUtil.getMinimalAnsweringTables(answeringParts, minimalStorageTables); if (minimalStorageTables.isEmpty()) { LOG.info("Not considering fact table:" + cfact + " as it does not" + " have any storage tables"); - cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses)); + cubeql.addFactPruningMsgs(cfact.fact, noCandidateStorages(skipStorageCauses)); i.remove(); continue; } @@ -463,10 +482,10 @@ class StorageTableResolver implements ContextRewriter { LinkedHashSet<FactPartition> rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table); LinkedHashSet<FactPartition> minimalPartsCopy = new LinkedHashSet<FactPartition>(minimalParts); minimalPartsCopy.retainAll(rangeParts); - if (!StringUtils.isEmpty(whereClasueForFallback.get(trange))) { + if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) { rangeToWhere.put( rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), - minimalPartsCopy) + " and " + whereClasueForFallback.get(trange), table); + minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table); } else { rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java index 0f1ad6e..e04d04c 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java @@ -947,12 +947,8 @@ public class TestCubeRewriter extends TestQueryRewrite { MISSING_PARTITIONS); assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(), NO_CANDIDATE_STORAGES); - assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() - .getCause(), NO_CANDIDATE_STORAGES); - assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() - .getStorageCauses().values().iterator().next().getCause(), SkipStorageCode.PART_COL_DOES_NOT_EXIST); - assertEquals(pruneCauses.getDetails().get("summary4").iterator().next() - .getStorageCauses().values().iterator().next().getNonExistantPartCols(), Arrays.asList("dt")); + assertEquals(pruneCauses.getDetails().get("summary4").iterator().next().getCause(), TIMEDIM_NOT_SUPPORTED); + assertTrue(pruneCauses.getDetails().get("summary4").iterator().next().getUnsupportedTimeDims().contains("d_time")); } @Test http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java index 4c2a7a4..720229d 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java @@ -68,7 +68,7 @@ import org.testng.annotations.Test; import com.google.common.base.Optional; import lombok.NonNull; -@Test +@Test(groups = "unit-test") public class QueryAPIErrorResponseTest extends LensJerseyTest { private static final String MOCK_QUERY = "mock-query";
