Repository: lens Updated Branches: refs/heads/lens-1381 b26632a15 -> 4d493597e
Fix failing TestCases in org.apache.lens.server.query.QueryAPIErrorResponseTest Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4d493597 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4d493597 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4d493597 Branch: refs/heads/lens-1381 Commit: 4d493597e56248f0ed5d3d1d690d52212e2e9abe Parents: b26632a Author: Puneet Gupta <[email protected]> Authored: Tue Feb 21 16:32:17 2017 +0530 Committer: Puneet <[email protected]> Committed: Tue Feb 21 16:32:17 2017 +0530 ---------------------------------------------------------------------- .../lens/cube/parse/CubeQueryRewriter.java | 49 ++++++++++++-------- .../lens/server/common/RestAPITestUtil.java | 11 +++++ .../lens/server/common/TestDataUtils.java | 4 ++ .../server/query/QueryAPIErrorResponseTest.java | 27 +++++++++++ 4 files changed, 71 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index 4dd3d00..f052a2f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -138,61 +138,70 @@ public class CubeQueryRewriter { rewriters.add(new ColumnResolver(conf)); // Rewrite base trees (groupby, having, orderby, limit) using aliases rewriters.add(new AliasReplacer(conf)); + ExpressionResolver exprResolver = new ExpressionResolver(conf); DenormalizationResolver denormResolver = new DenormalizationResolver(conf); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); - // Resolve expressions + + // Phase 1 of exprResolver: Resolve expressions rewriters.add(exprResolver); - // De-normalized columns resolved + // Phase 1 of denormResolver: De-normalized columns resolved rewriters.add(denormResolver); // Resolve time ranges rewriters.add(new TimerangeResolver(conf)); - // Resolve candidate fact tables and dimension tables for columns queried + // Phase 1 of candidateTblResolver: Resolve candidate storages and dimension tables for columns queried rewriters.add(candidateTblResolver); // Resolve aggregations and generate base select tree rewriters.add(new AggregateResolver()); rewriters.add(new GroupbyResolver(conf)); + //validate fields queryability (in case of derived cubes setup) rewriters.add(new FieldValidator()); - rewriters.add(storageTableResolver); - //TODO union: Add CoveringSetResolver which creates UnionCandidates and JoinCandidates. - //TODO union: Some code form candidateTblResolver(phase 2) to be moved to CoveringSetResolver - //TODO union: AggregateResolver,GroupbyResolver,FieldValidator before CoveringSetResolver // Resolve joins and generate base join tree rewriters.add(new JoinResolver(conf)); - // Do col life validation + // Do col life validation for the time range(s) queried rewriters.add(new TimeRangeChecker(conf)); - // Resolve candidate fact tables and dimension tables for columns included + // Phase 1 of storageTableResolver: Validate and prune candidate storages + rewriters.add(storageTableResolver); + // Phase 2 of candidateTblResolver: Resolve candidate storages and dimension tables for columns included // in join and denorm resolvers - //TODO union : this should be CoveringSetResolver now rewriters.add(candidateTblResolver); + // Find Union and Join combinations over Storage Candidates that can answer the queried time range(s) and all + // queried measures rewriters.add(new CandidateCoveringSetsResolver(conf)); - // Phase 1: resolve fact tables. - //TODO union: This phase 1 of storageTableResolver should happen before CoveringSetResolver + + // If lightest fact first option is enabled for this driver (via lens.cube.query.pick.lightest.fact.first = true), + // run LightestFactResolver and keep only the lighted combination(s) generated by CandidateCoveringSetsResolver if (lightFactFirst) { // Prune candidate tables for which denorm column references do not exist - //TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver:Prune candidate facts without any valid expressions rewriters.add(exprResolver); + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver(conf)); } - // Phase 2: resolve fact table partitions. + + // Phase 2 of storageTableResolver: resolve storage table partitions. rewriters.add(storageTableResolver); + // In case partial data is allowed (via lens.cube.query.fail.if.data.partial = false) and there are many + // combinations with partial data, pick the one that covers the maximum part of time ranges(s) queried rewriters.add(new MaxCoveringFactResolver(conf)); - // Phase 3: resolve dimension tables and partitions. + // Phase 3 of storageTableResolver: resolve dimension tables and partitions. rewriters.add(storageTableResolver); // Prune candidate tables for which denorm column references do not exist //TODO union: phase 2 of denormResolver needs to be moved before CoveringSetResolver.. check if this makes sense rewriters.add(denormResolver); - // Prune candidate facts without any valid expressions + // Phase 2 of exprResolver : Prune candidate facts without any valid expressions rewriters.add(exprResolver); - // We can have LightestFactResolver before LeastPartitionResolver - that says - // "if two facts have the same least weight, then the fact with least number of time partitions queried will be - // picked". This will be useful, if users did not set fact weights. + if (!lightFactFirst) { + // Pick the least cost combination(s) (and prune others) out of a set of combinations produced + // by CandidateCoveringSetsResolver rewriters.add(new LightestFactResolver(conf)); } + // if two combinations have the same least weight/cost, then the combination with least number of time partitions + // queried will be picked. Rest of the combinations will be pruned rewriters.add(new LeastPartitionResolver(conf)); rewriters.add(new LightestDimensionResolver(conf)); } http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java index 02e2f8b..df4e07a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java @@ -39,6 +39,7 @@ import org.apache.lens.api.LensSessionHandle; import org.apache.lens.api.metastore.ObjectFactory; import org.apache.lens.api.metastore.XCube; import org.apache.lens.api.metastore.XFactTable; +import org.apache.lens.api.metastore.XStorage; import org.apache.lens.api.query.*; import org.apache.lens.api.result.LensAPIResult; @@ -166,6 +167,16 @@ public class RestAPITestUtil { checkResponse(result); } + public static void createStorageFailFast(final WebTarget target, final LensSessionHandle sessionId, + final XStorage storage, MediaType mt) { + APIResult result = target.path("metastore").path("storages").queryParam("sessionid", sessionId) + .request(mt).post(Entity.entity( + new GenericEntity<JAXBElement<XStorage>>(cubeObjectFactory.createXStorage(storage)) { + }, mt), + APIResult.class); + checkResponse(result); + } + public static APIResult setCurrentDatabase(final WebTarget target, final LensSessionHandle sessionId, final String dbName, MediaType mt) { http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java index 0400519..fbfd898 100644 --- a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java +++ b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java @@ -50,6 +50,10 @@ public class TestDataUtils { return "db" + getRandomName(); } + public static String getRandomStorageName() { + return "storage" + getRandomName(); + } + public static String getRandomCubeName() { return "cube" + getRandomName(); } http://git-wip-us.apache.org/repos/asf/lens/blob/4d493597/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java index 567c929..064da01 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java @@ -47,9 +47,11 @@ import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.api.result.LensErrorTO; import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; import org.apache.lens.cube.error.ColUnAvailableInTimeRange; +import org.apache.lens.cube.metadata.HDFSStorage; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensRequestLoggingFilter; import org.apache.lens.server.common.ErrorResponseExpectedData; +import org.apache.lens.server.common.RestAPITestUtil; import org.apache.lens.server.error.GenericExceptionMapper; import org.apache.lens.server.error.LensJAXBValidationExceptionMapper; import org.apache.lens.server.metastore.MetastoreResource; @@ -255,6 +257,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest { final String testCube = getRandomCubeName(); final String testDimensionField = getRandomDimensionField(); final String testFact = getRandomFactName(); + final String testStorage = getRandomStorageName(); /* Setup: Begin */ LensSessionHandle sessionId = openSession(target, "foo", "bar", new LensConf(), mt); @@ -268,9 +271,21 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest { XCube xcube = createXCubeWithDummyMeasure(testCube, Optional.of("dt"), testXDim); createCubeFailFast(target, sessionId, xcube, mt); + /* Create Storage */ + XStorage xs = new XStorage(); + xs.setClassname(HDFSStorage.class.getCanonicalName()); + xs.setName(testStorage); + RestAPITestUtil.createStorageFailFast(target, sessionId, xs, mt); + /* Create a fact with test dimension field */ XColumn xColumn = createXColumn(testDimensionField); XFactTable xFactTable = createXFactTableWithColumns(testFact, testCube, xColumn); + + //Create a StorageTable + XStorageTables tables = new XStorageTables(); + tables.getStorageTable().add(createStorageTblElement(testStorage,"DAILY")); + xFactTable.setStorageTables(tables); + createFactFailFast(target, sessionId, xFactTable, mt); /* Setup: End */ @@ -343,6 +358,18 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest { } finally { closeSessionFailFast(target(), sessionId, mt); } + } + private XStorageTableElement createStorageTblElement(String storageName, String... updatePeriod) { + XStorageTableElement tbl = new XStorageTableElement(); + tbl.setUpdatePeriods(new XUpdatePeriods()); + tbl.setStorageName(storageName); + if (updatePeriod != null) { + for (String p : updatePeriod) { + tbl.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p)); + } + } + tbl.setTableDesc(new XStorageTableDesc()); + return tbl; } }
