Repository: lens Updated Branches: refs/heads/master 8a3657291 -> b3f993d8a
LENS-912 : Make keyword optional in queries Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b3f993d8 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b3f993d8 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b3f993d8 Branch: refs/heads/master Commit: b3f993d8af508f900b22b7c67813b734b50255d8 Parents: 8a36572 Author: Rajat Khandelwal <[email protected]> Authored: Wed Jan 27 16:58:18 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Wed Jan 27 16:58:18 2016 +0530 ---------------------------------------------------------------------- .../org/apache/lens/api/query/QueryStatus.java | 5 + .../lens/cube/metadata/CubeMetastoreClient.java | 20 +- .../org/apache/lens/cube/parse/HQLParser.java | 7 + .../server/query/QueryExecutionServiceImpl.java | 2 +- .../lens/server/query/QueryServiceResource.java | 3 +- .../lens/server/rewrite/CubeKeywordRemover.java | 41 ++++ .../apache/lens/server/rewrite/RewriteUtil.java | 150 +++++++++----- .../rewrite/UserQueryToCubeQueryRewriter.java | 2 +- .../lens/server/query/TestQueryService.java | 36 ++-- .../server/rewrite/CubeKeywordRemoverTest.java | 46 +++++ .../lens/server/rewrite/TestRewriting.java | 198 +++++++++++++++---- src/site/apt/user/olap-cube.apt | 2 +- 12 files changed, 403 insertions(+), 109 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java ---------------------------------------------------------------------- diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java index 91cbe39..44fd97e 100644 --- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java +++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java @@ -220,6 +220,11 @@ public class QueryStatus implements Serializable { return status.equals(Status.QUEUED); } + public boolean failed() { + return status.equals(Status.FAILED); + } + + /** * Checks if is valid transition. * http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index dcb932e..8969d1f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -40,14 +40,16 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; -import org.apache.hadoop.hive.ql.metadata.*; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.thrift.TException; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import lombok.extern.slf4j.Slf4j; /** @@ -234,6 +236,20 @@ public class CubeMetastoreClient { return latestParts; } + public boolean isLensQueryableTable(String tableName) { + try { + Table table = getTable(tableName); + String typeProperty = table.getProperty(MetastoreConstants.TABLE_TYPE_KEY); + if (StringUtils.isBlank(typeProperty)) { + return false; + } + CubeTableType type = CubeTableType.valueOf(typeProperty); + return type == CubeTableType.CUBE || type == CubeTableType.DIMENSION; + } catch (HiveException e) { + return false; + } + } + /** * In-memory storage of {@link PartitionTimeline} objects for each valid http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index b1deb07..c9aff5d 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -833,4 +833,11 @@ public final class HQLParser { return true; } + + public static ASTNode leftMostChild(ASTNode node) { + while (node.getChildren() != null) { + node = (ASTNode) node.getChild(0); + } + return node; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index 2dff9af..672f2be 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -432,7 +432,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE log.info("Driver {} for type {} is loaded", driverPath.getName(), driverType); } catch (Exception e) { log.error("Could not load driver {} of type {}", driverPath.getName(), driverType, e); - throw new LensException("Could not load driver "+driverPath.getName()+ " of type "+ driverType); + throw new LensException("Could not load driver "+driverPath.getName()+ " of type "+ driverType, e); } } } http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java index 08192bd..bb4cfd2 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java @@ -199,7 +199,8 @@ public class QueryServiceResource { * {@link org.apache.lens.api.query.SubmitOp#EXECUTE} operation. * {@link QueryPlan} in case of {@link org.apache.lens.api.query.SubmitOp#EXPLAIN} operation. * {@link QueryHandleWithResultSet} in case {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT} - * operation. {@link QueryCostTO} in case of {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation. + * operation. {@link org.apache.lens.api.result.QueryCostTO} in case of + * {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation. */ @POST @Path("queries") http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java new file mode 100644 index 0000000..fd8d6e7 --- /dev/null +++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/CubeKeywordRemover.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.rewrite; + + +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.rewrite.Phase1Rewriter; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; + +public class CubeKeywordRemover implements Phase1Rewriter { + @Override + public String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException { + return query.replaceAll("(?i)cube\\s+select", "select"); + } + + @Override + public void init(Configuration rewriteConf) { + + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java index abec2b3..1c0cd35 100644 --- a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java +++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java @@ -22,9 +22,8 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; +import org.apache.lens.cube.metadata.CubeMetastoreClient; import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.cube.parse.CubeQueryRewriter; import org.apache.lens.cube.parse.HQLParser; @@ -38,6 +37,7 @@ import org.apache.lens.server.api.query.AbstractQueryContext; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.HiveParser; @@ -52,14 +52,6 @@ public final class RewriteUtil { private RewriteUtil() { } - - /** The cube pattern. */ - static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE - | Pattern.DOTALL); - - /** The matcher. */ - static Matcher matcher = null; - /** * The Class CubeQueryInfo. */ @@ -92,14 +84,15 @@ public final class RewriteUtil { if (log.isDebugEnabled()) { log.debug("User query AST:{}", ast.dump()); } - List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>(); - findCubePositions(ast, cubeQueries, query); + List<CubeQueryInfo> cubeQueries = new ArrayList<>(); + findCubePositions(ast, cubeQueries, query, conf); for (CubeQueryInfo cqi : cubeQueries) { cqi.query = query.substring(cqi.startPos, cqi.endPos); } return cubeQueries; } + /** * Find cube positions. * @@ -108,7 +101,8 @@ public final class RewriteUtil { * @param originalQuery the original query * @throws LensException the lens exception */ - private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery) + private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery, + HiveConf conf) throws LensException { int childCount = ast.getChildCount(); if (ast.getToken() != null) { @@ -116,13 +110,22 @@ public final class RewriteUtil { log.debug("First child: {} Type:{}", ast.getChild(0), ((ASTNode) ast.getChild(0)).getToken().getType()); } if (ast.getToken().getType() == HiveParser.TOK_QUERY - && ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) { + && (isCubeKeywordNode((ASTNode) ast.getChild(0)) || isFromNodeWithCubeTable((ASTNode) ast.getChild(0), conf))) { log.debug("Inside cube clause"); CubeQueryInfo cqi = new CubeQueryInfo(); cqi.cubeAST = ast; if (ast.getParent() != null) { ASTNode parent = (ASTNode) ast.getParent(); - cqi.startPos = ast.getCharPositionInLine(); + if (isCubeKeywordNode((ASTNode) ast.getChild(0))) { + cqi.startPos = ast.getCharPositionInLine(); + } else { + ASTNode selectAST = (ASTNode) ast.getChild(1).getChild(1); + // Left most child of select AST will have char position just after select / select distinct + // Go back one "select[ distinct]" + cqi.startPos = getStartPos(originalQuery, HQLParser.leftMostChild(selectAST).getCharPositionInLine(), + "distinct"); + cqi.startPos = getStartPos(originalQuery, cqi.startPos, "select"); + } int ci = ast.getChildIndex(); if (parent.getToken() == null || parent.getToken().getType() == HiveParser.TOK_EXPLAIN || parent.getToken().getType() == HiveParser.TOK_CREATETABLE) { @@ -134,7 +137,17 @@ public final class RewriteUtil { cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine(), ")"); } else if (parent.getToken().getType() == HiveParser.TOK_UNION) { // one less for the next start and less the size of string 'UNION ALL' - cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL"); + ASTNode nextChild = (ASTNode) parent.getChild(ci + 1); + if (isCubeKeywordNode((ASTNode) nextChild.getChild(0))) { + cqi.endPos = getEndPos(originalQuery, nextChild.getCharPositionInLine() - 1, "UNION ALL"); + } else { + // Go back one "union all select[ distinct]" + cqi.endPos = getEndPos(originalQuery, nextChild.getChild(1).getChild(1).getCharPositionInLine() - 1, + "distinct"); + cqi.endPos = getEndPos(originalQuery, cqi.endPos, "select"); + cqi.endPos = getEndPos(originalQuery, cqi.endPos, "union all"); + } + } else { // Not expected to reach here log.warn("Unknown query pattern found with AST:{}", ast.dump()); @@ -156,7 +169,7 @@ public final class RewriteUtil { cubeQueries.add(cqi); } else { for (int childPos = 0; childPos < childCount; ++childPos) { - findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery); + findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery, conf); } } } else { @@ -164,6 +177,39 @@ public final class RewriteUtil { } } + private static boolean isCubeTableNode(ASTNode node, HiveConf conf) throws LensException { + if (node.getType() == HiveParser.TOK_TABREF || node.getType() == HiveParser.TOK_TABNAME) { + return isCubeTableNode((ASTNode) node.getChild(0), conf); + } + if (node.getText().contains("JOIN")) { + if (isCubeTableNode((ASTNode) node.getChild(0), conf)) { + for (int i = 1; i < node.getChildCount(); i += 2) { + if (!isCubeTableNode((ASTNode) node.getChild(i), conf)) { + return false; + } + } + return true; + } + } + return node.getType() == HiveParser.Identifier && getClient(conf).isLensQueryableTable(node.getText()); + } + + private static boolean isFromNodeWithCubeTable(ASTNode child, HiveConf conf) throws LensException { + return child.getType() == HiveParser.TOK_FROM && isCubeTableNode((ASTNode) child.getChild(0), conf); + } + + public static CubeMetastoreClient getClient(HiveConf conf) throws LensException { + try { + return CubeMetastoreClient.getInstance(conf); + } catch (HiveException e) { + throw new LensException("Couldn't get instance of metastore client", e); + } + } + + private static boolean isCubeKeywordNode(ASTNode child) { + return child.getToken().getType() == HiveParser.KW_CUBE; + } + /** * Gets the end pos. * @@ -173,17 +219,38 @@ public final class RewriteUtil { * @return the end pos */ private static int getEndPos(String query, int backTrackIndex, String... backTrackStr) { + backTrackIndex = backTrack(query, backTrackIndex, backTrackStr); + while (backTrackIndex > 0 && Character.isSpaceChar(query.charAt(backTrackIndex - 1))) { + backTrackIndex--; + } + return backTrackIndex; + } + + private static int backTrack(String query, int backTrackIndex, String... backTrackStr) { if (backTrackStr != null) { String q = query.substring(0, backTrackIndex).toLowerCase(); - for (int i = 0; i < backTrackStr.length; i++) { - if (q.trim().endsWith(backTrackStr[i].toLowerCase())) { - backTrackIndex = q.lastIndexOf(backTrackStr[i].toLowerCase()); + for (String aBackTrackStr : backTrackStr) { + if (q.trim().endsWith(aBackTrackStr.toLowerCase())) { + backTrackIndex = q.lastIndexOf(aBackTrackStr.toLowerCase()); break; } } } - while (Character.isSpaceChar(query.charAt(backTrackIndex - 1))) { - backTrackIndex--; + return backTrackIndex; + } + + /** + * Gets the end pos. + * + * @param query the query + * @param backTrackIndex the back track index + * @param backTrackStr the back track str + * @return the end pos + */ + private static int getStartPos(String query, int backTrackIndex, String... backTrackStr) { + backTrackIndex = backTrack(query, backTrackIndex, backTrackStr); + while (backTrackIndex < query.length() && Character.isSpaceChar(query.charAt(backTrackIndex))) { + backTrackIndex++; } return backTrackIndex; } @@ -224,7 +291,7 @@ public final class RewriteUtil { try { String replacedQuery = getReplacedQuery(ctx.getPhase1RewrittenQuery()); - Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<LensDriver, DriverRewriterRunnable>(); + Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<>(); List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf()); for (LensDriver driver : ctx.getDriverContext().getDrivers()) { @@ -246,8 +313,7 @@ public final class RewriteUtil { } public static DriverQueryPlan getRewriterPlan(DriverRewriterRunnable rewriter) { - RewriterPlan plan = new RewriterPlan(rewriter.cubeQueryCtx); - return plan; + return new RewriterPlan(rewriter.cubeQueryCtx); } public static class DriverRewriterRunnable implements Runnable { @@ -275,15 +341,15 @@ public final class RewriteUtil { private String rewrittenQuery; public DriverRewriterRunnable(LensDriver driver, - AbstractQueryContext ctx, - List<CubeQueryInfo> cubeQueries, - String replacedQuery) { + AbstractQueryContext ctx, + List<CubeQueryInfo> cubeQueries, + String replacedQuery) { this.driver = driver; this.ctx = ctx; this.cubeQueries = cubeQueries; this.replacedQuery = replacedQuery; if (cubeQueries != null) { - cubeQueryCtx = new ArrayList<CubeQueryContext>(cubeQueries.size()); + cubeQueryCtx = new ArrayList<>(cubeQueries.size()); } } @@ -296,7 +362,7 @@ public final class RewriteUtil { } MethodMetricsContext rewriteGauge = MethodMetricsFactory - .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE); + .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE); StringBuilder builder = new StringBuilder(); int start = 0; CubeQueryRewriter rewriter = null; @@ -320,7 +386,7 @@ public final class RewriteUtil { // Parse and rewrite individual cube query CubeQueryContext cqc = rewriter.rewrite(cqi.query); MethodMetricsContext toHQLGauge = MethodMetricsFactory - .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE); + .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE); // toHQL actually generates the rewritten query String hqlQuery = cqc.toHQL(); cubeQueryCtx.add(cqc); @@ -368,25 +434,9 @@ public final class RewriteUtil { log.warn("Driver : {} Skipped for the query rewriting due to ", driver, e); ctx.setDriverRewriteError(driver, e); failureCause = new StringBuilder(" Driver :") - .append(driver.getFullyQualifiedName()) - .append(" Cause :" + e.getLocalizedMessage()) - .toString(); - } - } - - /** - * Checks if is cube query. - * - * @param query the query - * @return true, if is cube query - */ - public static boolean isCubeQuery(String query) { - if (matcher == null) { - matcher = cubePattern.matcher(query); - } else { - matcher.reset(query); + .append(driver.getFullyQualifiedName()) + .append(" Cause :" + e.getLocalizedMessage()) + .toString(); } - return matcher.matches(); } - } http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java index 35de4d7..79cee62 100644 --- a/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java +++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java @@ -39,7 +39,7 @@ import com.google.common.collect.Lists; * @see LensConfConstants#QUERY_PHASE1_REWRITERS */ public class UserQueryToCubeQueryRewriter { - List<Phase1Rewriter> phase1RewriterList = Lists.newArrayList(); + List<Phase1Rewriter> phase1RewriterList = Lists.<Phase1Rewriter>newArrayList(new CubeKeywordRemover()); public UserQueryToCubeQueryRewriter(Configuration conf) throws LensException { try { http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 494bce5..3facded 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -228,7 +228,7 @@ public class TestQueryService extends LensJerseyTest { Collection<LensDriver> drivers = queryService.getDrivers(); assertEquals(drivers.size(), 4); Set<String> driverNames = new HashSet<String>(drivers.size()); - for(LensDriver driver : drivers){ + for (LensDriver driver : drivers) { assertEquals(driver.getConf().get("lens.driver.test.drivername"), driver.getFullyQualifiedName()); driverNames.add(driver.getFullyQualifiedName()); } @@ -458,7 +458,7 @@ public class TestQueryService extends LensJerseyTest { Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE)); LensAPIResult result = response.readEntity(LensAPIResult.class); List<LensErrorTO> childErrors = result.getLensErrorTO().getChildErrors(); - boolean hiveSemanticErrorExists=false; + boolean hiveSemanticErrorExists = false; for (LensErrorTO error : childErrors) { if (error.getCode() == LensDriverErrorCode.SEMANTIC_ERROR.getLensErrorInfo().getErrorCode()) { hiveSemanticErrorExists = true; @@ -1335,7 +1335,7 @@ public class TestQueryService extends LensJerseyTest { if (driver instanceof HiveDriver) { addedToHiveDriver = ((HiveDriver) driver).areDBResourcesAddedForSession(sessionHandle.getPublicId().toString(), DB_WITH_JARS); - if (addedToHiveDriver){ + if (addedToHiveDriver) { break; //There are two Hive drivers now both pointing to same hive server. So break after first success } } @@ -1382,7 +1382,8 @@ public class TestQueryService extends LensJerseyTest { final FormDataMultiPart mp = new FormDataMultiPart(); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "cube select ID from nonexist")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), + "cube sdfelect ID from cube_nonexist")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "estimate")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(), MediaType.APPLICATION_XML_TYPE)); @@ -1392,8 +1393,9 @@ public class TestQueryService extends LensJerseyTest { LensErrorTO expectedLensErrorTO = LensErrorTO.composedOf( - LensCubeErrorCode.NEITHER_CUBE_NOR_DIMENSION.getLensErrorInfo().getErrorCode(), - "Neither cube nor dimensions accessed in the query", TestDataUtils.MOCK_STACK_TRACE); + LensCubeErrorCode.SYNTAX_ERROR.getLensErrorInfo().getErrorCode(), + "Syntax Error: line 1:5 cannot recognize input near 'sdfelect' 'ID' 'from' in select clause", + TestDataUtils.MOCK_STACK_TRACE); ErrorResponseExpectedData expectedData = new ErrorResponseExpectedData(BAD_REQUEST, expectedLensErrorTO); expectedData.verify(response); @@ -1458,17 +1460,17 @@ public class TestQueryService extends LensJerseyTest { MetricRegistry reg = LensMetricsRegistry.getStaticRegistry(); assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-CUBE_REWRITE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-DRIVER_ESTIMATE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-RewriteUtil-rewriteQuery", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-CUBE_REWRITE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-DRIVER_ESTIMATE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-RewriteUtil-rewriteQuery", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-CUBE_REWRITE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-DRIVER_ESTIMATE", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-RewriteUtil-rewriteQuery", - "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")), + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-CUBE_REWRITE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-DRIVER_ESTIMATE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive1-RewriteUtil-rewriteQuery", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-CUBE_REWRITE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-DRIVER_ESTIMATE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-hive/hive2-RewriteUtil-rewriteQuery", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-CUBE_REWRITE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-DRIVER_ESTIMATE", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-jdbc/jdbc1-RewriteUtil-rewriteQuery", + "lens.MethodMetricGauge.TestQueryService-testEstimateGauges-PARALLEL_ESTIMATE")), reg.getGauges().keySet().toString()); } http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java new file mode 100644 index 0000000..956d433 --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/CubeKeywordRemoverTest.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * + */ +package org.apache.lens.server.rewrite; + +import static org.testng.Assert.assertEquals; + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class CubeKeywordRemoverTest { + CubeKeywordRemover cubeKeywordRemover = new CubeKeywordRemover(); + + @DataProvider + public Object[][] cubeQueryDataProvider() { + return new Object[][]{ + {"cube select blah blah", "select blah blah"}, + {"cube\tselect blah blah", "select blah blah"}, + {"cube\nselect blah blah", "select blah blah"}, + {"CUBE sElEct blAh blAh", "select blAh blAh"}, + }; + } + + @Test(dataProvider = "cubeQueryDataProvider") + public void testRewrite(String userQuery, String expected) throws Exception { + assertEquals(cubeKeywordRemover.rewrite(userQuery, null, null), expected); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java index 202db82..0e640ec 100644 --- a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java +++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java @@ -21,6 +21,7 @@ package org.apache.lens.server.rewrite; import java.util.*; import org.apache.lens.api.LensConf; +import org.apache.lens.cube.metadata.CubeMetastoreClient; import org.apache.lens.cube.parse.CubeQueryContext; import org.apache.lens.cube.parse.CubeQueryRewriter; import org.apache.lens.cube.parse.HQLParser; @@ -35,7 +36,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.parse.*; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; +import org.apache.hadoop.hive.ql.parse.ParseException; import org.mockito.Matchers; import org.mockito.Mockito; @@ -75,7 +78,19 @@ public class TestRewriting { // number of successful queries through mock rewriter // we use this number to mock failures after successful queries // change the number, if more tests for success needs to be added - static final int NUM_SUCCESS = 36; + static final int NUM_SUCCESS = 63; + + public static CubeMetastoreClient getMockedClient() { + CubeMetastoreClient client = Mockito.mock(CubeMetastoreClient.class); + Mockito.when(client.isLensQueryableTable(Matchers.any(String.class))).thenAnswer(new Answer<Boolean>() { + + @Override + public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { + return invocationOnMock.getArguments()[0].toString().toLowerCase().contains("cube"); + } + }); + return client; + } private CubeQueryRewriter getMockedRewriter() throws ParseException, LensException, HiveException { CubeQueryRewriter mockwriter = Mockito.mock(CubeQueryRewriter.class); @@ -83,9 +98,8 @@ public class TestRewriting { @Override public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - i++; // return query for first NUM_SUCCESS calls and fail later - if (i <= NUM_SUCCESS) { + if (++i <= NUM_SUCCESS) { return getMockedCubeContext((String) args[0]); } else { throw new RuntimeException("Mock fail"); @@ -114,7 +128,8 @@ public class TestRewriting { throws ParseException, LensException { CubeQueryContext context = Mockito.mock(CubeQueryContext.class); Mockito.when(context.toHQL()).thenReturn(query.substring(4)); - Mockito.when(context.toAST(Matchers.any(Context.class))).thenReturn(HQLParser.parseHQL(query.substring(4), hconf)); + Mockito.when(context.toAST(Matchers.any(Context.class))) + .thenReturn(HQLParser.parseHQL(query.toLowerCase().replaceFirst("^cube", ""), hconf)); return context; } @@ -181,9 +196,10 @@ public class TestRewriting { drivers.add(driver); CubeQueryRewriter mockWriter = getMockedRewriter(); + CubeMetastoreClient mockClient = getMockedClient(); PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter); + PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getClient")).toReturn(mockClient); String q1 = "select name from table"; - Assert.assertFalse(RewriteUtil.isCubeQuery(q1)); List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf); Assert.assertEquals(cubeQueries.size(), 0); QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers); @@ -192,7 +208,6 @@ public class TestRewriting { conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName()); driver.configure(conf, null, null); String q2 = "cube select name from table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -201,20 +216,43 @@ public class TestRewriting { MetricRegistry reg = LensMetricsRegistry.getStaticRegistry(); Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-RewriteUtil-rewriteQuery", - "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-1-RewriteUtil-rewriteQuery-toHQL"))); + "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-RewriteUtil-rewriteQuery", + "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-1-RewriteUtil-rewriteQuery-toHQL"))); conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY); + assertIsCubeQuery("select name from cube_table", lensConf, conf, drivers); + assertIsCubeQuery("select cube_table.name from cube_table", lensConf, conf, drivers); + assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim " + + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers); + assertIsCubeQuery("select distinct cube_dim.name, cube_dim2.name from cube_dim full outer " + + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers); + assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim left outer " + + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers); + assertIsCubeQuery("select cube_dim.name, cube_dim2.name from cube_dim inner " + + "join cube_dim2 on cube_dim.d2id=cube_dim2.id", lensConf, conf, drivers); + + q2 = "select * from (select name from cube_table) table1"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "insert overwrite directory 'target/rewrite' cube select name from table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "insert overwrite directory 'target/rewrite' select name from cube_table"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "insert overwrite local directory 'target/rewrite' cube select name from table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -222,7 +260,6 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "insert overwrite local directory 'target/example-output' cube select id,name from dim_table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table"); @@ -230,7 +267,6 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "explain cube select name from table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -238,7 +274,6 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "select * from (cube select name from table) a"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -246,15 +281,20 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "insert overwrite directory 'target/rewrite' select * from (cube select name from table) a"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "insert overwrite directory 'target/rewrite' select * from (select name from cube_table) a"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table)a"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -262,16 +302,21 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "select * from ( cube select name from table ) a"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from ( select name from cube_table ) a"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from ( cube select name from table where" + " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from" @@ -279,9 +324,17 @@ public class TestRewriting { ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from ( select name from cube_table where" + + " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a"; + cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from" + + " cube_table where (name = 'ABC' OR name = 'XYZ') AND (key=100)"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName() + "-multiple"); q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 2); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -290,15 +343,15 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); reg = LensMetricsRegistry.getStaticRegistry(); Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList( - "lens.MethodMetricGauge.TestRewriting-"+driver.getFullyQualifiedName()+"-1-RewriteUtil-rewriteQuery-toHQL", - "lens.MethodMetricGauge.TestRewriting-multiple-"+driver.getFullyQualifiedName() - +"-2-RewriteUtil-rewriteQuery-toHQL", - "lens.MethodMetricGauge.TestRewriting-multiple-"+driver.getFullyQualifiedName()+"-RewriteUtil-rewriteQuery"))); + "lens.MethodMetricGauge.TestRewriting-" + driver.getFullyQualifiedName() + "-1-RewriteUtil-rewriteQuery-toHQL", + "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName() + + "-2-RewriteUtil-rewriteQuery-toHQL", + "lens.MethodMetricGauge.TestRewriting-multiple-" + driver.getFullyQualifiedName() + + "-RewriteUtil-rewriteQuery"))); conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY); q2 = "select * from (cube select name from table) a full outer join" + " (cube select name2 from table2) b on a.name=b.name2"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 2); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -306,16 +359,31 @@ public class TestRewriting { ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (select name from cube_table) a full outer join" + + " (select name2 from cube_table2) b on a.name=b.name2"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table) a join (select name2 from table2) b"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table) a join (select name2 from cube_table2) b"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); + Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table union all cube select name2 from table2) u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -323,9 +391,24 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2"); + q2 = "select * from (select name from cube_table union all select distinct name2 from cube_table2) u"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + Assert.assertEquals(cubeQueries.get(1).query, "select distinct name2 from cube_table2"); + + q2 = "select * from (select distinct name from cube_table union all select name2 from cube_table2) u"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "select distinct name from cube_table"); + Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2"); + q2 = "insert overwrite directory 'target/rewrite' " + "select * from (cube select name from table union all cube select name2 from table2) u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -333,8 +416,16 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2"); + q2 = "insert overwrite directory 'target/rewrite' " + + "select * from (cube select name from table union all select name2 from cube_table2) u"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); + Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2"); + q2 = "select u.* from (select name from table union all cube select name2 from table2) u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2"); @@ -342,7 +433,6 @@ public class TestRewriting { runRewrites(RewriteUtil.rewriteQuery(ctx)); q2 = "select u.* from (select name from table union all cube select name2 from table2)u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2"); @@ -351,7 +441,6 @@ public class TestRewriting { q2 = "select * from (cube select name from table union all cube select name2" + " from table2 union all cube select name3 from table3) u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -362,7 +451,6 @@ public class TestRewriting { q2 = "select * from ( cube select name from table union all cube" + " select name2 from table2 union all cube select name3 from table3 ) u"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -372,7 +460,6 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3"); q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); Assert.assertEquals(cubeQueries.size(), 2); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); @@ -380,8 +467,15 @@ public class TestRewriting { ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table union all select" + " name2 from cube_table2) u group by u.name"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); + Assert.assertEquals(cubeQueries.get(1).query, "select name2 from cube_table2"); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -390,16 +484,21 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2"); q2 = "create table temp1 as cube select name from table"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); Assert.assertEquals(cubeQueries.size(), 1); Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); + q2 = "create table temp1 as select name from cube_table"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + q2 = "create table temp1 as select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -407,9 +506,18 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table"); Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2"); + q2 = "create table temp1 as select * from (select name from cube_table union all cube select" + + " name2 from table2) u group by u.name"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 2); + Assert.assertEquals(cubeQueries.get(0).query, "select name from cube_table"); + Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2"); + + q2 = "create table temp1 as cube select name from table where" + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')"; - Assert.assertTrue(RewriteUtil.isCubeQuery(q2)); cubeQueries = RewriteUtil.findCubePositions(q2, hconf); ctx = new QueryContext(q2, null, lensConf, conf, drivers); runRewrites(RewriteUtil.rewriteQuery(ctx)); @@ -417,6 +525,15 @@ public class TestRewriting { Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')"); + q2 = "create table temp1 as select name from cube_table where" + + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')"; + cubeQueries = RewriteUtil.findCubePositions(q2, hconf); + ctx = new QueryContext(q2, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, + "select name from cube_table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')"); + // failing query for second driver MockDriver driver2 = new MockDriver(); driver2.configure(conf, null, null); @@ -460,4 +577,13 @@ public class TestRewriting { Assert.assertNull(runnables.get(driver2).getRewrittenQuery()); Assert.assertNotNull(ctx.getDriverRewriteError(driver2)); } + + private void assertIsCubeQuery(String query, LensConf lensConf, Configuration conf, List<LensDriver> drivers) + throws LensException { + List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(query, hconf); + Assert.assertEquals(cubeQueries.size(), 1); + Assert.assertEquals(cubeQueries.get(0).query, query); + QueryContext ctx = new QueryContext(query, null, lensConf, conf, drivers); + runRewrites(RewriteUtil.rewriteQuery(ctx)); + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b3f993d8/src/site/apt/user/olap-cube.apt ---------------------------------------------------------------------- diff --git a/src/site/apt/user/olap-cube.apt b/src/site/apt/user/olap-cube.apt index c47ac30..4bed623 100644 --- a/src/site/apt/user/olap-cube.apt +++ b/src/site/apt/user/olap-cube.apt @@ -283,7 +283,7 @@ lens-shell> +---+ - CUBE SELECT [DISTINCT] select_expr, select_expr, ... + [CUBE] SELECT [DISTINCT] select_expr, select_expr, ... FROM cube_table_reference [WHERE [where_condition AND] [TIME_RANGE_IN(colName, from, to)]] [GROUP BY col_list]
