LENS-788 : Push expressions before flattening and convert filters to array_contains filters for bridge tables
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/dba885ca Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/dba885ca Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/dba885ca Branch: refs/heads/master Commit: dba885ca972e8a11281f1bc44ceb4b7dd3906ce0 Parents: 79b95f0 Author: Amareshwari Sriramadasu <[email protected]> Authored: Wed Mar 30 10:25:20 2016 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Wed Mar 30 10:25:20 2016 +0530 ---------------------------------------------------------------------- .../apache/lens/cube/parse/CandidateFact.java | 51 +- .../lens/cube/parse/CubeQueryConfUtil.java | 5 + .../lens/cube/parse/CubeQueryContext.java | 21 +- .../lens/cube/parse/DefaultAliasDecider.java | 13 +- .../apache/lens/cube/parse/DefaultQueryAST.java | 5 +- .../apache/lens/cube/parse/DimHQLContext.java | 29 +- .../lens/cube/parse/DimOnlyHQLContext.java | 10 - .../org/apache/lens/cube/parse/HQLParser.java | 49 +- .../apache/lens/cube/parse/JoinResolver.java | 9 +- .../lens/cube/parse/MultiFactHQLContext.java | 3 - .../org/apache/lens/cube/parse/QueryAST.java | 6 +- .../parse/SingleFactMultiStorageHQLContext.java | 8 - .../SingleFactSingleStorageHQLContext.java | 12 - .../lens/cube/parse/join/AutoJoinContext.java | 104 +--- .../cube/parse/join/BridgeTableJoinContext.java | 321 ++++++++++ .../src/main/resources/olap-query-conf.xml | 18 + .../apache/lens/cube/parse/CubeTestSetup.java | 19 +- .../lens/cube/parse/TestBridgeTableQueries.java | 618 +++++++++++++++++-- .../cube/parse/TestDefaultAliasDecider.java | 53 ++ .../apache/lens/cube/parse/TestHQLParser.java | 82 +++ .../org/apache/lens/cube/parse/TestQuery.java | 18 +- .../cube/parse/join/TestBridgeTableJoinCtx.java | 122 ++++ lens-dist/src/main/assembly/bin-dist.xml | 16 + .../lens/examples/PopulateSampleMetastore.java | 2 + .../apache/lens/examples/SampleMetastore.java | 4 + .../src/main/resources/cube-queries.sql | 6 +- .../resources/customer-interests-local-part.xml | 30 + .../resources/customer-interests-local.data | 6 + .../src/main/resources/customer-interests.xml | 31 + .../main/resources/customer_interests_table.xml | 52 ++ .../src/main/resources/db-storage-schema.sql | 18 + .../src/main/resources/interests-local-part.xml | 29 + .../src/main/resources/interests-local.data | 4 + lens-examples/src/main/resources/interests.xml | 31 + .../src/main/resources/interests_table.xml | 51 ++ lens-examples/src/main/resources/sales-cube.xml | 24 + src/site/apt/user/olap-cube.apt | 8 + src/site/apt/user/olap-query-conf.apt | 48 +- tools/conf/client/lens-client-site.xml | 16 +- 39 files changed, 1678 insertions(+), 274 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java index 82ca4f4..3f724b6 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java @@ -23,8 +23,6 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.*; import java.util.*; import org.apache.lens.cube.metadata.*; -import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor; -import org.apache.lens.cube.parse.HQLParser.TreeNode; import org.apache.lens.server.api.error.LensException; import org.apache.commons.lang.StringUtils; @@ -76,6 +74,8 @@ public class CandidateFact implements CandidateTable, QueryAST { @Getter @Setter private Integer limitValue; + @Getter + private String fromString; private final List<Integer> selectIndices = Lists.newArrayList(); private final List<Integer> dimFieldIndices = Lists.newArrayList(); private Collection<String> columns; @@ -166,7 +166,7 @@ public class CandidateFact implements CandidateTable, QueryAST { } public boolean isExpressionAnswerable(ASTNode node, CubeQueryContext context) throws LensException { - return getColumns().containsAll(getColsInExpr(context, context.getCube().getAllFieldNames(), node)); + return getColumns().containsAll(HQLParser.getColsInExpr(context.getAliasForTableName(context.getCube()), node)); } /** @@ -182,7 +182,7 @@ public class CandidateFact implements CandidateTable, QueryAST { int currentChild = 0; for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) { ASTNode selectExpr = (ASTNode) this.selectAST.getChild(currentChild); - Set<String> exprCols = getColsInExpr(cubeql, cubeCols, selectExpr); + Set<String> exprCols = HQLParser.getColsInExpr(cubeql.getAliasForTableName(cubeql.getCube()), selectExpr); if (getColumns().containsAll(exprCols)) { selectIndices.add(i); if (cubeql.getCube().getDimAttributeNames().containsAll(exprCols)) { @@ -216,39 +216,6 @@ public class CandidateFact implements CandidateTable, QueryAST { // push down of having clauses happens just after this call in cubequerycontext } - private Set<String> getColsInExpr(final CubeQueryContext cubeql, final Set<String> cubeCols, - ASTNode expr) throws LensException { - final Set<String> cubeColsInExpr = new HashSet<>(); - HQLParser.bft(expr, new ASTNodeVisitor() { - @Override - public void visit(TreeNode visited) { - ASTNode node = visited.getNode(); - ASTNode parent = null; - if (visited.getParent() != null) { - parent = visited.getParent().getNode(); - } - - if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) { - // Take child ident.totext - ASTNode ident = (ASTNode) node.getChild(0); - String column = ident.getText().toLowerCase(); - if (cubeCols.contains(column)) { - cubeColsInExpr.add(column); - } - } else if (node.getToken().getType() == DOT) { - String alias = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).getText().toLowerCase(); - ASTNode colIdent = (ASTNode) node.getChild(1); - String column = colIdent.getText().toLowerCase(); - if (cubeql.getAliasForTableName(cubeql.getCube()).equalsIgnoreCase(alias) && cubeCols.contains(column)) { - cubeColsInExpr.add(column); - } - } - } - }); - - return cubeColsInExpr; - } - @Override public String getStorageString(String alias) { return StringUtils.join(storageTables, ",") + " " + alias; @@ -371,4 +338,14 @@ public class CandidateFact implements CandidateTable, QueryAST { } return timePartDimensions; } + + public void updateFromString(CubeQueryContext query, Set<Dimension> queryDims, + Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + fromString = "%s"; // to update the storage alias later + if (query.isAutoJoinResolved()) { + fromString = + query.getAutoJoinCtx().getFromString(fromString, this, queryDims, dimsToQuery, + query, this); + } + } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java index d96b567..a57292c 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryConfUtil.java @@ -110,4 +110,9 @@ public final class CubeQueryConfUtil { public static final boolean DEFAULT_ENABLE_FLATTENING_FOR_BRIDGETABLES = false; public static final String BRIDGE_TABLE_FIELD_AGGREGATOR = "lens.cube.query.bridge.table.field.aggregator"; public static final String DEFAULT_BRIDGE_TABLE_FIELD_AGGREGATOR = "collect_set"; + public static final String DO_FLATTENING_OF_BRIDGE_TABLE_EARLY = + "lens.cube.query.flatten.bridge.tables.early"; + public static final boolean DEFAULT_DO_FLATTENING_OF_BRIDGE_TABLE_EARLY = false; + public static final String BRIDGE_TABLE_FIELD_ARRAY_FILTER = "lens.cube.query.bridge.table.field.array.filter"; + public static final String DEFAULT_BRIDGE_TABLE_FIELD_ARRAY_FILTER = "array_contains"; } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java index b8b6db9..c9b1475 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java @@ -170,7 +170,8 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { @Getter private Map<Dimension, PruneCauses<CubeDimensionTable>> dimPruningMsgs = new HashMap<Dimension, PruneCauses<CubeDimensionTable>>(); - + @Getter + private String fromString; public CubeQueryContext(ASTNode ast, QB qb, Configuration queryConf, HiveConf metastoreConf) throws LensException { this.ast = ast; @@ -628,6 +629,14 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } } + void updateFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery) throws LensException { + fromString = "%s"; // storage string is updated later + if (isAutoJoinResolved()) { + fromString = + getAutoJoinCtx().getFromString(fromString, fact, dimsToQuery.keySet(), dimsToQuery, this, this); + } + } + public String getSelectTree() { return HQLParser.getString(selectAST); } @@ -925,7 +934,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { log.info("facts:{}, dimsToQuery: {}", cfacts, dimsToQuery); // pick denorm tables for the picked fact and dimensions - Set<Dimension> denormTables = new HashSet<Dimension>(); + Set<Dimension> denormTables = new HashSet<>(); if (cfacts != null) { for (CandidateFact cfact : cfacts) { Set<Dimension> factDenormTables = deNormCtx.rewriteDenormctx(cfact, dimsToQuery, cfacts.size() > 1); @@ -946,7 +955,7 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { } if (autoJoinCtx != null) { // add optional dims from Join resolver - Set<Dimension> joiningTables = new HashSet<Dimension>(); + Set<Dimension> joiningTables = new HashSet<>(); if (cfacts != null && cfacts.size() > 1) { for (CandidateFact cfact : cfacts) { Set<Dimension> factJoiningTables = autoJoinCtx.pickOptionalTables(cfact, factDimMap.get(cfact), this); @@ -968,9 +977,15 @@ public class CubeQueryContext implements TrackQueriedColumns, QueryAST { cfact.updateASTs(this); } whereAST = MultiFactHQLContext.convertHavingToWhere(havingAST, this, cfacts, new DefaultAliasDecider()); + for (CandidateFact cFact : cfacts) { + cFact.updateFromString(this, factDimMap.get(cFact), dimsToQuery); + } } } + if (cfacts == null || cfacts.size() == 1) { + updateFromString(cfacts == null? null: cfacts.iterator().next(), dimsToQuery); + } hqlContext = createHQLContext(cfacts, dimsToQuery, factDimMap); return hqlContext.toHQL(); } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java index dadbfa0..80ceae4 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultAliasDecider.java @@ -25,7 +25,18 @@ public class DefaultAliasDecider implements AliasDecider { int counter = 0; private static final String ALIAS_PREFIX = "alias"; + final String aliasPrefix; + public DefaultAliasDecider(String alias) { + aliasPrefix = alias; + } + + DefaultAliasDecider() { + this(ALIAS_PREFIX); + } public String decideAlias(ASTNode node) { - return ALIAS_PREFIX + (counter++); + if (node == null) { + throw new NullPointerException("Node cannot be null"); + } + return aliasPrefix + (counter++); } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java index a403e36..0697e78 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DefaultQueryAST.java @@ -30,6 +30,7 @@ import lombok.Data; public class DefaultQueryAST implements QueryAST { private ASTNode selectAST, whereAST, groupByAST, havingAST, joinAST, orderByAST; private Integer limitValue; + private String fromString; public String getSelectTree() { return HQLParser.getString(selectAST); @@ -49,7 +50,6 @@ public class DefaultQueryAST implements QueryAST { return null; } - public String getHavingTree() { if (havingAST != null) { return HQLParser.getString(havingAST); @@ -69,6 +69,7 @@ public class DefaultQueryAST implements QueryAST { LensException { return new DefaultQueryAST(ast.getSelectAST(), fact.getStorageWhereClause(storageTable.substring(storageTable.indexOf(".") + 1)), - ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue()); + ast.getGroupByAST(), ast.getHavingAST(), ast.getJoinAST(), ast.getOrderByAST(), ast.getLimitValue(), + ast.getFromString()); } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java index 318c82a..7c14be7 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java @@ -40,27 +40,23 @@ abstract class DimHQLContext extends SimpleHQLContext { private final Set<Dimension> queriedDims; private String where; protected final CubeQueryContext query; + private final String astFromString; public CubeQueryContext getQuery() { return query; } DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery, Set<Dimension> queriedDims, QueryAST ast) throws LensException { - this(query, dimsToQuery, queriedDims, ast.getSelectTree(), ast.getWhereTree(), ast.getGroupByTree(), - ast.getOrderByTree(), ast.getHavingTree(), ast.getLimitValue()); - } - DimHQLContext(CubeQueryContext query, Map<Dimension, CandidateDim> dimsToQuery, - Set<Dimension> queriedDims, String select, String where, - String groupby, String orderby, String having, Integer limit) throws LensException { - super(select, groupby, orderby, having, limit); + super(ast.getSelectTree(), ast.getGroupByTree(), ast.getOrderByTree(), ast.getHavingTree(), ast.getLimitValue()); this.query = query; this.dimsToQuery = dimsToQuery; - this.where = where; + this.where = ast.getWhereTree(); this.queriedDims = queriedDims; + this.astFromString = ast.getFromString(); } protected void setMissingExpressions() throws LensException { - setFrom(getFromString()); + setFrom(String.format(astFromString, getFromTable())); setWhere(joinWithAnd( genWhereClauseWithDimPartitions(where), getQuery().getConf().getBoolean( CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, CubeQueryConfUtil.DEFAULT_REPLACE_TIMEDIM_WITH_PART_COL) @@ -71,21 +67,6 @@ abstract class DimHQLContext extends SimpleHQLContext { return null; } - - - protected String getFromString() throws LensException { - String fromString = getFromTable(); - if (query.isAutoJoinResolved()) { - fromString = - query.getAutoJoinCtx().getFromString(fromString, getQueriedFact(), getQueriedDimSet(), getDimsToQuery(), query); - } - return fromString; - } - - protected abstract Set<Dimension> getQueriedDimSet(); - - protected abstract CandidateFact getQueriedFact(); - protected abstract String getFromTable() throws LensException; public Map<Dimension, CandidateDim> getDimsToQuery() { http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java index d22287b..6f6572e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java @@ -55,14 +55,4 @@ class DimOnlyHQLContext extends DimHQLContext { return query.getQBFromString(null, getDimsToQuery()); } } - - @Override - protected Set<Dimension> getQueriedDimSet() { - return getDimsToQuery().keySet(); - } - - @Override - protected CandidateFact getQueriedFact() { - return null; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java index 8d6105f..0d00188 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java @@ -37,10 +37,10 @@ import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.parse.*; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.tree.Tree; import com.google.common.base.Optional; - import com.google.common.collect.Sets; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -60,6 +60,26 @@ public final class HQLParser { .getType() == HiveParser.TOK_TABLE_OR_COL && astNode.getChild(1).getType() == HiveParser.Identifier; } + public static boolean isPrimitiveBooleanExpression(ASTNode ast) { + return HQLParser.FILTER_OPERATORS.contains(ast.getType()); + } + + public static boolean isPrimitiveBooleanFunction(ASTNode ast) { + if (ast.getType() == TOK_FUNCTION) { + if (ast.getChild(0).getText().equals("in")) { + return true; + } + } + return false; + } + public static ASTNode getDotAST(String tableAlias, String fieldAlias) { + ASTNode child = new ASTNode(new CommonToken(DOT, ".")); + child.addChild(new ASTNode(new CommonToken(TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"))); + child.getChild(0).addChild(new ASTNode(new CommonToken(Identifier, tableAlias))); + child.addChild(new ASTNode(new CommonToken(Identifier, fieldAlias))); + return child; + } + public interface ASTNodeVisitor { void visit(TreeNode node) throws LensException; } @@ -147,8 +167,8 @@ public final class HQLParser { primitiveTypes.add(TOK_CHAR); PRIMITIVE_TYPES = Collections.unmodifiableSet(primitiveTypes); - FILTER_OPERATORS = Sets.newHashSet(KW_IN, GREATERTHAN, GREATERTHANOREQUALTO, LESSTHAN, LESSTHANOREQUALTO, EQUAL, - EQUAL_NS); + FILTER_OPERATORS = Sets.newHashSet(GREATERTHAN, GREATERTHANOREQUALTO, LESSTHAN, LESSTHANOREQUALTO, EQUAL, + EQUAL_NS, NOTEQUAL); } public static ASTNode parseHQL(String query, HiveConf conf) throws LensException { @@ -720,6 +740,29 @@ public final class HQLParser { return colname; } + public static Set<String> getColsInExpr(final String tableAlias, ASTNode expr) throws LensException { + final Set<String> colsInExpr = new HashSet<>(); + HQLParser.bft(expr, new ASTNodeVisitor() { + @Override + public void visit(TreeNode visited) { + ASTNode node = visited.getNode(); + ASTNode parent = null; + if (visited.getParent() != null) { + parent = visited.getParent().getNode(); + } + if (node.getToken().getType() == DOT) { + String alias = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).getText().toLowerCase(); + ASTNode colIdent = (ASTNode) node.getChild(1); + String column = colIdent.getText().toLowerCase(); + if (tableAlias.equalsIgnoreCase(alias)) { + colsInExpr.add(column); + } + } + } + }); + return colsInExpr; + } + public static boolean isAggregateAST(ASTNode node) { int exprTokenType = node.getType(); if (exprTokenType == HiveParser.TOK_FUNCTION || exprTokenType == HiveParser.TOK_FUNCTIONDI http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java index b861bb6..7dceee1 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java @@ -65,7 +65,8 @@ class JoinResolver implements ContextRewriter { if (!joinResolverDisabled && (!cubeql.getNonChainedDimensions().isEmpty() && cubeql.hasCubeInQuery()) || ((cubeql.getNonChainedDimensions().size() > 1) && !cubeql.hasCubeInQuery())) { - log.warn("Disabling auto join resolver as there are direct dimensions queried"); + log.warn("Disabling auto join resolver as there are direct dimensions queried {}", + cubeql.getNonChainedDimensions()); joinResolverDisabled = true; } if (joinResolverDisabled) { @@ -156,12 +157,16 @@ class JoinResolver implements ContextRewriter { CubeQueryConfUtil.DEFAULT_ENABLE_FLATTENING_FOR_BRIDGETABLES); String bridgeTableFieldAggr = cubeql.getConf().get(CubeQueryConfUtil.BRIDGE_TABLE_FIELD_AGGREGATOR, CubeQueryConfUtil.DEFAULT_BRIDGE_TABLE_FIELD_AGGREGATOR); + String bridgeTableFieldArrayFilter = cubeql.getConf().get(CubeQueryConfUtil.BRIDGE_TABLE_FIELD_ARRAY_FILTER, + CubeQueryConfUtil.DEFAULT_BRIDGE_TABLE_FIELD_ARRAY_FILTER); + boolean doFlatteningEarly = cubeql.getConf().getBoolean(CubeQueryConfUtil.DO_FLATTENING_OF_BRIDGE_TABLE_EARLY, + CubeQueryConfUtil.DEFAULT_DO_FLATTENING_OF_BRIDGE_TABLE_EARLY); Set<Dimension> requiredDimensions = Sets.newHashSet(cubeql.getDimensions()); requiredDimensions.removeAll(cubeql.getOptionalDimensions()); AutoJoinContext joinCtx = new AutoJoinContext(multipleJoinPaths, requiredDimensions, tableJoinTypeMap, target, cubeql.getConf().get(CubeQueryConfUtil.JOIN_TYPE_KEY), true, flattenBridgeTables, - bridgeTableFieldAggr); + bridgeTableFieldAggr, bridgeTableFieldArrayFilter, doFlatteningEarly); cubeql.setAutoJoinCtx(joinCtx); } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java index 9c18b7e..4d6ce9e 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java @@ -230,7 +230,4 @@ class MultiFactHQLContext extends SimpleHQLContext { return null; } - private static boolean isPrimitiveBooleanExpression(ASTNode ast) { - return HQLParser.FILTER_OPERATORS.contains(ast.getType()); - } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java index 31680ca..f064dcb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAST.java @@ -18,14 +18,14 @@ */ package org.apache.lens.cube.parse; - import org.apache.hadoop.hive.ql.parse.ASTNode; - -interface QueryAST { +public interface QueryAST { String getSelectTree(); + String getFromString(); + String getWhereTree(); String getHavingTree(); http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java index 63cb388..9ee94d3 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java @@ -192,14 +192,6 @@ public class SingleFactMultiStorageHQLContext extends UnionHQLContext { ast.getSelectAST().addChild(selectExprAST); } - private ASTNode getDotAST(String tableAlias, String fieldAlias) { - ASTNode child = new ASTNode(new CommonToken(DOT, ".")); - child.addChild(new ASTNode(new CommonToken(TOK_TABLE_OR_COL, "TOK_TABLE_OR_COL"))); - child.getChild(0).addChild(new ASTNode(new CommonToken(Identifier, tableAlias))); - child.addChild(new ASTNode(new CommonToken(Identifier, fieldAlias))); - return child; - } - private static ArrayList<HQLContextInterface> getUnionContexts(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query, QueryAST ast) throws LensException { http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java index b1a3b3f..dbc84ed 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactSingleStorageHQLContext.java @@ -33,7 +33,6 @@ import org.apache.lens.server.api.error.LensException; class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { private final CandidateFact fact; - private final Set<Dimension> queriedDimSet; private String storageAlias; SingleFactSingleStorageHQLContext(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery, @@ -47,7 +46,6 @@ class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { throws LensException { super(dimsToQuery, dimsQueried, query, ast); this.fact = fact; - this.queriedDimSet = dimsQueried; } SingleFactSingleStorageHQLContext(CandidateFact fact, String storageAlias, Map<Dimension, CandidateDim> dimsToQuery, @@ -72,14 +70,4 @@ class SingleFactSingleStorageHQLContext extends DimOnlyHQLContext { } } } - - @Override - protected CandidateFact getQueriedFact() { - return fact; - } - - @Override - public Set<Dimension> getQueriedDimSet() { - return queriedDimSet; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java index e14a898..8b24f70 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/AutoJoinContext.java @@ -70,12 +70,15 @@ public class AutoJoinContext { JoinClause minCostClause; private final boolean flattenBridgeTables; private final String bridgeTableFieldAggr; + private final String bridgeTableFieldArrayFilter; + private final boolean doFlatteningEarly; public AutoJoinContext(Map<Aliased<Dimension>, List<JoinPath>> allPaths, Set<Dimension> requiredDimensions, Map<AbstractCubeTable, JoinType> tableJoinTypeMap, AbstractCubeTable autoJoinTarget, String joinTypeCfg, boolean joinsResolved, - boolean flattenBridgeTables, String bridgeTableFieldAggr) { + boolean flattenBridgeTables, String bridgeTableFieldAggr, String bridgeTableFieldArrayFilter, + boolean doFlatteningEarly) { this.allPaths = allPaths; this.requiredDimensions = requiredDimensions; initJoinPathColumns(); @@ -85,6 +88,8 @@ public class AutoJoinContext { this.joinsResolved = joinsResolved; this.flattenBridgeTables = flattenBridgeTables; this.bridgeTableFieldAggr = bridgeTableFieldAggr; + this.bridgeTableFieldArrayFilter = bridgeTableFieldArrayFilter; + this.doFlatteningEarly = doFlatteningEarly; log.debug("All join paths:{}", allPaths); log.debug("Join path from columns:{}", joinPathFromColumns); log.debug("Join path to columns:{}", joinPathToColumns); @@ -165,22 +170,23 @@ public class AutoJoinContext { } public String getFromString(String fromTable, CandidateFact fact, Set<Dimension> qdims, - Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext cubeql) throws LensException { + Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext cubeql, QueryAST ast) throws LensException { String fromString = fromTable; log.info("All paths dump:{} Queried dims:{}", cubeql.getAutoJoinCtx().getAllPaths(), qdims); if (qdims == null || qdims.isEmpty()) { return fromString; } // Compute the merged join clause string for the min cost joinClause - String clause = getMergedJoinClause(cubeql, cubeql.getAutoJoinCtx().getJoinClause(fact), dimsToQuery); + String clause = getMergedJoinClause(cubeql, fact, ast, + cubeql.getAutoJoinCtx().getJoinClause(fact), dimsToQuery); fromString += clause; return fromString; } // Some refactoring needed to account for multiple join paths - public String getMergedJoinClause(CubeQueryContext cubeql, JoinClause joinClause, - Map<Dimension, CandidateDim> dimsToQuery) { + public String getMergedJoinClause(CubeQueryContext cubeql, CandidateFact fact, QueryAST ast, JoinClause joinClause, + Map<Dimension, CandidateDim> dimsToQuery) throws LensException { Set<String> clauses = new LinkedHashSet<>(); String joinTypeStr = ""; JoinType joinType = JoinType.INNER; @@ -192,12 +198,8 @@ public class AutoJoinContext { Iterator<JoinTree> iter = joinClause.getJoinTree().dft(); boolean hasBridgeTable = false; - boolean initedBridgeClauses = false; - StringBuilder bridgeSelectClause = new StringBuilder(); - StringBuilder bridgeFromClause = new StringBuilder(); - StringBuilder bridgeFilterClause = new StringBuilder(); - StringBuilder bridgeJoinClause = new StringBuilder(); - StringBuilder bridgeGroupbyClause = new StringBuilder(); + BridgeTableJoinContext bridgeTableJoinContext = new BridgeTableJoinContext(cubeql, fact, ast, bridgeTableFieldAggr, + bridgeTableFieldArrayFilter, doFlatteningEarly); while (iter.hasNext()) { JoinTree cur = iter.next(); @@ -252,90 +254,18 @@ public class AutoJoinContext { // if a bridge table is present in the path if (hasBridgeTable) { - // if any relation has bridge table, the clause becomes the following : - // join (" select " + joinkey + " aggr over fields from bridge table + from bridgeTable + [where user/storage - // filters] + groupby joinkey) on joincond" - // Or - // " join (select " + joinkey + " aggr over fields from table reached through bridge table + from bridge table - // join <next tables> on join condition + [and user/storage filters] + groupby joinkey) on joincond - if (!initedBridgeClauses) { - // we just found a bridge table in the path we need to initialize the clauses for subquery required for - // aggregating fields of bridge table - // initiliaze select clause with join key - bridgeSelectClause.append(" (select ").append(toAlias).append(".").append(rel.getToColumn()).append(" as ") - .append(rel.getToColumn()); - // group by join key - bridgeGroupbyClause.append(" group by ").append(toAlias).append(".").append(rel.getToColumn()); - // from clause with bridge table - bridgeFromClause.append(" from ").append(dimsToQuery.get(rel.getToTable()).getStorageString(toAlias)); - // we need to initialize filter clause with user filter clause or storage filter if applicable - if (StringUtils.isNotBlank(userFilter)) { - bridgeFilterClause.append(userFilter); - } - if (StringUtils.isNotBlank(storageFilter)) { - if (StringUtils.isNotBlank(bridgeFilterClause.toString())) { - bridgeFilterClause.append(" and "); - } - bridgeFilterClause.append(storageFilter); - } - // initialize final join clause - bridgeJoinClause.append(" on ").append(fromAlias).append(".") - .append(rel.getFromColumn()).append(" = ").append("%s") - .append(".").append(rel.getToColumn()); - initedBridgeClauses = true; - } else { - // if bridge clauses are already inited, this is a next table getting joined with bridge table - // we will append a simple join clause - bridgeFromClause.append(" join "); - bridgeFromClause.append(dimsToQuery.get(rel.getToTable()).getStorageString(toAlias)); - bridgeFromClause.append(" on ").append(fromAlias).append(".") - .append(rel.getFromColumn()).append(" = ").append(toAlias) - .append(".").append(rel.getToColumn()); - - if (StringUtils.isNotBlank(userFilter)) { - bridgeFromClause.append(" and ").append(userFilter); - } - if (StringUtils.isNotBlank(storageFilter)) { - bridgeFromClause.append(" and ").append(storageFilter); - } - } + bridgeTableJoinContext.updateBridgeClause(rel, fromAlias, toAlias, dimsToQuery.get(rel.getToTable()), + userFilter, storageFilter); if (cubeql.getTblAliasToColumns().get(toAlias) != null && !cubeql.getTblAliasToColumns().get(toAlias).isEmpty()) { // there are fields selected from this table after seeing bridge table in path // we should make subQuery for this selection - clause.append(joinTypeStr).append(" join "); - clause.append(bridgeSelectClause.toString()); - for (String col : cubeql.getTblAliasToColumns().get(toAlias)) { - clause.append(",").append(bridgeTableFieldAggr).append("(").append(toAlias) - .append(".").append(col) - .append(")") - .append(" as ").append(col); - } - String bridgeFrom = bridgeFromClause.toString(); - clause.append(bridgeFrom); - String bridgeFilter = bridgeFilterClause.toString(); - if (StringUtils.isNotBlank(bridgeFilter)) { - if (bridgeFrom.contains(" join ")) { - clause.append(" and "); - } else { - clause.append(" where"); - } - clause.append(bridgeFilter); - } - clause.append(bridgeGroupbyClause.toString()); - clause.append(") ").append(toAlias); - clause.append(String.format(bridgeJoinClause.toString(), toAlias)); - clauses.add(clause.toString()); + clauses.add(bridgeTableJoinContext.generateJoinClause(joinTypeStr, toAlias)); } if (cur.getSubtrees().isEmpty()) { // clear bridge flags and builders, as there are no more clauses in this tree. hasBridgeTable = false; - initedBridgeClauses = false; - bridgeSelectClause.setLength(0); - bridgeFromClause.setLength(0); - bridgeFilterClause.setLength(0); - bridgeJoinClause.setLength(0); - bridgeGroupbyClause.setLength(0); + bridgeTableJoinContext.resetContext(); } } else { // Simple join clause is : http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java new file mode 100644 index 0000000..cf74634 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/join/BridgeTableJoinContext.java @@ -0,0 +1,321 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse.join; + +import static org.apache.lens.cube.parse.HQLParser.*; + +import java.util.*; + +import org.apache.lens.cube.metadata.join.TableRelationship; +import org.apache.lens.cube.parse.*; +import org.apache.lens.server.api.error.LensException; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hive.ql.parse.ASTNode; +import org.apache.hadoop.hive.ql.parse.HiveParser; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +/** + * Join context related to Bridge tables + */ +@Slf4j +public class BridgeTableJoinContext { + private final String bridgeTableFieldAggr; + private final String arrayFilter; + private final CubeQueryContext cubeql; + private final CandidateFact fact; + private final QueryAST queryAST; + private final boolean doFlatteningEarly; + private boolean initedBridgeClauses = false; + private final StringBuilder bridgeSelectClause = new StringBuilder(); + private final StringBuilder bridgeFromClause = new StringBuilder(); + private final StringBuilder bridgeFilterClause = new StringBuilder(); + private final StringBuilder bridgeJoinClause = new StringBuilder(); + private final StringBuilder bridgeGroupbyClause = new StringBuilder(); + + public BridgeTableJoinContext(CubeQueryContext cubeql, CandidateFact fact, QueryAST queryAST, + String bridgeTableFieldAggr, String arrayFilter, boolean doFlatteningEarly) { + this.cubeql = cubeql; + this.queryAST = queryAST; + this.fact = fact; + this.bridgeTableFieldAggr = bridgeTableFieldAggr; + this.arrayFilter = arrayFilter; + this.doFlatteningEarly = doFlatteningEarly; + } + + public void resetContext() { + initedBridgeClauses = false; + bridgeSelectClause.setLength(0); + bridgeFromClause.setLength(0); + bridgeFilterClause.setLength(0); + bridgeJoinClause.setLength(0); + bridgeGroupbyClause.setLength(0); + } + + public void initBridgeClauses(TableRelationship rel, String fromAlias, String toAlias, CandidateDim toTable, String + userFilter, + String storageFilter) { + // we just found a bridge table in the path we need to initialize the clauses for subquery required for + // aggregating fields of bridge table + // initialize select clause with join key + bridgeSelectClause.append(" (select ").append(toAlias).append(".").append(rel.getToColumn()).append(" as ") + .append(rel.getToColumn()); + // group by join key + bridgeGroupbyClause.append(" group by ").append(toAlias).append(".").append(rel.getToColumn()); + // from clause with bridge table + bridgeFromClause.append(" from ").append(toTable.getStorageString(toAlias)); + // we need to initialize filter clause with user filter clause or storage filter if applicable + if (StringUtils.isNotBlank(userFilter)) { + bridgeFilterClause.append(userFilter); + } + if (StringUtils.isNotBlank(storageFilter)) { + if (StringUtils.isNotBlank(bridgeFilterClause.toString())) { + bridgeFilterClause.append(" and "); + } + bridgeFilterClause.append(storageFilter); + } + // initialize final join clause + bridgeJoinClause.append(" on ").append(fromAlias).append(".") + .append(rel.getFromColumn()).append(" = ").append("%s") + .append(".").append(rel.getToColumn()); + initedBridgeClauses = true; + } + + // if any relation has bridge table, the clause becomes the following : + // join (" select " + joinkey + " aggr over fields from bridge table + from bridgeTable + [where user/storage + // filters] + groupby joinkey) on joincond" + // Or + // " join (select " + joinkey + " aggr over fields from table reached through bridge table + from bridge table + // join <next tables> on join condition + [and user/storage filters] + groupby joinkey) on joincond + public void updateBridgeClause(TableRelationship rel, String fromAlias, String toAlias, CandidateDim toTable, + String userFilter, String storageFilter) { + if (!initedBridgeClauses) { + initBridgeClauses(rel, fromAlias, toAlias, toTable, userFilter, storageFilter); + } else { + addAnotherJoinClause(rel, fromAlias, toAlias, toTable, userFilter, storageFilter); + } + } + + public void addAnotherJoinClause(TableRelationship rel, String fromAlias, String toAlias, CandidateDim toTable, + String userFilter, String storageFilter) { + // if bridge clauses are already inited, this is a next table getting joined with bridge table + // we will append a simple join clause + bridgeFromClause.append(" join "); + bridgeFromClause.append(toTable.getStorageString(toAlias)); + bridgeFromClause.append(" on ").append(fromAlias).append(".") + .append(rel.getFromColumn()).append(" = ").append(toAlias) + .append(".").append(rel.getToColumn()); + + if (StringUtils.isNotBlank(userFilter)) { + bridgeFromClause.append(" and ").append(userFilter); + } + if (StringUtils.isNotBlank(storageFilter)) { + bridgeFromClause.append(" and ").append(storageFilter); + } + } + + public String generateJoinClause(String joinTypeStr, String toAlias) throws LensException { + StringBuilder clause = new StringBuilder(joinTypeStr); + clause.append(" join "); + clause.append(bridgeSelectClause.toString()); + // iterate over all select expressions and add them for select clause if do_flattening_early is disabled + if (!doFlatteningEarly) { + BridgeTableSelectCtx selectCtx = new BridgeTableSelectCtx(bridgeTableFieldAggr, arrayFilter, toAlias); + selectCtx.processSelectAST(queryAST.getSelectAST()); + selectCtx.processWhereClauses(fact); + selectCtx.processGroupbyAST(queryAST.getGroupByAST()); + selectCtx.processOrderbyAST(queryAST.getOrderByAST()); + clause.append(",").append(StringUtils.join(selectCtx.getSelectedBridgeExprs(), ",")); + } else { + for (String col : cubeql.getTblAliasToColumns().get(toAlias)) { + clause.append(",").append(bridgeTableFieldAggr).append("(").append(toAlias) + .append(".").append(col) + .append(")") + .append(" as ").append(col); + } + } + String bridgeFrom = bridgeFromClause.toString(); + clause.append(bridgeFrom); + String bridgeFilter = bridgeFilterClause.toString(); + if (StringUtils.isNotBlank(bridgeFilter)) { + if (bridgeFrom.contains(" join ")) { + clause.append(" and "); + } else { + clause.append(" where "); + } + clause.append(bridgeFilter); + } + clause.append(bridgeGroupbyClause.toString()); + clause.append(") ").append(toAlias); + clause.append(String.format(bridgeJoinClause.toString(), toAlias)); + return clause.toString(); + } + + @Data + static class BridgeTableSelectCtx { + private final HashMap<HashableASTNode, ASTNode> exprToDotAST = new HashMap<>(); + private final List<String> selectedBridgeExprs = new ArrayList<>(); + private final AliasDecider aliasDecider = new DefaultAliasDecider("balias"); + private final String bridgeTableFieldAggr; + private final String arrayFilter; + private final String tableAlias; + + List<String> processSelectAST(ASTNode selectAST) + throws LensException { + // iterate over children + for (int i = 0; i < selectAST.getChildCount(); i++) { + ASTNode selectExprNode = (ASTNode) selectAST.getChild(i); + ASTNode child = (ASTNode) selectExprNode.getChild(0); + if (hasBridgeCol(child, tableAlias)) { + selectExprNode.setChild(0, getDotASTForExprAST(child)); + } + } + return selectedBridgeExprs; + } + + private ASTNode getDotASTForExprAST(ASTNode child) { + HashableASTNode hashAST = new HashableASTNode(child); + if (!exprToDotAST.containsKey(hashAST)) { + // add selected expression to get selected from bridge table, with a generated alias + String colAlias = aliasDecider.decideAlias(child); + selectedBridgeExprs.add(bridgeTableFieldAggr + "(" + HQLParser.getString(child) + ") as " + colAlias); + + // replace bridge expression with tableAlias.colAlias. + ASTNode dot = HQLParser.getDotAST(tableAlias, colAlias); + exprToDotAST.put(hashAST, dot); + } + return exprToDotAST.get(hashAST); + } + + // process groupby + void processGroupbyAST(ASTNode ast) + throws LensException { + if (ast == null) { + return; + } + // iterate over children + for (int i = 0; i < ast.getChildCount(); i++) { + ASTNode exprNode = (ASTNode) ast.getChild(i); + if (hasBridgeCol(exprNode, tableAlias)) { + ast.setChild(i, getDotASTForExprAST(exprNode)); + } + } + } + + // process orderby + void processOrderbyAST(ASTNode ast) + throws LensException { + if (ast == null) { + return; + } + // iterate over children + for (int i = 0; i < ast.getChildCount(); i++) { + ASTNode exprNode = (ASTNode) ast.getChild(i); + ASTNode child = (ASTNode) exprNode.getChild(0); + if (hasBridgeCol(child, tableAlias)) { + exprNode.setChild(0, getDotASTForExprAST(child)); + } + } + } + + void processWhereClauses(CandidateFact fact) throws LensException { + + for (Map.Entry<String, ASTNode> whereEntry : fact.getStorgeWhereClauseMap().entrySet()) { + ASTNode whereAST = whereEntry.getValue(); + processWhereAST(whereAST, null, 0); + } + } + + void processWhereAST(ASTNode ast, ASTNode parent, int childPos) + throws LensException { + if (ast == null) { + return; + } + ASTNode child; + int replaceIndex = -1; + if (isPrimitiveBooleanExpression(ast)) { + replaceIndex = 0; + } else if (isPrimitiveBooleanFunction(ast)) { + replaceIndex = 1; + } + if (replaceIndex != -1) { + child = (ASTNode) ast.getChild(replaceIndex); + if (hasBridgeCol(child, tableAlias)) { + ast.setChild(replaceIndex, getDotASTForExprAST(child)); + parent.setChild(childPos, replaceDirectFiltersWithArrayFilter(ast, arrayFilter)); + } + } + // recurse down + for (int i = 0; i < ast.getChildCount(); i++) { + processWhereAST((ASTNode) ast.getChild(i), ast, i); + } + } + } + /** + * Update =, != and IN clause filters to arrayFilter. arrayFilter will have signature arrayFilter(col, value) + * + * @param ast AST for simple filter + * @param arrayFilter arrayFilter function + * @return ASTNode with converted filter + * + * @throws LensException + */ + static ASTNode replaceDirectFiltersWithArrayFilter(ASTNode ast, String arrayFilter) + throws LensException { + StringBuilder filterBuilder = new StringBuilder(); + if ((ast.getType() == HiveParser.EQUAL || ast.getType() == HiveParser.NOTEQUAL)) { + String colStr = getString((ASTNode) ast.getChild(0)); + if (ast.getType() == HiveParser.NOTEQUAL) { + filterBuilder.append(" NOT "); + } + filterBuilder.append(arrayFilter); + filterBuilder.append("("); + filterBuilder.append(colStr).append(","); + filterBuilder.append(getString((ASTNode)ast.getChild(1))); + filterBuilder.append(")"); + } else if (ast.getType() == HiveParser.TOK_FUNCTION) { + // This is IN clause as function + String colStr = getString((ASTNode) ast.getChild(1)); + filterBuilder.append("("); + for (int i = 2; i < ast.getChildCount(); i++) { + filterBuilder.append(arrayFilter); + filterBuilder.append("("); + filterBuilder.append(colStr).append(","); + filterBuilder.append(ast.getChild(i).getText()); + filterBuilder.append(")"); + if (i + 1 != ast.getChildCount()) { + filterBuilder.append(" OR "); + } + } + filterBuilder.append(")"); + } + String finalFilter = filterBuilder.toString(); + if (StringUtils.isNotBlank(finalFilter)) { + return HQLParser.parseExpr(finalFilter); + } + return ast; + } + + static boolean hasBridgeCol(ASTNode astNode, String tableAlias) throws LensException { + Set<String> bridgeCols = HQLParser.getColsInExpr(tableAlias, astNode); + return !bridgeCols.isEmpty(); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/main/resources/olap-query-conf.xml ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/resources/olap-query-conf.xml b/lens-cube/src/main/resources/olap-query-conf.xml index 1436cd1..6e9933e 100644 --- a/lens-cube/src/main/resources/olap-query-conf.xml +++ b/lens-cube/src/main/resources/olap-query-conf.xml @@ -211,4 +211,22 @@ be passed for each driver, as the function could be different for each driver. </description> </property> + <property> + <name>lens.cube.query.bridge.table.field.array.filter</name> + <value>array_contains</value> + <description>The array filter function to be used for filter on fields from bridge tables. Would be used + only when flattening is enabled and not flattening early. The value can be passed for each query. + During typical deployments it would be passed for each driver, as the function could be different for each + driver. Signature of the filter getting passed here should be very similar to array_contains. i.e. + array_contains(array, value) to return true if array contains value. + </description> + </property> + <property> + <name>lens.cube.query.flatten.bridge.tables.early</name> + <value>false</value> + <description>Flag specifies if fields selected have to be flattened earlier to applying any expression on top + of them, For ex: sub_str(bridge_field) is the expression, if the property is set to true, sub_str will be + applied on aggregated field. If set to false, aggregate will applied on top of the expression i.e. sub_str. + </description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/dba885ca/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java index 42decc6..379e873 100644 --- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java +++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java @@ -1069,6 +1069,16 @@ public class CubeTestSetup { }); } }); + joinChains.add(new JoinChain("userchain", "user-chain", "user chain") { + { + addPath(new ArrayList<TableReference>() { + { + add(new TableReference("basecube", "userid")); + add(new TableReference("userdim", "id")); + } + }); + } + }); joinChains.add(new JoinChain("userSports", "user-sports", "user sports") { { addPath(new ArrayList<TableReference>() { @@ -1131,7 +1141,14 @@ public class CubeTestSetup { Set<ExprColumn> baseExprs = new HashSet<>(exprs); baseExprs.add(new ExprColumn(new FieldSchema("substrsprorts", "String", "substr of sports"), "substr sports", "substr(sports, 10)")); - + baseExprs.add(new ExprColumn(new FieldSchema("xsports_abbr", "array<string>", ""), + "xuser sports", "substr(xsports, 3)")); + baseExprs.add(new ExprColumn(new FieldSchema("ysports_abbr", "array<string>", ""), + "yuser sports", "substr(ysports, 3)")); + baseExprs.add(new ExprColumn(new FieldSchema("sports_abbr", "array<string>", ""), + "user sports", "substr(sports, 3)")); + baseExprs.add(new ExprColumn(new FieldSchema("sportids_abbr", "array<string>", ""), + "user sports", "case when sportids == 1 then 'CKT' when sportids == 2 then 'FTB' else 'NON' end")); client.createCube(BASE_CUBE_NAME, cubeMeasures2, cubeDimensions2, baseExprs, joinChains, cubeProperties); Map<String, String> derivedProperties = new HashMap<>();
