Repository: atlas Updated Branches: refs/heads/master faeecf101 -> 5614bf0d6
ATLAS-2229: DSL implementation using ANTLR #3 - Select, GroupBy, OrderBy Signed-off-by: Madhan Neethiraj <mad...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5614bf0d Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5614bf0d Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5614bf0d Branch: refs/heads/master Commit: 5614bf0d6da416455c366cf44bbaace72b0893b2 Parents: faeecf1 Author: Ashutosh Mestry <ames...@hortonworks.com> Authored: Wed Dec 20 13:37:23 2017 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Wed Dec 20 23:48:55 2017 -0800 ---------------------------------------------------------------------- .../atlas/discovery/EntityDiscoveryService.java | 6 +- .../java/org/apache/atlas/query/DSLVisitor.java | 48 ++- .../apache/atlas/query/IdentifierHelper.java | 8 + .../org/apache/atlas/query/QueryProcessor.java | 338 +++++++++++++------ .../apache/atlas/query/QueryProcessorTest.java | 80 +++-- 5 files changed, 331 insertions(+), 149 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/5614bf0d/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index e742496..ece1516 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -720,8 +720,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private AttributeSearchResult toAttributesResult(List results, GremlinQuery query) { AttributeSearchResult ret = new AttributeSearchResult(); - List<String> names = extractNames(results); - List<List<Object>> values = extractValues(results); +// List<String> names = extractNames(results); +// List<List<Object>> values = extractValues(results); + List<String> names = (List<String>) results.get(0); + List<List<Object>> values = extractValues(results.subList(1, results.size())); ret.setName(names); ret.setValues(values); http://git-wip-us.apache.org/repos/asf/atlas/blob/5614bf0d/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java index 064f6dd..b597a0d 100644 --- a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java +++ b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java @@ -21,13 +21,12 @@ package org.apache.atlas.query; import org.apache.atlas.query.antlr4.AtlasDSLParser.*; import org.apache.atlas.query.antlr4.AtlasDSLParserBaseVisitor; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; +import java.util.Objects; public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class); @@ -68,7 +67,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { } queryProcessor.addLimit(ctx.limitClause().NUMBER().toString(), - (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); + (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); return super.visitLimitOffset(ctx); } @@ -78,17 +77,46 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { LOG.debug("=> DSLVisitor.visitSelectExpr({})", ctx); } + // Select can have only attributes, aliased attributes or aggregate functions + + // Groupby attr also represent select expr, no processing is needed in that case + // visit groupBy would handle the select expr appropriately if (!(ctx.getParent() instanceof GroupByExpressionContext)) { - List<Pair<String, String>> items = new ArrayList<>(); - for (int i = 0; i < ctx.selectExpression().size(); i++) { - String idf = ctx.selectExpression(i).expr().getText(); - String alias = (ctx.selectExpression(i).K_AS() != null) ? - ctx.selectExpression(i).identifier().getText() : ""; + String[] items = new String[ctx.selectExpression().size()]; + String[] labels = new String[ctx.selectExpression().size()]; + + QueryProcessor.SelectExprMetadata selectExprMetadata = new QueryProcessor.SelectExprMetadata(); - items.add(new MutablePair<String, String>(idf, alias)); + for (int i = 0; i < ctx.selectExpression().size(); i++) { + SelectExpressionContext selectExpression = ctx.selectExpression(i); + CountClauseContext countClause = selectExpression.expr().compE().countClause(); + SumClauseContext sumClause = selectExpression.expr().compE().sumClause(); + MinClauseContext minClause = selectExpression.expr().compE().minClause(); + MaxClauseContext maxClause = selectExpression.expr().compE().maxClause(); + IdentifierContext identifier = selectExpression.identifier(); + + labels[i] = identifier != null ? identifier.getText() : selectExpression.getText(); + + if (Objects.nonNull(countClause)) { + items[i] = "count"; + selectExprMetadata.setCountIdx(i); + } else if (Objects.nonNull(sumClause)) { + items[i] = sumClause.expr().getText(); + selectExprMetadata.setSumIdx(i); + } else if (Objects.nonNull(minClause)) { + items[i] = minClause.expr().getText(); + selectExprMetadata.setMinIdx(i); + } else if (Objects.nonNull(maxClause)) { + items[i] = maxClause.expr().getText(); + selectExprMetadata.setMaxIdx(i); + } else { + items[i] = selectExpression.expr().getText(); + } } - queryProcessor.addSelect(items); + selectExprMetadata.setItems(items); + selectExprMetadata.setLabels(labels); + queryProcessor.addSelect(selectExprMetadata); } return super.visitSelectExpr(ctx); } http://git-wip-us.apache.org/repos/asf/atlas/blob/5614bf0d/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java index 6bbadc4..0c7e2ed 100644 --- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java +++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java @@ -189,6 +189,14 @@ public class IdentifierHelper { return isPrimitive; } + public boolean isAttribute() { + return isAttribute; + } + + public String getAttributeName() { + return attributeName; + } + public String getEdgeLabel() { return edgeLabel; } http://git-wip-us.apache.org/repos/asf/atlas/blob/5614bf0d/repository/src/main/java/org/apache/atlas/query/QueryProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/QueryProcessor.java b/repository/src/main/java/org/apache/atlas/query/QueryProcessor.java index 599f370..76a10e3 100644 --- a/repository/src/main/java/org/apache/atlas/query/QueryProcessor.java +++ b/repository/src/main/java/org/apache/atlas/query/QueryProcessor.java @@ -21,20 +21,29 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.query.Expressions.Expression; -import org.apache.atlas.type.*; +import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasBuiltInTypes; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.stream.Stream; public class QueryProcessor { private static final Logger LOG = LoggerFactory.getLogger(QueryProcessor.class); @@ -42,13 +51,18 @@ public class QueryProcessor { private final int DEFAULT_QUERY_RESULT_LIMIT = 25; private final int DEFAULT_QUERY_RESULT_OFFSET = 0; + private final List<String> errorList = new ArrayList<>(); + private final GremlinClauseList queryClauses = new GremlinClauseList(); + + private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT; + private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET; + private boolean hasSelect = false; + private boolean isSelectNoop = false; + private boolean hasGrpBy = false; + + private final org.apache.atlas.query.Lookup lookup; private final boolean isNestedQuery; - private final List<String> errorList = new ArrayList<>(); - private final GremlinClauseList queryClauses = new GremlinClauseList(); - private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT; - private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET; private int currentStep; - private final org.apache.atlas.query.Lookup lookup; private Context context; @Inject @@ -83,15 +97,6 @@ public class QueryProcessor { return expression.isReady(); } - private void init() { - if (!isNestedQuery) { - add(GremlinClause.G); - add(GremlinClause.V); - } else { - add(GremlinClause.NESTED_START); - } - } - public void addFrom(String typeName) { if (LOG.isDebugEnabled()) { LOG.debug("addFrom(typeName={})", typeName); @@ -117,17 +122,6 @@ public class QueryProcessor { } } - private void introduceType(IdentifierHelper.Advice ia) { - if (!ia.isPrimitive() && ia.getIntroduceType()) { - add(GremlinClause.OUT, ia.getEdgeLabel()); - context.registerActive(ia.getTypeName()); - } - } - - private IdentifierHelper.Advice getAdvice(String actualTypeName) { - return IdentifierHelper.create(context, lookup, actualTypeName); - } - public void addFromProperty(String typeName, String attribute) { if (LOG.isDebugEnabled()) { LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute); @@ -202,51 +196,60 @@ public class QueryProcessor { queryClauses.add(GremlinClause.OR, StringUtils.join(clauses, ',')); } - public void addSelect(List<Pair<String, String>> items) { + public void addSelect(SelectExprMetadata selectExprMetadata) { + String[] items = selectExprMetadata.getItems(); + String[] labels = selectExprMetadata.getLabels(); if (LOG.isDebugEnabled()) { - LOG.debug("addSelect(items.length={})", items != null ? items.size() : -1); + LOG.debug("addSelect(items.length={})", items != null ? items.length : 0); } - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < items.size(); i++) { - IdentifierHelper.Advice ia = getAdvice(items.get(i).getLeft()); - if(StringUtils.isNotEmpty(items.get(i).getRight())) { - context.aliasMap.put(items.get(i).getRight(), ia.getQualifiedName()); - } + if (items != null) { + for (int i = 0; i < items.length; i++) { + IdentifierHelper.Advice ia = getAdvice(items[i]); - if(!ia.isPrimitive() && ia.getIntroduceType()) { - add(GremlinClause.OUT, ia.getEdgeLabel()); - add(GremlinClause.AS, getCurrentStep()); - addSelectClause(getCurrentStep()); - incrementCurrentStep(); - } else { - sb.append(quoted(ia.getQualifiedName())); - } + if(!labels[i].equals(items[i])) { + context.aliasMap.put(labels[i], ia.getQualifiedName()); + } - if (i != items.size() - 1) { - sb.append(","); + if (i == selectExprMetadata.getCountIdx()) { + items[i] = GremlinClause.INLINE_COUNT.get(); + } else if (i == selectExprMetadata.getMinIdx()) { + items[i] = GremlinClause.INLINE_MIN.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else if (i == selectExprMetadata.getMaxIdx()) { + items[i] = GremlinClause.INLINE_MAX.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else if (i == selectExprMetadata.getSumIdx()) { + items[i] = GremlinClause.INLINE_SUM.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else { + if (!ia.isPrimitive() && ia.getIntroduceType()) { + add(GremlinClause.OUT, ia.getEdgeLabel()); + context.registerActive(ia.getTypeName()); + + int dotIdx = ia.get().indexOf("."); + if (dotIdx != -1) { + IdentifierHelper.Advice iax = getAdvice(ia.get()); + items[i] = GremlinClause.INLINE_GET_PROPERTY.get(iax.getQualifiedName()); + } else { + isSelectNoop = true; + } + } else { + items[i] = GremlinClause.INLINE_GET_PROPERTY.get(ia.getQualifiedName()); + } + } } - } - if (!StringUtils.isEmpty(sb.toString())) { - addValueMapClause(sb.toString()); - } - } + // If GroupBy clause exists then the query spits out a List<Map<String, List<AtlasVertex>>> otherwise the query returns List<AtlasVertex> + // Different transformations are needed for DSLs with groupby and w/o groupby + GremlinClause transformationFn; + if (isSelectNoop) { + transformationFn = GremlinClause.SELECT_EXPR_NOOP_FN; + } else { + transformationFn = hasGrpBy ? GremlinClause.SELECT_WITH_GRPBY_HELPER_FN : GremlinClause.SELECT_EXPR_HELPER_FN; + } + queryClauses.add(0, transformationFn, getJoinedQuotedStr(labels), String.join(",", items)); + queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL); - private void addSelectClause(String s) { - if (LOG.isDebugEnabled()) { - LOG.debug("addSelectClause(s={})", s); + hasSelect = true; } - - add(GremlinClause.SELECT, s); - } - - private String getCurrentStep() { - return String.format("s%d", currentStep); - } - - private void incrementCurrentStep() { - currentStep++; } public QueryProcessor createNestedProcessor() { @@ -255,14 +258,6 @@ public class QueryProcessor { return qp; } - private void addValueMapClause(String s) { - if (LOG.isDebugEnabled()) { - LOG.debug("addValueMapClause(s={})", s); - } - - add(GremlinClause.VALUE_MAP, s); - } - public void addFromAlias(String typeName, String alias) { if (LOG.isDebugEnabled()) { LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias); @@ -281,19 +276,6 @@ public class QueryProcessor { add(GremlinClause.AS, stepName); } - private void add(GremlinClause clause, String... args) { - queryClauses.add(new GremlinClauseValue(clause, clause.get(args))); - } - - private void addRangeClause(String startIndex, String endIndex) { - if (LOG.isDebugEnabled()) { - LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex); - } - - add(GremlinClause.RANGE, startIndex, startIndex, endIndex); - } - - public void addGroupBy(String item) { if (LOG.isDebugEnabled()) { LOG.debug("addGroupBy(item={})", item); @@ -301,16 +283,7 @@ public class QueryProcessor { add(GremlinClause.GROUP); addByClause(item, false); - } - - private void addByClause(String name, boolean descr) { - if (LOG.isDebugEnabled()) { - LOG.debug("addByClause(name={})", name, descr); - } - - IdentifierHelper.Advice ia = getAdvice(name); - add((!descr) ? GremlinClause.BY : GremlinClause.BY_DESC, - ia.getQualifiedName()); + hasGrpBy = true; } public void addLimit(String limit, String offset) { @@ -335,27 +308,38 @@ public class QueryProcessor { addLimit(Integer.toString(providedLimit), Integer.toString(providedOffset)); } + updatePosition(GremlinClause.LIMIT); add(GremlinClause.TO_LIST); + updatePosition(GremlinClause.INLINE_TRANSFORM_CALL); } public String getText() { + String ret; String[] items = new String[queryClauses.size()]; - for (int i = 0; i < queryClauses.size(); i++) { + int startIdx = hasSelect ? 1 : 0; + int endIdx = hasSelect ? queryClauses.size() - 1 : queryClauses.size(); + for (int i = startIdx; i < endIdx; i++) { items[i] = queryClauses.getValue(i); } - String ret = StringUtils.join(items, "."); + if (hasSelect) { + String body = StringUtils.join(Stream.of(items).filter(Objects::nonNull).toArray(), "."); + String inlineFn = queryClauses.getValue(queryClauses.size() - 1); + String funCall = String.format(inlineFn, body); + ret = queryClauses.getValue(0) + funCall; + } else { + ret = String.join(".", items); + } if (LOG.isDebugEnabled()) { LOG.debug("getText() => {}", ret); } - return ret; } public boolean hasSelect() { - return (queryClauses.hasClause(GremlinClause.VALUE_MAP) != -1); + return hasSelect; } public void addOrderBy(String name, boolean isDesc) { @@ -365,19 +349,73 @@ public class QueryProcessor { add(GremlinClause.ORDER); addByClause(name, isDesc); - updateSelectClausePosition(); } - private void updateSelectClausePosition() { - int selectClauseIndex = queryClauses.hasClause(GremlinClause.VALUE_MAP); - if(-1 == selectClauseIndex) { + private void updatePosition(GremlinClause clause) { + int index = queryClauses.hasClause(clause); + if(-1 == index) { return; } - GremlinClauseValue gcv = queryClauses.remove(selectClauseIndex); + GremlinClauseValue gcv = queryClauses.remove(index); queryClauses.add(gcv); } + private void init() { + if (!isNestedQuery) { + add(GremlinClause.G); + add(GremlinClause.V); + } else { + add(GremlinClause.NESTED_START); + } + } + + private void introduceType(IdentifierHelper.Advice ia) { + if (!ia.isPrimitive() && ia.getIntroduceType()) { + add(GremlinClause.OUT, ia.getEdgeLabel()); + context.registerActive(ia.getTypeName()); + } + } + + private IdentifierHelper.Advice getAdvice(String actualTypeName) { + return IdentifierHelper.create(context, lookup, actualTypeName); + } + + private String getJoinedQuotedStr(String[] elements) { + StringJoiner joiner = new StringJoiner(","); + Arrays.stream(elements).map(x -> "'" + x + "'").forEach(joiner::add); + return joiner.toString(); + } + + private void add(GremlinClause clause, String... args) { + queryClauses.add(new GremlinClauseValue(clause, clause.get(args))); + } + + private void add(int idx, GremlinClause clause, String... args) { + queryClauses.add(idx, new GremlinClauseValue(clause, clause.get(args))); + } + + private void addRangeClause(String startIndex, String endIndex) { + if (LOG.isDebugEnabled()) { + LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex); + } + + if (hasSelect) { + add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex); + } else { + add(GremlinClause.RANGE, startIndex, startIndex, endIndex); + } + } + + private void addByClause(String name, boolean descr) { + if (LOG.isDebugEnabled()) { + LOG.debug("addByClause(name={})", name, descr); + } + + IdentifierHelper.Advice ia = getAdvice(name); + add((!descr) ? GremlinClause.BY : GremlinClause.BY_DESC, ia.getQualifiedName()); + } + private enum GremlinClause { AS("as('%s')"), BY("by('%s')"), @@ -407,6 +445,17 @@ public class QueryProcessor { TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"), TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"), TRAIT("has('__traitNames', within('%s'))"), + SELECT_EXPR_NOOP_FN("def f(r){ r }; "), + SELECT_EXPR_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({[%s]})).unique(); }; "), + SELECT_WITH_GRPBY_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({it.values()}).flatten().collect({[%s]})).unique(); }; "), + INLINE_COUNT("r.size()"), + INLINE_SUM("r.sum({it.value('%s')}).value('%s')"), + INLINE_MAX("r.max({it.value('%s')}).value('%s')"), + INLINE_MIN("r.min({it.value('%s')}).value('%s')"), + INLINE_GET_PROPERTY("it.value('%s')"), + INLINE_OUT_VERTEX("it.out('%s')"), + INLINE_OUT_VERTEX_VALUE("it.out('%s').value('%s')"), // This might require more closure introduction :( + INLINE_TRANSFORM_CALL("f(%s)"), V("V()"), VALUE_MAP("valueMap(%s)"); @@ -452,14 +501,30 @@ public class QueryProcessor { list.add(g); } + public void add(int idx, GremlinClauseValue g) { + list.add(idx, g); + } + public void add(GremlinClauseValue g, AtlasEntityType t) { add(g); } + public void add(int idx, GremlinClauseValue g, AtlasEntityType t) { + add(idx, g); + } + public void add(GremlinClause clause, String... args) { list.add(new GremlinClauseValue(clause, clause.get(args))); } + public void add(int i, GremlinClause clause, String... args) { + list.add(i, new GremlinClauseValue(clause, clause.get(args))); + } + + public GremlinClauseValue getAt(int i) { + return list.get(i); + } + public String getValue(int i) { return list.get(i).value; } @@ -500,8 +565,8 @@ public class QueryProcessor { static class Context { private final List<String> errorList; org.apache.atlas.query.Lookup lookup; - private AtlasType activeType; Map<String, String> aliasMap = new HashMap<>(); + private AtlasType activeType; public Context(List<String> errorList, org.apache.atlas.query.Lookup lookup) { this.lookup = lookup; @@ -682,16 +747,71 @@ public class QueryProcessor { @Override public boolean isDate(Context context, String attributeName) { AtlasEntityType et = context.getActiveEntityType(); - if(et == null) { + if (et == null) { return false; } AtlasType attr = et.getAttributeType(attributeName); - if(attr == null) { - return false; - } + return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE); + + } + } + + static class SelectExprMetadata { + private String[] items; + private String[] labels; + + private int countIdx = -1; + private int sumIdx = -1; + private int maxIdx = -1; + private int minIdx = -1; + + public String[] getItems() { + return items; + } + + public int getCountIdx() { + return countIdx; + } + + public void setCountIdx(final int countIdx) { + this.countIdx = countIdx; + } + + public int getSumIdx() { + return sumIdx; + } + + public void setSumIdx(final int sumIdx) { + this.sumIdx = sumIdx; + } + + public int getMaxIdx() { + return maxIdx; + } + + public void setMaxIdx(final int maxIdx) { + this.maxIdx = maxIdx; + } + + public int getMinIdx() { + return minIdx; + } + + public void setMinIdx(final int minIdx) { + this.minIdx = minIdx; + } + + public String[] getLabels() { + return labels; + } + + public void setItems(final String[] items) { + this.items = items; + } - return attr.getTypeName().equals("date"); + public void setLabels(final String[] labels) { + this.labels = labels; } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/5614bf0d/repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java b/repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java index 4c6d314..b873141 100644 --- a/repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java +++ b/repository/src/test/java/org/apache/atlas/query/QueryProcessorTest.java @@ -32,9 +32,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -72,8 +70,8 @@ public class QueryProcessorTest { @Test public void DBHasName() { String expected = "g.V().has('__typeName', 'DB').has('DB.name').limit(25).toList()"; - // verify("DB has name", expected); - verify("DB where DB has name", expected); + verify("DB has name", expected); + verify("DB where DB has name", expected); } @Test @@ -83,15 +81,19 @@ public class QueryProcessorTest { @Test public void DBasDSelect() { - String expected = "g.V().has('__typeName', 'DB').as('d').valueMap('DB.name','DB.owner')"; - verify("DB as d select d.name, d.owner", expected + ".limit(25).toList()"); - verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList()"); + String expected = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').as('d')"; + verify("DB as d select d.name, d.owner", expected + ".limit(25).toList())"); + verify("DB as d select d.name, d.owner limit 10", expected + ".limit(10).toList())"); } @Test public void tableSelectColumns() { - verify("Table select columns limit 10", "g.V().has('__typeName', 'Table').out('__Table.columns').as('s0').select('s0').limit(10).toList()"); - verify("Table select db.name", "g.V().has('__typeName', 'Table').out('__DB.Table').as('s0').select('s0').limit(25).toList()"); + String exMain = "g.V().has('__typeName', 'Table').out('__Table.columns').limit(10).toList()"; + String exSel = "def f(r){ r }"; + verify("Table select columns limit 10", getExpected(exSel, exMain)); + + String exMain2 = "g.V().has('__typeName', 'Table').out('__Table.db').limit(25).toList()"; + verify("Table select db.name", getExpected(exSel, exMain2)); } @Test(enabled = false) @@ -112,9 +114,16 @@ public class QueryProcessorTest { verify("from DB orderby name", expected); verify("from DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()"); verify("DB as d orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').limit(3).toList()"); - verify("DB as d select name, owner orderby d.owner limit 3", "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner').valueMap('DB.name','DB.owner').limit(3).toList()"); + + + String exSel = "def f(r){ return [['d.name','d.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }"; + String exMain = "g.V().has('__typeName', 'DB').as('d').order().by('DB.owner)').limit(25).toList()"; + verify("DB as d select d.name, d.owner orderby (d.owner) limit 25", getExpected(exSel, exMain)); + + String exMain2 = "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').limit(25).toList()"; + String exSel2 = "def f(r){ return [['_col_0','_col_1']].plus(r.collect({[it.value('Table.name'),it.value('Table.createTime')]})).unique(); }"; verify("Table where (name = \"sales_fact\" and createTime > \"2014-01-01\" ) select name as _col_0, createTime as _col_1 orderby _col_1", - "g.V().has('__typeName', 'Table').and(__.has('Table.name', eq(\"sales_fact\")),__.has('Table.createTime', gt('1388563200000'))).order().by('Table.createTime').valueMap('Table.name','Table.createTime').limit(25).toList()"); + getExpected(exSel2, exMain2)); } @Test @@ -124,47 +133,55 @@ public class QueryProcessorTest { @Test public void fromDBSelect() { - verify("from DB select DB.name, DB.owner", "g.V().has('__typeName', 'DB').valueMap('DB.name','DB.owner').limit(25).toList()"); + String expected = "def f(r){ return [['DB.name','DB.owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }; f(g.V().has('__typeName', 'DB').limit(25).toList())"; + verify("from DB select DB.name, DB.owner", expected); } @Test - public void fromDBSelectGroupBy() { + public void fromDBGroupBy() { verify("from DB groupby (DB.owner)", "g.V().has('__typeName', 'DB').group().by('DB.owner').limit(25).toList()"); } @Test public void whereClauseTextContains() { - String expected = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).valueMap('DB.name','DB.owner').limit(25).toList()"; - verify("from DB where name = \"Reporting\" select name, owner", expected); + String exMain = "g.V().has('__typeName', 'DB').has('DB.name', eq(\"Reporting\")).limit(25).toList()"; + String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('DB.name'),it.value('DB.owner')]})).unique(); }"; + verify("from DB where name = \"Reporting\" select name, owner", getExpected(exSel, exMain)); + verify("from DB where (name = \"Reporting\") select name, owner", getExpected(exSel, exMain)); verify("Table where Asset.name like \"Tab*\"", "g.V().has('__typeName', 'Table').has('Table.name', org.janusgraph.core.attribute.Text.textRegex(\"Tab.*\")).limit(25).toList()"); - verify("from DB where (name = \"Reporting\") select name, owner", expected); verify("from Table where (db.name = \"Reporting\")", - "g.V().has('__typeName', 'Table').out('__DB.Table').has('DB.name', eq(\"Reporting\")).in('__DB.Table').limit(25).toList()"); + "g.V().has('__typeName', 'Table').out('__Table.db').has('DB.name', eq(\"Reporting\")).in('__Table.db').limit(25).toList()"); } @Test public void whereClauseWithAsTextContains() { - verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", - "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).valueMap('Table.name','Table.owner').limit(25).toList()"); + String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }"; + String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.name', eq(\"testtable_1\")).limit(25).toList()"; + verify("Table as t where t.name = \"testtable_1\" select t.name, t.owner)", getExpected(exSel, exMain)); } @Test public void whereClauseWithDateCompare() { - verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", - "g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).valueMap('Table.name','Table.owner').limit(25).toList()"); + String exSel = "def f(r){ return [['t.name','t.owner']].plus(r.collect({[it.value('Table.name'),it.value('Table.owner')]})).unique(); }"; + String exMain = "g.V().has('__typeName', 'Table').as('t').has('Table.createdTime', eq('1513046158440')).limit(25).toList()"; + verify("Table as t where t.createdTime = \"2017-12-12T02:35:58.440Z\" select t.name, t.owner)", getExpected(exSel, exMain)); } @Test public void multipleWhereClauses() { - verify("Table where name=\"sales_fact\", columns as c select c.owner, c.name, c.dataType", - "g.V().has('__typeName', 'Table').has('Table.name', eq(\"sales_fact\")).out('__Table.columns').as('c').valueMap('Column.owner','Column.name','Column.dataType').limit(25).toList()"); + String exSel = "def f(r){ return [['c.owner','c.name','c.dataType']].plus(r.collect({[it.value('Column.owner'),it.value('Column.name'),it.value('Column.dataType')]})).unique(); }"; + String exMain = "g.V().has('__typeName', 'Table').has('Table.name', eq(\"sales_fact\")).out('__Table.columns').as('c').limit(25).toList()"; + verify("Table where name=\"sales_fact\", columns as c select c.owner, c.name, c.dataType", getExpected(exSel, exMain)); + ; } @Test public void subType() { - verify("Asset select name, owner", - "g.V().has('__typeName', within('Asset','Table')).valueMap('Asset.name','Asset.owner').limit(25).toList()"); + String exMain = "g.V().has('__typeName', within('Asset','Table')).limit(25).toList()"; + String exSel = "def f(r){ return [['name','owner']].plus(r.collect({[it.value('Asset.name'),it.value('Asset.owner')]})).unique(); }"; + + verify("Asset select name, owner", getExpected(exSel, exMain)); } @Test @@ -236,7 +253,7 @@ public class QueryProcessorTest { verify("hive_db where hive_db.name='Reporting' and hive_db.createTime < '2017-12-12T02:35:58.440Z'", "g.V().has('__typeName', 'hive_db').and(__.has('hive_db.name', eq('Reporting')),__.has('hive_db.createTime', lt('1513046158440'))).limit(25).toList()"); verify("Table where db.name='Sales' and db.clusterName='cl1'", - "g.V().has('__typeName', 'Table').and(__.out('__DB.Table').has('DB.name', eq('Sales')).in('__DB.Table'),__.out('__DB.Table').has('DB.clusterName', eq('cl1')).in('__DB.Table')).limit(25).toList()"); + "g.V().has('__typeName', 'Table').and(__.out('__Table.db').has('DB.name', eq('Sales')).in('__Table.db'),__.out('__Table.db').has('DB.clusterName', eq('cl1')).in('__Table.db')).limit(25).toList()"); } private void verify(String dsl, String expectedGremlin) { @@ -245,6 +262,10 @@ public class QueryProcessorTest { assertEquals(actualGremlin, expectedGremlin); } + private String getExpected(String select, String main) { + return String.format("%s; f(%s)", select, main); + } + private AtlasDSLParser.QueryContext getParsedQuery(String query) { AtlasDSLParser.QueryContext queryContext = null; InputStream stream = new ByteArrayInputStream(query.getBytes()); @@ -276,8 +297,9 @@ public class QueryProcessorTest { qv.visit(queryContext); queryProcessor.close(); - assertTrue(StringUtils.isNotEmpty(queryProcessor.getText())); - return queryProcessor.getText(); + String s = queryProcessor.getText(); + assertTrue(StringUtils.isNotEmpty(s)); + return s; } private static class TestLookup implements org.apache.atlas.query.Lookup { @@ -324,6 +346,8 @@ public class QueryProcessorTest { public String getRelationshipEdgeLabel(QueryProcessor.Context context, String attributeName) { if (attributeName.equalsIgnoreCase("columns")) return "__Table.columns"; + if (attributeName.equalsIgnoreCase("db")) + return "__Table.db"; else return "__DB.Table"; }
