ATLAS-2229: DSL implementation using ANTLR - #4 (multiple fixes) Signed-off-by: Madhan Neethiraj <mad...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8db8b5c7 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8db8b5c7 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8db8b5c7 Branch: refs/heads/master Commit: 8db8b5c7c199c25c6d83074940fc3020cd3815d0 Parents: 5384a74 Author: apoorvnaik <apoorvn...@apache.org> Authored: Fri Dec 22 12:55:28 2017 -0800 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Sat Dec 23 21:51:34 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/atlas/AtlasErrorCode.java | 3 +- .../atlas/discovery/EntityDiscoveryService.java | 17 +- .../java/org/apache/atlas/query/AtlasDSL.java | 135 +++ .../java/org/apache/atlas/query/DSLVisitor.java | 84 +- .../org/apache/atlas/query/Expressions.java | 41 - .../atlas/query/GremlinQueryComposer.java | 887 +++++++++++++++++++ .../apache/atlas/query/GremlinTranslator.java | 41 - .../apache/atlas/query/IdentifierHelper.java | 38 +- .../java/org/apache/atlas/query/Lookup.java | 18 +- .../org/apache/atlas/query/QueryParser.java | 66 -- .../org/apache/atlas/query/QueryProcessor.java | 823 ----------------- .../atlas/query/antlr4/AtlasDSLLexer.java | 14 +- .../atlas/query/antlr4/AtlasDSLLexer.tokens | 59 -- .../apache/atlas/query/antlr4/AtlasDSLParser.g4 | 2 +- .../atlas/query/antlr4/AtlasDSLParser.tokens | 59 -- .../store/graph/v1/AtlasAbstractDefStoreV1.java | 4 +- .../graph/v1/AtlasRelationshipDefStoreV1.java | 6 +- .../org/apache/atlas/query/DSLQueriesTest.java | 30 +- .../atlas/query/GremlinQueryComposerTest.java | 394 ++++++++ .../apache/atlas/query/QueryProcessorTest.java | 397 --------- 20 files changed, 1527 insertions(+), 1591 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index fb741f8..0953725 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -102,7 +102,8 @@ public enum AtlasErrorCode { INVALID_ENTITY_FOR_CLASSIFICATION (400, "ATLAS-400-00-055", "Entity (guid=â{0}â,typename=â{1}â) cannot be classified by Classification â{2}â, because â{1}â is not in the ClassificationDef's restrictions."), SAVED_SEARCH_CHANGE_USER(400, "ATLAS-400-00-056", "saved-search {0} can not be moved from user {1} to {2}"), INVALID_QUERY_PARAM_LENGTH(400, "ATLAS-400-00-057" , "Length of query param {0} exceeds the limit"), - INVALID_QUERY_LENGTH(400, "ATLAS-400-00-057" , "Invalid query length, update {0} to change the limit" ), + INVALID_QUERY_LENGTH(400, "ATLAS-400-00-058" , "Invalid query length, update {0} to change the limit" ), + INVALID_DSL_QUERY(400, "ATLAS-400-00-059" , "Invalid DSL query: {0} Reason: {1}. Please refer to Atlas DSL grammar for more information" ), // All Not found enums go here TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"), http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index ece1516..cbe9a5b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -33,12 +33,9 @@ import org.apache.atlas.model.discovery.SearchParameters; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.profile.AtlasUserSavedSearch; -import org.apache.atlas.query.Expressions.Expression; +import org.apache.atlas.query.AtlasDSL; import org.apache.atlas.query.GremlinQuery; -import org.apache.atlas.query.GremlinTranslator; import org.apache.atlas.query.QueryParams; -import org.apache.atlas.query.QueryParser; -import org.apache.atlas.query.QueryProcessor; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graph.GraphBackedSearchIndexer; import org.apache.atlas.repository.graph.GraphHelper; @@ -678,16 +675,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { } private GremlinQuery toGremlinQuery(String query, int limit, int offset) throws AtlasBaseException { - QueryParams params = validateSearchParams(limit, offset); - Expression expression = QueryParser.apply(query, params); - - if (expression == null) { - throw new AtlasBaseException(DISCOVERY_QUERY_FAILED, query); - } - - QueryProcessor queryProcessor = new QueryProcessor(typeRegistry, limit, offset); - Expression validExpression = queryProcessor.validate(expression); - GremlinQuery gremlinQuery = new GremlinTranslator(queryProcessor, validExpression).translate(); + QueryParams params = validateSearchParams(limit, offset); + GremlinQuery gremlinQuery = new AtlasDSL.Translator(AtlasDSL.Parser.parse(query), typeRegistry, params.offset(), params.limit()).translate(); if (LOG.isDebugEnabled()) { LOG.debug("Translated Gremlin Query: {}", gremlinQuery.queryStr()); http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java b/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java new file mode 100644 index 0000000..e4c8165 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/query/AtlasDSL.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.query; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.TokenStream; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.query.antlr4.AtlasDSLLexer; +import org.apache.atlas.query.antlr4.AtlasDSLParser; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class AtlasDSL { + + public static class Parser { + private static final Logger LOG = LoggerFactory.getLogger(Parser.class); + + private static final Set<String> RESERVED_KEYWORDS = + new HashSet<>(Arrays.asList("[", "]", "(", ")", "=", "<", ">", "!=", "<=", ">=", ",", "and", "or", "+", "-", + "*", "/", ".", "select", "from", "where", "groupby", "loop", "isa", "is", "has", + "as", "times", "withPath", "limit", "offset", "orderby", "count", "max", "min", + "sum", "by", "order", "like")); + + public static boolean isKeyword(String word) { + return RESERVED_KEYWORDS.contains(word); + } + + public static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException { + AtlasDSLParser.QueryContext ret; + try { + + InputStream stream = new ByteArrayInputStream(queryStr.getBytes()); + AtlasDSLLexer lexer = new AtlasDSLLexer(CharStreams.fromStream(stream)); + Validator validator = new Validator(); + TokenStream inputTokenStream = new CommonTokenStream(lexer); + AtlasDSLParser parser = new AtlasDSLParser(inputTokenStream); + + parser.removeErrorListeners(); + parser.addErrorListener(validator); + + // Validate the syntax of the query here + ret = parser.query(); + if (!validator.isValid()) { + LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg()); + throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg()); + } + + } catch (IOException e) { + throw new AtlasBaseException(e); + } + + return ret; + } + + } + + static class Validator extends BaseErrorListener { + private boolean isValid = true; + private String errorMsg = ""; + + @Override + public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line, final int charPositionInLine, final String msg, final RecognitionException e) { + // TODO: Capture multiple datapoints + isValid = false; + errorMsg = msg; + } + + public boolean isValid() { + return isValid; + } + + public String getErrorMsg() { + return errorMsg; + } + } + + public static class Translator { + private final AtlasDSLParser.QueryContext queryContext; + private final AtlasTypeRegistry typeRegistry; + private final int offset; + private final int limit; + + public Translator(final AtlasDSLParser.QueryContext queryContext, AtlasTypeRegistry typeRegistry, int offset, int limit) { + this.queryContext = queryContext; + this.typeRegistry = typeRegistry; + this.offset = offset; + this.limit = limit; + } + + public GremlinQuery translate() { + GremlinQueryComposer gremlinQueryComposer = new GremlinQueryComposer(typeRegistry); + + if (offset >= 0) { + if (!gremlinQueryComposer.hasLimitOffset()) { + gremlinQueryComposer.addLimit(Integer.toString(limit), Integer.toString(offset)); + } + } + + DSLVisitor dslVisitor = new DSLVisitor(gremlinQueryComposer); + + // Now process the Query and collect translation in + queryContext.accept(dslVisitor); + + return new GremlinQuery(gremlinQueryComposer.get(), gremlinQueryComposer.hasSelect()); + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java index b597a0d..a21e884 100644 --- a/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java +++ b/repository/src/main/java/org/apache/atlas/query/DSLVisitor.java @@ -28,51 +28,51 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { +public class DSLVisitor extends AtlasDSLParserBaseVisitor<Void> { private static final Logger LOG = LoggerFactory.getLogger(DSLVisitor.class); private static final String AND = "AND"; private static final String OR = "OR"; - private final QueryProcessor queryProcessor; + private final GremlinQueryComposer gremlinQueryComposer; - public DSLVisitor(QueryProcessor queryProcessor) { - this.queryProcessor = queryProcessor; + public DSLVisitor(GremlinQueryComposer gremlinQueryComposer) { + this.gremlinQueryComposer = gremlinQueryComposer; } @Override - public String visitIsClause(IsClauseContext ctx) { + public Void visitIsClause(IsClauseContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitIsClause({})", ctx); } - queryProcessor.addFromIsA(ctx.arithE().getText(), ctx.identifier().getText()); + gremlinQueryComposer.addFromIsA(ctx.arithE().getText(), ctx.identifier().getText()); return super.visitIsClause(ctx); } @Override - public String visitHasClause(HasClauseContext ctx) { + public Void visitHasClause(HasClauseContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitHasClause({})", ctx); } - queryProcessor.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText()); + gremlinQueryComposer.addFromProperty(ctx.arithE().getText(), ctx.identifier().getText()); return super.visitHasClause(ctx); } @Override - public String visitLimitOffset(LimitOffsetContext ctx) { + public Void visitLimitOffset(LimitOffsetContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitLimitOffset({})", ctx); } - queryProcessor.addLimit(ctx.limitClause().NUMBER().toString(), - (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); + gremlinQueryComposer.addLimit(ctx.limitClause().NUMBER().toString(), + (ctx.offsetClause() == null ? "0" : ctx.offsetClause().NUMBER().getText())); return super.visitLimitOffset(ctx); } @Override - public String visitSelectExpr(SelectExprContext ctx) { + public Void visitSelectExpr(SelectExprContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitSelectExpr({})", ctx); } @@ -85,7 +85,7 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { String[] items = new String[ctx.selectExpression().size()]; String[] labels = new String[ctx.selectExpression().size()]; - QueryProcessor.SelectExprMetadata selectExprMetadata = new QueryProcessor.SelectExprMetadata(); + GremlinQueryComposer.SelectExprMetadata selectExprMetadata = new GremlinQueryComposer.SelectExprMetadata(); for (int i = 0; i < ctx.selectExpression().size(); i++) { SelectExpressionContext selectExpression = ctx.selectExpression(i); @@ -116,23 +116,25 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { selectExprMetadata.setItems(items); selectExprMetadata.setLabels(labels); - queryProcessor.addSelect(selectExprMetadata); + gremlinQueryComposer.addSelect(selectExprMetadata); } return super.visitSelectExpr(ctx); } @Override - public String visitOrderByExpr(OrderByExprContext ctx) { + public Void visitOrderByExpr(OrderByExprContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitOrderByExpr({})", ctx); } - queryProcessor.addOrderBy(ctx.expr().getText(), (ctx.sortOrder() != null && ctx.sortOrder().getText().equalsIgnoreCase("desc"))); + // Extract the attribute from parentheses + String text = ctx.expr().getText().replace("(", "").replace(")", ""); + gremlinQueryComposer.addOrderBy(text, (ctx.sortOrder() != null && ctx.sortOrder().getText().equalsIgnoreCase("desc"))); return super.visitOrderByExpr(ctx); } @Override - public String visitWhereClause(WhereClauseContext ctx) { + public Void visitWhereClause(WhereClauseContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitWhereClause({})", ctx); } @@ -141,12 +143,12 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { // The first expr shouldn't be processed if there are following exprs ExprContext expr = ctx.expr(); - processExpr(expr, queryProcessor); + processExpr(expr, gremlinQueryComposer); return super.visitWhereClause(ctx); } @Override - public String visitFromExpression(final FromExpressionContext ctx) { + public Void visitFromExpression(final FromExpressionContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitFromExpression({})", ctx); } @@ -155,38 +157,38 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { AliasExprContext aliasExpr = fromSrc.aliasExpr(); if (aliasExpr != null) { - queryProcessor.addFromAlias(aliasExpr.identifier(0).getText(), aliasExpr.identifier(1).getText()); + gremlinQueryComposer.addFromAlias(aliasExpr.identifier(0).getText(), aliasExpr.identifier(1).getText()); } else { if (fromSrc.identifier() != null) { - queryProcessor.addFrom(fromSrc.identifier().getText()); + gremlinQueryComposer.addFrom(fromSrc.identifier().getText()); } else { - queryProcessor.addFrom(fromSrc.literal().getText()); + gremlinQueryComposer.addFrom(fromSrc.literal().getText()); } } return super.visitFromExpression(ctx); } @Override - public String visitGroupByExpression(GroupByExpressionContext ctx) { + public Void visitGroupByExpression(GroupByExpressionContext ctx) { if (LOG.isDebugEnabled()) { LOG.debug("=> DSLVisitor.visitGroupByExpression({})", ctx); } String s = ctx.selectExpr().getText(); - queryProcessor.addGroupBy(s); + gremlinQueryComposer.addGroupBy(s); return super.visitGroupByExpression(ctx); } - private void processExpr(final ExprContext expr, QueryProcessor queryProcessor) { + private void processExpr(final ExprContext expr, GremlinQueryComposer gremlinQueryComposer) { if (CollectionUtils.isNotEmpty(expr.exprRight())) { - processExprRight(expr, queryProcessor); + processExprRight(expr, gremlinQueryComposer); } else { - processExpr(expr.compE(), queryProcessor); + processExpr(expr.compE(), gremlinQueryComposer); } } - private void processExprRight(final ExprContext expr, QueryProcessor queryProcessor) { - QueryProcessor nestedProcessor = queryProcessor.createNestedProcessor(); + private void processExprRight(final ExprContext expr, GremlinQueryComposer gremlinQueryComposer) { + GremlinQueryComposer nestedProcessor = gremlinQueryComposer.createNestedProcessor(); List<String> nestedQueries = new ArrayList<>(); String prev = null; @@ -194,20 +196,20 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { // Process first expression then proceed with the others // expr -> compE exprRight* processExpr(expr.compE(), nestedProcessor); - nestedQueries.add(nestedProcessor.getText()); + nestedQueries.add(nestedProcessor.get()); for (ExprRightContext exprRight : expr.exprRight()) { - nestedProcessor = queryProcessor.createNestedProcessor(); + nestedProcessor = gremlinQueryComposer.createNestedProcessor(); // AND expression if (exprRight.K_AND() != null) { if (prev == null) prev = AND; if (OR.equalsIgnoreCase(prev)) { // Change of context - QueryProcessor orClause = nestedProcessor.createNestedProcessor(); + GremlinQueryComposer orClause = nestedProcessor.createNestedProcessor(); orClause.addOrClauses(nestedQueries); nestedQueries.clear(); - nestedQueries.add(orClause.getText()); + nestedQueries.add(orClause.get()); } prev = AND; } @@ -216,25 +218,25 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { if (prev == null) prev = OR; if (AND.equalsIgnoreCase(prev)) { // Change of context - QueryProcessor andClause = nestedProcessor.createNestedProcessor(); + GremlinQueryComposer andClause = nestedProcessor.createNestedProcessor(); andClause.addAndClauses(nestedQueries); nestedQueries.clear(); - nestedQueries.add(andClause.getText()); + nestedQueries.add(andClause.get()); } prev = OR; } processExpr(exprRight.compE(), nestedProcessor); - nestedQueries.add(nestedProcessor.getText()); + nestedQueries.add(nestedProcessor.get()); } if (AND.equalsIgnoreCase(prev)) { - queryProcessor.addAndClauses(nestedQueries); + gremlinQueryComposer.addAndClauses(nestedQueries); } if (OR.equalsIgnoreCase(prev)) { - queryProcessor.addOrClauses(nestedQueries); + gremlinQueryComposer.addOrClauses(nestedQueries); } } - private void processExpr(final CompEContext compE, final QueryProcessor queryProcessor) { + private void processExpr(final CompEContext compE, final GremlinQueryComposer gremlinQueryComposer) { if (compE != null && compE.isClause() == null && compE.hasClause() == null && compE.isClause() == null) { ComparisonClauseContext comparisonClause = compE.comparisonClause(); @@ -252,9 +254,9 @@ public class DSLVisitor extends AtlasDSLParserBaseVisitor<String> { String op = comparisonClause.operator().getText().toUpperCase(); String rhs = comparisonClause.arithE(1).getText(); - queryProcessor.addWhere(lhs, op, rhs); + gremlinQueryComposer.addWhere(lhs, op, rhs); } else { - processExpr(compE.arithE().multiE().atomE().expr(), queryProcessor); + processExpr(compE.arithE().multiE().atomE().expr(), gremlinQueryComposer); } } } http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/Expressions.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/Expressions.java b/repository/src/main/java/org/apache/atlas/query/Expressions.java deleted file mode 100644 index 77350f0..0000000 --- a/repository/src/main/java/org/apache/atlas/query/Expressions.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.query; - - -import org.apache.atlas.query.antlr4.AtlasDSLParser.QueryContext; - - -public class Expressions { - public static class Expression { - private final QueryContext parsedQuery; - - - public Expression(QueryContext q) { - parsedQuery = q; - } - - public Expression isReady() { - return (parsedQuery != null ? this : null); - } - - public void accept(DSLVisitor qv) { - qv.visit(parsedQuery); - } - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java new file mode 100644 index 0000000..6af1db6 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java @@ -0,0 +1,887 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.query; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.type.AtlasArrayType; +import org.apache.atlas.type.AtlasBuiltInTypes; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasStructType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class GremlinQueryComposer { + private static final Logger LOG = LoggerFactory.getLogger(GremlinQueryComposer.class); + + private final int DEFAULT_QUERY_RESULT_LIMIT = 25; + private final int DEFAULT_QUERY_RESULT_OFFSET = 0; + + private final List<String> errorList = new ArrayList<>(); + private final GremlinClauseList queryClauses = new GremlinClauseList(); + private final Lookup lookup; + private final boolean isNestedQuery; + private int providedLimit = DEFAULT_QUERY_RESULT_LIMIT; + private int providedOffset = DEFAULT_QUERY_RESULT_OFFSET; + private boolean hasSelect = false; + private boolean isSelectNoop = false; + private boolean hasGroupBy = false; + private boolean hasOrderBy = false; + private boolean hasLimitOffset = false; + private String offset = null; + private String limit = null; + private Context context; + private SelectExprMetadata selectExprMetadata; + + @Inject + public GremlinQueryComposer(AtlasTypeRegistry typeRegistry) { + isNestedQuery = false; + lookup = new RegistryBasedLookup(errorList, typeRegistry); + context = new Context(errorList, lookup); + + init(); + } + + public GremlinQueryComposer(AtlasTypeRegistry typeRegistry, int limit, int offset) { + this(typeRegistry); + + providedLimit = limit; + providedOffset = offset < 0 ? DEFAULT_QUERY_RESULT_OFFSET : offset; + } + + @VisibleForTesting + GremlinQueryComposer(Lookup lookup, Context context) { + this.isNestedQuery = false; + this.lookup = lookup; + this.context = context; + + init(); + } + + public GremlinQueryComposer(Lookup registryLookup, boolean isNestedQuery) { + this.isNestedQuery = isNestedQuery; + this.lookup = registryLookup; + + init(); + } + + public void addFrom(String typeName) { + if (LOG.isDebugEnabled()) { + LOG.debug("addFrom(typeName={})", typeName); + } + + IdentifierHelper.Advice ta = getAdvice(typeName); + + if(context.shouldRegister(ta.get())) { + context.registerActive(ta.get()); + + IdentifierHelper.Advice ia = getAdvice(ta.get()); + + if (ia.isTrait()) { + add(GremlinClause.TRAIT, ia.get()); + } else { + if (ia.hasSubtypes()) { + add(GremlinClause.HAS_TYPE_WITHIN, ia.getSubTypes()); + } else { + add(GremlinClause.HAS_TYPE, ia.get()); + } + } + } else { + IdentifierHelper.Advice ia = getAdvice(ta.get()); + introduceType(ia); + } + } + + public void addFromProperty(String typeName, String attribute) { + if (LOG.isDebugEnabled()) { + LOG.debug("addFromProperty(typeName={}, attribute={})", typeName, attribute); + } + + addFrom(typeName); + add(GremlinClause.HAS_PROPERTY, + IdentifierHelper.getQualifiedName(lookup, context, attribute)); + } + + + public void addFromIsA(String typeName, String traitName) { + addFrom(typeName); + add(GremlinClause.TRAIT, traitName); + } + + public void addWhere(String lhs, String operator, String rhs) { + if (LOG.isDebugEnabled()) { + LOG.debug("addWhere(lhs={}, operator={}, rhs={})", lhs, operator, rhs); + } + + String currentType = context.getActiveTypeName(); + SearchParameters.Operator op = SearchParameters.Operator.fromString(operator); + IdentifierHelper.Advice org = null; + IdentifierHelper.Advice lhsI = getAdvice(lhs); + if(!lhsI.isPrimitive()) { + introduceType(lhsI); + org = lhsI; + lhsI = getAdvice(lhs); + } + + if(lhsI.isDate()) { + rhs = parseDate(rhs); + } + + rhs = addQuotesIfNecessary(rhs); + if(op == SearchParameters.Operator.LIKE) { + add(GremlinClause.TEXT_CONTAINS, lhsI.getQualifiedName(), rhs.replace("*", ".*").replace('?', '.')); + } else if(op == SearchParameters.Operator.IN) { + add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), "within", rhs); + } else { + add(GremlinClause.HAS_OPERATOR, lhsI.getQualifiedName(), op.getSymbols()[1], rhs); + } + + if(org != null && !org.isPrimitive() && org.getIntroduceType()) { + add(GremlinClause.DEDUP); + add(GremlinClause.IN, org.getEdgeLabel()); + context.registerActive(currentType); + } + } + + public void addAndClauses(List<String> clauses) { + queryClauses.add(GremlinClause.AND, StringUtils.join(clauses, ',')); + } + + public void addOrClauses(List<String> clauses) { + queryClauses.add(GremlinClause.OR, StringUtils.join(clauses, ',')); + } + + public void addSelect(SelectExprMetadata selectExprMetadata) { + String[] items = selectExprMetadata.getItems(); + String[] labels = selectExprMetadata.getLabels(); + if (LOG.isDebugEnabled()) { + LOG.debug("addSelect(items.length={})", items != null ? items.length : 0); + } + + if (items != null) { + for (int i = 0; i < items.length; i++) { + IdentifierHelper.Advice ia = getAdvice(items[i]); + + if(!labels[i].equals(items[i])) { + context.aliasMap.put(labels[i], ia.getQualifiedName()); + } + + if (i == selectExprMetadata.getCountIdx()) { + items[i] = GremlinClause.INLINE_COUNT.get(); + } else if (i == selectExprMetadata.getMinIdx()) { + items[i] = GremlinClause.INLINE_MIN.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else if (i == selectExprMetadata.getMaxIdx()) { + items[i] = GremlinClause.INLINE_MAX.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else if (i == selectExprMetadata.getSumIdx()) { + items[i] = GremlinClause.INLINE_SUM.get(ia.getQualifiedName(), ia.getQualifiedName()); + } else { + if (!ia.isPrimitive() && ia.getIntroduceType()) { + add(GremlinClause.OUT, ia.getEdgeLabel()); + context.registerActive(ia.getTypeName()); + + int dotIdx = ia.get().indexOf("."); + if (dotIdx != -1) { + IdentifierHelper.Advice iax = getAdvice(ia.get()); + items[i] = GremlinClause.INLINE_GET_PROPERTY.get(iax.getQualifiedName()); + } else { + isSelectNoop = true; + } + } else { + items[i] = GremlinClause.INLINE_GET_PROPERTY.get(ia.getQualifiedName()); + } + } + } + + // If GroupBy clause exists then the query spits out a List<Map<String, List<AtlasVertex>>> otherwise the query returns List<AtlasVertex> + // Different transformations are needed for DSLs with groupby and w/o groupby + GremlinClause transformationFn; + if (isSelectNoop) { + transformationFn = GremlinClause.SELECT_EXPR_NOOP_FN; + } else { + transformationFn = hasGroupBy ? GremlinClause.SELECT_WITH_GRPBY_HELPER_FN : GremlinClause.SELECT_EXPR_HELPER_FN; + } + queryClauses.add(0, transformationFn, getJoinedQuotedStr(labels), String.join(",", items)); + queryClauses.add(GremlinClause.INLINE_TRANSFORM_CALL); + + hasSelect = true; + this.selectExprMetadata = selectExprMetadata; + } + } + + public GremlinQueryComposer createNestedProcessor() { + GremlinQueryComposer qp = new GremlinQueryComposer(lookup, true); + qp.context = this.context; + return qp; + } + + public void addFromAlias(String typeName, String alias) { + if (LOG.isDebugEnabled()) { + LOG.debug("addFromAlias(typeName={}, alias={})", typeName, alias); + } + + addFrom(typeName); + addAsClause(alias); + context.registerAlias(alias); + } + + public void addAsClause(String stepName) { + if (LOG.isDebugEnabled()) { + LOG.debug("addAsClause(stepName={})", stepName); + } + + add(GremlinClause.AS, stepName); + } + + public void addGroupBy(String item) { + if (LOG.isDebugEnabled()) { + LOG.debug("addGroupBy(item={})", item); + } + + addGroupByClause(item); + hasGroupBy = true; + } + + public void addLimit(String limit, String offset) { + if (LOG.isDebugEnabled()) { + LOG.debug("addLimit(limit={}, offset={})", limit, offset); + } + + this.limit = limit; + this.offset = offset; + + hasLimitOffset = true; + } + + public String get() { + close(); + + String ret; + String[] items = new String[queryClauses.size()]; + + boolean needTransformation = needTransformation(); + int startIdx = needTransformation ? 1 : 0; + int endIdx = needTransformation ? queryClauses.size() - 1 : queryClauses.size(); + + for (int i = startIdx; i < endIdx; i++) { + items[i] = queryClauses.getValue(i); + } + + if (needTransformation) { + String body = String.join(".", Stream.of(items).filter(Objects::nonNull).collect(Collectors.toList())); + String inlineFn = queryClauses.getValue(queryClauses.size() - 1); + String funCall = String.format(inlineFn, body); + ret = queryClauses.getValue(0) + funCall; + } else { + ret = String.join(".", items); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("get() => {}", ret); + } + return ret; + } + + public boolean hasSelect() { + return hasSelect; + } + + public boolean hasLimitOffset() { + return hasLimitOffset; + } + + public void addOrderBy(String name, boolean isDesc) { + if (LOG.isDebugEnabled()) { + LOG.debug("addOrderBy(name={}, isDesc={})", name, isDesc); + } + + AtlasAttribute attribute = ((AtlasStructType) context.getActiveType()).getAttribute(getAttributeName(name)); + if (hasGroupBy) { + GremlinClause transformationFn = isDesc ? GremlinClause.GRPBY_ORDERBY_DESC_HELPER_FN : GremlinClause.GRPBY_ORDERBY_ASC_HELPER_FN; + add(0, transformationFn, attribute.getQualifiedName(), attribute.getQualifiedName()); + add(GremlinClause.INLINE_TRANSFORM_CALL); + } else { + addOrderByClause(attribute.getQualifiedName(), isDesc); + } + hasOrderBy = true; + } + + private String getAttributeName(String fqdn) { + final String ret; + + int lastSepIdx = fqdn.lastIndexOf('.'); + + return lastSepIdx == -1 ? fqdn : fqdn.substring(lastSepIdx + 1); + } + + private boolean needTransformation() { + return (hasGroupBy && hasSelect && hasOrderBy) || (hasGroupBy && hasOrderBy) || hasSelect; + } + + private static String quoted(String rhs) { + return IdentifierHelper.getQuoted(rhs); + } + + private String addQuotesIfNecessary(String rhs) { + if(IdentifierHelper.isQuoted(rhs)) return rhs; + return quoted(rhs); + } + + private String parseDate(String rhs) { + String s = IdentifierHelper.isQuoted(rhs) ? + IdentifierHelper.removeQuotes(rhs) : + rhs; + return String.format("'%d'", DateTime.parse(s).getMillis()); + } + + private void close() { + // No limits or toList() need to be added to the nested queries + if (isNestedQuery) return; + + if (hasLimitOffset) { + // If there are any aggregator functions then implicit limits shouldn't be applied + if (selectExprMetadata == null || !selectExprMetadata.hasAggregatorFunction()) { + if (offset.equalsIgnoreCase("0")) { + add(GremlinClause.LIMIT, limit); + } else { + addRangeClause(offset, limit); + } + } else { + LOG.warn("Query has aggregator function. Performance might be slow for large dataset"); + } + } + + if (queryClauses.isEmpty()) { + queryClauses.clear(); + return; + } + + updatePosition(GremlinClause.LIMIT); + add(GremlinClause.TO_LIST); + updatePosition(GremlinClause.INLINE_TRANSFORM_CALL); + } + + private void updatePosition(GremlinClause clause) { + int index = queryClauses.hasClause(clause); + if(-1 == index) { + return; + } + + GremlinClauseValue gcv = queryClauses.remove(index); + queryClauses.add(gcv); + } + + private void init() { + if (!isNestedQuery) { + add(GremlinClause.G); + add(GremlinClause.V); + } else { + add(GremlinClause.NESTED_START); + } + } + + private void introduceType(IdentifierHelper.Advice ia) { + if (!ia.isPrimitive() && ia.getIntroduceType()) { + add(GremlinClause.OUT, ia.getEdgeLabel()); + context.registerActive(ia.getTypeName()); + } + } + + private IdentifierHelper.Advice getAdvice(String actualTypeName) { + return IdentifierHelper.create(context, lookup, actualTypeName); + } + + private String getJoinedQuotedStr(String[] elements) { + StringJoiner joiner = new StringJoiner(","); + Arrays.stream(elements) + .map(x -> x.contains("'") ? "\"" + x + "\"" : "'" + x + "'") + .forEach(joiner::add); + return joiner.toString(); + } + + private void add(GremlinClause clause, String... args) { + queryClauses.add(new GremlinClauseValue(clause, clause.get(args))); + } + + private void add(int idx, GremlinClause clause, String... args) { + queryClauses.add(idx, new GremlinClauseValue(clause, clause.get(args))); + } + + private void addRangeClause(String startIndex, String endIndex) { + if (LOG.isDebugEnabled()) { + LOG.debug("addRangeClause(startIndex={}, endIndex={})", startIndex, endIndex); + } + + if (hasSelect) { + add(queryClauses.size() - 1, GremlinClause.RANGE, startIndex, startIndex, endIndex); + } else { + add(GremlinClause.RANGE, startIndex, startIndex, endIndex); + } + } + + private void addOrderByClause(String name, boolean descr) { + if (LOG.isDebugEnabled()) { + LOG.debug("addOrderByClause(name={})", name, descr); + } + + IdentifierHelper.Advice ia = getAdvice(name); + add((!descr) ? GremlinClause.ORDER_BY : GremlinClause.ORDER_BY_DESC, ia.getQualifiedName()); + } + + private void addGroupByClause(String name) { + if (LOG.isDebugEnabled()) { + LOG.debug("addGroupByClause(name={})", name); + } + + IdentifierHelper.Advice ia = getAdvice(name); + add(GremlinClause.GROUP_BY, ia.getQualifiedName()); + } + + private enum GremlinClause { + AS("as('%s')"), + DEDUP("dedup()"), + G("g"), + GROUP_BY("group().by('%s')"), + HAS("has('%s', %s)"), + HAS_OPERATOR("has('%s', %s(%s))"), + HAS_PROPERTY("has('%s')"), + WHERE("where(%s)"), + HAS_NOT_PROPERTY("hasNot('%s')"), + HAS_TYPE("has('__typeName', '%s')"), + HAS_TYPE_WITHIN("has('__typeName', within(%s))"), + HAS_WITHIN("has('%s', within(%s))"), + IN("in('%s')"), + OR("or(%s)"), + AND("and(%s)"), + NESTED_START("__"), + NESTED_HAS_OPERATOR("has('%s', %s(%s))"), + LIMIT("limit(%s)"), + ORDER_BY("order().by('%s')"), + ORDER_BY_DESC("order().by('%s', decr)"), + OUT("out('%s')"), + RANGE("range(%s, %s + %s)"), + SELECT("select('%s')"), + TO_LIST("toList()"), + TEXT_CONTAINS("has('%s', org.janusgraph.core.attribute.Text.textRegex(%s))"), + TEXT_PREFIX("has('%s', org.janusgraph.core.attribute.Text.textPrefix(%s))"), + TEXT_SUFFIX("has('%s', org.janusgraph.core.attribute.Text.textRegex(\".*\" + %s))"), + TRAIT("has('__traitNames', within('%s'))"), + SELECT_EXPR_NOOP_FN("def f(r){ r }; "), + SELECT_EXPR_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({[%s]})).unique(); }; "), + SELECT_WITH_GRPBY_HELPER_FN("def f(r){ return [[%s]].plus(r.collect({it.values()}).flatten().collect({[%s]})).unique(); }; "), + GRPBY_ORDERBY_ASC_HELPER_FN("def f(r){ m=r.get(0); m.each({ k,v -> m[k] = v.sort{a,b -> a.value('%s') <=> b.value('%s')}}); r }; "), + GRPBY_ORDERBY_DESC_HELPER_FN("def f(r){ m=r.get(0); m.each({ k,v -> m[k] = v.sort{a,b -> b.value('%s') <=> a.value('%s')}}); r; }; "), + INLINE_COUNT("r.size()"), + INLINE_SUM("r.sum({it.value('%s')}).value('%s')"), + INLINE_MAX("r.max({it.value('%s')}).value('%s')"), + INLINE_MIN("r.min({it.value('%s')}).value('%s')"), + INLINE_GET_PROPERTY("it.value('%s')"), + INLINE_OUT_VERTEX("it.out('%s')"), + INLINE_OUT_VERTEX_VALUE("it.out('%s').value('%s')"), // This might require more closure introduction :( + INLINE_TRANSFORM_CALL("f(%s)"), + V("V()"), + VALUE_MAP("valueMap(%s)"); + + private final String format; + + GremlinClause(String format) { + this.format = format; + } + + String get(String... args) { + return (args == null || args.length == 0) ? + format : + String.format(format, args); + } + } + + private static class GremlinClauseValue { + private final GremlinClause clause; + private final String value; + + public GremlinClauseValue(GremlinClause clause, String value) { + this.clause = clause; + this.value = value; + } + + public GremlinClause getClause() { + return clause; + } + + public String getValue() { + return value; + } + } + + private static class GremlinClauseList { + private final List<GremlinClauseValue> list; + + private GremlinClauseList() { + this.list = new LinkedList<>(); + } + + public void add(GremlinClauseValue g) { + list.add(g); + } + + public void add(int idx, GremlinClauseValue g) { + list.add(idx, g); + } + + public void add(GremlinClauseValue g, AtlasEntityType t) { + add(g); + } + + public void add(int idx, GremlinClauseValue g, AtlasEntityType t) { + add(idx, g); + } + + public void add(GremlinClause clause, String... args) { + list.add(new GremlinClauseValue(clause, clause.get(args))); + } + + public void add(int i, GremlinClause clause, String... args) { + list.add(i, new GremlinClauseValue(clause, clause.get(args))); + } + + public GremlinClauseValue getAt(int i) { + return list.get(i); + } + + public String getValue(int i) { + return list.get(i).value; + } + + public GremlinClauseValue get(int i) { + return list.get(i); + } + + public int size() { + return list.size(); + } + + public int hasClause(GremlinClause clause) { + for (int i = 0; i < list.size(); i++) { + if (list.get(i).getClause() == clause) + return i; + } + + return -1; + } + + public boolean isEmpty() { + return list.size() == 0; + } + + public void clear() { + list.clear(); + } + + public GremlinClauseValue remove(int index) { + GremlinClauseValue gcv = get(index); + list.remove(index); + return gcv; + } + } + + @VisibleForTesting + static class Context { + private final List<String> errorList; + Lookup lookup; + Map<String, String> aliasMap = new HashMap<>(); + private AtlasType activeType; + + public Context(List<String> errorList, Lookup lookup) { + this.lookup = lookup; + this.errorList = errorList; + } + + public void registerActive(String typeName) { + if(shouldRegister(typeName)) { + activeType = lookup.getType(typeName); + } + + aliasMap.put(typeName, typeName); + } + + public AtlasType getActiveType() { + return activeType; + } + + public AtlasEntityType getActiveEntityType() { + return (activeType instanceof AtlasEntityType) ? + (AtlasEntityType) activeType : + null; + } + + public String getActiveTypeName() { + return activeType.getTypeName(); + } + + public boolean shouldRegister(String typeName) { + return activeType == null || + (activeType != null && !StringUtils.equals(getActiveTypeName(), typeName)) && + (activeType != null && !lookup.hasAttribute(this, typeName)); + } + + public void registerAlias(String alias) { + if(aliasMap.containsKey(alias)) { + errorList.add(String.format("Duplicate alias found: %s for type %s already present.", alias, getActiveEntityType())); + return; + } + + aliasMap.put(alias, getActiveTypeName()); + } + + public boolean hasAlias(String alias) { + return aliasMap.containsKey(alias); + } + + public String getTypeNameFromAlias(String alias) { + return aliasMap.get(alias); + } + + public boolean isEmpty() { + return activeType == null; + } + } + + private static class RegistryBasedLookup implements Lookup { + private final List<String> errorList; + private final AtlasTypeRegistry typeRegistry; + + public RegistryBasedLookup(List<String> errorList, AtlasTypeRegistry typeRegistry) { + this.errorList = errorList; + this.typeRegistry = typeRegistry; + } + + @Override + public AtlasType getType(String typeName) { + try { + return typeRegistry.getType(typeName); + } catch (AtlasBaseException e) { + addError(e.getMessage()); + } + + return null; + } + + @Override + public String getQualifiedName(Context context, String name) { + try { + AtlasEntityType et = context.getActiveEntityType(); + if(et == null) { + return ""; + } + + return et.getQualifiedAttributeName(name); + } catch (AtlasBaseException e) { + addError(e.getMessage()); + } + + return ""; + } + + @Override + public boolean isPrimitive(Context context, String attributeName) { + AtlasEntityType et = context.getActiveEntityType(); + if(et == null) { + return false; + } + + AtlasType attr = et.getAttributeType(attributeName); + if(attr == null) { + return false; + } + + TypeCategory attrTypeCategory = attr.getTypeCategory(); + return (attrTypeCategory != null) && (attrTypeCategory == TypeCategory.PRIMITIVE || attrTypeCategory == TypeCategory.ENUM); + } + + @Override + public String getRelationshipEdgeLabel(Context context, String attributeName) { + AtlasEntityType et = context.getActiveEntityType(); + if(et == null) { + return ""; + } + + AtlasAttribute attr = et.getAttribute(attributeName); + return (attr != null) ? attr.getRelationshipEdgeLabel() : ""; + } + + @Override + public boolean hasAttribute(Context context, String typeName) { + return (context.getActiveEntityType() != null) && context.getActiveEntityType().getAttribute(typeName) != null; + } + + @Override + public boolean doesTypeHaveSubTypes(Context context) { + return (context.getActiveEntityType() != null && context.getActiveEntityType().getAllSubTypes().size() > 0); + } + + @Override + public String getTypeAndSubTypes(Context context) { + String[] str = context.getActiveEntityType() != null ? + context.getActiveEntityType().getTypeAndAllSubTypes().toArray(new String[]{}) : + new String[]{}; + if(str.length == 0) { + return null; + } + + String[] quoted = new String[str.length]; + for (int i = 0; i < str.length; i++) { + quoted[i] = quoted(str[i]); + } + + return StringUtils.join(quoted, ","); + } + + @Override + public boolean isTraitType(Context context) { + return (context.getActiveType() != null && + context.getActiveType().getTypeCategory() == TypeCategory.CLASSIFICATION); + } + + @Override + public String getTypeFromEdge(Context context, String item) { + AtlasEntityType et = context.getActiveEntityType(); + if(et == null) { + return ""; + } + + AtlasAttribute attr = et.getAttribute(item); + if(attr == null) { + return null; + } + + AtlasType at = attr.getAttributeType(); + if(at.getTypeCategory() == TypeCategory.ARRAY) { + AtlasArrayType arrType = ((AtlasArrayType)at); + return ((AtlasBuiltInTypes.AtlasObjectIdType) arrType.getElementType()).getObjectType(); + } + + return context.getActiveEntityType().getAttribute(item).getTypeName(); + } + + @Override + public boolean isDate(Context context, String attributeName) { + AtlasEntityType et = context.getActiveEntityType(); + if (et == null) { + return false; + } + + AtlasType attr = et.getAttributeType(attributeName); + return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE); + + } + + protected void addError(String s) { + errorList.add(s); + } + } + + static class SelectExprMetadata { + private String[] items; + private String[] labels; + + private int countIdx = -1; + private int sumIdx = -1; + private int maxIdx = -1; + private int minIdx = -1; + private boolean hasAggregator = false; + + public String[] getItems() { + return items; + } + + public void setItems(final String[] items) { + this.items = items; + } + + public int getCountIdx() { + return countIdx; + } + + public void setCountIdx(final int countIdx) { + this.countIdx = countIdx; + setHasAggregator(); + } + + public int getSumIdx() { + return sumIdx; + } + + public void setSumIdx(final int sumIdx) { + this.sumIdx = sumIdx; + setHasAggregator(); + } + + public int getMaxIdx() { + return maxIdx; + } + + public void setMaxIdx(final int maxIdx) { + this.maxIdx = maxIdx; + setHasAggregator(); + } + + public int getMinIdx() { + return minIdx; + } + + public void setMinIdx(final int minIdx) { + this.minIdx = minIdx; + setHasAggregator(); + } + + public String[] getLabels() { + return labels; + } + + public void setLabels(final String[] labels) { + this.labels = labels; + } + + public boolean hasAggregatorFunction(){ + return hasAggregator; + } + + private void setHasAggregator() { + hasAggregator = true; + } + } +} http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java b/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java deleted file mode 100644 index 6ce90e6..0000000 --- a/repository/src/main/java/org/apache/atlas/query/GremlinTranslator.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.query; - -import org.apache.atlas.query.Expressions.Expression; - - -public class GremlinTranslator { - private final QueryProcessor queryProcessor; - private Expression expression; - - public GremlinTranslator(QueryProcessor queryProcessor, Expression expression) { - this.expression = expression; - this.queryProcessor = queryProcessor; - } - - public GremlinQuery translate() { - DSLVisitor qv = new DSLVisitor(queryProcessor); - - expression.accept(qv); - queryProcessor.close(); - - GremlinQuery ret = new GremlinQuery(queryProcessor.getText(), queryProcessor.hasSelect()); - return ret; - } -} http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java index 0c7e2ed..fe1c5bd 100644 --- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java +++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java @@ -25,17 +25,27 @@ import java.util.regex.Pattern; public class IdentifierHelper { - public static String stripQuotes(String quotedIdentifier) { - String ret = quotedIdentifier; + private static final Pattern SINGLE_QUOTED_IDENTIFIER = Pattern.compile("'(\\w[\\w\\d\\.\\s]*)'"); + private static final Pattern DOUBLE_QUOTED_IDENTIFIER = Pattern.compile("\"(\\w[\\w\\d\\.\\s]*)\""); + private static final Pattern BACKTICK_QUOTED_IDENTIFIER = Pattern.compile("`(\\w[\\w\\d\\.\\s]*)`"); - if (isQuoted(quotedIdentifier)) { - ret = quotedIdentifier.substring(1, quotedIdentifier.length() - 1); + public static String get(String quotedIdentifier) { + String ret; + + if (quotedIdentifier.charAt(0) == '`') { + ret = extract(BACKTICK_QUOTED_IDENTIFIER, quotedIdentifier); + } else if (quotedIdentifier.charAt(0) == '\'') { + ret = extract(SINGLE_QUOTED_IDENTIFIER, quotedIdentifier); + } else if (quotedIdentifier.charAt(0) == '"') { + ret = extract(DOUBLE_QUOTED_IDENTIFIER, quotedIdentifier); + } else { + ret = quotedIdentifier; } return ret; } - public static Advice create(QueryProcessor.Context context, + public static Advice create(GremlinQueryComposer.Context context, org.apache.atlas.query.Lookup lookup, String identifier) { Advice ia = new Advice(identifier); @@ -49,7 +59,7 @@ public class IdentifierHelper { } public static String getQualifiedName(org.apache.atlas.query.Lookup lookup, - QueryProcessor.Context context, + GremlinQueryComposer.Context context, String name) { return lookup.getQualifiedName(context, name); } @@ -70,7 +80,9 @@ public class IdentifierHelper { } public static String removeQuotes(String rhs) { - return rhs.replace("\"", "").replace("'", ""); + return rhs.replace("\"", "") + .replace("'", "") + .replace("`", ""); } public static String getQuoted(String s) { @@ -97,10 +109,10 @@ public class IdentifierHelper { public Advice(String s) { this.raw = removeQuotes(s); - this.actual = IdentifierHelper.stripQuotes(raw); + this.actual = IdentifierHelper.get(raw); } - private void update(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) { + private void update(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) { newContext = context.isEmpty(); if(!newContext) { if(context.aliasMap.containsKey(this.raw)) { @@ -116,7 +128,7 @@ public class IdentifierHelper { } } - private void updateSubTypes(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) { + private void updateSubTypes(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) { if(isTrait) { return; } @@ -127,7 +139,7 @@ public class IdentifierHelper { } } - private void updateEdgeInfo(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) { + private void updateEdgeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) { if(isPrimitive == false && isTrait == false) { edgeLabel = lookup.getRelationshipEdgeLabel(context, attributeName); edgeDirection = "OUT"; @@ -135,7 +147,7 @@ public class IdentifierHelper { } } - private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, QueryProcessor.Context context) { + private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) { if(parts.length == 1) { typeName = context.getActiveTypeName(); attributeName = parts[0]; @@ -171,7 +183,7 @@ public class IdentifierHelper { } } - private void setIsDate(Lookup lookup, QueryProcessor.Context context) { + private void setIsDate(Lookup lookup, GremlinQueryComposer.Context context) { if(isPrimitive) { isDate = lookup.isDate(context, attributeName); } http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/Lookup.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/Lookup.java b/repository/src/main/java/org/apache/atlas/query/Lookup.java index a64b688..d09b207 100644 --- a/repository/src/main/java/org/apache/atlas/query/Lookup.java +++ b/repository/src/main/java/org/apache/atlas/query/Lookup.java @@ -23,21 +23,21 @@ import org.apache.atlas.type.AtlasType; public interface Lookup { AtlasType getType(String typeName); - String getQualifiedName(QueryProcessor.Context context, String name); + String getQualifiedName(GremlinQueryComposer.Context context, String name); - boolean isPrimitive(QueryProcessor.Context context, String attributeName); + boolean isPrimitive(GremlinQueryComposer.Context context, String attributeName); - String getRelationshipEdgeLabel(QueryProcessor.Context context, String attributeName); + String getRelationshipEdgeLabel(GremlinQueryComposer.Context context, String attributeName); - boolean hasAttribute(QueryProcessor.Context context, String typeName); + boolean hasAttribute(GremlinQueryComposer.Context context, String typeName); - boolean doesTypeHaveSubTypes(QueryProcessor.Context context); + boolean doesTypeHaveSubTypes(GremlinQueryComposer.Context context); - String getTypeAndSubTypes(QueryProcessor.Context context); + String getTypeAndSubTypes(GremlinQueryComposer.Context context); - boolean isTraitType(QueryProcessor.Context context); + boolean isTraitType(GremlinQueryComposer.Context context); - String getTypeFromEdge(QueryProcessor.Context context, String item); + String getTypeFromEdge(GremlinQueryComposer.Context context, String item); - boolean isDate(QueryProcessor.Context context, String attributeName); + boolean isDate(GremlinQueryComposer.Context context, String attributeName); } http://git-wip-us.apache.org/repos/asf/atlas/blob/8db8b5c7/repository/src/main/java/org/apache/atlas/query/QueryParser.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/query/QueryParser.java b/repository/src/main/java/org/apache/atlas/query/QueryParser.java deleted file mode 100644 index 6e378a1..0000000 --- a/repository/src/main/java/org/apache/atlas/query/QueryParser.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.query; - -import org.antlr.v4.runtime.CharStreams; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.TokenStream; -import org.apache.atlas.query.Expressions.Expression; -import org.apache.atlas.query.antlr4.AtlasDSLLexer; -import org.apache.atlas.query.antlr4.AtlasDSLParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class QueryParser { - private static final Logger LOG = LoggerFactory.getLogger(QueryParser.class); - - private static final Set<String> RESERVED_KEYWORDS = - new HashSet<>(Arrays.asList("[", "]", "(", ")", "=", "<", ">", "!=", "<=", ">=", ",", "and", "or", "+", "-", - "*", "/", ".", "select", "from", "where", "groupby", "loop", "isa", "is", "has", - "as", "times", "withPath", "limit", "offset", "orderby", "count", "max", "min", - "sum", "by", "order", "like")); - - public static boolean isKeyword(String word) { - return RESERVED_KEYWORDS.contains(word); - } - - public static Expression apply(String queryStr, QueryParams params) { - Expression ret = null; - - try { - InputStream stream = new ByteArrayInputStream(queryStr.getBytes()); - AtlasDSLLexer lexer = new AtlasDSLLexer(CharStreams.fromStream(stream)); - TokenStream inputTokenStream = new CommonTokenStream(lexer); - AtlasDSLParser parser = new AtlasDSLParser(inputTokenStream); - - ret = new Expression(parser.query()); - } catch (IOException e) { - ret = null; - LOG.error(e.getMessage(), e); - } - - return ret; - } -}
