Repository: asterixdb Updated Branches: refs/heads/master 714cd44c4 -> 746ea8c72
[NO ISSUE][API] added parse-only request parameter - user model changes: parse-only request parameter has been added - storage format changes: no - interface changes: no Details: - Support for new request parameter named parse-only; Returns as result an object with single key-value pair. The key is statement-parameters and the is a sorted array containing positional and named free parameters. - Added test cases Change-Id: Idd2f461c22b05a5fcaa50a6e4f9b7dcd91acc184 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3085 Sonar-Qube: Jenkins <[email protected]> Tested-by: Till Westmann <[email protected]> Integration-Tests: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/746ea8c7 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/746ea8c7 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/746ea8c7 Branch: refs/heads/master Commit: 746ea8c729e67f00f9e1a1d22a8af860adc370e6 Parents: 714cd44 Author: sandeepgupta <[email protected]> Authored: Mon Jan 7 10:32:51 2019 +0530 Committer: Till Westmann <[email protected]> Committed: Mon Jan 7 08:10:41 2019 -0800 ---------------------------------------------------------------------- .../server/QueryServiceRequestParameters.java | 9 +++ .../api/http/server/QueryServiceServlet.java | 73 ++++++++++++++++---- .../asterix/api/http/server/ResultUtil.java | 52 ++++++++++++++ .../asterix/app/translator/QueryTranslator.java | 9 ++- .../asterix/test/common/TestExecutor.java | 13 +++- .../asterix/test/runtime/ParseOnlyTest.java | 65 +++++++++++++++++ .../parseonly/001/parseonly_01.1.query.sqlpp | 28 ++++++++ .../parseonly/001/parseonly_01.2.query.sqlpp | 54 +++++++++++++++ .../parseonly/001/parseonly_01.3.query.sqlpp | 30 ++++++++ .../parseonly/001/parseonly_01.4.query.sqlpp | 29 ++++++++ .../parseonly/001/parseonly_01.5.query.sqlpp | 34 +++++++++ .../parseonly/002/parseonly_02.1.parse.sqlpp | 28 ++++++++ .../results/parseonly/001/parseonly_01.1.adm | 1 + .../results/parseonly/001/parseonly_01.2.adm | 1 + .../results/parseonly/001/parseonly_01.3.adm | 1 + .../results/parseonly/001/parseonly_01.4.adm | 1 + .../results/parseonly/001/parseonly_01.5.adm | 1 + .../resources/runtimets/testsuite_parseonly.xml | 35 ++++++++++ .../lang/aql/rewrites/AqlQueryRewriter.java | 5 ++ .../lang/common/base/IQueryRewriter.java | 6 ++ .../lang/sqlpp/rewrites/SqlppQueryRewriter.java | 18 +++++ .../lang/sqlpp/util/SqlppVariableUtil.java | 15 +++- 22 files changed, 484 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java index 16a2105..e0af3bd 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java @@ -44,6 +44,7 @@ public class QueryServiceRequestParameters { private String planFormat; private Map<String, JsonNode> statementParams; private boolean expressionTree; + private boolean parseOnly; //don't execute; simply check for syntax correctness and named parameters. private boolean rewrittenExpressionTree; private boolean logicalPlan; private boolean optimizedLogicalPlan; @@ -171,6 +172,14 @@ public class QueryServiceRequestParameters { this.optimizedLogicalPlan = optimizedLogicalPlan; } + public void setParseOnly(boolean parseOnly) { + this.parseOnly = parseOnly; + } + + public boolean isParseOnly() { + return parseOnly; + } + public boolean isJob() { return job; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 7e563e1..fce9365 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -33,23 +33,29 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.function.BiFunction; import java.util.function.Function; import org.apache.asterix.algebra.base.ILangExtension; +import org.apache.asterix.app.translator.QueryTranslator; import org.apache.asterix.common.api.Duration; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.api.IClusterManagementWork; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.context.IStorageComponentProvider; import org.apache.asterix.common.dataflow.ICcApplicationContext; +import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.lang.aql.parser.TokenMgrError; import org.apache.asterix.lang.common.base.IParser; +import org.apache.asterix.lang.common.base.IParserFactory; import org.apache.asterix.lang.common.base.Statement; +import org.apache.asterix.lang.common.expression.VariableExpr; +import org.apache.asterix.lang.common.statement.Query; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.translator.ExecutionPlans; @@ -141,6 +147,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { REWRITTEN_EXPRESSION_TREE("rewritten-expression-tree"), LOGICAL_PLAN("logical-plan"), OPTIMIZED_LOGICAL_PLAN("optimized-logical-plan"), + PARSE_ONLY("parse-only"), JOB("job"), SIGNATURE("signature"), MULTI_STATEMENT("multi-statement"); @@ -411,6 +418,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false)); param.setRewrittenExpressionTree(getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false)); param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false)); + param.setParseOnly(getOptBoolean(jsonRequest, Parameter.PARSE_ONLY.str(), false)); param.setOptimizedLogicalPlan(getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false)); param.setJob(getOptBoolean(jsonRequest, Parameter.JOB.str(), false)); param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE.str(), true)); @@ -435,6 +443,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { param.setTimeout(request.getParameter(Parameter.TIMEOUT.str())); param.setMaxResultReads(request.getParameter(Parameter.MAX_RESULT_READS.str())); param.setPlanFormat(request.getParameter(Parameter.PLAN_FORMAT.str())); + param.setParseOnly(Boolean.parseBoolean(request.getParameter(Parameter.PARSE_ONLY.str()))); final String multiStatementParam = request.getParameter(Parameter.MULTI_STATEMENT.str()); param.setMultiStatement(multiStatementParam == null || Boolean.parseBoolean(multiStatementParam)); try { @@ -445,6 +454,14 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } } + private void setAccessControlHeaders(IServletRequest request, IServletResponse response) throws IOException { + //CORS + if (request.getHeader("Origin") != null) { + response.setHeader("Access-Control-Allow-Origin", request.getHeader("Origin")); + } + response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + } + private static ResultDelivery parseResultDelivery(String mode) { if ("async".equals(mode)) { return ResultDelivery.ASYNC; @@ -506,29 +523,36 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { LOGGER.info("handleRequest: {}", param); ResultDelivery delivery = parseResultDelivery(param.getMode()); setSessionConfig(sessionOutput, param, delivery); - ResultProperties resultProperties = param.getMaxResultReads() == null ? new ResultProperties(delivery) + final ResultProperties resultProperties = param.getMaxResultReads() == null ? new ResultProperties(delivery) : new ResultProperties(delivery, Long.parseLong(param.getMaxResultReads())); printAdditionalResultFields(sessionOutput.out()); printRequestId(sessionOutput.out()); printClientContextID(sessionOutput.out(), param); - printSignature(sessionOutput.out(), param); + if (!param.isParseOnly()) { + printSignature(sessionOutput.out(), param); + } printType(sessionOutput.out(), sessionOutput.config()); if (param.getStatement() == null || param.getStatement().isEmpty()) { throw new RuntimeDataException(ErrorCode.NO_STATEMENT_PROVIDED); } String statementsText = param.getStatement() + ";"; - Map<String, byte[]> statementParams = org.apache.asterix.app.translator.RequestParameters - .serializeParameterValues(param.getStatementParams()); - // CORS - if (request.getHeader("Origin") != null) { - response.setHeader("Access-Control-Allow-Origin", request.getHeader("Origin")); - } - response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - response.setStatus(execution.getHttpStatus()); - executeStatement(statementsText, sessionOutput, resultProperties, stats, param, execution, optionalParams, - statementParams); - if (ResultDelivery.IMMEDIATE == delivery || ResultDelivery.DEFERRED == delivery) { + if (param.isParseOnly()) { + ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statementsText); + setAccessControlHeaders(request, response); + response.setStatus(HttpResponseStatus.OK); + printParseOnlyValueResult(sessionOutput, parseOnlyResult); ResultUtil.printStatus(sessionOutput, execution.getResultStatus()); + } else { + Map<String, byte[]> statementParams = org.apache.asterix.app.translator.RequestParameters + .serializeParameterValues(param.getStatementParams()); + + setAccessControlHeaders(request, response); + response.setStatus(execution.getHttpStatus()); + executeStatement(statementsText, sessionOutput, resultProperties, stats, param, execution, + optionalParams, statementParams); + if (ResultDelivery.IMMEDIATE == delivery || ResultDelivery.DEFERRED == delivery) { + ResultUtil.printStatus(sessionOutput, execution.getResultStatus()); + } } if (!warnings.isEmpty()) { printWarnings(sessionOutput.out(), warnings); @@ -553,6 +577,18 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } } + protected ResultUtil.ParseOnlyResult parseStatement(String statementsText) throws CompilationException { + IParserFactory factory = compilationProvider.getParserFactory(); + IParser parser = factory.createParser(statementsText); + List<Statement> stmts = parser.parse(); + QueryTranslator.validateStatements(stmts); + Query query = (Query) stmts.get(stmts.size() - 1); + Set<VariableExpr> extVars = + compilationProvider.getRewriterFactory().createQueryRewriter().getExternalVariables(query.getBody()); + ResultUtil.ParseOnlyResult parseOnlyResult = new ResultUtil.ParseOnlyResult(extVars); + return parseOnlyResult; + } + protected void executeStatement(String statementsText, SessionOutput sessionOutput, ResultProperties resultProperties, Stats stats, QueryServiceRequestParameters param, RequestExecutionState execution, Map<String, String> optionalParameters, @@ -646,10 +682,19 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { // do nothing } - private void printWarnings(PrintWriter pw, List<ExecutionWarning> warnings) { + protected void printWarnings(PrintWriter pw, List<ExecutionWarning> warnings) { ResultUtil.printWarnings(pw, warnings); } + protected void printParseOnlyValueResult(SessionOutput output, ResultUtil.ParseOnlyResult parseOnlyResult) { + final PrintWriter pw = output.out(); + pw.print("\t\""); + pw.print(ResultFields.RESULTS.str()); //TODO: use ResultUtil, ResultPrinter + pw.print("\":"); + pw.print(parseOnlyResult.asJson()); + pw.print(",\n"); + } + protected void printExecutionPlans(SessionOutput output, ExecutionPlans executionPlans) { final PrintWriter pw = output.out(); pw.print("\t\""); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java index 8824f6a..fa3c03d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java @@ -24,9 +24,11 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -35,6 +37,8 @@ import org.apache.asterix.app.result.ResultPrinter; import org.apache.asterix.app.result.ResultReader; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.lang.aql.parser.TokenMgrError; +import org.apache.asterix.lang.common.expression.VariableExpr; +import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.translator.IStatementExecutor.Stats; import org.apache.asterix.translator.SessionOutput; @@ -379,4 +383,52 @@ public class ResultUtil { return (app, status) -> app.append("\t\"").append(AbstractQueryApiServlet.ResultFields.STATUS.str()) .append("\": \"").append(status).append("\",\n"); } + + public static class ParseOnlyResult { + private Set<VariableExpr> externalVariables; + + private static final String STMT_PARAM_LBL = "statement-parameters"; + + public ParseOnlyResult(Set<VariableExpr> extVars) { + this.externalVariables = extVars; + } + + public String asJson() { + + ArrayList<String> positionalVars = new ArrayList<>(); + ArrayList<String> namedVars = new ArrayList<>(); + + for (VariableExpr extVarRef : externalVariables) { + String varname = extVarRef.getVar().getValue(); + if (SqlppVariableUtil.isPositionalVariableIdentifier(extVarRef.getVar())) { + positionalVars.add(SqlppVariableUtil.toUserDefinedName(varname)); + } else { + namedVars.add(SqlppVariableUtil.toUserDefinedName(varname)); + } + } + Collections.sort(positionalVars); + Collections.sort(namedVars); + final StringBuilder output = new StringBuilder(); + output.append("{\"").append(STMT_PARAM_LBL).append("\":["); + boolean first = true; + for (String posVar : positionalVars) { + if (first) { + first = false; + } else { + output.append(","); + } + output.append(posVar); + } + for (String namedVar : namedVars) { + if (first) { + first = false; + } else { + output.append(","); + } + output.append("\"").append(namedVar).append("\""); + } + output.append("]}"); + return output.toString(); + } + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index 9139d14..64a3d51 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -65,7 +65,6 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.MetadataException; -import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.common.functions.FunctionSignature; import org.apache.asterix.common.utils.JobUtils; import org.apache.asterix.common.utils.JobUtils.ProgressState; @@ -2932,13 +2931,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } } - protected void validateStatements(List<Statement> statements) throws RuntimeDataException { - if (statements.stream().filter(this::isNotAllowedMultiStatement).count() > 1) { - throw new RuntimeDataException(ErrorCode.UNSUPPORTED_MULTIPLE_STATEMENTS); + public static void validateStatements(List<Statement> statements) throws CompilationException { + if (statements.stream().filter(QueryTranslator::isNotAllowedMultiStatement).count() > 1) { + throw new CompilationException(ErrorCode.UNSUPPORTED_MULTIPLE_STATEMENTS); } } - protected boolean isNotAllowedMultiStatement(Statement statement) { + protected static boolean isNotAllowedMultiStatement(Statement statement) { switch (statement.getKind()) { case DATAVERSE_DECL: case FUNCTION_DECL: http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index bd7510f..3aa8807 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -132,7 +132,7 @@ public class TestExecutor { private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)"); private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)"); private static final Pattern HTTP_PARAM_PATTERN = - Pattern.compile("param (?<name>[\\w$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE); + Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE); private static final Pattern HTTP_BODY_PATTERN = Pattern.compile("body=(.*)", Pattern.MULTILINE); private static final Pattern HTTP_STATUSCODE_PATTERN = Pattern.compile("statuscode (.*)", Pattern.MULTILINE); private static final Pattern MAX_RESULT_READS_PATTERN = @@ -890,6 +890,7 @@ public class TestExecutor { break; case "query": case "async": + case "parse": case "deferred": case "metrics": // isDmlRecoveryTest: insert Crash and Recovery @@ -1220,8 +1221,14 @@ public class TestExecutor { if (DELIVERY_IMMEDIATE.equals(delivery)) { resultStream = executeQueryService(statement, fmt, uri, params, isJsonEncoded, null, isCancellable(reqType)); - resultStream = METRICS_QUERY_TYPE.equals(reqType) ? ResultExtractor.extractMetrics(resultStream) - : ResultExtractor.extract(resultStream); + switch (reqType) { + case METRICS_QUERY_TYPE: + resultStream = ResultExtractor.extractMetrics(resultStream); + break; + default: + resultStream = ResultExtractor.extract(resultStream); + break; + } } else { String handleVar = getHandleVariable(statement); resultStream = executeQueryService(statement, fmt, uri, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ParseOnlyTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ParseOnlyTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ParseOnlyTest.java new file mode 100644 index 0000000..92edf47 --- /dev/null +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ParseOnlyTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.runtime; + +import java.util.Collection; + +import org.apache.asterix.test.common.TestExecutor; + +import org.apache.asterix.testframework.context.TestCaseContext; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@RunWith(Parameterized.class) +public class ParseOnlyTest { + + protected static final String TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf"; + + @BeforeClass + public static void setUp() throws Exception { + LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, new TestExecutor()); + } + + @AfterClass + public static void tearDown() throws Exception { + LangExecutionUtil.tearDown(); + } + + @Parameters(name = "ParseOnlyTest {index}: {0}") + public static Collection<Object[]> tests() throws Exception { + return LangExecutionUtil.tests("only.xml", "testsuite_parseonly.xml"); + } + + protected TestCaseContext tcCtx; + + public ParseOnlyTest(TestCaseContext tcCtx) { + this.tcCtx = tcCtx; + } + + @Test + public void test() throws Exception { + LangExecutionUtil.test(tcCtx); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.1.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.1.query.sqlpp new file mode 100644 index 0000000..e24410f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.1.query.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Success + * Date : Jun 2018 + */ + +-- param parse-only:string=true + +select count(*) from ChirpUsers where name=$qname; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.2.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.2.query.sqlpp new file mode 100644 index 0000000..ce0bf45 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.2.query.sqlpp @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Success + * Date : Jun 2018 + */ + +-- param parse-only:string=true + +// requesttype=application/json + +{ + "t1": { + "p_null": $p_null, + "p_bool": $p_bool, + "p_int": $p_int, + "p_dec": $p_dec, + "p_dbl": $p_dbl, + "p_str": $p_str, + "p_arr": $p_arr, + "p_obj": $p_obj + }, + + "t2": { + "p_null_type": $p_null is null, + "p_bool_type": is_boolean($p_bool), + "p_int_type": is_number($p_int), + "p_dec_type": is_number($p_dec), + "p_dbl_type": is_number($p_dbl), + "p_str_type": is_string($p_str), + "p_arr_type": is_array($p_arr), + "p_obj_type": is_object($p_obj) + }, + + "t3": [ $p_null, $p_bool, $p_int, $p_dec, $p_dbl, $p_str, $p_arr, $p_obj ] +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.3.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.3.query.sqlpp new file mode 100644 index 0000000..3730708 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.3.query.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Success + * Date : Jun 2018 + */ + +-- param parse-only:string=true + + +select count(*) from ChirpUsers where field1=$qname and field2=$val and field3=$1; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.4.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.4.query.sqlpp new file mode 100644 index 0000000..2a25662 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.4.query.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Success + * Date : Jun 2018 + */ + +-- param parse-only:string=true + + +select $p_int, $p_str http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.5.query.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.5.query.sqlpp new file mode 100644 index 0000000..9975f8b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/001/parseonly_01.5.query.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Success + * Date : Jun 2018 + */ + +-- param parse-only:string=true + +// requesttype=application/json + + +{ + "t1": $p_int + ? + $1, + "t2": $p_str || ? || $2 +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/002/parseonly_02.1.parse.sqlpp ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/002/parseonly_02.1.parse.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/002/parseonly_02.1.parse.sqlpp new file mode 100644 index 0000000..13b9de5 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/parseonly/002/parseonly_02.1.parse.sqlpp @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Description : Test named statement parameters with json encoded request + * Expected Res : Failure + * Date : Jun 2018 + */ + +-- param parse-only:string=true + +select count(*) from ChirpUsers name=$qname" http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.1.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.1.adm new file mode 100644 index 0000000..70522ea --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.1.adm @@ -0,0 +1 @@ +{"statement-parameters":["qname"]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.2.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.2.adm new file mode 100644 index 0000000..3f0a194 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.2.adm @@ -0,0 +1 @@ +{"statement-parameters":["p_arr","p_bool","p_dbl","p_dec","p_int","p_null","p_obj","p_str"]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.3.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.3.adm new file mode 100644 index 0000000..d2b0d34 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.3.adm @@ -0,0 +1 @@ +{"statement-parameters":[1,"qname","val"]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.4.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.4.adm new file mode 100644 index 0000000..782b0af --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.4.adm @@ -0,0 +1 @@ +{"statement-parameters":["p_int","p_str"]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.5.adm ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.5.adm new file mode 100644 index 0000000..c7715a1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/parseonly/001/parseonly_01.5.adm @@ -0,0 +1 @@ +{"statement-parameters":[1,2,"p_int","p_str"]} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_parseonly.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_parseonly.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_parseonly.xml new file mode 100644 index 0000000..f7cea01 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_parseonly.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ! Licensed to the Apache Software Foundation (ASF) under one + ! or more contributor license agreements. See the NOTICE file + ! distributed with this work for additional information + ! regarding copyright ownership. The ASF licenses this file + ! to you under the Apache License, Version 2.0 (the + ! "License"); you may not use this file except in compliance + ! with the License. You may obtain a copy of the License at + ! + ! http://www.apache.org/licenses/LICENSE-2.0 + ! + ! Unless required by applicable law or agreed to in writing, + ! software distributed under the License is distributed on an + ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ! KIND, either express or implied. See the License for the + ! specific language governing permissions and limitations + ! under the License. + !--> +<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries" + QueryFileExtension=".sqlpp"> + <test-group name="parseonly"> + <test-case FilePath="parseonly"> + <compilation-unit name="001"> + <output-dir compare="Text">001</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="parseonly"> + <compilation-unit name="002"> + <output-dir compare="Text">named_02</output-dir> + <expected-error>ASX1001</expected-error> + </compilation-unit> + </test-case> + </test-group> +</test-suite> http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java index 61cecfb..9aaf5b7 100644 --- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java +++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java @@ -143,6 +143,11 @@ class AqlQueryRewriter implements IQueryRewriter { return gfc.getCalls(); } + @Override + public Set<VariableExpr> getExternalVariables(Expression expr) { + throw new UnsupportedOperationException("getExternalVariables not implemented for AQL"); + } + private static class GatherFunctionCalls extends GatherFunctionCallsVisitor implements IAQLVisitor<Void, Void> { public GatherFunctionCalls() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java index 8f3b8a9..7500ab9 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IQueryRewriter.java @@ -24,6 +24,7 @@ import java.util.Set; import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.struct.VarIdentifier; @@ -53,4 +54,9 @@ public interface IQueryRewriter { */ Set<CallExpr> getFunctionCalls(Expression expression) throws CompilationException; + /** + * Find all external variables (positional and named variables) in given expression + */ + Set<VariableExpr> getExternalVariables(Expression expr) throws CompilationException; + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java index 69a3e5e..942fc88 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.HashSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -33,6 +34,7 @@ import org.apache.asterix.lang.common.base.IQueryRewriter; import org.apache.asterix.lang.common.base.IReturningStatement; import org.apache.asterix.lang.common.clause.LetClause; import org.apache.asterix.lang.common.expression.CallExpr; +import org.apache.asterix.lang.common.expression.VariableExpr; import org.apache.asterix.lang.common.rewrites.LangRewritingContext; import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.struct.Identifier; @@ -77,6 +79,9 @@ import org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVis import org.apache.asterix.lang.sqlpp.struct.SetOperationRight; import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil; import org.apache.asterix.lang.sqlpp.util.SqlppAstPrintUtil; +import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil; +import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor; +import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor; import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -297,6 +302,19 @@ public class SqlppQueryRewriter implements IQueryRewriter { return gfc.getCalls(); } + @Override + public Set<VariableExpr> getExternalVariables(Expression expr) throws CompilationException { + Set<VariableExpr> freeVars = SqlppVariableUtil.getFreeVariables(expr); + + Set<VariableExpr> extVars = new HashSet<>(); + for (VariableExpr ve : freeVars) { + if (SqlppVariableUtil.isExternalVariableReference(ve)) { + extVars.add(ve); + } + } + return extVars; + } + private static class GatherFunctionCalls extends GatherFunctionCallsVisitor implements ISqlppVisitor<Void, Void> { public GatherFunctionCalls() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/746ea8c7/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java index 8fabf13..f71ae5b 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java +++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/SqlppVariableUtil.java @@ -19,7 +19,6 @@ package org.apache.asterix.lang.sqlpp.util; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -39,7 +38,6 @@ import org.apache.asterix.lang.common.struct.VarIdentifier; import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause; import org.apache.asterix.lang.sqlpp.clause.FromClause; import org.apache.asterix.lang.sqlpp.clause.FromTerm; -import org.apache.asterix.lang.sqlpp.clause.SelectBlock; import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -80,6 +78,10 @@ public class SqlppVariableUtil { if (varName.startsWith(USER_VAR_PREFIX)) { return varName.substring(1); } + if (varName.startsWith(EXTERNAL_VAR_PREFIX)) { + return varName.substring(1); + } + return varName; } @@ -95,6 +97,15 @@ public class SqlppVariableUtil { return EXTERNAL_VAR_PREFIX + varName; } + public static boolean isPositionalVariableIdentifier(VarIdentifier varId) { + try { + Integer.parseInt(toUserDefinedName(varId.getValue())); + return true; + } catch (NumberFormatException ignored) { + return false; + } + } + public static boolean isExternalVariableIdentifier(VarIdentifier varId) { return varId.getValue().startsWith(EXTERNAL_VAR_PREFIX); }
