This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit c77ab0dc75b6b9affd2a0f81b1514a0bd83a38ea Author: Till Westmann <[email protected]> AuthorDate: Sun Jun 7 17:37:13 2020 -0700 [NO ISSUE][API][SQL] retrieve request and statement information - user model changes: no - storage format changes: no - interface changes: no details: - Get user agent and remote address from request reference. - Extract statement properties that are determined during parsing. Change-Id: I26dab3ccee21293963496f38abe44b9a653a35e4 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5783 Tested-by: Michael Blow <[email protected]> Reviewed-by: Michael Blow <[email protected]> Integration-Tests: Jenkins <[email protected]> --- .../algebra/extension/ExtensionStatement.java | 2 + .../asterix/translator/IRequestParameters.java | 7 ++ .../asterix/translator/IStatementExecutor.java | 33 +++++++ .../apache/asterix/api/http/server/ApiServlet.java | 2 +- .../api/http/server/NCQueryServiceServlet.java | 27 ++++-- .../http/server/QueryServiceRequestParameters.java | 10 ++ .../api/http/server/QueryServiceServlet.java | 106 +++++++++++++-------- .../apache/asterix/api/java/AsterixJavaClient.java | 4 +- .../message/ExecuteStatementRequestMessage.java | 7 +- .../message/ExecuteStatementResponseMessage.java | 10 ++ .../asterix/app/translator/QueryTranslator.java | 12 ++- .../asterix/app/translator/RequestParameters.java | 27 ++++-- .../http/servlet/QueryCancellationServletTest.java | 8 +- .../asterix/test/common/ResultExtractor.java | 6 +- .../apache/asterix/test/common/TestExecutor.java | 1 - .../asterix/common/api/IRequestReference.java | 14 +++ 16 files changed, 203 insertions(+), 73 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java index 15267aa..9507e16 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionStatement.java @@ -36,6 +36,8 @@ public abstract class ExtensionStatement extends AbstractStatement { return Kind.EXTENSION; } + public abstract String getName(); + /** * Called when the {@code IStatementExecutor} encounters an extension statement. * An implementation class should implement the actual processing of the statement in this method. diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java index e242258..417a130 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.asterix.common.api.ICommonRequestParameters; import org.apache.asterix.om.base.IAObject; +import org.apache.asterix.translator.IStatementExecutor.StatementProperties; import org.apache.asterix.translator.IStatementExecutor.Stats; import org.apache.hyracks.api.result.IResultSet; @@ -45,6 +46,12 @@ public interface IRequestParameters extends ICommonRequestParameters { Stats getStats(); /** + * @return a reference on which to write properties of executed queries (e.g. what kind of statement was parsed + * by the parser) + */ + StatementProperties getStatementProperties(); + + /** * @return a reference to write the metadata of executed queries */ IStatementExecutor.ResultMetadata getOutMetadata(); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java index e7fec0c..5911c1e 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java @@ -36,6 +36,7 @@ import org.apache.asterix.common.api.IResponsePrinter; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.lang.common.base.IStatementRewriter; +import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.statement.Query; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.om.base.IAObject; @@ -216,6 +217,38 @@ public interface IStatementExecutor { } } + class StatementProperties implements Serializable { + private static final long serialVersionUID = -1L; + + private Statement.Kind kind; + private String name; + + public Statement.Kind getKind() { + return kind; + } + + public void setKind(Statement.Kind kind) { + this.kind = kind; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean isValid() { + return kind != null && (kind != Statement.Kind.EXTENSION || name != null); + } + + @Override + public String toString() { + return Statement.Kind.EXTENSION == kind ? String.valueOf(name) : String.valueOf(kind); + } + } + /** * Compiles and executes a list of statements * diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java index b6ac3f1..47e897d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java @@ -155,7 +155,7 @@ public class ApiServlet extends AbstractServlet { long startTime = System.currentTimeMillis(); final IRequestParameters requestParameters = new RequestParameters(requestReference, query, resultSet, new ResultProperties(IStatementExecutor.ResultDelivery.IMMEDIATE), new IStatementExecutor.Stats(), - null, null, null, null, true); + new IStatementExecutor.StatementProperties(), null, null, null, null, true); translator.compileAndExecute(hcc, requestParameters); long endTime = System.currentTimeMillis(); duration = (endTime - startTime) / 1000.00; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java index 6953d1f..1b2b00d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java @@ -69,8 +69,9 @@ public class NCQueryServiceServlet extends QueryServiceServlet { @Override protected void executeStatement(IRequestReference requestReference, String statementsText, - SessionOutput sessionOutput, ResultProperties resultProperties, IStatementExecutor.Stats stats, - QueryServiceRequestParameters param, RequestExecutionState execution, + SessionOutput sessionOutput, ResultProperties resultProperties, + IStatementExecutor.StatementProperties statementProperties, IStatementExecutor.Stats stats, + QueryServiceRequestParameters param, RequestExecutionState executionState, Map<String, String> optionalParameters, Map<String, byte[]> statementParameters, ResponsePrinter responsePrinter, List<Warning> warnings) throws Exception { // Running on NC -> send 'execute' message to CC @@ -89,7 +90,7 @@ public class NCQueryServiceServlet extends QueryServiceServlet { resultProperties.getNcToCcResultProperties(), param.getClientContextID(), handleUrl, optionalParameters, statementParameters, param.isMultiStatement(), param.getProfileType(), stmtCategoryRestrictionMask, requestReference); - execution.start(); + executionState.start(); ncMb.sendMessageToPrimaryCC(requestMsg); try { responseMsg = (ExecuteStatementResponseMessage) responseFuture.get(timeout, TimeUnit.MILLISECONDS); @@ -103,11 +104,12 @@ public class NCQueryServiceServlet extends QueryServiceServlet { cancelQuery(ncMb, ncCtx.getNodeId(), requestReference.getUuid(), param.getClientContextID(), hde, true); throw hde; } - execution.end(); + executionState.end(); } finally { ncMb.deregisterMessageFuture(responseFuture.getFutureId()); } + updatePropertiesFromCC(statementProperties, responseMsg); Throwable err = responseMsg.getError(); if (err != null) { if (err instanceof Error) { @@ -118,6 +120,8 @@ public class NCQueryServiceServlet extends QueryServiceServlet { throw new Exception(err.toString(), err); } } + // if the was no error, we can set the result status to success + executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK); updateStatsFromCC(stats, responseMsg); if (hasResult(responseMsg)) { responsePrinter.addResultPrinter( @@ -151,14 +155,14 @@ public class NCQueryServiceServlet extends QueryServiceServlet { } @Override - protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, + protected void handleExecuteStatementException(Throwable t, RequestExecutionState executionState, QueryServiceRequestParameters param) { if (t instanceof TimeoutException // TODO(mblow): I don't think t can ever been an instance of TimeoutException || ExceptionUtils.matchingCause(t, candidate -> candidate instanceof IPCException)) { GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, t.toString(), t); - state.setStatus(ResultStatus.FAILED, HttpResponseStatus.SERVICE_UNAVAILABLE); + executionState.setStatus(ResultStatus.FAILED, HttpResponseStatus.SERVICE_UNAVAILABLE); } else { - super.handleExecuteStatementException(t, state, param); + super.handleExecuteStatementException(t, executionState, param); } } @@ -177,4 +181,13 @@ public class NCQueryServiceServlet extends QueryServiceServlet { stats.setProcessedObjects(responseStats.getProcessedObjects()); stats.updateTotalWarningsCount(responseStats.getTotalWarningsCount()); } + + private static void updatePropertiesFromCC(IStatementExecutor.StatementProperties statementProperties, + ExecuteStatementResponseMessage responseMsg) { + IStatementExecutor.StatementProperties responseStmtProps = responseMsg.getStatementProperties(); + if (responseStmtProps != null) { + statementProperties.setKind(responseStmtProps.getKind()); + statementProperties.setName(responseStmtProps.getName()); + } + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java index 6e70d27..2379e69 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java @@ -117,6 +117,7 @@ public class QueryServiceRequestParameters { private ResultDelivery mode = ResultDelivery.IMMEDIATE; private PlanFormat planFormat = PlanFormat.JSON; private ProfileType profileType = ProfileType.COUNTS; + private Map<String, String> optionalParams = null; private Map<String, JsonNode> statementParams = null; private boolean pretty = false; private boolean expressionTree = false; @@ -220,6 +221,14 @@ public class QueryServiceRequestParameters { this.planFormat = planFormat; } + public Map<String, String> getOptionalParams() { + return optionalParams; + } + + public void setOptionalParams(Map<String, String> optionalParams) { + this.optionalParams = optionalParams; + } + public Map<String, JsonNode> getStatementParams() { return statementParams; } @@ -367,6 +376,7 @@ public class QueryServiceRequestParameters { setHost(servlet.host(request)); setPath(servlet.servletPath(request)); String contentType = HttpUtil.getContentTypeOnly(request); + setOptionalParams(optionalParams); try { if (HttpUtil.ContentType.APPLICATION_JSON.equals(contentType)) { setParamFromJSON(request, optionalParams); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 319d959..cb1d6cf 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -88,6 +88,7 @@ import org.apache.asterix.translator.SessionConfig; import org.apache.asterix.translator.SessionOutput; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.application.IServiceContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.HyracksException; import org.apache.hyracks.api.exceptions.Warning; import org.apache.hyracks.control.common.controllers.CCConfig; @@ -152,18 +153,18 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { response.setStatus(HttpResponseStatus.OK); } - protected static final class RequestExecutionState { + protected static class RequestExecutionState { private long execStart = -1; private long execEnd = -1; - private ResultStatus resultStatus = ResultStatus.SUCCESS; - private HttpResponseStatus httpResponseStatus = HttpResponseStatus.OK; + private ResultStatus resultStatus = ResultStatus.FATAL; + private HttpResponseStatus httpResponseStatus = HttpResponseStatus.INTERNAL_SERVER_ERROR; public void setStatus(ResultStatus resultStatus, HttpResponseStatus httpResponseStatus) { this.resultStatus = resultStatus; this.httpResponseStatus = httpResponseStatus; } - ResultStatus getResultStatus() { + public ResultStatus getResultStatus() { return resultStatus; } @@ -187,9 +188,19 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } } - long duration() { + public long duration() { return execEnd - execStart; } + + protected StringBuilder append(StringBuilder sb) { + return sb.append("ResultStatus: ").append(resultStatus.str()).append(" HTTPStatus: ") + .append(String.valueOf(httpResponseStatus)); + } + + @Override + public String toString() { + return append(new StringBuilder()).toString(); + } } private static SessionOutput createSessionOutput(PrintWriter resultWriter) { @@ -201,7 +212,12 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } protected void setRequestParam(IServletRequest request, QueryServiceRequestParameters param, - Map<String, String> optionalParams) throws IOException, AlgebricksException { + Function<IServletRequest, Map<String, String>> optionalParamProvider, RequestExecutionState executionState) + throws IOException, AlgebricksException { + Map<String, String> optionalParams = null; + if (optionalParamProvider != null) { + optionalParams = optionalParamProvider.apply(request); + } param.setParameters(this, request, optionalParams); } @@ -248,22 +264,18 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { long elapsedStart = System.nanoTime(); long errorCount = 1; Stats stats = new Stats(); - RequestExecutionState execution = new RequestExecutionState(); List<Warning> warnings = new ArrayList<>(); Charset resultCharset = HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request); PrintWriter httpWriter = response.writer(); SessionOutput sessionOutput = createSessionOutput(httpWriter); - QueryServiceRequestParameters param = newRequestParameters(); ResponsePrinter responsePrinter = new ResponsePrinter(sessionOutput); ResultDelivery delivery = ResultDelivery.IMMEDIATE; + QueryServiceRequestParameters param = newRequestParameters(); + RequestExecutionState executionState = newRequestExecutionState(); try { // buffer the output until we are ready to set the status of the response message correctly responsePrinter.begin(); - Map<String, String> optionalParams = null; - if (optionalParamProvider != null) { - optionalParams = optionalParamProvider.apply(request); - } - setRequestParam(request, param, optionalParams); + setRequestParam(request, param, optionalParamProvider, executionState); if (forceReadOnly) { param.setReadOnly(true); } @@ -278,27 +290,32 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { if (param.isParseOnly()) { ResultUtil.ParseOnlyResult parseOnlyResult = parseStatement(statementsText); setAccessControlHeaders(request, response); - response.setStatus(execution.getHttpStatus()); + executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK); + response.setStatus(executionState.getHttpStatus()); responsePrinter.addResultPrinter(new ParseOnlyResultPrinter(parseOnlyResult)); } else { Map<String, byte[]> statementParams = org.apache.asterix.app.translator.RequestParameters .serializeParameterValues(param.getStatementParams()); setAccessControlHeaders(request, response); - response.setStatus(execution.getHttpStatus()); stats.setProfileType(param.getProfileType()); - executeStatement(requestRef, statementsText, sessionOutput, resultProperties, stats, param, execution, - optionalParams, statementParams, responsePrinter, warnings); + IStatementExecutor.StatementProperties statementProperties = + new IStatementExecutor.StatementProperties(); + response.setStatus(HttpResponseStatus.OK); + executeStatement(requestRef, statementsText, sessionOutput, resultProperties, statementProperties, + stats, param, executionState, param.getOptionalParams(), statementParams, responsePrinter, + warnings); + executionState.setStatus(ResultStatus.SUCCESS, HttpResponseStatus.OK); } errorCount = 0; } catch (Exception | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError e) { - handleExecuteStatementException(e, execution, param); - response.setStatus(execution.getHttpStatus()); + handleExecuteStatementException(e, executionState, param); + response.setStatus(executionState.getHttpStatus()); requestFailed(e, responsePrinter); } finally { - execution.finish(); + executionState.finish(); } responsePrinter.printResults(); - buildResponseFooters(elapsedStart, errorCount, stats, execution, resultCharset, responsePrinter, delivery); + buildResponseFooters(elapsedStart, errorCount, stats, executionState, resultCharset, responsePrinter, delivery); responsePrinter.printFooters(); responsePrinter.end(); if (sessionOutput.out().checkError()) { @@ -306,6 +323,10 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } } + protected RequestExecutionState newRequestExecutionState() throws HyracksDataException { + return new RequestExecutionState(); + } + protected void buildResponseHeaders(IRequestReference requestRef, SessionOutput sessionOutput, QueryServiceRequestParameters param, ResponsePrinter responsePrinter, ResultDelivery delivery) { responsePrinter.addHeaderPrinter(new RequestIdPrinter(requestRef.getUuid())); @@ -322,7 +343,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } protected void buildResponseResults(ResponsePrinter responsePrinter, SessionOutput sessionOutput, - ExecutionPlans plans, List<Warning> warnings) { + ExecutionPlans plans, List<Warning> warnings) throws HyracksDataException { responsePrinter.addResultPrinter(new PlansPrinter(plans, sessionOutput.config().getPlanFormat())); if (!warnings.isEmpty()) { List<ICodedMessage> codedWarnings = new ArrayList<>(); @@ -331,20 +352,21 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } } - protected void buildResponseFooters(long elapsedStart, long errorCount, Stats stats, - RequestExecutionState execution, Charset resultCharset, ResponsePrinter responsePrinter, + protected ResponseMetrics buildResponseFooters(long elapsedStart, long errorCount, Stats stats, + RequestExecutionState executionState, Charset resultCharset, ResponsePrinter responsePrinter, ResultDelivery delivery) { if (ResultDelivery.ASYNC != delivery) { // in case of ASYNC delivery, the status is printed by query translator - responsePrinter.addFooterPrinter(new StatusPrinter(execution.getResultStatus())); + responsePrinter.addFooterPrinter(new StatusPrinter(executionState.getResultStatus())); } final ResponseMetrics metrics = - ResponseMetrics.of(System.nanoTime() - elapsedStart, execution.duration(), stats.getCount(), + ResponseMetrics.of(System.nanoTime() - elapsedStart, executionState.duration(), stats.getCount(), stats.getSize(), stats.getProcessedObjects(), errorCount, stats.getTotalWarningsCount()); responsePrinter.addFooterPrinter(new MetricsPrinter(metrics, resultCharset)); if (isPrintingProfile(stats)) { responsePrinter.addFooterPrinter(new ProfilePrinter(stats.getJobProfile())); } + return metrics; } protected void validateStatement(String statement) throws RuntimeDataException { @@ -365,8 +387,9 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { } protected void executeStatement(IRequestReference requestReference, String statementsText, - SessionOutput sessionOutput, ResultProperties resultProperties, Stats stats, - QueryServiceRequestParameters param, RequestExecutionState execution, + SessionOutput sessionOutput, ResultProperties resultProperties, + IStatementExecutor.StatementProperties statementProperties, Stats stats, + QueryServiceRequestParameters param, RequestExecutionState executionState, Map<String, String> optionalParameters, Map<String, byte[]> statementParameters, ResponsePrinter responsePrinter, List<Warning> warnings) throws Exception { IClusterManagementWork.ClusterState clusterState = @@ -383,22 +406,23 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { MetadataManager.INSTANCE.init(); IStatementExecutor translator = statementExecutorFactory.create((ICcApplicationContext) appCtx, statements, sessionOutput, compilationProvider, componentProvider, responsePrinter); - execution.start(); + executionState.start(); Map<String, IAObject> stmtParams = org.apache.asterix.app.translator.RequestParameters.deserializeParameterValues(statementParameters); int stmtCategoryRestriction = org.apache.asterix.app.translator.RequestParameters .getStatementCategoryRestrictionMask(param.isReadOnly()); - IRequestParameters requestParameters = new org.apache.asterix.app.translator.RequestParameters(requestReference, - statementsText, getResultSet(), resultProperties, stats, null, param.getClientContextID(), - optionalParameters, stmtParams, param.isMultiStatement(), stmtCategoryRestriction); + IRequestParameters requestParameters = + new org.apache.asterix.app.translator.RequestParameters(requestReference, statementsText, + getResultSet(), resultProperties, stats, statementProperties, null, param.getClientContextID(), + optionalParameters, stmtParams, param.isMultiStatement(), stmtCategoryRestriction); translator.compileAndExecute(getHyracksClientConnection(), requestParameters); - execution.end(); + executionState.end(); translator.getWarnings(warnings, maxWarnings - warnings.size()); stats.updateTotalWarningsCount(parserTotalWarningsCount); buildResponseResults(responsePrinter, sessionOutput, translator.getExecutionPlans(), warnings); } - protected void handleExecuteStatementException(Throwable t, RequestExecutionState state, + protected void handleExecuteStatementException(Throwable t, RequestExecutionState executionState, QueryServiceRequestParameters param) { if (t instanceof org.apache.asterix.aqlplus.parser.TokenMgrError || t instanceof TokenMgrError || t instanceof AlgebricksException) { @@ -408,37 +432,37 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + LogRedactionUtil.userData(param.toString())); } - state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); + executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); } else if (t instanceof HyracksException) { HyracksException he = (HyracksException) t; switch (he.getComponent() + he.getErrorCode()) { case ASTERIX + REQUEST_TIMEOUT: LOGGER.info(() -> "handleException: request execution timed out: " + LogRedactionUtil.userData(param.toString())); - state.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK); + executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK); break; case ASTERIX + REJECT_BAD_CLUSTER_STATE: case ASTERIX + REJECT_NODE_UNREGISTERED: LOGGER.warn(() -> "handleException: " + he.getMessage() + ": " + LogRedactionUtil.userData(param.toString())); - state.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE); + executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE); break; case ASTERIX + INVALID_REQ_PARAM_VAL: case ASTERIX + INVALID_REQ_JSON_VAL: case ASTERIX + NO_STATEMENT_PROVIDED: case HYRACKS + JOB_REQUIREMENTS_EXCEED_CAPACITY: - state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); + executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST); break; default: LOGGER.warn(() -> "handleException: unexpected exception " + he.getMessage() + ": " + LogRedactionUtil.userData(param.toString()), he); - state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); + executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); break; } } else { LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), t); - state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); + executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR); } } @@ -473,7 +497,7 @@ public class QueryServiceServlet extends AbstractQueryApiServlet { return new QueryServiceRequestParameters(); } - private static boolean isPrintingProfile(IStatementExecutor.Stats stats) { + protected static boolean isPrintingProfile(IStatementExecutor.Stats stats) { return stats.getProfileType() == Stats.ProfileType.FULL && stats.getJobProfile() != null; } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java index 1cdf6f7..4fe6582 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java @@ -136,8 +136,8 @@ public class AsterixJavaClient { final RequestReference requestReference = RequestReference.of(UUID.randomUUID().toString(), "CC", System.currentTimeMillis()); final IRequestParameters requestParameters = new RequestParameters(requestReference, statement, null, - new ResultProperties(IStatementExecutor.ResultDelivery.IMMEDIATE), new IStatementExecutor.Stats(), null, - null, null, statementParams, true); + new ResultProperties(IStatementExecutor.ResultDelivery.IMMEDIATE), new IStatementExecutor.Stats(), + new IStatementExecutor.StatementProperties(), null, null, null, statementParams, true); translator.compileAndExecute(hcc, requestParameters); executionPlans = translator.getExecutionPlans(); writer.flush(); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java index 94080da..149ed33 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java @@ -146,10 +146,12 @@ public final class ExecuteStatementRequestMessage implements ICcAddressedMessage compilationProvider, storageComponentProvider, new ResponsePrinter(sessionOutput)); final IStatementExecutor.Stats stats = new IStatementExecutor.Stats(); stats.setProfileType(profileType); + final IStatementExecutor.StatementProperties statementProperties = + new IStatementExecutor.StatementProperties(); Map<String, IAObject> stmtParams = RequestParameters.deserializeParameterValues(statementParameters); final IRequestParameters requestParameters = new RequestParameters(requestReference, statementsText, null, - resultProperties, stats, outMetadata, clientContextID, optionalParameters, stmtParams, - multiStatement, statementCategoryRestrictionMask); + resultProperties, stats, statementProperties, outMetadata, clientContextID, optionalParameters, + stmtParams, multiStatement, statementCategoryRestrictionMask); translator.compileAndExecute(ccApp.getHcc(), requestParameters); translator.getWarnings(warnings, maxWarnings - warnings.size()); stats.updateTotalWarningsCount(parserTotalWarningsCount); @@ -157,6 +159,7 @@ public final class ExecuteStatementRequestMessage implements ICcAddressedMessage responseMsg.setResult(outWriter.toString()); responseMsg.setMetadata(outMetadata); responseMsg.setStats(stats); + responseMsg.setStatementProperties(statementProperties); responseMsg.setExecutionPlans(translator.getExecutionPlans()); responseMsg.setWarnings(warnings); } catch (AlgebricksException | HyracksException | TokenMgrError diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java index 58898cf..2cdede1 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementResponseMessage.java @@ -41,6 +41,8 @@ public final class ExecuteStatementResponseMessage implements INcAddressedMessag private IStatementExecutor.Stats stats; + private IStatementExecutor.StatementProperties statementProperties; + private Throwable error; private ExecutionPlans executionPlans; @@ -92,6 +94,14 @@ public final class ExecuteStatementResponseMessage implements INcAddressedMessag this.stats = stats; } + public IStatementExecutor.StatementProperties getStatementProperties() { + return statementProperties; + } + + public void setStatementProperties(IStatementExecutor.StatementProperties statementProperties) { + this.statementProperties = statementProperties; + } + public ExecutionPlans getExecutionPlans() { return executionPlans; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index ed3eb8b..b814b0a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -291,6 +291,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen final ResultDelivery resultDelivery = requestParameters.getResultProperties().getDelivery(); final long maxResultReads = requestParameters.getResultProperties().getMaxReads(); final Stats stats = requestParameters.getStats(); + final StatementProperties statementProperties = requestParameters.getStatementProperties(); final ResultMetadata outMetadata = requestParameters.getOutMetadata(); final Map<String, IAObject> stmtParams = requestParameters.getStatementParameters(); warningCollector.setMaxWarnings(sessionConfig.getMaxWarnings()); @@ -307,7 +308,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen metadataProvider.setWriterFactory(writerFactory); metadataProvider.setResultSerializerFactoryProvider(resultSerializerFactoryProvider); metadataProvider.setOutputFile(outputFile); - switch (stmt.getKind()) { + Statement.Kind kind = stmt.getKind(); + statementProperties.setKind(kind); + switch (kind) { case SET: handleSetStatement(stmt, config); break; @@ -417,12 +420,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen // No op break; case EXTENSION: - ((ExtensionStatement) stmt).handle(hcc, this, requestParameters, metadataProvider, - resultSetIdCounter); + final ExtensionStatement extStmt = (ExtensionStatement) stmt; + statementProperties.setName(extStmt.getName()); + extStmt.handle(hcc, this, requestParameters, metadataProvider, resultSetIdCounter); break; default: throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(), - "Unexpected statement: " + stmt.getKind()); + "Unexpected statement: " + kind); } } } finally { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java index 90602e7..5ebc9ba 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java @@ -31,6 +31,7 @@ import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.translator.IRequestParameters; import org.apache.asterix.translator.IStatementExecutor; +import org.apache.asterix.translator.IStatementExecutor.StatementProperties; import org.apache.asterix.translator.IStatementExecutor.Stats; import org.apache.asterix.translator.ResultProperties; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; @@ -49,6 +50,7 @@ public class RequestParameters implements IRequestParameters { private final IResultSet resultSet; private final ResultProperties resultProperties; private final Stats stats; + private final StatementProperties statementProperties; private final Map<String, String> optionalParameters; private final IStatementExecutor.ResultMetadata outMetadata; private final String clientContextId; @@ -58,22 +60,24 @@ public class RequestParameters implements IRequestParameters { private final String statement; public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet, - ResultProperties resultProperties, Stats stats, IStatementExecutor.ResultMetadata outMetadata, - String clientContextId, Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, - boolean multiStatement) { - this(requestReference, statement, resultSet, resultProperties, stats, outMetadata, clientContextId, - optionalParameters, statementParameters, multiStatement, NO_CATEGORY_RESTRICTION_MASK); + ResultProperties resultProperties, Stats stats, StatementProperties statementProperties, + IStatementExecutor.ResultMetadata outMetadata, String clientContextId, + Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement) { + this(requestReference, statement, resultSet, resultProperties, stats, statementProperties, outMetadata, + clientContextId, optionalParameters, statementParameters, multiStatement, NO_CATEGORY_RESTRICTION_MASK); } public RequestParameters(IRequestReference requestReference, String statement, IResultSet resultSet, - ResultProperties resultProperties, Stats stats, IStatementExecutor.ResultMetadata outMetadata, - String clientContextId, Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, - boolean multiStatement, int statementCategoryRestrictionMask) { + ResultProperties resultProperties, Stats stats, StatementProperties statementProperties, + IStatementExecutor.ResultMetadata outMetadata, String clientContextId, + Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement, + int statementCategoryRestrictionMask) { this.requestReference = requestReference; this.statement = statement; this.resultSet = resultSet; this.resultProperties = resultProperties; this.stats = stats; + this.statementProperties = statementProperties; this.outMetadata = outMetadata; this.clientContextId = clientContextId; this.optionalParameters = optionalParameters; @@ -93,11 +97,16 @@ public class RequestParameters implements IRequestParameters { } @Override - public IStatementExecutor.Stats getStats() { + public Stats getStats() { return stats; } @Override + public StatementProperties getStatementProperties() { + return statementProperties; + } + + @Override public Map<String, String> getOptionalParameters() { return optionalParameters; } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryCancellationServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryCancellationServletTest.java index 348b947..50e1155 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryCancellationServletTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/QueryCancellationServletTest.java @@ -70,8 +70,8 @@ public class QueryCancellationServletTest { verify(mockResponse, times(1)).setStatus(HttpResponseStatus.NOT_FOUND); final RequestReference requestReference = RequestReference.of("1", "node1", System.currentTimeMillis()); - RequestParameters requestParameters = - new RequestParameters(requestReference, "select 1", null, null, null, null, "1", null, null, true); + RequestParameters requestParameters = new RequestParameters(requestReference, "select 1", null, null, null, + null, null, "1", null, null, true); ClientRequest request = new ClientRequest(requestParameters); request.setJobId(new JobId(1)); request.markCancellable(); @@ -87,8 +87,8 @@ public class QueryCancellationServletTest { // Tests the case that the job cancellation hit some exception from Hyracks. final RequestReference requestReference2 = RequestReference.of("2", "node1", System.currentTimeMillis()); - requestParameters = - new RequestParameters(requestReference2, "select 1", null, null, null, null, "2", null, null, true); + requestParameters = new RequestParameters(requestReference2, "select 1", null, null, null, null, null, "2", + null, null, true); ClientRequest request2 = new ClientRequest(requestParameters); request2.setJobId(new JobId(2)); request2.markCancellable(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java index ed0c802..17ba250 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java @@ -144,16 +144,18 @@ public class ResultExtractor { return extract(resultStream, resultFields, resultCharset, "", "", ""); } - throw new AsterixException("Unkown output format for result of test query"); + throw new AsterixException("Unknown output format for result of test query"); } private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields, Charset resultCharset, String openMarker, String separator, String closeMarker) throws Exception { ExtractedResult extractedResult = new ExtractedResult(); final String resultStr = IOUtils.toString(resultStream, resultCharset); + + LOGGER.debug("+++++++\n" + resultStr + "\n+++++++\n"); + final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class); - LOGGER.debug("+++++++\n" + result + "\n+++++++\n"); // if we have errors field in the results, we will always return it checkForErrors(result); final StringBuilder resultBuilder = new StringBuilder(); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 56dbac2..6bae914 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -1415,7 +1415,6 @@ public class TestExecutor { FilenameUtils.getExtension(ctx.getFile().getName())); return executeQuery(fmt, statement, variableCtx, ctx, expectedResultFile, actualResultFile, queryCount, numResultFiles, params, compare, uri); - } private void polldynamic(TestCaseContext testCaseCtx, TestFileContext ctx, Map<String, Object> variableCtx, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestReference.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestReference.java index 8a25ed2..c7f8d30 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestReference.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestReference.java @@ -42,4 +42,18 @@ public interface IRequestReference extends Serializable { * @return the time at which the request was received. */ long getTime(); + + /** + * Gets the user agent from which the request was received. + * + * @return user agent from which the request was received. + */ + String getUserAgent(); + + /** + * Gets the remote address from which the request was received. + * + * @return remote address from which the request was received. + */ + String getRemoteAddr(); }
