Till Westmann has submitted this change and it was merged. Change subject: add support for JSON encoded requests ......................................................................
add support for JSON encoded requests - run SQL++ execution test *queries* using JSON encoded requests - fix metadata cleanup at the end of the tests Change-Id: I1cc934d5dd984b476d4adb1755572d2e2f451985 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1201 Sonar-Qube: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java 2 files changed, 111 insertions(+), 58 deletions(-) Approvals: Yingyi Bu: Looks good to me, approved Jenkins: Verified; No violations found; Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java index 132737b..856aa40 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java @@ -56,6 +56,8 @@ import org.apache.hyracks.api.client.IHyracksClientConnection; import org.apache.hyracks.api.dataset.IHyracksDataset; import org.apache.hyracks.client.dataset.HyracksDataset; +import org.json.JSONException; +import org.json.JSONObject; public class QueryServiceServlet extends HttpServlet { private static final long serialVersionUID = 1L; @@ -71,10 +73,9 @@ } public enum Parameter { - // Standard STATEMENT("statement"), FORMAT("format"), - // Asterix + CLIENT_ID("client_context_id"), PRETTY("pretty"); private final String str; @@ -121,6 +122,7 @@ public enum ResultFields { REQUEST_ID("requestID"), + CLIENT_ID("clientContextID"), SIGNATURE("signature"), TYPE("type"), STATUS("status"), @@ -217,6 +219,13 @@ } } + static class RequestParameters { + String statement; + String format; + boolean pretty; + String clientContextID; + } + private static String getParameterValue(String content, String attribute) { if (content == null || attribute == null) { return null; @@ -255,11 +264,7 @@ return SessionConfig.OutputFormat.CLEAN_JSON; } - /** - * Construct a SessionConfig with the appropriate output writer and - * output-format based on the Accept: header and other servlet parameters. - */ - private static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) { + private static SessionConfig createSessionConfig(RequestParameters param, PrintWriter resultWriter) { SessionConfig.ResultDecorator resultPrefix = (AlgebricksAppendable app) -> { app.append("\t\""); app.append(ResultFields.RESULTS.str()); @@ -272,16 +277,14 @@ return app; }; - final String formatstr = toLower(request.getParameter(Parameter.FORMAT.str())); - SessionConfig.OutputFormat format = getFormat(formatstr); + SessionConfig.OutputFormat format = getFormat(param.format); SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix); sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true); - boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str())); - sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, indentJson); + sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, param.pretty); sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD, format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON); - sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, - format == SessionConfig.OutputFormat.CSV && "present".equals(getParameterValue(formatstr, "header"))); + sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, format == SessionConfig.OutputFormat.CSV + && "present".equals(getParameterValue(param.format, "header"))); return sessionConfig; } @@ -305,6 +308,12 @@ UUID requestId = UUID.randomUUID(); printField(pw, ResultFields.REQUEST_ID.str(), requestId.toString()); return requestId; + } + + private static void printClientContextID(PrintWriter pw, RequestParameters params) { + if (params.clientContextID != null && !params.clientContextID.isEmpty()) { + printField(pw, ResultFields.CLIENT_ID.str(), params.clientContextID); + } } private static void printSignature(PrintWriter pw) { @@ -370,16 +379,9 @@ } @Override - protected void doPost(HttpServletRequest request, HttpServletResponse response) - throws ServletException, IOException { - String query = request.getParameter(Parameter.STATEMENT.str()); + protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException { try { - if (query == null) { - StringWriter sw = new StringWriter(); - IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name()); - query = sw.toString(); - } - handleRequest(request, response, query); + handleRequest(getRequestParameters(request), response); } catch (IOException e) { // Servlet methods should not throw exceptions // http://cwe.mitre.org/data/definitions/600.html @@ -387,13 +389,46 @@ } } - private void handleRequest(HttpServletRequest request, HttpServletResponse response, String query) - throws IOException { + private RequestParameters getRequestParameters(HttpServletRequest request) throws IOException { + final String contentTypeParam = request.getContentType(); + int sep = contentTypeParam.indexOf(';'); + final String contentType = sep < 0 ? contentTypeParam.trim() : contentTypeParam.substring(0, sep).trim(); + RequestParameters param = new RequestParameters(); + if (MediaType.JSON.str().equals(contentType)) { + try { + JSONObject jsonRequest = new JSONObject(getRequestBody(request)); + param.statement = jsonRequest.getString(Parameter.STATEMENT.str()); + param.format = toLower(jsonRequest.optString(Parameter.FORMAT.str())); + param.pretty = jsonRequest.optBoolean(Parameter.PRETTY.str()); + param.clientContextID = jsonRequest.optString(Parameter.CLIENT_ID.str()); + } catch (JSONException e) { + // if the JSON parsing fails, the statement is empty and we get an empty statement error + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } else { + param.statement = request.getParameter(Parameter.STATEMENT.str()); + if (param.statement == null) { + param.statement = getRequestBody(request); + } + param.format = toLower(request.getParameter(Parameter.FORMAT.str())); + param.pretty = Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str())); + param.clientContextID = request.getParameter(Parameter.CLIENT_ID.str()); + } + return param; + } + + private static String getRequestBody(HttpServletRequest request) throws IOException { + StringWriter sw = new StringWriter(); + IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name()); + return sw.toString(); + } + + private void handleRequest(RequestParameters param, HttpServletResponse response) throws IOException { long elapsedStart = System.nanoTime(); final StringWriter stringWriter = new StringWriter(); final PrintWriter resultWriter = new PrintWriter(stringWriter); - SessionConfig sessionConfig = createSessionConfig(request, resultWriter); + SessionConfig sessionConfig = createSessionConfig(param, resultWriter); response.setCharacterEncoding("utf-8"); response.setContentType(MediaType.JSON.str()); @@ -404,10 +439,11 @@ resultWriter.print("{\n"); printRequestId(resultWriter); + printClientContextID(resultWriter, param); printSignature(resultWriter); printType(resultWriter, sessionConfig); try { - if (query == null || query.isEmpty()) { + if (param.statement == null || param.statement.isEmpty()) { throw new AsterixException("Empty request, no statement provided"); } IHyracksClientConnection hcc; @@ -421,7 +457,7 @@ context.setAttribute(HYRACKS_DATASET_ATTR, hds); } } - IParser parser = compilationProvider.getParserFactory().createParser(query); + IParser parser = compilationProvider.getParserFactory().createParser(param.statement); List<Statement> aqlStatements = parser.parse(); MetadataManager.INSTANCE.init(); IStatementExecutor translator = statementExecutorFactory.create(aqlStatements, sessionConfig, diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java index 1c48dbb..08a0342 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java @@ -60,10 +60,13 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.StandardHttpRequestRetryHandler; import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; public class TestExecutor { @@ -347,9 +350,7 @@ } protected HttpResponse executeHttpRequest(HttpUriRequest method) throws Exception { - HttpClient client = HttpClients.custom() - .setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE) - .build(); + HttpClient client = HttpClients.custom().setRetryHandler(StandardHttpRequestRetryHandler.INSTANCE).build(); try { return client.execute(method); } catch (Exception e) { @@ -395,13 +396,14 @@ } public InputStream executeQueryService(String str, String url) throws Exception { - return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>()); + return executeQueryService(str, OutputFormat.CLEAN_JSON, url, new ArrayList<>(), false); } public InputStream executeQueryService(String str, OutputFormat fmt, String url, - List<CompilationUnit.Parameter> params) throws Exception { + List<CompilationUnit.Parameter> params, boolean jsonEncoded) throws Exception { setFormatParam(params, fmt); - HttpUriRequest method = constructPostMethod(str, url, "statement", true, params); + HttpUriRequest method = jsonEncoded ? constructPostMethodJson(str, url, "statement", params) + : constructPostMethodUrl(str, url, "statement", params); // Set accepted output response type method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType()); HttpResponse response = executeHttpRequest(method); @@ -409,19 +411,16 @@ } protected void setFormatParam(List<CompilationUnit.Parameter> params, OutputFormat fmt) { - boolean formatSet = false; for (CompilationUnit.Parameter param : params) { if ("format".equals(param.getName())) { param.setValue(fmt.mimeType()); - formatSet = true; + return; } } - if (!formatSet) { - CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter(); - formatParam.setName("format"); - formatParam.setValue(fmt.mimeType()); - params.add(formatParam); - } + CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter(); + formatParam.setName("format"); + formatParam.setValue(fmt.mimeType()); + params.add(formatParam); } private HttpUriRequest constructHttpMethod(String statement, String endpoint, String stmtParam, @@ -431,7 +430,8 @@ return constructGetMethod(statement, endpoint, stmtParam, otherParams); } else { // Use POST for bigger ones to avoid 413 FULL_HEAD - return constructPostMethod(statement, endpoint, stmtParam, postStmtAsParam, otherParams); + String stmtParamName = (postStmtAsParam ? stmtParam : null); + return constructPostMethodUrl(statement, endpoint, stmtParamName, otherParams); } } @@ -445,10 +445,10 @@ return builder.build(); } - protected HttpUriRequest constructPostMethod(String statement, String endpoint, String stmtParam, - boolean postStmtAsParam, List<CompilationUnit.Parameter> otherParams) { + protected HttpUriRequest constructPostMethodUrl(String statement, String endpoint, String stmtParam, + List<CompilationUnit.Parameter> otherParams) { RequestBuilder builder = RequestBuilder.post(endpoint); - if (postStmtAsParam) { + if (stmtParam != null) { for (CompilationUnit.Parameter param : otherParams) { builder.addParameter(param.getName(), param.getValue()); } @@ -457,6 +457,26 @@ // this seems pretty bad - we should probably fix the API and not the client builder.setEntity(new StringEntity(statement, StandardCharsets.UTF_8)); } + builder.setCharset(StandardCharsets.UTF_8); + return builder.build(); + } + + protected HttpUriRequest constructPostMethodJson(String statement, String endpoint, String stmtParam, + List<CompilationUnit.Parameter> otherParams) { + if (stmtParam == null) { + throw new NullPointerException("Statement parameter required."); + } + RequestBuilder builder = RequestBuilder.post(endpoint); + JSONObject content = new JSONObject(); + try { + content.put(stmtParam, statement); + for (CompilationUnit.Parameter param : otherParams) { + content.put(param.getName(), param.getValue()); + } + } catch (JSONException e) { + throw new IllegalArgumentException("Request object construction failed.", e); + } + builder.setEntity(new StringEntity(content.toString(), ContentType.APPLICATION_JSON)); builder.setCharset(StandardCharsets.UTF_8); return builder.build(); } @@ -485,9 +505,7 @@ // Create a method instance. HttpUriRequest request = RequestBuilder.post(url) .addParameter("mode", defer ? "asynchronous-deferred" : "asynchronous") - .setEntity(new StringEntity(str, StandardCharsets.UTF_8)) - .setHeader("Accept", fmt.mimeType()) - .build(); + .setEntity(new StringEntity(str, StandardCharsets.UTF_8)).setHeader("Accept", fmt.mimeType()).build(); HttpResponse response = executeAndCheckHttpRequest(request); InputStream resultStream = response.getEntity().getContent(); @@ -664,7 +682,7 @@ } else { if (ctx.getType().equalsIgnoreCase("query")) { resultStream = executeQueryService(statement, fmt, getEndpoint(Servlets.QUERY_SERVICE), - cUnit.getParameter()); + cUnit.getParameter(), true); resultStream = ResultExtractor.extract(resultStream); } else if (ctx.getType().equalsIgnoreCase("async")) { resultStream = executeAnyAQLAsync(statement, false, fmt, getEndpoint(Servlets.SQLPP)); @@ -968,20 +986,18 @@ public void cleanup(String testCase, List<String> badtestcases) throws Exception { try { ArrayList<String> toBeDropped = new ArrayList<>(); - InputStream resultStream = null; - OutputFormat fmt = OutputFormat.ADM; - resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse` as dv;", fmt, - getEndpoint(Servlets.QUERY_SERVICE), new ArrayList<>()); + InputStream resultStream = executeQueryService("select dv.DataverseName from Metadata.`Dataverse` as dv;", + getEndpoint(Servlets.QUERY_SERVICE)); resultStream = ResultExtractor.extract(resultStream); - BufferedReader reader = new BufferedReader(new InputStreamReader(resultStream)); - String dataverse = reader.readLine(); - while (dataverse != null) { - JSONObject json = new JSONObject(dataverse); + StringWriter sw = new StringWriter(); + IOUtils.copy(resultStream, sw, StandardCharsets.UTF_8.name()); + JSONArray result = new JSONArray(sw.toString()); + for (int i = 0; i < result.length(); ++i) { + JSONObject json = result.getJSONObject(i); String dvName = json.getString("DataverseName"); if (!dvName.equals("Metadata") && !dvName.equals("Default")) { toBeDropped.add(dvName); } - dataverse = reader.readLine(); } if (!toBeDropped.isEmpty()) { badtestcases.add(testCase); @@ -998,6 +1014,7 @@ } } catch (Throwable th) { th.printStackTrace(); + throw th; } } } -- To view, visit https://asterix-gerrit.ics.uci.edu/1201 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1cc934d5dd984b476d4adb1755572d2e2f451985 Gerrit-PatchSet: 3 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
