Michael Blow has submitted this change and it was merged. Change subject: [NO ISSUE] Handle Accept-Charset in QueryResultApiServlet ......................................................................
[NO ISSUE] Handle Accept-Charset in QueryResultApiServlet - exercise non-UTF8 Accept-Charset in TestExecutor - remove double-buffering on http responses - minor refactoring / cleanup Change-Id: I8f37eb684bf2457e5ff451bf5c8fbca742d531f2 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3191 Reviewed-by: Murtadha Hubail <[email protected]> Sonar-Qube: Jenkins <[email protected]> Tested-by: Michael Blow <[email protected]> --- M asterixdb/asterix-app/pom.xml M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java D asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java M asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh M asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh M asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java M asterixdb/pom.xml M hyracks-fullstack/hyracks/hyracks-http/pom.xml M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java M hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java M hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java M hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java A hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java 25 files changed, 430 insertions(+), 452 deletions(-) Approvals: Jenkins: No violations found Michael Blow: Verified Murtadha Hubail: Looks good to me, approved diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml index 8ef9c45..281e1d4 100644 --- a/asterixdb/asterix-app/pom.xml +++ b/asterixdb/asterix-app/pom.xml @@ -666,7 +666,6 @@ <dependency> <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId> - <version>${hyracks.version}</version> </dependency> </dependencies> </project> diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java index 8fdcc41..7f74c92 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java @@ -18,12 +18,14 @@ */ package org.apache.asterix.api.http.server; +import java.io.IOException; import java.util.concurrent.ConcurrentMap; import org.apache.asterix.app.result.ResultHandle; import org.apache.asterix.app.result.ResultReader; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.translator.IStatementExecutor.Stats; +import org.apache.asterix.translator.SessionConfig; import org.apache.asterix.translator.SessionOutput; import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -91,7 +93,7 @@ // way to send the same OutputFormat value here as was // originally determined there. Need to save this value on // some object that we can obtain here. - SessionOutput sessionOutput = RestApiServlet.initResponse(request, response); + SessionOutput sessionOutput = initResponse(request, response); ResultUtil.printResults(appCtx, resultReader, sessionOutput, new Stats(), null); } catch (HyracksDataException e) { final int errorCode = e.getErrorCode(); @@ -112,4 +114,82 @@ } } + /** + * Initialize the Content-Type of the response, and construct a + * SessionConfig with the appropriate output writer and output-format + * based on the Accept: header and other servlet parameters. + */ + static SessionOutput initResponse(IServletRequest request, IServletResponse response) throws IOException { + // CLEAN_JSON output is the default; most generally useful for a + // programmatic HTTP API + SessionConfig.OutputFormat format = SessionConfig.OutputFormat.CLEAN_JSON; + // First check the "output" servlet parameter. + String output = request.getParameter("output"); + String accept = request.getHeader("Accept", ""); + if (output != null) { + if ("CSV".equals(output)) { + format = SessionConfig.OutputFormat.CSV; + } else if ("ADM".equals(output)) { + format = SessionConfig.OutputFormat.ADM; + } + } else { + // Second check the Accept: HTTP header. + if (accept.contains("application/x-adm")) { + format = SessionConfig.OutputFormat.ADM; + } else if (accept.contains("text/csv")) { + format = SessionConfig.OutputFormat.CSV; + } + } + SessionConfig.PlanFormat planFormat = SessionConfig.PlanFormat.get(request.getParameter("plan-format"), + "plan format", SessionConfig.PlanFormat.STRING, LOGGER); + + // If it's JSON, check for the "lossless" flag + + if (format == SessionConfig.OutputFormat.CLEAN_JSON + && ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true"))) { + format = SessionConfig.OutputFormat.LOSSLESS_JSON; + } + + SessionOutput.ResultAppender appendHandle = (app, handle) -> app.append("{ \"").append("handle") + .append("\":" + " \"").append(handle).append("\" }"); + SessionConfig sessionConfig = new SessionConfig(format, planFormat); + + // If it's JSON or ADM, check for the "wrapper-array" flag. Default is + // "true" for JSON and "false" for ADM. (Not applicable for CSV.) + boolean wrapperArray = + format == SessionConfig.OutputFormat.CLEAN_JSON || format == SessionConfig.OutputFormat.LOSSLESS_JSON; + String wrapperParam = request.getParameter("wrapper-array"); + if (wrapperParam != null) { + wrapperArray = Boolean.valueOf(wrapperParam); + } else if (accept.contains("wrap-array=true")) { + wrapperArray = true; + } else if (accept.contains("wrap-array=false")) { + wrapperArray = false; + } + sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, wrapperArray); + // Now that format is set, output the content-type + switch (format) { + case ADM: + HttpUtil.setContentType(response, "application/x-adm", request); + break; + case CLEAN_JSON: + // No need to reflect "clean-ness" in output type; fall through + case LOSSLESS_JSON: + HttpUtil.setContentType(response, "application/json", request); + break; + case CSV: + // Check for header parameter or in Accept:. + if ("present".equals(request.getParameter("header")) || accept.contains("header=present")) { + HttpUtil.setContentType(response, "text/csv; header=present", request); + sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true); + } else { + HttpUtil.setContentType(response, "text/csv; header=absent", request); + } + break; + default: + throw new IOException("Unknown format " + format); + } + return new SessionOutput(sessionConfig, response.writer(), null, null, appendHandle, null); + } + } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java index 625834f..79401b8 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java @@ -28,6 +28,8 @@ import java.io.PrintWriter; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -77,7 +79,6 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; - import io.netty.handler.codec.http.HttpResponseStatus; public class QueryServiceServlet extends AbstractQueryApiServlet { @@ -339,16 +340,18 @@ } private static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, - long resultSize, long processedObjects, long errorCount, long warnCount) { + long resultSize, long processedObjects, long errorCount, long warnCount, Charset resultCharset) { boolean hasErrors = errorCount != 0; boolean hasWarnings = warnCount != 0; + boolean useAscii = !StandardCharsets.UTF_8.equals(resultCharset) + && !"μ".contentEquals(resultCharset.decode(resultCharset.encode("μ"))); pw.print("\t\""); pw.print(ResultFields.METRICS.str()); pw.print("\": {\n"); pw.print("\t"); - ResultUtil.printField(pw, Metrics.ELAPSED_TIME.str(), Duration.formatNanos(elapsedTime)); + ResultUtil.printField(pw, Metrics.ELAPSED_TIME.str(), Duration.formatNanos(elapsedTime, useAscii)); pw.print("\t"); - ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(), Duration.formatNanos(executionTime)); + ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(), Duration.formatNanos(executionTime, useAscii)); pw.print("\t"); ResultUtil.printField(pw, Metrics.RESULT_COUNT.str(), resultCount, true); pw.print("\t"); @@ -366,14 +369,31 @@ pw.print("\t}\n"); } + private String getOptText(JsonNode node, Parameter parameter) { + return getOptText(node, parameter.str()); + } + private String getOptText(JsonNode node, String fieldName) { final JsonNode value = node.get(fieldName); return value != null ? value.asText() : null; } + private boolean getOptBoolean(JsonNode node, Parameter parameter, boolean defaultValue) { + return getOptBoolean(node, parameter.str(), defaultValue); + } + private boolean getOptBoolean(JsonNode node, String fieldName, boolean defaultValue) { final JsonNode value = node.get(fieldName); return value != null ? value.asBoolean() : defaultValue; + } + + private String getParameter(IServletRequest request, Parameter parameter) { + return request.getParameter(parameter.str()); + } + + private boolean getOptBoolean(IServletRequest request, Parameter parameter, boolean defaultValue) { + String value = request.getParameter(parameter.str()); + return value == null ? defaultValue : Boolean.parseBoolean(value); } @FunctionalInterface @@ -419,46 +439,41 @@ if (HttpUtil.ContentType.APPLICATION_JSON.equals(contentType)) { try { JsonNode jsonRequest = OBJECT_MAPPER.readTree(HttpUtil.getRequestBody(request)); - final String statementParam = Parameter.STATEMENT.str(); - if (jsonRequest.has(statementParam)) { - param.setStatement(jsonRequest.get(statementParam).asText()); - } - param.setFormat(toLower(getOptText(jsonRequest, Parameter.FORMAT.str()))); - param.setPretty(getOptBoolean(jsonRequest, Parameter.PRETTY.str(), false)); - param.setMode(toLower(getOptText(jsonRequest, Parameter.MODE.str()))); - param.setClientContextID(getOptText(jsonRequest, Parameter.CLIENT_ID.str())); - param.setTimeout(getOptText(jsonRequest, Parameter.TIMEOUT.str())); - param.setMaxResultReads(getOptText(jsonRequest, Parameter.MAX_RESULT_READS.str())); - param.setPlanFormat(getOptText(jsonRequest, Parameter.PLAN_FORMAT.str())); - param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE.str(), false)); + param.setStatement(getOptText(jsonRequest, Parameter.STATEMENT)); + param.setFormat(toLower(getOptText(jsonRequest, Parameter.FORMAT))); + param.setPretty(getOptBoolean(jsonRequest, Parameter.PRETTY, false)); + param.setMode(toLower(getOptText(jsonRequest, Parameter.MODE))); + param.setClientContextID(getOptText(jsonRequest, Parameter.CLIENT_ID)); + param.setTimeout(getOptText(jsonRequest, Parameter.TIMEOUT)); + param.setMaxResultReads(getOptText(jsonRequest, Parameter.MAX_RESULT_READS)); + param.setPlanFormat(getOptText(jsonRequest, Parameter.PLAN_FORMAT)); + param.setExpressionTree(getOptBoolean(jsonRequest, Parameter.EXPRESSION_TREE, false)); param.setRewrittenExpressionTree( - getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE.str(), false)); - param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN.str(), false)); - param.setOptimizedLogicalPlan( - getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN.str(), false)); - param.setJob(getOptBoolean(jsonRequest, Parameter.JOB.str(), false)); - param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE.str(), true)); + getOptBoolean(jsonRequest, Parameter.REWRITTEN_EXPRESSION_TREE, false)); + param.setLogicalPlan(getOptBoolean(jsonRequest, Parameter.LOGICAL_PLAN, false)); + param.setOptimizedLogicalPlan(getOptBoolean(jsonRequest, Parameter.OPTIMIZED_LOGICAL_PLAN, false)); + param.setJob(getOptBoolean(jsonRequest, Parameter.JOB, false)); + param.setSignature(getOptBoolean(jsonRequest, Parameter.SIGNATURE, true)); param.setStatementParams( getOptStatementParameters(jsonRequest, jsonRequest.fieldNames(), JsonNode::get, v -> v)); - param.setMultiStatement(getOptBoolean(jsonRequest, Parameter.MULTI_STATEMENT.str(), true)); + param.setMultiStatement(getOptBoolean(jsonRequest, Parameter.MULTI_STATEMENT, true)); } catch (JsonParseException | JsonMappingException e) { // if the JSON parsing fails, the statement is empty and we get an empty statement error GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e); } } else { - param.setStatement(request.getParameter(Parameter.STATEMENT.str())); + param.setStatement(getParameter(request, Parameter.STATEMENT)); if (param.getStatement() == null) { param.setStatement(HttpUtil.getRequestBody(request)); } - param.setFormat(toLower(request.getParameter(Parameter.FORMAT.str()))); - param.setPretty(Boolean.parseBoolean(request.getParameter(Parameter.PRETTY.str()))); - param.setMode(toLower(request.getParameter(Parameter.MODE.str()))); - param.setClientContextID(request.getParameter(Parameter.CLIENT_ID.str())); - param.setTimeout(request.getParameter(Parameter.TIMEOUT.str())); - param.setMaxResultReads(request.getParameter(Parameter.MAX_RESULT_READS.str())); - param.setPlanFormat(request.getParameter(Parameter.PLAN_FORMAT.str())); - final String multiStatementParam = request.getParameter(Parameter.MULTI_STATEMENT.str()); - param.setMultiStatement(multiStatementParam == null || Boolean.parseBoolean(multiStatementParam)); + param.setFormat(toLower(getParameter(request, Parameter.FORMAT))); + param.setPretty(Boolean.parseBoolean(getParameter(request, Parameter.PRETTY))); + param.setMode(toLower(getParameter(request, Parameter.MODE))); + param.setClientContextID(getParameter(request, Parameter.CLIENT_ID)); + param.setTimeout(getParameter(request, Parameter.TIMEOUT)); + param.setMaxResultReads(getParameter(request, Parameter.MAX_RESULT_READS)); + param.setPlanFormat(getParameter(request, Parameter.PLAN_FORMAT)); + param.setMultiStatement(getOptBoolean(request, Parameter.MULTI_STATEMENT, true)); try { param.setStatementParams(getOptStatementParameters(request, request.getParameterNames().iterator(), IServletRequest::getParameter, OBJECT_MAPPER::readTree)); @@ -512,7 +527,7 @@ QueryServiceRequestParameters param = getRequestParameters(request); LOGGER.info("handleRequest: {}", param); long elapsedStart = System.nanoTime(); - HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request); + Charset resultCharset = HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request); final PrintWriter httpWriter = response.writer(); ResultDelivery delivery = parseResultDelivery(param.getMode()); @@ -573,7 +588,7 @@ execution.finish(); } printMetrics(sessionOutput.out(), System.nanoTime() - elapsedStart, execution.duration(), stats.getCount(), - stats.getSize(), stats.getProcessedObjects(), errorCount, warnings.size()); + stats.getSize(), stats.getProcessedObjects(), errorCount, warnings.size(), resultCharset); sessionOutput.out().print("}\n"); sessionOutput.out().flush(); if (sessionOutput.out().checkError()) { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java deleted file mode 100644 index 347b2d7..0000000 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.api.http.server; - -import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.ConcurrentMap; - -import org.apache.asterix.app.translator.QueryTranslator; -import org.apache.asterix.app.translator.RequestParameters; -import org.apache.asterix.common.config.GlobalConfig; -import org.apache.asterix.common.context.IStorageComponentProvider; -import org.apache.asterix.common.dataflow.ICcApplicationContext; -import org.apache.asterix.common.exceptions.AsterixException; -import org.apache.asterix.compiler.provider.ILangCompilationProvider; -import org.apache.asterix.lang.aql.parser.TokenMgrError; -import org.apache.asterix.lang.common.base.IParser; -import org.apache.asterix.lang.common.base.IParserFactory; -import org.apache.asterix.lang.common.base.Statement; -import org.apache.asterix.metadata.MetadataManager; -import org.apache.asterix.translator.IRequestParameters; -import org.apache.asterix.translator.IStatementExecutor; -import org.apache.asterix.translator.IStatementExecutor.ResultDelivery; -import org.apache.asterix.translator.IStatementExecutorFactory; -import org.apache.asterix.translator.ResultProperties; -import org.apache.asterix.translator.SessionConfig; -import org.apache.asterix.translator.SessionConfig.OutputFormat; -import org.apache.asterix.translator.SessionConfig.PlanFormat; -import org.apache.asterix.translator.SessionOutput; -import org.apache.hyracks.api.client.IHyracksClientConnection; -import org.apache.hyracks.api.result.IResultSet; -import org.apache.hyracks.http.api.IServletRequest; -import org.apache.hyracks.http.api.IServletResponse; -import org.apache.hyracks.http.server.AbstractServlet; -import org.apache.hyracks.http.server.utils.HttpUtil; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.node.ObjectNode; - -import io.netty.handler.codec.http.HttpMethod; -import io.netty.handler.codec.http.HttpResponseStatus; - -public abstract class RestApiServlet extends AbstractServlet { - private static final Logger LOGGER = LogManager.getLogger(); - private final ICcApplicationContext appCtx; - private final ILangCompilationProvider compilationProvider; - private final IParserFactory parserFactory; - private final IStatementExecutorFactory statementExecutorFactory; - private final IStorageComponentProvider componentProvider; - - public RestApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, ICcApplicationContext appCtx, - ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory, - IStorageComponentProvider componentProvider) { - super(ctx, paths); - this.appCtx = appCtx; - this.compilationProvider = compilationProvider; - this.parserFactory = compilationProvider.getParserFactory(); - this.statementExecutorFactory = statementExecutorFactory; - this.componentProvider = componentProvider; - } - - /** - * Initialize the Content-Type of the response, and construct a - * SessionConfig with the appropriate output writer and output-format - * based on the Accept: header and other servlet parameters. - */ - static SessionOutput initResponse(IServletRequest request, IServletResponse response) throws IOException { - HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN, request); - // CLEAN_JSON output is the default; most generally useful for a - // programmatic HTTP API - OutputFormat format = OutputFormat.CLEAN_JSON; - // First check the "output" servlet parameter. - String output = request.getParameter("output"); - String accept = request.getHeader("Accept", ""); - if (output != null) { - if ("CSV".equals(output)) { - format = OutputFormat.CSV; - } else if ("ADM".equals(output)) { - format = OutputFormat.ADM; - } - } else { - // Second check the Accept: HTTP header. - if (accept.contains("application/x-adm")) { - format = OutputFormat.ADM; - } else if (accept.contains("text/csv")) { - format = OutputFormat.CSV; - } - } - PlanFormat planFormat = - PlanFormat.get(request.getParameter("plan-format"), "plan format", PlanFormat.STRING, LOGGER); - - // If it's JSON, check for the "lossless" flag - - if (format == OutputFormat.CLEAN_JSON - && ("true".equals(request.getParameter("lossless")) || accept.contains("lossless=true"))) { - format = OutputFormat.LOSSLESS_JSON; - } - - SessionOutput.ResultAppender appendHandle = (app, handle) -> app.append("{ \"").append("handle") - .append("\":" + " \"").append(handle).append("\" }"); - SessionConfig sessionConfig = new SessionConfig(format, planFormat); - - // If it's JSON or ADM, check for the "wrapper-array" flag. Default is - // "true" for JSON and "false" for ADM. (Not applicable for CSV.) - boolean wrapperArray = format == OutputFormat.CLEAN_JSON || format == OutputFormat.LOSSLESS_JSON; - String wrapperParam = request.getParameter("wrapper-array"); - if (wrapperParam != null) { - wrapperArray = Boolean.valueOf(wrapperParam); - } else if (accept.contains("wrap-array=true")) { - wrapperArray = true; - } else if (accept.contains("wrap-array=false")) { - wrapperArray = false; - } - sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, wrapperArray); - // Now that format is set, output the content-type - switch (format) { - case ADM: - HttpUtil.setContentType(response, "application/x-adm"); - break; - case CLEAN_JSON: - // No need to reflect "clean-ness" in output type; fall through - case LOSSLESS_JSON: - HttpUtil.setContentType(response, "application/json"); - break; - case CSV: - // Check for header parameter or in Accept:. - if ("present".equals(request.getParameter("header")) || accept.contains("header=present")) { - HttpUtil.setContentType(response, "text/csv; header=present"); - sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true); - } else { - HttpUtil.setContentType(response, "text/csv; header=absent"); - } - break; - default: - throw new IOException("Unknown format " + format); - } - return new SessionOutput(sessionConfig, response.writer(), null, null, appendHandle, null); - } - - @Override - protected void get(IServletRequest request, IServletResponse response) { - getOrPost(request, response); - } - - @Override - protected void post(IServletRequest request, IServletResponse response) { - getOrPost(request, response); - } - - private void getOrPost(IServletRequest request, IServletResponse response) { - try { - String query = query(request); - // enable cross-origin resource sharing - response.setHeader("Access-Control-Allow-Origin", "*"); - response.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - SessionOutput sessionOutput = initResponse(request, response); - QueryTranslator.ResultDelivery resultDelivery = whichResultDelivery(request); - doHandle(response, query, sessionOutput, resultDelivery); - } catch (Exception e) { - response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - LOGGER.log(Level.WARN, "Failure handling request", e); - return; - } - } - - private void doHandle(IServletResponse response, String query, SessionOutput sessionOutput, - ResultDelivery resultDelivery) throws JsonProcessingException { - try { - response.setStatus(HttpResponseStatus.OK); - IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR); - IParser parser = parserFactory.createParser(query); - List<Statement> aqlStatements = parser.parse(); - validate(aqlStatements); - MetadataManager.INSTANCE.init(); - IStatementExecutor translator = statementExecutorFactory.create(appCtx, aqlStatements, sessionOutput, - compilationProvider, componentProvider); - final IResultSet resultSet = ServletUtil.getResultSet(hcc, appCtx, ctx); - final IRequestParameters requestParameters = new RequestParameters(resultSet, - new ResultProperties(resultDelivery), new IStatementExecutor.Stats(), null, null, null, null, true); - translator.compileAndExecute(hcc, null, requestParameters); - } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) { - response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, pe.getMessage(), pe); - String errorMessage = ResultUtil.buildParseExceptionMessage(pe, query); - ObjectNode errorResp = - ResultUtil.getErrorResponse(2, errorMessage, "", ResultUtil.extractFullStackTrace(pe)); - sessionOutput.out().write(OBJECT_MAPPER.writeValueAsString(errorResp)); - } catch (Exception e) { - GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, e.getMessage(), e); - response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); - ResultUtil.apiErrorHandler(sessionOutput.out(), e); - } - } - - //TODO: Both Get and Post of this API must use the same parameter names - private String query(IServletRequest request) { - if (request.getHttpRequest().method() == HttpMethod.POST) { - return HttpUtil.getRequestBody(request); - } else { - return getQueryParameter(request); - } - } - - private void validate(List<Statement> aqlStatements) throws AsterixException { - for (Statement st : aqlStatements) { - if ((st.getCategory() & getAllowedCategories()) == 0) { - throw new AsterixException(String.format(getErrorMessage(), st.getKind())); - } - } - } - - protected QueryTranslator.ResultDelivery whichResultDelivery(IServletRequest request) { - String mode = request.getParameter("mode"); - if (mode != null) { - if ("asynchronous".equals(mode) || "async".equals(mode)) { - return QueryTranslator.ResultDelivery.ASYNC; - } else if ("asynchronous-deferred".equals(mode) || "deferred".equals(mode)) { - return QueryTranslator.ResultDelivery.DEFERRED; - } - } - return QueryTranslator.ResultDelivery.IMMEDIATE; - } - - protected abstract String getQueryParameter(IServletRequest request); - - protected abstract byte getAllowedCategories(); - - protected abstract String getErrorMessage(); -} diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java index 7e851bf..3335cbf 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/result/ResultPrinterTest.java @@ -28,7 +28,6 @@ import org.apache.asterix.api.http.server.ResultUtil; import org.apache.asterix.common.api.IApplicationContext; import org.apache.asterix.common.config.CompilerProperties; -import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.test.common.ResultExtractor; import org.apache.asterix.translator.IStatementExecutor; import org.apache.asterix.translator.SessionConfig; @@ -75,7 +74,7 @@ boolean exceptionThrown = false; try { // ensure result is valid json and error will be returned and not results. - ResultExtractor.extract(IOUtils.toInputStream(resultStr, StandardCharsets.UTF_8)); + ResultExtractor.extract(IOUtils.toInputStream(resultStr, StandardCharsets.UTF_8), StandardCharsets.UTF_8); } catch (Exception e) { exceptionThrown = true; Assert.assertTrue(e.getMessage().contains(expectedException.getMessage())); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java index ab8969e..37f471e 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java @@ -19,6 +19,7 @@ package org.apache.asterix.common; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.rmi.RemoteException; import java.util.Arrays; import java.util.LinkedHashSet; @@ -131,7 +132,7 @@ public static long getDatasetCount(String datasetName) throws Exception { final String query = "SELECT VALUE COUNT(*) FROM `" + datasetName + "`;"; final InputStream responseStream = TEST_EXECUTOR.executeQueryService(query, - TEST_EXECUTOR.getEndpoint(Servlets.QUERY_SERVICE), OUTPUT_FORMAT); + TEST_EXECUTOR.getEndpoint(Servlets.QUERY_SERVICE), OUTPUT_FORMAT, StandardCharsets.UTF_8); final ObjectNode response = OBJECT_MAPPER.readValue(responseStream, ObjectNode.class); final JsonNode result = response.get("results"); // make sure there is a single value in result diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java index d20d72d..48488a4 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ParseDurationTest.java @@ -142,6 +142,7 @@ Assert.assertEquals("1.234567ms", Duration.formatNanos(1234567l)); Assert.assertEquals("123.456µs", Duration.formatNanos(123456l)); Assert.assertEquals("12.345µs", Duration.formatNanos(12345l)); + Assert.assertEquals("12.345us", Duration.formatNanos(12345l, true)); Assert.assertEquals("1.234µs", Duration.formatNanos(1234l)); Assert.assertEquals("123ns", Duration.formatNanos(123l)); Assert.assertEquals("12ns", Duration.formatNanos(12l)); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java index e85fedf..969d23d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java @@ -21,6 +21,7 @@ import java.io.InputStream; import java.net.URI; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.List; @@ -49,7 +50,7 @@ @Override public InputStream executeQueryService(String str, TestCaseContext.OutputFormat fmt, URI uri, - List<TestCase.CompilationUnit.Parameter> params, boolean jsonEncoded, + List<TestCase.CompilationUnit.Parameter> params, boolean jsonEncoded, Charset responseCharset, Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception { String clientContextId = UUID.randomUUID().toString(); final List<TestCase.CompilationUnit.Parameter> newParams = cancellable @@ -57,7 +58,7 @@ Callable<InputStream> query = () -> { try { return CancellationTestExecutor.super.executeQueryService(str, fmt, uri, newParams, jsonEncoded, - responseCodeValidator, true); + responseCharset, responseCodeValidator, true); } catch (Exception e) { e.printStackTrace(); throw e; diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java index 37e1213..de7dac2 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java @@ -83,16 +83,16 @@ private static final Logger LOGGER = LogManager.getLogger(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static InputStream extract(InputStream resultStream) throws Exception { - return extract(resultStream, EnumSet.of(ResultField.RESULTS)); + public static InputStream extract(InputStream resultStream, Charset resultCharset) throws Exception { + return extract(resultStream, EnumSet.of(ResultField.RESULTS), resultCharset); } - public static InputStream extractMetrics(InputStream resultStream) throws Exception { - return extract(resultStream, EnumSet.of(ResultField.METRICS)); + public static InputStream extractMetrics(InputStream resultStream, Charset resultCharset) throws Exception { + return extract(resultStream, EnumSet.of(ResultField.METRICS), resultCharset); } - public static String extractHandle(InputStream resultStream) throws Exception { - String result = IOUtils.toString(resultStream, StandardCharsets.UTF_8); + public static String extractHandle(InputStream resultStream, Charset responseCharset) throws Exception { + String result = IOUtils.toString(resultStream, responseCharset); ObjectNode resultJson = OBJECT_MAPPER.readValue(result, ObjectNode.class); final JsonNode handle = resultJson.get("handle"); if (handle != null) { @@ -107,8 +107,9 @@ return null; } - private static InputStream extract(InputStream resultStream, EnumSet<ResultField> resultFields) throws Exception { - final String resultStr = IOUtils.toString(resultStream, Charset.defaultCharset()); + private static InputStream extract(InputStream resultStream, EnumSet<ResultField> resultFields, + Charset resultCharset) throws Exception { + final String resultStr = IOUtils.toString(resultStream, resultCharset); final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class); LOGGER.debug("+++++++\n" + result + "\n+++++++\n"); @@ -171,7 +172,7 @@ throw new IllegalStateException("Unexpected result field: " + fieldKind); } } - return IOUtils.toInputStream(resultBuilder.toString(), StandardCharsets.UTF_8); + return IOUtils.toInputStream(resultBuilder, resultCharset); } private static void checkForErrors(ObjectNode result) throws Exception { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index b143ea9..683d5c8 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.test.common; +import static java.nio.charset.StandardCharsets.UTF_8; + import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; @@ -35,12 +37,14 @@ import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; +import java.nio.CharBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -56,6 +60,7 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.asterix.api.http.server.QueryServiceServlet; @@ -139,11 +144,7 @@ public static final Set<String> NON_CANCELLABLE = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("store", "validate"))); - private final IPollTask plainExecutor = (testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, - queryCount, expectedResultFileCtxs, testFile, actualPath) -> { - executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit, queryCount, - expectedResultFileCtxs, testFile, actualPath); - }; + private final IPollTask plainExecutor = this::executeTestFile; public static final String DELIVERY_ASYNC = "async"; public static final String DELIVERY_DEFERRED = "deferred"; @@ -154,6 +155,8 @@ private static final HashMap<Integer, ITestServer> runningTestServers = new HashMap<>(); private static Map<String, InetSocketAddress> ncEndPoints; private static Map<String, InetSocketAddress> replicationAddress; + + private static final List<Charset> charsetsRemaining = new ArrayList<>(); /* * Instance members @@ -211,24 +214,23 @@ } public void runScriptAndCompareWithResult(File scriptFile, File expectedFile, File actualFile, - ComparisonEnum compare) throws Exception { + ComparisonEnum compare, Charset actualEncoding) throws Exception { LOGGER.info("Expected results file: {} ", expectedFile); - BufferedReader readerExpected = - new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); - BufferedReader readerActual = - new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8")); boolean regex = false; - try { + try (BufferedReader readerExpected = + new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8)); + BufferedReader readerActual = + new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), actualEncoding))) { if (ComparisonEnum.BINARY.equals(compare)) { if (!IOUtils.contentEquals(new FileInputStream(actualFile), new FileInputStream(expectedFile))) { throw new Exception("Result for " + scriptFile + ": actual file did not match expected result"); } return; } else if (actualFile.toString().endsWith(".regex")) { - runScriptAndCompareWithResultRegex(scriptFile, expectedFile, actualFile); + runScriptAndCompareWithResultRegex(scriptFile, readerExpected, readerActual); return; } else if (actualFile.toString().endsWith(".regexadm")) { - runScriptAndCompareWithResultRegexAdm(scriptFile, expectedFile, actualFile); + runScriptAndCompareWithResultRegexAdm(scriptFile, readerExpected, readerActual); return; } String lineExpected, lineActual; @@ -278,11 +280,8 @@ throw createLineChangedException(scriptFile, "<EOF>", lineActual, num); } } catch (Exception e) { - LOGGER.info("Actual results file: {}", actualFile); + LOGGER.info("Actual results file: {} encoding: {}", actualFile, actualEncoding); throw e; - } finally { - readerExpected.close(); - readerActual.close(); } } @@ -396,54 +395,48 @@ return true; } - public void runScriptAndCompareWithResultRegex(File scriptFile, File expectedFile, File actualFile) - throws Exception { + public void runScriptAndCompareWithResultRegex(File scriptFile, BufferedReader readerExpected, + BufferedReader readerActual) throws Exception { String lineExpected, lineActual; - try (BufferedReader readerExpected = - new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), "UTF-8")); - BufferedReader readerActual = - new BufferedReader(new InputStreamReader(new FileInputStream(actualFile), "UTF-8"))) { - StringBuilder actual = new StringBuilder(); - while ((lineActual = readerActual.readLine()) != null) { - actual.append(lineActual).append('\n'); + StringBuilder actual = new StringBuilder(); + while ((lineActual = readerActual.readLine()) != null) { + actual.append(lineActual).append('\n'); + } + while ((lineExpected = readerExpected.readLine()) != null) { + if ("".equals(lineExpected.trim())) { + continue; } - while ((lineExpected = readerExpected.readLine()) != null) { - if ("".equals(lineExpected.trim())) { - continue; - } - Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected); - if (!m.matches()) { - throw new IllegalArgumentException( - "Each line of regex file must conform to: [-]/regex/[flags]: " + expectedFile); - } - String negateStr = m.group(1); - String expression = m.group(2); - String flagStr = m.group(3); - boolean negate = "-".equals(negateStr); - int flags = Pattern.MULTILINE; - if (flagStr.contains("m")) { - flags |= Pattern.DOTALL; - } - if (flagStr.contains("i")) { - flags |= Pattern.CASE_INSENSITIVE; - } - Pattern linePattern = Pattern.compile(expression, flags); - boolean match = linePattern.matcher(actual).find(); - if (match && !negate || negate && !match) { - continue; - } - throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression - + "' not found in result: " + actual); + Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected); + if (!m.matches()) { + throw new IllegalArgumentException("Each line of regex file must conform to: [-]/regex/[flags]"); } + String negateStr = m.group(1); + String expression = m.group(2); + String flagStr = m.group(3); + boolean negate = "-".equals(negateStr); + int flags = Pattern.MULTILINE; + if (flagStr.contains("m")) { + flags |= Pattern.DOTALL; + } + if (flagStr.contains("i")) { + flags |= Pattern.CASE_INSENSITIVE; + } + Pattern linePattern = Pattern.compile(expression, flags); + boolean match = linePattern.matcher(actual).find(); + if (match && !negate || negate && !match) { + continue; + } + throw new Exception("Result for " + scriptFile + ": expected pattern '" + expression + + "' not found in result: " + actual); } } - public void runScriptAndCompareWithResultRegexAdm(File scriptFile, File expectedFile, File actualFile) - throws Exception { + public void runScriptAndCompareWithResultRegexAdm(File scriptFile, BufferedReader expectedFile, + BufferedReader actualFile) throws Exception { StringWriter actual = new StringWriter(); StringWriter expected = new StringWriter(); - IOUtils.copy(new FileInputStream(actualFile), actual, StandardCharsets.UTF_8); - IOUtils.copy(new FileInputStream(expectedFile), expected, StandardCharsets.UTF_8); + IOUtils.copy(actualFile, actual); + IOUtils.copy(expectedFile, expected); Pattern pattern = Pattern.compile(expected.toString(), Pattern.DOTALL | Pattern.MULTILINE); if (!pattern.matcher(actual.toString()).matches()) { // figure out where the problem first occurs... @@ -570,21 +563,27 @@ } public InputStream executeQueryService(String str, URI uri, OutputFormat fmt) throws Exception { - return executeQueryService(str, fmt, uri, new ArrayList<>(), false); + return executeQueryService(str, fmt, uri, new ArrayList<>(), false, UTF_8); + } + + public InputStream executeQueryService(String str, URI uri, OutputFormat fmt, Charset resultCharset) + throws Exception { + return executeQueryService(str, fmt, uri, new ArrayList<>(), false, resultCharset); } public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params, - boolean jsonEncoded) throws Exception { - return executeQueryService(str, fmt, uri, params, jsonEncoded, null, false); + boolean jsonEncoded, Charset responseCharset) throws Exception { + return executeQueryService(str, fmt, uri, params, jsonEncoded, responseCharset, null, false); } public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params, boolean jsonEncoded, Predicate<Integer> responseCodeValidator) throws Exception { - return executeQueryService(str, fmt, uri, params, jsonEncoded, responseCodeValidator, false); + return executeQueryService(str, fmt, uri, params, jsonEncoded, UTF_8, responseCodeValidator, false); } public InputStream executeQueryService(String str, OutputFormat fmt, URI uri, List<Parameter> params, - boolean jsonEncoded, Predicate<Integer> responseCodeValidator, boolean cancellable) throws Exception { + boolean jsonEncoded, Charset responseCharset, Predicate<Integer> responseCodeValidator, boolean cancellable) + throws Exception { List<Parameter> newParams = upsertParam(params, "format", ParameterTypeEnum.STRING, fmt.mimeType()); newParams = upsertParam(newParams, QueryServiceServlet.Parameter.PLAN_FORMAT.str(), ParameterTypeEnum.STRING, DEFAULT_PLAN_FORMAT); @@ -602,11 +601,58 @@ // Set accepted output response type method.setHeader("Origin", uri.getScheme() + uri.getAuthority()); method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType()); + method.setHeader("Accept-Charset", responseCharset.name()); + if (!responseCharset.equals(UTF_8)) { + LOGGER.info("using Accept-Charset: {}", responseCharset.name()); + } HttpResponse response = executeHttpRequest(method); if (responseCodeValidator != null) { checkResponse(response, responseCodeValidator); } return response.getEntity().getContent(); + } + + private Charset selectCharset(File result) throws IOException { + // choose an encoding that works for this input + return selectCharset(FileUtils.readFileToString(result, UTF_8)); + } + + private Charset selectCharset(String payload) { + // choose an encoding that works for this input + return nextCharset(charset -> canEncodeDecode(charset, payload)); + } + + public static Charset nextCharset(Predicate<Charset> test) { + synchronized (charsetsRemaining) { + while (true) { + for (Iterator<Charset> iter = charsetsRemaining.iterator(); iter.hasNext();) { + Charset next = iter.next(); + if (test.test(next)) { + iter.remove(); + return next; + } + } + List<Charset> allCharsets = Charset.availableCharsets().values().stream() + .filter(c -> canEncodeDecode(c, "\n\t\\[]{}'\"")).collect(Collectors.toList()); + Collections.shuffle(allCharsets); + charsetsRemaining.addAll(allCharsets); + } + } + } + + // duplicated from hyracks-test-support as transitive dependencies on test-jars are not handled correctly + private static boolean canEncodeDecode(Charset charset, String input) { + try { + if (input.equals(new String(input.getBytes(charset), charset))) { + // workaround for https://bugs.openjdk.java.net/browse/JDK-6392670 and similar + if (input.equals(charset.decode(charset.encode(CharBuffer.wrap(input))).toString())) { + return true; + } + } + } catch (Exception e) { + LOGGER.debug("cannot encode / decode {} with {} due to exception", input, charset.displayName(), e); + } + return false; } protected List<Parameter> upsertParam(List<Parameter> params, String name, ParameterTypeEnum type, String value) { @@ -646,7 +692,6 @@ for (Parameter param : params) { builder.addParameter(param.getName(), param.getValue()); } - builder.setCharset(StandardCharsets.UTF_8); return builder.build(); } @@ -656,8 +701,8 @@ for (Parameter param : params) { builder.addParameter(param.getName(), param.getValue()); } - builder.setCharset(StandardCharsets.UTF_8); - body.ifPresent(s -> builder.setEntity(new StringEntity(s, StandardCharsets.UTF_8))); + builder.setCharset(UTF_8); + body.ifPresent(s -> builder.setEntity(new StringEntity(s, UTF_8))); return builder.build(); } @@ -681,7 +726,7 @@ for (Parameter param : params) { builder.addParameter(param.getName(), param.getValue()); } - builder.setCharset(StandardCharsets.UTF_8); + builder.setCharset(UTF_8); return builder.build(); } @@ -695,9 +740,9 @@ builder.addParameter(stmtParam, statement); } else { // this seems pretty bad - we should probably fix the API and not the client - builder.setEntity(new StringEntity(statement, StandardCharsets.UTF_8)); + builder.setEntity(new StringEntity(statement, UTF_8)); } - builder.setCharset(StandardCharsets.UTF_8); + builder.setCharset(UTF_8); return builder.build(); } @@ -732,7 +777,7 @@ } catch (JsonProcessingException e) { e.printStackTrace(); } - builder.setCharset(StandardCharsets.UTF_8); + builder.setCharset(UTF_8); return builder.build(); } @@ -765,8 +810,7 @@ // and returns the contents as a string // This string is later passed to REST API for execution. public String readTestFile(File testFile) throws Exception { - BufferedReader reader = - new BufferedReader(new InputStreamReader(new FileInputStream(testFile), StandardCharsets.UTF_8)); + BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(testFile), UTF_8)); String line; StringBuilder stringBuilder = new StringBuilder(); String ls = System.getProperty("line.separator"); @@ -820,9 +864,9 @@ future.get(); ByteArrayInputStream bisIn = new ByteArrayInputStream(baos.toByteArray()); StringWriter writerIn = new StringWriter(); - IOUtils.copy(bisIn, writerIn, StandardCharsets.UTF_8); + IOUtils.copy(bisIn, writerIn, UTF_8); StringWriter writerErr = new StringWriter(); - IOUtils.copy(p.getErrorStream(), writerErr, StandardCharsets.UTF_8); + IOUtils.copy(p.getErrorStream(), writerErr, UTF_8); StringBuffer stdOut = writerIn.getBuffer(); if (writerErr.getBuffer().length() > 0) { @@ -918,18 +962,18 @@ expectedResultFileCtxs); break; case "txnqbc": // qbc represents query before crash - resultStream = query(cUnit, testFile.getName(), statement); + resultStream = query(cUnit, testFile.getName(), statement, UTF_8); qbcFile = getTestCaseQueryBeforeCrashFile(actualPath, testCaseCtx, cUnit); writeOutputToFile(qbcFile, resultStream); break; case "txnqar": // qar represents query after recovery - resultStream = query(cUnit, testFile.getName(), statement); + resultStream = query(cUnit, testFile.getName(), statement, UTF_8); File qarFile = new File(actualPath + File.separator + testCaseCtx.getTestCase().getFilePath().replace(File.separator, "_") + "_" + cUnit.getName() + "_qar.adm"); writeOutputToFile(qarFile, resultStream); qbcFile = getTestCaseQueryBeforeCrashFile(actualPath, testCaseCtx, cUnit); - runScriptAndCompareWithResult(testFile, qbcFile, qarFile, ComparisonEnum.TEXT); + runScriptAndCompareWithResult(testFile, qbcFile, qarFile, ComparisonEnum.TEXT, UTF_8); break; case "txneu": // eu represents erroneous update try { @@ -1172,7 +1216,7 @@ throw new IllegalArgumentException("Unexpected format for method " + reqType + ": " + extension); } if (handleVar != null) { - String handle = ResultExtractor.extractHandle(resultStream); + String handle = ResultExtractor.extractHandle(resultStream, UTF_8); if (handle != null) { variableCtx.put(handleVar, handle); } else { @@ -1181,15 +1225,15 @@ } else { if (expectedResultFile == null) { if (testFile.getName().startsWith(DIAGNOSE)) { - LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8)); + LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, UTF_8)); } else { - LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8)); + LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, UTF_8)); Assert.fail("no result file for " + testFile.toString() + "; queryCount: " + queryCount + ", filectxs.size: " + numResultFiles); } } else { writeOutputToFile(actualResultFile, resultStream); - runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare); + runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, UTF_8); } } queryCount.increment(); @@ -1207,23 +1251,25 @@ URI uri = testFile.getName().endsWith("aql") ? getEndpoint(Servlets.QUERY_AQL) : getEndpoint(Servlets.QUERY_SERVICE); boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement)); + Charset responseCharset = expectedResultFile == null ? UTF_8 : selectCharset(expectedResultFile); InputStream resultStream; if (DELIVERY_IMMEDIATE.equals(delivery)) { + resultStream = executeQueryService(statement, fmt, uri, params, isJsonEncoded, responseCharset, null, + isCancellable(reqType)); resultStream = - executeQueryService(statement, fmt, uri, params, isJsonEncoded, null, isCancellable(reqType)); - resultStream = METRICS_QUERY_TYPE.equals(reqType) ? ResultExtractor.extractMetrics(resultStream) - : ResultExtractor.extract(resultStream); + METRICS_QUERY_TYPE.equals(reqType) ? ResultExtractor.extractMetrics(resultStream, responseCharset) + : ResultExtractor.extract(resultStream, responseCharset); } else { String handleVar = getHandleVariable(statement); resultStream = executeQueryService(statement, fmt, uri, - upsertParam(params, "mode", ParameterTypeEnum.STRING, delivery), isJsonEncoded); - String handle = ResultExtractor.extractHandle(resultStream); + upsertParam(params, "mode", ParameterTypeEnum.STRING, delivery), isJsonEncoded, responseCharset); + String handle = ResultExtractor.extractHandle(resultStream, responseCharset); Assert.assertNotNull("no handle for " + reqType + " test " + testFile.toString(), handleVar); variableCtx.put(handleVar, toQueryServiceHandle(handle)); } if (actualResultFile == null) { if (testFile.getName().startsWith(DIAGNOSE)) { - LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, StandardCharsets.UTF_8)); + LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, responseCharset)); } else { Assert.fail("no result file for " + testFile.toString() + "; queryCount: " + queryCount + ", filectxs.size: " + numResultFiles); @@ -1238,7 +1284,7 @@ + ", filectxs.size: " + numResultFiles); } } - runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare); + runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, responseCharset); if (!reqType.equals("validate")) { queryCount.increment(); } @@ -1266,8 +1312,7 @@ throw new Exception( "Failed to delete an existing result file: " + actualResultFile.getAbsolutePath()); } - writeOutputToFile(actualResultFile, - new ByteArrayInputStream(poller.poll().getBytes(StandardCharsets.UTF_8))); + writeOutputToFile(actualResultFile, new ByteArrayInputStream(poller.poll().getBytes(UTF_8))); variableCtx.put(key, actualResultFile); validate(actualPath, testCaseCtx, cUnit, statement, variableCtx, testFile, ctx, queryCount, expectedResultFileCtxs); @@ -1399,8 +1444,8 @@ private InputStream executeUpdateOrDdl(String statement, OutputFormat outputFormat, URI serviceUri) throws Exception { - InputStream resultStream = executeQueryService(statement, serviceUri, outputFormat); - return ResultExtractor.extract(resultStream); + InputStream resultStream = executeQueryService(statement, serviceUri, outputFormat, UTF_8); + return ResultExtractor.extract(resultStream, UTF_8); } protected static boolean isExpected(Exception e, CompilationUnit cUnit) { @@ -1554,7 +1599,7 @@ String endpoint = "/admin/cluster/node/" + nodeId + "/config"; InputStream executeJSONGet = executeJSONGet(fmt, createEndpointURI(endpoint, null)); StringWriter actual = new StringWriter(); - IOUtils.copy(executeJSONGet, actual, StandardCharsets.UTF_8); + IOUtils.copy(executeJSONGet, actual, UTF_8); String config = actual.toString(); int nodePid = new ObjectMapper().readValue(config, ObjectNode.class).get("pid").asInt(); if (nodePid <= 1) { @@ -1584,7 +1629,7 @@ String endpoint = "/admin/cluster/node/" + nodeId + "/config"; InputStream executeJSONGet = executeJSONGet(fmt, createEndpointURI(endpoint, null)); StringWriter actual = new StringWriter(); - IOUtils.copy(executeJSONGet, actual, StandardCharsets.UTF_8); + IOUtils.copy(executeJSONGet, actual, UTF_8); String config = actual.toString(); ObjectMapper om = new ObjectMapper(); String logDir = om.readTree(config).findPath("txn.log.dir").asText(); @@ -1761,8 +1806,8 @@ ArrayList<String> toBeDropped = new ArrayList<>(); InputStream resultStream = executeQueryService( "select dv.DataverseName from Metadata.`Dataverse` as dv order by dv.DataverseName;", - getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON); - String out = IOUtils.toString(resultStream, StandardCharsets.UTF_8); + getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.CLEAN_JSON, UTF_8); + String out = IOUtils.toString(resultStream, UTF_8); ObjectMapper om = new ObjectMapper(); om.setConfig(om.getDeserializationConfig().with(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT)); JsonNode result; @@ -1794,8 +1839,8 @@ dropStatement.append(dv); dropStatement.append(";\n"); resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE), - OutputFormat.CLEAN_JSON); - ResultExtractor.extract(resultStream); + OutputFormat.CLEAN_JSON, UTF_8); + ResultExtractor.extract(resultStream, UTF_8); } } } catch (Throwable th) { @@ -1964,11 +2009,12 @@ return !NON_CANCELLABLE.contains(type); } - private InputStream query(CompilationUnit cUnit, String testFile, String statement) throws Exception { + private InputStream query(CompilationUnit cUnit, String testFile, String statement, Charset responseCharset) + throws Exception { final URI uri = getQueryServiceUri(testFile); final InputStream inputStream = executeQueryService(statement, OutputFormat.forCompilationUnit(cUnit), uri, - cUnit.getParameter(), true, null, false); - return ResultExtractor.extract(inputStream); + cUnit.getParameter(), true, responseCharset, null, false); + return ResultExtractor.extract(inputStream, responseCharset); } private URI getQueryServiceUri(String extension) throws URISyntaxException { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java index e4e300a..5dd41c3 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -149,7 +150,8 @@ } writer.close(); // Compares the actual result and the expected result. - runScriptAndCompareWithResult(queryFile, expectedFile, actualResultFile, ComparisonEnum.TEXT); + runScriptAndCompareWithResult(queryFile, expectedFile, actualResultFile, ComparisonEnum.TEXT, + StandardCharsets.UTF_8); } catch (Exception e) { GlobalConfig.ASTERIX_LOGGER.warn("Failed while testing file " + queryFile); throw e; diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java index a8b43b7..905497e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/Duration.java @@ -47,13 +47,25 @@ this.nanoDigits = nanoDigits; } + public String asciiSafeUnit() { + return this == MICRO ? "us" : unit; + } + public static String formatNanos(long nanoTime) { + return formatNanos(nanoTime, false); + } + + public static String formatNanos(long nanoTime, boolean asciiSafe) { StringBuilder sb = new StringBuilder(); - formatNanos(nanoTime, sb); + formatNanos(nanoTime, sb, asciiSafe); return sb.toString(); } public static void formatNanos(long nanoTime, StringBuilder out) { + formatNanos(nanoTime, out, false); + } + + public static void formatNanos(long nanoTime, StringBuilder out, boolean asciiSafe) { final String strTime = String.valueOf(Math.abs(nanoTime)); final int len = strTime.length(); for (Duration tu : VALUES) { @@ -67,7 +79,7 @@ if (k > 0) { out.append('.').append(strTime, n, k + 1); } - out.append(tu.unit); + out.append(asciiSafe ? tu.asciiSafeUnit() : tu.unit); break; } } diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java index b762517..7e29ed5 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/base/TestMethodTracer.java @@ -56,7 +56,7 @@ @Override protected void failed(Throwable e, Description description) { - LOGGER.log(level, "### {} FAILED ({})", description.getMethodName(), e.getClass().getName()); + LOGGER.log(level, "### {} FAILED", description.getMethodName(), e); } @Override diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java index 639e036..17c98bc 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java @@ -67,13 +67,12 @@ String feedType = req.getParameter(FEED_TYPE); feedType = (feedType != null) ? feedType : defaultFeedType; feed.setFeedType(feedType); - HttpUtil.setContentType(res, MIME_TYPE); + HttpUtil.setContentType(res, MIME_TYPE, req); SyndFeedOutput output = new SyndFeedOutput(); output.output(feed, res.writer()); } catch (FeedException | ParseException ex) { GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, ex.getMessage(), ex); - String msg = COULD_NOT_GENERATE_FEED_ERROR; - res.writer().print(msg); + res.writer().print(COULD_NOT_GENERATE_FEED_ERROR); res.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR); } } diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh index e337cb0..e0cff32 100755 --- a/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh +++ b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh @@ -86,7 +86,7 @@ fi fi -export JAVA_VERSION=$(java -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"') +export JAVA_VERSION=$($JAVACMD -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"') case $JAVA_VERSION in 1.8*|1.9*|10*|11*) ;; diff --git a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh index 522fb7c..80647d4 100755 --- a/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh +++ b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh @@ -99,11 +99,14 @@ JAVACMD=`which java` fi fi -"$JAVACMD" -version 2>&1 | grep -q '1\.[89]' || { - echo "JAVA_HOME must be at version 1.8 or later:" - "$JAVACMD" -version +export JAVA_VERSION=$($JAVACMD -version 2>&1 | head -1 | awk '{ print $3 }' | tr -d '"') +case $JAVA_VERSION in + 1.8*|1.9*|10*|11*) + ;; + *) + echo JAVA_HOME must be at version 1.8 or later, but is: $JAVA_VERSION exit 2 -} +esac DIRNAME=$(dirname "$0") [ $(echo $DIRNAME | wc -l) -ne 1 ] && { echo "Paths with spaces are not supported" diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java index 0479bb2..cd72591 100644 --- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java +++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/SampleLocalClusterIT.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -121,7 +122,7 @@ public void test1_sanityQuery() throws Exception { TestExecutor testExecutor = new TestExecutor(); InputStream resultStream = testExecutor.executeQueryService("1+1;", - testExecutor.getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.ADM); + testExecutor.getEndpoint(Servlets.QUERY_SERVICE), OutputFormat.ADM, StandardCharsets.UTF_8); final ObjectMapper objectMapper = new ObjectMapper(); final ObjectNode response = objectMapper.readValue(resultStream, ObjectNode.class); final JsonNode result = response.get("results"); diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml index fbf17c1..0336790 100644 --- a/asterixdb/pom.xml +++ b/asterixdb/pom.xml @@ -1085,6 +1085,12 @@ </dependency> <dependency> <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-util</artifactId> + <version>${hyracks.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> <artifactId>hyracks-dataflow-std</artifactId> <version>${hyracks.version}</version> </dependency> diff --git a/hyracks-fullstack/hyracks/hyracks-http/pom.xml b/hyracks-fullstack/hyracks/hyracks-http/pom.xml index a67fa15..46e2004 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-http/pom.xml @@ -95,5 +95,11 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hyracks</groupId> + <artifactId>hyracks-test-support</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> </dependencies> </project> \ No newline at end of file diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java index e72bee9..ac87592 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java @@ -107,7 +107,7 @@ public synchronized PrintWriter writer() { if (writer == null) { Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8); - writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(outputStream, charset))); + writer = new PrintWriter(new OutputStreamWriter(outputStream, charset)); } return writer; } diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java index 85a0a43..b42db39 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java @@ -52,7 +52,7 @@ public FullResponse(ChannelHandlerContext ctx, FullHttpRequest request) { this.ctx = ctx; - baos = new ByteArrayOutputStream(); + baos = new ByteArrayOutputStream(4096); response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.INTERNAL_SERVER_ERROR); keepAlive = HttpUtil.isKeepAlive(request); if (keepAlive) { @@ -89,7 +89,7 @@ public synchronized PrintWriter writer() { if (writer == null) { Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(response, StandardCharsets.UTF_8); - writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(baos, charset))); + writer = new PrintWriter(new OutputStreamWriter(baos, charset)); } return writer; } diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java index b34b9dc..6e4a273 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java @@ -45,7 +45,7 @@ public class HttpUtil { private static final Logger LOGGER = LogManager.getLogger(); private static final Pattern PARENT_DIR = Pattern.compile("/[^./]+/\\.\\./"); - private static final String DEFAULT_RESPONSE_CHARSET = StandardCharsets.UTF_8.name(); + private static final Charset DEFAULT_RESPONSE_CHARSET = StandardCharsets.UTF_8; private HttpUtil() { } @@ -100,15 +100,20 @@ return contentType == null ? null : contentType.split(";")[0]; } - public static String getRequestBody(IServletRequest request) { - FullHttpRequest httpRequest = request.getHttpRequest(); - Charset charset = io.netty.handler.codec.http.HttpUtil.getCharset(httpRequest, StandardCharsets.UTF_8); - return httpRequest.content().toString(charset); + public static Charset getRequestCharset(HttpRequest request) { + return io.netty.handler.codec.http.HttpUtil.getCharset(request, StandardCharsets.UTF_8); } - public static void setContentType(IServletResponse response, String type, IServletRequest fromRequest) + public static String getRequestBody(IServletRequest request) { + FullHttpRequest httpRequest = request.getHttpRequest(); + return httpRequest.content().toString(getRequestCharset(httpRequest)); + } + + public static Charset setContentType(IServletResponse response, String type, IServletRequest fromRequest) throws IOException { - response.setHeader(HttpHeaderNames.CONTENT_TYPE, type + "; charset=" + getPreferredCharset(fromRequest)); + Charset preferredCharset = getPreferredCharset(fromRequest); + response.setHeader(HttpHeaderNames.CONTENT_TYPE, type + "; charset=" + preferredCharset.name()); + return preferredCharset; } public static void setContentType(IServletResponse response, String type, String charset) throws IOException { @@ -169,25 +174,24 @@ return clusterURL; } - public static String getPreferredCharset(IServletRequest request) { + public static Charset getPreferredCharset(IServletRequest request) { return getPreferredCharset(request, DEFAULT_RESPONSE_CHARSET); } - public static String getPreferredCharset(IServletRequest request, String defaultCharset) { + public static Charset getPreferredCharset(IServletRequest request, Charset defaultCharset) { String acceptCharset = request.getHeader(HttpHeaderNames.ACCEPT_CHARSET); if (acceptCharset == null) { return defaultCharset; } // If no "q" parameter is present, the default weight is 1 [https://tools.ietf.org/html/rfc7231#section-5.3.1] - Optional<String> preferredCharset = Stream.of(StringUtils.split(acceptCharset, ",")) - .map(WeightedHeaderValue::new).sorted().map(WeightedHeaderValue::getValue) - .map(a -> "*".equals(a) ? defaultCharset : a).filter(value -> { + Optional<Charset> preferredCharset = Stream.of(StringUtils.split(acceptCharset, ",")) + .map(WeightedHeaderValue::new).sorted().map(WeightedHeaderValue::getValueDefaultStar).filter(value -> { if (!Charset.isSupported(value)) { LOGGER.info("disregarding unsupported charset '{}'", value); return false; } return true; - }).findFirst(); + }).map(Charset::forName).findFirst(); return preferredCharset.orElse(defaultCharset); } @@ -216,6 +220,10 @@ return value; } + public String getValueDefaultStar() { + return "*".equals(value) ? DEFAULT_RESPONSE_CHARSET.name() : value; + } + public double getWeight() { return weight; } diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java index a166e52..260da99 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpAcceptCharsetTest.java @@ -61,7 +61,7 @@ @Test public void testAmbiguous() { IServletRequest request = withCharset("utf-8;q=.75,utf-16;q=.75,utf-32;q=.5"); - String preferredCharset = HttpUtil.getPreferredCharset(request); + String preferredCharset = HttpUtil.getPreferredCharset(request).name(); Assert.assertTrue("ambiguous by weight (got: " + preferredCharset + ")", preferredCharset.toLowerCase().matches("utf-(8|16)")); } @@ -83,7 +83,7 @@ } else if (charsetObject instanceof String) { return ((String) charsetObject).toLowerCase(); } else if (charsetObject instanceof IServletRequest) { - return HttpUtil.getPreferredCharset((IServletRequest) charsetObject).toLowerCase(); + return HttpUtil.getPreferredCharset((IServletRequest) charsetObject).name().toLowerCase(); } throw new IllegalArgumentException("unknown type: " + charsetObject.getClass()); } diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java index c71c5e4..b522122 100644 --- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java +++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerEncodingTest.java @@ -27,7 +27,6 @@ import java.util.Collection; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.http.HttpEntity; @@ -44,8 +43,7 @@ import org.apache.hyracks.http.server.HttpServerConfigBuilder; import org.apache.hyracks.http.server.WebManager; import org.apache.hyracks.test.http.servlet.CompliantEchoServlet; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.hyracks.test.string.EncodingUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -57,8 +55,6 @@ @RunWith(Parameterized.class) public class HttpServerEncodingTest { - private static final Logger LOGGER = LogManager.getLogger(); - private static final int PORT = 9898; private static final String HOST = "localhost"; private static final String PROTOCOL = "http"; @@ -72,25 +68,12 @@ List<Object[]> tests = new ArrayList<>(); Stream.of("encoding is hard", "中文字符", "لا يوجد ترجمة لكُ", STRING_NEEDS_2_JAVA_CHARS_1, STRING_NEEDS_2_JAVA_CHARS_2).forEach(input -> { - Set<Charset> legalCharsets = getLegalCharsetsFor(input); + Set<Charset> legalCharsets = EncodingUtils.getLegalCharsetsFor(input); legalCharsets.forEach(charsetIn -> legalCharsets.forEach(charsetOut -> tests .add(new Object[] { input + ":" + charsetIn.displayName() + "->" + charsetOut.displayName(), input, charsetIn, charsetOut }))); }); return tests; - } - - private static Set<Charset> getLegalCharsetsFor(String input) { - return Charset.availableCharsets().values().stream().filter(Charset::canEncode) - .filter(test -> canEncodeDecode(input, test)).collect(Collectors.toSet()); - } - - private static boolean canEncodeDecode(String input, Charset charset) { - if (input.equals(new String(input.getBytes(charset), charset))) { - return true; - } - LOGGER.info("cannot encode / decode {} with {}", input, charset.displayName()); - return false; } @Parameter(0) diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java new file mode 100644 index 0000000..9ba4817 --- /dev/null +++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/string/EncodingUtils.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hyracks.test.string; + +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class EncodingUtils { + private static final Logger LOGGER = LogManager.getLogger(); + + private EncodingUtils() { + } + + // duplicated in [asterix-app]TestExecutor.java as transitive dependencies on test-jars are not handled correctly + public static boolean canEncodeDecode(String input, Charset charset, boolean quiet) { + try { + if (input.equals(new String(input.getBytes(charset), charset))) { + if (!input.equals(charset.decode(charset.encode(CharBuffer.wrap(input))).toString())) { + // workaround for https://bugs.openjdk.java.net/browse/JDK-6392670 and similar + if (!quiet) { + LOGGER.info("cannot encode / decode {} with {} using CharBuffer.wrap(<String>)", input, + charset.displayName()); + } + } else { + return true; + } + } + if (!quiet) { + LOGGER.info("cannot encode / decode {} with {}", input, charset.displayName()); + } + } catch (Exception e) { + if (!quiet) { + LOGGER.info("cannot encode / decode {} with {}, got exception ({})", input, charset.displayName(), + String.valueOf(e)); + } + } + return false; + } + + public static Set<Charset> getLegalCharsetsFor(String input) { + return Charset.availableCharsets().values().stream().filter(Charset::canEncode) + .filter(test -> canEncodeDecode(input, test, false)).collect(Collectors.toSet()); + } +} -- To view, visit https://asterix-gerrit.ics.uci.edu/3191 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8f37eb684bf2457e5ff451bf5c8fbca742d531f2 Gerrit-PatchSet: 25 Gerrit-Project: asterixdb Gerrit-Branch: stabilization-f69489 Gerrit-Owner: Michael Blow <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]>
