Till Westmann has submitted this change and it was merged. Change subject: QueryService API updates ......................................................................
QueryService API updates - API returns non-JSON results (ADM/CSV) as arrays of (escaped) strings - fix encoding and content-length of response - run SQL++ query tests through QueryService API - fix tests/expected errors - correct execution times in the case of errors - re-structure printing of CSV headers - improve parameter handling - small API cleanup Change-Id: Ie67ad4ea31699400726c8c026c4a91edc698f2b5 Reviewed-on: https://asterix-gerrit.ics.uci.edu/896 Tested-by: Jenkins <[email protected]> Reviewed-by: Yingyi Bu <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml R asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java A asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java M asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java 11 files changed, 575 insertions(+), 289 deletions(-) Approvals: Yingyi Bu: Looks good to me, approved Jenkins: Verified Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java index 7bfc55c..c09b424 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java @@ -95,7 +95,12 @@ /** * Format flag: indent JSON results. */ - public static final String INDENT_JSON = "indent-json"; + public static final String FORMAT_INDENT_JSON = "indent-json"; + + /** + * Format flag: quote records in the results array. + */ + public static final String FORMAT_QUOTE_RECORD = "quote-record"; public interface ResultDecorator { PrintWriter print(PrintWriter pw); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java index ba8644a..7a47ca9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java @@ -27,6 +27,7 @@ import java.io.PrintWriter; import java.util.List; import java.util.logging.Level; +import java.util.logging.Logger; import javax.imageio.ImageIO; import javax.servlet.ServletContext; @@ -57,53 +58,34 @@ public class APIServlet extends HttpServlet { private static final long serialVersionUID = 1L; - private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION"; + private static final Logger LOGGER = Logger.getLogger(APIServlet.class.getName()); + private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION"; private static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET"; private final ILangCompilationProvider aqlCompilationProvider; - private final IParserFactory aqlParserFactory; private final ILangCompilationProvider sqlppCompilationProvider; - private final IParserFactory sqlppParserFactory; public APIServlet() { this.aqlCompilationProvider = new AqlCompilationProvider(); - this.aqlParserFactory = aqlCompilationProvider.getParserFactory(); - this.sqlppCompilationProvider = new SqlppCompilationProvider(); - this.sqlppParserFactory = sqlppCompilationProvider.getParserFactory(); } @Override public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException { // Query language - ILangCompilationProvider compilationProvider; - IParserFactory parserFactory; - String lang = request.getParameter("query-language"); - if (lang.equals("AQL")) { - // Uses AQL compiler. - compilationProvider = aqlCompilationProvider; - parserFactory = aqlParserFactory; - } else { - // Uses SQL++ compiler. - compilationProvider = sqlppCompilationProvider; - parserFactory = sqlppParserFactory; - } + ILangCompilationProvider compilationProvider = "AQL".equals(request.getParameter("query-language")) + ? aqlCompilationProvider : sqlppCompilationProvider; + IParserFactory parserFactory = compilationProvider.getParserFactory(); // Output format. OutputFormat format; boolean csv_and_header = false; String output = request.getParameter("output-format"); - if (output.equals("ADM")) { - format = OutputFormat.ADM; - } else if (output.equals("CSV")) { - format = OutputFormat.CSV; - } else if (output.equals("CSV-Header")) { - format = OutputFormat.CSV; - csv_and_header = true; - } else if (output.equals("LOSSLESS_JSON")) { - format = OutputFormat.LOSSLESS_JSON; - } else { + try { + format = OutputFormat.valueOf(output); + } catch (IllegalArgumentException e) { + LOGGER.info(output + ": unsupported output-format, using " + OutputFormat.CLEAN_JSON + " instead"); // Default output format format = OutputFormat.CLEAN_JSON; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java index d150e5d..3198759 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java @@ -88,7 +88,7 @@ // originally determined there. Need to save this value on // some object that we can obtain here. SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response); - ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats()); + ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats(), null); } catch (Exception e) { out.println(e.getMessage()); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java index c62389f..8fa09a4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java @@ -37,6 +37,7 @@ import org.apache.asterix.aql.translator.QueryTranslator; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.utils.JSONUtil; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.lang.aql.parser.TokenMgrError; @@ -55,24 +56,21 @@ private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName()); - public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION"; - public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET"; + private static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION"; + private static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET"; + + private transient final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider(); public enum Parameter { // Standard - statement, - format, + STATEMENT("statement"), + FORMAT("format"), // Asterix - header, - indent - } - - public enum Header { - Accept("Accept"); + INDENT("indent"); private final String str; - Header(String str) { + Parameter(String str) { this.str = str; } @@ -81,9 +79,10 @@ } } - public enum MediaType { + private enum MediaType { CSV("text/csv"), - JSON("application/json"); + JSON("application/json"), + ADM("application/x-adm"); private final String str; @@ -96,36 +95,92 @@ } } - public enum ResultFields { - requestID, - signature, - status, - results, - errors, - metrics + private enum Attribute { + HEADER("header"), + LOSSLESS("lossless"); + + private final String str; + + Attribute(String str) { + this.str = str; + } + + public String str() { + return str; + } } - public enum ResultStatus { - success, - timeout, - errors, - fatal + private enum ResultFields { + REQUEST_ID("requestID"), + SIGNATURE("signature"), + TYPE("type"), + STATUS("status"), + RESULTS("results"), + ERRORS("errors"), + METRICS("metrics"); + + private final String str; + + ResultFields(String str) { + this.str = str; + } + + public String str() { + return str; + } } - public enum ErrorField { - code, - msg, - stack + private enum ResultStatus { + SUCCESS("success"), + TIMEOUT("timeout"), + ERRORS("errors"), + FATAL("fatal"); + + private final String str; + + ResultStatus(String str) { + this.str = str; + } + + public String str() { + return str; + } } - public enum Metrics { - elapsedTime, - executionTime, - resultCount, - resultSize + private enum ErrorField { + CODE("code"), + MSG("msg"), + STACK("stack"); + + private final String str; + + ErrorField(String str) { + this.str = str; + } + + public String str() { + return str; + } } - public enum TimeUnit { + private enum Metrics { + ELAPSED_TIME("elapsedTime"), + EXECUTION_TIME("executionTime"), + RESULT_COUNT("resultCount"), + RESULT_SIZE("resultSize"); + + private final String str; + + Metrics(String str) { + this.str = str; + } + + public String str() { + return str; + } + } + + enum TimeUnit { SEC("s", 9), MILLI("ms", 6), MICRO("µs", 3), @@ -153,18 +208,33 @@ } } - private final ILangCompilationProvider compilationProvider = new SqlppCompilationProvider(); - - static SessionConfig.OutputFormat getFormat(HttpServletRequest request) { - // First check the "format" parameter. - String format = request.getParameter(Parameter.format.name()); - if (format != null && format.equals("CSV")) { - return SessionConfig.OutputFormat.CSV; + private static String getParameterValue(String content, String attribute) { + int sc = content.indexOf(';'); + if (sc < 0) { + return null; } - // Second check the Accept: HTTP header. - String accept = request.getHeader(Header.Accept.str()); - if (accept != null && accept.contains(MediaType.CSV.str())) { - return SessionConfig.OutputFormat.CSV; + int eq = content.indexOf('=', sc + 1); + if (eq < 0) { + return null; + } + if (content.substring(sc + 1, eq).trim().equalsIgnoreCase(attribute)) { + return content.substring(eq + 1).trim().toLowerCase(); + } + return null; + } + + private static SessionConfig.OutputFormat getFormat(String format) { + if (format != null) { + if (format.startsWith(MediaType.CSV.str())) { + return SessionConfig.OutputFormat.CSV; + } + if (format.equals(MediaType.ADM.str())) { + return SessionConfig.OutputFormat.ADM; + } + if (format.startsWith(MediaType.JSON.str())) { + return Boolean.parseBoolean(getParameterValue(format, "lossless")) + ? SessionConfig.OutputFormat.LOSSLESS_JSON : SessionConfig.OutputFormat.CLEAN_JSON; + } } return SessionConfig.OutputFormat.CLEAN_JSON; } @@ -173,60 +243,37 @@ * Construct a SessionConfig with the appropriate output writer and * output-format based on the Accept: header and other servlet parameters. */ - static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) { - SessionConfig.ResultDecorator resultPrefix = new SessionConfig.ResultDecorator() { - @Override - public PrintWriter print(PrintWriter pw) { - pw.print("\t\""); - pw.print(ResultFields.results.name()); - pw.print("\": "); - return pw; - } + private static SessionConfig createSessionConfig(HttpServletRequest request, PrintWriter resultWriter) { + SessionConfig.ResultDecorator resultPrefix = (PrintWriter pw) -> { + pw.print("\t\""); + pw.print(ResultFields.RESULTS.str()); + pw.print("\": "); + return pw; }; - SessionConfig.ResultDecorator resultPostfix = new SessionConfig.ResultDecorator() { - @Override - public PrintWriter print(PrintWriter pw) { - pw.print("\t,\n"); - return pw; - } + SessionConfig.ResultDecorator resultPostfix = (PrintWriter pw) -> { + pw.print("\t,\n"); + return pw; }; - SessionConfig.OutputFormat format = getFormat(request); + String formatstr = request.getParameter(Parameter.FORMAT.str()).toLowerCase(); + SessionConfig.OutputFormat format = getFormat(formatstr); SessionConfig sessionConfig = new SessionConfig(resultWriter, format, resultPrefix, resultPostfix); - sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, format == SessionConfig.OutputFormat.CLEAN_JSON); - boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.indent.name())); - sessionConfig.set(SessionConfig.INDENT_JSON, indentJson); - - if (format == SessionConfig.OutputFormat.CSV && ("present".equals(request.getParameter(Parameter.header.name())) - || request.getHeader(Header.Accept.str()).contains("header=present"))) { - sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true); - } + sessionConfig.set(SessionConfig.FORMAT_WRAPPER_ARRAY, true); + boolean indentJson = Boolean.parseBoolean(request.getParameter(Parameter.INDENT.str())); + sessionConfig.set(SessionConfig.FORMAT_INDENT_JSON, indentJson); + sessionConfig.set(SessionConfig.FORMAT_QUOTE_RECORD, + format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON); + sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, + format == SessionConfig.OutputFormat.CSV && "present".equals(getParameterValue(formatstr, "header"))); return sessionConfig; } - /** - * Initialize the Content-Type of the response based on a SessionConfig. - */ - static void initResponse(HttpServletResponse response, SessionConfig sessionConfig) throws IOException { - response.setCharacterEncoding("utf-8"); - switch (sessionConfig.fmt()) { - case CLEAN_JSON: - response.setContentType(MediaType.JSON.str()); - break; - case CSV: - String contentType = MediaType.CSV.str() + "; header=" - + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent"); - response.setContentType(contentType); - break; - } - } - - static void printField(PrintWriter pw, String name, String value) { + private static void printField(PrintWriter pw, String name, String value) { printField(pw, name, value, true); } - static void printField(PrintWriter pw, String name, String value, boolean comma) { + private static void printField(PrintWriter pw, String name, String value, boolean comma) { pw.print("\t\""); pw.print(name); pw.print("\": \""); @@ -238,89 +285,120 @@ pw.print('\n'); } - static UUID printRequestId(PrintWriter pw) { + private static UUID printRequestId(PrintWriter pw) { UUID requestId = UUID.randomUUID(); - printField(pw, ResultFields.requestID.name(), requestId.toString()); + printField(pw, ResultFields.REQUEST_ID.str(), requestId.toString()); return requestId; } - static void printSignature(PrintWriter pw) { - printField(pw, ResultFields.signature.name(), "*"); + private static void printSignature(PrintWriter pw) { + printField(pw, ResultFields.SIGNATURE.str(), "*"); } - static void printStatus(PrintWriter pw, ResultStatus rs) { - printField(pw, ResultFields.status.name(), rs.name()); + private static void printType(PrintWriter pw, SessionConfig sessionConfig) { + switch (sessionConfig.fmt()) { + case ADM: + printField(pw, ResultFields.TYPE.str(), MediaType.ADM.str()); + break; + case CSV: + String contentType = MediaType.CSV.str() + "; header=" + + (sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER) ? "present" : "absent"); + printField(pw, ResultFields.TYPE.str(), contentType); + break; + default: + break; + } } - static void printError(PrintWriter pw, Throwable e) { + private static void printStatus(PrintWriter pw, ResultStatus rs) { + printField(pw, ResultFields.STATUS.str(), rs.str()); + } + + private static void printError(PrintWriter pw, Throwable e) { final boolean addStack = false; pw.print("\t\""); - pw.print(ResultFields.errors.name()); + pw.print(ResultFields.ERRORS.str()); pw.print("\": [{ \n"); - printField(pw, ErrorField.code.name(), "1"); - printField(pw, ErrorField.msg.name(), JSONUtil.escape(e.getMessage()), addStack); + printField(pw, ErrorField.CODE.str(), "1"); + final String msg = e.getMessage(); + printField(pw, ErrorField.MSG.str(), JSONUtil.escape(msg != null ? msg : e.getClass().getSimpleName()), + addStack); if (addStack) { StringWriter sw = new StringWriter(); PrintWriter stackWriter = new PrintWriter(sw); - e.printStackTrace(stackWriter); + LOGGER.info(stackWriter.toString()); stackWriter.close(); - printField(pw, ErrorField.stack.name(), JSONUtil.escape(sw.toString()), false); + printField(pw, ErrorField.STACK.str(), JSONUtil.escape(sw.toString()), false); } pw.print("\t}],\n"); } - static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, long resultSize) { + private static void printMetrics(PrintWriter pw, long elapsedTime, long executionTime, long resultCount, + long resultSize) { pw.print("\t\""); - pw.print(ResultFields.metrics.name()); + pw.print(ResultFields.METRICS.str()); pw.print("\": {\n"); pw.print("\t"); - printField(pw, Metrics.elapsedTime.name(), TimeUnit.formatNanos(elapsedTime)); + printField(pw, Metrics.ELAPSED_TIME.str(), TimeUnit.formatNanos(elapsedTime)); pw.print("\t"); - printField(pw, Metrics.executionTime.name(), TimeUnit.formatNanos(executionTime)); + printField(pw, Metrics.EXECUTION_TIME.str(), TimeUnit.formatNanos(executionTime)); pw.print("\t"); - printField(pw, Metrics.resultCount.name(), String.valueOf(resultCount)); + printField(pw, Metrics.RESULT_COUNT.str(), String.valueOf(resultCount)); pw.print("\t"); - printField(pw, Metrics.resultSize.name(), String.valueOf(resultSize), false); + printField(pw, Metrics.RESULT_SIZE.str(), String.valueOf(resultSize), false); pw.print("\t}\n"); } @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - String query = request.getParameter(Parameter.statement.name()); - if (query == null) { - StringWriter sw = new StringWriter(); - IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name()); - query = sw.toString(); + String query = request.getParameter(Parameter.STATEMENT.str()); + try { + if (query == null) { + StringWriter sw = new StringWriter(); + IOUtils.copy(request.getInputStream(), sw, StandardCharsets.UTF_8.name()); + query = sw.toString(); + } + handleRequest(request, response, query); + } catch (IOException e) { + // Servlet methods should not throw exceptions + // http://cwe.mitre.org/data/definitions/600.html + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e); } - handleRequest(request, response, query); } @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - String query = request.getParameter(Parameter.statement.name()); - handleRequest(request, response, query); + String query = request.getParameter(Parameter.STATEMENT.str()); + try { + handleRequest(request, response, query); + } catch (IOException e) { + // Servlet methods should not throw exceptions + // http://cwe.mitre.org/data/definitions/600.html + GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e); + } } - public void handleRequest(HttpServletRequest request, HttpServletResponse response, String query) + private void handleRequest(HttpServletRequest request, HttpServletResponse response, String query) throws IOException { long elapsedStart = System.nanoTime(); - - query = query + ";"; final StringWriter stringWriter = new StringWriter(); final PrintWriter resultWriter = new PrintWriter(stringWriter); SessionConfig sessionConfig = createSessionConfig(request, resultWriter); - initResponse(response, sessionConfig); + response.setCharacterEncoding("utf-8"); + response.setContentType(MediaType.JSON.str()); int respCode = HttpServletResponse.SC_OK; ResultUtils.Stats stats = new ResultUtils.Stats(); - long execStart = 0, execEnd = 0; + long execStart = 0; + long execEnd = -1; resultWriter.print("{\n"); UUID requestId = printRequestId(resultWriter); printSignature(resultWriter); + printType(resultWriter, sessionConfig); try { IHyracksClientConnection hcc; IHyracksDataset hds; @@ -340,17 +418,21 @@ execStart = System.nanoTime(); translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats); execEnd = System.nanoTime(); - printStatus(resultWriter, ResultStatus.success); + printStatus(resultWriter, ResultStatus.SUCCESS); } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) { GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe); printError(resultWriter, pe); - printStatus(resultWriter, ResultStatus.fatal); + printStatus(resultWriter, ResultStatus.FATAL); respCode = HttpServletResponse.SC_BAD_REQUEST; } catch (Exception e) { GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e); printError(resultWriter, e); - printStatus(resultWriter, ResultStatus.fatal); + printStatus(resultWriter, ResultStatus.FATAL); respCode = HttpServletResponse.SC_INTERNAL_SERVER_ERROR; + } finally { + if (execEnd == -1) { + execEnd = System.nanoTime(); + } } printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.count, stats.size); resultWriter.print("}\n"); @@ -358,7 +440,6 @@ String result = stringWriter.toString(); GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, result); - //result = JSONUtil.indent(result); response.getWriter().print(result); if (response.getWriter().checkError()) { @@ -366,5 +447,4 @@ } response.setStatus(respCode); } - } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java index 7656006..95afc5b 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java @@ -39,7 +39,6 @@ import org.apache.asterix.api.common.APIFramework; import org.apache.asterix.api.common.SessionConfig; -import org.apache.asterix.api.common.SessionConfig.OutputFormat; import org.apache.asterix.app.external.ExternalIndexingOperations; import org.apache.asterix.app.external.FeedJoint; import org.apache.asterix.app.external.FeedLifecycleListener; @@ -2562,15 +2561,8 @@ hcc.waitForCompletion(jobId); ResultReader resultReader = new ResultReader(hcc, hdc); resultReader.open(jobId, metadataProvider.getResultSetId()); - - // In this case (the normal case), we don't use the - // "response" JSONObject - just stream the results - // to the "out" PrintWriter - if (sessionConfig.fmt() == OutputFormat.CSV - && sessionConfig.is(SessionConfig.FORMAT_CSV_HEADER)) { - ResultUtils.displayCSVHeader(metadataProvider.findOutputRecordType(), sessionConfig); - } - ResultUtils.displayResults(resultReader, sessionConfig, stats); + ResultUtils.displayResults(resultReader, sessionConfig, stats, + metadataProvider.findOutputRecordType()); break; case ASYNC_DEFERRED: handle = new JSONArray(); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java index b140bef..73dd706 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java @@ -34,8 +34,7 @@ import org.apache.asterix.api.common.SessionConfig; import org.apache.asterix.api.common.SessionConfig.OutputFormat; import org.apache.asterix.api.http.servlet.APIServlet; -import org.apache.asterix.api.http.servlet.JSONUtil; -import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.utils.JSONUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.http.ParseException; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -74,35 +73,25 @@ return s; } - public static void displayCSVHeader(ARecordType recordType, SessionConfig conf) throws AsterixException { - if (recordType == null) { - throw new AsterixException("Cannot output CSV with header without specifying output-record-type"); - } - // If HTML-ifying, we have to output this here before the header - - // pretty ugly - if (conf.is(SessionConfig.FORMAT_HTML)) { - conf.out().println("<h4>Results:</h4>"); - conf.out().println("<pre>"); - } - + private static void printCSVHeader(ARecordType recordType, PrintWriter out) { String[] fieldNames = recordType.getFieldNames(); boolean notfirst = false; for (String name : fieldNames) { if (notfirst) { - conf.out().print(','); + out.print(','); } notfirst = true; - conf.out().print('"'); - conf.out().print(name.replace("\"", "\"\"")); - conf.out().print('"'); + out.print('"'); + out.print(name.replace("\"", "\"\"")); + out.print('"'); } - conf.out().print("\r\n"); + out.print("\r\n"); } public static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE); - public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats) - throws HyracksDataException { + public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats, + ARecordType recordType) throws HyracksDataException { // Whether we are wrapping the output sequence in an array boolean wrap_array = false; // Whether this is the first instance being output @@ -110,31 +99,37 @@ // If we're outputting CSV with a header, the HTML header was already // output by displayCSVHeader(), so skip it here - if (conf.is(SessionConfig.FORMAT_HTML) - && !(conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER))) { + if (conf.is(SessionConfig.FORMAT_HTML)) { conf.out().println("<h4>Results:</h4>"); conf.out().println("<pre>"); } conf.resultPrefix(conf.out()); - switch (conf.fmt()) { - case LOSSLESS_JSON: - case CLEAN_JSON: - case ADM: - if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) { - // Conveniently, LOSSLESS_JSON and ADM have the same syntax for an - // "ordered list", and our representation of the result of a - // statement is an ordered list of instances. - conf.out().print("[ "); - wrap_array = true; - } - break; - default: - break; + if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) { + conf.out().print("[ "); + wrap_array = true; } - final boolean indentJSON = conf.is(SessionConfig.INDENT_JSON); + final boolean indentJSON = conf.is(SessionConfig.FORMAT_INDENT_JSON); + final boolean quoteRecord = conf.is(SessionConfig.FORMAT_QUOTE_RECORD); + + if (conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER)) { + if (recordType == null) { + throw new HyracksDataException("Cannot print CSV with header without specifying output-record-type"); + } + if (quoteRecord) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + printCSVHeader(recordType, pw); + pw.close(); + conf.out().print(JSONUtil.quoteAndEscape(sw.toString())); + conf.out().print("\n"); + notfirst = true; + } else { + printCSVHeader(recordType, conf.out()); + } + } final IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor(); final IFrame frame = new VSizeFrame(resultDisplayFrameMgr); @@ -161,10 +156,15 @@ // TODO(tillw): this is inefficient - do this during result generation result = JSONUtil.indent(result, 2); } - conf.out().print(result); if (conf.fmt() == OutputFormat.CSV) { - conf.out().print("\r\n"); + // TODO(tillw): this is inefficient as well + result = result + "\r\n"; } + if (quoteRecord) { + // TODO(tillw): this is inefficient as well + result = JSONUtil.quoteAndEscape(result); + } + conf.out().print(result); ++stats.count; // TODO(tillw) fix this approximation stats.size += result.length(); @@ -193,7 +193,7 @@ errorArray.put(errorMessage); try { errorResp.put("error-code", errorArray); - if (! "".equals(errorSummary)) { + if (!"".equals(errorSummary)) { errorResp.put("summary", errorSummary); } else { //parse exception diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml index 6a61b67..7c2b0a4 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/records/RecordsQueries.xml @@ -115,19 +115,19 @@ <test-case FilePath="records"> <compilation-unit name="closed-closed-fieldname-conflict_issue173"> <output-dir compare="Text">closed-closed-fieldname-conflict_issue173</output-dir> - <expected-error>java.lang.IllegalStateException: Closed fields 0 and 1 have the same field name "name"</expected-error> + <expected-error>Closed fields 0 and 1 have the same field name "name"</expected-error> </compilation-unit> </test-case> <test-case FilePath="records"> <compilation-unit name="open-closed-fieldname-conflict_issue173"> <output-dir compare="Text">open-closed-fieldname-conflict_issue173</output-dir> - <expected-error>org.apache.hyracks.api.exceptions.HyracksDataException: Open field "name" has the same field name as closed field at index 0</expected-error> + <expected-error>Open field "name" has the same field name as closed field at index 0</expected-error> </compilation-unit> </test-case> <test-case FilePath="records"> <compilation-unit name="open-open-fieldname-conflict_issue173"> <output-dir compare="Text">open-open-fieldname-conflict_issue173</output-dir> - <expected-error>org.apache.hyracks.api.exceptions.HyracksDataException: Open fields 0 and 1 have the same field name "name"</expected-error> + <expected-error>Open fields 0 and 1 have the same field name "name"</expected-error> </compilation-unit> </test-case> </test-group> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 195906b..96d7c4f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -17,10 +17,9 @@ ! under the License. !--> <!DOCTYPE test-suite [ + <!ENTITY RecordsQueries SYSTEM "queries_sqlpp/records/RecordsQueries.xml"> - <!ENTITY RecordsQueries SYSTEM "queries_sqlpp/records/RecordsQueries.xml"> - - ]> +]> <test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" @@ -1076,43 +1075,43 @@ <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_duration"> <output-dir compare="Text">issue363_inequality_duration</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the DURATION type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the DURATION type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_interval"> <output-dir compare="Text">issue363_inequality_interval</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the INTERVAL type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the INTERVAL type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_point"> <output-dir compare="Text">issue363_inequality_point</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the POINT type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the POINT type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_line"> <output-dir compare="Text">issue363_inequality_line</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the LINE type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the LINE type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_polygon"> <output-dir compare="Text">issue363_inequality_polygon</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the POLYGON type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the POLYGON type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_rectangle"> <output-dir compare="Text">issue363_inequality_rectangle</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the RECTANGLE type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the RECTANGLE type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> <compilation-unit name="issue363_inequality_circle"> <output-dir compare="Text">issue363_inequality_circle</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Comparison operations (GT, GE, LT, and LE) for the CIRCLE type are not defined</expected-error> + <expected-error>Comparison operations (GT, GE, LT, and LE) for the CIRCLE type are not defined</expected-error> </compilation-unit> </test-case> <test-case FilePath="comparison"> @@ -1350,7 +1349,7 @@ <test-case FilePath="custord"> <compilation-unit name="join_q_07"> <output-dir compare="Text">join_q_06</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier c</expected-error> + <expected-error>Undefined alias (variable) reference for identifier c</expected-error> </compilation-unit> </test-case> <test-case FilePath="custord"> @@ -2336,19 +2335,19 @@ <test-case FilePath="global-aggregate"> <compilation-unit name="q05_error"> <output-dir compare="Text">q01</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: The first argument should be a RECORD, but it is</expected-error> + <expected-error>The first argument should be a RECORD, but it is [ FacebookUserType: open</expected-error> </compilation-unit> </test-case> <test-case FilePath="global-aggregate"> <compilation-unit name="q06_error"> <output-dir compare="Text">q01</output-dir> - <expected-error>Caused by: org.apache.asterix.common.exceptions.AsterixException: Unsupported type: STRING</expected-error> + <expected-error>Unsupported type: STRING</expected-error> </compilation-unit> </test-case> <test-case FilePath="global-aggregate"> <compilation-unit name="q07_error"> <output-dir compare="Text">q01</output-dir> - <expected-error>org.apache.asterix.common.exceptions.AsterixException: COUNT is a SQL-92 aggregate function. The SQL++ core aggregate function coll_count could potentially express the intent.</expected-error> + <expected-error>COUNT is a SQL-92 aggregate function. The SQL++ core aggregate function coll_count could potentially express the intent.</expected-error> </compilation-unit> </test-case> <test-case FilePath="global-aggregate"> @@ -2805,8 +2804,8 @@ <compilation-unit name="partition-by-nonexistent-field"> <output-dir compare="Text">partition-by-nonexistent-field</output-dir> <expected-error>java.lang.NullPointerException</expected-error> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Cannot find dataset</expected-error> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier testds</expected-error> + <expected-error>Cannot find dataset</expected-error> + <expected-error>Undefined alias (variable) reference for identifier testds</expected-error> </compilation-unit> </test-case> <test-case FilePath="misc"> @@ -6332,7 +6331,7 @@ <test-case FilePath="cross-dataverse"> <compilation-unit name="cross-dv13"> <output-dir compare="Text">cross-dv13</output-dir> - <expected-error>Error: Recursive invocation testdv2.fun03@0</expected-error> + <expected-error>Recursive invocation testdv2.fun03@0</expected-error> </compilation-unit> </test-case> <test-case FilePath="cross-dataverse"> @@ -6348,7 +6347,7 @@ <test-case FilePath="cross-dataverse"> <compilation-unit name="cross-dv16"> <output-dir compare="Text">cross-dv16</output-dir> - <expected-error>Error: Recursive invocation testdv1.fun04@0</expected-error> + <expected-error>Recursive invocation testdv1.fun04@0</expected-error> </compilation-unit> </test-case> <!--NotImplementedException: No binary comparator factory implemented for type RECORD. @@ -6410,7 +6409,7 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="query-issue455"> <output-dir compare="Text">query-issue455</output-dir> - <expected-error>Error: function test.printName@0 is undefined</expected-error> + <expected-error>function test.printName@0 is undefined</expected-error> </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> @@ -6554,7 +6553,7 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="udf26"> <output-dir compare="Text">udf26</output-dir> - <expected-error>Error: function test.needs_f1@1 depends upon function test.f1@0 which is undefined</expected-error> + <expected-error>function test.needs_f1@1 depends upon function test.f1@0 which is undefined</expected-error> </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> @@ -6576,13 +6575,13 @@ <test-case FilePath="user-defined-functions"> <compilation-unit name="udf30"> <output-dir compare="Text">udf30</output-dir> - <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException: Undefined alias (variable) reference for identifier y</expected-error> + <expected-error>Undefined alias (variable) reference for identifier y</expected-error> </compilation-unit> </test-case> <test-case FilePath="user-defined-functions"> <compilation-unit name="f01"> <output-dir compare="Text">f01</output-dir> - <expected-error>Error: function test.int8@0 is undefined</expected-error> + <expected-error>function test.int8@0 is undefined</expected-error> </compilation-unit> </test-case> <!-- This test case is not valid anymore since we do not required "IMPORT_PRIVATE_FUNCTIONS" flag anymore --> diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java similarity index 63% rename from asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java rename to asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java index d69c3c1..0b973e1 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/JSONUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JSONUtil.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.asterix.api.http.servlet; +package org.apache.asterix.common.utils; import java.util.Iterator; +import java.util.logging.Logger; import org.json.JSONArray; import org.json.JSONException; @@ -26,21 +27,23 @@ public class JSONUtil { - static final String INDENT = "\t"; + private static final Logger LOGGER = Logger.getLogger(JSONUtil.class.getName()); - public static String indent(String str) { - return indent(str, 0); + private static final String INDENT = "\t"; + + private JSONUtil() { } public static String indent(String str, int initialIndent) { try { return append(new StringBuilder(), new JSONObject(str), initialIndent).toString(); } catch (JSONException e) { + LOGGER.finest("Could not indent JSON string, returning the input string: " + str); return str; } } - static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException { + private static StringBuilder append(StringBuilder sb, Object o, int indent) throws JSONException { if (o instanceof JSONObject) { return append(sb, (JSONObject) o, indent); } else if (o instanceof JSONArray) { @@ -53,8 +56,8 @@ throw new UnsupportedOperationException(o.getClass().getSimpleName()); } - static StringBuilder append(StringBuilder sb, JSONObject jobj, int indent) throws JSONException { - sb = sb.append("{\n"); + private static StringBuilder append(StringBuilder builder, JSONObject jobj, int indent) throws JSONException { + StringBuilder sb = builder.append("{\n"); boolean first = true; for (Iterator it = jobj.keys(); it.hasNext();) { final String key = (String) it.next(); @@ -72,8 +75,8 @@ return indent(sb, indent).append("}"); } - static StringBuilder append(StringBuilder sb, JSONArray jarr, int indent) throws JSONException { - sb = sb.append("[\n"); + private static StringBuilder append(StringBuilder builder, JSONArray jarr, int indent) throws JSONException { + StringBuilder sb = builder.append("[\n"); for (int i = 0; i < jarr.length(); ++i) { if (i > 0) { sb = sb.append(",\n"); @@ -85,11 +88,12 @@ return indent(sb, indent).append("]"); } - static StringBuilder quote(StringBuilder sb, String str) { + private static StringBuilder quote(StringBuilder sb, String str) { return sb.append('"').append(str).append('"'); } - static StringBuilder indent(StringBuilder sb, int indent) { + private static StringBuilder indent(StringBuilder sb, int i) { + int indent = i; while (indent > 0) { sb.append(INDENT); --indent; @@ -97,40 +101,49 @@ return sb; } - public static String escape(String str) { - StringBuilder result = new StringBuilder(); - for (int i = 0; i < str.length(); ++i) { - appendEsc(result, str.charAt(i)); - } - return result.toString(); + public static String quoteAndEscape(String str) { + StringBuilder sb = new StringBuilder(); + sb.append('"'); + escape(sb, str); + return sb.append('"').toString(); } - public static StringBuilder appendEsc(StringBuilder sb, char c) { + public static String escape(String str) { + return escape(new StringBuilder(), str).toString(); + } + + private static StringBuilder escape(StringBuilder sb, String str) { + for (int i = 0; i < str.length(); ++i) { + appendEsc(sb, str.charAt(i)); + } + return sb; + } + + private static StringBuilder appendEsc(StringBuilder sb, char c) { + CharSequence cs = esc(c); + return cs != null ? sb.append(cs) : sb.append(c); + } + + public static CharSequence esc(char c) { switch (c) { case '"': - return sb.append("\\\""); + return "\\\""; case '\\': - return sb.append("\\\\"); + return "\\\\"; case '/': - return sb.append("\\/"); + return "\\/"; case '\b': - return sb.append("\\b"); + return "\\b"; case '\n': - return sb.append("\\n"); + return "\\n"; case '\f': - return sb.append("\\f"); + return "\\f"; case '\r': - return sb.append("\\r"); + return "\\r"; case '\t': - return sb.append("\\t"); + return "\\t"; default: - return sb.append(c); + return null; } - } - - public static void main(String[] args) { - String json = args.length > 0 ? args[0] : "{\"a\":[\"b\",\"c\\\nd\"],\"e\":42}"; - System.out.println(json); - System.out.println(indent(json)); } } diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java new file mode 100644 index 0000000..13ab717 --- /dev/null +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/ResultExtractor.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.test.aql; + +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +/** + * extracts results from the response of the QueryServiceServlet. + * + * As the response is not necessarily valid JSON, non-JSON content has to be extracted in some cases. + * The current implementation creates a toomany copies of the data to be usable for larger results. + */ +public class ResultExtractor { + + private static final Logger LOGGER = Logger.getLogger(ResultExtractor.class.getName()); + + static InputStream extract(InputStream resultStream) throws Exception { + String result = IOUtils.toString(resultStream, Charset.forName("UTF-8")); + + LOGGER.fine("+++++++\n" + result + "\n+++++++\n"); + + JSONTokener tokener = new JSONTokener(result); + tokener.nextTo('{'); + tokener.next('{'); + String name; + String type = null; + String results = ""; + while ((name = getFieldName(tokener)) != null) { + if ("requestID".equals(name) || "signature".equals(name) || "status".equals(name)) { + getStringField(tokener); + } else if ("type".equals(name)) { + type = getStringField(tokener); + } else if ("metrics".equals(name)) { + JSONObject metrics = getObjectField(tokener); + LOGGER.fine(name + ": " + metrics); + } else if ("errors".equals(name)) { + JSONArray errors = getArrayField(tokener); + LOGGER.fine(name + ": " + errors); + JSONObject err = errors.getJSONObject(0); + throw new Exception(err.getString("msg")); + } else if ("results".equals(name)) { + results = getResults(tokener, type); + } else { + throw tokener.syntaxError(name + ": unanticipated field"); + } + } + while (tokener.more() && tokener.skipTo('}') != '}') { + // skip along + } + tokener.next('}'); + return IOUtils.toInputStream(results); + } + + private static String getFieldName(JSONTokener tokener) throws JSONException { + char c = tokener.skipTo('"'); + if (c != '"') { + return null; + } + tokener.next('"'); + return tokener.nextString('"'); + } + + private static String getStringField(JSONTokener tokener) throws JSONException { + tokener.skipTo('"'); + tokener.next('"'); + return tokener.nextString('"'); + } + + private static JSONArray getArrayField(JSONTokener tokener) throws JSONException { + tokener.skipTo(':'); + tokener.next(':'); + Object obj = tokener.nextValue(); + if (obj instanceof JSONArray) { + return (JSONArray) obj; + } else { + throw tokener.syntaxError(String.valueOf(obj) + ": unexpected value"); + } + } + + private static JSONObject getObjectField(JSONTokener tokener) throws JSONException { + tokener.skipTo(':'); + tokener.next(':'); + Object obj = tokener.nextValue(); + if (obj instanceof JSONObject) { + return (JSONObject) obj; + } else { + throw tokener.syntaxError(String.valueOf(obj) + ": unexpected value"); + } + } + + private static String getResults(JSONTokener tokener, String type) throws JSONException { + tokener.skipTo(':'); + tokener.next(':'); + StringBuilder result = new StringBuilder(); + if (type != null) { + // a type was provided in the response and so the result is encoded as an array of escaped strings that + // need to be concatenated + Object obj = tokener.nextValue(); + if (!(obj instanceof JSONArray)) { + throw tokener.syntaxError("array expected"); + } + JSONArray array = (JSONArray) obj; + for (int i = 0; i < array.length(); ++i) { + result.append(array.getString(i)); + } + return result.toString(); + } else { + int level = 0; + boolean inQuote = false; + while (tokener.more()) { + char c = tokener.next(); + switch (c) { + case '{': + case '[': + ++level; + result.append(c); + break; + case '}': + case ']': + --level; + result.append(c); + break; + case '"': + if (inQuote) { + --level; + } else { + ++level; + } + inQuote = !inQuote; + result.append(c); + break; + case ',': + if (level == 0) { + return result.toString().trim(); + } else { + result.append(c); + } + break; + default: + result.append(c); + break; + } + } + } + return null; + } +} diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java index 1af6b80..58c6a91 100644 --- a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java +++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java @@ -46,6 +46,7 @@ import org.apache.asterix.testframework.context.TestCaseContext; import org.apache.asterix.testframework.context.TestCaseContext.OutputFormat; import org.apache.asterix.testframework.context.TestFileContext; +import org.apache.asterix.testframework.xml.TestCase; import org.apache.asterix.testframework.xml.TestCase.CompilationUnit; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler; @@ -277,33 +278,74 @@ return statusCode; } - // Executes Query and returns results as JSONArray - public InputStream executeQuery(String str, OutputFormat fmt, String url, List<CompilationUnit.Parameter> params) - throws Exception { - HttpMethodBase method = null; - if (str.length() + url.length() < MAX_URL_LENGTH) { - // Use GET for small-ish queries - method = new GetMethod(url); - NameValuePair[] parameters = new NameValuePair[params.size() + 1]; - parameters[0] = new NameValuePair("query", str); - int i = 1; - for (CompilationUnit.Parameter param : params) { - parameters[i++] = new NameValuePair(param.getName(), param.getValue()); - } - method.setQueryString(parameters); - } else { - // Use POST for bigger ones to avoid 413 FULL_HEAD - // QQQ POST API doesn't allow encoding additional parameters - method = new PostMethod(url); - ((PostMethod) method).setRequestEntity(new StringRequestEntity(str)); - } - + public InputStream executeQuery(String str, OutputFormat fmt, String url, + List<CompilationUnit.Parameter> params) throws Exception { + HttpMethod method = constructHttpMethod(str, url, "query", false, params); // Set accepted output response type method.setRequestHeader("Accept", fmt.mimeType()); - // Provide custom retry handler is necessary - method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); executeHttpMethod(method); return method.getResponseBodyAsStream(); + } + + public InputStream executeQueryService(String str, OutputFormat fmt, String url, + List<CompilationUnit.Parameter> params) throws Exception { + setFormatParam(params, fmt); + HttpMethod method = constructHttpMethod(str, url, "statement", true, params); + // Set accepted output response type + method.setRequestHeader("Accept", OutputFormat.CLEAN_JSON.mimeType()); + executeHttpMethod(method); + return method.getResponseBodyAsStream(); + } + + private void setFormatParam(List<CompilationUnit.Parameter> params, OutputFormat fmt) { + boolean formatSet = false; + for (CompilationUnit.Parameter param : params) { + if ("format".equals(param.getName())) { + param.setValue(fmt.mimeType()); + formatSet = true; + } + } + if (!formatSet) { + CompilationUnit.Parameter formatParam = new CompilationUnit.Parameter(); + formatParam.setName("format"); + formatParam.setValue(fmt.mimeType()); + params.add(formatParam); + } + } + + private HttpMethod constructHttpMethod(String statement, String endpoint, String stmtParam, boolean postStmtAsParam, + List<CompilationUnit.Parameter> otherParams) { + HttpMethod method; + if (statement.length() + endpoint.length() < MAX_URL_LENGTH) { + // Use GET for small-ish queries + GetMethod getMethod = new GetMethod(endpoint); + NameValuePair[] parameters = new NameValuePair[otherParams.size() + 1]; + parameters[0] = new NameValuePair(stmtParam, statement); + int i = 1; + for (CompilationUnit.Parameter param : otherParams) { + parameters[i++] = new NameValuePair(param.getName(), param.getValue()); + } + getMethod.setQueryString(parameters); + method = getMethod; + } else { + // Use POST for bigger ones to avoid 413 FULL_HEAD + PostMethod postMethod = new PostMethod(endpoint); + if (postStmtAsParam) { + for (CompilationUnit.Parameter param : otherParams) { + postMethod.setParameter(param.getName(), param.getValue()); + } + postMethod.setParameter("statement", statement); + } else { + // this seems pretty bad - we should probably fix the API and not the client + postMethod.setRequestEntity(new StringRequestEntity(statement)); + } + method = postMethod; + } + // Provide custom retry handler is necessary + HttpMethodParams httpMethodParams = method.getParams(); + httpMethodParams.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false)); + httpMethodParams.setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, StandardCharsets.UTF_8.name()); + return method; } public InputStream executeClusterStateQuery(OutputFormat fmt, String url) throws Exception { @@ -473,6 +515,7 @@ executeTest(actualPath, testCaseCtx, pb, isDmlRecoveryTest, null); } + public void executeTest(TestCaseContext testCaseCtx, TestFileContext ctx, String statement, boolean isDmlRecoveryTest, ProcessBuilder pb, CompilationUnit cUnit, MutableInt queryCount, List<TestFileContext> expectedResultFileCtxs, File testFile, String actualPath) throws Exception { @@ -523,8 +566,9 @@ } } else { if (ctx.getType().equalsIgnoreCase("query")) { - resultStream = executeQuery(statement, fmt, - "http://"; + host + ":" + port + Servlets.SQLPP_QUERY.getPath(), cUnit.getParameter()); + resultStream = executeQueryService(statement, fmt, + "http://"; + host + ":" + port + Servlets.QUERY_SERVICE.getPath(), cUnit.getParameter()); + resultStream = ResultExtractor.extract(resultStream); } else if (ctx.getType().equalsIgnoreCase("async")) { resultStream = executeAnyAQLAsync(statement, false, fmt, "http://"; + host + ":" + port + Servlets.SQLPP.getPath()); @@ -533,7 +577,6 @@ "http://"; + host + ":" + port + Servlets.SQLPP.getPath()); } } - if (queryCount.intValue() >= expectedResultFileCtxs.size()) { throw new IllegalStateException("no result file for " + testFile.toString() + "; queryCount: " + queryCount + ", filectxs.size: " + expectedResultFileCtxs.size()); @@ -770,6 +813,7 @@ } else { // Get the expected exception String expectedError = cUnit.getExpectedError().get(numOfErrors - 1); + System.err.println("+++++\n" + expectedError + "\n+++++\n"); if (e.toString().contains(expectedError)) { System.err.println("...but that was expected."); } else { -- To view, visit https://asterix-gerrit.ics.uci.edu/896 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ie67ad4ea31699400726c8c026c4a91edc698f2b5 Gerrit-PatchSet: 22 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Till Westmann <[email protected]> Gerrit-Reviewer: Ian Maxon <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Michael Blow <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]>
