This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d7ef925f3c3852bce601903fa7bc6068bf4cc6fa Author: sandeepgupta <[email protected]> AuthorDate: Tue Oct 8 11:56:35 2019 +0530 [NO ISSUE][TEST] Fix extraction of multi-document result in test executor - user model changes: no - storage format changes: no - interface changes: no Details: Fix extraction/writing within test suite for multiple document result as proper JSON document Change-Id: I879e045e2ae7f64f662993afb46cad7b29c781b3 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/3623 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> Tested-by: Till Westmann <[email protected]> --- .../asterix/test/common/ResultExtractor.java | 29 ++++++++++++ .../apache/asterix/test/common/TestExecutor.java | 54 ++++++++++++++++++++-- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java index 60d44a1..5e53d54 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java @@ -86,6 +86,12 @@ public class ResultExtractor { private static final Logger LOGGER = LogManager.getLogger(); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static ExtractedResult extract(InputStream resultStream, Charset resultCharset, String outputFormat) + throws Exception { + return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset, + outputFormat); + } + public static ExtractedResult extract(InputStream resultStream, Charset resultCharset) throws Exception { return extract(resultStream, EnumSet.of(ResultField.RESULTS, ResultField.WARNINGS), resultCharset); } @@ -120,6 +126,25 @@ public class ResultExtractor { private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields, Charset resultCharset) throws Exception { + return extract(resultStream, resultFields, resultCharset, "jsonl"); //default output format type is jsonl + } + + private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields, + Charset resultCharset, String fmt) throws Exception { + + if (fmt.equals("json")) { + return extract(resultStream, resultFields, resultCharset, "[", ",", "]"); + } + + if (fmt.equals("jsonl")) { + return extract(resultStream, resultFields, resultCharset, "", "", ""); + } + + throw new AsterixException("Unkown output format for result of test query"); + } + + private static ExtractedResult extract(InputStream resultStream, EnumSet<ResultField> resultFields, + Charset resultCharset, String openMarker, String separator, String closeMarker) throws Exception { ExtractedResult extractedResult = new ExtractedResult(); final String resultStr = IOUtils.toString(resultStream, resultCharset); final ObjectNode result = OBJECT_MAPPER.readValue(resultStr, ObjectNode.class); @@ -158,7 +183,10 @@ public class ResultExtractor { } else { JsonNode[] fields = Iterators.toArray(fieldValue.elements(), JsonNode.class); if (fields.length > 1) { + String sep = openMarker; for (JsonNode f : fields) { + resultBuilder.append(sep); + sep = separator; if (f.isObject()) { resultBuilder.append(OBJECT_MAPPER.writeValueAsString(f)); @@ -166,6 +194,7 @@ public class ResultExtractor { resultBuilder.append(f.asText()); } } + resultBuilder.append(closeMarker); } } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index 15711d0..8dc04eb 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -138,6 +138,9 @@ public class TestExecutor { Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", Pattern.MULTILINE); private static final Pattern POLL_DELAY_PATTERN = Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE); private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)"); + private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)"); + private static final Pattern OUTPUTFORMAT_VARIABLE_PATTERN = Pattern.compile("outputformat=(\\w+)"); + private static final Pattern VARIABLE_REF_PATTERN = Pattern.compile("\\$(\\w+)"); private static final Pattern HTTP_PARAM_PATTERN = Pattern.compile("param (?<name>[\\w-$]+)(?::(?<type>\\w+))?=(?<value>.*)", Pattern.MULTILINE); @@ -236,6 +239,9 @@ public class TestExecutor { ComparisonEnum compare, Charset actualEncoding) throws Exception { LOGGER.info("Expected results file: {} ", expectedFile); boolean regex = false; + if (expectedFile.getName().endsWith(".ignore")) { + return; //skip the comparison + } try (BufferedReader readerExpected = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile), UTF_8)); BufferedReader readerActual = @@ -1318,13 +1324,14 @@ public class TestExecutor { } boolean isJsonEncoded = isJsonEncoded(extractHttpRequestType(statement)); - Charset responseCharset = getResponseCharset(expectedResultFile); InputStream resultStream; ExtractedResult extractedResult = null; + final String variablesReplaced = replaceVarRefRelaxed(statement, variableCtx); + String resultVar = getResultVariable(statement); //Is the result of the statement/query to be used in later tests if (DELIVERY_IMMEDIATE.equals(delivery)) { - resultStream = executeQueryService(statement, ctx, fmt, uri, params, isJsonEncoded, responseCharset, null, - isCancellable(reqType)); + resultStream = executeQueryService(variablesReplaced, ctx, fmt, uri, params, isJsonEncoded, responseCharset, + null, isCancellable(reqType)); switch (reqType) { case METRICS_QUERY_TYPE: resultStream = ResultExtractor.extractMetrics(resultStream, responseCharset); @@ -1336,7 +1343,13 @@ public class TestExecutor { resultStream = ResultExtractor.extractPlans(resultStream, responseCharset); break; default: - extractedResult = ResultExtractor.extract(resultStream, responseCharset); + String outputFormatVariable = getOutputFormatVariable(statement); + if ((outputFormatVariable == null) || (outputFormatVariable.equals("jsonl"))) { + extractedResult = ResultExtractor.extract(resultStream, responseCharset); + } else { + extractedResult = ResultExtractor.extract(resultStream, responseCharset, "json"); + } + resultStream = extractedResult.getResult(); break; } @@ -1356,7 +1369,13 @@ public class TestExecutor { + ", filectxs.size: " + numResultFiles); } } else { - writeOutputToFile(actualResultFile, resultStream); + if (resultVar != null) { + String result = IOUtils.toString(resultStream, responseCharset); + variableCtx.put(resultVar, result); + writeOutputToFile(actualResultFile, new ByteArrayInputStream(result.getBytes(responseCharset))); + } else { + writeOutputToFile(actualResultFile, resultStream); + } if (expectedResultFile == null) { if (reqType.equals("store")) { return extractedResult; @@ -1584,6 +1603,16 @@ public class TestExecutor { return handleVariableMatcher.find() ? handleVariableMatcher.group(1) : null; } + protected static String getResultVariable(String statement) { + final Matcher resultVariableMatcher = RESULT_VARIABLE_PATTERN.matcher(statement); + return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null; + } + + protected static String getOutputFormatVariable(String statement) { + final Matcher outputFormatVariableMatcher = OUTPUTFORMAT_VARIABLE_PATTERN.matcher(statement); + return outputFormatVariableMatcher.find() ? outputFormatVariableMatcher.group(1) : null; + } + protected static String replaceVarRef(String statement, Map<String, Object> variableCtx) { String tmpStmt = statement; Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt); @@ -1597,6 +1626,21 @@ public class TestExecutor { return tmpStmt; } + protected static String replaceVarRefRelaxed(String statement, Map<String, Object> variableCtx) { + String tmpStmt = statement; + Matcher variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt); + while (variableReferenceMatcher.find()) { + String var = variableReferenceMatcher.group(1); + Object value = variableCtx.get(var); + if (value == null) { + continue; + } + tmpStmt = tmpStmt.replace("$" + var, String.valueOf(value)); + variableReferenceMatcher = VARIABLE_REF_PATTERN.matcher(tmpStmt); + } + return tmpStmt; + } + protected static Optional<String> extractMaxResultReads(String statement) { final Matcher m = MAX_RESULT_READS_PATTERN.matcher(statement); while (m.find()) {
