Michael Blow has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/3191
Change subject: [NO ISSUE] Handle Accept-Charset in RestApiServlet
......................................................................
[NO ISSUE] Handle Accept-Charset in RestApiServlet
also, exercise accept-charset in TestExecutor
Change-Id: I8f37eb684bf2457e5ff451bf5c8fbca742d531f2
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M
asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
M
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
6 files changed, 81 insertions(+), 34 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/91/3191/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
index 347b2d7..221984e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
@@ -86,7 +86,6 @@
* based on the Accept: header and other servlet parameters.
*/
static SessionOutput initResponse(IServletRequest request,
IServletResponse response) throws IOException {
- HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_PLAIN,
request);
// CLEAN_JSON output is the default; most generally useful for a
// programmatic HTTP API
OutputFormat format = OutputFormat.CLEAN_JSON;
@@ -136,20 +135,20 @@
// Now that format is set, output the content-type
switch (format) {
case ADM:
- HttpUtil.setContentType(response, "application/x-adm");
+ HttpUtil.setContentType(response, "application/x-adm",
request);
break;
case CLEAN_JSON:
// No need to reflect "clean-ness" in output type; fall through
case LOSSLESS_JSON:
- HttpUtil.setContentType(response, "application/json");
+ HttpUtil.setContentType(response, "application/json", request);
break;
case CSV:
// Check for header parameter or in Accept:.
if ("present".equals(request.getParameter("header")) ||
accept.contains("header=present")) {
- HttpUtil.setContentType(response, "text/csv;
header=present");
+ HttpUtil.setContentType(response, "text/csv;
header=present", request);
sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, true);
} else {
- HttpUtil.setContentType(response, "text/csv;
header=absent");
+ HttpUtil.setContentType(response, "text/csv;
header=absent", request);
}
break;
default:
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
index e85fedf..969d23d 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/CancellationTestExecutor.java
@@ -21,6 +21,7 @@
import java.io.InputStream;
import java.net.URI;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
@@ -49,7 +50,7 @@
@Override
public InputStream executeQueryService(String str,
TestCaseContext.OutputFormat fmt, URI uri,
- List<TestCase.CompilationUnit.Parameter> params, boolean
jsonEncoded,
+ List<TestCase.CompilationUnit.Parameter> params, boolean
jsonEncoded, Charset responseCharset,
Predicate<Integer> responseCodeValidator, boolean cancellable)
throws Exception {
String clientContextId = UUID.randomUUID().toString();
final List<TestCase.CompilationUnit.Parameter> newParams = cancellable
@@ -57,7 +58,7 @@
Callable<InputStream> query = () -> {
try {
return CancellationTestExecutor.super.executeQueryService(str,
fmt, uri, newParams, jsonEncoded,
- responseCodeValidator, true);
+ responseCharset, responseCodeValidator, true);
} catch (Exception e) {
e.printStackTrace();
throw e;
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index 37e1213..3b57039 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -91,8 +91,8 @@
return extract(resultStream, EnumSet.of(ResultField.METRICS));
}
- public static String extractHandle(InputStream resultStream) throws
Exception {
- String result = IOUtils.toString(resultStream, StandardCharsets.UTF_8);
+ public static String extractHandle(InputStream resultStream, Charset
responseCharset) throws Exception {
+ String result = IOUtils.toString(resultStream, responseCharset);
ObjectNode resultJson = OBJECT_MAPPER.readValue(result,
ObjectNode.class);
final JsonNode handle = resultJson.get("handle");
if (handle != null) {
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index b143ea9..16db6f8 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -35,12 +35,14 @@
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
@@ -155,6 +157,8 @@
private static Map<String, InetSocketAddress> ncEndPoints;
private static Map<String, InetSocketAddress> replicationAddress;
+ private static final List<Charset> charsetsRemaining = new ArrayList<>();
+
/*
* Instance members
*/
@@ -211,12 +215,12 @@
}
public void runScriptAndCompareWithResult(File scriptFile, File
expectedFile, File actualFile,
- ComparisonEnum compare) throws Exception {
+ ComparisonEnum compare, Charset actualEncoding) throws Exception {
LOGGER.info("Expected results file: {} ", expectedFile);
BufferedReader readerExpected =
- new BufferedReader(new InputStreamReader(new
FileInputStream(expectedFile), "UTF-8"));
+ new BufferedReader(new InputStreamReader(new
FileInputStream(expectedFile), StandardCharsets.UTF_8));
BufferedReader readerActual =
- new BufferedReader(new InputStreamReader(new
FileInputStream(actualFile), "UTF-8"));
+ new BufferedReader(new InputStreamReader(new
FileInputStream(actualFile), actualEncoding));
boolean regex = false;
try {
if (ComparisonEnum.BINARY.equals(compare)) {
@@ -570,21 +574,23 @@
}
public InputStream executeQueryService(String str, URI uri, OutputFormat
fmt) throws Exception {
- return executeQueryService(str, fmt, uri, new ArrayList<>(), false);
+ return executeQueryService(str, fmt, uri, new ArrayList<>(), false,
StandardCharsets.UTF_8);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI
uri, List<Parameter> params,
- boolean jsonEncoded) throws Exception {
- return executeQueryService(str, fmt, uri, params, jsonEncoded, null,
false);
+ boolean jsonEncoded, Charset responseCharset) throws Exception {
+ return executeQueryService(str, fmt, uri, params, jsonEncoded,
responseCharset, null, false);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI
uri, List<Parameter> params,
boolean jsonEncoded, Predicate<Integer> responseCodeValidator)
throws Exception {
- return executeQueryService(str, fmt, uri, params, jsonEncoded,
responseCodeValidator, false);
+ return executeQueryService(str, fmt, uri, params, jsonEncoded,
StandardCharsets.UTF_8, responseCodeValidator,
+ false);
}
public InputStream executeQueryService(String str, OutputFormat fmt, URI
uri, List<Parameter> params,
- boolean jsonEncoded, Predicate<Integer> responseCodeValidator,
boolean cancellable) throws Exception {
+ boolean jsonEncoded, Charset responseCharset, Predicate<Integer>
responseCodeValidator, boolean cancellable)
+ throws Exception {
List<Parameter> newParams = upsertParam(params, "format",
ParameterTypeEnum.STRING, fmt.mimeType());
newParams = upsertParam(newParams,
QueryServiceServlet.Parameter.PLAN_FORMAT.str(), ParameterTypeEnum.STRING,
DEFAULT_PLAN_FORMAT);
@@ -602,11 +608,47 @@
// Set accepted output response type
method.setHeader("Origin", uri.getScheme() + uri.getAuthority());
method.setHeader("Accept", OutputFormat.CLEAN_JSON.mimeType());
+ method.setHeader("Accept-Charset", responseCharset.name());
HttpResponse response = executeHttpRequest(method);
if (responseCodeValidator != null) {
checkResponse(response, responseCodeValidator);
}
return response.getEntity().getContent();
+ }
+
+ private Charset selectCharset(File result) throws IOException {
+ // choose an encoding that works for this input
+ return selectCharset(FileUtils.readFileToString(result,
StandardCharsets.UTF_8));
+ }
+
+ private Charset selectCharset(String payload) {
+ // choose an encoding that works for this input
+ return nextCharset(charset -> canEncodeDecode(charset, payload));
+ }
+
+ public static Charset nextCharset(Predicate<Charset> test) {
+ synchronized (charsetsRemaining) {
+ while (true) {
+ for (Iterator<Charset> iter = charsetsRemaining.iterator();
iter.hasNext();) {
+ Charset next = iter.next();
+ if (test.test(next)) {
+ iter.remove();
+ return next;
+ }
+ }
+ List<Charset> allCharsets = new
ArrayList<>(Charset.availableCharsets().values());
+ Collections.shuffle(allCharsets);
+ charsetsRemaining.addAll(allCharsets);
+ }
+ }
+ }
+
+ private static boolean canEncodeDecode(Charset charset, String input) {
+ if (input.equals(new String(input.getBytes(charset), charset))) {
+ return true;
+ }
+ LOGGER.debug("cannot encode / decode {} with {}", input,
charset.displayName());
+ return false;
}
protected List<Parameter> upsertParam(List<Parameter> params, String name,
ParameterTypeEnum type, String value) {
@@ -918,18 +960,18 @@
expectedResultFileCtxs);
break;
case "txnqbc": // qbc represents query before crash
- resultStream = query(cUnit, testFile.getName(), statement);
+ resultStream = query(cUnit, testFile.getName(), statement,
StandardCharsets.UTF_8);
qbcFile = getTestCaseQueryBeforeCrashFile(actualPath,
testCaseCtx, cUnit);
writeOutputToFile(qbcFile, resultStream);
break;
case "txnqar": // qar represents query after recovery
- resultStream = query(cUnit, testFile.getName(), statement);
+ resultStream = query(cUnit, testFile.getName(), statement,
StandardCharsets.UTF_8);
File qarFile = new File(actualPath + File.separator
+
testCaseCtx.getTestCase().getFilePath().replace(File.separator, "_") + "_" +
cUnit.getName()
+ "_qar.adm");
writeOutputToFile(qarFile, resultStream);
qbcFile = getTestCaseQueryBeforeCrashFile(actualPath,
testCaseCtx, cUnit);
- runScriptAndCompareWithResult(testFile, qbcFile, qarFile,
ComparisonEnum.TEXT);
+ runScriptAndCompareWithResult(testFile, qbcFile, qarFile,
ComparisonEnum.TEXT, StandardCharsets.UTF_8);
break;
case "txneu": // eu represents erroneous update
try {
@@ -1171,8 +1213,9 @@
} else {
throw new IllegalArgumentException("Unexpected format for method "
+ reqType + ": " + extension);
}
+ Charset responseCharset = selectCharset(expectedResultFile);
if (handleVar != null) {
- String handle = ResultExtractor.extractHandle(resultStream);
+ String handle = ResultExtractor.extractHandle(resultStream,
responseCharset);
if (handle != null) {
variableCtx.put(handleVar, handle);
} else {
@@ -1189,7 +1232,7 @@
}
} else {
writeOutputToFile(actualResultFile, resultStream);
- runScriptAndCompareWithResult(testFile, expectedResultFile,
actualResultFile, compare);
+ runScriptAndCompareWithResult(testFile, expectedResultFile,
actualResultFile, compare, responseCharset);
}
}
queryCount.increment();
@@ -1207,23 +1250,25 @@
URI uri = testFile.getName().endsWith("aql") ?
getEndpoint(Servlets.QUERY_AQL)
: getEndpoint(Servlets.QUERY_SERVICE);
boolean isJsonEncoded =
isJsonEncoded(extractHttpRequestType(statement));
+ Charset responseCharset =
+ expectedResultFile == null ? StandardCharsets.UTF_8 :
selectCharset(expectedResultFile);
InputStream resultStream;
if (DELIVERY_IMMEDIATE.equals(delivery)) {
- resultStream =
- executeQueryService(statement, fmt, uri, params,
isJsonEncoded, null, isCancellable(reqType));
+ resultStream = executeQueryService(statement, fmt, uri, params,
isJsonEncoded, responseCharset, null,
+ isCancellable(reqType));
resultStream = METRICS_QUERY_TYPE.equals(reqType) ?
ResultExtractor.extractMetrics(resultStream)
: ResultExtractor.extract(resultStream);
} else {
String handleVar = getHandleVariable(statement);
resultStream = executeQueryService(statement, fmt, uri,
- upsertParam(params, "mode", ParameterTypeEnum.STRING,
delivery), isJsonEncoded);
- String handle = ResultExtractor.extractHandle(resultStream);
+ upsertParam(params, "mode", ParameterTypeEnum.STRING,
delivery), isJsonEncoded, responseCharset);
+ String handle = ResultExtractor.extractHandle(resultStream,
responseCharset);
Assert.assertNotNull("no handle for " + reqType + " test " +
testFile.toString(), handleVar);
variableCtx.put(handleVar, toQueryServiceHandle(handle));
}
if (actualResultFile == null) {
if (testFile.getName().startsWith(DIAGNOSE)) {
- LOGGER.info("Diagnostic output: {}",
IOUtils.toString(resultStream, StandardCharsets.UTF_8));
+ LOGGER.info("Diagnostic output: {}",
IOUtils.toString(resultStream, responseCharset));
} else {
Assert.fail("no result file for " + testFile.toString() + ";
queryCount: " + queryCount
+ ", filectxs.size: " + numResultFiles);
@@ -1238,7 +1283,7 @@
+ ", filectxs.size: " + numResultFiles);
}
}
- runScriptAndCompareWithResult(testFile, expectedResultFile,
actualResultFile, compare);
+ runScriptAndCompareWithResult(testFile, expectedResultFile,
actualResultFile, compare, responseCharset);
if (!reqType.equals("validate")) {
queryCount.increment();
}
@@ -1964,10 +2009,11 @@
return !NON_CANCELLABLE.contains(type);
}
- private InputStream query(CompilationUnit cUnit, String testFile, String
statement) throws Exception {
+ private InputStream query(CompilationUnit cUnit, String testFile, String
statement, Charset responseCharset)
+ throws Exception {
final URI uri = getQueryServiceUri(testFile);
final InputStream inputStream = executeQueryService(statement,
OutputFormat.forCompilationUnit(cUnit), uri,
- cUnit.getParameter(), true, null, false);
+ cUnit.getParameter(), true, responseCharset, null, false);
return ResultExtractor.extract(inputStream);
}
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
index e4e300a..5dd41c3 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/sqlpp/ParserTestExecutor.java
@@ -25,6 +25,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -149,7 +150,8 @@
}
writer.close();
// Compares the actual result and the expected result.
- runScriptAndCompareWithResult(queryFile, expectedFile,
actualResultFile, ComparisonEnum.TEXT);
+ runScriptAndCompareWithResult(queryFile, expectedFile,
actualResultFile, ComparisonEnum.TEXT,
+ StandardCharsets.UTF_8);
} catch (Exception e) {
GlobalConfig.ASTERIX_LOGGER.warn("Failed while testing file " +
queryFile);
throw e;
diff --git
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
index 639e036..17c98bc 100644
---
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
+++
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/server/RSSFeedServlet.java
@@ -67,13 +67,12 @@
String feedType = req.getParameter(FEED_TYPE);
feedType = (feedType != null) ? feedType : defaultFeedType;
feed.setFeedType(feedType);
- HttpUtil.setContentType(res, MIME_TYPE);
+ HttpUtil.setContentType(res, MIME_TYPE, req);
SyndFeedOutput output = new SyndFeedOutput();
output.output(feed, res.writer());
} catch (FeedException | ParseException ex) {
GlobalConfig.ASTERIX_LOGGER.log(Level.WARN, ex.getMessage(), ex);
- String msg = COULD_NOT_GENERATE_FEED_ERROR;
- res.writer().print(msg);
+ res.writer().print(COULD_NOT_GENERATE_FEED_ERROR);
res.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/3191
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I8f37eb684bf2457e5ff451bf5c8fbca742d531f2
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: stabilization-f69489
Gerrit-Owner: Michael Blow <[email protected]>
