This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 1bd0e9c61a DRILL-8365: HTTP Plugin Places Parameters in Wrong Place
(#2715)
1bd0e9c61a is described below
commit 1bd0e9c61aa7c8951d01ab4832e930951e0a4db7
Author: Charles S. Givre <[email protected]>
AuthorDate: Mon Dec 5 07:32:55 2022 -0500
DRILL-8365: HTTP Plugin Places Parameters in Wrong Place (#2715)
---
.../drill/exec/store/http/HttpBatchReader.java | 4 +--
.../exec/store/http/HttpScanBatchCreator.java | 11 ++++++-
.../drill/exec/store/http/TestPagination.java | 37 ++++++++++++++++++++++
3 files changed, 49 insertions(+), 3 deletions(-)
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
index cc17636bcc..579643ea80 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpBatchReader.java
@@ -263,8 +263,8 @@ public class HttpBatchReader implements
ManagedReader<SchemaNegotiator> {
logger.debug("Building URL from {}", baseUrl);
HttpApiConfig apiConfig = subScan.tableSpec().connectionConfig();
- // Append table name, if available.
- if (subScan.tableSpec().tableName() != null) {
+ // Append table name, if present. When pagination is used, the paginator
adds this.
+ if (subScan.tableSpec().tableName() != null && paginator == null) {
baseUrl += subScan.tableSpec().tableName();
}
diff --git
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
index 3de783f7da..0da06a4c43 100644
---
a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
+++
b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpScanBatchCreator.java
@@ -110,7 +110,16 @@ public class HttpScanBatchCreator implements
BatchCreator<HttpSubScan> {
private Paginator getPaginator() {
HttpUrl.Builder urlBuilder;
- HttpUrl rawUrl =
HttpUrl.parse(subScan.tableSpec().connectionConfig().url());
+ HttpUrl rawUrl;
+
+ // Append table name, if present.
+ if (subScan.tableSpec().tableName() != null) {
+ rawUrl = HttpUrl.parse(subScan.tableSpec().connectionConfig().url() +
subScan.tableSpec().tableName());
+ } else {
+ rawUrl = HttpUrl.parse(subScan.tableSpec().connectionConfig().url());
+ }
+
+
// If the URL is not parsable or otherwise invalid
if (rawUrl == null) {
diff --git
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
index 5931e0d032..ed30d8d8b1 100644
---
a/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
+++
b/contrib/storage-http/src/test/java/org/apache/drill/exec/store/http/TestPagination.java
@@ -216,6 +216,15 @@ public class TestPagination extends ClusterTest {
.inputType("json")
.build();
+ HttpApiConfig mockJsonConfigWithPaginatorAndTail = HttpApiConfig.builder()
+ .url("http://localhost:8092/json";)
+ .method("get")
+ .headers(headers)
+ .requireTail(true)
+ .paginator(offsetPaginatorForJson)
+ .inputType("json")
+ .build();
+
HttpPaginatorConfig pagePaginatorForXML = HttpPaginatorConfig.builder()
.method("page")
.pageParam("page")
@@ -268,6 +277,7 @@ public class TestPagination extends ClusterTest {
configs.put("nested_keyset", mockJsonConfigWitNestedKeyset);
configs.put("nested_keyset_and_datapath",
mockJsonConfigWitNestedKeysetAndDataPath);
configs.put("json_paginator", mockJsonConfigWithPaginator);
+ configs.put("json_tail", mockJsonConfigWithPaginatorAndTail);
configs.put("xml_paginator", mockXmlConfigWithPaginator);
configs.put("xml_paginator_url_params",
mockXmlConfigWithPaginatorAndUrlParams);
@@ -318,6 +328,33 @@ public class TestPagination extends ClusterTest {
}
}
+ @Test
+ public void simpleJSONPaginatorQueryWithTail() throws Exception {
+ String sql = "SELECT * FROM `local`.`json_tail`.`?arg1=foo` LIMIT 4";
+ try (MockWebServer server = startServer()) {
+
+ server.enqueue(new
MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE1));
+ server.enqueue(new
MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE2));
+ server.enqueue(new
MockResponse().setResponseCode(200).setBody(TEST_JSON_PAGE3));
+
+ List<QueryDataBatch> results = client.queryBuilder()
+ .sql(sql)
+ .results();
+
+ int count = 0;
+ for(QueryDataBatch b : results){
+ count += b.getHeader().getRowCount();
+ b.release();
+ }
+ assertEquals(2, results.size());
+ assertEquals(4, count);
+
+ // Verify that the URLs are correct
+ RecordedRequest recordedRequest = server.takeRequest();
+ assertEquals("http://localhost:8092/json?arg1=foo&offset=0&limit=2";,
recordedRequest.getRequestUrl().toString());
+ }
+ }
+
@Test
public void simpleJSONPaginatorQueryWith429() throws Exception {
// This test simulates an http request that hits a burst limit. In this
situation,
