This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ae6c43d Add an integration test for HTTP inputSource (#10620)
ae6c43d is described below
commit ae6c43de71f9983564b440f8db851e5e7f1ae16d
Author: Jihoon Son <[email protected]>
AuthorDate: Thu Dec 3 15:51:56 2020 -0800
Add an integration test for HTTP inputSource (#10620)
---
.../druid/tests/indexer/ITHttpInputSourceTest.java | 53 ++++++++++++
.../wikipedia_http_inputsource_queries.json | 98 ++++++++++++++++++++++
.../indexer/wikipedia_http_inputsource_task.json | 90 ++++++++++++++++++++
3 files changed, 241 insertions(+)
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java
new file mode 100644
index 0000000..69d1eea
--- /dev/null
+++
b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.tests.indexer;
+
+import org.apache.druid.testing.guice.DruidTestModuleFactory;
+import org.apache.druid.tests.TestNGGroup;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.UUID;
+
+@Test(groups = TestNGGroup.INPUT_SOURCE)
+@Guice(moduleFactory = DruidTestModuleFactory.class)
+public class ITHttpInputSourceTest extends AbstractITBatchIndexTest
+{
+ private static final String INDEX_TASK =
"/indexer/wikipedia_http_inputsource_task.json";
+ private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_http_inputsource_queries.json";
+
+ @Test
+ public void doTest() throws IOException
+ {
+ final String indexDatasource = "wikipedia_http_inputsource_test_" +
UUID.randomUUID();
+ try (final Closeable ignored1 = unloader(indexDatasource +
config.getExtraDatasourceNameSuffix())) {
+ doIndexTest(
+ indexDatasource,
+ INDEX_TASK,
+ INDEX_QUERIES_RESOURCE,
+ false,
+ true,
+ true
+ );
+ }
+ }
+}
diff --git
a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json
b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json
new file mode 100644
index 0000000..11496c2
--- /dev/null
+++
b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json
@@ -0,0 +1,98 @@
+[
+ {
+ "description": "timeseries, 1 agg, all",
+ "query":{
+ "queryType" : "timeBoundary",
+ "dataSource": "%%DATASOURCE%%"
+ },
+ "expectedResults":[
+ {
+ "timestamp" : "2016-06-27T00:00:11.000Z",
+ "result" : {
+ "minTime" : "2016-06-27T00:00:11.000Z",
+ "maxTime" : "2016-06-27T21:31:02.000Z"
+ }
+ }
+ ]
+ },
+ {
+ "description": "timeseries, datasketch aggs, all",
+ "query":{
+ "queryType" : "timeseries",
+ "dataSource": "%%DATASOURCE%%",
+ "granularity":"day",
+ "intervals":[
+ "2016-06-27/P1D"
+ ],
+ "filter":null,
+ "aggregations":[
+ {
+ "type": "HLLSketchMerge",
+ "name": "approxCountHLL",
+ "fieldName": "HLLSketchBuild",
+ "lgK": 12,
+ "tgtHllType": "HLL_4",
+ "round": true
+ },
+ {
+ "type":"thetaSketch",
+ "name":"approxCountTheta",
+ "fieldName":"thetaSketch",
+ "size":16384,
+ "shouldFinalize":true,
+ "isInputThetaSketch":false,
+ "errorBoundsStdDev":null
+ },
+ {
+ "type":"quantilesDoublesSketch",
+ "name":"quantilesSketch",
+ "fieldName":"quantilesDoublesSketch",
+ "k":128
+ }
+ ]
+ },
+ "expectedResults":[
+ {
+ "timestamp" : "2016-06-27T00:00:00.000Z",
+ "result" : {
+ "quantilesSketch":48866,
+ "approxCountTheta":7862.0,
+ "approxCountHLL":7961
+ }
+ }
+ ]
+ },
+ {
+ "description": "timeseries, stringFirst/stringLast aggs, all",
+ "query":{
+ "queryType" : "timeseries",
+ "dataSource": "%%DATASOURCE%%",
+ "granularity":"day",
+ "intervals":[
+ "2016-06-27/P1D"
+ ],
+ "filter":null,
+ "aggregations":[
+ {
+ "type": "stringFirst",
+ "name": "first_user",
+ "fieldName": "user"
+ },
+ {
+ "type":"stringLast",
+ "name":"last_user",
+ "fieldName":"user"
+ }
+ ]
+ },
+ "expectedResults":[
+ {
+ "timestamp" : "2016-06-27T00:00:00.000Z",
+ "result" : {
+ "first_user":"Lsjbot",
+ "last_user":"EmausBot"
+ }
+ }
+ ]
+ }
+]
\ No newline at end of file
diff --git
a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json
b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json
new file mode 100644
index 0000000..6038327
--- /dev/null
+++
b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json
@@ -0,0 +1,90 @@
+{
+ "type": "index_parallel",
+ "spec": {
+ "dataSchema": {
+ "dataSource": "%%DATASOURCE%%",
+ "timestampSpec": {
+ "column": "timestamp"
+ },
+ "dimensionsSpec": {
+ "dimensions": [
+ "page",
+ {"type": "string", "name": "language", "createBitmapIndex": false},
+ "user",
+ "unpatrolled",
+ "newPage",
+ "robot",
+ "anonymous",
+ "namespace",
+ "continent",
+ "country",
+ "region",
+ "city"
+ ]
+ },
+ "metricsSpec": [
+ {
+ "type": "count",
+ "name": "count"
+ },
+ {
+ "type": "doubleSum",
+ "name": "added",
+ "fieldName": "added"
+ },
+ {
+ "type": "doubleSum",
+ "name": "deleted",
+ "fieldName": "deleted"
+ },
+ {
+ "type": "doubleSum",
+ "name": "delta",
+ "fieldName": "delta"
+ },
+ {
+ "name": "thetaSketch",
+ "type": "thetaSketch",
+ "fieldName": "user"
+ },
+ {
+ "name": "quantilesDoublesSketch",
+ "type": "quantilesDoublesSketch",
+ "fieldName": "delta"
+ },
+ {
+ "name": "HLLSketchBuild",
+ "type": "HLLSketchBuild",
+ "fieldName": "user"
+ }
+ ],
+ "granularitySpec": {
+ "segmentGranularity": "DAY",
+ "queryGranularity": "second",
+ "intervals" : [ "2016-06/P1M" ]
+ }
+ },
+ "ioConfig": {
+ "type": "index_parallel",
+ "inputSource": {
+ "type": "http",
+ "uris": ["https://druid.apache.org/data/wikipedia.json.gz";,
"https://druid.apache.org/data/wikipedia.json.gz";]
+ },
+ "inputFormat": {
+ "type": "json"
+ }
+ },
+ "tuningConfig": {
+ "type": "index_parallel",
+ "maxNumConcurrentSubTasks": 10,
+ "partitionsSpec": {
+ "type": "hashed"
+ },
+ "forceGuaranteedRollup": true,
+ "splitHintSpec": {
+ "type": "maxSize",
+ "maxNumFiles": 1
+ }
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
