This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 5cb0d09c933 Move ITDruidExactCountBitmapTest to embedded tests (#18474)
5cb0d09c933 is described below
commit 5cb0d09c933fdd01745f21ab810332e6d307509f
Author: Kashif Faraz <[email protected]>
AuthorDate: Wed Sep 3 22:24:25 2025 +0530
Move ITDruidExactCountBitmapTest to embedded tests (#18474)
Changes:
- Delete `ITDruidExactCountBitmapTest` and related files
- Add steps in `UnionQueryTest` to perform the same verifications
---
.github/workflows/revised-its.yml | 2 +-
embedded-tests/pom.xml | 6 +
.../testing/embedded/query/UnionQueryTest.java | 63 ++++++-
.../DruidExactCountBitmap/docker-compose.py | 36 ----
integration-tests-ex/cases/pom.xml | 20 --
.../DruidExactCountBitmapTest.java | 205 ---------------------
.../ITDruidExactCountBitmapTest.java | 31 ----
.../testsEx/categories/DruidExactCountBitmap.java | 24 ---
.../resources/druid-exact-count-bitmap/data.json | 10 -
.../kafka_supervisor_template.json | 74 --------
.../druid-exact-count-bitmap/queries.json | 115 ------------
11 files changed, 65 insertions(+), 521 deletions(-)
diff --git a/.github/workflows/revised-its.yml
b/.github/workflows/revised-its.yml
index c662cb47d8f..b88662deefb 100644
--- a/.github/workflows/revised-its.yml
+++ b/.github/workflows/revised-its.yml
@@ -49,7 +49,7 @@ jobs:
fail-fast: false
matrix:
jdk: [17]
- it: [MultiStageQuery, BatchIndex, InputSource, InputFormat,
DruidExactCountBitmap]
+ it: [MultiStageQuery, BatchIndex, InputSource, InputFormat]
indexer: [middleManager]
uses: ./.github/workflows/reusable-revised-its.yml
if: ${{ needs.changes.outputs.core == 'true' ||
needs.changes.outputs.common-extensions == 'true' }}
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index aa137005207..fb83b4c7421 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -131,6 +131,12 @@
<version>${project.parent.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions.contrib</groupId>
+ <artifactId>druid-exact-count-bitmap</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java
index 05a47e8f2d6..c7c1efcea58 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/UnionQueryTest.java
@@ -33,9 +33,9 @@ import org.apache.druid.query.UnionDataSource;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule;
-import
org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule;
-import org.apache.druid.query.aggregation.datasketches.theta.SketchModule;
+import
org.apache.druid.query.aggregation.exact.count.bitmap64.Bitmap64ExactCountBuildAggregatorFactory;
+import
org.apache.druid.query.aggregation.exact.count.bitmap64.Bitmap64ExactCountMergeAggregatorFactory;
+import
org.apache.druid.query.aggregation.exact.count.bitmap64.Bitmap64ExactCountModule;
import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.search.InsensitiveContainsSearchQuerySpec;
@@ -60,6 +60,14 @@ import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+/**
+ * Embedded test to verify the following:
+ * <ul>
+ * <li>Native queries against a {@link UnionDataSource}</li>
+ * <li>SQL UNION ALL queries</li>
+ * <li>Native queries for exact cardinality count using Bitmap64 count
extension</li>
+ * </ul>
+ */
public class UnionQueryTest extends EmbeddedClusterTestBase
{
private final EmbeddedOverlord overlord = new EmbeddedOverlord();
@@ -71,7 +79,7 @@ public class UnionQueryTest extends EmbeddedClusterTestBase
return EmbeddedDruidCluster
.withEmbeddedDerbyAndZookeeper()
.useLatchableEmitter()
- .addExtensions(SketchModule.class, HllSketchModule.class,
DoublesSketchModule.class)
+ .addExtension(Bitmap64ExactCountModule.class)
.addServer(overlord)
.addServer(coordinator)
.addServer(new EmbeddedIndexer())
@@ -93,6 +101,13 @@ public class UnionQueryTest extends EmbeddedClusterTestBase
final Task task = MoreResources.Task.INDEX_TASK_WITH_AGGREGATORS
.get()
.dataSource(datasourceName)
+ .metricAggregates(
+ new CountAggregatorFactory("ingested_events"),
+ new DoubleSumAggregatorFactory("added", "added"),
+ new DoubleSumAggregatorFactory("deleted", "deleted"),
+ new DoubleSumAggregatorFactory("delta", "delta"),
+ new Bitmap64ExactCountBuildAggregatorFactory("unique_deleted",
"deleted")
+ )
.withId(IdUtils.getRandomId());
cluster.callApi().runTask(task, overlord);
cluster.callApi().waitForAllSegmentsToBeAvailable(datasourceName,
coordinator);
@@ -182,7 +197,12 @@ public class UnionQueryTest extends EmbeddedClusterTestBase
)
);
- // Verify some SQL queries
+ verifyBitmap64CardinalityQueries(unionDatasource);
+ verifyUnionAllSqlQueries(datasourceNames);
+ }
+
+ private void verifyUnionAllSqlQueries(List<String> datasourceNames)
+ {
cluster.callApi().verifySqlQuery(
"SELECT page, COUNT(*), SUM(ingested_events), SUM(added),
SUM(deleted), SUM(delta) FROM (%s)"
+ " WHERE __time >= '2013-08-31' AND __time < '2013-09-01'"
@@ -217,6 +237,39 @@ public class UnionQueryTest extends EmbeddedClusterTestBase
);
}
+ private void verifyBitmap64CardinalityQueries(DataSource dataSource)
+ {
+ verifyQuery(
+ Druids
+ .newTimeseriesQueryBuilder()
+ .intervals("2013-08-31/2013-09-01")
+ .dataSource(dataSource)
+ .granularity(Granularities.ALL)
+ .aggregators(
+ new CountAggregatorFactory("rows"),
+ new
Bitmap64ExactCountBuildAggregatorFactory("added_cardinality", "added"),
+ new
Bitmap64ExactCountBuildAggregatorFactory("deleted_cardinality", "deleted"),
+ new
Bitmap64ExactCountMergeAggregatorFactory("unique_deleted_cardinality",
"unique_deleted"),
+ new
Bitmap64ExactCountBuildAggregatorFactory("time_cardinality", "__time")
+ )
+ .build(),
+ List.of(
+ Map.of(
+ "timestamp",
+ "2013-08-31T01:02:33.000Z",
+ "result",
+ Map.of(
+ "rows", 15,
+ "added_cardinality", 5,
+ "deleted_cardinality", 5,
+ "unique_deleted_cardinality", 5,
+ "time_cardinality", 5
+ )
+ )
+ )
+ );
+ }
+
/**
* Creates a SQL for each of the datasources and then combines them with
{@code UNION ALL}.
*/
diff --git
a/integration-tests-ex/cases/cluster/DruidExactCountBitmap/docker-compose.py
b/integration-tests-ex/cases/cluster/DruidExactCountBitmap/docker-compose.py
deleted file mode 100644
index 8cb9b56fddf..00000000000
--- a/integration-tests-ex/cases/cluster/DruidExactCountBitmap/docker-compose.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from template import BaseTemplate, generate
-from template import COORDINATOR, MIDDLE_MANAGER
-
-class Template(BaseTemplate):
- def extend_druid_service(self, service):
- self.add_env(service, 'druid_test_loadList',
'druid-exact-count-bitmap')
-
- def define_coordinator(self):
- service_name = COORDINATOR
- service = self.define_master_service(service_name, COORDINATOR)
- self.add_env(service, 'druid_host', service_name)
- self.add_env(service, 'druid_manager_segments_pollDuration', 'PT5S')
- self.add_env(service, 'druid_coordinator_period', 'PT10S')
-
- def define_indexer(self):
- '''
- Override the indexer to MIDDLE_MANAGER
- '''
- return self.define_std_indexer(MIDDLE_MANAGER)
-
-generate(__file__, Template())
diff --git a/integration-tests-ex/cases/pom.xml
b/integration-tests-ex/cases/pom.xml
index 703287d4857..f8c25e426e6 100644
--- a/integration-tests-ex/cases/pom.xml
+++ b/integration-tests-ex/cases/pom.xml
@@ -74,11 +74,6 @@
<artifactId>druid-indexing-service</artifactId>
<version>${project.parent.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-kafka-indexing-service</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.druid.integration-tests</groupId>
<artifactId>druid-it-tools</artifactId>
@@ -259,12 +254,6 @@
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.druid.extensions.contrib</groupId>
- <artifactId>druid-exact-count-bitmap</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -509,14 +498,5 @@
</plugins>
</build>
</profile>
- <profile>
- <id>IT-DruidExactCountBitmap</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <it.category>DruidExactCountBitmap</it.category>
- </properties>
- </profile>
</profiles>
</project>
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/DruidExactCountBitmapTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/DruidExactCountBitmapTest.java
deleted file mode 100644
index b4c83cb58dd..00000000000
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/DruidExactCountBitmapTest.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.testsEx.DruidExactCountBitmap;
-
-import org.apache.druid.indexing.kafka.KafkaConsumerConfigs;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.testing.tools.ITRetryUtil;
-import org.apache.druid.testing.tools.IntegrationTestingConfig;
-import org.apache.druid.testing.tools.KafkaEventWriter;
-import org.apache.druid.testing.tools.KafkaUtil;
-import org.apache.druid.testing.tools.StreamEventWriter;
-import org.apache.druid.testing.utils.KafkaAdminClient;
-import org.apache.druid.testsEx.indexer.AbstractIndexerTest;
-import org.joda.time.Interval;
-import org.junit.Test;
-
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-import java.util.function.Function;
-
-public class DruidExactCountBitmapTest extends AbstractIndexerTest
-{
- private static final Logger LOG = new
Logger(DruidExactCountBitmapTest.class);
- private static final String SUPERVISOR_TEMPLATE =
"/druid-exact-count-bitmap/kafka_supervisor_template.json";
- private static final String QUERY_FILE =
"/druid-exact-count-bitmap/queries.json";
- private static final String DATA_FILE =
"/druid-exact-count-bitmap/data.json";
- private static final String DATASOURCE = "wikipedia_index_test";
-
- @Test
- public void testQuery() throws Exception
- {
- final String fullDatasourceName = DATASOURCE +
config.getExtraDatasourceNameSuffix();
- String baseName = fullDatasourceName + UUID.randomUUID();
- KafkaAdminClient streamAdminClient = new KafkaAdminClient(config);
- List<String> supervisors = new ArrayList<>();
-
- final int numDatasources = 3;
- for (int i = 0; i < numDatasources; i++) {
- String datasource = baseName + "-" + i;
- streamAdminClient.createStream(datasource, 1, Collections.emptyMap());
- ITRetryUtil.retryUntil(
- () -> streamAdminClient.isStreamActive(datasource),
- true,
- 10000,
- 30,
- "Wait for stream active"
- );
- String supervisorSpec = generateStreamIngestionPropsTransform(
- datasource,
- datasource,
- config
- ).apply(getResourceAsString(SUPERVISOR_TEMPLATE));
- LOG.info("supervisorSpec: [%s]\n", supervisorSpec);
- // Start supervisor
- String specResponse = indexer.submitSupervisor(supervisorSpec);
- LOG.info("Submitted supervisor [%s]", specResponse);
- supervisors.add(specResponse);
-
- int ctr = 0;
- try (
- StreamEventWriter streamEventWriter = new KafkaEventWriter(config,
false);
- BufferedReader reader = new BufferedReader(
- new InputStreamReader(getResourceAsStream(DATA_FILE),
StandardCharsets.UTF_8)
- )
- ) {
- String line;
- while ((line = reader.readLine()) != null) {
- streamEventWriter.write(datasource, StringUtils.toUtf8(line));
- ctr++;
- }
- }
- final int numWritten = ctr;
-
- LOG.info("Waiting for stream indexing tasks to consume events");
-
- ITRetryUtil.retryUntilTrue(
- () ->
- numWritten == this.queryHelper.countRows(
- datasource,
- Intervals.ETERNITY,
- name -> new LongSumAggregatorFactory(name, "count")
- ),
- StringUtils.format(
- "dataSource[%s] consumed [%,d] events, expected [%,d]",
- datasource,
- this.queryHelper.countRows(
- datasource,
- Intervals.ETERNITY,
- name -> new LongSumAggregatorFactory(name, "count")
- ),
- numWritten
- )
- );
- }
-
- String queryResponseTemplate = StringUtils.replace(
- getResourceAsString(QUERY_FILE),
- "%%DATASOURCE%%",
- baseName
- );
-
- queryHelper.testQueriesFromString(queryResponseTemplate);
-
-
- for (int i = 0; i < numDatasources; i++) {
- indexer.terminateSupervisor(supervisors.get(i));
- streamAdminClient.deleteStream(baseName + "-" + i);
- }
-
- for (int i = 0; i < numDatasources; i++) {
- final int datasourceNumber = i;
- ITRetryUtil.retryUntil(
- () -> coordinator.areSegmentsLoaded(baseName + "-" +
datasourceNumber),
- true,
- 10000,
- 10,
- "Kafka segments loaded"
- );
- }
-
- queryHelper.testQueriesFromString(queryResponseTemplate);
-
- for (int i = 0; i < numDatasources; i++) {
- final String datasource = baseName + "-" + i;
- List<String> intervals = coordinator.getSegmentIntervals(datasource);
-
- Collections.sort(intervals);
- String first = intervals.get(0).split("/")[0];
- String last = intervals.get(intervals.size() - 1).split("/")[1];
- Interval interval = Intervals.of(first + "/" + last);
- coordinator.unloadSegmentsForDataSource(baseName + "-" + i);
- ITRetryUtil.retryUntilFalse(
- () -> coordinator.areSegmentsLoaded(datasource),
- "Segment Unloading"
- );
- coordinator.deleteSegmentsDataSource(baseName + "-" + i, interval);
- }
- }
-
- /**
- * sad version of
- * {@link
org.apache.druid.tests.indexer.AbstractKafkaIndexingServiceTest#generateStreamIngestionPropsTransform}
- */
- private Function<String, String> generateStreamIngestionPropsTransform(
- String streamName,
- String fullDatasourceName,
- IntegrationTestingConfig config
- )
- {
- final Map<String, Object> consumerConfigs =
KafkaConsumerConfigs.getConsumerProperties();
- final Properties consumerProperties = new Properties();
- consumerProperties.putAll(consumerConfigs);
- consumerProperties.setProperty("bootstrap.servers",
config.getKafkaInternalHost());
- KafkaUtil.addPropertiesFromTestConfig(config, consumerProperties);
- return spec -> {
- try {
- spec = StringUtils.replace(
- spec,
- "%%DATASOURCE%%",
- fullDatasourceName
- );
- spec = StringUtils.replace(
- spec,
- "%%TOPIC_VALUE%%",
- streamName
- );
- return StringUtils.replace(
- spec,
- "%%STREAM_PROPERTIES_VALUE%%",
- jsonMapper.writeValueAsString(consumerProperties)
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- };
- }
-}
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/ITDruidExactCountBitmapTest.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/ITDruidExactCountBitmapTest.java
deleted file mode 100644
index 21241515657..00000000000
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/DruidExactCountBitmap/ITDruidExactCountBitmapTest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.testsEx.DruidExactCountBitmap;
-
-import org.apache.druid.testsEx.categories.DruidExactCountBitmap;
-import org.apache.druid.testsEx.config.DruidTestRunner;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-
-@RunWith(DruidTestRunner.class)
-@Category(DruidExactCountBitmap.class)
-public class ITDruidExactCountBitmapTest extends DruidExactCountBitmapTest
-{
-}
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCountBitmap.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCountBitmap.java
deleted file mode 100644
index 42799ff2b48..00000000000
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/categories/DruidExactCountBitmap.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.testsEx.categories;
-
-public class DruidExactCountBitmap
-{
-}
diff --git
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/data.json
b/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/data.json
deleted file mode 100644
index f39dcd7c104..00000000000
---
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/data.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{"timestamp": "2013-08-31T01:02:33Z", "page": "Gypsy Danger", "language" :
"en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot":
"false", "anonymous": "false", "namespace":"article", "continent":"North
America", "country":"United States", "region":"Bay Area", "city":"San
Francisco", "added": 459, "deleted": 200, "delta": -143}
-{"timestamp": "2013-08-31T03:32:45Z", "page": "Striker Eureka", "language" :
"en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot":
"true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia",
"country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459,
"deleted": 129, "delta": 330}
-{"timestamp": "2013-08-31T07:11:21Z", "page": "Cherno Alpha", "language" :
"ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true",
"robot": "true", "anonymous": "false", "namespace":"article",
"continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow",
"added": 123, "deleted": 12, "delta": 111}
-{"timestamp": "2013-08-31T11:58:39Z", "page": "Crimson Typhoon", "language" :
"zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false",
"robot": "true", "anonymous": "false", "namespace":"wikipedia",
"continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan",
"added": 905, "deleted": 5, "delta": 900}
-{"timestamp": "2013-08-31T12:41:27Z", "page": "Coyote Tango", "language" :
"ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false",
"robot": "true", "anonymous": "false", "namespace":"wikipedia",
"continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo",
"added": 1, "deleted": 10, "delta": -9}
-{"timestamp": "2013-09-01T01:02:33Z", "page": "Gypsy Danger", "language" :
"en", "user" : "nuclear", "unpatrolled" : "true", "newPage" : "true", "robot":
"false", "anonymous": "false", "namespace":"article", "continent":"North
America", "country":"United States", "region":"Bay Area", "city":"San
Francisco", "added": 459, "deleted": 200, "delta": -143}
-{"timestamp": "2013-09-01T03:32:45Z", "page": "Striker Eureka", "language" :
"en", "user" : "speed", "unpatrolled" : "false", "newPage" : "true", "robot":
"true", "anonymous": "false", "namespace":"wikipedia", "continent":"Australia",
"country":"Australia", "region":"Cantebury", "city":"Syndey", "added": 459,
"deleted": 129, "delta": 330}
-{"timestamp": "2013-09-01T07:11:21Z", "page": "Cherno Alpha", "language" :
"ru", "user" : "masterYi", "unpatrolled" : "false", "newPage" : "true",
"robot": "true", "anonymous": "false", "namespace":"article",
"continent":"Asia", "country":"Russia", "region":"Oblast", "city":"Moscow",
"added": 123, "deleted": 12, "delta": 111}
-{"timestamp": "2013-09-01T11:58:39Z", "page": "Crimson Typhoon", "language" :
"zh", "user" : "triplets", "unpatrolled" : "true", "newPage" : "false",
"robot": "true", "anonymous": "false", "namespace":"wikipedia",
"continent":"Asia", "country":"China", "region":"Shanxi", "city":"Taiyuan",
"added": 905, "deleted": 5, "delta": 900}
-{"timestamp": "2013-09-01T12:41:27Z", "page": "Coyote Tango", "language" :
"ja", "user" : "stringer", "unpatrolled" : "true", "newPage" : "false",
"robot": "true", "anonymous": "false", "namespace":"wikipedia",
"continent":"Asia", "country":"Japan", "region":"Kanto", "city":"Tokyo",
"added": 1, "deleted": 10, "delta": -9}
diff --git
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/kafka_supervisor_template.json
b/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/kafka_supervisor_template.json
deleted file mode 100644
index bd9dd3f111f..00000000000
---
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/kafka_supervisor_template.json
+++ /dev/null
@@ -1,74 +0,0 @@
-{
- "type": "kafka",
- "dataSchema": {
- "dataSource": "%%DATASOURCE%%",
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [
- "page",
- "language",
- "user",
- "unpatrolled",
- "newPage",
- "robot",
- "anonymous",
- "namespace",
- "continent",
- "country",
- "region",
- "city"
- ],
- "dimensionExclusions": [],
- "spatialDimensions": []
- },
- "metricsSpec": [
- {
- "type": "count",
- "name": "count"
- },
- {
- "type": "doubleSum",
- "name": "added",
- "fieldName": "added"
- },
- {
- "type": "doubleSum",
- "name": "deleted",
- "fieldName": "deleted"
- },
- {
- "type": "doubleSum",
- "name": "delta",
- "fieldName": "delta"
- },
- {
- "type": "Bitmap64ExactCountBuild",
- "name": "unique_deleted",
- "fieldName": "deleted"
- }
- ],
- "granularitySpec": {
- "type": "uniform",
- "segmentGranularity": "DAY",
- "queryGranularity": "second"
- }
- },
- "tuningConfig": {
- "type": "kafka",
- "intermediatePersistPeriod": "PT1H",
- "maxRowsPerSegment": 5000000,
- "maxRowsInMemory": 500000
- },
- "ioConfig": {
- "topic": "%%TOPIC_VALUE%%",
- "consumerProperties": %%STREAM_PROPERTIES_VALUE%%,
- "taskCount": 2,
- "replicas": 1,
- "taskDuration": "PT120S",
- "useEarliestOffset": true,
- "inputFormat" : {"type": "json"}
- }
-}
diff --git
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/queries.json
b/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/queries.json
deleted file mode 100644
index cd7e90af3fe..00000000000
---
a/integration-tests-ex/cases/src/test/resources/druid-exact-count-bitmap/queries.json
+++ /dev/null
@@ -1,115 +0,0 @@
-[
- {
- "description": "Timeseries query on non-rolled-up columns",
- "query": {
- "queryType": "timeseries",
- "dataSource": {
- "type": "union",
- "dataSources": [
- "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3",
- "%%DATASOURCE%%-0"
- ]
- },
- "intervals": ["2013-08-31/2013-09-01"],
- "granularity": "all",
- "aggregations": [
- {
- "type": "count",
- "name": "added_count",
- "fieldName": "added"
- },
- {
- "type": "Bitmap64ExactCountBuild",
- "name": "added_cardinality",
- "fieldName": "added"
- }
- ]
- },
- "expectedResults": [
- {
- "timestamp": "2013-08-31T01:02:33.000Z",
- "result": {
- "added_count": 15,
- "added_cardinality": 4
- }
- }
- ]
- },
- {
- "description": "Timeseries query on rolled-up columns",
- "query": {
- "queryType": "timeseries",
- "dataSource": {
- "type": "union",
- "dataSources": [
- "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3",
- "%%DATASOURCE%%-0"
- ]
- },
- "intervals": ["2013-08-31/2013-09-01"],
- "granularity": "all",
- "aggregations": [
- {
- "type": "count",
- "name": "deleted_count",
- "fieldName": "deleted"
- },
- {
- "type": "Bitmap64ExactCountBuild",
- "name": "deleted_cardinality",
- "fieldName": "deleted"
- },
- {
- "type": "Bitmap64ExactCountMerge",
- "name": "deleted_cardinality_from_rollup_columns",
- "fieldName": "unique_deleted"
- }
- ]
- },
- "expectedResults": [
- {
- "timestamp": "2013-08-31T01:02:33.000Z",
- "result": {
- "deleted_count": 15,
- "deleted_cardinality": 5,
- "deleted_cardinality_from_rollup_columns": 5
- }
- }
- ]
- },
- {
- "description": "Bitmap64 works on time column",
- "query": {
- "queryType": "timeseries",
- "dataSource": {
- "type": "union",
- "dataSources": [
- "%%DATASOURCE%%-1", "%%DATASOURCE%%-2", "%%DATASOURCE%%-3",
- "%%DATASOURCE%%-0"
- ]
- },
- "intervals": ["2013-08-31/2013-09-01"],
- "granularity": "all",
- "aggregations": [
- {
- "type": "count",
- "name": "time_count"
- },
- {
- "type": "Bitmap64ExactCountBuild",
- "name": "time_cardinality",
- "fieldName": "__time"
- }
- ]
- },
- "expectedResults": [
- {
- "timestamp": "2013-08-31T01:02:33.000Z",
- "result": {
- "time_count": 15,
- "time_cardinality": 5
- }
- }
- ]
- }
-]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
