github-advanced-security[bot] commented on code in PR #17383:
URL: https://github.com/apache/druid/pull/17383#discussion_r1807155486
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java:
##########
@@ -298,179 +183,15 @@
})
private String query;
- private SqlEngine engine;
- @Nullable
- private PlannerFactory plannerFactory;
- private final Closer closer = Closer.create();
-
- @Setup(Level.Trial)
- public void setup()
- {
- final GeneratorSchemaInfo schemaInfo =
GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
-
- final DataSegment dataSegment = DataSegment.builder()
- .dataSource("foo")
-
.interval(schemaInfo.getDataInterval())
- .version("1")
- .shardSpec(new
LinearShardSpec(0))
- .size(0)
- .build();
-
-
- final PlannerConfig plannerConfig = new PlannerConfig();
-
- final SegmentGenerator segmentGenerator = closer.register(new
SegmentGenerator());
- log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].",
segmentGenerator.getCacheDir(), rowsPerSegment);
-
- TransformSpec transformSpec = new TransformSpec(
- null,
- ImmutableList.of(
- new ExpressionTransform(
- "nested",
- "json_object('long1', long1, 'nesteder',
json_object('string1', string1, 'long2', long2, 'double3',double3, 'string5',
string5))",
- TestExprMacroTable.INSTANCE
- )
- )
- );
-
-
-
- StringEncodingStrategy encodingStrategy;
- if (stringEncoding.startsWith("front-coded")) {
- String[] split = stringEncoding.split("-");
- int bucketSize = Integer.parseInt(split[2]);
- encodingStrategy = new StringEncodingStrategy.FrontCoded(bucketSize,
FrontCodedIndexed.V1);
- } else {
- encodingStrategy = new StringEncodingStrategy.Utf8();
- }
- final QueryableIndex index;
- if ("auto".equals(schema)) {
- Iterable<DimensionSchema> columnSchemas = Iterables.concat(
- schemaInfo.getDimensionsSpec()
- .getDimensions()
- .stream()
- .map(x -> new AutoTypeColumnSchema(x.getName(), null))
- .collect(Collectors.toList()),
- Collections.singletonList(new AutoTypeColumnSchema("nested", null))
- );
- index = segmentGenerator.generate(
- dataSegment,
- schemaInfo,
-
DimensionsSpec.builder().setDimensions(ImmutableList.copyOf(columnSchemas.iterator())).build(),
- transformSpec,
-
IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(),
- Granularities.NONE,
- rowsPerSegment
- );
- } else {
- Iterable<DimensionSchema> columnSchemas = Iterables.concat(
- schemaInfo.getDimensionsSpec().getDimensions(),
- Collections.singletonList(new AutoTypeColumnSchema("nested", null))
- );
- index = segmentGenerator.generate(
- dataSegment,
- schemaInfo,
-
DimensionsSpec.builder().setDimensions(ImmutableList.copyOf(columnSchemas.iterator())).build(),
- transformSpec,
-
IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(),
- Granularities.NONE,
- rowsPerSegment
- );
- }
-
- final QueryRunnerFactoryConglomerate conglomerate =
QueryStackTests.createQueryRunnerFactoryConglomerate(
- closer,
- PROCESSING_CONFIG
- );
-
- final SpecificSegmentsQuerySegmentWalker walker =
SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate).add(
- dataSegment,
- index
- );
- closer.register(walker);
-
- final DruidSchemaCatalog rootSchema =
- CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig,
AuthTestUtils.TEST_AUTHORIZER_MAPPER);
- engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
- plannerFactory = new PlannerFactory(
- rootSchema,
- CalciteTests.createOperatorTable(),
- CalciteTests.createExprMacroTable(),
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER,
- CalciteTests.getJsonMapper(),
- CalciteTests.DRUID_SCHEMA_NAME,
- new CalciteRulesManager(ImmutableSet.of()),
- CalciteTests.createJoinableFactoryWrapper(),
- CatalogResolver.NULL_RESOLVER,
- new AuthConfig(),
- new DruidHookDispatcher()
- );
-
- try {
- SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
- engine,
- plannerFactory,
- QUERIES.get(Integer.parseInt(query))
- );
- log.info("non-vectorized and vectorized results match");
- }
- catch (Throwable ex) {
- log.warn(ex, "non-vectorized and vectorized results do not match");
- }
-
- final String sql = QUERIES.get(Integer.parseInt(query));
- final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, "EXPLAIN PLAN FOR " + sql,
ImmutableMap.of("useNativeQueryExplain", true))) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Object[] planResult = resultSequence.toList().get(0);
- log.info("Native query plan:\n" +
- jsonMapper.writerWithDefaultPrettyPrinter()
- .writeValueAsString(jsonMapper.readValue((String)
planResult[0], List.class))
- );
- }
- catch (JsonProcessingException ex) {
- log.warn(ex, "explain failed");
- }
-
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Yielder<Object[]> yielder = Yielders.each(resultSequence);
- int rowCounter = 0;
- while (!yielder.isDone()) {
- rowCounter++;
- yielder.next(yielder.get());
- }
- log.info("Total result row count:" + rowCounter);
- }
- catch (Throwable ex) {
- log.warn(ex, "failed to count rows");
- }
- }
-
- @TearDown(Level.Trial)
- public void tearDown() throws Exception
+ @Override
+ public String getQuery()
{
- closer.close();
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8423)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java:
##########
@@ -304,154 +204,26 @@
})
private String query;
- private SqlEngine engine;
- @Nullable
- private PlannerFactory plannerFactory;
- private Closer closer = Closer.create();
-
- @Setup(Level.Trial)
- public void setup()
+ @Override
+ public String getQuery()
{
- final GeneratorSchemaInfo schemaInfo =
GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
-
- final DataSegment dataSegment = DataSegment.builder()
- .dataSource("foo")
-
.interval(schemaInfo.getDataInterval())
- .version("1")
- .shardSpec(new
LinearShardSpec(0))
- .size(0)
- .build();
-
- final PlannerConfig plannerConfig = new PlannerConfig();
-
- final SegmentGenerator segmentGenerator = closer.register(new
SegmentGenerator());
- log.info(
- "Starting benchmark setup using cacheDir[%s], rows[%,d], schema[%s].",
- segmentGenerator.getCacheDir(),
- rowsPerSegment,
- schema
- );
- final QueryableIndex index;
- if ("auto".equals(schema)) {
- List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
- .getDimensions()
- .stream()
- .map(x -> new
AutoTypeColumnSchema(x.getName(), null))
-
.collect(Collectors.toList());
- index = segmentGenerator.generate(
- dataSegment,
- schemaInfo,
- DimensionsSpec.builder().setDimensions(columnSchemas).build(),
- TransformSpec.NONE,
- IndexSpec.DEFAULT,
- Granularities.NONE,
- rowsPerSegment
- );
- } else {
- index = segmentGenerator.generate(dataSegment, schemaInfo,
Granularities.NONE, rowsPerSegment);
- }
-
- final QueryRunnerFactoryConglomerate conglomerate =
QueryStackTests.createQueryRunnerFactoryConglomerate(
- closer,
- PROCESSING_CONFIG
- );
-
- final SpecificSegmentsQuerySegmentWalker walker =
SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate).add(
- dataSegment,
- index
- );
- closer.register(walker);
- final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
- final DruidSchemaCatalog rootSchema =
- CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig,
AuthTestUtils.TEST_AUTHORIZER_MAPPER);
- engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
- plannerFactory = new PlannerFactory(
- rootSchema,
- CalciteTests.createOperatorTable(),
- CalciteTests.createExprMacroTable(),
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER,
- jsonMapper,
- CalciteTests.DRUID_SCHEMA_NAME,
- new CalciteRulesManager(ImmutableSet.of()),
- CalciteTests.createJoinableFactoryWrapper(),
- CatalogResolver.NULL_RESOLVER,
- new AuthConfig(),
- new DruidHookDispatcher()
- );
-
- try {
- SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
- engine,
- plannerFactory,
- QUERIES.get(Integer.parseInt(query))
- );
- log.info("non-vectorized and vectorized results match");
- }
- catch (Throwable ex) {
- log.warn(ex, "non-vectorized and vectorized results do not match");
- }
-
- final String sql = QUERIES.get(Integer.parseInt(query));
-
- try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(
- engine,
- "EXPLAIN PLAN FOR " + sql,
- ImmutableMap.of(
- "useNativeQueryExplain",
- true
- )
- )) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Object[] planResult = resultSequence.toList().get(0);
- log.info("Native query plan:\n" +
- jsonMapper.writerWithDefaultPrettyPrinter()
- .writeValueAsString(jsonMapper.readValue((String)
planResult[0], List.class))
- );
- }
- catch (JsonProcessingException ex) {
- log.warn(ex, "explain failed");
- }
-
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Yielder<Object[]> yielder = Yielders.each(resultSequence);
- int rowCounter = 0;
- while (!yielder.isDone()) {
- rowCounter++;
- yielder.next(yielder.get());
- }
- log.info("Total result row count:" + rowCounter);
- }
- catch (Throwable ex) {
- log.warn(ex, "failed to count rows");
- }
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8420)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlWindowFunctionsBenchmark.java:
##########
@@ -121,319 +45,55 @@
@Fork(value = 1)
@Warmup(iterations = 3)
@Measurement(iterations = 5)
-public class SqlWindowFunctionsBenchmark
+public class SqlWindowFunctionsBenchmark extends SqlBaseQueryBenchmark
{
- static {
- NullHandling.initializeForTests();
- }
-
- private static final Logger log = new
Logger(SqlWindowFunctionsBenchmark.class);
+ private static final List<String> QUERIES = ImmutableList.of(
+ "SELECT SUM(dimSequentialHalfNull) FROM druid.basic GROUP BY dimUniform",
+ "SELECT SUM(SUM(dimSequentialHalfNull)) OVER (ORDER BY dimUniform) FROM
druid.basic GROUP BY dimUniform",
+ "SELECT ROW_NUMBER() OVER (PARTITION BY dimUniform ORDER BY
dimSequential) FROM druid.basic",
+ "SELECT COUNT(*) OVER (PARTITION BY dimUniform RANGE BETWEEN UNBOUNDED
PRECEDING AND UNBOUNDED FOLLOWING) FROM druid.basic",
+ "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential
RANGE UNBOUNDED PRECEDING) FROM druid.basic",
+ "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential
RANGE UNBOUNDED FOLLOWING) FROM druid.basic",
+ "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential)
FROM druid.basic GROUP BY dimSequential, dimUniform",
+ "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential)
FROM druid.basic GROUP BY dimUniform, dimSequential",
+ "SELECT SUM(dimSequentialHalfNull) + SUM(dimZipf),
LAG(SUM(dimSequentialHalfNull + dimZipf)) OVER (PARTITION BY dimUniform ORDER
BY dimSequential) FROM druid.basic GROUP BY __time, dimUniform, dimSequential"
+ );
- private static final String STORAGE_MMAP = "mmap";
- private static final String STORAGE_FRAME_ROW = "frame-row";
- private static final String STORAGE_FRAME_COLUMNAR = "frame-columnar";
-
- @Param({"2000000"})
- private int rowsPerSegment;
@Param({
- "auto"
+ "0",
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7",
+ "8"
})
- private String schema;
-
- // Can be STORAGE_MMAP, STORAGE_FRAME_ROW, or STORAGE_FRAME_COLUMNAR
- @Param({STORAGE_MMAP})
- private String storageType;
-
- private SqlEngine engine;
-
- @Nullable
- private PlannerFactory plannerFactory;
- private final Closer closer = Closer.create();
-
- private static final DruidProcessingConfig PROCESSING_CONFIG = new
DruidProcessingConfig() {
-
- @Override
- public int getNumMergeBuffers()
- {
- return 3;
- }
-
- @Override
- public int intermediateComputeSizeBytes()
- {
- return 200_000_000;
- }
- };
-
- @Setup(Level.Trial)
- public void setup()
- {
- final GeneratorSchemaInfo schemaInfo =
GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
- final DataSegment dataSegment = schemaInfo.makeSegmentDescriptor("foo");
- final SegmentGenerator segmentGenerator = closer.register(new
SegmentGenerator());
-
- log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].",
segmentGenerator.getCacheDir(), rowsPerSegment);
- final QueryableIndex index;
- if ("auto".equals(schema)) {
- List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
- .getDimensions()
- .stream()
- .map(x -> new
AutoTypeColumnSchema(x.getName(), null))
-
.collect(Collectors.toList());
- index = segmentGenerator.generate(
- dataSegment,
- schemaInfo,
- DimensionsSpec.builder().setDimensions(columnSchemas).build(),
- TransformSpec.NONE,
-
IndexSpec.builder().withStringDictionaryEncoding(getStringEncodingStrategy()).build(),
- Granularities.NONE,
- rowsPerSegment
- );
- } else {
- index = segmentGenerator.generate(dataSegment, schemaInfo,
Granularities.NONE, rowsPerSegment);
- }
-
- final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
- ImmutableMap.of(dataSegment, index),
- Collections.emptyMap(),
- null,
- closer
- );
-
- plannerFactory = sqlSystem.lhs;
- engine = sqlSystem.rhs;
- }
-
- private StringEncodingStrategy getStringEncodingStrategy()
- {
- return new StringEncodingStrategy.Utf8();
- }
-
- public static Pair<PlannerFactory, SqlEngine> createSqlSystem(
- final Map<DataSegment, QueryableIndex> segmentMap,
- final Map<String, LookupExtractor> lookupMap,
- @Nullable final String storageType,
- final Closer closer
- )
- {
- final QueryRunnerFactoryConglomerate conglomerate =
QueryStackTests.createQueryRunnerFactoryConglomerate(closer, PROCESSING_CONFIG);
- final SpecificSegmentsQuerySegmentWalker walker =
SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate);
- final PlannerConfig plannerConfig = new PlannerConfig();
-
- for (final Map.Entry<DataSegment, QueryableIndex> segmentEntry :
segmentMap.entrySet()) {
- addSegmentToWalker(walker, segmentEntry.getKey(),
segmentEntry.getValue(), storageType);
- }
-
- // Child injector that adds additional lookups.
- final Injector injector = new StartupInjectorBuilder()
- .withEmptyProperties()
- .add(
- new ExpressionModule(),
- new SegmentWranglerModule(),
- new LookylooModule(),
- new SqlAggregationModule(),
- new CalciteTestOperatorModule(),
- binder -> {
- for (Map.Entry<String, LookupExtractor> entry :
lookupMap.entrySet()) {
- MapBinder.newMapBinder(binder, String.class,
LookupExtractor.class)
- .addBinding(entry.getKey())
- .toProvider(entry::getValue)
- .in(LazySingleton.class);
- }
- }
- )
- .build();
-
- final DruidSchemaCatalog rootSchema =
- QueryFrameworkUtils.createMockRootSchema(
- injector,
- conglomerate,
- walker,
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER
- );
-
- final SqlEngine engine = CalciteTests.createMockSqlEngine(walker,
conglomerate);
-
- final PlannerFactory plannerFactory = new PlannerFactory(
- rootSchema,
- createOperatorTable(injector),
- injector.getInstance(ExprMacroTable.class),
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER,
- injector.getInstance(Key.get(ObjectMapper.class, Json.class)),
- CalciteTests.DRUID_SCHEMA_NAME,
- new CalciteRulesManager(ImmutableSet.of()),
- new
JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
- CatalogResolver.NULL_RESOLVER,
- new AuthConfig(),
- new DruidHookDispatcher()
- );
-
- return Pair.of(plannerFactory, engine);
- }
+ private String query;
- private static void addSegmentToWalker(
- final SpecificSegmentsQuerySegmentWalker walker,
- final DataSegment descriptor,
- final QueryableIndex index,
- @Nullable final String storageType
- )
- {
- if (storageType == null || STORAGE_MMAP.equals(storageType)) {
- walker.add(descriptor, new QueryableIndexSegment(index,
descriptor.getId()));
- } else if (STORAGE_FRAME_ROW.equals(storageType)) {
- walker.add(
- descriptor,
- FrameTestUtil.cursorFactoryToFrameSegment(
- new QueryableIndexCursorFactory(index),
- FrameType.ROW_BASED,
- descriptor.getId()
- )
- );
- } else if (STORAGE_FRAME_COLUMNAR.equals(storageType)) {
- walker.add(
- descriptor,
- FrameTestUtil.cursorFactoryToFrameSegment(
- new QueryableIndexCursorFactory(index),
- FrameType.COLUMNAR,
- descriptor.getId()
- )
- );
- } else {
- throw new IAE("Invalid storageType[%s]", storageType);
- }
- }
- private static DruidOperatorTable createOperatorTable(final Injector
injector)
+ @Override
+ public String getQuery()
{
- try {
- final Set<SqlOperatorConversion> extractionOperators = new HashSet<>();
-
extractionOperators.add(injector.getInstance(QueryLookupOperatorConversion.class));
- final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator =
- new ApproxCountDistinctSqlAggregator(new
HllSketchApproxCountDistinctSqlAggregator());
- final Set<SqlAggregator> aggregators = new HashSet<>(
- ImmutableList.of(
- new DoublesSketchApproxQuantileSqlAggregator(),
- new DoublesSketchObjectSqlAggregator(),
- new HllSketchApproxCountDistinctSqlAggregator(),
- new HllSketchApproxCountDistinctUtf8SqlAggregator(),
- new ThetaSketchApproxCountDistinctSqlAggregator(),
- new CountSqlAggregator(countDistinctSqlAggregator),
- countDistinctSqlAggregator
- )
- );
- return new DruidOperatorTable(aggregators, extractionOperators);
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8424)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlProjectionsBenchmark.java:
##########
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.query.QueryContexts;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+import java.util.Map;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlProjectionsBenchmark extends SqlBaseQueryBenchmark
+{
+ private static final List<String> QUERIES = ImmutableList.of(
+ "SELECT string2, APPROX_COUNT_DISTINCT_DS_HLL(string5) FROM
druid.projections GROUP BY 1 ORDER BY 2",
+ "SELECT string2, SUM(long4) FROM druid.projections GROUP BY 1 ORDER BY 2"
+ );
+
+ @Param({
+ "0",
+ "1"
+ })
+ private String query;
+
+ @Param({
+ "true",
+ "false"
+ })
+ private boolean useProjections;
+
+ @Override
+ public String getQuery()
+ {
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8422)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlComplexMetricsColumnsBenchmark.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlComplexMetricsColumnsBenchmark extends SqlBaseQueryBenchmark
+{
+ private static final List<String> QUERIES = ImmutableList.of(
+ "SELECT APPROX_COUNT_DISTINCT_DS_HLL(hll_string5) FROM
druid.datasketches",
+ "SELECT APPROX_COUNT_DISTINCT_DS_THETA(theta_string5) FROM
druid.datasketches",
+ "SELECT DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_float4), 0.5) FROM
druid.datasketches",
+ "SELECT DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_long3), 0.9) FROM
druid.datasketches",
+ "SELECT string2, APPROX_COUNT_DISTINCT_DS_HLL(hll_string5) FROM
druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+ "SELECT string2, APPROX_COUNT_DISTINCT_DS_THETA(theta_string5, 4096)
FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+ "SELECT string2, DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_float4),
0.5) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+ "SELECT string2, DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_long3),
0.9) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC"
+ );
+
+ @Param({
+ "0",
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7"
+ })
+ private String query;
+
+ @Override
+ public String getQuery()
+ {
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8419)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlPlanBenchmark.java:
##########
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlPlanBenchmark extends SqlBasePlanBenchmark
+{
+ @Param({
+ "0",
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7",
+ "8",
+ "9",
+ "10",
+ "11",
+ "12",
+ "13",
+ "14",
+ "15",
+ "16",
+ "17",
+ "18",
+ "19",
+ "20",
+ "21",
+ "22",
+ "23",
+ "24",
+ "25",
+ "26",
+ "27",
+ "28",
+ "29",
+ "30",
+ "31",
+ "32",
+ "33",
+ "34",
+ "35",
+ "36",
+ "37",
+ "38",
+ "39",
+ "40",
+ "41"
+ })
+ private String query;
+
+ @Override
+ public String getQuery()
+ {
+ return SqlBenchmark.QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8421)
##########
benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java:
##########
@@ -528,254 +412,15 @@
})
private String query;
- // Can be STORAGE_MMAP, STORAGE_FRAME_ROW, or STORAGE_FRAME_COLUMNAR
- @Param({STORAGE_MMAP})
- private String storageType;
-
- private SqlEngine engine;
-
- @Nullable
- private PlannerFactory plannerFactory;
- private final Closer closer = Closer.create();
-
- @Setup(Level.Trial)
- public void setup()
- {
- final GeneratorSchemaInfo schemaInfo =
GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
- final DataSegment dataSegment = schemaInfo.makeSegmentDescriptor("foo");
- final SegmentGenerator segmentGenerator = closer.register(new
SegmentGenerator());
-
- log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].",
segmentGenerator.getCacheDir(), rowsPerSegment);
- final QueryableIndex index;
- if ("auto".equals(schema)) {
- List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
- .getDimensions()
- .stream()
- .map(x -> new
AutoTypeColumnSchema(x.getName(), null))
-
.collect(Collectors.toList());
- index = segmentGenerator.generate(
- dataSegment,
- schemaInfo,
- DimensionsSpec.builder().setDimensions(columnSchemas).build(),
- TransformSpec.NONE,
-
IndexSpec.builder().withStringDictionaryEncoding(getStringEncodingStrategy()).build(),
- Granularities.NONE,
- rowsPerSegment
- );
- } else {
- index = segmentGenerator.generate(dataSegment, schemaInfo,
Granularities.NONE, rowsPerSegment);
- }
-
- final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
- ImmutableMap.of(dataSegment, index),
- Collections.emptyMap(),
- null,
- closer
- );
-
- plannerFactory = sqlSystem.lhs;
- engine = sqlSystem.rhs;
-
- final String sql = QUERIES.get(Integer.parseInt(query));
- final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, "EXPLAIN PLAN FOR " + sql,
ImmutableMap.of("useNativeQueryExplain", true))) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Object[] planResult = resultSequence.toList().get(0);
- log.info("Native query plan:\n" +
- jsonMapper.writerWithDefaultPrettyPrinter()
- .writeValueAsString(jsonMapper.readValue((String)
planResult[0], List.class))
- );
- }
- catch (JsonProcessingException ignored) {
-
- }
-
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Yielder<Object[]> yielder = Yielders.each(resultSequence);
- int rowCounter = 0;
- while (!yielder.isDone()) {
- rowCounter++;
- yielder.next(yielder.get());
- }
- log.info("Total result row count:" + rowCounter);
- }
- catch (Throwable ignored) {
-
- }
- }
-
- private StringEncodingStrategy getStringEncodingStrategy()
- {
- if (stringEncoding.startsWith("front-coded")) {
- String[] split = stringEncoding.split("-");
- int bucketSize = Integer.parseInt(split[2]);
- return new StringEncodingStrategy.FrontCoded(bucketSize,
FrontCodedIndexed.V1);
- } else {
- return new StringEncodingStrategy.Utf8();
- }
- }
-
- public static Pair<PlannerFactory, SqlEngine> createSqlSystem(
- final Map<DataSegment, QueryableIndex> segmentMap,
- final Map<String, LookupExtractor> lookupMap,
- @Nullable final String storageType,
- final Closer closer
- )
- {
- final QueryRunnerFactoryConglomerate conglomerate =
QueryStackTests.createQueryRunnerFactoryConglomerate(closer);
- final SpecificSegmentsQuerySegmentWalker walker =
SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate);
- final PlannerConfig plannerConfig = new PlannerConfig();
-
- for (final Map.Entry<DataSegment, QueryableIndex> segmentEntry :
segmentMap.entrySet()) {
- addSegmentToWalker(walker, segmentEntry.getKey(),
segmentEntry.getValue(), storageType);
- }
-
- // Child injector that adds additional lookups.
- final Injector injector = new StartupInjectorBuilder()
- .withEmptyProperties()
- .add(
- new ExpressionModule(),
- new SegmentWranglerModule(),
- new LookylooModule(),
- new SqlAggregationModule(),
- new CalciteTestOperatorModule(),
- binder -> {
- for (Map.Entry<String, LookupExtractor> entry :
lookupMap.entrySet()) {
- MapBinder.newMapBinder(binder, String.class,
LookupExtractor.class)
- .addBinding(entry.getKey())
- .toProvider(entry::getValue)
- .in(LazySingleton.class);
- }
- }
- )
- .build();
-
- final DruidSchemaCatalog rootSchema =
- QueryFrameworkUtils.createMockRootSchema(
- injector,
- conglomerate,
- walker,
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER
- );
-
- final SqlEngine engine = CalciteTests.createMockSqlEngine(walker,
conglomerate);
-
- final PlannerFactory plannerFactory = new PlannerFactory(
- rootSchema,
- createOperatorTable(injector),
- injector.getInstance(ExprMacroTable.class),
- plannerConfig,
- AuthTestUtils.TEST_AUTHORIZER_MAPPER,
- injector.getInstance(Key.get(ObjectMapper.class, Json.class)),
- CalciteTests.DRUID_SCHEMA_NAME,
- new CalciteRulesManager(ImmutableSet.of()),
- new
JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
- CatalogResolver.NULL_RESOLVER,
- new AuthConfig(),
- new DruidHookDispatcher()
- );
-
- return Pair.of(plannerFactory, engine);
- }
-
- private static void addSegmentToWalker(
- final SpecificSegmentsQuerySegmentWalker walker,
- final DataSegment descriptor,
- final QueryableIndex index,
- @Nullable final String storageType
- )
- {
- if (storageType == null || STORAGE_MMAP.equals(storageType)) {
- walker.add(descriptor, new QueryableIndexSegment(index,
descriptor.getId()));
- } else if (STORAGE_FRAME_ROW.equals(storageType)) {
- walker.add(
- descriptor,
- FrameTestUtil.cursorFactoryToFrameSegment(
- new QueryableIndexCursorFactory(index),
- FrameType.ROW_BASED,
- descriptor.getId()
- )
- );
- } else if (STORAGE_FRAME_COLUMNAR.equals(storageType)) {
- walker.add(
- descriptor,
- FrameTestUtil.cursorFactoryToFrameSegment(
- new QueryableIndexCursorFactory(index),
- FrameType.COLUMNAR,
- descriptor.getId()
- )
- );
- } else {
- throw new IAE("Invalid storageType[%s]", storageType);
- }
- }
-
- private static DruidOperatorTable createOperatorTable(final Injector
injector)
- {
- try {
- final Set<SqlOperatorConversion> extractionOperators = new HashSet<>();
-
extractionOperators.add(injector.getInstance(QueryLookupOperatorConversion.class));
- final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator =
- new ApproxCountDistinctSqlAggregator(new
HllSketchApproxCountDistinctSqlAggregator());
- final Set<SqlAggregator> aggregators = new HashSet<>(
- ImmutableList.of(
- new DoublesSketchApproxQuantileSqlAggregator(),
- new DoublesSketchObjectSqlAggregator(),
- new HllSketchApproxCountDistinctSqlAggregator(),
- new HllSketchApproxCountDistinctUtf8SqlAggregator(),
- new ThetaSketchApproxCountDistinctSqlAggregator(),
- new CountSqlAggregator(countDistinctSqlAggregator),
- countDistinctSqlAggregator
- )
- );
- return new DruidOperatorTable(aggregators, extractionOperators);
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @TearDown(Level.Trial)
- public void tearDown() throws Exception
- {
- closer.close();
- }
-
- @Benchmark
- @BenchmarkMode(Mode.AverageTime)
- @OutputTimeUnit(TimeUnit.MILLISECONDS)
- public void querySql(Blackhole blackhole)
+ @Override
+ public String getQuery()
{
- final Map<String, Object> context = ImmutableMap.of(
- QueryContexts.VECTORIZE_KEY, vectorize,
- QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
- );
- final String sql = QUERIES.get(Integer.parseInt(query));
- try (final DruidPlanner planner =
plannerFactory.createPlannerForTesting(engine, sql, context)) {
- final PlannerResult plannerResult = planner.plan();
- final Sequence<Object[]> resultSequence =
plannerResult.run().getResults();
- final Object[] lastRow = resultSequence.accumulate(null, (accumulated,
in) -> in);
- blackhole.consume(lastRow);
- }
+ return QUERIES.get(Integer.parseInt(query));
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more
details](https://github.com/apache/druid/security/code-scanning/8418)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
