This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 272a19992d Adding query function override for Aggregate functions of
multi valued column (#11307)
272a19992d is described below
commit 272a19992d561cff939fc6a3fe154a4456a2361c
Author: Eaugene Thomas <[email protected]>
AuthorDate: Mon Sep 25 06:52:41 2023 +0530
Adding query function override for Aggregate functions of multi valued
column (#11307)
---
.../requesthandler/BaseBrokerRequestHandler.java | 76 +++++++++++++++++++++-
.../broker/requesthandler/QueryOverrideTest.java | 20 ++++++
2 files changed, 95 insertions(+), 1 deletion(-)
diff --git
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
index f0b30544f4..420dc15eb2 100644
---
a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
+++
b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/BaseBrokerRequestHandler.java
@@ -21,6 +21,7 @@ package org.apache.pinot.broker.requesthandler;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
@@ -86,6 +87,7 @@ import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.QueryConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.exception.BadQueryRequestException;
@@ -113,6 +115,14 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
private static final Expression TRUE =
RequestUtils.getLiteralExpression(true);
private static final Expression STAR =
RequestUtils.getIdentifierExpression("*");
private static final int
MAX_UNAVAILABLE_SEGMENTS_TO_PRINT_IN_QUERY_EXCEPTION = 10;
+ private static final Map<String, String>
DISTINCT_MV_COL_FUNCTION_OVERRIDE_MAP =
+ ImmutableMap.<String, String>builder().put("distinctcount",
"distinctcountmv")
+ .put("distinctcountbitmap",
"distinctcountbitmapmv").put("distinctcounthll", "distinctcounthllmv")
+ .put("distinctcountrawhll",
"distinctcountrawhllmv").put("distinctsum", "distinctsummv")
+ .put("distinctavg", "distinctavgmv").put("count",
"countmv").put("min", "minmv").put("max", "maxmv")
+ .put("avg", "avgmv").put("sum", "summv").put("minmaxrange",
"minmaxrangemv")
+ .put("distinctcounthllplus", "distinctcounthllplusmv")
+ .put("distinctcountrawhllplus", "distinctcountrawhllplusmv").build();
protected final PinotConfiguration _config;
protected final BrokerRoutingManager _routingManager;
@@ -380,6 +390,11 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
handleDistinctCountBitmapOverride(serverPinotQuery);
}
+ Schema schema = _tableCache.getSchema(rawTableName);
+ if (schema != null) {
+ handleDistinctMultiValuedOverride(serverPinotQuery, schema);
+ }
+
long compilationEndTimeNs = System.nanoTime();
// full request compile time = compilationTimeNs + parserTimeNs
_brokerMetrics.addPhaseTiming(rawTableName,
BrokerQueryPhase.REQUEST_COMPILATION,
@@ -489,7 +504,6 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
BrokerRequest offlineBrokerRequest = null;
BrokerRequest realtimeBrokerRequest = null;
TimeBoundaryInfo timeBoundaryInfo = null;
- Schema schema = _tableCache.getSchema(rawTableName);
if (offlineTableName != null && realtimeTableName != null) {
// Time boundary info might be null when there is no segment in the
offline table, query real-time side only
timeBoundaryInfo =
_routingManager.getTimeBoundaryInfo(offlineTableName);
@@ -940,6 +954,20 @@ public abstract class BaseBrokerRequestHandler implements
BrokerRequestHandler {
return segmentPartitionedColumns;
}
+ /**
+ * Retrieve multivalued columns for a table.
+ * From the table Schema , we get the multi valued columns of dimension
fields.
+ *
+ * @param tableSchema
+ * @param columnName
+ * @return multivalued columns of the table .
+ */
+ private static boolean isMultiValueColumn(Schema tableSchema, String
columnName) {
+
+ DimensionFieldSpec dimensionFieldSpec =
tableSchema.getDimensionSpec(columnName);
+ return dimensionFieldSpec != null &&
!dimensionFieldSpec.isSingleValueField();
+ }
+
/**
* Sets the table name in the given broker request.
* NOTE: Set table name in broker request because it is used for access
control, query routing etc.
@@ -1073,6 +1101,52 @@ public abstract class BaseBrokerRequestHandler
implements BrokerRequestHandler {
}
}
+ /**
+ * Rewrites selected 'Distinct' prefixed function to 'Distinct----MV'
function for the field of multivalued type.
+ */
+ @VisibleForTesting
+ static void handleDistinctMultiValuedOverride(PinotQuery pinotQuery, Schema
tableSchema) {
+ for (Expression expression : pinotQuery.getSelectList()) {
+ handleDistinctMultiValuedOverride(expression, tableSchema);
+ }
+ List<Expression> orderByExpressions = pinotQuery.getOrderByList();
+ if (orderByExpressions != null) {
+ for (Expression expression : orderByExpressions) {
+ // NOTE: Order-by is always a Function with the ordering of the
Expression
+
handleDistinctMultiValuedOverride(expression.getFunctionCall().getOperands().get(0),
tableSchema);
+ }
+ }
+ Expression havingExpression = pinotQuery.getHavingExpression();
+ if (havingExpression != null) {
+ handleDistinctMultiValuedOverride(havingExpression, tableSchema);
+ }
+ }
+
+ /**
+ * Rewrites selected 'Distinct' prefixed function to 'Distinct----MV'
function for the field of multivalued type.
+ */
+ private static void handleDistinctMultiValuedOverride(Expression expression,
Schema tableSchema) {
+ Function function = expression.getFunctionCall();
+ if (function == null) {
+ return;
+ }
+
+ String overrideOperator =
DISTINCT_MV_COL_FUNCTION_OVERRIDE_MAP.get(function.getOperator());
+ if (overrideOperator != null) {
+ List<Expression> operands = function.getOperands();
+ if (operands.size() >= 1 && operands.get(0).isSetIdentifier() &&
isMultiValueColumn(tableSchema,
+ operands.get(0).getIdentifier().getName())) {
+ // we are only checking the first operand that if its a MV column as
all the overriding agg. fn.'s have
+ // first operator is column name
+ function.setOperator(overrideOperator);
+ }
+ } else {
+ for (Expression operand : function.getOperands()) {
+ handleDistinctMultiValuedOverride(operand, tableSchema);
+ }
+ }
+ }
+
/**
* Rewrites 'DistinctCount' to 'DistinctCountBitmap' for the given
expression.
*/
diff --git
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryOverrideTest.java
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryOverrideTest.java
index 8ed7ceb3b0..df9e07b446 100644
---
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryOverrideTest.java
+++
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/QueryOverrideTest.java
@@ -19,9 +19,11 @@
package org.apache.pinot.broker.requesthandler;
import com.google.common.collect.ImmutableSet;
+import java.io.IOException;
import java.util.Arrays;
import org.apache.pinot.common.request.PinotQuery;
import org.apache.pinot.common.utils.request.RequestUtils;
+import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.testng.annotations.Test;
@@ -63,6 +65,24 @@ public class QueryOverrideTest {
assertEquals(pinotQuery.getSelectList().get(0).getFunctionCall().getOperator(),
"segmentpartitioneddistinctcount");
}
+ @Test
+ public void testDistinctMultiValuedOverride()
+ throws IOException {
+ String query1 = "SELECT DISTINCT_COUNT(col1) FROM myTable";
+ PinotQuery pinotQuery1 = CalciteSqlParser.compileToPinotQuery(query1);
+ String query2 = "SELECT DISTINCT_COUNT(col2) FROM myTable";
+ PinotQuery pinotQuery2 = CalciteSqlParser.compileToPinotQuery(query2);
+ Schema tableSchema = Schema.fromString("{\"schemaName\":\"testSchema\","
+ + "\"dimensionFieldSpecs\":[
{\"name\":\"col2\",\"dataType\":\"LONG\",\"singleValueField\":\"false\"},"
+ +
"{\"name\":\"col3\",\"dataType\":\"LONG\",\"singleValueField\":\"false\"}],"
+ +
"\"dateTimeFieldSpecs\":[{\"name\":\"dt1\",\"dataType\":\"INT\",\"format\":\"x:HOURS:EPOCH\","
+ + "\"granularity\":\"1:HOURS\"}]}");
+ BaseBrokerRequestHandler.handleDistinctMultiValuedOverride(pinotQuery1,
tableSchema);
+
assertEquals(pinotQuery1.getSelectList().get(0).getFunctionCall().getOperator(),
"distinctcount");
+ BaseBrokerRequestHandler.handleDistinctMultiValuedOverride(pinotQuery2,
tableSchema);
+
assertEquals(pinotQuery2.getSelectList().get(0).getFunctionCall().getOperator(),
"distinctcountmv");
+ }
+
@Test
public void testApproximateFunctionOverride() {
{
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
