This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 974e1ea01f Cleanup null value handling in data table (#9429)
974e1ea01f is described below
commit 974e1ea01f1d2635e2343473b089048139024bd6
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Mon Sep 19 16:12:21 2022 -0700
Cleanup null value handling in data table (#9429)
---
.../org/apache/pinot/common/utils/DataSchema.java | 69 +++++++------
.../blocks/results/AggregationResultsBlock.java | 14 +--
.../blocks/results/GroupByResultsBlock.java | 9 +-
.../function/CoalesceTransformFunction.java | 82 ++++++++-------
.../pinot/core/query/distinct/DistinctTable.java | 10 +-
.../query/selection/SelectionOperatorUtils.java | 13 +--
.../function/CoalesceTransformFunctionTest.java | 111 ++++++++++-----------
.../org/apache/pinot/spi/utils/NullValueUtils.java | 46 ---------
8 files changed, 157 insertions(+), 197 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
index d8a6fa5cf8..4370e7623a 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -51,9 +51,9 @@ public class DataSchema {
/** Used by both Broker and Server to generate results for EXPLAIN PLAN
queries. */
public static final DataSchema EXPLAIN_RESULT_SCHEMA =
- new DataSchema(new String[]{"Operator", "Operator_Id", "Parent_Id"},
- new DataSchema.ColumnDataType[]{DataSchema.ColumnDataType.STRING,
DataSchema.ColumnDataType.INT,
- DataSchema.ColumnDataType.INT});
+ new DataSchema(new String[]{"Operator", "Operator_Id", "Parent_Id"}, new
ColumnDataType[]{
+ ColumnDataType.STRING, ColumnDataType.INT, ColumnDataType.INT
+ });
@JsonCreator
public DataSchema(@JsonProperty("columnNames") String[] columnNames,
@@ -228,8 +228,8 @@ public class DataSchema {
}
if (anObject instanceof DataSchema) {
DataSchema anotherDataSchema = (DataSchema) anObject;
- return Arrays.equals(_columnNames, anotherDataSchema._columnNames) &&
Arrays
- .equals(_columnDataTypes, anotherDataSchema._columnDataTypes);
+ return Arrays.equals(_columnNames, anotherDataSchema._columnNames) &&
Arrays.equals(_columnDataTypes,
+ anotherDataSchema._columnDataTypes);
}
return false;
}
@@ -240,33 +240,46 @@ public class DataSchema {
}
public enum ColumnDataType {
- INT,
- LONG,
- FLOAT,
- DOUBLE,
- BIG_DECIMAL,
- BOOLEAN /* Stored as INT */,
- TIMESTAMP /* Stored as LONG */,
- STRING,
- JSON /* Stored as STRING */,
- BYTES,
- OBJECT,
- INT_ARRAY,
- LONG_ARRAY,
- FLOAT_ARRAY,
- DOUBLE_ARRAY,
- BOOLEAN_ARRAY /* Stored as INT_ARRAY */,
- TIMESTAMP_ARRAY /* Stored as LONG_ARRAY */,
- STRING_ARRAY,
- BYTES_ARRAY;
+ INT(0),
+ LONG(0L),
+ FLOAT(0f),
+ DOUBLE(0d),
+ BIG_DECIMAL(BigDecimal.ZERO),
+ BOOLEAN(0) /* Stored as INT */,
+ TIMESTAMP(0L) /* Stored as LONG */,
+ STRING(""),
+ JSON("") /* Stored as STRING */,
+ BYTES(new ByteArray(new byte[0])),
+ OBJECT(null),
+ INT_ARRAY(new int[0]),
+ LONG_ARRAY(new long[0]),
+ FLOAT_ARRAY(new float[0]),
+ DOUBLE_ARRAY(new double[0]),
+ BOOLEAN_ARRAY(new int[0]) /* Stored as INT_ARRAY */,
+ TIMESTAMP_ARRAY(new long[0]) /* Stored as LONG_ARRAY */,
+ STRING_ARRAY(new String[0]),
+ BYTES_ARRAY(new byte[0][]);
private static final EnumSet<ColumnDataType> NUMERIC_TYPES =
EnumSet.of(INT, LONG, FLOAT, DOUBLE, BIG_DECIMAL);
private static final EnumSet<ColumnDataType> INTEGRAL_TYPES =
EnumSet.of(INT, LONG);
- private static final EnumSet<ColumnDataType> ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
- DOUBLE_ARRAY, STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
BYTES_ARRAY);
- private static final EnumSet<ColumnDataType> NUMERIC_ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY,
- DOUBLE_ARRAY);
+ private static final EnumSet<ColumnDataType> ARRAY_TYPES =
+ EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY,
STRING_ARRAY, BOOLEAN_ARRAY, TIMESTAMP_ARRAY,
+ BYTES_ARRAY);
+ private static final EnumSet<ColumnDataType> NUMERIC_ARRAY_TYPES =
+ EnumSet.of(INT_ARRAY, LONG_ARRAY, FLOAT_ARRAY, DOUBLE_ARRAY);
private static final EnumSet<ColumnDataType> INTEGRAL_ARRAY_TYPES =
EnumSet.of(INT_ARRAY, LONG_ARRAY);
+
+ // Placeholder for null. We need a placeholder for null so that it can be
serialized in the data table
+ private final Object _nullPlaceholder;
+
+ ColumnDataType(Object nullPlaceHolder) {
+ _nullPlaceholder = nullPlaceHolder;
+ }
+
+ public Object getNullPlaceholder() {
+ return _nullPlaceholder;
+ }
+
/**
* Returns the data type stored in Pinot.
*/
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
index bba0c6df61..02da5baf44 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/AggregationResultsBlock.java
@@ -30,7 +30,6 @@ import
org.apache.pinot.core.common.datatable.DataTableFactory;
import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.spi.utils.ByteArray;
-import org.apache.pinot.spi.utils.NullValueUtils;
import org.roaringbitmap.RoaringBitmap;
@@ -82,18 +81,13 @@ public class AggregationResultsBlock extends
BaseResultsBlock {
dataTableBuilder.startRow();
for (int i = 0; i < numColumns; i++) {
Object result = _results.get(i);
+ if (result == null) {
+ result = columnDataTypes[i].getNullPlaceholder();
+ nullBitmaps[i].add(0);
+ }
if (!returnFinalResult) {
- if (result == null && columnDataTypes[i] != ColumnDataType.OBJECT) {
- result =
NullValueUtils.getDefaultNullValue(columnDataTypes[i].toDataType());
- nullBitmaps[i].add(0);
- }
setIntermediateResult(dataTableBuilder, columnDataTypes, i, result);
} else {
- result = _aggregationFunctions[i].extractFinalResult(result);
- if (result == null) {
- result =
NullValueUtils.getDefaultNullValue(columnDataTypes[i].toDataType());
- nullBitmaps[i].add(0);
- }
setFinalResult(dataTableBuilder, columnDataTypes, i, result);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
index 1d0a7c5089..3e5c534a65 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/blocks/results/GroupByResultsBlock.java
@@ -37,7 +37,6 @@ import org.apache.pinot.core.data.table.Table;
import
org.apache.pinot.core.query.aggregation.groupby.AggregationGroupByResult;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.spi.utils.ByteArray;
-import org.apache.pinot.spi.utils.NullValueUtils;
import org.roaringbitmap.RoaringBitmap;
@@ -134,12 +133,10 @@ public class GroupByResultsBlock extends BaseResultsBlock
{
Iterator<Record> iterator = _table.iterator();
if (queryContext.isNullHandlingEnabled()) {
RoaringBitmap[] nullBitmaps = new RoaringBitmap[numColumns];
- Object[] defaultNullValues = new Object[numColumns];
+ Object[] nullPlaceholders = new Object[numColumns];
for (int colId = 0; colId < numColumns; colId++) {
nullBitmaps[colId] = new RoaringBitmap();
- if (storedColumnDataTypes[colId] != ColumnDataType.OBJECT) {
- defaultNullValues[colId] =
NullValueUtils.getDefaultNullValue(storedColumnDataTypes[colId].toDataType());
- }
+ nullPlaceholders[colId] =
storedColumnDataTypes[colId].getNullPlaceholder();
}
int rowId = 0;
while (iterator.hasNext()) {
@@ -148,7 +145,7 @@ public class GroupByResultsBlock extends BaseResultsBlock {
for (int colId = 0; colId < numColumns; colId++) {
Object value = values[colId];
if (value == null && storedColumnDataTypes[colId] !=
ColumnDataType.OBJECT) {
- value = defaultNullValues[colId];
+ value = nullPlaceholders[colId];
nullBitmaps[colId].add(rowId);
}
setDataTableColumn(storedColumnDataTypes[colId], dataTableBuilder,
colId, value);
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunction.java
index dfc45a8036..0019c49305 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunction.java
@@ -26,8 +26,7 @@ import org.apache.pinot.common.function.TransformFunctionType;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.segment.spi.datasource.DataSource;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.utils.NullValueUtils;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.roaringbitmap.RoaringBitmap;
@@ -49,8 +48,15 @@ import org.roaringbitmap.RoaringBitmap;
* Coalesce(columnA, columnB)
*/
public class CoalesceTransformFunction extends BaseTransformFunction {
+ public static final int NULL_INT = Integer.MIN_VALUE;
+ public static final long NULL_LONG = Long.MIN_VALUE;
+ public static final float NULL_FLOAT = Float.NEGATIVE_INFINITY;
+ public static final double NULL_DOUBLE = Double.NEGATIVE_INFINITY;
+ public static final BigDecimal NULL_BIG_DECIMAL =
BigDecimal.valueOf(Long.MIN_VALUE);
+ public static final String NULL_STRING = "null";
+
private TransformFunction[] _transformFunctions;
- private FieldSpec.DataType _storedType;
+ private DataType _dataType;
private TransformResultMetadata _resultMetadata;
/**
@@ -96,7 +102,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (int) NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_INT;
}
}
return results;
@@ -129,7 +135,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (long) NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_LONG;
}
}
return results;
@@ -162,7 +168,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (float) NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_FLOAT;
}
}
return results;
@@ -172,7 +178,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
* Get transform double results based on store type.
* @param projectionBlock
*/
- private double[] getDoublelTransformResults(ProjectionBlock projectionBlock)
{
+ private double[] getDoubleTransformResults(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
double[] results = new double[length];
int width = _transformFunctions.length;
@@ -195,7 +201,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (double) NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_DOUBLE;
}
}
return results;
@@ -228,13 +234,12 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (BigDecimal)
NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_BIG_DECIMAL;
}
}
return results;
}
-
/**
* Get transform String results based on store type.
* @param projectionBlock
@@ -262,7 +267,7 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
break;
}
if (!hasNonNullValue) {
- results[i] = (String) NullValueUtils.getDefaultNullValue(_storedType);
+ results[i] = NULL_STRING;
}
}
return results;
@@ -282,15 +287,16 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
TransformFunction func = arguments.get(i);
Preconditions.checkArgument(func instanceof IdentifierTransformFunction,
"Only column names are supported in COALESCE.");
- FieldSpec.DataType storedType =
func.getResultMetadata().getDataType().getStoredType();
- if (_storedType == null) {
- _storedType = storedType;
+ DataType dataType = func.getResultMetadata().getDataType();
+ if (_dataType == null) {
+ _dataType = dataType;
} else {
- Preconditions.checkArgument(storedType.equals(_storedType), "Argument
types have to be the same.");
+ Preconditions.checkArgument(dataType == _dataType, "Argument types
have to be the same.");
}
_transformFunctions[i] = func;
}
- switch (_storedType) {
+ // TODO: Support BOOLEAN, TIMESTAMP, BYTES
+ switch (_dataType) {
case INT:
_resultMetadata = INT_SV_NO_DICTIONARY_METADATA;
break;
@@ -319,17 +325,9 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
return _resultMetadata;
}
- @Override
- public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
- if (_storedType != FieldSpec.DataType.STRING) {
- return super.transformToStringValuesSV(projectionBlock);
- }
- return getStringTransformResults(projectionBlock);
- }
-
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
- if (_storedType != FieldSpec.DataType.INT) {
+ if (_dataType != DataType.INT) {
return super.transformToIntValuesSV(projectionBlock);
}
return getIntTransformResults(projectionBlock);
@@ -337,33 +335,45 @@ public class CoalesceTransformFunction extends
BaseTransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
- if (_storedType != FieldSpec.DataType.LONG) {
+ if (_dataType != DataType.LONG) {
return super.transformToLongValuesSV(projectionBlock);
}
return getLongTransformResults(projectionBlock);
}
@Override
- public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock
projectionBlock) {
- if (_storedType != FieldSpec.DataType.BIG_DECIMAL) {
- return super.transformToBigDecimalValuesSV(projectionBlock);
+ public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
+ if (_dataType != DataType.FLOAT) {
+ return super.transformToFloatValuesSV(projectionBlock);
}
- return getBigDecimalTransformResults(projectionBlock);
+ return getFloatTransformResults(projectionBlock);
}
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- if (_storedType != FieldSpec.DataType.DOUBLE) {
+ if (_dataType != DataType.DOUBLE) {
return super.transformToDoubleValuesSV(projectionBlock);
}
- return getDoublelTransformResults(projectionBlock);
+ return getDoubleTransformResults(projectionBlock);
}
@Override
- public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
- if (_storedType != FieldSpec.DataType.FLOAT) {
- return super.transformToFloatValuesSV(projectionBlock);
+ public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock
projectionBlock) {
+ if (_dataType != DataType.BIG_DECIMAL) {
+ return super.transformToBigDecimalValuesSV(projectionBlock);
}
- return getFloatTransformResults(projectionBlock);
+ return getBigDecimalTransformResults(projectionBlock);
+ }
+
+ @Override
+ public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
+ if (_dataType != DataType.STRING) {
+ return super.transformToStringValuesSV(projectionBlock);
+ }
+ return getStringTransformResults(projectionBlock);
+ }
+
+ public static void main(String[] args) {
+ System.out.println(BigDecimal.valueOf(Long.MIN_VALUE));
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/DistinctTable.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/DistinctTable.java
index 49e85a5a2d..8a8dddf308 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/DistinctTable.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/distinct/DistinctTable.java
@@ -40,7 +40,6 @@ import
org.apache.pinot.core.common.datatable.DataTableBuilder;
import org.apache.pinot.core.common.datatable.DataTableFactory;
import org.apache.pinot.core.data.table.Record;
import org.apache.pinot.spi.utils.ByteArray;
-import org.apache.pinot.spi.utils.NullValueUtils;
import org.roaringbitmap.RoaringBitmap;
@@ -277,16 +276,15 @@ public class DistinctTable {
public byte[] toBytes()
throws IOException {
// NOTE: Serialize the DistinctTable as a DataTable
- DataTableBuilder dataTableBuilder = DataTableFactory.getDataTableBuilder(
- _dataSchema);
+ DataTableBuilder dataTableBuilder =
DataTableFactory.getDataTableBuilder(_dataSchema);
ColumnDataType[] storedColumnDataTypes =
_dataSchema.getStoredColumnDataTypes();
int numColumns = storedColumnDataTypes.length;
RoaringBitmap[] nullBitmaps = null;
if (_nullHandlingEnabled) {
nullBitmaps = new RoaringBitmap[numColumns];
- Object[] colDefaultNullValues = new Object[numColumns];
+ Object[] nullPlaceholders = new Object[numColumns];
for (int colId = 0; colId < numColumns; colId++) {
- colDefaultNullValues[colId] =
NullValueUtils.getDefaultNullValue(storedColumnDataTypes[colId].toDataType());
+ nullPlaceholders[colId] =
storedColumnDataTypes[colId].getNullPlaceholder();
nullBitmaps[colId] = new RoaringBitmap();
}
@@ -295,7 +293,7 @@ public class DistinctTable {
Object[] values = record.getValues();
for (int colId = 0; colId < numColumns; colId++) {
if (values[colId] == null) {
- values[colId] = colDefaultNullValues[colId];
+ values[colId] = nullPlaceholders[colId];
nullBitmaps[colId].add(rowId);
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
index 851608330d..e6536d7fe4 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/query/selection/SelectionOperatorUtils.java
@@ -43,7 +43,6 @@ import
org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.spi.utils.ArrayCopyUtils;
import org.apache.pinot.spi.utils.ByteArray;
-import org.apache.pinot.spi.utils.NullValueUtils;
import org.roaringbitmap.RoaringBitmap;
@@ -244,21 +243,17 @@ public class SelectionOperatorUtils {
RoaringBitmap[] nullBitmaps = null;
if (nullHandlingEnabled) {
nullBitmaps = new RoaringBitmap[numColumns];
- Object[] colDefaultNullValues = new Object[numColumns];
+ Object[] nullPlaceholders = new Object[numColumns];
for (int colId = 0; colId < numColumns; colId++) {
- if (storedColumnDataTypes[colId] != ColumnDataType.OBJECT &&
!storedColumnDataTypes[colId].isArray()) {
- colDefaultNullValues[colId] =
-
NullValueUtils.getDefaultNullValue(storedColumnDataTypes[colId].toDataType());
- }
nullBitmaps[colId] = new RoaringBitmap();
+ nullPlaceholders[colId] =
storedColumnDataTypes[colId].getNullPlaceholder();
}
-
int rowId = 0;
for (Object[] row : rows) {
for (int i = 0; i < numColumns; i++) {
Object columnValue = row[i];
if (columnValue == null) {
- row[i] = colDefaultNullValues[i];
+ row[i] = nullPlaceholders[i];
nullBitmaps[i].add(rowId);
}
}
@@ -434,7 +429,7 @@ public class SelectionOperatorUtils {
int numColumns = dataTable.getDataSchema().size();
int numRows = dataTable.getNumberOfRows();
if (nullHandlingEnabled) {
- RoaringBitmap[] nullBitmaps = new RoaringBitmap[numColumns];;
+ RoaringBitmap[] nullBitmaps = new RoaringBitmap[numColumns];
for (int coldId = 0; coldId < numColumns; coldId++) {
nullBitmaps[coldId] = dataTable.getNullRowIds(coldId);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunctionTest.java
index 8f7584388e..00569eca15 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/CoalesceTransformFunctionTest.java
@@ -45,7 +45,6 @@ import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.data.readers.GenericRow;
-import org.apache.pinot.spi.utils.NullValueUtils;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.Assert;
@@ -267,7 +266,7 @@ public class CoalesceTransformFunctionTest extends
BaseTransformFunctionTest {
int[] expectedResults = new int[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (int)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.INT);
+ expectedResults[i] = CoalesceTransformFunction.NULL_INT;
} else if (isColumn1Null(i)) {
expectedResults[i] = _intSVValues[i] + INT_VALUE_SHIFT;
} else if (isColumn2Null(i)) {
@@ -280,126 +279,126 @@ public class CoalesceTransformFunctionTest extends
BaseTransformFunctionTest {
testIntTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
}
- // Test the Coalesce on two String columns where one or the other or both
can be null.
+ // Test the Coalesce on two long columns where one or the other or both can
be null.
@Test
- public void testCoalesceStringColumns()
+ public void testCoalesceLongColumns()
throws Exception {
ExpressionContext coalesceExpr =
- RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
STRING_SV_COLUMN1, STRING_SV_COLUMN2));
+ RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
LONG_SV_COLUMN1, LONG_SV_COLUMN2));
TransformFunction coalesceTransformFunction =
TransformFunctionFactory.get(coalesceExpr, _enableNullDataSourceMap);
Assert.assertEquals(coalesceTransformFunction.getName(), "coalesce");
- String[] expectedResults = new String[NUM_ROWS];
+ long[] expectedResults = new long[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (String)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.STRING);
+ expectedResults[i] = CoalesceTransformFunction.NULL_LONG;
} else if (isColumn1Null(i)) {
- expectedResults[i] = _stringSVValues[i] + SUFFIX;
+ expectedResults[i] = _intSVValues[i] + INT_VALUE_SHIFT;
} else if (isColumn2Null(i)) {
- expectedResults[i] = _stringSVValues[i];
+ expectedResults[i] = _intSVValues[i];
} else {
- expectedResults[i] = _stringSVValues[i];
+ expectedResults[i] = _intSVValues[i];
}
}
- testStringTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
- testStringTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
+ testLongTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
+ testLongTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
}
- // Test the Coalesce on two big decimal columns where one or the other or
both can be null.
+ // Test the Coalesce on two float columns where one or the other or both can
be null.
@Test
- public void testCoalesceBigDecimalColumns()
+ public void testCoalesceFloatColumns()
throws Exception {
- ExpressionContext coalesceExpr = RequestContextUtils.getExpression(
- String.format("COALESCE(%s,%s)", BIG_DECIMAL_SV_COLUMN1,
BIG_DECIMAL_SV_COLUMN2));
+ ExpressionContext coalesceExpr =
+ RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
FLOAT_SV_COLUMN1, FLOAT_SV_COLUMN2));
TransformFunction coalesceTransformFunction =
TransformFunctionFactory.get(coalesceExpr, _enableNullDataSourceMap);
Assert.assertEquals(coalesceTransformFunction.getName(), "coalesce");
- BigDecimal[] expectedResults = new BigDecimal[NUM_ROWS];
+ float[] expectedResults = new float[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (BigDecimal)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.BIG_DECIMAL);
+ expectedResults[i] = CoalesceTransformFunction.NULL_FLOAT;
} else if (isColumn1Null(i)) {
- expectedResults[i] = BigDecimal.valueOf(_intSVValues[i] +
INT_VALUE_SHIFT);
+ expectedResults[i] = _floatValues[i] + FLOAT_VALUE_SHIFT;
} else if (isColumn2Null(i)) {
- expectedResults[i] = BigDecimal.valueOf(_intSVValues[i]);
+ expectedResults[i] = _floatValues[i];
} else {
- expectedResults[i] = BigDecimal.valueOf(_intSVValues[i]);
+ expectedResults[i] = _floatValues[i];
}
}
- testBigDecimalTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock,
- _enableNullDataSourceMap);
- testBigDecimalTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock,
- _disableNullDataSourceMap);
+ testFloatTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
+ testFloatTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
}
- // Test the Coalesce on two long columns where one or the other or both can
be null.
+ // Test the Coalesce on two double columns where one or the other or both
can be null.
@Test
- public void testCoalesceLongColumns()
+ public void testCoalesceDoubleColumns()
throws Exception {
ExpressionContext coalesceExpr =
- RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
LONG_SV_COLUMN1, LONG_SV_COLUMN2));
+ RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
DOUBLE_SV_COLUMN1, DOUBLE_SV_COLUMN2));
TransformFunction coalesceTransformFunction =
TransformFunctionFactory.get(coalesceExpr, _enableNullDataSourceMap);
Assert.assertEquals(coalesceTransformFunction.getName(), "coalesce");
- long[] expectedResults = new long[NUM_ROWS];
+ double[] expectedResults = new double[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (long)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.LONG);
+ expectedResults[i] = CoalesceTransformFunction.NULL_DOUBLE;
} else if (isColumn1Null(i)) {
- expectedResults[i] = _intSVValues[i] + INT_VALUE_SHIFT;
+ expectedResults[i] = _doubleValues[i] + DOUBLE_VALUE_SHIFT;
} else if (isColumn2Null(i)) {
- expectedResults[i] = _intSVValues[i];
+ expectedResults[i] = _doubleValues[i];
} else {
- expectedResults[i] = _intSVValues[i];
+ expectedResults[i] = _doubleValues[i];
}
}
- testLongTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
- testLongTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
+ testDoubleTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
+ testDoubleTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
}
- // Test the Coalesce on two double columns where one or the other or both
can be null.
+ // Test the Coalesce on two big decimal columns where one or the other or
both can be null.
@Test
- public void testCoalesceDoubleColumns()
+ public void testCoalesceBigDecimalColumns()
throws Exception {
- ExpressionContext coalesceExpr =
- RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
DOUBLE_SV_COLUMN1, DOUBLE_SV_COLUMN2));
+ ExpressionContext coalesceExpr = RequestContextUtils.getExpression(
+ String.format("COALESCE(%s,%s)", BIG_DECIMAL_SV_COLUMN1,
BIG_DECIMAL_SV_COLUMN2));
TransformFunction coalesceTransformFunction =
TransformFunctionFactory.get(coalesceExpr, _enableNullDataSourceMap);
Assert.assertEquals(coalesceTransformFunction.getName(), "coalesce");
- double[] expectedResults = new double[NUM_ROWS];
+ BigDecimal[] expectedResults = new BigDecimal[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (double)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.DOUBLE);
+ expectedResults[i] = CoalesceTransformFunction.NULL_BIG_DECIMAL;
} else if (isColumn1Null(i)) {
- expectedResults[i] = _doubleValues[i] + DOUBLE_VALUE_SHIFT;
+ expectedResults[i] = BigDecimal.valueOf(_intSVValues[i] +
INT_VALUE_SHIFT);
} else if (isColumn2Null(i)) {
- expectedResults[i] = _doubleValues[i];
+ expectedResults[i] = BigDecimal.valueOf(_intSVValues[i]);
} else {
- expectedResults[i] = _doubleValues[i];
+ expectedResults[i] = BigDecimal.valueOf(_intSVValues[i]);
}
}
- testDoubleTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
- testDoubleTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
+ testBigDecimalTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock,
+ _enableNullDataSourceMap);
+ testBigDecimalTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock,
+ _disableNullDataSourceMap);
}
- // Test the Coalesce on two float columns where one or the other or both can
be null.
+ // Test the Coalesce on two String columns where one or the other or both
can be null.
@Test
- public void testCoalesceFloatColumns()
+ public void testCoalesceStringColumns()
throws Exception {
ExpressionContext coalesceExpr =
- RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
FLOAT_SV_COLUMN1, FLOAT_SV_COLUMN2));
+ RequestContextUtils.getExpression(String.format("COALESCE(%s,%s)",
STRING_SV_COLUMN1, STRING_SV_COLUMN2));
TransformFunction coalesceTransformFunction =
TransformFunctionFactory.get(coalesceExpr, _enableNullDataSourceMap);
Assert.assertEquals(coalesceTransformFunction.getName(), "coalesce");
- float[] expectedResults = new float[NUM_ROWS];
+ String[] expectedResults = new String[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
if (isColumn1Null(i) && isColumn2Null(i)) {
- expectedResults[i] = (float)
NullValueUtils.getDefaultNullValue(FieldSpec.DataType.FLOAT);
+ expectedResults[i] = CoalesceTransformFunction.NULL_STRING;
} else if (isColumn1Null(i)) {
- expectedResults[i] = _floatValues[i] + FLOAT_VALUE_SHIFT;
+ expectedResults[i] = _stringSVValues[i] + SUFFIX;
} else if (isColumn2Null(i)) {
- expectedResults[i] = _floatValues[i];
+ expectedResults[i] = _stringSVValues[i];
} else {
- expectedResults[i] = _floatValues[i];
+ expectedResults[i] = _stringSVValues[i];
}
}
- testFloatTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
- testFloatTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
+ testStringTransformFunction(coalesceExpr, expectedResults,
_enableNullProjectionBlock, _enableNullDataSourceMap);
+ testStringTransformFunction(coalesceExpr, expectedResults,
_disableNullProjectionBlock, _disableNullDataSourceMap);
}
// Test that non-column-names appear in one of the argument.
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/NullValueUtils.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/NullValueUtils.java
deleted file mode 100644
index 09b4531e59..0000000000
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/NullValueUtils.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.spi.utils;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.pinot.spi.data.FieldSpec;
-import org.apache.pinot.spi.data.FieldSpec.DataType;
-
-
-public class NullValueUtils {
- private NullValueUtils() {
- }
-
- private static final Map<DataType, Object> FIELD_TYPE_DEFAULT_NULL_VALUE_MAP
= new HashMap<>();
-
- static {
- DataType[] types = {DataType.INT, DataType.LONG, DataType.FLOAT,
DataType.DOUBLE, DataType.BIG_DECIMAL,
- DataType.BOOLEAN, DataType.TIMESTAMP, DataType.STRING, DataType.JSON,
DataType.BYTES};
- for (DataType type : types) {
- FieldSpec.FieldType fieldType =
type.equals(FieldSpec.DataType.BIG_DECIMAL)
- ? FieldSpec.FieldType.METRIC : FieldSpec.FieldType.DIMENSION;
- FIELD_TYPE_DEFAULT_NULL_VALUE_MAP.put(type,
FieldSpec.getDefaultNullValue(fieldType, type, null));
- }
- }
-
- public static Object getDefaultNullValue(FieldSpec.DataType dataType) {
- return FIELD_TYPE_DEFAULT_NULL_VALUE_MAP.get(dataType);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
