This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 74fd91aa18 Fix transformation to string for BOOLEAN and TIMESTAMP
(#9287)
74fd91aa18 is described below
commit 74fd91aa185ca4f26033174781567bd60b84cddf
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Aug 30 15:05:14 2022 -0700
Fix transformation to string for BOOLEAN and TIMESTAMP (#9287)
---
.../org/apache/pinot/core/common/DataFetcher.java | 126 +++++++++--
.../transform/function/BaseTransformFunction.java | 114 +++++++---
.../transform/function/CastTransformFunction.java | 249 ++-------------------
.../function/BaseTransformFunctionTest.java | 13 +-
.../NullHandlingTransformFunctionTest.java | 42 ++--
5 files changed, 228 insertions(+), 316 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
index b075e99d0b..1fc1fa2c05 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/common/DataFetcher.java
@@ -21,6 +21,7 @@ package org.apache.pinot.core.common;
import java.io.Closeable;
import java.io.IOException;
import java.math.BigDecimal;
+import java.sql.Timestamp;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -32,7 +33,7 @@ import
org.apache.pinot.segment.spi.evaluator.TransformEvaluator;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReader;
import org.apache.pinot.segment.spi.index.reader.ForwardIndexReaderContext;
-import org.apache.pinot.spi.data.FieldSpec;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.trace.Tracing;
import org.apache.pinot.spi.utils.BytesUtils;
@@ -66,10 +67,11 @@ public class DataFetcher {
for (Map.Entry<String, DataSource> entry : dataSourceMap.entrySet()) {
String column = entry.getKey();
DataSource dataSource = entry.getValue();
+ DataSourceMetadata dataSourceMetadata =
dataSource.getDataSourceMetadata();
ColumnValueReader columnValueReader =
- new ColumnValueReader(dataSource.getForwardIndex(),
dataSource.getDictionary());
+ new ColumnValueReader(dataSource.getForwardIndex(),
dataSource.getDictionary(),
+ dataSourceMetadata.getDataType());
_columnValueReaderMap.put(column, columnValueReader);
- DataSourceMetadata dataSourceMetadata =
dataSource.getDataSourceMetadata();
if (!dataSourceMetadata.isSingleValue()) {
maxNumValuesPerMVEntry = Math.max(maxNumValuesPerMVEntry,
dataSourceMetadata.getMaxNumValuesPerMVEntry());
}
@@ -427,16 +429,16 @@ public class DataFetcher {
private class ColumnValueReader implements Closeable {
final ForwardIndexReader _reader;
final Dictionary _dictionary;
- final FieldSpec.DataType _dataType;
+ final DataType _dataType;
final boolean _singleValue;
boolean _readerContextCreated;
ForwardIndexReaderContext _readerContext;
- ColumnValueReader(ForwardIndexReader reader, @Nullable Dictionary
dictionary) {
+ ColumnValueReader(ForwardIndexReader reader, @Nullable Dictionary
dictionary, DataType dataType) {
_reader = reader;
_dictionary = dictionary;
- _dataType = reader.getStoredType();
+ _dataType = dataType;
_singleValue = reader.isSingleValue();
}
@@ -550,9 +552,19 @@ public class DataFetcher {
if (_dictionary != null) {
int[] dictIdBuffer = THREAD_LOCAL_DICT_IDS.get();
_reader.readDictIds(docIds, length, dictIdBuffer, readerContext);
- _dictionary.readStringValues(dictIdBuffer, length, valueBuffer);
+ if (_dataType == DataType.BOOLEAN) {
+ for (int i = 0; i < length; i++) {
+ valueBuffer[i] =
Boolean.toString(_dictionary.getIntValue(dictIdBuffer[i]) == 1);
+ }
+ } else if (_dataType == DataType.TIMESTAMP) {
+ for (int i = 0; i < length; i++) {
+ valueBuffer[i] = new
Timestamp(_dictionary.getLongValue(dictIdBuffer[i])).toString();
+ }
+ } else {
+ _dictionary.readStringValues(dictIdBuffer, length, valueBuffer);
+ }
} else {
- switch (_reader.getStoredType()) {
+ switch (_dataType) {
case INT:
for (int i = 0; i < length; i++) {
valueBuffer[i] = Integer.toString(_reader.getInt(docIds[i],
readerContext));
@@ -573,7 +585,23 @@ public class DataFetcher {
valueBuffer[i] = Double.toString(_reader.getDouble(docIds[i],
readerContext));
}
break;
+ case BIG_DECIMAL:
+ for (int i = 0; i < length; i++) {
+ valueBuffer[i] = _reader.getBigDecimal(docIds[i],
readerContext).toPlainString();
+ }
+ break;
+ case BOOLEAN:
+ for (int i = 0; i < length; i++) {
+ valueBuffer[i] = Boolean.toString(_reader.getInt(docIds[i],
readerContext) == 1);
+ }
+ break;
+ case TIMESTAMP:
+ for (int i = 0; i < length; i++) {
+ valueBuffer[i] = new Timestamp(_reader.getLong(docIds[i],
readerContext)).toString();
+ }
+ break;
case STRING:
+ case JSON:
for (int i = 0; i < length; i++) {
valueBuffer[i] = _reader.getString(docIds[i], readerContext);
}
@@ -622,23 +650,25 @@ public class DataFetcher {
void readDictIdsMV(int[] docIds, int length, int[][] dictIdsBuffer) {
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
dictIdsBuffer[i] = Arrays.copyOfRange(_reusableMVDictIds, 0,
numValues);
}
}
void readIntValuesMV(int[] docIds, int length, int[][] valuesBuffer) {
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
if (_dictionary != null) {
for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
int[] values = new int[numValues];
_dictionary.readIntValues(_reusableMVDictIds, numValues, values);
valuesBuffer[i] = values;
}
} else {
- _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, getReaderContext());
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, readerContext);
}
}
@@ -650,15 +680,16 @@ public class DataFetcher {
void readLongValuesMV(int[] docIds, int length, long[][] valuesBuffer) {
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
if (_dictionary != null) {
for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
long[] values = new long[numValues];
_dictionary.readLongValues(_reusableMVDictIds, numValues, values);
valuesBuffer[i] = values;
}
} else {
- _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, getReaderContext());
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, readerContext);
}
}
@@ -670,15 +701,16 @@ public class DataFetcher {
void readFloatValuesMV(int[] docIds, int length, float[][] valuesBuffer) {
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
if (_dictionary != null) {
for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
float[] values = new float[numValues];
_dictionary.readFloatValues(_reusableMVDictIds, numValues, values);
valuesBuffer[i] = values;
}
} else {
- _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, getReaderContext());
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, readerContext);
}
}
@@ -690,15 +722,16 @@ public class DataFetcher {
void readDoubleValuesMV(int[] docIds, int length, double[][] valuesBuffer)
{
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
if (_dictionary != null) {
for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
double[] values = new double[numValues];
_dictionary.readDoubleValues(_reusableMVDictIds, numValues, values);
valuesBuffer[i] = values;
}
} else {
- _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, getReaderContext());
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, readerContext);
}
}
@@ -710,15 +743,62 @@ public class DataFetcher {
void readStringValuesMV(int[] docIds, int length, String[][] valuesBuffer)
{
Tracing.activeRecording().setInputDataType(_dataType, _singleValue);
+ ForwardIndexReaderContext readerContext = getReaderContext();
if (_dictionary != null) {
- for (int i = 0; i < length; i++) {
- int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
getReaderContext());
- String[] values = new String[numValues];
- _dictionary.readStringValues(_reusableMVDictIds, numValues, values);
- valuesBuffer[i] = values;
+ if (_dataType == DataType.BOOLEAN) {
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
+ int[] intValues = new int[numValues];
+ _dictionary.readIntValues(_reusableMVDictIds, numValues,
intValues);
+ String[] values = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i] = Boolean.toString(intValues[i] == 1);
+ }
+ valuesBuffer[i] = values;
+ }
+ } else if (_dataType == DataType.TIMESTAMP) {
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
+ long[] longValues = new long[numValues];
+ _dictionary.readLongValues(_reusableMVDictIds, numValues,
longValues);
+ String[] values = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i] = new Timestamp(longValues[i]).toString();
+ }
+ valuesBuffer[i] = values;
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getDictIdMV(docIds[i], _reusableMVDictIds,
readerContext);
+ String[] values = new String[numValues];
+ _dictionary.readStringValues(_reusableMVDictIds, numValues,
values);
+ valuesBuffer[i] = values;
+ }
}
} else {
- _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, getReaderContext());
+ if (_dataType == DataType.BOOLEAN) {
+ int[] intValueBuffer = new int[_maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getIntMV(docIds[i], intValueBuffer,
readerContext);
+ String[] values = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i] = Boolean.toString(intValueBuffer[i] == 1);
+ }
+ valuesBuffer[i] = values;
+ }
+ } else if (_dataType == DataType.TIMESTAMP) {
+ long[] longValueBuffer = new long[_maxNumValuesPerMVEntry];
+ for (int i = 0; i < length; i++) {
+ int numValues = _reader.getLongMV(docIds[i], longValueBuffer,
readerContext);
+ String[] values = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ values[i] = new Timestamp(longValueBuffer[i]).toString();
+ }
+ valuesBuffer[i] = values;
+ }
+ } else {
+ _reader.readValuesMV(docIds, length, _maxNumValuesPerMVEntry,
valuesBuffer, readerContext);
+ }
}
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
index 2479afd94c..11d1c0bb01 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.operator.transform.function;
import com.google.common.base.Preconditions;
import java.math.BigDecimal;
+import java.sql.Timestamp;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.segment.spi.index.reader.Dictionary;
@@ -85,11 +86,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_intValuesSV == null) {
_intValuesSV = new int[length];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -126,11 +125,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_longValuesSV == null) {
_longValuesSV = new long[length];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -167,11 +164,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_floatValuesSV == null) {
_floatValuesSV = new float[length];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -208,11 +203,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_doubleValuesSV == null) {
_doubleValuesSV = new double[length];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -249,10 +242,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock
projectionBlock) {
int length = projectionBlock.getNumDocs();
- if (_bigDecimalValuesSV == null || _bigDecimalValuesSV.length < length) {
+ if (_bigDecimalValuesSV == null) {
_bigDecimalValuesSV = new BigDecimal[length];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -293,17 +285,26 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_stringValuesSV == null) {
_stringValuesSV = new String[length];
}
-
+ DataType dataType = getResultMetadata().getDataType();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
- dictionary.readStringValues(dictIds, length, _stringValuesSV);
+ if (dataType == DataType.BOOLEAN) {
+ for (int i = 0; i < length; i++) {
+ _stringValuesSV[i] =
Boolean.toString(dictionary.getIntValue(dictIds[i]) == 1);
+ }
+ } else if (dataType == DataType.TIMESTAMP) {
+ for (int i = 0; i < length; i++) {
+ _stringValuesSV[i] = new
Timestamp(dictionary.getLongValue(dictIds[i])).toString();
+ }
+ } else {
+ dictionary.readStringValues(dictIds, length, _stringValuesSV);
+ }
} else {
- switch (getResultMetadata().getDataType().getStoredType()) {
+ switch (dataType) {
case INT:
int[] intValues = transformToIntValuesSV(projectionBlock);
ArrayCopyUtils.copy(intValues, _stringValuesSV, length);
@@ -324,6 +325,18 @@ public abstract class BaseTransformFunction implements
TransformFunction {
BigDecimal[] bigDecimalValues =
transformToBigDecimalValuesSV(projectionBlock);
ArrayCopyUtils.copy(bigDecimalValues, _stringValuesSV, length);
break;
+ case BOOLEAN:
+ intValues = transformToIntValuesSV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ _stringValuesSV[i] = Boolean.toString(intValues[i] == 1);
+ }
+ break;
+ case TIMESTAMP:
+ longValues = transformToLongValuesSV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ _stringValuesSV[i] = new Timestamp(longValues[i]).toString();
+ }
+ break;
case BYTES:
byte[][] bytesValues = transformToBytesValuesSV(projectionBlock);
ArrayCopyUtils.copy(bytesValues, _stringValuesSV, length);
@@ -338,11 +351,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_byteValuesSV == null) {
_byteValuesSV = new byte[length][];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[] dictIds = transformToDictIdsSV(projectionBlock);
@@ -363,11 +374,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_intValuesMV == null) {
_intValuesMV = new int[length][];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -430,11 +439,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_longValuesMV == null) {
_longValuesMV = new long[length][];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -497,11 +504,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) {
int length = projectionBlock.getNumDocs();
-
if (_floatValuesMV == null) {
_floatValuesMV = new float[length][];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -564,11 +569,9 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock)
{
int length = projectionBlock.getNumDocs();
-
if (_doubleValuesMV == null) {
_doubleValuesMV = new double[length][];
}
-
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
@@ -631,23 +634,44 @@ public abstract class BaseTransformFunction implements
TransformFunction {
@Override
public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock)
{
int length = projectionBlock.getNumDocs();
-
if (_stringValuesMV == null) {
_stringValuesMV = new String[length][];
}
-
+ DataType dataType = getResultMetadata().getDataType();
Dictionary dictionary = getDictionary();
if (dictionary != null) {
int[][] dictIdsMV = transformToDictIdsMV(projectionBlock);
- for (int i = 0; i < length; i++) {
- int[] dictIds = dictIdsMV[i];
- int numValues = dictIds.length;
- String[] stringValues = new String[numValues];
- dictionary.readStringValues(dictIds, numValues, stringValues);
- _stringValuesMV[i] = stringValues;
+ if (dataType == DataType.BOOLEAN) {
+ for (int i = 0; i < length; i++) {
+ int[] dictIds = dictIdsMV[i];
+ int numValues = dictIds.length;
+ String[] stringValues = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ stringValues[j] =
Boolean.toString(dictionary.getIntValue(dictIds[i]) == 1);
+ }
+ _stringValuesMV[i] = stringValues;
+ }
+ } else if (dataType == DataType.TIMESTAMP) {
+ for (int i = 0; i < length; i++) {
+ int[] dictIds = dictIdsMV[i];
+ int numValues = dictIds.length;
+ String[] stringValues = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ stringValues[j] = new
Timestamp(dictionary.getLongValue(dictIds[i])).toString();
+ }
+ _stringValuesMV[i] = stringValues;
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ int[] dictIds = dictIdsMV[i];
+ int numValues = dictIds.length;
+ String[] stringValues = new String[numValues];
+ dictionary.readStringValues(dictIds, numValues, stringValues);
+ _stringValuesMV[i] = stringValues;
+ }
}
} else {
- switch (getResultMetadata().getDataType().getStoredType()) {
+ switch (dataType) {
case INT:
int[][] intValuesMV = transformToIntValuesMV(projectionBlock);
for (int i = 0; i < length; i++) {
@@ -688,6 +712,30 @@ public abstract class BaseTransformFunction implements
TransformFunction {
_stringValuesMV[i] = stringValues;
}
break;
+ case BOOLEAN:
+ intValuesMV = transformToIntValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ int[] intValues = intValuesMV[i];
+ int numValues = intValues.length;
+ String[] stringValues = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ stringValues[j] = Boolean.toString(intValues[i] == 1);
+ }
+ _stringValuesMV[i] = stringValues;
+ }
+ break;
+ case TIMESTAMP:
+ longValuesMV = transformToLongValuesMV(projectionBlock);
+ for (int i = 0; i < length; i++) {
+ long[] longValues = longValuesMV[i];
+ int numValues = longValues.length;
+ String[] stringValues = new String[numValues];
+ for (int j = 0; j < numValues; j++) {
+ stringValues[j] = new Timestamp(longValues[i]).toString();
+ }
+ _stringValuesMV[i] = stringValues;
+ }
+ break;
default:
throw new IllegalStateException();
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
index 751efe4eb0..4a527ad868 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java
@@ -19,14 +19,12 @@
package org.apache.pinot.core.operator.transform.function;
import java.math.BigDecimal;
-import java.sql.Timestamp;
import java.util.List;
import java.util.Map;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.core.operator.transform.TransformResultMetadata;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.spi.data.FieldSpec.DataType;
-import org.apache.pinot.spi.utils.ArrayCopyUtils;
public class CastTransformFunction extends BaseTransformFunction {
@@ -71,6 +69,7 @@ public class CastTransformFunction extends
BaseTransformFunction {
case "BIG_DECIMAL":
_resultMetadata = BIG_DECIMAL_SV_NO_DICTIONARY_METADATA;
break;
+ case "BOOL":
case "BOOLEAN":
_resultMetadata = BOOLEAN_SV_NO_DICTIONARY_METADATA;
break;
@@ -99,275 +98,55 @@ public class CastTransformFunction extends
BaseTransformFunction {
@Override
public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) {
- // When casting to types other than INT, need to first read as the result
type then convert to int values
- DataType resultStoredType = _resultMetadata.getDataType().getStoredType();
- if (resultStoredType == DataType.INT) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.INT) {
return _transformFunction.transformToIntValuesSV(projectionBlock);
} else {
- int length = projectionBlock.getNumDocs();
- if (_intValuesSV == null || _intValuesSV.length < length) {
- _intValuesSV = new int[length];
- }
- switch (resultStoredType) {
- case LONG:
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _intValuesSV, length);
- break;
- case FLOAT:
- float[] floatValues =
_transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _intValuesSV, length);
- break;
- case DOUBLE:
- double[] doubleValues =
_transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _intValuesSV, length);
- break;
- case BIG_DECIMAL:
- BigDecimal[] bigDecimalValues =
_transformFunction.transformToBigDecimalValuesSV(projectionBlock);
- ArrayCopyUtils.copy(bigDecimalValues, _intValuesSV, length);
- break;
- case STRING:
- String[] stringValues =
_transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _intValuesSV, length);
- break;
- default:
- throw new IllegalStateException();
- }
- return _intValuesSV;
+ return super.transformToIntValuesSV(projectionBlock);
}
}
@Override
public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) {
- // When casting to types other than LONG, need to first read as the result
type then convert to long values
- DataType resultStoredType = _resultMetadata.getDataType().getStoredType();
- if (resultStoredType == DataType.LONG) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.LONG) {
return _transformFunction.transformToLongValuesSV(projectionBlock);
} else {
- int length = projectionBlock.getNumDocs();
-
- if (_longValuesSV == null || _longValuesSV.length < length) {
- _longValuesSV = new long[length];
- }
- switch (resultStoredType) {
- case INT:
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _longValuesSV, length);
- break;
- case FLOAT:
- float[] floatValues =
_transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _longValuesSV, length);
- break;
- case DOUBLE:
- double[] doubleValues =
_transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _longValuesSV, length);
- break;
- case BIG_DECIMAL:
- BigDecimal[] bigDecimalValues =
_transformFunction.transformToBigDecimalValuesSV(projectionBlock);
- ArrayCopyUtils.copy(bigDecimalValues, _longValuesSV, length);
- break;
- case STRING:
- String[] stringValues =
_transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _longValuesSV, length);
- break;
- default:
- throw new IllegalStateException();
- }
- return _longValuesSV;
+ return super.transformToLongValuesSV(projectionBlock);
}
}
@Override
public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) {
- // When casting to types other than FLOAT, need to first read as the
result type then convert to float values
- DataType resultStoredType = _resultMetadata.getDataType().getStoredType();
- if (resultStoredType == DataType.FLOAT) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.FLOAT) {
return _transformFunction.transformToFloatValuesSV(projectionBlock);
} else {
- int length = projectionBlock.getNumDocs();
-
- if (_floatValuesSV == null || _floatValuesSV.length < length) {
- _floatValuesSV = new float[length];
- }
- switch (resultStoredType) {
- case INT:
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _floatValuesSV, length);
- break;
- case LONG:
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _floatValuesSV, length);
- break;
- case DOUBLE:
- double[] doubleValues =
_transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _floatValuesSV, length);
- break;
- case BIG_DECIMAL:
- BigDecimal[] bigDecimalValues =
_transformFunction.transformToBigDecimalValuesSV(projectionBlock);
- ArrayCopyUtils.copy(bigDecimalValues, _floatValuesSV, length);
- break;
- case STRING:
- String[] stringValues =
_transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _floatValuesSV, length);
- break;
- default:
- throw new IllegalStateException();
- }
- return _floatValuesSV;
+ return super.transformToFloatValuesSV(projectionBlock);
}
}
@Override
public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) {
- // When casting to types other than DOUBLE, need to first read as the
result type then convert to double values
- DataType resultStoredType = _resultMetadata.getDataType().getStoredType();
- if (resultStoredType == DataType.DOUBLE) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.DOUBLE) {
return _transformFunction.transformToDoubleValuesSV(projectionBlock);
} else {
- int length = projectionBlock.getNumDocs();
-
- if (_doubleValuesSV == null || _doubleValuesSV.length < length) {
- _doubleValuesSV = new double[length];
- }
- switch (resultStoredType) {
- case INT:
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _doubleValuesSV, length);
- break;
- case LONG:
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _doubleValuesSV, length);
- break;
- case FLOAT:
- float[] floatValues =
_transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _doubleValuesSV, length);
- break;
- case BIG_DECIMAL:
- BigDecimal[] bigDecimalValues =
_transformFunction.transformToBigDecimalValuesSV(projectionBlock);
- ArrayCopyUtils.copy(bigDecimalValues, _doubleValuesSV, length);
- break;
- case STRING:
- String[] stringValues =
_transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _doubleValuesSV, length);
- break;
- default:
- throw new IllegalStateException();
- }
- return _doubleValuesSV;
+ return super.transformToDoubleValuesSV(projectionBlock);
}
}
@Override
public BigDecimal[] transformToBigDecimalValuesSV(ProjectionBlock
projectionBlock) {
- // When casting to types other than BIG_DECIMAL, need to first read as the
result type then convert to
- // BigDecimal values
- DataType dataType = _resultMetadata.getDataType();
- DataType resultStoredType = dataType.getStoredType();
- if (dataType == DataType.BIG_DECIMAL) {
+ if (_resultMetadata.getDataType().getStoredType() == DataType.BIG_DECIMAL)
{
return _transformFunction.transformToBigDecimalValuesSV(projectionBlock);
} else {
- int length = projectionBlock.getNumDocs();
- if (_bigDecimalValuesSV == null || _bigDecimalValuesSV.length < length) {
- _bigDecimalValuesSV = new BigDecimal[length];
- }
- int numDocs = projectionBlock.getNumDocs();
- switch (resultStoredType) {
- case INT:
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _bigDecimalValuesSV, numDocs);
- break;
- case LONG:
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _bigDecimalValuesSV, numDocs);
- break;
- case FLOAT:
- float[] floatValues =
_transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _bigDecimalValuesSV, numDocs);
- break;
- case DOUBLE:
- double[] doubleValues =
_transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _bigDecimalValuesSV, numDocs);
- break;
- case STRING:
- String[] stringValues =
_transformFunction.transformToStringValuesSV(projectionBlock);
- ArrayCopyUtils.copy(stringValues, _bigDecimalValuesSV, numDocs);
- break;
- default:
- throw new IllegalStateException();
- }
- return _bigDecimalValuesSV;
+ return super.transformToBigDecimalValuesSV(projectionBlock);
}
}
@Override
public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) {
- // When casting to types other than STRING, need to first read as the
result type then convert to string values
- DataType resultDataType = _resultMetadata.getDataType();
- DataType resultStoredType = resultDataType.getStoredType();
- int length = projectionBlock.getNumDocs();
- if (resultStoredType == DataType.STRING) {
- // Specialize BOOlEAN and TIMESTAMP when casting to STRING
- DataType inputDataType =
_transformFunction.getResultMetadata().getDataType();
- if (inputDataType.getStoredType() != inputDataType) {
- if (_stringValuesSV == null || _stringValuesSV.length < length) {
- _stringValuesSV = new String[length];
- }
- if (inputDataType == DataType.BOOLEAN) {
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- for (int i = 0; i < length; i++) {
- _stringValuesSV[i] = Boolean.toString(intValues[i] == 1);
- }
- } else {
- assert inputDataType == DataType.TIMESTAMP;
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- for (int i = 0; i < length; i++) {
- _stringValuesSV[i] = new Timestamp(longValues[i]).toString();
- }
- }
- return _stringValuesSV;
- } else {
- return _transformFunction.transformToStringValuesSV(projectionBlock);
- }
+ if (_resultMetadata.getDataType().getStoredType() == DataType.STRING) {
+ return _transformFunction.transformToStringValuesSV(projectionBlock);
} else {
- if (_stringValuesSV == null || _stringValuesSV.length < length) {
- _stringValuesSV = new String[length];
- }
- switch (resultDataType) {
- case INT:
- int[] intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- ArrayCopyUtils.copy(intValues, _stringValuesSV, length);
- break;
- case LONG:
- long[] longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- ArrayCopyUtils.copy(longValues, _stringValuesSV, length);
- break;
- case FLOAT:
- float[] floatValues =
_transformFunction.transformToFloatValuesSV(projectionBlock);
- ArrayCopyUtils.copy(floatValues, _stringValuesSV, length);
- break;
- case DOUBLE:
- double[] doubleValues =
_transformFunction.transformToDoubleValuesSV(projectionBlock);
- ArrayCopyUtils.copy(doubleValues, _stringValuesSV, length);
- break;
- case BIG_DECIMAL:
- BigDecimal[] bigDecimalValues =
_transformFunction.transformToBigDecimalValuesSV(projectionBlock);
- ArrayCopyUtils.copy(bigDecimalValues, _stringValuesSV, length);
- break;
- case BOOLEAN:
- intValues =
_transformFunction.transformToIntValuesSV(projectionBlock);
- for (int i = 0; i < length; i++) {
- _stringValuesSV[i] = Boolean.toString(intValues[i] == 1);
- }
- break;
- case TIMESTAMP:
- longValues =
_transformFunction.transformToLongValuesSV(projectionBlock);
- for (int i = 0; i < length; i++) {
- _stringValuesSV[i] = new Timestamp(longValues[i]).toString();
- }
- break;
- default:
- throw new IllegalStateException();
- }
- return _stringValuesSV;
+ return super.transformToStringValuesSV(projectionBlock);
}
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
index d40d482a6d..2d37fb168c 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunctionTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.core.operator.transform.function;
import java.io.File;
import java.math.BigDecimal;
+import java.sql.Timestamp;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
@@ -219,7 +220,11 @@ public abstract class BaseTransformFunctionTest {
Assert.assertEquals(floatValues[i], (float) expectedValues[i]);
Assert.assertEquals(doubleValues[i], (double) expectedValues[i]);
Assert.assertEquals(bigDecimalValues[i].intValue(), expectedValues[i]);
- Assert.assertEquals(stringValues[i],
Integer.toString(expectedValues[i]));
+ if (transformFunction.getResultMetadata().getDataType() ==
FieldSpec.DataType.BOOLEAN) {
+ Assert.assertEquals(stringValues[i],
Boolean.toString(expectedValues[i] == 1));
+ } else {
+ Assert.assertEquals(stringValues[i],
Integer.toString(expectedValues[i]));
+ }
}
}
@@ -236,7 +241,11 @@ public abstract class BaseTransformFunctionTest {
Assert.assertEquals(floatValues[i], (float) expectedValues[i]);
Assert.assertEquals(doubleValues[i], (double) expectedValues[i]);
Assert.assertEquals(bigDecimalValues[i].longValue(), expectedValues[i]);
- Assert.assertEquals(stringValues[i], Long.toString(expectedValues[i]));
+ if (transformFunction.getResultMetadata().getDataType() ==
FieldSpec.DataType.TIMESTAMP) {
+ Assert.assertEquals(stringValues[i], new
Timestamp(expectedValues[i]).toString());
+ } else {
+ Assert.assertEquals(stringValues[i], Long.toString(expectedValues[i]));
+ }
}
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java
index c8f83dad6e..4130cdd399 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/NullHandlingTransformFunctionTest.java
@@ -22,7 +22,6 @@ import java.io.File;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -54,10 +53,12 @@ import org.apache.pinot.spi.data.TimeGranularitySpec;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.apache.pinot.spi.utils.ReadMode;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
-import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
public class NullHandlingTransformFunctionTest {
private static final String SEGMENT_NAME = "testSegmentWithNulls";
@@ -175,13 +176,11 @@ public class NullHandlingTransformFunctionTest {
throws Exception {
ExpressionContext expression =
RequestContextUtils.getExpression(String.format("%s IS NULL", columnName));
TransformFunction transformFunction =
TransformFunctionFactory.get(expression, _dataSourceMap);
- Assert.assertTrue(transformFunction instanceof IsNullTransformFunction);
- Assert.assertEquals(transformFunction.getName(),
TransformFunctionType.IS_NULL.getName());
- int[] expectedValues = new int[NUM_ROWS];
+ assertTrue(transformFunction instanceof IsNullTransformFunction);
+ assertEquals(transformFunction.getName(),
TransformFunctionType.IS_NULL.getName());
+ boolean[] expectedValues = new boolean[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
- if (i % NULL_VALUE_MOD == 0) {
- expectedValues[i] = 1;
- }
+ expectedValues[i] = i % NULL_VALUE_MOD == 0;
}
testTransformFunction(expression, expectedValues);
}
@@ -199,33 +198,30 @@ public class NullHandlingTransformFunctionTest {
public void testIsNotNullTransformFunction(String columnName)
throws Exception {
- ExpressionContext expression =
- RequestContextUtils.getExpression(String.format("%s IS NOT NULL",
columnName));
+ ExpressionContext expression =
RequestContextUtils.getExpression(String.format("%s IS NOT NULL", columnName));
TransformFunction transformFunction =
TransformFunctionFactory.get(expression, _dataSourceMap);
- Assert.assertTrue(transformFunction instanceof IsNotNullTransformFunction);
- Assert.assertEquals(transformFunction.getName(),
TransformFunctionType.IS_NOT_NULL.getName());
- int[] expectedValues = new int[NUM_ROWS];
- Arrays.fill(expectedValues, 1);
+ assertTrue(transformFunction instanceof IsNotNullTransformFunction);
+ assertEquals(transformFunction.getName(),
TransformFunctionType.IS_NOT_NULL.getName());
+ boolean[] expectedValues = new boolean[NUM_ROWS];
for (int i = 0; i < NUM_ROWS; i++) {
- if (i % NULL_VALUE_MOD == 0) {
- expectedValues[i] = 0;
- }
+ expectedValues[i] = i % NULL_VALUE_MOD != 0;
}
testTransformFunction(expression, expectedValues);
}
- protected void testTransformFunction(ExpressionContext expression, int[]
expectedValues) throws Exception {
+ protected void testTransformFunction(ExpressionContext expression, boolean[]
expectedValues)
+ throws Exception {
int[] intValues =
getTransformFunctionInstance(expression).transformToIntValuesSV(_projectionBlock);
long[] longValues =
getTransformFunctionInstance(expression).transformToLongValuesSV(_projectionBlock);
float[] floatValues =
getTransformFunctionInstance(expression).transformToFloatValuesSV(_projectionBlock);
double[] doubleValues =
getTransformFunctionInstance(expression).transformToDoubleValuesSV(_projectionBlock);
String[] stringValues =
getTransformFunctionInstance(expression).transformToStringValuesSV(_projectionBlock);
for (int i = 0; i < NUM_ROWS; i++) {
- Assert.assertEquals(intValues[i], expectedValues[i]);
- Assert.assertEquals(longValues[i], expectedValues[i]);
- Assert.assertEquals(floatValues[i], (float) expectedValues[i]);
- Assert.assertEquals(doubleValues[i], (double) expectedValues[i]);
- Assert.assertEquals(stringValues[i],
Integer.toString(expectedValues[i]));
+ assertEquals(intValues[i] == 1, expectedValues[i]);
+ assertEquals(longValues[i] == 1, expectedValues[i]);
+ assertEquals(floatValues[i] == 1, expectedValues[i]);
+ assertEquals(doubleValues[i] == 1, expectedValues[i]);
+ assertEquals(stringValues[i], Boolean.toString(expectedValues[i]));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
