This is an automated email from the ASF dual-hosted git repository. kharekartik pushed a commit to branch optimise_memory in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 6706e2a8b59349505b059e0c03ff83d76b55b436 Author: KKcorps <[email protected]> AuthorDate: Sat Mar 26 19:03:16 2022 +0530 Only create results array for length of input and not for maxLength --- .../function/AdditionTransformFunction.java | 5 +- .../function/ArrayAverageTransformFunction.java | 5 +- .../function/ArrayLengthTransformFunction.java | 5 +- .../function/ArrayMaxTransformFunction.java | 30 ++++++++---- .../function/ArrayMinTransformFunction.java | 34 +++++++++---- .../function/ArraySumTransformFunction.java | 6 ++- .../transform/function/BaseTransformFunction.java | 55 +++++++++++++--------- .../transform/function/CastTransformFunction.java | 24 ++++++---- .../DateTimeConversionTransformFunction.java | 10 ++-- .../function/DateTruncTransformFunction.java | 5 +- .../function/DivisionTransformFunction.java | 6 +-- .../function/InIdSetTransformFunction.java | 5 +- .../transform/function/InTransformFunction.java | 5 +- .../function/JsonExtractKeyTransformFunction.java | 5 +- .../function/LogicalOperatorTransformFunction.java | 5 +- .../function/MapValueTransformFunction.java | 5 +- .../function/ModuloTransformFunction.java | 6 +-- .../function/MultiplicationTransformFunction.java | 5 +- .../function/ScalarTransformFunctionWrapper.java | 55 +++++++++++++--------- .../function/SingleParamMathTransformFunction.java | 21 +++++---- .../function/SubtractionTransformFunction.java | 6 +-- .../function/TimeConversionTransformFunction.java | 6 ++- .../function/ValueInTransformFunction.java | 25 +++++----- 23 files changed, 201 insertions(+), 133 deletions(-) diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java index bfc4904..43a5da3 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/AdditionTransformFunction.java @@ -66,11 +66,12 @@ public class AdditionTransformFunction extends BaseTransformFunction { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_sums == null) { - _sums = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _sums = new double[length]; } - int length = projectionBlock.getNumDocs(); Arrays.fill(_sums, 0, length, _literalSum); for (TransformFunction transformFunction : _transformFunctions) { double[] values = transformFunction.transformToDoubleValuesSV(projectionBlock); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java index 0ccd058..e712656 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayAverageTransformFunction.java @@ -71,11 +71,12 @@ public class ArrayAverageTransformFunction extends BaseTransformFunction { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int numDocs = projectionBlock.getNumDocs(); + if (_results == null) { - _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _results = new double[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (_argument.getResultMetadata().getDataType().getStoredType()) { case INT: int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java index 75efe80..9285337 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLengthTransformFunction.java @@ -68,11 +68,12 @@ public class ArrayLengthTransformFunction extends BaseTransformFunction { @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + int numDocs = projectionBlock.getNumDocs(); + if (_results == null) { - _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _results = new int[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (_argument.getResultMetadata().getDataType().getStoredType()) { case INT: int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java index 81d7479..b93aebc 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMaxTransformFunction.java @@ -79,10 +79,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.INT) { return super.transformToIntValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { - _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intValuesSV = new int[length]; } - int length = projectionBlock.getNumDocs(); int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock); for (int i = 0; i < length; i++) { int maxRes = Integer.MIN_VALUE; @@ -99,10 +101,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.LONG) { return super.transformToLongValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_longValuesSV == null) { - _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longValuesSV = new long[length]; } - int length = projectionBlock.getNumDocs(); long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock); for (int i = 0; i < length; i++) { long maxRes = Long.MIN_VALUE; @@ -119,10 +123,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.FLOAT) { return super.transformToFloatValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_floatValuesSV == null) { - _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _floatValuesSV = new float[length]; } - int length = projectionBlock.getNumDocs(); float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock); for (int i = 0; i < length; i++) { float maxRes = Float.NEGATIVE_INFINITY; @@ -139,10 +145,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.DOUBLE) { return super.transformToDoubleValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_doubleValuesSV == null) { - _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _doubleValuesSV = new double[length]; } - int length = projectionBlock.getNumDocs(); double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock); for (int i = 0; i < length; i++) { double maxRes = Double.NEGATIVE_INFINITY; @@ -159,10 +167,12 @@ public class ArrayMaxTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.STRING) { return super.transformToStringValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { - _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringValuesSV = new String[length]; } - int length = projectionBlock.getNumDocs(); String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock); for (int i = 0; i < length; i++) { String maxRes = null; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java index d3de1da..f2be2f9 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayMinTransformFunction.java @@ -79,10 +79,12 @@ public class ArrayMinTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.INT) { return super.transformToIntValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { - _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intValuesSV = new int[length]; } - int length = projectionBlock.getNumDocs(); int[][] intValuesMV = _argument.transformToIntValuesMV(projectionBlock); for (int i = 0; i < length; i++) { int minRes = Integer.MAX_VALUE; @@ -99,10 +101,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.LONG) { return super.transformToLongValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_longValuesSV == null) { - _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longValuesSV = new long[length]; } - int length = projectionBlock.getNumDocs(); + long[][] longValuesMV = _argument.transformToLongValuesMV(projectionBlock); for (int i = 0; i < length; i++) { long minRes = Long.MAX_VALUE; @@ -119,10 +124,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.FLOAT) { return super.transformToFloatValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_floatValuesSV == null) { - _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _floatValuesSV = new float[length]; } - int length = projectionBlock.getNumDocs(); + float[][] floatValuesMV = _argument.transformToFloatValuesMV(projectionBlock); for (int i = 0; i < length; i++) { float minRes = Float.POSITIVE_INFINITY; @@ -139,10 +147,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.DOUBLE) { return super.transformToDoubleValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_doubleValuesSV == null) { - _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _doubleValuesSV = new double[length]; } - int length = projectionBlock.getNumDocs(); + double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock); for (int i = 0; i < length; i++) { double minRes = Double.POSITIVE_INFINITY; @@ -159,10 +170,13 @@ public class ArrayMinTransformFunction extends BaseTransformFunction { if (_argument.getResultMetadata().getDataType().getStoredType() != DataType.STRING) { return super.transformToStringValuesSV(projectionBlock); } + + int length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { - _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringValuesSV = new String[length]; } - int length = projectionBlock.getNumDocs(); + String[][] stringValuesMV = _argument.transformToStringValuesMV(projectionBlock); for (int i = 0; i < length; i++) { String minRes = null; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java index 33ae2a1..85c86ce 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArraySumTransformFunction.java @@ -71,10 +71,12 @@ public class ArraySumTransformFunction extends BaseTransformFunction { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_results == null) { - _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _results = new double[length]; } - int length = projectionBlock.getNumDocs(); + double[][] doubleValuesMV = _argument.transformToDoubleValuesMV(projectionBlock); for (int i = 0; i < length; i++) { double sumRes = 0; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java index b7a3d91..da75d30 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/BaseTransformFunction.java @@ -81,11 +81,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { - _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intValuesSV = new int[length]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -117,11 +118,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_longValuesSV == null) { - _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longValuesSV = new long[length]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -153,11 +155,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public float[] transformToFloatValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_floatValuesSV == null) { - _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _floatValuesSV = new float[length]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -189,11 +192,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_doubleValuesSV == null) { - _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _doubleValuesSV = new double[length]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -225,11 +229,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_stringValuesSV == null) { - _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringValuesSV = new String[length]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -265,11 +270,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public byte[][] transformToBytesValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_byteValuesSV == null) { - _byteValuesSV = new byte[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _byteValuesSV = new byte[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[] dictIds = transformToDictIdsSV(projectionBlock); @@ -284,11 +290,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public int[][] transformToIntValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_intValuesMV == null) { - _intValuesMV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _intValuesMV = new int[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); @@ -350,11 +357,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public long[][] transformToLongValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_longValuesMV == null) { - _longValuesMV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _longValuesMV = new long[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); @@ -416,11 +424,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public float[][] transformToFloatValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_floatValuesMV == null) { - _floatValuesMV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _floatValuesMV = new float[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); @@ -482,11 +491,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public double[][] transformToDoubleValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_doubleValuesMV == null) { - _doubleValuesMV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _doubleValuesMV = new double[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); @@ -548,11 +558,12 @@ public abstract class BaseTransformFunction implements TransformFunction { @Override public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_stringValuesMV == null) { - _stringValuesMV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _stringValuesMV = new String[length][]; } - int length = projectionBlock.getNumDocs(); Dictionary dictionary = getDictionary(); if (dictionary != null) { int[][] dictIdsMV = transformToDictIdsMV(projectionBlock); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java index 503474e..eb361ca 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/CastTransformFunction.java @@ -99,10 +99,11 @@ public class CastTransformFunction extends BaseTransformFunction { if (resultStoredType == DataType.INT) { return _transformFunction.transformToIntValuesSV(projectionBlock); } else { + int numDocs = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { - _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intValuesSV = new int[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (resultStoredType) { case LONG: long[] longValues = _transformFunction.transformToLongValuesSV(projectionBlock); @@ -134,10 +135,11 @@ public class CastTransformFunction extends BaseTransformFunction { if (resultStoredType == DataType.LONG) { return _transformFunction.transformToLongValuesSV(projectionBlock); } else { + int numDocs = projectionBlock.getNumDocs(); + if (_longValuesSV == null) { - _longValuesSV = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longValuesSV = new long[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (resultStoredType) { case INT: int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); @@ -169,10 +171,11 @@ public class CastTransformFunction extends BaseTransformFunction { if (resultStoredType == DataType.FLOAT) { return _transformFunction.transformToFloatValuesSV(projectionBlock); } else { + int numDocs = projectionBlock.getNumDocs(); + if (_floatValuesSV == null) { - _floatValuesSV = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _floatValuesSV = new float[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (resultStoredType) { case INT: int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); @@ -204,10 +207,11 @@ public class CastTransformFunction extends BaseTransformFunction { if (resultStoredType == DataType.DOUBLE) { return _transformFunction.transformToDoubleValuesSV(projectionBlock); } else { + int numDocs = projectionBlock.getNumDocs(); + if (_doubleValuesSV == null) { - _doubleValuesSV = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _doubleValuesSV = new double[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); switch (resultStoredType) { case INT: int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); @@ -243,7 +247,7 @@ public class CastTransformFunction extends BaseTransformFunction { DataType inputDataType = _transformFunction.getResultMetadata().getDataType(); if (inputDataType.getStoredType() != inputDataType) { if (_stringValuesSV == null) { - _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringValuesSV = new String[numDocs]; } if (inputDataType == DataType.BOOLEAN) { int[] intValues = _transformFunction.transformToIntValuesSV(projectionBlock); @@ -263,7 +267,7 @@ public class CastTransformFunction extends BaseTransformFunction { } } else { if (_stringValuesSV == null) { - _stringValuesSV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringValuesSV = new String[numDocs]; } switch (resultDataType) { case INT: diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java index 99500bd..d895387 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTimeConversionTransformFunction.java @@ -127,11 +127,12 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction { @Override public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { if (_resultMetadata == LONG_SV_NO_DICTIONARY_METADATA) { + int length = projectionBlock.getNumDocs(); + if (_longOutputTimes == null) { - _longOutputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longOutputTimes = new long[length]; } - int length = projectionBlock.getNumDocs(); if (_dateTimeTransformer instanceof EpochToEpochTransformer) { EpochToEpochTransformer dateTimeTransformer = (EpochToEpochTransformer) _dateTimeTransformer; dateTimeTransformer @@ -150,11 +151,12 @@ public class DateTimeConversionTransformFunction extends BaseTransformFunction { @Override public String[] transformToStringValuesSV(ProjectionBlock projectionBlock) { if (_resultMetadata == STRING_SV_NO_DICTIONARY_METADATA) { + int length = projectionBlock.getNumDocs(); + if (_stringOutputTimes == null) { - _stringOutputTimes = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringOutputTimes = new String[length]; } - int length = projectionBlock.getNumDocs(); if (_dateTimeTransformer instanceof EpochToSDFTransformer) { EpochToSDFTransformer dateTimeTransformer = (EpochToSDFTransformer) _dateTimeTransformer; dateTimeTransformer diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java index 70e5e0d..7d97b3c 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DateTruncTransformFunction.java @@ -130,11 +130,12 @@ public class DateTruncTransformFunction extends BaseTransformFunction { @Override public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_longOutputTimes == null) { - _longOutputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longOutputTimes = new long[length]; } - int length = projectionBlock.getNumDocs(); long[] input = _mainTransformFunction.transformToLongValuesSV(projectionBlock); for (int i = 0; i < length; i++) { _longOutputTimes[i] = _outputTimeUnit diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java index dad8345..c04cb5d 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/DivisionTransformFunction.java @@ -77,12 +77,12 @@ public class DivisionTransformFunction extends BaseTransformFunction { @SuppressWarnings("Duplicates") @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_quotients == null) { - _quotients = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _quotients = new double[length]; } - int length = projectionBlock.getNumDocs(); - if (_firstTransformFunction == null) { Arrays.fill(_quotients, 0, length, _firstLiteral); } else { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java index 4dcff5a..9938c65 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InIdSetTransformFunction.java @@ -75,11 +75,12 @@ public class InIdSetTransformFunction extends BaseTransformFunction { @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_results == null) { - _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _results = new int[length]; } - int length = projectionBlock.getNumDocs(); DataType storedType = _transformFunction.getResultMetadata().getDataType().getStoredType(); switch (storedType) { case INT: diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java index 6245108..8fcfd81 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/InTransformFunction.java @@ -139,13 +139,14 @@ public class InTransformFunction extends BaseTransformFunction { @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_intValuesSV == null) { - _intValuesSV = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intValuesSV = new int[length]; } else { Arrays.fill(_intValuesSV, 0); } - int length = projectionBlock.getNumDocs(); TransformResultMetadata mainFunctionMetadata = _mainFunction.getResultMetadata(); DataType storedType = mainFunctionMetadata.getDataType().getStoredType(); if (_valueSet != null) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java index b5864c3..df00077 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/JsonExtractKeyTransformFunction.java @@ -86,12 +86,13 @@ public class JsonExtractKeyTransformFunction extends BaseTransformFunction { @Override public String[][] transformToStringValuesMV(ProjectionBlock projectionBlock) { + int numDocs = projectionBlock.getNumDocs(); + if (_stringValuesMV == null) { - _stringValuesMV = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _stringValuesMV = new String[numDocs][]; } String[] jsonStrings = _jsonFieldTransformFunction.transformToStringValuesSV(projectionBlock); - int numDocs = projectionBlock.getNumDocs(); for (int i = 0; i < numDocs; i++) { List<String> values = JSON_PARSER_CONTEXT.parse(jsonStrings[i]).read(_jsonPath); _stringValuesMV[i] = values.toArray(new String[0]); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java index 3fe232c..7818245 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/LogicalOperatorTransformFunction.java @@ -60,10 +60,11 @@ public abstract class LogicalOperatorTransformFunction extends BaseTransformFunc @Override public int[] transformToIntValuesSV(ProjectionBlock projectionBlock) { + int numDocs = projectionBlock.getNumDocs(); + if (_results == null) { - _results = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _results = new int[numDocs]; } - int numDocs = projectionBlock.getNumDocs(); ArrayCopyUtils.copy(_arguments.get(0).transformToIntValuesSV(projectionBlock), _results, numDocs); int numArguments = _arguments.size(); for (int i = 1; i < numArguments; i++) { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java index 395b20d..bdb6b39 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MapValueTransformFunction.java @@ -97,13 +97,14 @@ public class MapValueTransformFunction extends BaseTransformFunction { @Override public int[] transformToDictIdsSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_dictIds == null) { - _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _dictIds = new int[length]; } int[][] keyDictIdsMV = _keyColumnFunction.transformToDictIdsMV(projectionBlock); int[][] valueDictIdsMV = _valueColumnFunction.transformToDictIdsMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { int[] keyDictIds = keyDictIdsMV[i]; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java index 4339eec..68cb165 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ModuloTransformFunction.java @@ -77,12 +77,12 @@ public class ModuloTransformFunction extends BaseTransformFunction { @SuppressWarnings("Duplicates") @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_modulos == null) { - _modulos = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _modulos = new double[length]; } - int length = projectionBlock.getNumDocs(); - if (_firstTransformFunction == null) { Arrays.fill(_modulos, 0, length, _firstLiteral); } else { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java index 63d3986..df24e8c 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/MultiplicationTransformFunction.java @@ -66,11 +66,12 @@ public class MultiplicationTransformFunction extends BaseTransformFunction { @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_products == null) { - _products = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _products = new double[length]; } - int length = projectionBlock.getNumDocs(); Arrays.fill(_products, 0, length, _literalProduct); for (TransformFunction transformFunction : _transformFunctions) { double[] values = transformFunction.transformToDoubleValuesSV(projectionBlock); diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java index 34d44d3..518776d 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ScalarTransformFunctionWrapper.java @@ -125,11 +125,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.INT) { return super.transformToIntValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_intResults == null) { - _intResults = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _intResults = new int[length]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -144,11 +145,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) { return super.transformToLongValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_longResults == null) { - _longResults = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _longResults = new long[length]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -163,11 +165,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) { return super.transformToFloatValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_floatResults == null) { - _floatResults = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _floatResults = new float[length]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -182,11 +185,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) { return super.transformToDoubleValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_doubleResults == null) { - _doubleResults = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _doubleResults = new double[length]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -201,11 +205,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) { return super.transformToStringValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_stringResults == null) { - _stringResults = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _stringResults = new String[length]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -222,11 +227,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.BYTES) { return super.transformToBytesValuesSV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_bytesResults == null) { - _bytesResults = new byte[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _bytesResults = new byte[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -241,11 +247,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.INT) { return super.transformToIntValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_intMVResults == null) { - _intMVResults = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _intMVResults = new int[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -260,11 +267,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.LONG) { return super.transformToLongValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_longMVResults == null) { - _longMVResults = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _longMVResults = new long[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -279,11 +287,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.FLOAT) { return super.transformToFloatValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_floatMVResults == null) { - _floatMVResults = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _floatMVResults = new float[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -298,11 +307,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.DOUBLE) { return super.transformToDoubleValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_doubleMVResults == null) { - _doubleMVResults = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _doubleMVResults = new double[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; @@ -317,11 +327,12 @@ public class ScalarTransformFunctionWrapper extends BaseTransformFunction { if (_resultMetadata.getDataType().getStoredType() != DataType.STRING) { return super.transformToStringValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); + if (_stringMVResults == null) { - _stringMVResults = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _stringMVResults = new String[length][]; } getNonLiteralValues(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { for (int j = 0; j < _numNonLiteralArguments; j++) { _arguments[_nonLiteralIndices[j]] = _nonLiteralValues[j][i]; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java index 9dcb6a3..e667078 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SingleParamMathTransformFunction.java @@ -33,7 +33,6 @@ import org.apache.pinot.segment.spi.datasource.DataSource; */ public abstract class SingleParamMathTransformFunction extends BaseTransformFunction { private TransformFunction _transformFunction; - protected double[] _results; @Override public void init(List<TransformFunction> arguments, Map<String, DataSource> dataSourceMap) { @@ -55,13 +54,15 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { - if (_results == null) { - _results = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + int length = projectionBlock.getNumDocs(); + + if (_doubleValuesSV == null) { + _doubleValuesSV = new double[length]; } double[] values = _transformFunction.transformToDoubleValuesSV(projectionBlock); applyMathOperator(values, projectionBlock.getNumDocs()); - return _results; + return _doubleValuesSV; } abstract protected void applyMathOperator(double[] values, int length); @@ -77,7 +78,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.abs(values[i]); + _doubleValuesSV[i] = Math.abs(values[i]); } } } @@ -93,7 +94,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.ceil(values[i]); + _doubleValuesSV[i] = Math.ceil(values[i]); } } } @@ -109,7 +110,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.exp(values[i]); + _doubleValuesSV[i] = Math.exp(values[i]); } } } @@ -125,7 +126,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.floor(values[i]); + _doubleValuesSV[i] = Math.floor(values[i]); } } } @@ -141,7 +142,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.log(values[i]); + _doubleValuesSV[i] = Math.log(values[i]); } } } @@ -157,7 +158,7 @@ public abstract class SingleParamMathTransformFunction extends BaseTransformFunc @Override protected void applyMathOperator(double[] values, int length) { for (int i = 0; i < length; i++) { - _results[i] = Math.sqrt(values[i]); + _doubleValuesSV[i] = Math.sqrt(values[i]); } } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java index bd7b50a..f1a9540 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/SubtractionTransformFunction.java @@ -77,12 +77,12 @@ public class SubtractionTransformFunction extends BaseTransformFunction { @SuppressWarnings("Duplicates") @Override public double[] transformToDoubleValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_differences == null) { - _differences = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _differences = new double[length]; } - int length = projectionBlock.getNumDocs(); - if (_firstTransformFunction == null) { Arrays.fill(_differences, 0, length, _firstLiteral); } else { diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java index ac1a535..a32546d 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/TimeConversionTransformFunction.java @@ -68,12 +68,14 @@ public class TimeConversionTransformFunction extends BaseTransformFunction { @Override public long[] transformToLongValuesSV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_outputTimes == null) { - _outputTimes = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL]; + _outputTimes = new long[length]; } _timeUnitTransformer.transform(_mainTransformFunction.transformToLongValuesSV(projectionBlock), _outputTimes, - projectionBlock.getNumDocs()); + length); return _outputTimes; } } diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java index 0d1f20b..8635b99 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ValueInTransformFunction.java @@ -109,6 +109,8 @@ public class ValueInTransformFunction extends BaseTransformFunction { @Override public int[][] transformToDictIdsMV(ProjectionBlock projectionBlock) { + int length = projectionBlock.getNumDocs(); + if (_dictIdSet == null) { _dictIdSet = new IntOpenHashSet(); assert _dictionary != null; @@ -118,10 +120,9 @@ public class ValueInTransformFunction extends BaseTransformFunction { _dictIdSet.add(dictId); } } - _dictIds = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _dictIds = new int[length][]; } int[][] unFilteredDictIds = _mainTransformFunction.transformToDictIdsMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _dictIds[i] = filterInts(_dictIdSet, unFilteredDictIds[i]); } @@ -134,15 +135,15 @@ public class ValueInTransformFunction extends BaseTransformFunction { return super.transformToIntValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); if (_intValueSet == null) { _intValueSet = new IntOpenHashSet(); for (String inValue : _stringValueSet) { _intValueSet.add(Integer.parseInt(inValue)); } - _intValues = new int[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _intValues = new int[length][]; } int[][] unFilteredIntValues = _mainTransformFunction.transformToIntValuesMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _intValues[i] = filterInts(_intValueSet, unFilteredIntValues[i]); } @@ -155,15 +156,15 @@ public class ValueInTransformFunction extends BaseTransformFunction { return super.transformToLongValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); if (_longValueSet == null) { _longValueSet = new LongOpenHashSet(); for (String inValue : _stringValueSet) { _longValueSet.add(Long.parseLong(inValue)); } - _longValues = new long[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _longValues = new long[length][]; } long[][] unFilteredLongValues = _mainTransformFunction.transformToLongValuesMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _longValues[i] = filterLongs(_longValueSet, unFilteredLongValues[i]); } @@ -176,15 +177,15 @@ public class ValueInTransformFunction extends BaseTransformFunction { return super.transformToFloatValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); if (_floatValueSet == null) { _floatValueSet = new FloatOpenHashSet(); for (String inValue : _stringValueSet) { _floatValueSet.add(Float.parseFloat(inValue)); } - _floatValues = new float[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _floatValues = new float[length][]; } float[][] unFilteredFloatValues = _mainTransformFunction.transformToFloatValuesMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _floatValues[i] = filterFloats(_floatValueSet, unFilteredFloatValues[i]); } @@ -197,15 +198,15 @@ public class ValueInTransformFunction extends BaseTransformFunction { return super.transformToDoubleValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); if (_doubleValueSet == null) { _doubleValueSet = new DoubleOpenHashSet(); for (String inValue : _stringValueSet) { _doubleValueSet.add(Double.parseDouble(inValue)); } - _doubleValues = new double[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _doubleValues = new double[length][]; } double[][] unFilteredDoubleValues = _mainTransformFunction.transformToDoubleValuesMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _doubleValues[i] = filterDoubles(_doubleValueSet, unFilteredDoubleValues[i]); } @@ -218,11 +219,11 @@ public class ValueInTransformFunction extends BaseTransformFunction { return super.transformToStringValuesMV(projectionBlock); } + int length = projectionBlock.getNumDocs(); if (_stringValues == null) { - _stringValues = new String[DocIdSetPlanNode.MAX_DOC_PER_CALL][]; + _stringValues = new String[length][]; } String[][] unFilteredStringValues = _mainTransformFunction.transformToStringValuesMV(projectionBlock); - int length = projectionBlock.getNumDocs(); for (int i = 0; i < length; i++) { _stringValues[i] = filterStrings(_stringValueSet, unFilteredStringValues[i]); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
