This is an automated email from the ASF dual-hosted git repository. jackie pushed a commit to branch empty_array_input in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit d5394f3789d89084048740271f9620d23fc6a15f Author: Jackie (Xiaotian) Jiang <[email protected]> AuthorDate: Wed May 8 17:13:40 2019 -0700 In DataTypeTransformer, support filling in default null value for empty array input In normal case, RecordReader should fill in default null value for null/empty array input. In case where RecordReader is not implemented to handle default null value, handle it in DataTypeTransformer. --- .../org/apache/pinot/common/data/FieldSpec.java | 26 +++++++++++----------- .../recordtransformer/DataTypeTransformer.java | 7 +++--- .../recordtransformer/RecordTransformerTest.java | 18 +++++++++++++++ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java index 30be748..00192a6 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/data/FieldSpec.java @@ -49,19 +49,19 @@ public abstract class FieldSpec implements Comparable<FieldSpec>, ConfigNodeLife // TODO: revisit to see if we allow 0-length byte array private static final byte[] NULL_BYTE_ARRAY_VALUE = new byte[0]; - private static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE; - private static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE; - private static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY; - private static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY; - private static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE; - - private static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null"; - private static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0; - private static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L; - private static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F; - private static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D; - private static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null"; - private static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE; + public static final Integer DEFAULT_DIMENSION_NULL_VALUE_OF_INT = Integer.MIN_VALUE; + public static final Long DEFAULT_DIMENSION_NULL_VALUE_OF_LONG = Long.MIN_VALUE; + public static final Float DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT = Float.NEGATIVE_INFINITY; + public static final Double DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE = Double.NEGATIVE_INFINITY; + public static final String DEFAULT_DIMENSION_NULL_VALUE_OF_STRING = "null"; + public static final byte[] DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE; + + public static final Integer DEFAULT_METRIC_NULL_VALUE_OF_INT = 0; + public static final Long DEFAULT_METRIC_NULL_VALUE_OF_LONG = 0L; + public static final Float DEFAULT_METRIC_NULL_VALUE_OF_FLOAT = 0.0F; + public static final Double DEFAULT_METRIC_NULL_VALUE_OF_DOUBLE = 0.0D; + public static final String DEFAULT_METRIC_NULL_VALUE_OF_STRING = "null"; + public static final byte[] DEFAULT_METRIC_NULL_VALUE_OF_BYTES = NULL_BYTE_ARRAY_VALUE; @ConfigKey("name") protected String _name; diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java index a021f4b..4fff5fc 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/DataTypeTransformer.java @@ -74,9 +74,9 @@ public class DataTypeTransformer implements RecordTransformer { PinotDataType dest = entry.getValue(); Object value = record.getValue(column); - // NOTE: should not be null in normal case if combined with other transformers (without TimeTransformer, outgoing - // time column might be null) - if (value == null) { + // NOTE: should not need to set default null value in normal case (RecordReader is responsible for filling in the + // default null value; TimeTransformer is responsible for filling in the outgoing time value if not exists) + if (value == null || (value instanceof Object[] && ((Object[]) value).length == 0)) { // Set default null value FieldSpec fieldSpec = _schema.getFieldSpecFor(column); Object defaultNullValue = fieldSpec.getDefaultNullValue(); @@ -91,7 +91,6 @@ public class DataTypeTransformer implements RecordTransformer { if (value instanceof Object[]) { // Multi-valued column Object[] values = (Object[]) value; - assert values.length > 0; source = MULTI_VALUE_TYPE_MAP.get(values[0].getClass()); if (source == null) { source = PinotDataType.OBJECT_ARRAY; diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java index 73b4476..81283cf 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/recordtransformer/RecordTransformerTest.java @@ -88,6 +88,24 @@ public class RecordTransformerTest { assertEquals(record.getValue("svStringWithNullCharacters"), "1\0002\0003"); assertEquals(record.getValue("svStringWithLengthLimit"), "123"); } + + // Test empty record + record = new GenericRow(); + for (int i = 0; i < NUM_ROUNDS; i++) { + record = transformer.transform(record); + assertNotNull(record); + assertEquals(record.getValue("svInt"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT); + assertEquals(record.getValue("svLong"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG); + assertEquals(record.getValue("svFloat"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT); + assertEquals(record.getValue("svDouble"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE); + assertEquals(record.getValue("svBytes"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_BYTES); + assertEquals(record.getValue("mvInt"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_INT}); + assertEquals(record.getValue("mvLong"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_LONG}); + assertEquals(record.getValue("mvFloat"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_FLOAT}); + assertEquals(record.getValue("mvDouble"), new Object[]{FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_DOUBLE}); + assertEquals(record.getValue("svStringWithNullCharacters"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING); + assertEquals(record.getValue("svStringWithLengthLimit"), FieldSpec.DEFAULT_DIMENSION_NULL_VALUE_OF_STRING); + } } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
