This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0cd233b6db6 Fix complex field ser de (#17685)
0cd233b6db6 is described below
commit 0cd233b6db6d87d2eb3efa6901f3744a89143767
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Wed Feb 25 15:04:27 2026 -0800
Fix complex field ser de (#17685)
---
.../java/org/apache/pinot/spi/data/FieldSpec.java | 53 +++-
.../pinot/spi/data/SchemaSerializationTest.java | 353 +++++++++++++++------
2 files changed, 299 insertions(+), 107 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
index 5e8ef58666b..a67682aec9c 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
@@ -25,10 +25,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.OptBoolean;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.Serializable;
import java.math.BigDecimal;
import java.sql.Timestamp;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -336,13 +338,10 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
return getStringValue(_defaultNullValue);
}
- /**
- * Helper method to return the String value for the given object.
- * This is required as not all data types have a toString() (e.g. byte[]).
- *
- * @param value Value for which String value needs to be returned
- * @return String value for the object.
- */
+ /// Returns the [String] representation of the given object.
+ /// The input value could be:
+ /// - Default null value stored in [FieldSpec]
+ /// - Value from the records (post transform)
public static String getStringValue(Object value) {
if (value instanceof BigDecimal) {
return ((BigDecimal) value).toPlainString();
@@ -350,10 +349,32 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
if (value instanceof byte[]) {
return BytesUtils.toHexString((byte[]) value);
}
+ if (value instanceof List || value instanceof Map) {
+ try {
+ return JsonUtils.objectToString(value);
+ } catch (Exception e) {
+ throw new RuntimeException("Caught exception serializing value: " +
value, e);
+ }
+ }
return value.toString();
}
- // Required by JSON de-serializer. DO NOT REMOVE.
+ @JsonProperty
+ private void setDefaultNullValue(@Nullable JsonNode defaultNullValue) {
+ if (defaultNullValue != null && !defaultNullValue.isNull()) {
+ if (defaultNullValue.isValueNode()) {
+ _stringDefaultNullValue = defaultNullValue.asText();
+ } else {
+ // For ARRAY and OBJECT
+ _stringDefaultNullValue = defaultNullValue.toString();
+ }
+ }
+ if (_dataType != null) {
+ _defaultNullValue = getDefaultNullValue(getFieldType(), _dataType,
_stringDefaultNullValue);
+ }
+ }
+
+ @JsonIgnore
public void setDefaultNullValue(@Nullable Object defaultNullValue) {
if (defaultNullValue != null) {
_stringDefaultNullValue = getStringValue(defaultNullValue);
@@ -535,10 +556,8 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
jsonNode.put(key, BytesUtils.toHexString((byte[])
_defaultNullValue));
break;
case MAP:
- jsonNode.put(key, JsonUtils.objectToJsonNode(_defaultNullValue));
- break;
case LIST:
- jsonNode.put(key, JsonUtils.objectToJsonNode(_defaultNullValue));
+ jsonNode.set(key, JsonUtils.objectToJsonNode(_defaultNullValue));
break;
default:
throw new IllegalStateException("Unsupported data type: " + this);
@@ -568,7 +587,7 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
&& Objects.equals(_maxLength, that._maxLength)
&& Objects.equals(_maxLengthExceedStrategy,
that._maxLengthExceedStrategy)
&& _allowTrailingZeros == that._allowTrailingZeros
- &&
getStringValue(_defaultNullValue).equals(getStringValue(that._defaultNullValue))
+ && _dataType.equals(_defaultNullValue, that._defaultNullValue)
&& Objects.equals(_transformFunction, that._transformFunction)
&& Objects.equals(_virtualColumnProvider, that._virtualColumnProvider);
}
@@ -576,7 +595,7 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
@Override
public int hashCode() {
return Objects.hash(_name, _dataType, _singleValueField, _notNull,
_maxLength, _maxLengthExceedStrategy,
- _allowTrailingZeros, getStringValue(_defaultNullValue),
_transformFunction, _virtualColumnProvider);
+ _allowTrailingZeros, _dataType.hashCode(_defaultNullValue),
_transformFunction, _virtualColumnProvider);
}
/**
@@ -720,6 +739,14 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
}
}
+ public boolean equals(Object value1, Object value2) {
+ return this == BYTES ? Arrays.equals((byte[]) value1, (byte[]) value2) :
value1.equals(value2);
+ }
+
+ public int hashCode(Object value) {
+ return this == BYTES ? Arrays.hashCode((byte[]) value) :
value.hashCode();
+ }
+
/**
* Compares the given values of the data type.
*
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaSerializationTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaSerializationTest.java
index f6a8b7ce535..3acb8cdc551 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaSerializationTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/SchemaSerializationTest.java
@@ -20,12 +20,19 @@ package org.apache.pinot.spi.data;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.List;
import java.util.Map;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.JsonUtils;
-import org.testng.Assert;
import org.testng.annotations.Test;
import org.testng.collections.Lists;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
/**
* Unit tests for Schema serialization with @JsonValue annotation.
@@ -43,22 +50,22 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
- .addMetric("metric1", FieldSpec.DataType.LONG)
+ .addSingleValueDimension("dim1", DataType.STRING)
+ .addMetric("metric1", DataType.LONG)
.build();
// Serialize using Jackson (which should use @JsonValue -> toJsonObject())
final String jsonString = JsonUtils.objectToString(schema);
// Verify that defaultNullValueString is NOT present in the output
- Assert.assertFalse(jsonString.contains("defaultNullValueString"),
+ assertFalse(jsonString.contains("defaultNullValueString"),
"defaultNullValueString should not be present in serialized output");
// Verify it can be deserialized back
final Schema deserializedSchema = Schema.fromString(jsonString);
- Assert.assertEquals(deserializedSchema.getSchemaName(), "testSchema");
- Assert.assertNotNull(deserializedSchema.getDimensionSpec("dim1"));
- Assert.assertNotNull(deserializedSchema.getMetricSpec("metric1"));
+ assertEquals(deserializedSchema.getSchemaName(), "testSchema");
+ assertNotNull(deserializedSchema.getDimensionSpec("dim1"));
+ assertNotNull(deserializedSchema.getMetricSpec("metric1"));
}
/**
@@ -69,8 +76,8 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING) // default
null value = "null"
- .addMetric("metric1", FieldSpec.DataType.DOUBLE) // default null value
= 0.0
+ .addSingleValueDimension("dim1", DataType.STRING) // default null
value = "null"
+ .addMetric("metric1", DataType.DOUBLE) // default null value = 0.0
.build();
final String jsonString = JsonUtils.objectToString(schema);
@@ -78,24 +85,24 @@ public class SchemaSerializationTest {
// Check dimension field spec - should not have defaultNullValue since
"null" is the default
final JsonNode dimSpecs = jsonNode.get("dimensionFieldSpecs");
- Assert.assertNotNull(dimSpecs);
- Assert.assertEquals(dimSpecs.size(), 1);
+ assertNotNull(dimSpecs);
+ assertEquals(dimSpecs.size(), 1);
final JsonNode dimSpec = dimSpecs.get(0);
- Assert.assertFalse(dimSpec.has("defaultNullValue"),
+ assertFalse(dimSpec.has("defaultNullValue"),
"defaultNullValue should not be present for STRING dimension with
default value");
- Assert.assertFalse(dimSpec.has("notNull"),
+ assertFalse(dimSpec.has("notNull"),
"notNull should not be present when false (default)");
- Assert.assertFalse(dimSpec.has("singleValueField"),
+ assertFalse(dimSpec.has("singleValueField"),
"singleValueField should not be present when true (default)");
- Assert.assertFalse(dimSpec.has("allowTrailingZeros"),
+ assertFalse(dimSpec.has("allowTrailingZeros"),
"allowTrailingZeros should not be present when false (default)");
// Check metric field spec - should not have defaultNullValue since 0.0 is
the default
final JsonNode metricSpecs = jsonNode.get("metricFieldSpecs");
- Assert.assertNotNull(metricSpecs);
- Assert.assertEquals(metricSpecs.size(), 1);
+ assertNotNull(metricSpecs);
+ assertEquals(metricSpecs.size(), 1);
final JsonNode metricSpec = metricSpecs.get(0);
- Assert.assertFalse(metricSpec.has("defaultNullValue"),
+ assertFalse(metricSpec.has("defaultNullValue"),
"defaultNullValue should not be present for DOUBLE metric with default
value");
}
@@ -107,9 +114,9 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING,
"custom_default")
- .addMetric("metric1", FieldSpec.DataType.DOUBLE, 99.9)
- .addMultiValueDimension("mvDim", FieldSpec.DataType.INT)
+ .addSingleValueDimension("dim1", DataType.STRING, "custom_default")
+ .addMetric("metric1", DataType.DOUBLE, 99.9)
+ .addMultiValueDimension("mvDim", DataType.INT)
.build();
final String jsonString = JsonUtils.objectToString(schema);
@@ -128,24 +135,24 @@ public class SchemaSerializationTest {
}
}
- Assert.assertNotNull(dim1);
- Assert.assertTrue(dim1.has("defaultNullValue"),
+ assertNotNull(dim1);
+ assertTrue(dim1.has("defaultNullValue"),
"defaultNullValue should be present for non-default value");
- Assert.assertEquals(dim1.get("defaultNullValue").asText(),
"custom_default");
+ assertEquals(dim1.get("defaultNullValue").asText(), "custom_default");
// Check multi-value dimension has singleValueField: false
- Assert.assertNotNull(mvDim);
- Assert.assertTrue(mvDim.has("singleValueField"),
+ assertNotNull(mvDim);
+ assertTrue(mvDim.has("singleValueField"),
"singleValueField should be present when false (non-default)");
- Assert.assertFalse(mvDim.get("singleValueField").asBoolean());
+ assertFalse(mvDim.get("singleValueField").asBoolean());
// Check metric with custom default
final JsonNode metricSpecs = jsonNode.get("metricFieldSpecs");
- Assert.assertNotNull(metricSpecs);
+ assertNotNull(metricSpecs);
final JsonNode metric1 = metricSpecs.get(0);
- Assert.assertTrue(metric1.has("defaultNullValue"),
+ assertTrue(metric1.has("defaultNullValue"),
"defaultNullValue should be present for non-default value");
- Assert.assertEquals(metric1.get("defaultNullValue").asDouble(), 99.9);
+ assertEquals(metric1.get("defaultNullValue").asDouble(), 99.9);
}
/**
@@ -156,25 +163,25 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("dim1", DataType.STRING)
.build();
final String jsonString = JsonUtils.objectToString(schema);
final JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
// Should have dimensionFieldSpecs
- Assert.assertTrue(jsonNode.has("dimensionFieldSpecs"));
+ assertTrue(jsonNode.has("dimensionFieldSpecs"));
// Should NOT have empty metricFieldSpecs, dateTimeFieldSpecs,
complexFieldSpecs
- Assert.assertFalse(jsonNode.has("metricFieldSpecs"),
+ assertFalse(jsonNode.has("metricFieldSpecs"),
"Empty metricFieldSpecs should be omitted");
- Assert.assertFalse(jsonNode.has("dateTimeFieldSpecs"),
+ assertFalse(jsonNode.has("dateTimeFieldSpecs"),
"Empty dateTimeFieldSpecs should be omitted");
- Assert.assertFalse(jsonNode.has("complexFieldSpecs"),
+ assertFalse(jsonNode.has("complexFieldSpecs"),
"Empty complexFieldSpecs should be omitted");
- Assert.assertFalse(jsonNode.has("timeFieldSpec"),
+ assertFalse(jsonNode.has("timeFieldSpec"),
"Null timeFieldSpec should be omitted");
- Assert.assertFalse(jsonNode.has("primaryKeyColumns"),
+ assertFalse(jsonNode.has("primaryKeyColumns"),
"Empty primaryKeyColumns should be omitted");
}
@@ -187,28 +194,28 @@ public class SchemaSerializationTest {
// Test with default value (false)
final Schema schemaWithDefault = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("dim1", DataType.STRING)
.build();
final String jsonStringDefault =
JsonUtils.objectToString(schemaWithDefault);
final JsonNode jsonNodeDefault =
JsonUtils.stringToJsonNode(jsonStringDefault);
- Assert.assertTrue(jsonNodeDefault.has("enableColumnBasedNullHandling"),
+ assertTrue(jsonNodeDefault.has("enableColumnBasedNullHandling"),
"enableColumnBasedNullHandling should always be present");
-
Assert.assertFalse(jsonNodeDefault.get("enableColumnBasedNullHandling").asBoolean());
+
assertFalse(jsonNodeDefault.get("enableColumnBasedNullHandling").asBoolean());
// Test with non-default value (true)
final Schema schemaWithEnabled = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
.setEnableColumnBasedNullHandling(true)
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
+ .addSingleValueDimension("dim1", DataType.STRING)
.build();
final String jsonStringEnabled =
JsonUtils.objectToString(schemaWithEnabled);
final JsonNode jsonNodeEnabled =
JsonUtils.stringToJsonNode(jsonStringEnabled);
- Assert.assertTrue(jsonNodeEnabled.has("enableColumnBasedNullHandling"));
-
Assert.assertTrue(jsonNodeEnabled.get("enableColumnBasedNullHandling").asBoolean());
+ assertTrue(jsonNodeEnabled.has("enableColumnBasedNullHandling"));
+
assertTrue(jsonNodeEnabled.get("enableColumnBasedNullHandling").asBoolean());
}
/**
@@ -217,7 +224,7 @@ public class SchemaSerializationTest {
@Test
public void testJsonValueSerializationWithComplexFieldSpecMap()
throws Exception {
- final ComplexFieldSpec mapField = new ComplexFieldSpec("mapField",
FieldSpec.DataType.MAP, true, Map.of());
+ final ComplexFieldSpec mapField = new ComplexFieldSpec("mapField",
DataType.MAP, true, Map.of());
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
@@ -228,23 +235,23 @@ public class SchemaSerializationTest {
final JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
// Should have complexFieldSpecs
- Assert.assertTrue(jsonNode.has("complexFieldSpecs"));
+ assertTrue(jsonNode.has("complexFieldSpecs"));
final JsonNode complexSpecs = jsonNode.get("complexFieldSpecs");
- Assert.assertEquals(complexSpecs.size(), 1);
+ assertEquals(complexSpecs.size(), 1);
final JsonNode mapSpec = complexSpecs.get(0);
- Assert.assertEquals(mapSpec.get("name").asText(), "mapField");
- Assert.assertEquals(mapSpec.get("dataType").asText(), "MAP");
- Assert.assertEquals(mapSpec.get("fieldType").asText(), "COMPLEX");
+ assertEquals(mapSpec.get("name").asText(), "mapField");
+ assertEquals(mapSpec.get("dataType").asText(), "MAP");
+ assertEquals(mapSpec.get("fieldType").asText(), "COMPLEX");
// defaultNullValue should be omitted since empty Map is the default
- Assert.assertFalse(mapSpec.has("defaultNullValue"),
+ assertFalse(mapSpec.has("defaultNullValue"),
"Empty Map default should not be serialized");
// Verify round-trip
final Schema deserializedSchema = Schema.fromString(jsonString);
- Assert.assertNotNull(deserializedSchema.getFieldSpecFor("mapField"));
-
Assert.assertEquals(deserializedSchema.getFieldSpecFor("mapField").getDataType(),
FieldSpec.DataType.MAP);
+ assertNotNull(deserializedSchema.getFieldSpecFor("mapField"));
+ assertEquals(deserializedSchema.getFieldSpecFor("mapField").getDataType(),
DataType.MAP);
}
/**
@@ -253,7 +260,7 @@ public class SchemaSerializationTest {
@Test
public void testJsonValueSerializationWithComplexFieldSpecList()
throws Exception {
- final ComplexFieldSpec listField = new ComplexFieldSpec("listField",
FieldSpec.DataType.LIST, true, Map.of());
+ final ComplexFieldSpec listField = new ComplexFieldSpec("listField",
DataType.LIST, true, Map.of());
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
@@ -264,18 +271,18 @@ public class SchemaSerializationTest {
final JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
// Should have complexFieldSpecs
- Assert.assertTrue(jsonNode.has("complexFieldSpecs"));
+ assertTrue(jsonNode.has("complexFieldSpecs"));
final JsonNode complexSpecs = jsonNode.get("complexFieldSpecs");
- Assert.assertEquals(complexSpecs.size(), 1);
+ assertEquals(complexSpecs.size(), 1);
final JsonNode listSpec = complexSpecs.get(0);
- Assert.assertEquals(listSpec.get("name").asText(), "listField");
- Assert.assertEquals(listSpec.get("dataType").asText(), "LIST");
+ assertEquals(listSpec.get("name").asText(), "listField");
+ assertEquals(listSpec.get("dataType").asText(), "LIST");
// Verify round-trip
final Schema deserializedSchema = Schema.fromString(jsonString);
- Assert.assertNotNull(deserializedSchema.getFieldSpecFor("listField"));
-
Assert.assertEquals(deserializedSchema.getFieldSpecFor("listField").getDataType(),
FieldSpec.DataType.LIST);
+ assertNotNull(deserializedSchema.getFieldSpecFor("listField"));
+
assertEquals(deserializedSchema.getFieldSpecFor("listField").getDataType(),
DataType.LIST);
}
/**
@@ -286,24 +293,24 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addDateTime("timestamp", FieldSpec.DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
+ .addDateTime("timestamp", DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:MILLISECONDS")
.build();
final String jsonString = JsonUtils.objectToString(schema);
final JsonNode jsonNode = JsonUtils.stringToJsonNode(jsonString);
- Assert.assertTrue(jsonNode.has("dateTimeFieldSpecs"));
+ assertTrue(jsonNode.has("dateTimeFieldSpecs"));
final JsonNode dateTimeSpecs = jsonNode.get("dateTimeFieldSpecs");
- Assert.assertEquals(dateTimeSpecs.size(), 1);
+ assertEquals(dateTimeSpecs.size(), 1);
final JsonNode dtSpec = dateTimeSpecs.get(0);
- Assert.assertEquals(dtSpec.get("name").asText(), "timestamp");
- Assert.assertEquals(dtSpec.get("dataType").asText(), "LONG");
- Assert.assertEquals(dtSpec.get("format").asText(), "1:MILLISECONDS:EPOCH");
- Assert.assertEquals(dtSpec.get("granularity").asText(), "1:MILLISECONDS");
+ assertEquals(dtSpec.get("name").asText(), "timestamp");
+ assertEquals(dtSpec.get("dataType").asText(), "LONG");
+ assertEquals(dtSpec.get("format").asText(), "1:MILLISECONDS:EPOCH");
+ assertEquals(dtSpec.get("granularity").asText(), "1:MILLISECONDS");
// defaultNullValue should be omitted since Long.MIN_VALUE is the default
for DATE_TIME LONG
- Assert.assertFalse(dtSpec.has("defaultNullValue"),
+ assertFalse(dtSpec.has("defaultNullValue"),
"Default null value should not be serialized for DATE_TIME LONG");
}
@@ -316,11 +323,11 @@ public class SchemaSerializationTest {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
.setEnableColumnBasedNullHandling(true)
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
- .addSingleValueDimension("dim2", FieldSpec.DataType.INT, 42)
- .addMultiValueDimension("mvDim", FieldSpec.DataType.DOUBLE)
- .addMetric("metric1", FieldSpec.DataType.LONG)
- .addDateTime("ts", FieldSpec.DataType.LONG, "1:HOURS:EPOCH", "1:HOURS")
+ .addSingleValueDimension("dim1", DataType.STRING)
+ .addSingleValueDimension("dim2", DataType.INT, 42)
+ .addMultiValueDimension("mvDim", DataType.DOUBLE)
+ .addMetric("metric1", DataType.LONG)
+ .addDateTime("ts", DataType.LONG, "1:HOURS:EPOCH", "1:HOURS")
.setPrimaryKeyColumns(Lists.newArrayList("dim1"))
.build();
@@ -332,7 +339,7 @@ public class SchemaSerializationTest {
final JsonNode toJsonObjectNode = schema.toJsonObject();
// They should be equal
- Assert.assertEquals(jacksonNode, toJsonObjectNode,
+ assertEquals(jacksonNode, toJsonObjectNode,
"Jackson serialization should match toJsonObject() output");
}
@@ -345,13 +352,13 @@ public class SchemaSerializationTest {
final Schema originalSchema = new Schema.SchemaBuilder()
.setSchemaName("complexTestSchema")
.setEnableColumnBasedNullHandling(true)
- .addSingleValueDimension("stringDim", FieldSpec.DataType.STRING)
- .addSingleValueDimension("intDimWithDefault", FieldSpec.DataType.INT,
100)
- .addMultiValueDimension("mvStringDim", FieldSpec.DataType.STRING,
"default")
- .addMetric("longMetric", FieldSpec.DataType.LONG)
- .addMetric("doubleMetricWithDefault", FieldSpec.DataType.DOUBLE, 3.14)
- .addDateTime("eventTime", FieldSpec.DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
- .addDateTime("dayTime", FieldSpec.DataType.STRING,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd", "1:DAYS")
+ .addSingleValueDimension("stringDim", DataType.STRING)
+ .addSingleValueDimension("intDimWithDefault", DataType.INT, 100)
+ .addMultiValueDimension("mvStringDim", DataType.STRING, "default")
+ .addMetric("longMetric", DataType.LONG)
+ .addMetric("doubleMetricWithDefault", DataType.DOUBLE, 3.14)
+ .addDateTime("eventTime", DataType.LONG, "1:MILLISECONDS:EPOCH",
"1:MILLISECONDS")
+ .addDateTime("dayTime", DataType.STRING,
"1:DAYS:SIMPLE_DATE_FORMAT:yyyy-MM-dd", "1:DAYS")
.setPrimaryKeyColumns(Lists.newArrayList("stringDim", "eventTime"))
.build();
@@ -362,24 +369,24 @@ public class SchemaSerializationTest {
final Schema deserializedSchema = Schema.fromString(jsonString);
// Verify all fields
- Assert.assertEquals(deserializedSchema.getSchemaName(),
"complexTestSchema");
- Assert.assertTrue(deserializedSchema.isEnableColumnBasedNullHandling());
+ assertEquals(deserializedSchema.getSchemaName(), "complexTestSchema");
+ assertTrue(deserializedSchema.isEnableColumnBasedNullHandling());
// Verify dimensions
- Assert.assertNotNull(deserializedSchema.getDimensionSpec("stringDim"));
-
Assert.assertEquals(deserializedSchema.getDimensionSpec("intDimWithDefault").getDefaultNullValue(),
100);
-
Assert.assertFalse(deserializedSchema.getDimensionSpec("mvStringDim").isSingleValueField());
+ assertNotNull(deserializedSchema.getDimensionSpec("stringDim"));
+
assertEquals(deserializedSchema.getDimensionSpec("intDimWithDefault").getDefaultNullValue(),
100);
+
assertFalse(deserializedSchema.getDimensionSpec("mvStringDim").isSingleValueField());
// Verify metrics
- Assert.assertNotNull(deserializedSchema.getMetricSpec("longMetric"));
-
Assert.assertEquals(deserializedSchema.getMetricSpec("doubleMetricWithDefault").getDefaultNullValue(),
3.14);
+ assertNotNull(deserializedSchema.getMetricSpec("longMetric"));
+
assertEquals(deserializedSchema.getMetricSpec("doubleMetricWithDefault").getDefaultNullValue(),
3.14);
// Verify date time
- Assert.assertNotNull(deserializedSchema.getDateTimeSpec("eventTime"));
-
Assert.assertEquals(deserializedSchema.getDateTimeSpec("eventTime").getFormat(),
"1:MILLISECONDS:EPOCH");
+ assertNotNull(deserializedSchema.getDateTimeSpec("eventTime"));
+ assertEquals(deserializedSchema.getDateTimeSpec("eventTime").getFormat(),
"1:MILLISECONDS:EPOCH");
// Verify primary keys
- Assert.assertEquals(deserializedSchema.getPrimaryKeyColumns(),
Lists.newArrayList("stringDim", "eventTime"));
+ assertEquals(deserializedSchema.getPrimaryKeyColumns(),
Lists.newArrayList("stringDim", "eventTime"));
}
/**
@@ -391,8 +398,8 @@ public class SchemaSerializationTest {
throws Exception {
final Schema schema = new Schema.SchemaBuilder()
.setSchemaName("testSchema")
- .addSingleValueDimension("dim1", FieldSpec.DataType.STRING)
- .addMetric("metric1", FieldSpec.DataType.LONG)
+ .addSingleValueDimension("dim1", DataType.STRING)
+ .addMetric("metric1", DataType.LONG)
.build();
// Use a fresh ObjectMapper (simulates different Jackson configurations)
@@ -400,11 +407,169 @@ public class SchemaSerializationTest {
final String freshMapperJson = freshMapper.writeValueAsString(schema);
// Should still produce toJsonObject() format
- Assert.assertFalse(freshMapperJson.contains("defaultNullValueString"),
+ assertFalse(freshMapperJson.contains("defaultNullValueString"),
"Fresh ObjectMapper should also use @JsonValue and omit
defaultNullValueString");
// Should be deserializable
final Schema deserializedSchema = Schema.fromString(freshMapperJson);
- Assert.assertEquals(deserializedSchema.getSchemaName(), "testSchema");
+ assertEquals(deserializedSchema.getSchemaName(), "testSchema");
+ }
+
+ @Test
+ public void testComplexFieldDefaultNullValue()
+ throws Exception {
+ // Test LIST
+ ComplexFieldSpec listFieldSpec = new ComplexFieldSpec("list",
DataType.LIST, true, Map.of());
+
+ // Test no defaultNullValue
+ Object defaultNullValue = listFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof List);
+ assertEquals(defaultNullValue, List.of());
+ ObjectNode jsonObject = listFieldSpec.toJsonObject();
+ assertFalse(jsonObject.has("defaultNullValue"));
+ ComplexFieldSpec deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ String serialized = jsonObject.toString();
+ assertFalse(serialized.contains("defaultNullValue"));
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test null defaultNullValue
+ serialized = "{"
+ + "\"name\":\"list\","
+ + "\"dataType\":\"LIST\","
+ + "\"fieldType\":\"COMPLEX\","
+ + "\"defaultNullValue\":null,"
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test numeric
+ listFieldSpec.setDefaultNullValue(List.of(1, 2, 3));
+ defaultNullValue = listFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof List);
+ assertEquals(defaultNullValue, List.of(1, 2, 3));
+ jsonObject = listFieldSpec.toJsonObject();
+ JsonNode defaultNullValueNode = jsonObject.get("defaultNullValue");
+ assertTrue(defaultNullValueNode.isArray());
+ assertEquals(defaultNullValueNode.toString(), "[1,2,3]");
+ deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ serialized = jsonObject.toString();
+ assertTrue(serialized.contains("\"defaultNullValue\":[1,2,3]"));
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ // Test compatibility with serialized JSON ARRAY
+ serialized = "{"
+ + "\"name\":\"list\","
+ + "\"dataType\":\"LIST\","
+ + "\"fieldType\":\"COMPLEX\","
+ + "\"defaultNullValue\":\"[1,2,3]\","
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test text
+ listFieldSpec.setDefaultNullValue(List.of("a", "b", "c"));
+ defaultNullValue = listFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof List);
+ assertEquals(defaultNullValue, List.of("a", "b", "c"));
+ jsonObject = listFieldSpec.toJsonObject();
+ defaultNullValueNode = jsonObject.get("defaultNullValue");
+ assertTrue(defaultNullValueNode.isArray());
+ assertEquals(defaultNullValueNode.toString(), "[\"a\",\"b\",\"c\"]");
+ deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ serialized = jsonObject.toString();
+
assertTrue(serialized.contains("\"defaultNullValue\":[\"a\",\"b\",\"c\"]"));
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ // Test compatibility with serialized JSON ARRAY
+ serialized = "{"
+ + "\"name\":\"list\","
+ + "\"dataType\":\"LIST\","
+ + "\"fieldType\":\"COMPLEX\","
+ + "\"defaultNullValue\":\"[\\\"a\\\",\\\"b\\\",\\\"c\\\"]\","
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test MAP
+ ComplexFieldSpec mapFieldSpec = new ComplexFieldSpec("map", DataType.MAP,
true, Map.of());
+
+ // Test no defaultNullValue
+ defaultNullValue = mapFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof Map);
+ assertEquals(defaultNullValue, Map.of());
+ jsonObject = mapFieldSpec.toJsonObject();
+ assertFalse(jsonObject.has("defaultNullValue"));
+ deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ serialized = jsonObject.toString();
+ assertFalse(serialized.contains("defaultNullValue"));
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test null defaultNullValue
+ serialized = "{"
+ + "\"name\":\"map\","
+ + "\"dataType\":\"MAP\","
+ + "\"fieldType\":\"COMPLEX\","
+ + "\"defaultNullValue\":null,"
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test numeric
+ mapFieldSpec.setDefaultNullValue(Map.of("a", 1, "b", 2));
+ defaultNullValue = mapFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof Map);
+ assertEquals(defaultNullValue, Map.of("a", 1, "b", 2));
+ jsonObject = mapFieldSpec.toJsonObject();
+ defaultNullValueNode = jsonObject.get("defaultNullValue");
+ assertTrue(defaultNullValueNode.isObject());
+ deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ serialized = jsonObject.toString();
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ // Test compatibility with serialized JSON OBJECT
+ serialized = "{"
+ + "\"name\":\"map\","
+ + "\"dataType\":\"MAP\","
+ + "\"fieldType\":\"COMPLEX\","
+ + "\"defaultNullValue\":\"{\\\"a\\\":1,\\\"b\\\":2}\","
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+
+ // Test text
+ mapFieldSpec.setDefaultNullValue(Map.of("key", "a", "value", "b"));
+ defaultNullValue = mapFieldSpec.getDefaultNullValue();
+ assertTrue(defaultNullValue instanceof Map);
+ assertEquals(defaultNullValue, Map.of("key", "a", "value", "b"));
+ jsonObject = mapFieldSpec.toJsonObject();
+ defaultNullValueNode = jsonObject.get("defaultNullValue");
+ assertTrue(defaultNullValueNode.isObject());
+ deserialized = JsonUtils.jsonNodeToObject(jsonObject,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ serialized = jsonObject.toString();
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
+ // Test compatibility with serialized JSON OBJECT
+ serialized = "{"
+ + "\"name\":\"map\","
+ + "\"dataType\":\"MAP\","
+ + "\"fieldType\":\"COMPLEX\","
+ +
"\"defaultNullValue\":\"{\\\"key\\\":\\\"a\\\",\\\"value\\\":\\\"b\\\"}\","
+ + "\"childFieldSpecs\":{}"
+ + "}";
+ deserialized = JsonUtils.stringToObject(serialized,
ComplexFieldSpec.class);
+ assertEquals(deserialized.getDefaultNullValue(), defaultNullValue);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
