This is an automated email from the ASF dual-hosted git repository.
abhishekrb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a44006c9983 Fix decimal type support for the delta input format.
(#17376)
a44006c9983 is described below
commit a44006c99832ed59c714604bdabe33711cf4c4b1
Author: Abhishek Radhakrishnan <[email protected]>
AuthorDate: Fri Oct 18 08:09:35 2024 -0700
Fix decimal type support for the delta input format. (#17376)
The Delta Decimal type wasn't handled correctly in the Druid Delta
connector, resulting in the error: Unsupported fieldType[Decimal(4, 2)] for
fieldName[price].
There were no tests or existing tables with the Decimal type, so I've
updated the existing table, complex-types-table to include this data type.
Note that the Decimal type can only be handled as a double at most in
Druid. For a big decimal that cannot fit inside a double, it should be ingested
as a string.
---
.../java/org/apache/druid/delta/input/DeltaInputRow.java | 2 +-
.../main/java/org/apache/druid/delta/input/RowSerde.java | 3 +++
.../apache/druid/delta/input/ComplexTypesDeltaTable.java | 11 ++++++-----
...3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc | Bin 0 -> 24 bytes
...8-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc | Bin 20 -> 0 bytes
...8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc | Bin 36 -> 0 bytes
...d-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
...0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
...7-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc | Bin 36 -> 0 bytes
...e-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
...5-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc | Bin 36 -> 0 bytes
...7-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc | Bin 36 -> 0 bytes
...d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
...6-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc | Bin 36 -> 0 bytes
...6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc | Bin 0 -> 40 bytes
.../_delta_log/.00000000000000000000.json.crc | Bin 52 -> 56 bytes
.../_delta_log/00000000000000000000.json | 14 +++++++-------
...51ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet | Bin 0 -> 1604 bytes
...53008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet | Bin 1488 -> 0 bytes
...fecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet | Bin 3288 -> 0 bytes
...d58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet | Bin 0 -> 3588 bytes
...9e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet | Bin 0 -> 3591 bytes
...f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet | Bin 3291 -> 0 bytes
...1a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet | Bin 0 -> 3589 bytes
...ee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet | Bin 3289 -> 0 bytes
...88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet | Bin 3290 -> 0 bytes
...86f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet | Bin 0 -> 3590 bytes
...60316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet | Bin 3291 -> 0 bytes
...abfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet | Bin 0 -> 3591 bytes
.../src/test/resources/create_delta_table.py | 7 ++++---
30 files changed, 21 insertions(+), 16 deletions(-)
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
index 442412dd154..a8cd0d5020e 100644
---
a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
+++
b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
@@ -201,7 +201,7 @@ public class DeltaInputRow implements InputRow
}
return String.valueOf(charArray);
} else if (dataType instanceof DecimalType) {
- return dataRow.getDecimal(columnOrdinal).longValue();
+ return dataRow.getDecimal(columnOrdinal);
} else if (dataType instanceof StructType) {
final io.delta.kernel.data.Row structRow =
dataRow.getStruct(columnOrdinal);
return RowSerde.convertRowToJsonObject(structRow);
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java
b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java
index bad6191496d..d7c6fcccdba 100644
---
a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java
+++
b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java
@@ -32,6 +32,7 @@ import io.delta.kernel.types.BooleanType;
import io.delta.kernel.types.ByteType;
import io.delta.kernel.types.DataType;
import io.delta.kernel.types.DateType;
+import io.delta.kernel.types.DecimalType;
import io.delta.kernel.types.DoubleType;
import io.delta.kernel.types.FloatType;
import io.delta.kernel.types.IntegerType;
@@ -126,6 +127,8 @@ public class RowSerde
value = row.getFloat(fieldId);
} else if (fieldType instanceof DoubleType) {
value = row.getDouble(fieldId);
+ } else if (fieldType instanceof DecimalType) {
+ value = row.getDecimal(fieldId);
} else if (fieldType instanceof DateType) {
value = DeltaTimeUtils.getSecondsFromDate(row.getInt(fieldId));
} else if (fieldType instanceof TimestampType) {
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
index 7fdffc03041..81edba92777 100644
---
a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
+++
b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java
@@ -28,6 +28,7 @@ import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.segment.AutoTypeColumnSchema;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -66,35 +67,35 @@ public class ComplexTypesDeltaTable
"id", 0L,
"array_info", ImmutableList.of(0, 1, 2, 3),
"struct_info", ImmutableMap.of("id", 0L, "name", "0"),
- "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0",
"nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0)),
+ "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0",
"nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0,
"nested_decimal", BigDecimal.valueOf(0.23))),
"map_info", ImmutableMap.of("key1", 1.0f, "key2", 1.0f)
),
ImmutableMap.of(
"id", 1L,
"array_info", ImmutableList.of(1, 2, 3, 4),
"struct_info", ImmutableMap.of("id", 1L, "name", "1"),
- "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1",
"nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0)),
+ "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1",
"nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0,
"nested_decimal", BigDecimal.valueOf(1.23))),
"map_info", ImmutableMap.of("key1", 2.0f, "key2", 2.0f)
),
ImmutableMap.of(
"id", 2L,
"array_info", ImmutableList.of(2, 3, 4, 5),
"struct_info", ImmutableMap.of("id", 2L, "name", "2"),
- "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2",
"nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0)),
+ "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2",
"nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0,
"nested_decimal", BigDecimal.valueOf(2.23))),
"map_info", ImmutableMap.of("key1", 3.0f, "key2", 3.0f)
),
ImmutableMap.of(
"id", 3L,
"array_info", ImmutableList.of(3, 4, 5, 6),
"struct_info", ImmutableMap.of("id", 3L, "name", "3"),
- "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3",
"nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0)),
+ "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3",
"nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0,
"nested_decimal", BigDecimal.valueOf(3.23))),
"map_info", ImmutableMap.of("key1", 4.0f, "key2", 4.0f)
),
ImmutableMap.of(
"id", 4L,
"array_info", ImmutableList.of(4, 5, 6, 7),
"struct_info", ImmutableMap.of("id", 4L, "name", "4"),
- "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4",
"nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0)),
+ "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4",
"nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0,
"nested_decimal", BigDecimal.valueOf(4.23))),
"map_info", ImmutableMap.of("key1", 5.0f, "key2", 5.0f)
)
)
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..92c3e2d8fba
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc
deleted file mode 100644
index a56f68f447b..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc
deleted file mode 100644
index 6b7e86bcf54..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..7bba84c69e6
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..4b77b095336
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc
deleted file mode 100644
index 88b089e9503..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..e669a893a30
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc
deleted file mode 100644
index 7f497252005..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc
deleted file mode 100644
index cd8fc7a087f..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..d464223f06f
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc
deleted file mode 100644
index 038082a933b..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc
new file mode 100644
index 00000000000..92345f4f98a
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
index 311d2a22b04..fca90d03dc1 100644
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
index 84803f9483c..5383b51e0b7 100644
---
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
+++
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json
@@ -1,8 +1,8 @@
-{"commitInfo":{"timestamp":1723511561738,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"17937"},"engineInfo":"Apache-Spark/3.5.0
Delta-Lake/3.1.0","txnId":"b9eae5f4-d55b-4c38-b365-8228ec09248e"}}
-{"metaData":{"id":"ce998219-9bde-4831-b78c-14b11f919fbe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata
[...]
+{"commitInfo":{"timestamp":1729202194177,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"19553"},"engineInfo":"Apache-Spark/3.5.0
Delta-Lake/3.1.0","txnId":"ff724e89-210e-47dd-a31f-edec6f2c6f6f"}}
+{"metaData":{"id":"68614ef0-e5ca-4caa-bbd9-b3427eecadc9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata
[...]
{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}
-{"add":{"path":"part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet","partitionValues":{},"size":3288,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"
[...]
-{"add":{"path":"part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"
[...]
-{"add":{"path":"part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet","partitionValues":{},"size":3289,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"
[...]
-{"add":{"path":"part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet","partitionValues":{},"size":3290,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"
[...]
-{"add":{"path":"part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"
[...]
+{"add":{"path":"part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet","partitionValues":{},"size":3588,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0,\"nested_decimal\":0.23}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"nam
[...]
+{"add":{"path":"part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0,\"nested_decimal\":1.23}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"nam
[...]
+{"add":{"path":"part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet","partitionValues":{},"size":3589,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0,\"nested_decimal\":2.23}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"nam
[...]
+{"add":{"path":"part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet","partitionValues":{},"size":3590,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0,\"nested_decimal\":3.23}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"nam
[...]
+{"add":{"path":"part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0,\"nested_decimal\":4.23}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"nam
[...]
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet
new file mode 100644
index 00000000000..fdd3237e37c
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet
deleted file mode 100644
index 7a105c0f74f..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet
deleted file mode 100644
index bb2fb67389d..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet
new file mode 100644
index 00000000000..8b6fdcfab64
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet
new file mode 100644
index 00000000000..c349328307e
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet
deleted file mode 100644
index 641396cb6f7..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet
new file mode 100644
index 00000000000..dcfcdd89bc0
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet
deleted file mode 100644
index abee0ea5d0a..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet
deleted file mode 100644
index 453c3879fa3..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet
new file mode 100644
index 00000000000..ab5b52c9091
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet
deleted file mode 100644
index aadf1e152c8..00000000000
Binary files
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet
and /dev/null differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet
new file mode 100644
index 00000000000..507b56bbfc1
Binary files /dev/null and
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet
differ
diff --git
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
index ffca055e73c..52cd3ed3de5 100755
---
a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
+++
b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
@@ -17,10 +17,10 @@
import argparse
from enum import Enum
-
+from decimal import Decimal
from delta import *
import pyspark
-from pyspark.sql.types import MapType, StructType, StructField, ShortType,
StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType,
DateType, BooleanType, ArrayType
+from pyspark.sql.types import MapType, StructType, StructField, ShortType,
StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType,
DateType, BooleanType, ArrayType, DecimalType
from pyspark.sql.functions import expr
from datetime import datetime, timedelta
import random
@@ -73,6 +73,7 @@ def create_dataset_with_complex_types(num_records):
StructField("nested", StructType([
StructField("nested_int", IntegerType(), False),
StructField("nested_double", DoubleType(), True),
+ StructField("nested_decimal", DecimalType(4, 2), True),
]))
])),
StructField("map_info", MapType(StringType(), FloatType()))
@@ -85,7 +86,7 @@ def create_dataset_with_complex_types(num_records):
idx,
(idx, idx + 1, idx + 2, idx + 3),
(idx, f"{idx}"),
- (idx, f"{idx}", (idx, idx + 1.0)),
+ (idx, f"{idx}", (idx, idx + 1.0, Decimal(idx + 0.23))),
{"key1": idx + 1.0, "key2": idx + 1.0}
)
data.append(record)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
