voonhous commented on code in PR #17769:
URL: https://github.com/apache/hudi/pull/17769#discussion_r2663604776
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/TestHoodieSparkLanceReader.java:
##########
@@ -143,6 +150,141 @@ public void testReadAllSupportedTypes() throws Exception {
}
}
+ @Test
+ public void testReadDecimalType() throws Exception {
+ // Create schema with decimal types of different precision and scale
+ StructType schema = new StructType()
+ .add("id", DataTypes.IntegerType, false)
+ .add("decimal_small", DataTypes.createDecimalType(10, 2), false)
+ .add("decimal_large", DataTypes.createDecimalType(20, 5), false)
+ .add("decimal_zero_scale", DataTypes.createDecimalType(10, 0), false);
+
+ // Create test data with 3 records
+ List<InternalRow> expectedRows = new ArrayList<>();
+ expectedRows.add(createRow(1, Decimal.apply(new BigDecimal("999.99"), 10,
2),
+ Decimal.apply(new BigDecimal("123456789.12345"), 20, 5),
Decimal.apply(12345L, 10, 0)));
+ expectedRows.add(createRow(2, Decimal.apply(new BigDecimal("-123.45"), 10,
2),
+ Decimal.apply(new BigDecimal("-987654.54321"), 20, 5),
Decimal.apply(-5000L, 10, 0)));
+ expectedRows.add(createRow(3, Decimal.apply(0L, 10, 2),
+ Decimal.apply(0L, 20, 5), Decimal.apply(0L, 10, 0)));
+
+ // Write and read back
+ StoragePath path = new StoragePath(tempDir.getAbsolutePath() +
"/test_decimal.lance");
+ try (HoodieSparkLanceReader reader = writeAndCreateReader(path, schema,
expectedRows)) {
+ assertNotNull(reader.getSchema(), "Schema should not be null");
+ List<InternalRow> actualRows = readAllRows(reader, schema);
+ assertEquals(expectedRows, actualRows);
+ }
+ }
+
+ @Test
+ public void testReadTimestampAndDateTypes() throws Exception {
+ // Create schema with timestamp and date types
+ StructType schema = new StructType()
+ .add("id", DataTypes.IntegerType, false)
+ .add("timestamp_field", DataTypes.TimestampType, false)
+ .add("date_field", DataTypes.DateType, false);
+
+ // Create test data with 3 records
+ // Timestamps are microseconds since Unix epoch, dates are days since Unix
epoch
+ List<InternalRow> expectedRows = new ArrayList<>();
+ expectedRows.add(createRow(1, 1609459200000000L, 18628)); // 2021-01-01
+ expectedRows.add(createRow(2, 1640995200000000L, 18993)); // 2022-01-01
+ expectedRows.add(createRow(3, 0L, 0)); // 1970-01-01 (epoch)
+
+ // Write and read back
+ StoragePath path = new StoragePath(tempDir.getAbsolutePath() +
"/test_timestamp_date.lance");
+ try (HoodieSparkLanceReader reader = writeAndCreateReader(path, schema,
expectedRows)) {
+ assertNotNull(reader.getSchema(), "Schema should not be null");
+ List<InternalRow> actualRows = readAllRows(reader, schema);
+ assertEquals(expectedRows, actualRows);
+ }
+ }
+
+ @Test
+ public void testReadArrayType() throws Exception {
+ // Create schema with array types
+ StructType schema = new StructType()
+ .add("id", DataTypes.IntegerType, false)
+ .add("int_array", DataTypes.createArrayType(DataTypes.IntegerType,
false), false)
+ .add("string_array", DataTypes.createArrayType(DataTypes.StringType,
false), false);
+
+ // Create test data with 3 records
+ List<InternalRow> expectedRows = new ArrayList<>();
+ expectedRows.add(createRow(1, new Object[]{}, new Object[]{})); // Empty
arrays
+ expectedRows.add(createRow(2, new Object[]{42}, new Object[]{"Alice"}));
// Single element
+ expectedRows.add(createRow(3, new Object[]{1, 2, 3, 4, 5}, new
Object[]{"Bob", "Charlie", "David"})); // Multi-element
Review Comment:
Let's add an extra test case to check if we can read null data types
correctly as Spark's `ArrayData` supports null elements. It will be good to
verify that `Lance` preservers them.
```java
expectedRows.add(createRow(4, new Object[]{1, null, 3}, new Object[]{"A",
null, "C"}));
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
