voonhous commented on code in PR #19809:
URL: https://github.com/apache/hudi/pull/19809#discussion_r3916155984
##########
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieInternalRowUtils.scala:
##########
@@ -123,7 +123,7 @@ object HoodieInternalRowUtils {
/**
* Get or create [[StructType]] for provided [[Schema]]
- * @param schema [[Schema]] to convert to [[StructType]], NOTE: It is best
that the schema passed in is cached through
[[org.apache.hudi.common.avro.AvroSchemaCache]], so that we can reduce the
overhead of schema lookup in the map
+ * @param schema [[Schema]] to convert to [[StructType]], NOTE: It is best
that the schema passed in is cached through
[[org.apache.hudi.common.schema.HoodieAvroSchemaCache]], so that we can reduce
the overhead of schema lookup in the map
* @return [[StructType]] for provided [[Schema]]
Review Comment:
Done in d74b3cd4f3b6.
##########
hudi-common/src/test/java/org/apache/hudi/common/avro/TestHoodieAvroUtils.java:
##########
@@ -566,6 +567,129 @@ public void
testConvertValueForAvroLogicalTypesCrossAvroVersion() {
assertEquals(FIXTURE_EPOCH_MICROS,
HoodieAvroUtils.convertValueForAvroLogicalTypes(LOCAL_TS_MICROS_SCHEMA,
FIXTURE_LOCAL_DT_MICROS, false));
}
+ @Test
+ public void testConvertValueForSpecificDataTypes_NullSchema() {
+ // Test with null schema - should return value unchanged
+ String testValue = "test_value";
+ Object result = HoodieAvroUtils.convertValueForSpecificDataTypes(null,
testValue, false);
+ assertEquals(testValue, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_NullValue_NullableSchema() {
+ // Test with null value and nullable schema - should return null
+ Schema nullableIntSchema =
HoodieSchema.createNullable(HoodieSchema.create(HoodieSchemaType.INT)).toAvroSchema();
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(nullableIntSchema, null,
false);
+ assertNull(result);
+ }
+
+ @Test
+ public void
testConvertValueForSpecificDataTypes_NullValue_NonNullableSchema() {
+ // Test with null value and non-nullable schema - should throw exception
+ Schema nonNullableSchema = Schema.create(Schema.Type.STRING);
+ assertThrows(IllegalStateException.class, () ->
+ HoodieAvroUtils.convertValueForSpecificDataTypes(nonNullableSchema,
null, false));
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_DateLogicalType() {
+ // Create date schema
+ Schema dateSchema = HoodieSchema.createDate().toAvroSchema();
+
+ // Test value: epoch days for 2023-01-01
+ int epochDays = 19358;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(dateSchema, epochDays, false);
+ assertNotNull(result);
+ assertTrue(result instanceof LocalDate);
+ assertEquals(LocalDate.of(2023, 1, 1), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMillis_Enabled() {
+ // Create timestamp-millis schema
+ Schema timestampMillisSchema =
HoodieSchema.createTimestampMillis().toAvroSchema();
+
+ // Test value: milliseconds for 2023-01-01 00:00:00
+ long millis = 1672560000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(timestampMillisSchema, millis,
true);
+ assertNotNull(result);
+ assertTrue(result instanceof Timestamp);
+ assertEquals(new Timestamp(millis), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMillis_Disabled() {
+ // Create timestamp-millis schema
+ Schema timestampMillisSchema =
HoodieSchema.createTimestampMillis().toAvroSchema();
+ long millis = 1672560000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(timestampMillisSchema, millis,
false);
+ assertEquals(millis, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_TimestampMicros_Enabled() {
+ // Create timestamp-micros schema
+ Schema timestampMicrosSchema =
HoodieSchema.createTimestampMicros().toAvroSchema();
+
+ // Test value: microseconds for 2023-01-01 00:00:00
+ long micros = 1672560000000000L;
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(timestampMicrosSchema, micros,
true);
+ assertNotNull(result);
+ assertTrue(result instanceof Timestamp);
+ assertEquals(new Timestamp(micros / 1000), result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_DecimalBytes() {
+ // Create decimal schema with precision=10, scale=2
+ Schema decimalSchema = HoodieSchema.createDecimal(10, 2).toAvroSchema();
+
+ // Create test value: 1234.56
+ BigDecimal expectedDecimal = new BigDecimal("1234.56");
+ ByteBuffer byteBuffer =
ByteBuffer.wrap(expectedDecimal.unscaledValue().toByteArray());
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(decimalSchema, byteBuffer,
false);
+ assertNotNull(result);
+ assertTrue(result instanceof BigDecimal);
+ assertEquals(expectedDecimal, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_NonLogicalType() {
+ // Test with non-logical type (plain string) - should return unchanged
+ Schema stringSchema = Schema.create(Schema.Type.STRING);
+ String testValue = "test_string";
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(stringSchema, testValue,
false);
+ assertEquals(testValue, result);
+ }
+
+ @Test
+ public void testConvertValueForSpecificDataTypes_UnionWithNull() {
+ // Test with union type containing null
+ Schema nullableDateSchema =
HoodieSchema.createNullable(HoodieSchema.createDate()).toAvroSchema();
+
+ // Test with non-null value
+ int epochDays = 19358; // 2023-01-01
+ Object result =
HoodieAvroUtils.convertValueForSpecificDataTypes(nullableDateSchema, epochDays,
false);
+ assertNotNull(result);
+ assertTrue(result instanceof LocalDate);
+ assertEquals(LocalDate.of(2023, 1, 1), result);
+ }
+
+ @Test
+ public void testConvertBytesToBigDecimalWithHoodieSchema() {
+ HoodieSchema decimalSchema = HoodieSchema.createDecimal(10, 2);
+ BigDecimal expected = new BigDecimal("1234.56");
+ assertEquals(expected,
+
HoodieAvroUtils.convertBytesToBigDecimal(expected.unscaledValue().toByteArray(),
decimalSchema));
+ }
+
+ @Test
+ public void testConvertBytesToBigDecimalWithNonDecimalHoodieSchema() {
+ HoodieSchema stringSchema = HoodieSchema.create(HoodieSchemaType.STRING);
+ assertThrows(IllegalArgumentException.class, () ->
+ HoodieAvroUtils.convertBytesToBigDecimal(new byte[] {0x01},
stringSchema));
+ }
Review Comment:
Done in 2abcfe1e3a9c.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
