yihua commented on code in PR #17573:
URL: https://github.com/apache/hudi/pull/17573#discussion_r2628659457
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBucketIndexSupport.scala:
##########
@@ -66,7 +66,7 @@ class TestBucketIndexSupport extends
HoodieSparkClientTestBase with PredicateHel
)
)
- val avroSchema = new Schema.Parser().parse(avroSchemaStr)
+ val hoodieSchema = HoodieSchema.parse(avroSchemaStr)
Review Comment:
nit: should we name this as `schema` for brevity?
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala:
##########
@@ -625,14 +624,14 @@ class TestDataSourceDefaults extends
ScalaAssertionSupport {
val props = new TypedProperties()
OrderingFieldsTestUtils.setOrderingFieldsConfig(props, key,
"favoriteIntNumber")
val baseOrderingVal: Object = baseRecord.get("favoriteIntNumber")
- val fieldSchema: Schema =
baseRecord.getSchema().getField("favoriteIntNumber").schema()
+ val fieldSchema: HoodieSchema =
HoodieSchema.fromAvroSchema(baseRecord.getSchema().getField("favoriteIntNumber").schema())
- val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema, baseOrderingVal,
false).asInstanceOf[Comparable[_]])
+ val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
org.apache.hudi.common.schema.HoodieSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
baseOrderingVal, false).asInstanceOf[Comparable[_]])
val laterRecord = SchemaTestUtil
.generateAvroRecordFromJson(schema, 2, "001", "f1")
val laterOrderingVal: Object = laterRecord.get("favoriteIntNumber")
- val newerPayload = new OverwriteWithLatestAvroPayload(laterRecord,
HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema, laterOrderingVal,
false).asInstanceOf[Comparable[_]])
+ val newerPayload = new OverwriteWithLatestAvroPayload(laterRecord,
org.apache.hudi.common.schema.HoodieSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
laterOrderingVal, false).asInstanceOf[Comparable[_]])
Review Comment:
Same here and other places
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionBucketIndexSupport.scala:
##########
@@ -67,19 +67,19 @@ class TestPartitionBucketIndexSupport extends
TestBucketIndexSupport {
// init
val testKeyGenerator = new NonpartitionedKeyGenerator(configProperties)
- var record = new GenericData.Record(avroSchema)
+ var record = new GenericData.Record(hoodieSchema.toAvroSchema)
Review Comment:
nit: same on `schema` instead of `hoodieSchema`
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestColumnStatsIndex.scala:
##########
@@ -1101,8 +1100,8 @@ class TestColumnStatsIndex extends
ColumnStatIndexTestBase {
def testDeserialize(description: String, expected: JBigDecimal,
wrapperCreator: WrapperCreator): Unit = {
val dt = DecimalType(10, 2)
// Get the schema from the DecimalWrapper's Avro definition.
- val schema: Schema = DecimalWrapper.SCHEMA$.getField("value").schema()
- val wrapper = wrapperCreator.create(expected, schema)
+ val schema: HoodieSchema =
HoodieSchema.fromAvroSchema(DecimalWrapper.SCHEMA$.getField("value").schema())
+ val wrapper = wrapperCreator.create(expected, schema.toAvroSchema)
Review Comment:
nit: these changes seem unnecessary as they convert the schema back and forth
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSourceDefaults.scala:
##########
@@ -625,14 +624,14 @@ class TestDataSourceDefaults extends
ScalaAssertionSupport {
val props = new TypedProperties()
OrderingFieldsTestUtils.setOrderingFieldsConfig(props, key,
"favoriteIntNumber")
val baseOrderingVal: Object = baseRecord.get("favoriteIntNumber")
- val fieldSchema: Schema =
baseRecord.getSchema().getField("favoriteIntNumber").schema()
+ val fieldSchema: HoodieSchema =
HoodieSchema.fromAvroSchema(baseRecord.getSchema().getField("favoriteIntNumber").schema())
- val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
HoodieAvroUtils.convertValueForSpecificDataTypes(fieldSchema, baseOrderingVal,
false).asInstanceOf[Comparable[_]])
+ val basePayload = new OverwriteWithLatestAvroPayload(baseRecord,
org.apache.hudi.common.schema.HoodieSchemaUtils.convertValueForSpecificDataTypes(fieldSchema,
baseOrderingVal, false).asInstanceOf[Comparable[_]])
Review Comment:
nit: no need of `org.apache.hudi.common.schema.` prefix
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestAvroConversionHelper.scala:
##########
@@ -41,11 +42,11 @@ class TestAvroConversionHelper extends FunSuite with
Matchers {
val dateInputData = Seq(7, 365, 0)
test("Logical type: date") {
- val schema = new Schema.Parser().parse(dateSchema)
- val convertor = AvroConversionUtils.createConverterToRow(schema,
AvroConversionUtils.convertAvroSchemaToStructType(schema))
+ val schema = HoodieSchema.parse(dateSchema)
+ val convertor =
AvroConversionUtils.createConverterToRow(schema.toAvroSchema,
HoodieSchemaConversionUtils.convertHoodieSchemaToStructType(schema))
Review Comment:
Is there any test on `createGenericRecordToInternalRowConverter`? If not,
should we add add another test to use
`createGenericRecordToInternalRowConverter` for testing the conversion?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
