voonhous commented on code in PR #17763:
URL: https://github.com/apache/hudi/pull/17763#discussion_r2656089177
##########
hudi-hadoop-common/src/main/java/org/apache/parquet/avro/AvroSchemaConverterWithTimestampNTZ.java:
##########
@@ -116,96 +118,112 @@ public AvroSchemaConverterWithTimestampNTZ(Configuration
conf) {
this.pathsToInt96 = new
HashSet<>(Arrays.asList(conf.getStrings("parquet.avro.writeFixedAsInt96", new
String[0])));
}
- /**
- * Given a schema, check to see if it is a union of a null type and a
regular schema,
- * and then return the non-null sub-schema. Otherwise, return the given
schema.
- *
- * @param schema The schema to check
- * @return The non-null portion of a union schema, or the given schema
- */
- public static Schema getNonNull(Schema schema) {
- if (schema.getType().equals(Schema.Type.UNION)) {
- List<Schema> schemas = schema.getTypes();
- if (schemas.size() == 2) {
- if (schemas.get(0).getType().equals(Schema.Type.NULL)) {
- return schemas.get(1);
- } else if (schemas.get(1).getType().equals(Schema.Type.NULL)) {
- return schemas.get(0);
- } else {
- return schema;
- }
- } else {
- return schema;
- }
- } else {
- return schema;
- }
- }
-
@Override
- public MessageType convert(Schema avroSchema) {
- if (!avroSchema.getType().equals(Schema.Type.RECORD)) {
- throw new IllegalArgumentException("Avro schema must be a record.");
+ public MessageType convert(HoodieSchema schema) {
+ if (schema.getType() != HoodieSchemaType.RECORD) {
+ throw new IllegalArgumentException("Hoodie schema must be a record.");
}
- return new MessageType(avroSchema.getFullName(),
convertFields(avroSchema.getFields(), ""));
+ return new MessageType(schema.getFullName(),
convertFields(schema.getFields(), ""));
}
- private List<Type> convertFields(List<Schema.Field> fields, String
schemaPath) {
+ private List<Type> convertFields(List<HoodieSchemaField> fields, String
schemaPath) {
List<Type> types = new ArrayList<Type>();
- for (Schema.Field field : fields) {
- if (field.schema().getType().equals(Schema.Type.NULL)) {
+ for (HoodieSchemaField field : fields) {
+ if (field.schema().getType() == HoodieSchemaType.NULL) {
continue; // Avro nulls are not encoded, unless they are null unions
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
