This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6d01bcfa6a2 [HUDI-7966] Handle NPE from
AvroSchemaUtils.createNewSchemaFromFieldsWithReference (#11585)
6d01bcfa6a2 is described below
commit 6d01bcfa6a26fb890d64802f6e4a16debb0bb11f
Author: Sagar Sumit <[email protected]>
AuthorDate: Mon Jul 8 10:38:23 2024 +0530
[HUDI-7966] Handle NPE from
AvroSchemaUtils.createNewSchemaFromFieldsWithReference (#11585)
---
.../java/org/apache/hudi/avro/AvroSchemaUtils.java | 15 +++++++-
.../read/HoodieFileGroupReaderSchemaHandler.java | 8 ++---
.../org/apache/hudi/avro/TestAvroSchemaUtils.java | 40 ++++++++++++++++++++++
3 files changed, 56 insertions(+), 7 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java
b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java
index 642624e0ed3..59917f6c652 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaUtils.java
@@ -27,6 +27,8 @@ import org.apache.hudi.exception.SchemaCompatibilityException;
import org.apache.avro.Schema;
import org.apache.avro.SchemaCompatibility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -46,6 +48,8 @@ import static
org.apache.hudi.common.util.ValidationUtils.checkState;
*/
public class AvroSchemaUtils {
+ private static final Logger LOG =
LoggerFactory.getLogger(AvroSchemaUtils.class);
+
private AvroSchemaUtils() {}
/**
@@ -302,8 +306,17 @@ public class AvroSchemaUtils {
* @return schema with fields from fields, and metadata from schema
*/
public static Schema createNewSchemaFromFieldsWithReference(Schema schema,
List<Schema.Field> fields) {
+ if (schema == null) {
+ throw new IllegalArgumentException("Schema must not be null");
+ }
Schema newSchema = Schema.createRecord(schema.getName(), schema.getDoc(),
schema.getNamespace(), schema.isError());
- for (Map.Entry<String, Object> prop : schema.getObjectProps().entrySet()) {
+ Map<String, Object> schemaProps = Collections.emptyMap();
+ try {
+ schemaProps = schema.getObjectProps();
+ } catch (Exception e) {
+ LOG.warn("Error while getting object properties from schema: {}",
schema, e);
+ }
+ for (Map.Entry<String, Object> prop : schemaProps.entrySet()) {
newSchema.addProp(prop.getKey(), prop.getValue());
}
newSchema.setFields(fields);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderSchemaHandler.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderSchemaHandler.java
index 85d0cef2893..8a962fdc2a7 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderSchemaHandler.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReaderSchemaHandler.java
@@ -40,6 +40,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static
org.apache.hudi.avro.AvroSchemaUtils.appendFieldsToSchemaDedupNested;
+import static
org.apache.hudi.avro.AvroSchemaUtils.createNewSchemaFromFieldsWithReference;
import static org.apache.hudi.avro.AvroSchemaUtils.findNestedField;
/**
@@ -178,11 +179,6 @@ public class HoodieFileGroupReaderSchemaHandler<T> {
Schema.Field curr = fields.get(i);
fields.set(i, new Schema.Field(curr.name(), curr.schema(), curr.doc(),
curr.defaultVal()));
}
- Schema newSchema = Schema.createRecord(dataSchema.getName(),
dataSchema.getDoc(), dataSchema.getNamespace(), dataSchema.isError());
- for (Map.Entry<String, Object> prop :
dataSchema.getObjectProps().entrySet()) {
- newSchema.addProp(prop.getKey(), prop.getValue());
- }
- newSchema.setFields(fields);
- return newSchema;
+ return createNewSchemaFromFieldsWithReference(dataSchema, fields);
}
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
index 37cea0d93a9..a11338d4017 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
@@ -115,6 +115,46 @@ public class TestAvroSchemaUtils {
+ " ]\n"
+ "}\n";
+ @Test
+ public void testCreateNewSchemaFromFieldsWithReference_NullSchema() {
+ // This test should throw an IllegalArgumentException
+ assertThrows(IllegalArgumentException.class, () ->
AvroSchemaUtils.createNewSchemaFromFieldsWithReference(null,
Collections.emptyList()));
+ }
+
+ @Test
+ public void testCreateNewSchemaFromFieldsWithReference_NullObjectProps() {
+ // Create a schema without any object properties
+ String schemaStr = "{ \"type\": \"record\", \"name\": \"TestRecord\",
\"fields\": [] }";
+ Schema schema = new Schema.Parser().parse(schemaStr);
+
+ // Ensure getObjectProps returns null by mocking or creating a schema
without props
+ Schema newSchema =
AvroSchemaUtils.createNewSchemaFromFieldsWithReference(schema,
Collections.emptyList());
+
+ // Validate the new schema
+ assertEquals("TestRecord", newSchema.getName());
+ assertEquals(0, newSchema.getFields().size());
+ }
+
+ @Test
+ public void testCreateNewSchemaFromFieldsWithReference_WithObjectProps() {
+ // Create a schema with object properties
+ String schemaStr = "{ \"type\": \"record\", \"name\": \"TestRecord\",
\"fields\": [], \"prop1\": \"value1\" }";
+ Schema schema = new Schema.Parser().parse(schemaStr);
+
+ // Add an object property to the schema
+ schema.addProp("prop1", "value1");
+
+ // Create new fields to add
+ Schema.Field newField = new Schema.Field("newField",
Schema.create(Schema.Type.STRING), null, (Object) null);
+ Schema newSchema =
AvroSchemaUtils.createNewSchemaFromFieldsWithReference(schema,
Collections.singletonList(newField));
+
+ // Validate the new schema
+ assertEquals("TestRecord", newSchema.getName());
+ assertEquals(1, newSchema.getFields().size());
+ assertEquals("value1", newSchema.getProp("prop1"));
+ assertEquals("newField", newSchema.getFields().get(0).name());
+ }
+
@Test
public void testIsStrictProjection() {
Schema sourceSchema = new Schema.Parser().parse(SOURCE_SCHEMA);
