This is an automated email from the ASF dual-hosted git repository.
tpalfy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 9d835e078b NIFI-14174: Protobuf Reader fails to generate schema for
proto files containing circular reference
9d835e078b is described below
commit 9d835e078b567dd32797a31fcae64f26365cd646
Author: Mark Bathori <[email protected]>
AuthorDate: Wed Jan 22 10:59:09 2025 +0100
NIFI-14174: Protobuf Reader fails to generate schema for proto files
containing circular reference
This closes #9657.
Signed-off-by: Tamas Palfy <[email protected]>
---
.../protobuf/schema/ProtoSchemaParser.java | 25 ++++++++++++----
.../nifi/services/protobuf/ProtoTestUtil.java | 6 ++++
.../protobuf/schema/TestProtoSchemaParser.java | 33 ++++++++++++++++++++++
.../test/resources/test_circular_reference.desc | 10 +++++++
.../test/resources/test_circular_reference.proto | 30 ++++++++++++++++++++
5 files changed, 99 insertions(+), 5 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/main/java/org/apache/nifi/services/protobuf/schema/ProtoSchemaParser.java
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/main/java/org/apache/nifi/services/protobuf/schema/ProtoSchemaParser.java
index 3e5777c1dd..18c4514c0d 100644
---
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/main/java/org/apache/nifi/services/protobuf/schema/ProtoSchemaParser.java
+++
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/main/java/org/apache/nifi/services/protobuf/schema/ProtoSchemaParser.java
@@ -29,13 +29,17 @@ import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.RecordField;
import org.apache.nifi.serialization.record.RecordFieldType;
import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.serialization.record.SchemaIdentifier;
+import org.apache.nifi.serialization.record.StandardSchemaIdentifier;
import org.apache.nifi.serialization.record.type.EnumDataType;
import org.apache.nifi.serialization.record.type.MapDataType;
import org.apache.nifi.serialization.record.type.RecordDataType;
import org.apache.nifi.services.protobuf.FieldType;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
/**
@@ -43,6 +47,8 @@ import java.util.Objects;
*/
public class ProtoSchemaParser {
+ private final Map<String, RecordSchema> parsedRecordSchemas = new
HashMap<>();
+
private final Schema schema;
public ProtoSchemaParser(Schema schema) {
@@ -57,13 +63,22 @@ public class ProtoSchemaParser {
public RecordSchema createSchema(String messageTypeName) {
final MessageType messageType = (MessageType)
schema.getType(messageTypeName);
Objects.requireNonNull(messageType, String.format("Message type with
name [%s] not found in the provided proto files", messageTypeName));
- List<RecordField> recordFields = new ArrayList<>();
- recordFields.addAll(processFields(messageType.getDeclaredFields()));
- recordFields.addAll(processFields(messageType.getExtensionFields()));
- recordFields.addAll(processOneOfFields(messageType));
+ if (parsedRecordSchemas.containsKey(messageTypeName)) {
+ return parsedRecordSchemas.get(messageTypeName);
+ } else {
+ final SchemaIdentifier identifier = new
StandardSchemaIdentifier.Builder().name(messageTypeName).build();
+ final SimpleRecordSchema recordSchema = new
SimpleRecordSchema(identifier);
+ parsedRecordSchemas.put(messageTypeName, recordSchema);
- return new SimpleRecordSchema(recordFields);
+ List<RecordField> recordFields = new ArrayList<>();
+
recordFields.addAll(processFields(messageType.getDeclaredFields()));
+
recordFields.addAll(processFields(messageType.getExtensionFields()));
+ recordFields.addAll(processOneOfFields(messageType));
+
+ recordSchema.setFields(recordFields);
+ return recordSchema;
+ }
}
/**
diff --git
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/ProtoTestUtil.java
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/ProtoTestUtil.java
index c0d273da95..2d0b8bd3b8 100644
---
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/ProtoTestUtil.java
+++
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/ProtoTestUtil.java
@@ -58,6 +58,12 @@ public class ProtoTestUtil {
return schemaLoader.loadSchema();
}
+ public static Schema loadCircularReferenceTestSchema() {
+ final SchemaLoader schemaLoader = new
SchemaLoader(FileSystems.getDefault());
+
schemaLoader.initRoots(Collections.singletonList(Location.get(BASE_TEST_PATH +
"test_circular_reference.proto")), Collections.emptyList());
+ return schemaLoader.loadSchema();
+ }
+
public static InputStream generateInputDataForProto3() throws IOException,
Descriptors.DescriptorValidationException {
DescriptorProtos.FileDescriptorSet descriptorSet =
DescriptorProtos.FileDescriptorSet.parseFrom(new FileInputStream(BASE_TEST_PATH
+ "test_proto3.desc"));
Descriptors.FileDescriptor fileDescriptor =
Descriptors.FileDescriptor.buildFrom(descriptorSet.getFile(0), new
Descriptors.FileDescriptor[0]);
diff --git
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/schema/TestProtoSchemaParser.java
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/schema/TestProtoSchemaParser.java
index 42bb858eab..14009576fe 100644
---
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/schema/TestProtoSchemaParser.java
+++
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/java/org/apache/nifi/services/protobuf/schema/TestProtoSchemaParser.java
@@ -25,11 +25,14 @@ import
org.apache.nifi.serialization.record.type.RecordDataType;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
+import java.util.Optional;
+import static
org.apache.nifi.services.protobuf.ProtoTestUtil.loadCircularReferenceTestSchema;
import static
org.apache.nifi.services.protobuf.ProtoTestUtil.loadProto2TestSchema;
import static
org.apache.nifi.services.protobuf.ProtoTestUtil.loadProto3TestSchema;
import static
org.apache.nifi.services.protobuf.ProtoTestUtil.loadRepeatedProto3TestSchema;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestProtoSchemaParser {
@@ -115,4 +118,34 @@ public class TestProtoSchemaParser {
final RecordSchema actual = schemaParser.createSchema("Proto2Message");
assertEquals(expected, actual);
}
+
+ @Test
+ public void testSchemaParserWithSelfCircularReference() {
+ final ProtoSchemaParser schemaParser = new
ProtoSchemaParser(loadCircularReferenceTestSchema());
+ final RecordSchema recordCSchema = schemaParser.createSchema("C");
+
+ final Optional<RecordField> parentField =
recordCSchema.getField("parent");
+ assertTrue(parentField.isPresent());
+ assertEquals(RecordFieldType.RECORD,
parentField.get().getDataType().getFieldType());
+
+ assertEquals(recordCSchema, ((RecordDataType)
parentField.get().getDataType()).getChildSchema());
+ }
+
+ @Test
+ public void testSchemaParserWithMutualCircularReference() {
+ final ProtoSchemaParser schemaParser = new
ProtoSchemaParser(loadCircularReferenceTestSchema());
+ final RecordSchema recordASchema = schemaParser.createSchema("A");
+ final Optional<RecordField> bOfA = recordASchema.getField("b");
+
+ assertTrue(bOfA.isPresent());
+ assertEquals(RecordFieldType.RECORD,
bOfA.get().getDataType().getFieldType());
+
+ final RecordSchema recordBSchema = schemaParser.createSchema("B");
+ final Optional<RecordField> aOfB = recordBSchema.getField("a");
+
+ assertTrue(aOfB.isPresent());
+ assertEquals(RecordFieldType.RECORD,
aOfB.get().getDataType().getFieldType());
+ assertEquals(recordBSchema, ((RecordDataType)
bOfA.get().getDataType()).getChildSchema());
+ assertEquals(recordASchema, ((RecordDataType)
aOfB.get().getDataType()).getChildSchema());
+ }
}
diff --git
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.desc
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.desc
new file mode 100644
index 0000000000..a8defd3833
--- /dev/null
+++
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.desc
@@ -0,0 +1,10 @@
+
+�
+ test_circular_reference.proto"
+A
+b ( 2.BRb"
+B
+a ( 2.ARa"5
+C
+value ( Rvalue
+parent ( 2.CRparentbproto3
\ No newline at end of file
diff --git
a/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.proto
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.proto
new file mode 100644
index 0000000000..0eb4bab38c
--- /dev/null
+++
b/nifi-extension-bundles/nifi-protobuf-bundle/nifi-protobuf-services/src/test/resources/test_circular_reference.proto
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto3";
+
+message A {
+ B b = 1;
+}
+
+message B {
+ A a = 1;
+}
+
+message C {
+ string value = 1;
+ C parent = 2;
+}
\ No newline at end of file
