http://git-wip-us.apache.org/repos/asf/nifi/blob/d21bd387/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLRecordReader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLRecordReader.java new file mode 100755 index 0000000..ef3d692 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLRecordReader.java @@ -0,0 +1,1436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.xml; + +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class TestXMLRecordReader { + private final String dateFormat = RecordFieldType.DATE.getDefaultFormat(); + private final String timeFormat = RecordFieldType.TIME.getDefaultFormat(); + private final String timestampFormat = RecordFieldType.TIMESTAMP.getDefaultFormat(); + + @Test + public void testSingleRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/person.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), false, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertNull(reader.nextRecord()); + } + + @Test + public void testMap() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_map.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaForMap(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(); + assertEquals("P1", first.getValue("ID")); + Map firstMap = (Map) first.getValue("MAP"); + assertEquals("Cleve Butler", firstMap.get("NAME")); + assertEquals("42", firstMap.get("AGE")); + assertEquals("USA", firstMap.get("COUNTRY")); + + Record second = reader.nextRecord(); + assertEquals("P2", second.getValue("ID")); + Map secondMap = (Map) second.getValue("MAP"); + assertEquals("Ainslie Fletcher", secondMap.get("NAME")); + assertEquals("33", secondMap.get("AGE")); + assertEquals("UK", secondMap.get("COUNTRY")); + } + + @Test + public void testMapWithRecords() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_map2.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaForRecordMap(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(); + assertEquals("P1", first.getValue("ID")); + Map firstMap = (Map) first.getValue("MAP"); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, ((Record) firstMap.get("ENTRY")).getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, ((Record) firstMap.get("ENTRY2")).getValues()); + + Record second = reader.nextRecord(); + assertEquals("P2", second.getValue("ID")); + Map secondMap = (Map) second.getValue("MAP"); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, ((Record) secondMap.get("ENTRY")).getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, ((Record) secondMap.get("ENTRY2")).getValues()); + } + + @Test + public void testTagInCharactersSimpleField() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, null}, reader.nextRecord().getValues()); + } + + @Test + public void testTagInCharactersRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaWithNestedRecord3(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(true, true); + assertEquals(42, first.getValue("AGE")); + Record firstNested = (Record) first.getValue("NAME"); + assertEquals("Cleve Butler", firstNested.getValue("CONTENT")); + assertEquals("attr content", firstNested.getValue("ATTR")); + assertEquals("inner content", firstNested.getValue("INNER")); + + Record second = reader.nextRecord(true, true); + assertEquals(33, second.getValue("AGE")); + Record secondNested = (Record) second.getValue("NAME"); + assertEquals("Ainslie Fletcher", secondNested.getValue("CONTENT")); + assertEquals("attr content", secondNested.getValue("ATTR")); + assertEquals("inner content", secondNested.getValue("INNER")); + + Record third = reader.nextRecord(true, true); + assertEquals(74, third.getValue("AGE")); + Record thirdNested = (Record) third.getValue("NAME"); + assertEquals("Amélie Bonfils", thirdNested.getValue("CONTENT")); + assertEquals("attr content", thirdNested.getValue("ATTR")); + assertEquals("inner content", thirdNested.getValue("INNER")); + + Record fourth = reader.nextRecord(true, true); + assertEquals(16, fourth.getValue("AGE")); + Record fourthNested = (Record) fourth.getValue("NAME"); + assertEquals("Elenora Scrivens", fourthNested.getValue("CONTENT")); + assertEquals("attr content", fourthNested.getValue("ATTR")); + assertEquals("inner content", fourthNested.getValue("INNER")); + + Record fifth = reader.nextRecord(true, true); + assertNull(fifth.getValue("AGE")); + Record fifthNested = (Record) fifth.getValue("NAME"); + assertNull(fifthNested.getValue("CONTENT")); + assertNull(fifthNested.getValue("ATTR")); + assertEquals("inner content", fifthNested.getValue("INNER")); + } + + @Test + public void testTagInCharactersCoerceTrueDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaWithNestedRecord3(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(true, false); + assertEquals("P1", first.getValue("ID")); + assertEquals(42, first.getValue("AGE")); + Record firstNested = (Record) first.getValue("NAME"); + assertEquals("Cleve Butler", firstNested.getValue("CONTENT")); + assertEquals("attr content", firstNested.getValue("ATTR")); + assertEquals("inner content", firstNested.getValue("INNER")); + + Record second = reader.nextRecord(true, false); + assertEquals("P2", second.getValue("ID")); + assertEquals(33, second.getValue("AGE")); + Record secondNested = (Record) second.getValue("NAME"); + assertEquals("Ainslie Fletcher", secondNested.getValue("CONTENT")); + assertEquals("attr content", secondNested.getValue("ATTR")); + assertEquals("inner content", secondNested.getValue("INNER")); + + Record third = reader.nextRecord(true, false); + assertEquals("P3", third.getValue("ID")); + assertEquals(74, third.getValue("AGE")); + Record thirdNested = (Record) third.getValue("NAME"); + assertEquals("Amélie Bonfils", thirdNested.getValue("CONTENT")); + assertEquals("attr content", thirdNested.getValue("ATTR")); + assertEquals("inner content", thirdNested.getValue("INNER")); + + Record fourth = reader.nextRecord(true, false); + assertEquals("P4", fourth.getValue("ID")); + assertEquals(16, fourth.getValue("AGE")); + Record fourthNested = (Record) fourth.getValue("NAME"); + assertEquals("Elenora Scrivens", fourthNested.getValue("CONTENT")); + assertEquals("attr content", fourthNested.getValue("ATTR")); + assertEquals("inner content", fourthNested.getValue("INNER")); + + Record fifth = reader.nextRecord(true, false); + assertEquals("P5", fifth.getValue("ID")); + assertNull(fifth.getValue("AGE")); + Record fifthNested = (Record) fifth.getValue("NAME"); + assertNull(fifthNested.getValue("CONTENT")); + assertNull(fifthNested.getValue("ATTR")); + assertEquals("inner content", fifthNested.getValue("INNER")); + } + + @Test + public void testTagInCharactersCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + assertEquals("P1", first.getValue("ID")); + assertEquals("42", first.getValue("AGE")); + Record firstNested = (Record) first.getValue("NAME"); + assertEquals("Cleve Butler", firstNested.getValue("CONTENT")); + assertEquals("attr content", firstNested.getValue("ATTR")); + assertEquals("inner content", firstNested.getValue("INNER")); + + Record second = reader.nextRecord(false, false); + assertEquals("P2", second.getValue("ID")); + assertEquals("33", second.getValue("AGE")); + Record secondNested = (Record) second.getValue("NAME"); + assertEquals("Ainslie Fletcher", secondNested.getValue("CONTENT")); + assertEquals("attr content", secondNested.getValue("ATTR")); + assertEquals("inner content", secondNested.getValue("INNER")); + + Record third = reader.nextRecord(false, false); + assertEquals("P3", third.getValue("ID")); + assertEquals("74", third.getValue("AGE")); + Record thirdNested = (Record) third.getValue("NAME"); + assertEquals("Amélie Bonfils", thirdNested.getValue("CONTENT")); + assertEquals("attr content", thirdNested.getValue("ATTR")); + assertEquals("inner content", thirdNested.getValue("INNER")); + + Record fourth = reader.nextRecord(false, false); + assertEquals("P4", fourth.getValue("ID")); + assertEquals("16", fourth.getValue("AGE")); + Record fourthNested = (Record) fourth.getValue("NAME"); + assertEquals("Elenora Scrivens", fourthNested.getValue("CONTENT")); + assertEquals("attr content", fourthNested.getValue("ATTR")); + assertEquals("inner content", fourthNested.getValue("INNER")); + + Record fifth = reader.nextRecord(false, false); + assertEquals("P5", fifth.getValue("ID")); + assertNull(fifth.getValue("AGE")); + Record fifthNested = (Record) fifth.getValue("NAME"); + assertNull(fifthNested.getValue("CONTENT")); + assertNull(fifthNested.getValue("ATTR")); + assertEquals("inner content", fifthNested.getValue("INNER")); + } + + @Test + public void testSimpleRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues()); + } + + @Test + public void testSimpleRecord2() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema2(), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertNull(reader.nextRecord(true, true).getValue("AGE")); + assertNull(reader.nextRecord(false, true).getValue("AGE")); + assertNotNull(reader.nextRecord(true, false).getValue("AGE")); + assertNotNull(reader.nextRecord(false, false).getValue("AGE")); + } + + @Test + public void testSimpleRecord3() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertEquals(Integer.class, reader.nextRecord(true, true).getValue("AGE").getClass()); + assertEquals(String.class, reader.nextRecord(false, true).getValue("AGE").getClass()); + } + + @Test + public void testSimpleRecord4() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.remove(2); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertEquals(Integer.class, reader.nextRecord(true, false).getValue("AGE").getClass()); + assertEquals(String.class, reader.nextRecord(false, false).getValue("AGE").getClass()); + } + + @Test + public void testSimpleRecordCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_no_attributes.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "42", "USA"}, reader.nextRecord(false, false).getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "33", "UK"}, reader.nextRecord(false, false).getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "74", "FR"}, reader.nextRecord(false, false).getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "16", "USA"}, reader.nextRecord(false, false).getValues()); + } + + @Test + public void testSimpleRecordWithAttribute() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(); + assertTrue(Arrays.asList(first.getValues()).containsAll(Arrays.asList("Cleve Butler", 42, "USA", "P1"))); + assertEquals("P1", first.getAsString("ID")); + + Record second = reader.nextRecord(); + assertTrue(Arrays.asList(second.getValues()).containsAll(Arrays.asList("Ainslie Fletcher", 33, "UK", "P2"))); + assertEquals("P2", second.getAsString("ID")); + + Record third = reader.nextRecord(); + assertTrue(Arrays.asList(third.getValues()).containsAll(Arrays.asList("Amélie Bonfils", 74, "FR", "P3"))); + assertEquals("P3", third.getAsString("ID")); + + Record fourth = reader.nextRecord(); + assertTrue(Arrays.asList(fourth.getValues()).containsAll(Arrays.asList("Elenora Scrivens", 16, "USA", "P4"))); + assertEquals("P4", fourth.getAsString("ID")); + } + + @Test + public void testSimpleRecordWithAttribute2() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + "ATTR_", "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(); + assertTrue(Arrays.asList(first.getValues()).containsAll(Arrays.asList("Cleve Butler", 42, "USA"))); + assertEquals("P1", first.getAsString("ATTR_ID")); + + Record second = reader.nextRecord(); + assertTrue(Arrays.asList(second.getValues()).containsAll(Arrays.asList("Ainslie Fletcher", 33, "UK"))); + assertEquals("P2", second.getAsString("ATTR_ID")); + + Record third = reader.nextRecord(); + assertTrue(Arrays.asList(third.getValues()).containsAll(Arrays.asList("Amélie Bonfils", 74, "FR"))); + assertEquals("P3", third.getAsString("ATTR_ID")); + + Record fourth = reader.nextRecord(); + assertTrue(Arrays.asList(fourth.getValues()).containsAll(Arrays.asList("Elenora Scrivens", 16, "USA"))); + assertEquals("P4", fourth.getAsString("ATTR_ID")); + } + + @Test + public void testSimpleRecordWithAttribute3() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(Collections.emptyList()), + true, null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(true, true); + assertEquals(null, first.getAsString("ID")); + + Record second = reader.nextRecord(false, false); + assertEquals("P2", second.getAsString("ID")); + + Record third = reader.nextRecord(true, false); + assertEquals("P3", third.getAsString("ID")); + + Record fourth = reader.nextRecord(false, true); + assertEquals(null, fourth.getAsString("ID")); + } + + @Test + public void testSimpleRecordWithAttribute4() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people2.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ID", RecordFieldType.INT.getDataType())); + + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertEquals(Integer.class, reader.nextRecord(true, true).getValue("ID").getClass()); + assertEquals(String.class, reader.nextRecord(false, true).getValue("ID").getClass()); + } + + @Test + public void testSimpleRecordWithAttribute5() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people2.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ID", RecordFieldType.INT.getDataType())); + + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertEquals(Integer.class, reader.nextRecord(true, false).getValue("ID").getClass()); + assertEquals(String.class, reader.nextRecord(false, false).getValue("ID").getClass()); + } + + @Test + public void testSimpleRecordWithAttributeCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + assertTrue(Arrays.asList(new Object[] {"Cleve Butler", "42", "USA", "P1"}).containsAll(Arrays.asList(first.getValues()))); + assertEquals("P1", first.getAsString("ID")); + + Record second = reader.nextRecord(false, false); + assertTrue(Arrays.asList(new Object[] {"Ainslie Fletcher", "33", "UK", "P2"}).containsAll(Arrays.asList(second.getValues()))); + assertEquals("P2", second.getAsString("ID")); + + Record third = reader.nextRecord(false, false); + assertTrue(Arrays.asList(new Object[] {"Amélie Bonfils", "74", "FR", "P3"}).containsAll(Arrays.asList(third.getValues()))); + assertEquals("P3", third.getAsString("ID")); + + Record fourth = reader.nextRecord(false, false); + assertTrue(Arrays.asList(new Object[] {"Elenora Scrivens", "16", "USA", "P4"}).containsAll(Arrays.asList(fourth.getValues()))); + assertEquals("P4", fourth.getAsString("ID")); + } + + @Test + public void testSimpleTypeWithAttributeAsRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people3.xml"); + final List<RecordField> fields = new ArrayList<>(); + + final List<RecordField> nestedFields1 = new ArrayList<>(); + nestedFields1.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + nestedFields1.add(new RecordField("CONTENT", RecordFieldType.STRING.getDataType())); + + final DataType recordType1 = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(nestedFields1)); + fields.add(new RecordField("NAME", recordType1)); + + final List<RecordField> nestedFields2 = new ArrayList<>(); + nestedFields2.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + nestedFields2.add(new RecordField("CONTENT", RecordFieldType.INT.getDataType())); + + final DataType recordType2 = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(nestedFields2)); + fields.add(new RecordField("AGE", recordType2)); + + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(true, true); + assertTrue(first.getValue("NAME") instanceof Record); + Record first_nested1 = (Record) first.getValue("NAME"); + assertTrue(first.getValue("AGE") instanceof Record); + Record first_nested2 = (Record) first.getValue("AGE"); + assertEquals("name1", first_nested1.getValue("ID")); + assertEquals("Cleve Butler", first_nested1.getValue("CONTENT")); + assertEquals("age1", first_nested2.getValue("ID")); + assertEquals(42, first_nested2.getValue("CONTENT")); + + Record second = reader.nextRecord(true, true); + assertTrue(second.getValue("NAME") instanceof Record); + Record second_nested1 = (Record) second.getValue("NAME"); + assertTrue(second.getValue("AGE") instanceof Record); + Record second_nested2 = (Record) second.getValue("AGE"); + assertEquals("name2", second_nested1.getValue("ID")); + assertEquals("Ainslie Fletcher", second_nested1.getValue("CONTENT")); + assertEquals("age2", second_nested2.getValue("ID")); + assertEquals(33, second_nested2.getValue("CONTENT")); + } + + @Test + public void testSimpleTypeWithAttributeAsRecordCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people3.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + assertTrue(first.getValue("NAME") instanceof Record); + Record first_nested1 = (Record) first.getValue("NAME"); + assertTrue(first.getValue("AGE") instanceof Record); + Record first_nested2 = (Record) first.getValue("AGE"); + assertEquals("name1", first_nested1.getValue("ID")); + assertEquals("Cleve Butler", first_nested1.getValue("CONTENT")); + assertEquals("age1", first_nested2.getValue("ID")); + assertEquals("42", first_nested2.getValue("CONTENT")); + assertEquals("USA", first.getValue("COUNTRY")); + + Record second = reader.nextRecord(false, false); + assertTrue(second.getValue("NAME") instanceof Record); + Record second_nested1 = (Record) second.getValue("NAME"); + assertTrue(second.getValue("AGE") instanceof Record); + Record second_nested2 = (Record) second.getValue("AGE"); + assertEquals("name2", second_nested1.getValue("ID")); + assertEquals("Ainslie Fletcher", second_nested1.getValue("CONTENT")); + assertEquals("age2", second_nested2.getValue("ID")); + assertEquals("33", second_nested2.getValue("CONTENT")); + assertEquals("UK", second.getValue("COUNTRY")); + } + + @Test + public void testSimpleRecordWithHeader() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_with_header_and_comments.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, null, dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues()); + } + + @Test + public void testSimpleRecordWithHeaderNoValidation() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_with_header_and_comments.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, null, null, dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues()); + } + + @Test + public void testInvalidXml() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_invalid.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + int count = 0; + + /* + Due to the missing starting tag <PERSON> for the third entry in people_invalid.xml, the reader assumes <NAME> and + <AGE> to be records. The tag <COUNTRY> is also assumed to be a record, but the exception is thrown + before the "record" for <COUNTRY> is returned. Even a tracking of the parsing depth would not help to overcome this problem. + */ + + try { + while ((reader.nextRecord()) != null) { + count++; + } + } catch (MalformedRecordException e) { + assertEquals("Could not parse XML", e.getMessage()); + assertEquals(4, count); + } + + } + + @Test + public void testChoiceForSimpleField() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields2(); + fields.add(new RecordField("AGE", RecordFieldType.CHOICE.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record record = reader.nextRecord(); + assertTrue(record.getValue("AGE") instanceof String); + assertEquals("42", record.getValue("AGE")); + } + + @Test + public void testChoiceForRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ADDRESS", RecordFieldType.CHOICE.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record record = reader.nextRecord(); + assertTrue(record.getValue("ADDRESS") instanceof Record); + + Record nested = (Record) record.getValue("ADDRESS"); + assertEquals("292 West Street", nested.getValue("STREET")); + assertEquals("Jersey City", nested.getValue("CITY")); + } + + @Test + public void testNameSpaces() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_namespace.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues()); + } + + @Test + public void testCData() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_cdata.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, reader.nextRecord().getValues()); + } + + @Test + public void testRecordExpectedSimpleFieldFoundAndNoContentFieldConfigured() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people.xml"); + List<RecordField> fields = getSimpleRecordFields2(); + final DataType recordType = RecordFieldType.RECORD.getRecordDataType(getNestedSchema()); + fields.add(new RecordField("AGE", recordType)); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA", null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK", null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR", null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA", null}, reader.nextRecord().getValues()); + } + + @Test + public void testSimpleFieldExpectedButRecordFound() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + List<RecordField> fields = getSimpleRecordFields(); + fields.add(new RecordField("ADDRESS", RecordFieldType.STRING.getDataType())); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(fields), true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + assertNull(reader.nextRecord().getValue("ADDRESS")); + assertNull(reader.nextRecord().getValue("ADDRESS")); + assertNull(reader.nextRecord().getValue("ADDRESS")); + assertNull(reader.nextRecord().getValue("ADDRESS")); + } + + @Test + public void testParseEmptyFields() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_empty.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {null, null, null}, reader.nextRecord().getValues()); + Assert.assertArrayEquals(new Object[] {null, null, null}, reader.nextRecord().getValues()); + } + + @Test + public void testParseEmptyFieldsCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_empty.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertArrayEquals(new Object[] {null, null, null}, reader.nextRecord(false, false).getValues()); + Assert.assertArrayEquals(new Object[] {null, null, null}, reader.nextRecord(false, false).getValues()); + } + + @Test(expected = MalformedRecordException.class) + public void testEmptyStreamAsSingleRecord() throws IOException, MalformedRecordException { + InputStream is = new ByteArrayInputStream(new byte[0]); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), false, null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + reader.nextRecord(true, true); + } + + @Test(expected = MalformedRecordException.class) + public void testEmptyStreamAsArray() throws IOException, MalformedRecordException { + InputStream is = new ByteArrayInputStream(new byte[0]); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + reader.nextRecord(true, true); + } + + @Test(expected = MalformedRecordException.class) + public void testEmptyStreamWIthXmlHeader() throws IOException, MalformedRecordException { + InputStream is = new ByteArrayInputStream(("<?xml version=\"1.0\" encoding=\"utf-8\"?>").getBytes()); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + Record record = reader.nextRecord(false, false); + assertNull(record); + } + + @Test + public void testParseEmptyArray() throws IOException, MalformedRecordException { + InputStream is = new ByteArrayInputStream("<root></root>".getBytes()); + XMLRecordReader reader = new XMLRecordReader(is, getSimpleSchema(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Assert.assertNull(reader.nextRecord()); + } + + @Test + public void testNestedRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + RecordSchema schema = getSchemaWithNestedRecord(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Object[] valuesFirstRecord = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Assert.assertArrayEquals(new Object[] {"292 West Street", "Jersey City"},((Record) valuesFirstRecord[valuesFirstRecord.length - 1]).getValues()); + + Object[] valuesSecondRecord = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Assert.assertArrayEquals(new Object[] {"123 6th St.", "Seattle"},((Record) valuesSecondRecord[valuesSecondRecord.length - 1]).getValues()); + + Object[] valuesThirdRecord = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Assert.assertArrayEquals(new Object[] {"44 Shirley Ave.", "Los Angeles"},((Record) valuesThirdRecord[valuesThirdRecord.length - 1]).getValues()); + + Object[] valuesFourthRecord = reader.nextRecord().getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + Assert.assertArrayEquals(new Object[] {"70 Bowman St." , "Columbus"},((Record) valuesFourthRecord[valuesFourthRecord.length - 1]).getValues()); + } + + @Test + public void testNestedRecordCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + RecordSchema schema = getSchemaWithNestedRecord(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + Object[] valuesFirstRecord = first.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "42", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + assertEquals("P1", first.getAsString("ID")); + + Record nestedFirstRecord = (Record) first.getValue("ADDRESS"); + Assert.assertEquals("Jersey City", nestedFirstRecord.getAsString("CITY")); + Assert.assertEquals("292 West Street", nestedFirstRecord.getAsString("STREET")); + + Record second = reader.nextRecord(false, false); + Object[] valuesSecondRecord = second.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "33", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + assertEquals("P2", second.getAsString("ID")); + + Record nestedSecondRecord = (Record) second.getValue("ADDRESS"); + Assert.assertEquals("Seattle", nestedSecondRecord.getAsString("CITY")); + Assert.assertEquals("123 6th St.", nestedSecondRecord.getAsString("STREET")); + + Record third = reader.nextRecord(false, false); + Object[] valuesThirdRecord = third.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "74", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + assertEquals("P3", third.getAsString("ID")); + + Record nestedThirdRecord = (Record) third.getValue("ADDRESS"); + Assert.assertEquals("Los Angeles", nestedThirdRecord.getAsString("CITY")); + Assert.assertEquals("44 Shirley Ave.", nestedThirdRecord.getAsString("STREET")); + + Record fourth = reader.nextRecord(false, false); + Object[] valuesFourthRecord = fourth.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "16", "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + assertEquals("P4", fourth.getAsString("ID")); + + Record nestedFourthRecord = (Record) fourth.getValue("ADDRESS"); + Assert.assertEquals("Columbus", nestedFourthRecord.getAsString("CITY")); + Assert.assertEquals("70 Bowman St.", nestedFourthRecord.getAsString("STREET")); + } + + @Test + public void testNestedRecordFieldsToIgnoreCoerceTrueDropTrue() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + + // Fields "AGE" and "ADDRESS/CITY" are not defined here + RecordSchema schema = getSchemaWithNestedRecord2(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(true, true); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Record firstRecordNested = (Record) firstRecord.getValue("ADDRESS"); + Assert.assertEquals("292 West Street", firstRecordNested.getValue("STREET")); + Assert.assertNull(firstRecord.getValue("AGE")); + Assert.assertNull(firstRecordNested.getValue("CITY")); + + Record secondRecord = reader.nextRecord(true, true); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Record secondRecordNested = (Record) secondRecord.getValue("ADDRESS"); + Assert.assertEquals("123 6th St.", secondRecordNested.getValue("STREET")); + Assert.assertNull(secondRecord.getValue("AGE")); + Assert.assertNull(secondRecordNested.getValue("CITY")); + + Record thirdRecord = reader.nextRecord(true, true); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Record thirdRecordNested = (Record) thirdRecord.getValue("ADDRESS"); + Assert.assertEquals("44 Shirley Ave.", thirdRecordNested.getValue("STREET")); + Assert.assertNull(thirdRecord.getValue("AGE")); + Assert.assertNull(thirdRecordNested.getValue("CITY")); + + Record fourthRecord = reader.nextRecord(true, true); + Object[] valuesFourthRecord = fourthRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + Record fourthRecordNested = (Record) fourthRecord.getValue("ADDRESS"); + Assert.assertEquals("70 Bowman St.", fourthRecordNested.getValue("STREET")); + Assert.assertNull(fourthRecord.getValue("AGE")); + Assert.assertNull(fourthRecordNested.getValue("CITY")); + } + + @Test + public void testNestedRecordFieldsToIgnoreCoerceFalseDropTrue() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + + // Fields "AGE" and "ADDRESS/CITY" are not defined here + RecordSchema schema = getSchemaWithNestedRecord2(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(false, true); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Record firstRecordNested = (Record) firstRecord.getValue("ADDRESS"); + Assert.assertEquals("292 West Street", firstRecordNested.getValue("STREET")); + Assert.assertNull(firstRecord.getValue("AGE")); + Assert.assertNull(firstRecordNested.getValue("CITY")); + + Record secondRecord = reader.nextRecord(false, true); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Record secondRecordNested = (Record) secondRecord.getValue("ADDRESS"); + Assert.assertEquals("123 6th St.", secondRecordNested.getValue("STREET")); + Assert.assertNull(secondRecord.getValue("AGE")); + Assert.assertNull(secondRecordNested.getValue("CITY")); + + Record thirdRecord = reader.nextRecord(false, true); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Record thirdRecordNested = (Record) thirdRecord.getValue("ADDRESS"); + Assert.assertEquals("44 Shirley Ave.", thirdRecordNested.getValue("STREET")); + Assert.assertNull(thirdRecord.getValue("AGE")); + Assert.assertNull(thirdRecordNested.getValue("CITY")); + + Record fourthRecord = reader.nextRecord(false, true); + Object[] valuesFourthRecord = fourthRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + Record fourthRecordNested = (Record) fourthRecord.getValue("ADDRESS"); + Assert.assertEquals("70 Bowman St.", fourthRecordNested.getValue("STREET")); + Assert.assertNull(fourthRecord.getValue("AGE")); + Assert.assertNull(fourthRecordNested.getValue("CITY")); + } + + @Test + public void testNestedRecordFieldsToIgnoreCoerceTrueDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + + // Fields "AGE" and "ADDRESS/CITY" are not defined here + RecordSchema schema = getSchemaWithNestedRecord2(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(true, false); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Record firstRecordNested = (Record) firstRecord.getValue("ADDRESS"); + Assert.assertEquals("292 West Street", firstRecordNested.getValue("STREET")); + Assert.assertNotNull(firstRecord.getValue("AGE")); + Assert.assertEquals("42", firstRecord.getValue("AGE")); + Assert.assertNotNull(firstRecordNested.getValue("CITY")); + Assert.assertEquals("Jersey City", firstRecordNested.getValue("CITY")); + + Record secondRecord = reader.nextRecord(true, false); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Record secondRecordNested = (Record) secondRecord.getValue("ADDRESS"); + Assert.assertEquals("123 6th St.", secondRecordNested.getValue("STREET")); + Assert.assertNotNull(secondRecord.getValue("AGE")); + Assert.assertEquals("33", secondRecord.getValue("AGE")); + Assert.assertNotNull(secondRecordNested.getValue("CITY")); + Assert.assertEquals("Seattle", secondRecordNested.getValue("CITY")); + + Record thirdRecord = reader.nextRecord(true, false); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Record thirdRecordNested = (Record) thirdRecord.getValue("ADDRESS"); + Assert.assertEquals("44 Shirley Ave.", thirdRecordNested.getValue("STREET")); + Assert.assertNotNull(thirdRecord.getValue("AGE")); + Assert.assertEquals("74", thirdRecord.getValue("AGE")); + Assert.assertNotNull(thirdRecordNested.getValue("CITY")); + Assert.assertEquals("Los Angeles", thirdRecordNested.getValue("CITY")); + + Record fourthRecord = reader.nextRecord(true, false); + Object[] valuesFourthRecord = fourthRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + Record fourthRecordNested = (Record) fourthRecord.getValue("ADDRESS"); + Assert.assertEquals("70 Bowman St.", fourthRecordNested.getValue("STREET")); + Assert.assertNotNull(fourthRecord.getValue("AGE")); + Assert.assertEquals("16", fourthRecord.getValue("AGE")); + Assert.assertNotNull(fourthRecordNested.getValue("CITY")); + Assert.assertEquals("Columbus", fourthRecordNested.getValue("CITY")); + } + + @Test + public void testNestedRecordFieldsToIgnoreCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_nested.xml"); + + // Fields "AGE" and "ADDRESS/CITY" are not defined here + RecordSchema schema = getSchemaWithNestedRecord2(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(false, false); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Record firstRecordNested = (Record) firstRecord.getValue("ADDRESS"); + Assert.assertEquals("292 West Street", firstRecordNested.getValue("STREET")); + Assert.assertNotNull(firstRecord.getValue("AGE")); + Assert.assertEquals("42", firstRecord.getValue("AGE")); + Assert.assertNotNull(firstRecordNested.getValue("CITY")); + Assert.assertEquals("Jersey City", firstRecordNested.getValue("CITY")); + + Record secondRecord = reader.nextRecord(false, false); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Record secondRecordNested = (Record) secondRecord.getValue("ADDRESS"); + Assert.assertEquals("123 6th St.", secondRecordNested.getValue("STREET")); + Assert.assertNotNull(secondRecord.getValue("AGE")); + Assert.assertEquals("33", secondRecord.getValue("AGE")); + Assert.assertNotNull(secondRecordNested.getValue("CITY")); + Assert.assertEquals("Seattle", secondRecordNested.getValue("CITY")); + + Record thirdRecord = reader.nextRecord(false, false); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Record thirdRecordNested = (Record) thirdRecord.getValue("ADDRESS"); + Assert.assertEquals("44 Shirley Ave.", thirdRecordNested.getValue("STREET")); + Assert.assertNotNull(thirdRecord.getValue("AGE")); + Assert.assertEquals("74", thirdRecord.getValue("AGE")); + Assert.assertNotNull(thirdRecordNested.getValue("CITY")); + Assert.assertEquals("Los Angeles", thirdRecordNested.getValue("CITY")); + + Record fourthRecord = reader.nextRecord(false, false); + Object[] valuesFourthRecord = fourthRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + Record fourthRecordNested = (Record) fourthRecord.getValue("ADDRESS"); + Assert.assertEquals("70 Bowman St.", fourthRecordNested.getValue("STREET")); + Assert.assertNotNull(fourthRecord.getValue("AGE")); + Assert.assertEquals("16", fourthRecord.getValue("AGE")); + Assert.assertNotNull(fourthRecordNested.getValue("CITY")); + Assert.assertEquals("Columbus", fourthRecordNested.getValue("CITY")); + } + + + @Test + public void testSimpleArray() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_array_simple.xml"); + RecordSchema schema = getSchemaWithSimpleArray(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Object[] nestedArrayFirstRecord = (Object[]) valuesFirstRecord[valuesFirstRecord.length - 1]; + assertEquals(2, nestedArrayFirstRecord.length); + Assert.assertArrayEquals(new Object[] {"child1", "child2"}, nestedArrayFirstRecord); + assertNotEquals(null, firstRecord.getValue("CHILD")); + + Record secondRecord = reader.nextRecord(); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + Object[] nestedArraySecondRecord = (Object[]) valuesSecondRecord[valuesSecondRecord.length - 1]; + assertEquals(1, nestedArraySecondRecord.length); + Assert.assertArrayEquals(new Object[] {"child1"}, nestedArraySecondRecord); + assertNotEquals(null, secondRecord.getValue("CHILD")); + + Record thirdRecord = reader.nextRecord(); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Object[] nestedArrayThirdRecord = (Object[]) valuesThirdRecord[valuesThirdRecord.length - 1]; + assertEquals(3, nestedArrayThirdRecord.length); + Assert.assertArrayEquals(new Object[] {"child1", "child2", "child3"}, nestedArrayThirdRecord); + assertNotEquals(null, thirdRecord.getValue("CHILD")); + + Record valuesFourthRecord = reader.nextRecord(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, Arrays.copyOfRange(valuesFourthRecord.getValues(), 0, valuesFourthRecord.getValues().length - 1)); + assertEquals(null, valuesFourthRecord.getValue("CHILD")); + } + + @Test + public void testSimpleArrayCoerceFalseDropFalse() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_array_simple.xml"); + RecordSchema schema = getSchemaWithSimpleArray(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + Object[] valuesFirstRecord = first.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", "42", "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + Object[] nestedArrayFirstRecord = (Object[]) valuesFirstRecord[valuesFirstRecord.length - 1]; + assertEquals(2, nestedArrayFirstRecord.length); + Assert.assertArrayEquals(new Object[] {"child1", "child2"}, nestedArrayFirstRecord); + assertNotEquals(null, first.getValue("CHILD")); + + Record second = reader.nextRecord(false, false); + Object[] valuesSecondRecord = second.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", "33", "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + String nestedArraySecondRecord = (String) valuesSecondRecord[valuesSecondRecord.length - 1]; + Assert.assertEquals("child1", nestedArraySecondRecord); + assertNotEquals(null, second.getValue("CHILD")); + + Record third = reader.nextRecord(false, false); + Object[] valuesThirdRecord = third.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", "74", "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + Object[] nestedArrayThirdRecord = (Object[]) valuesThirdRecord[valuesThirdRecord.length - 1]; + assertEquals(3, nestedArrayThirdRecord.length); + Assert.assertArrayEquals(new Object[] {"child1", "child2", "child3"}, nestedArrayThirdRecord); + assertNotEquals(null, third.getValue("CHILD")); + + Record fourth = reader.nextRecord(false, false); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", "16", "USA"}, Arrays.copyOfRange(fourth.getValues(), 0, fourth.getValues().length - 1)); + assertEquals(null, fourth.getValue("CHILD")); + } + + @Test + public void testNestedArrayInNestedRecord() throws IOException, MalformedRecordException { + InputStream is = new FileInputStream("src/test/resources/xml/people_array.xml"); + RecordSchema schema = getSchemaWithNestedArray(); + XMLRecordReader reader = new XMLRecordReader(is, schema, true, null, + "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record firstRecord = reader.nextRecord(); + Object[] valuesFirstRecord = firstRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Cleve Butler", 42, "USA"}, Arrays.copyOfRange(valuesFirstRecord, 0, valuesFirstRecord.length - 1)); + + Record nestedArrayFirstRecord = (Record) firstRecord.getValue("CHILDREN"); + assertEquals(2, ((Object[]) nestedArrayFirstRecord.getValue("CHILD")).length); + Assert.assertArrayEquals(new Object[] {"child1", "child2"}, ((Object[]) nestedArrayFirstRecord.getValue("CHILD"))); + + Record secondRecord = reader.nextRecord(); + Object[] valuesSecondRecord = secondRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Ainslie Fletcher", 33, "UK"}, Arrays.copyOfRange(valuesSecondRecord, 0, valuesSecondRecord.length - 1)); + + Record nestedArraySecondRecord = (Record) secondRecord.getValue("CHILDREN"); + assertEquals(1, ((Object[]) nestedArraySecondRecord.getValue("CHILD")).length); + Assert.assertArrayEquals(new Object[] {"child1"}, ((Object[]) nestedArraySecondRecord.getValue("CHILD"))); + + Record thirdRecord = reader.nextRecord(); + Object[] valuesThirdRecord = thirdRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Amélie Bonfils", 74, "FR"}, Arrays.copyOfRange(valuesThirdRecord, 0, valuesThirdRecord.length - 1)); + + Record nestedArrayThirdRecord = (Record) thirdRecord.getValue("CHILDREN"); + assertEquals(3, ((Object[]) nestedArrayThirdRecord.getValue("CHILD")).length); + Assert.assertArrayEquals(new Object[] {"child1", "child2", "child3"}, ((Object[]) nestedArrayThirdRecord.getValue("CHILD"))); + + Record fourthRecord = reader.nextRecord(); + Object[] valuesFourthRecord = fourthRecord.getValues(); + Assert.assertArrayEquals(new Object[] {"Elenora Scrivens", 16, "USA"}, Arrays.copyOfRange(valuesFourthRecord, 0, valuesFourthRecord.length - 1)); + + Assert.assertEquals(null, fourthRecord.getValue("CHILDREN")); + } + + @Test + public void testDeeplyNestedArraysAndRecords() throws IOException, MalformedRecordException { + // test records in nested arrays + InputStream is = new FileInputStream("src/test/resources/xml/people_complex1.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaForComplexData(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(true, true); + Object[] grandchildren_arr = (Object[]) first.getValue("CHILDREN"); + + Record first_1_1_1 = (Record)(((Object[])((Record) grandchildren_arr[0]).getValue("CHILD"))[0]); + assertEquals("daughter", first_1_1_1.getValue("ROLE")); + assertEquals("1-1-1", first_1_1_1.getValue("ID")); + assertEquals("Selina", first_1_1_1.getValue("NAME")); + + Record first_1_1_2 = (Record)(((Object[])((Record) grandchildren_arr[0]).getValue("CHILD"))[1]); + assertEquals("son", first_1_1_2.getValue("ROLE")); + assertEquals("1-1-2", first_1_1_2.getValue("ID")); + assertEquals("Hans", first_1_1_2.getValue("NAME")); + + Record first_1_1_3 = (Record)(((Object[])((Record) grandchildren_arr[1]).getValue("CHILD"))[0]); + assertEquals("daughter", first_1_1_3.getValue("ROLE")); + assertEquals("1-2-1", first_1_1_3.getValue("ID")); + assertEquals("Selina2", first_1_1_3.getValue("NAME")); + + Record first_1_1_4 = (Record)(((Object[])((Record) grandchildren_arr[1]).getValue("CHILD"))[1]); + assertEquals("son", first_1_1_4.getValue("ROLE")); + assertEquals("1-2-2", first_1_1_4.getValue("ID")); + assertEquals("Hans2", first_1_1_4.getValue("NAME")); + + Record second = reader.nextRecord(true, true); + Object[] grandchildren_arr2 = (Object[]) second.getValue("CHILDREN"); + + Record second_2_1_1 = (Record)(((Object[])((Record) grandchildren_arr2[0]).getValue("CHILD"))[0]); + assertEquals("daughter", second_2_1_1.getValue("ROLE")); + assertEquals("2-1-1", second_2_1_1.getValue("ID")); + assertEquals("Selina3", second_2_1_1.getValue("NAME")); + } + + @Test + public void testDeeplyNestedArraysAndRecords2() throws IOException, MalformedRecordException { + // test multiply nested arrays and records (recursion) + InputStream is = new FileInputStream("src/test/resources/xml/people_complex2.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaForComplexData2(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(); + assertEquals("grandmother", first.getValue("ROLE")); + assertEquals("1", first.getValue("ID")); + assertEquals("Lisa", first.getValue("NAME")); + + Object[] gm_spouses = (Object[]) first.getValue("CHILDREN"); + assertEquals(2, gm_spouses.length); + + Object[] gm_spouse1_parents = (Object[]) ((Record) gm_spouses[0]).getValue("CHILD"); + assertEquals(2, gm_spouse1_parents.length); + + Record first_1_1 = (Record) gm_spouse1_parents[0]; + assertEquals("mother", first_1_1.getValue("ROLE")); + assertEquals("1-1", first_1_1.getValue("ID")); + assertEquals("Anna", first_1_1.getValue("NAME")); + + Object[] gm_spouse1_parent1_first_husband = (Object[]) first_1_1.getValue("CHILDREN"); + assertEquals(1, gm_spouse1_parent1_first_husband.length); + Object[] gm_spouse1_parent1_children = (Object[])((Record) gm_spouse1_parent1_first_husband[0]).getValue("CHILD"); + + Record first_1_1_1 = (Record) gm_spouse1_parent1_children[0]; + assertEquals("daughter", first_1_1_1.getValue("ROLE")); + assertEquals("1-1-1", first_1_1_1.getValue("ID")); + assertEquals("Selina", first_1_1_1.getValue("NAME")); + + Record first_1_1_2 = (Record) gm_spouse1_parent1_children[1]; + assertEquals("son", first_1_1_2.getValue("ROLE")); + assertEquals("1-1-2", first_1_1_2.getValue("ID")); + assertEquals("Hans", first_1_1_2.getValue("NAME")); + + Record first_1_2 = (Record) gm_spouse1_parents[1]; + assertEquals("mother", first_1_2.getValue("ROLE")); + assertEquals("1-2", first_1_2.getValue("ID")); + assertEquals("Catrina", first_1_2.getValue("NAME")); + + Object[] gm_spouse2_parents = (Object[]) ((Record) gm_spouses[1]).getValue("CHILD"); + assertEquals(1, gm_spouse2_parents.length); + + Record second = reader.nextRecord(); + Record second_2_1_1 = (Record)((Object[])((Record)((Object[])((Record)((Object[])((Record)((Object[]) second + .getValue("CHILDREN"))[0]) + .getValue("CHILD"))[0]) + .getValue("CHILDREN"))[0]) + .getValue("CHILD"))[0]; + assertEquals("daughter", second_2_1_1.getValue("ROLE")); + assertEquals("2-1-1", second_2_1_1.getValue("ID")); + assertEquals("Selina3", second_2_1_1.getValue("NAME")); + } + + @Test + public void testDeeplyNestedArraysAndRecordsCoerceFalseDropTrue() throws IOException, MalformedRecordException { + // test multiply nested arrays and records (recursion) + InputStream is = new FileInputStream("src/test/resources/xml/people_complex2.xml"); + XMLRecordReader reader = new XMLRecordReader(is, getSchemaForComplexData2(), true, + null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, true); + + assertEquals("grandmother", first.getValue("ROLE")); + assertEquals("1", first.getValue("ID")); + assertEquals("Lisa", first.getValue("NAME")); + + Object[] gm_spouses = (Object[]) first.getValue("CHILDREN"); + assertEquals(2, gm_spouses.length); + + Object[] gm_spouse1_parents = (Object[]) ((Record) gm_spouses[0]).getValue("CHILD"); + assertEquals(2, gm_spouse1_parents.length); + + Record first_1_1 = (Record) gm_spouse1_parents[0]; + assertEquals("mother", first_1_1.getValue("ROLE")); + assertEquals("1-1", first_1_1.getValue("ID")); + assertEquals("Anna", first_1_1.getValue("NAME")); + + Record gm_spouse1_parent1_first_husband = (Record) first_1_1.getValue("CHILDREN"); + Object[] gm_spouse1_parent1_children = (Object[])gm_spouse1_parent1_first_husband.getValue("CHILD"); + + Record first_1_1_1 = (Record) gm_spouse1_parent1_children[0]; + assertEquals("daughter", first_1_1_1.getValue("ROLE")); + assertEquals("1-1-1", first_1_1_1.getValue("ID")); + assertEquals("Selina", first_1_1_1.getValue("NAME")); + + Record first_1_1_2 = (Record) gm_spouse1_parent1_children[1]; + assertEquals("son", first_1_1_2.getValue("ROLE")); + assertEquals("1-1-2", first_1_1_2.getValue("ID")); + assertEquals("Hans", first_1_1_2.getValue("NAME")); + + Record first_1_2 = (Record) gm_spouse1_parents[1]; + assertEquals("mother", first_1_2.getValue("ROLE")); + assertEquals("1-2", first_1_2.getValue("ID")); + assertEquals("Catrina", first_1_2.getValue("NAME")); + + Record gm_spouse2_parents = (Record) ((Record) gm_spouses[1]).getValue("CHILD"); + assertEquals("1-3", gm_spouse2_parents.getValue("ID")); + + Record second = reader.nextRecord(false, true); + Record second_2_1_1 = (Record)((Record)((Record)((Record) second + .getValue("CHILDREN")) + .getValue("CHILD")) + .getValue("CHILDREN")) + .getValue("CHILD"); + assertEquals("daughter", second_2_1_1.getValue("ROLE")); + assertEquals("2-1-1", second_2_1_1.getValue("ID")); + assertEquals("Selina3", second_2_1_1.getValue("NAME")); + } + + @Test + public void testDeeplyNestedArraysAndRecordsCoerceFalseDropFalse() throws IOException, MalformedRecordException { + // test multiply nested arrays and records (recursion) + InputStream is = new FileInputStream("src/test/resources/xml/people_complex2.xml"); + XMLRecordReader reader = new XMLRecordReader(is, new SimpleRecordSchema(Collections.emptyList()), + true, null, "CONTENT", dateFormat, timeFormat, timestampFormat, Mockito.mock(ComponentLog.class)); + + Record first = reader.nextRecord(false, false); + assertEquals("1", first.getValue("ID")); + assertEquals("Lisa", first.getValue("NAME")); + assertEquals("grandmother", first.getValue("ROLE")); + Object[] gm_arr = (Object[]) first.getValue("CHILDREN"); + assertEquals(2, gm_arr.length); + + Record gm_hus1_arr_rec = (Record) gm_arr[0]; + assertEquals("husband1", gm_hus1_arr_rec.getValue("SPOUSE")); + Object[] gm_hus1_arr_rec_arr = (Object[]) gm_hus1_arr_rec.getValue("CHILD"); + assertEquals(2, gm_hus1_arr_rec_arr.length); + + Record child1_1 = (Record) gm_hus1_arr_rec_arr[0]; + assertEquals("1-1", child1_1.getValue("ID")); + assertEquals("Anna", child1_1.getValue("NAME")); + assertEquals("mother", child1_1.getValue("ROLE")); + + Record child1_1_rec = (Record) child1_1.getValue("CHILDREN"); + assertEquals("first husband", child1_1_rec.getValue("ID")); + Object[] child1_1_rec_arr = (Object[]) child1_1_rec.getValue("CHILD"); + assertEquals(2, child1_1_rec_arr.length); + + Record child1_1_1 = (Record) child1_1_rec_arr[0]; + assertEquals("1-1-1", child1_1_1.getValue("ID")); + assertEquals("Selina", child1_1_1.getValue("NAME")); + assertEquals("daughter", child1_1_1.getValue("ROLE")); + + Record child1_1_2 = (Record) child1_1_rec_arr[1]; + assertEquals("1-1-2", child1_1_2.getValue("ID")); + assertEquals("Hans", child1_1_2.getValue("NAME")); + assertEquals("son", child1_1_2.getValue("ROLE")); + + Record child1_2 = (Record) gm_hus1_arr_rec_arr[1]; + assertEquals("1-2", child1_2.getValue("ID")); + assertEquals("Catrina", child1_2.getValue("NAME")); + assertEquals("mother", child1_2.getValue("ROLE")); + + Record gm_hus2_arr_rec = (Record) gm_arr[1]; + assertEquals("husband2", gm_hus2_arr_rec.getValue("SPOUSE")); + + Record child1_3 = (Record) gm_hus2_arr_rec.getValue("CHILD"); + assertEquals("1-3", child1_3.getValue("ID")); + assertEquals("Anna2", child1_3.getValue("NAME")); + assertEquals("mother", child1_3.getValue("ROLE")); + assertEquals(2, ((Object[])((Record) child1_3.getValue("CHILDREN")).getValue("CHILD")).length); + + Record second = reader.nextRecord(false, false); + assertEquals("2-1-1", ((Record)((Record)((Record)((Record) second.getValue("CHILDREN")) + .getValue("CHILD")) + .getValue("CHILDREN")) + .getValue("CHILD")) + .getValue("ID")); + } + + private List<RecordField> getSimpleRecordFields() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("NAME", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("AGE", RecordFieldType.INT.getDataType())); + fields.add(new RecordField("COUNTRY", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getSimpleRecordFields2() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("NAME", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("COUNTRY", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getSimpleRecordFields3() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("AGE", RecordFieldType.INT.getDataType())); + fields.add(new RecordField("COUNTRY", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getNestedRecordFields() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("STREET", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("CITY", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getNestedRecordFields2() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("STREET", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getNestedRecordFields3() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("CONTENT", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("ATTR", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("INNER", RecordFieldType.STRING.getDataType())); + return fields; + } + + private List<RecordField> getNameSpaceFields() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("F:NAME", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("F:AGE", RecordFieldType.INT.getDataType())); + fields.add(new RecordField("F:COUNTRY", RecordFieldType.STRING.getDataType())); + return fields; + } + + private RecordSchema getSimpleSchema() { + return new SimpleRecordSchema(getSimpleRecordFields()); + } + + private RecordSchema getSimpleSchema2() { + return new SimpleRecordSchema(getSimpleRecordFields2()); + } + + private RecordSchema getNestedSchema() { + return new SimpleRecordSchema(getNestedRecordFields()); + } + + private RecordSchema getNestedSchema2() { + return new SimpleRecordSchema(getNestedRecordFields2()); + } + + private RecordSchema getNestedSchema3() { + return new SimpleRecordSchema(getNestedRecordFields3()); + } + + private RecordSchema getNameSpaceSchema() { + return new SimpleRecordSchema(getNameSpaceFields()); + } + + private RecordSchema getSchemaWithNestedRecord() { + final List<RecordField> fields = getSimpleRecordFields(); + final DataType recordType = RecordFieldType.RECORD.getRecordDataType(getNestedSchema()); + fields.add(new RecordField("ADDRESS", recordType)); + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaWithNestedRecord2() { + final List<RecordField> fields = getSimpleRecordFields2(); + final DataType recordType = RecordFieldType.RECORD.getRecordDataType(getNestedSchema2()); + fields.add(new RecordField("ADDRESS", recordType)); + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaWithNestedRecord3() { + final List<RecordField> fields = getSimpleRecordFields3(); + final DataType recordType = RecordFieldType.RECORD.getRecordDataType(getNestedSchema3()); + fields.add(new RecordField("NAME", recordType)); + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaWithSimpleArray() { + final List<RecordField> fields = getSimpleRecordFields(); + final DataType arrayType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()); + fields.add(new RecordField("CHILD", arrayType)); + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaWithNestedArray() { + final List<RecordField> fields = getSimpleRecordFields(); + final DataType arrayType = RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()); + final List<RecordField> nestedArrayField = new ArrayList<RecordField>() {{ add(new RecordField("CHILD", arrayType)); }}; + + final DataType recordType = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(nestedArrayField)); + fields.add(new RecordField("CHILDREN", recordType)); + return new SimpleRecordSchema(fields); + } + + private List<RecordField> getSimpleFieldsForComplexData() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("NAME", RecordFieldType.STRING.getDataType())); + fields.add(new RecordField("ROLE", RecordFieldType.STRING.getDataType())); + return fields; + } + + private RecordSchema getSchemaForMap() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + + final DataType map = RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()); + fields.add(new RecordField("MAP", map)); + + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaForRecordMap() { + final List<RecordField> fields = new ArrayList<>(); + fields.add(new RecordField("ID", RecordFieldType.STRING.getDataType())); + + final DataType record = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(getSimpleRecordFields())); + + final DataType map = RecordFieldType.MAP.getMapDataType(record); + fields.add(new RecordField("MAP", map)); + + return new SimpleRecordSchema(fields); + } + + private RecordSchema getSchemaForComplexData() { + final DataType grandchild = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(getSimpleFieldsForComplexData())); + final DataType grandchild_arr1 = RecordFieldType.ARRAY.getArrayDataType(grandchild); + + final DataType grandchildren = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema( + new ArrayList<RecordField>() {{ add(new RecordField("CHILD", grandchild_arr1)); }})); + final DataType grandchild_arr = RecordFieldType.ARRAY.getArrayDataType(grandchildren); + + return new SimpleRecordSchema( + new ArrayList<RecordField>() {{ add(new RecordField("CHILDREN", grandchild_arr)); }}); + } + + private RecordSchema getSchemaForComplexData2() { + final DataType grandchild = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema(getSimpleFieldsForComplexData())); + final DataType grandchild_arr = RecordFieldType.ARRAY.getArrayDataType(grandchild); + + final DataType grandchildren = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema( + new ArrayList<RecordField>() {{ add(new RecordField("CHILD", grandchild_arr)); }})); + final DataType grandchildren_arr = RecordFieldType.ARRAY.getArrayDataType(grandchildren); + + final DataType parent = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema( + new ArrayList<RecordField>() {{ + add(new RecordField("CHILDREN", grandchildren_arr)); + addAll(getSimpleFieldsForComplexData()); + }})); + final DataType parent_arr = RecordFieldType.ARRAY.getArrayDataType(parent); + + final DataType parents = RecordFieldType.RECORD.getRecordDataType(new SimpleRecordSchema( + new ArrayList<RecordField>() {{ + add(new RecordField("CHILD", parent_arr)); + }})); + final DataType parents_arr = RecordFieldType.ARRAY.getArrayDataType(parents); + + final List<RecordField> fields = new ArrayList<RecordField>() {{ + add(new RecordField("CHILDREN", parents_arr)); + addAll(getSimpleFieldsForComplexData()); + }}; + return new SimpleRecordSchema(fields); + } +}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d21bd387/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people.xml new file mode 100755 index 0000000..db5ba8d --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people.xml @@ -0,0 +1,22 @@ +<PEOPLE attr="attr1"> + <PERSON ID="P1"> + <NAME>Cleve Butler</NAME> + <AGE>42</AGE> + <COUNTRY>USA</COUNTRY> + </PERSON> + <PERSON ID="P2"> + <NAME>Ainslie Fletcher</NAME> + <AGE>33</AGE> + <COUNTRY>UK</COUNTRY> + </PERSON> + <PERSON ID="P3"> + <NAME>Amélie Bonfils</NAME> + <AGE>74</AGE> + <COUNTRY>FR</COUNTRY> + </PERSON> + <PERSON ID="P4"> + <NAME>Elenora Scrivens</NAME> + <AGE>16</AGE> + <COUNTRY>USA</COUNTRY> + </PERSON> +</PEOPLE> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/d21bd387/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people2.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people2.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people2.xml new file mode 100755 index 0000000..fee3389 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people2.xml @@ -0,0 +1,12 @@ +<PEOPLE attr="attr1"> + <PERSON ID="1"> + <NAME>Cleve Butler</NAME> + <AGE>42</AGE> + <COUNTRY>USA</COUNTRY> + </PERSON> + <PERSON ID="2"> + <NAME>Ainslie Fletcher</NAME> + <AGE>33</AGE> + <COUNTRY>UK</COUNTRY> + </PERSON> +</PEOPLE> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/d21bd387/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people3.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people3.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people3.xml new file mode 100755 index 0000000..fc85045 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/people3.xml @@ -0,0 +1,12 @@ +<PEOPLE> + <PERSON> + <NAME ID="name1">Cleve Butler</NAME> + <AGE ID="age1">42</AGE> + <COUNTRY>USA</COUNTRY> + </PERSON> + <PERSON> + <NAME ID="name2">Ainslie Fletcher</NAME> + <AGE ID="age2">33</AGE> + <COUNTRY>UK</COUNTRY> + </PERSON> +</PEOPLE> \ No newline at end of file
