Repository: nifi Updated Branches: refs/heads/master e8248353d -> 671445f10
NIFI-844: Initial import of the ConvertAvroToJSON processor Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a9c2bf58 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a9c2bf58 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a9c2bf58 Branch: refs/heads/master Commit: a9c2bf58b80ad94428d33cc014f11c8c7875592b Parents: 4f6c1cf Author: Mark Payne <[email protected]> Authored: Wed Aug 19 16:53:59 2015 -0400 Committer: Mark Payne <[email protected]> Committed: Fri Aug 21 17:11:56 2015 -0400 ---------------------------------------------------------------------- nifi-assembly/pom.xml | 6 +- .../nifi-avro-bundle/nifi-avro-nar/pom.xml | 37 ++++++ .../src/main/resources/META-INF/NOTICE | 71 +++++++++++ .../nifi-avro-processors/.gitignore | 1 + .../nifi-avro-processors/pom.xml | 72 +++++++++++ .../nifi/processors/avro/ConvertAvroToJSON.java | 122 ++++++++++++++++++ .../org.apache.nifi.processor.Processor | 15 +++ .../processors/avro/TestConvertAvroToJSON.java | 102 +++++++++++++++ .../src/test/resources/user.avsc | 9 ++ nifi-nar-bundles/nifi-avro-bundle/pom.xml | 35 ++++++ .../src/main/resources/META-INF/NOTICE | 18 +++ .../nifi-standard-processors/pom.xml | 2 +- nifi-nar-bundles/nifi-standard-bundle/pom.xml | 5 - nifi-nar-bundles/pom.xml | 1 + pom.xml | 124 +++++++++++-------- 15 files changed, 561 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index bd4a5aa..0431a76 100644 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -219,8 +219,12 @@ language governing permissions and limitations under the License. --> </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-image-nar</artifactId> - <version>0.3.0-SNAPSHOT</version> <type>nar</type> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/pom.xml b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/pom.xml new file mode 100644 index 0000000..52cc180 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/pom.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-bundle</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-avro-nar</artifactId> + <version>0.3.0-SNAPSHOT</version> + <packaging>nar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-processors</artifactId> + <version>0.3.0-SNAPSHOT</version> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/src/main/resources/META-INF/NOTICE ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000..e5164e4 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,71 @@ +nifi-avro-nar +Copyright 2014-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2015 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Apache Commons Compress + The following NOTICE information applies: + Apache Commons Compress + Copyright 2002-2014 The Apache Software Foundation + + The files in the package org.apache.commons.compress.archivers.sevenz + were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), + which has been placed in the public domain: + + "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + + (ASLv2) Apache Avro + The following NOTICE information applies: + Apache Avro + Copyright 2009-2013 The Apache Software Foundation + + (ASLv2) Snappy Java + The following NOTICE information applies: + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta ([email protected]), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/.gitignore ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/.gitignore b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/.gitignore @@ -0,0 +1 @@ +/target/ http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/pom.xml b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/pom.xml new file mode 100644 index 0000000..989f762 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/pom.xml @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-bundle</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + + <artifactId>nifi-avro-processors</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-processor-utils</artifactId> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes combine.children="append"> + <exclude>src/test/resources/user.avsc</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java new file mode 100644 index 0000000..8832a73 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/java/org/apache/nifi/processors/avro/ConvertAvroToJSON.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.avro; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Set; + +import org.apache.avro.file.DataFileStream; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericRecord; +import org.apache.nifi.annotation.behavior.SideEffectFree; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.io.StreamCallback; + +@SideEffectFree +@SupportsBatching +@Tags({ "json", "avro", "binary" }) +@CapabilityDescription("Converts a Binary Avro record into a JSON object. This processor provides a direct mapping of an Avro field to a JSON field, such " + + "that the resulting JSON will have the same hierarchical structure as the Avro document. Note that the Avro schema information will be lost, as this " + + "is not a translation from binary Avro to JSON formatted Avro. The output JSON is encoded the UTF-8 encoding. If an incoming FlowFile contains a stream of " + + "multiple Avro records, the resultant FlowFile will contain a JSON Array containing all of the Avro records.") +@WritesAttribute(attribute = "mime.type", description = "Sets the mime type to application/json") +public class ConvertAvroToJSON extends AbstractProcessor { + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("A FlowFile is routed to this relationship after it has been converted to JSON") + .build(); + static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("A FlowFile is routed to this relationship if it cannot be parsed as Avro or cannot be converted to JSON for any reason") + .build(); + + @Override + public Set<Relationship> getRelationships() { + final Set<Relationship> rels = new HashSet<>(); + rels.add(REL_SUCCESS); + rels.add(REL_FAILURE); + return rels; + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + FlowFile flowFile = session.get(); + if (flowFile == null) { + return; + } + + try { + flowFile = session.write(flowFile, new StreamCallback() { + @Override + public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException { + try (final InputStream in = new BufferedInputStream(rawIn); + final OutputStream out = new BufferedOutputStream(rawOut); + final DataFileStream<GenericRecord> reader = new DataFileStream<>(in, new GenericDatumReader<GenericRecord>())) { + + final GenericData genericData = GenericData.get(); + GenericRecord record = reader.next(); + final String json = genericData.toString(record); + + int recordCount = 0; + if (reader.hasNext()) { + out.write('['); + } + + out.write(json.getBytes(StandardCharsets.UTF_8)); + recordCount++; + + while (reader.hasNext()) { + out.write(','); + final GenericRecord nextRecord = reader.next(record); + out.write(genericData.toString(nextRecord).getBytes(StandardCharsets.UTF_8)); + recordCount++; + } + + if (recordCount > 1) { + out.write(']'); + } + } + } + }); + } catch (final ProcessException pe) { + getLogger().error("Failed to convert {} from Avro to JSON due to {}; transferring to failure", new Object[] {flowFile, pe}); + session.transfer(flowFile, REL_FAILURE); + return; + } + + flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json"); + session.transfer(flowFile, REL_SUCCESS); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 0000000..67a6cd3 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.avro.ConvertAvroToJSON \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/java/org/apache/nifi/processors/avro/TestConvertAvroToJSON.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/java/org/apache/nifi/processors/avro/TestConvertAvroToJSON.java b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/java/org/apache/nifi/processors/avro/TestConvertAvroToJSON.java new file mode 100644 index 0000000..2d84202 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/java/org/apache/nifi/processors/avro/TestConvertAvroToJSON.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.avro; + +import java.io.File; +import java.io.IOException; + +import org.apache.avro.Schema; +import org.apache.avro.file.DataFileWriter; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.io.DatumWriter; +import org.apache.nifi.processors.avro.ConvertAvroToJSON; +import org.apache.nifi.stream.io.ByteArrayOutputStream; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Test; + +public class TestConvertAvroToJSON { + + @Test + public void testSingleAvroMessage() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON()); + final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc")); + + final GenericRecord user1 = new GenericData.Record(schema); + user1.put("name", "Alyssa"); + user1.put("favorite_number", 256); + + final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema); + final ByteArrayOutputStream out1 = serializeAvroRecord(schema, datumWriter, user1); + runner.enqueue(out1.toByteArray()); + + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0); + out.assertContentEquals("{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null}"); + } + + @Test + public void testMultipleAvroMessages() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON()); + final Schema schema = new Schema.Parser().parse(new File("src/test/resources/user.avsc")); + + final GenericRecord user1 = new GenericData.Record(schema); + user1.put("name", "Alyssa"); + user1.put("favorite_number", 256); + + final GenericRecord user2 = new GenericData.Record(schema); + user2.put("name", "George"); + user2.put("favorite_number", 1024); + user2.put("favorite_color", "red"); + + final DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(schema); + final ByteArrayOutputStream out1 = serializeAvroRecord(schema, datumWriter, user1, user2); + runner.enqueue(out1.toByteArray()); + + runner.run(); + + runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_SUCCESS, 1); + final MockFlowFile out = runner.getFlowFilesForRelationship(ConvertAvroToJSON.REL_SUCCESS).get(0); + out.assertContentEquals("[{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null},{\"name\": \"George\", \"favorite_number\": 1024, \"favorite_color\": \"red\"}]"); + } + + @Test + public void testNonJsonHandledProperly() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new ConvertAvroToJSON()); + runner.enqueue("hello".getBytes()); + runner.run(); + runner.assertAllFlowFilesTransferred(ConvertAvroToJSON.REL_FAILURE, 1); + } + + private ByteArrayOutputStream serializeAvroRecord(final Schema schema, final DatumWriter<GenericRecord> datumWriter, final GenericRecord... users) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter); + dataFileWriter.create(schema, out); + for (final GenericRecord user : users) { + dataFileWriter.append(user); + } + + dataFileWriter.close(); + return out; + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/resources/user.avsc ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/resources/user.avsc b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/resources/user.avsc new file mode 100644 index 0000000..117ea70 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/nifi-avro-processors/src/test/resources/user.avsc @@ -0,0 +1,9 @@ +{"namespace": "example.avro", + "type": "record", + "name": "User", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_number", "type": ["int", "null"]}, + {"name": "favorite_color", "type": ["string", "null"]} + ] +} http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-avro-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-avro-bundle/pom.xml b/nifi-nar-bundles/nifi-avro-bundle/pom.xml new file mode 100644 index 0000000..fe523e7 --- /dev/null +++ b/nifi-nar-bundles/nifi-avro-bundle/pom.xml @@ -0,0 +1,35 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-nar-bundles</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-bundle</artifactId> + <version>0.3.0-SNAPSHOT</version> + <packaging>pom</packaging> + + <modules> + <module>nifi-avro-processors</module> + <module>nifi-avro-nar</module> + </modules> + +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE index 152e3bc..85d8e65 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/src/main/resources/META-INF/NOTICE @@ -137,6 +137,24 @@ The following binary components are provided under the Apache Software License v in some artifacts (usually source distributions); but is always available from the source code management (SCM) system project uses. + (ASLv2) Apache Avro + The following NOTICE information applies: + Apache Avro + Copyright 2009-2013 The Apache Software Foundation + + (ASLv2) Snappy Java + The following NOTICE information applies: + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + ************************ Common Development and Distribution License 1.1 ************************ http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 729b495..3c68b17 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -190,8 +190,8 @@ language governing permissions and limitations under the License. --> <artifactId>derby</artifactId> <scope>test</scope> </dependency> - </dependencies> + <build> <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/nifi-standard-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 481241a..b7cc2b4 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -53,11 +53,6 @@ <type>war</type> <version>0.3.0-SNAPSHOT</version> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>2.4.5</version> - </dependency> </dependencies> </dependencyManagement> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/nifi-nar-bundles/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/pom.xml b/nifi-nar-bundles/pom.xml index 1be2cfa..4d9edbf 100644 --- a/nifi-nar-bundles/pom.xml +++ b/nifi-nar-bundles/pom.xml @@ -44,6 +44,7 @@ <module>nifi-flume-bundle</module> <module>nifi-ambari-bundle</module> <module>nifi-image-bundle</module> + <module>nifi-avro-bundle</module> </modules> <dependencyManagement> <dependencies> http://git-wip-us.apache.org/repos/asf/nifi/blob/a9c2bf58/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index af1c8fb..03f3e02 100644 --- a/pom.xml +++ b/pom.xml @@ -1,19 +1,16 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache</groupId> @@ -83,7 +80,7 @@ <issueManagement> <system>JIRA</system> <url>https://issues.apache.org/jira/browse/NIFI</url> - </issueManagement> + </issueManagement> <properties> <maven.compiler.source>1.7</maven.compiler.source> <maven.compiler.target>1.7</maven.compiler.target> @@ -188,7 +185,7 @@ <groupId>com.wordnik</groupId> <artifactId>swagger-annotations</artifactId> <version>1.5.3-M1</version> - </dependency> + </dependency> <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> @@ -254,10 +251,9 @@ <artifactId>quartz</artifactId> <version>2.2.1</version> <exclusions> - <!-- - | Exclude the quartz 2.2.1 bundled version of c3p0 because it is lgpl licensed - | We also don't use the JDBC related features of quartz for which the dependency would matter - --> + <!-- | Exclude the quartz 2.2.1 bundled version of c3p0 because it is + lgpl licensed | We also don't use the JDBC related features of quartz for + which the dependency would matter --> <exclusion> <groupId>c3p0</groupId> <artifactId>c3p0</artifactId> @@ -326,8 +322,8 @@ <artifactId>spring-core</artifactId> <version>${spring.version}</version> <exclusions> - <!-- <artifactId>jcl-over-slf4j</artifactId> is used in dependencies - section --> + <!-- <artifactId>jcl-over-slf4j</artifactId> is used in dependencies + section --> <exclusion> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> @@ -670,11 +666,16 @@ <artifactId>jasypt</artifactId> <version>1.9.2</version> </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - <version>1.9.13</version> - </dependency> + <dependency> + <groupId>org.codehaus.jackson</groupId> + <artifactId>jackson-mapper-asl</artifactId> + <version>1.9.13</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.6.1</version> + </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> @@ -896,6 +897,18 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-avro-nar</artifactId> + <version>0.3.0-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-image-nar</artifactId> + <version>0.3.0-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-properties</artifactId> <version>0.3.0-SNAPSHOT</version> </dependency> @@ -1101,7 +1114,7 @@ </dependency> </dependencies> </plugin> - </plugins> + </plugins> </pluginManagement> <plugins> <plugin> @@ -1141,7 +1154,7 @@ <requireMavenVersion> <version>${maven.min-version}</version> </requireMavenVersion> - </rules> + </rules> </configuration> </execution> </executions> @@ -1154,7 +1167,7 @@ <module name="Checker"> <property name="charset" value="UTF-8" /> <property name="severity" value="warning" /> - <!-- Checks for whitespace --> + <!-- Checks for whitespace --> <!-- See http://checkstyle.sf.net/config_whitespace.html --> <module name="FileTabCharacter"> <property name="eachLine" value="true" /> @@ -1166,13 +1179,16 @@ </module> <module name="RegexpSinglelineJava"> <property name="format" value="[@]see\s+[{][@]link" /> - <property name="message" value="Javadoc @see does not need @link: pick one or the other." /> + <property name="message" + value="Javadoc @see does not need @link: pick one or the other." /> </module> <module name="OuterTypeFilename" /> <module name="LineLength"> - <!-- needs extra, because Eclipse formatter ignores the ending left brace --> + <!-- needs extra, because Eclipse formatter ignores the ending left + brace --> <property name="max" value="200" /> - <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"; /> + <property name="ignorePattern" + value="^package.*|^import.*|a href|href|http://|https://|ftp://"; /> </module> <module name="AvoidStarImport" /> <module name="UnusedImports"> @@ -1185,7 +1201,8 @@ <module name="RightCurly" /> <module name="RightCurly"> <property name="option" value="alone" /> - <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT" /> + <property name="tokens" + value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT" /> </module> <module name="SeparatorWrap"> <property name="tokens" value="DOT" /> @@ -1199,19 +1216,23 @@ <property name="format" value="^[a-z]+(\.[a-z][a-zA-Z0-9]*)*$" /> </module> <module name="MethodTypeParameterName"> - <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)" /> + <property name="format" + value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)" /> </module> <module name="MethodParamPad" /> <module name="OperatorWrap"> <property name="option" value="NL" /> - <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, QUESTION, SL, SR, STAR " /> + <property name="tokens" + value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, QUESTION, SL, SR, STAR " /> </module> <module name="AnnotationLocation"> - <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF" /> + <property name="tokens" + value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF" /> </module> <module name="AnnotationLocation"> <property name="tokens" value="VARIABLE_DEF" /> - <property name="allowSamelineMultipleAnnotations" value="true" /> + <property name="allowSamelineMultipleAnnotations" + value="true" /> </module> <module name="NonEmptyAtclauseDescription" /> <module name="JavadocMethod"> @@ -1219,7 +1240,8 @@ <property name="allowMissingParamTags" value="true" /> <property name="allowMissingThrowsTags" value="true" /> <property name="allowMissingReturnTag" value="true" /> - <property name="allowedAnnotations" value="Override,Test,BeforeClass,AfterClass,Before,After" /> + <property name="allowedAnnotations" + value="Override,Test,BeforeClass,AfterClass,Before,After" /> <property name="allowThrowsTagsForSubclasses" value="true" /> </module> <module name="SingleLineJavadoc" /> @@ -1229,7 +1251,7 @@ <violationSeverity>warning</violationSeverity> <includeTestSourceDirectory>true</includeTestSourceDirectory> </configuration> - </plugin> + </plugin> <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> @@ -1254,19 +1276,17 @@ </exclusions> </dependency> </dependencies> - </plugin> + </plugin> </plugins> </build> <profiles> <profile> - <!-- - Checks style and licensing requirements. This is a good idea to - run for contributions and for the release process. While it - would be nice to run always these plugins can considerably slow - the build and have proven to create unstable builds in our - multi-module project and when building using multiple threads. - The stability issues seen with Checkstyle in multi-module builds - include false-positives and false negatives.--> + <!-- Checks style and licensing requirements. This is a good idea to run + for contributions and for the release process. While it would be nice to + run always these plugins can considerably slow the build and have proven + to create unstable builds in our multi-module project and when building using + multiple threads. The stability issues seen with Checkstyle in multi-module + builds include false-positives and false negatives. --> <id>contrib-check</id> <build> <plugins> @@ -1292,10 +1312,10 @@ <goal>check</goal> </goals> </execution> - </executions> + </executions> </plugin> </plugins> </build> </profile> - </profiles> + </profiles> </project>
