http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/README.md ---------------------------------------------------------------------- diff --git a/nifi-assembly/README.md b/nifi-assembly/README.md new file mode 100644 index 0000000..4fb4d3b --- /dev/null +++ b/nifi-assembly/README.md @@ -0,0 +1,110 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +# Apache NiFi + +Apache NiFi is an easy to use, powerful, and reliable system to process and distribute data. + +## Table of Contents + +- [Features](#features) +- [Getting Started](#getting-started) +- [Getting Help](#getting-help) +- [Requirements](#requirements) +- [License](#license) +- [Export Control] (#export-control) + +## Features + +Apache NiFi was made for dataflow. It supports highly configurable directed graphs of data routing, transformation, and system mediation logic. Some of its key features include: + +- Web-based user interface + - Seamless experience for design, control, and monitoring +- Highly configurable + - Loss tolerant vs guaranteed delivery + - Low latency vs high throughput + - Dynamic prioritization + - Flows can be modified at runtime + - Back pressure +- Data Provenance + - Track dataflow from beginning to end +- Designed for extension + - Build your own processors and more + - Enables rapid development and effective testing +- Secure + - SSL, SSH, HTTPS, encrypted content, etc... + - Pluggable role-based authentication/authorization + +## Getting Started + +To start NiFi: +- [linux/osx] execute bin/nifi.sh start +- [windows] execute bin/start-nifi.bat +- Direct your browser to http://localhost:8080/nifi/ + +## Getting Help +If you have questions, you can reach out to our mailing list: [email protected] +([archive](http://mail-archives.apache.org/mod_mbox/nifi-dev)). +We're also often available in IRC: #nifi on +[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi). + +## Requirements +* JDK 1.7 or higher + +## License + +Except as otherwise noted this software is licensed under the +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Export Control + +This distribution includes cryptographic software. The country in which you +currently reside may have restrictions on the import, possession, use, and/or +re-export to another country, of encryption software. BEFORE using any +encryption software, please check your country's laws, regulations and +policies concerning the import, possession, or use, and re-export of encryption +software, to see if this is permitted. See <http://www.wassenaar.org/> for more +information. + +The U.S. Government Department of Commerce, Bureau of Industry and Security +(BIS), has classified this software as Export Commodity Control Number (ECCN) +5D002.C.1, which includes information security software using or performing +cryptographic functions with asymmetric algorithms. The form and manner of this +Apache Software Foundation distribution makes it eligible for export under the +License Exception ENC Technology Software Unrestricted (TSU) exception (see the +BIS Export Administration Regulations, Section 740.13) for both object code and +source code. + +The following provides more details on the included cryptographic software: + +Apache NiFi uses BouncyCastle, Jasypt, JCraft Inc., and the built-in +java cryptography libraries for SSL, SSH, and the protection +of sensitive configuration parameters. See +http://bouncycastle.org/about.html +http://www.jasypt.org/faq.html +http://jcraft.com/c-info.html +http://www.oracle.com/us/products/export/export-regulations-345813.html +for more details on each of these libraries cryptography features.
http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml new file mode 100644 index 0000000..60a6545 --- /dev/null +++ b/nifi-assembly/pom.xml @@ -0,0 +1,501 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor +license agreements. See the NOTICE file distributed with this work for additional +information regarding copyright ownership. The ASF licenses this file to +You under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +OF ANY KIND, either express or implied. See the License for the specific +language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>nifi-assembly</artifactId> + <packaging>pom</packaging> + <description>This is the assembly Apache NiFi</description> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <finalName>nifi-${project.version}</finalName> + <attach>false</attach> + </configuration> + <executions> + <execution> + <id>make shared resource</id> + <goals> + <goal>single</goal> + </goals> + <phase>package</phase> + <configuration> + <archiverConfig> + <defaultDirectoryMode>0775</defaultDirectoryMode> + <directoryMode>0775</directoryMode> + <fileMode>0664</fileMode> + </archiverConfig> + <descriptors> + <descriptor>src/main/assembly/dependencies.xml</descriptor> + </descriptors> + <tarLongFileMode>posix</tarLongFileMode> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencies> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-runtime</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-bootstrap</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-resources</artifactId> + <classifier>resources</classifier> + <scope>runtime</scope> + <type>zip</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-docs</artifactId> + <classifier>resources</classifier> + <scope>runtime</scope> + <type>zip</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-provenance-repository-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-services-api-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-ssl-context-service-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-distributed-cache-services-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-jetty-bundle</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-update-attribute-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hadoop-libraries-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hadoop-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kafka-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-http-context-map-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-kite-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-flume-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-dbcp-service-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mongodb-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-solr-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-social-media-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-hl7-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-language-translation-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-geo-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-aws-nar</artifactId> + <type>nar</type> + </dependency> + </dependencies> + + <properties> + <!--Wrapper Properties --> + <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb> + <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb> + <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb> + <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb> + <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize> + <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles> + + <!-- nifi.properties: core properties --> + <nifi.version>${project.version}</nifi.version> + <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> + <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> + <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> + <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration> + <nifi.bored.yield.duration>10 millis</nifi.bored.yield.duration> + + <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file> + <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir> + <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file> + <nifi.templates.directory>./conf/templates</nifi.templates.directory> + <nifi.database.directory>./database_repository</nifi.database.directory> + + <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation> + <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory> + <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions> + <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval> + <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync> + <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation> + <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold> + <nifi.swap.in.period>5 sec</nifi.swap.in.period> + <nifi.swap.in.threads>1</nifi.swap.in.threads> + <nifi.swap.out.period>5 sec</nifi.swap.out.period> + <nifi.swap.out.threads>4</nifi.swap.out.threads> + + <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation> + <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size> + <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files> + <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default> + <nifi.content.repository.archive.max.retention.period>12 hours</nifi.content.repository.archive.max.retention.period> + <nifi.content.repository.archive.max.usage.percentage>50%</nifi.content.repository.archive.max.usage.percentage> + <nifi.content.repository.archive.enabled>true</nifi.content.repository.archive.enabled> + <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync> + <nifi.content.viewer.url>/nifi-content-viewer/</nifi.content.viewer.url> + + <nifi.restore.directory /> + <nifi.ui.banner.text /> + <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval> + <nifi.nar.library.directory>./lib</nifi.nar.library.directory> + <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> + <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> + + <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> + <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> + <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> + + <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> + + <!-- persistent provenance repository properties --> + <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> + <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> + <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> + <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> + <nifi.provenance.repository.rollover.time>30 secs</nifi.provenance.repository.rollover.time> + <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size> + <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads> + <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover> + <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID, Relationship</nifi.provenance.repository.indexed.fields> + <nifi.provenance.repository.indexed.attributes /> + <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size> + <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync> + <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count> + <nifi.provenance.repository.max.attribute.length>65536</nifi.provenance.repository.max.attribute.length> + + <!-- volatile provenance repository properties --> + <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size> + + <!-- Component status repository properties --> + <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation> + <nifi.components.status.repository.buffer.size>1440</nifi.components.status.repository.buffer.size> + <nifi.components.status.snapshot.frequency>1 min</nifi.components.status.snapshot.frequency> + + <!-- nifi.properties: web properties --> + <nifi.web.war.directory>./lib</nifi.web.war.directory> + <nifi.web.http.host /> + <nifi.web.http.port>8080</nifi.web.http.port> + <nifi.web.https.host /> + <nifi.web.https.port /> + <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir> + <nifi.web.jetty.threads>200</nifi.web.jetty.threads> + + <!-- nifi.properties: security properties --> + <nifi.security.keystore /> + <nifi.security.keystoreType /> + <nifi.security.keystorePasswd /> + <nifi.security.keyPasswd /> + <nifi.security.truststore /> + <nifi.security.truststoreType /> + <nifi.security.truststorePasswd /> + <nifi.security.needClientAuth /> + <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file> + <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration> + <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider> + <nifi.security.x509.principal.extractor /> + <nifi.security.support.new.account.requests /> + <nifi.security.ocsp.responder.url /> + <nifi.security.ocsp.responder.certificate /> + + <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) --> + <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> + <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> + <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout> + <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout> + <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast> + <nifi.cluster.protocol.multicast.address /> + <nifi.cluster.protocol.multicast.port /> + <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay> + <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> + <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> + + <!-- nifi.properties: cluster node properties (only configure for cluster nodes) --> + <nifi.cluster.is.node>false</nifi.cluster.is.node> + <nifi.cluster.node.address /> + <nifi.cluster.node.protocol.port /> + <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads> + <nifi.cluster.node.unicast.manager.address /> + <nifi.cluster.node.unicast.manager.protocol.port /> + + <!-- nifi.properties: cluster manager properties (only configure for cluster manager) --> + <nifi.cluster.is.manager>false</nifi.cluster.is.manager> + <nifi.cluster.manager.address /> + <nifi.cluster.manager.protocol.port /> + <nifi.cluster.manager.node.firewall.file /> + <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size> + <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout> + <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout> + <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads> + <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay> + <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads> + <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration> + </properties> + <profiles> + <profile> + <id>rpm</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>unpack-shared-resources</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <phase>generate-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/generated-resources</outputDirectory> + <includeArtifactIds>nifi-resources</includeArtifactIds> + <includeGroupIds>org.apache.nifi</includeGroupIds> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + <execution> + <id>unpack-docs</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <phase>generate-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/generated-docs</outputDirectory> + <includeArtifactIds>nifi-docs</includeArtifactIds> + <includeGroupIds>org.apache.nifi</includeGroupIds> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rpm-maven-plugin</artifactId> + <configuration> + <summary>Apache NiFi</summary> + <description>Apache NiFi is dataflow system based on the Flow-Based Programming concepts.</description> + <license>Apache License, Version 2.0 and others (see included LICENSE file)</license> + <url>http://nifi.apache.org</url> + <group>Utilities</group> + <prefix>/opt/nifi</prefix> + <defineStatements> + <defineStatement>_use_internal_dependency_generator 0</defineStatement> + </defineStatements> + <defaultDirmode>750</defaultDirmode> + <defaultFilemode>640</defaultFilemode> + <defaultUsername>root</defaultUsername> + <defaultGroupname>root</defaultGroupname> + </configuration> + <executions> + <execution> + <id>build-bin-rpm</id> + <goals> + <goal>attached-rpm</goal> + </goals> + <configuration> + <classifier>bin</classifier> + <provides> + <provide>nifi</provide> + </provides> + <mappings> + <mapping> + <directory>/opt/nifi/nifi-${project.version}</directory> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}</directory> + <sources> + <source> + <location>./LICENSE</location> + </source> + <source> + <location>./NOTICE</location> + </source> + <source> + <location>./README.md</location> + <destination>README</destination> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/bin</directory> + <filemode>750</filemode> + <sources> + <source> + <location>${project.build.directory}/generated-resources/bin/nifi.sh</location> + <destination>nifi.sh</destination> + <filter>true</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/conf</directory> + <configuration>true</configuration> + <sources> + <source> + <location>${project.build.directory}/generated-resources/conf</location> + <filter>true</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/lib</directory> + <dependency> + <excludes> + <exclude>org.apache.nifi:nifi-bootstrap</exclude> + <exclude>org.apache.nifi:nifi-resources</exclude> + <exclude>org.apache.nifi:nifi-docs</exclude> + </excludes> + </dependency> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory> + <dependency> + <includes> + <include>org.apache.nifi:nifi-bootstrap</include> + </includes> + </dependency> + </mapping> + <mapping> + <directory>/opt/nifi/nifi-${project.version}/docs</directory> + <sources> + <source> + <location>${project.build.directory}/generated-docs</location> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-assembly/src/main/assembly/dependencies.xml ---------------------------------------------------------------------- diff --git a/nifi-assembly/src/main/assembly/dependencies.xml b/nifi-assembly/src/main/assembly/dependencies.xml new file mode 100644 index 0000000..0e5362c --- /dev/null +++ b/nifi-assembly/src/main/assembly/dependencies.xml @@ -0,0 +1,140 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>bin</id> + <formats> + <format>dir</format> + <format>zip</format> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>nifi-${project.version}</baseDirectory> + + <dependencySets> + <!-- Write out all dependency artifacts to lib directory --> + <dependencySet> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>lib</outputDirectory> + <directoryMode>0770</directoryMode> + <fileMode>0660</fileMode> + <useTransitiveFiltering>true</useTransitiveFiltering> + <excludes> + <exclude>nifi-bootstrap</exclude> + <exclude>nifi-resources</exclude> + <exclude>nifi-docs</exclude> + </excludes> + </dependencySet> + + <!-- Write out the bootstrap lib component to its own dir --> + <dependencySet> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>lib/bootstrap</outputDirectory> + <directoryMode>0770</directoryMode> + <fileMode>0660</fileMode> + <useTransitiveFiltering>true</useTransitiveFiltering> + <includes> + <include>nifi-bootstrap</include> + <include>slf4j-api</include> + <include>logback-classic</include> + </includes> + </dependencySet> + + <!-- Write out the conf directory contents --> + <dependencySet> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./</outputDirectory> + <directoryMode>0770</directoryMode> + <fileMode>0664</fileMode> + <useTransitiveFiltering>true</useTransitiveFiltering> + <includes> + <include>nifi-resources</include> + </includes> + <unpack>true</unpack> + <unpackOptions> + <filtered>true</filtered> + <includes> + <include>conf/*</include> + </includes> + </unpackOptions> + </dependencySet> + + <!-- Write out the bin directory contents --> + <dependencySet> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>./</outputDirectory> + <directoryMode>0770</directoryMode> + <fileMode>0770</fileMode> + <useTransitiveFiltering>true</useTransitiveFiltering> + <includes> + <include>nifi-resources</include> + </includes> + <unpack>true</unpack> + <unpackOptions> + <filtered>true</filtered> + <includes> + <include>bin/*</include> + </includes> + </unpackOptions> + </dependencySet> + + <!-- Writes out the docs directory contents --> + <dependencySet> + <scope>runtime</scope> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>docs/</outputDirectory> + <useTransitiveFiltering>true</useTransitiveFiltering> + <includes> + <include>nifi-docs</include> + </includes> + <unpack>true</unpack> + <unpackOptions> + <filtered>false</filtered> + <excludes> + <!-- LICENSE and NOTICE both covered by top-level --> + <exclude>LICENSE</exclude> + <exclude>NOTICE</exclude> + </excludes> + </unpackOptions> + </dependencySet> + </dependencySets> + <files> + <file> + <source>./README.md</source> + <outputDirectory>./</outputDirectory> + <destName>README</destName> + <fileMode>0644</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>./LICENSE</source> + <outputDirectory>./</outputDirectory> + <destName>LICENSE</destName> + <fileMode>0644</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>./NOTICE</source> + <outputDirectory>./</outputDirectory> + <destName>NOTICE</destName> + <fileMode>0644</fileMode> + <filtered>true</filtered> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-bootstrap/pom.xml b/nifi-bootstrap/pom.xml new file mode 100644 index 0000000..2975a5a --- /dev/null +++ b/nifi-bootstrap/pom.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi</artifactId> + <version>0.3.0-SNAPSHOT</version> + </parent> + <artifactId>nifi-bootstrap</artifactId> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java ---------------------------------------------------------------------- diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java new file mode 100644 index 0000000..3817875 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/BootstrapCodec.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.Arrays; + +import org.apache.nifi.bootstrap.exception.InvalidCommandException; + +public class BootstrapCodec { + + private final RunNiFi runner; + private final BufferedReader reader; + private final BufferedWriter writer; + + public BootstrapCodec(final RunNiFi runner, final InputStream in, final OutputStream out) { + this.runner = runner; + this.reader = new BufferedReader(new InputStreamReader(in)); + this.writer = new BufferedWriter(new OutputStreamWriter(out)); + } + + public void communicate() throws IOException { + final String line = reader.readLine(); + final String[] splits = line.split(" "); + if (splits.length < 0) { + throw new IOException("Received invalid command from NiFi: " + line); + } + + final String cmd = splits[0]; + final String[] args; + if (splits.length == 1) { + args = new String[0]; + } else { + args = Arrays.copyOfRange(splits, 1, splits.length); + } + + try { + processRequest(cmd, args); + } catch (final InvalidCommandException ice) { + throw new IOException("Received invalid command from NiFi: " + line + " : " + ice.getMessage() == null ? "" : "Details: " + ice.toString()); + } + } + + private void processRequest(final String cmd, final String[] args) throws InvalidCommandException, IOException { + switch (cmd) { + case "PORT": { + if (args.length != 2) { + throw new InvalidCommandException(); + } + + final int port; + try { + port = Integer.parseInt(args[0]); + } catch (final NumberFormatException nfe) { + throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); + } + + if (port < 1 || port > 65535) { + throw new InvalidCommandException("Invalid Port number; should be integer between 1 and 65535"); + } + + final String secretKey = args[1]; + + runner.setNiFiCommandControlPort(port, secretKey); + writer.write("OK"); + writer.newLine(); + writer.flush(); + } + break; + } + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/aa998847/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java ---------------------------------------------------------------------- diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java new file mode 100644 index 0000000..8d74f16 --- /dev/null +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/NiFiListener.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.bootstrap; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.bootstrap.util.LimitingInputStream; + +public class NiFiListener { + + private ServerSocket serverSocket; + private volatile Listener listener; + + int start(final RunNiFi runner) throws IOException { + serverSocket = new ServerSocket(); + serverSocket.bind(new InetSocketAddress("localhost", 0)); + + final int localPort = serverSocket.getLocalPort(); + listener = new Listener(serverSocket, runner); + final Thread listenThread = new Thread(listener); + listenThread.setName("Listen to NiFi"); + listenThread.setDaemon(true); + listenThread.start(); + return localPort; + } + + public void stop() throws IOException { + final Listener listener = this.listener; + if (listener == null) { + return; + } + + listener.stop(); + } + + private class Listener implements Runnable { + + private final ServerSocket serverSocket; + private final ExecutorService executor; + private final RunNiFi runner; + private volatile boolean stopped = false; + + public Listener(final ServerSocket serverSocket, final RunNiFi runner) { + this.serverSocket = serverSocket; + this.executor = Executors.newFixedThreadPool(2, new ThreadFactory() { + @Override + public Thread newThread(final Runnable runnable) { + final Thread t = Executors.defaultThreadFactory().newThread(runnable); + t.setDaemon(true); + t.setName("NiFi Bootstrap Command Listener"); + return t; + } + }); + + this.runner = runner; + } + + public void stop() throws IOException { + stopped = true; + + executor.shutdown(); + try { + executor.awaitTermination(3, TimeUnit.SECONDS); + } catch (final InterruptedException ie) { + } + + serverSocket.close(); + } + + @Override + public void run() { + while (!serverSocket.isClosed()) { + try { + if (stopped) { + return; + } + + final Socket socket; + try { + socket = serverSocket.accept(); + } catch (final IOException ioe) { + if (stopped) { + return; + } + + throw ioe; + } + + executor.submit(new Runnable() { + @Override + public void run() { + try { + // we want to ensure that we don't try to read data from an InputStream directly + // by a BufferedReader because any user on the system could open a socket and send + // a multi-gigabyte file without any new lines in order to crash the Bootstrap, + // which in turn may cause the Shutdown Hook to shutdown NiFi. + // So we will limit the amount of data to read to 4 KB + final InputStream limitingIn = new LimitingInputStream(socket.getInputStream(), 4096); + final BootstrapCodec codec = new BootstrapCodec(runner, limitingIn, socket.getOutputStream()); + codec.communicate(); + } catch (final Throwable t) { + System.out.println("Failed to communicate with NiFi due to " + t); + t.printStackTrace(); + } finally { + try { + socket.close(); + } catch (final IOException ioe) { + } + } + } + }); + } catch (final Throwable t) { + System.err.println("Failed to receive information from NiFi due to " + t); + t.printStackTrace(); + } + } + } + } +}
