http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..21c67f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +target +.project +.settings +nbactions.xml + +# Intellij +.idea/ +*.iml +*.iws
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/DISCLAIMER ---------------------------------------------------------------------- diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 0000000..0f8e7a1 --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,15 @@ +Apache NiFi is an effort undergoing incubation at the Apache Software +Foundation (ASF), sponsored by the Apache Incubator PMC. + +Incubation is required of all newly accepted projects until a further review +indicates that the infrastructure, communications, and decision making process +have stabilized in a manner consistent with other successful ASF projects. + +While incubation status is not necessarily a reflection of the completeness +or stability of the code, it does indicate that the project has yet to be +fully endorsed by the ASF. + +For more information about the incubation status of the Apache NiFi project +you can go to the following page: + +http://nifi.incubator.apache.org/ http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/LICENSE ---------------------------------------------------------------------- diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/NOTICE ---------------------------------------------------------------------- diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..e02ae44 --- /dev/null +++ b/NOTICE @@ -0,0 +1,7 @@ +Apache NiFi +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Please see LICENSE for additional copyright and licensing information. http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md new file mode 100644 index 0000000..3f05134 --- /dev/null +++ b/README.md @@ -0,0 +1,65 @@ +# Apache NiFi + +Apache NiFi is a dataflow system based on the concepts of flow-based programming. It is currently apart of the Apache Incubator. + +## Table of Contents + +- [Features](#features) +- [Getting Started](#getting-started) +- [Getting Help](#getting-help) +- [Requirements](#requirements) +- [License](#license) +- [Disclaimer](#disclaimer) + +## Features + +Apache NiFi supports powerful and scalable directed graphs of data routing, transformation, and system mediation logic. Some of the high-level capabilities and objectives of Apache NiFi include: + +- Web-based user interface for seamless experience between design, control, feedback, and monitoring of data flows +- Highly configurable along several dimensions of quality of service such as loss tolerant versus guaranteed delivery, low latency versus high throughput, and priority based queuing +- Fine-grained data provenance for all data received, forked, joined, cloned, modified, sent, and ultimately dropped as data reaches its configured end-state +- Component-based extension model along well defined interfaces enabling rapid development and effective testing + +## Getting Started + +Execute <nifi install dir>/bin/nifi.sh + +## Getting Help +If you have questions, you can reach out to our mailing list: [email protected] +([archive](http://mail-archives.apache.org/mod_mbox/incubator-nifi-dev)). +We're also often available in IRC: #nifi on +[irc.freenode.net](http://webchat.freenode.net/?channels=#nifi). + + +## Requirements +* JDK 1.7 or higher + +## License + +Except as otherwise noted this software is licensed under the +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Disclaimer + +Apache NiFi is an effort undergoing incubation at the Apache Software +Foundation (ASF), sponsored by the Apache Incubator PMC. + +Incubation is required of all newly accepted projects until a further review +indicates that the infrastructure, communications, and decision making process +have stabilized in a manner consistent with other successful ASF projects. + +While incubation status is not necessarily a reflection of the completeness +or stability of the code, it does indicate that the project has yet to be +fully endorsed by the ASF. http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/assemblies/nifi/pom.xml ---------------------------------------------------------------------- diff --git a/assemblies/nifi/pom.xml b/assemblies/nifi/pom.xml new file mode 100644 index 0000000..ec06a23 --- /dev/null +++ b/assemblies/nifi/pom.xml @@ -0,0 +1,485 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-parent</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <artifactId>nifi</artifactId> + <version>0.0.1-SNAPSHOT</version> + <packaging>pom</packaging> + + <name>NiFi Release</name> + <description>This is the assembly for the Main Standard Release of NiFi</description> + + <build> + <plugins> + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>unpack-shared-resources</id> + <goals> + <goal>unpack-dependencies</goal> + </goals> + <phase>generate-resources</phase> + <configuration> + <outputDirectory>${project.build.directory}/generated-resources</outputDirectory> + <includeArtifactIds>nifi-resources</includeArtifactIds> + <includeGroupIds>org.apache.nifi</includeGroupIds> + <excludeTransitive>false</excludeTransitive> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptors> + <descriptor>src/main/assembly/dependencies.xml</descriptor> + </descriptors> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.3.2</version> + <executions> + <execution> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>${project.build.directory}/${project.artifactId}-${project.version}-${nifi.assembly.id}.dir/${nifi.executable}</executable> + </configuration> + </plugin> + </plugins> + <resources> + <resource> + <directory>${basedir}/src/main/resources</directory> + </resource> + <resource> + <directory>${project.build.directory}/generated-resources</directory> + <filtering>true</filtering> + </resource> + </resources> + </build> + <dependencies> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jcl-over-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>jul-to-slf4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>log4j-over-slf4j</artifactId> + <version>${org.slf4j.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + <version>0.0.1-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-runtime</artifactId> + <version>${framework.version}</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-resources</artifactId> + <version>${framework.version}</version> + <classifier>resources</classifier> + <scope>provided</scope> <!-- Provided - we don't want the zip in the libs --> + <type>zip</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-nar</artifactId> + <version>${framework.version}</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>volatile-provenance-repository-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>persistent-provenance-repository-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>standard-services-api-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>ssl-context-service-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>distributed-cache-services-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-jetty-bundle</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>update-attribute-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>monitor-threshold-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>hadoop-libraries-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>hadoop-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency> + <!-- <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>execute-script-nar</artifactId> + <version>0.0.1-SNAPSHOT</version> + <type>nar</type> + </dependency>--> + </dependencies> + + <properties> + <framework.version>0.0.1-SNAPSHOT</framework.version> + + <!--Wrapper Properties--> + <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb> + <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb> + <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb> + <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb> + <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize> + <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles> + + <!-- nifi.properties: core properties --> + <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> + <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> + <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> + <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration> + + <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file> + <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir> + <nifi.reporting.task.configuration.file>./conf/reporting-tasks.xml</nifi.reporting.task.configuration.file> + <nifi.controller.service.configuration.file>./conf/controller-services.xml</nifi.controller.service.configuration.file> + <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file> + <nifi.templates.directory>./conf/templates</nifi.templates.directory> + <nifi.database.directory>./database_repository</nifi.database.directory> + + <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation> + <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory> + <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions> + <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval> + <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync> + <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation> + <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold> + <nifi.swap.storage.directory>./flowfile_repository/swap</nifi.swap.storage.directory> + <nifi.swap.in.period>5 sec</nifi.swap.in.period> + <nifi.swap.in.threads>1</nifi.swap.in.threads> + <nifi.swap.out.period>5 sec</nifi.swap.out.period> + <nifi.swap.out.threads>4</nifi.swap.out.threads> + + <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation> + <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size> + <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files> + <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default> + <nifi.content.repository.archive.max.retention.period /> + <nifi.content.repository.archive.max.usage.percentage /> + <nifi.content.repository.archive.enabled>false</nifi.content.repository.archive.enabled> + <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync> + <nifi.content.viewer.url /> + + + <nifi.restore.directory /> + <nifi.ui.banner.text></nifi.ui.banner.text> + <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval> + <nifi.nar.library.directory>./lib</nifi.nar.library.directory> + <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> + <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> + + <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> + <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> + <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> + + <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> + + <!-- persistent provenance repository properties --> + <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> + <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> + <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> + <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> + <nifi.provenance.repository.rollover.time>5 mins</nifi.provenance.repository.rollover.time> + <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size> + <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads> + <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover> + <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID</nifi.provenance.repository.indexed.fields> + <nifi.provenance.repository.indexed.attributes /> + <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size> + <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync> + <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count> + + <!-- volatile provenance repository properties --> + <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size> + + <!-- Component status repository properties --> + <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation> + <nifi.components.status.repository.buffer.size>288</nifi.components.status.repository.buffer.size> + <nifi.components.status.snapshot.frequency>5 mins</nifi.components.status.snapshot.frequency> + + <!-- nifi.properties: web properties --> + <nifi.web.war.directory>./lib</nifi.web.war.directory> + <nifi.web.http.host /> + <nifi.web.http.port>8080</nifi.web.http.port> + <nifi.web.https.host /> + <nifi.web.https.port /> + <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir> + + <!-- nifi.properties: security properties --> + <nifi.security.keystore /> + <nifi.security.keystoreType /> + <nifi.security.keystorePasswd /> + <nifi.security.keyPasswd /> + <nifi.security.truststore /> + <nifi.security.truststoreType /> + <nifi.security.truststorePasswd /> + <nifi.security.needClientAuth /> + <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file> + <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration> + <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider> + <nifi.security.x509.principal.extractor /> + <nifi.security.support.new.account.requests /> + <nifi.security.ocsp.responder.url /> + <nifi.security.ocsp.responder.certificate /> + + <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) --> + <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> + <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> + <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout> + <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout> + <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast> + <nifi.cluster.protocol.multicast.address /> + <nifi.cluster.protocol.multicast.port /> + <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay> + <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> + <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> + + <!-- nifi.properties: cluster node properties (only configure for cluster nodes) --> + <nifi.cluster.is.node>false</nifi.cluster.is.node> + <nifi.cluster.node.address /> + <nifi.cluster.node.protocol.port /> + <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads> + <nifi.cluster.node.unicast.manager.address /> + <nifi.cluster.node.unicast.manager.protocol.port /> + + <!-- nifi.properties: cluster manager properties (only configure for cluster manager) --> + <nifi.cluster.is.manager>false</nifi.cluster.is.manager> + <nifi.cluster.manager.address /> + <nifi.cluster.manager.protocol.port /> + <nifi.cluster.manager.node.firewall.file /> + <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size> + <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout> + <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout> + <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads> + <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay> + <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads> + <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration> + + <!-- + Properties to execute nifi using Maven. It is assumed that the assembly has a + format of type 'dir' so that the executable can be referenced in a defined location. + --> + <nifi.assembly.id>bin</nifi.assembly.id> + <nifi.executable>${project.artifactId}-${project.version}/bin/${project.artifactId}</nifi.executable> + + </properties> + <profiles> + <profile> + <id>standard-rpm</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>rpm-maven-plugin</artifactId> + <version>2.0.1</version> + <executions> + <execution> + <goals> + <goal>rpm</goal> + </goals> + </execution> + </executions> + <configuration> + <name>${project.artifactId}</name> + <projversion>${project.version}</projversion> + <summary>NiFi data flow framework</summary> + <description> + Nifi is a simple event processing framework largely + based on the Flow-Based Programming concepts. + </description> + <copyright>NONE</copyright> + <url>http://nifi.apache.org</url> + <group>Utilities</group> + <requires> + <require>jdk</require> + </requires> + <prefix>/opt/${project.artifactId}</prefix> + <defineStatements> + <defineStatement>_use_internal_dependency_generator 0</defineStatement> + </defineStatements> + <defaultDirmode>750</defaultDirmode> + <defaultFilemode>640</defaultFilemode> + <defaultUsername>dataflow</defaultUsername> + <defaultGroupname>dataflow</defaultGroupname> + <mappings> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}</directory> + </mapping> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/bin</directory> + <filemode>750</filemode> + <sources> + <source> + <location>${project.build.directory}/generated-resources/bin/nifi</location> + <destination>${project.artifactId}</destination> + <filter>true</filter> + </source> + <source> + <location>${project.build.directory}/generated-resources/bin/wrapper-linux-x86-32</location> + </source> + <source> + <location>${project.build.directory}/generated-resources/bin/wrapper-linux-x86-64</location> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/conf</directory> + <configuration>true</configuration> + <sources> + <source> + <location>${project.build.directory}/generated-resources/conf</location> + <excludes> + <exclude>nifi.properties</exclude> + </excludes> + <filter>true</filter> + </source> + <source> + <location>${project.build.directory}/generated-resources/conf/nifi.properties</location> + <destination>${project.artifactId}.properties</destination> + <filter>true</filter> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/lib</directory> + <dependency /> + <sources> + <source> + <location>${project.build.directory}/generated-resources/lib</location> + <excludes> + <exclude>**/*-aix*</exclude> + <exclude>**/*-freebsd*</exclude> + <exclude>**/*-hpux*</exclude> + <exclude>**/*-linux-ia*</exclude> + <exclude>**/*-linux-ppc*</exclude> + <exclude>**/*-macosx*</exclude> + <exclude>**/*-solaris*</exclude> + <exclude>**/*-windows*</exclude> + </excludes> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/docs</directory> + <sources> + <source> + <location>${project.build.directory}/generated-resources/docs</location> + </source> + </sources> + </mapping> + <mapping> + <directory>/opt/${project.artifactId}/${project.artifactId}-${project.version}/logs</directory> + <sources> + <source> + <location>${project.build.directory}/generated-resources/logs</location> + </source> + </sources> + </mapping> + </mappings> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/assemblies/nifi/src/main/assembly/dependencies.xml ---------------------------------------------------------------------- diff --git a/assemblies/nifi/src/main/assembly/dependencies.xml b/assemblies/nifi/src/main/assembly/dependencies.xml new file mode 100644 index 0000000..339275b --- /dev/null +++ b/assemblies/nifi/src/main/assembly/dependencies.xml @@ -0,0 +1,123 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id>bin</id> + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>${project.artifactId}-${project.version}</baseDirectory> + + <dependencySets> + <dependencySet> + <scope>runtime</scope> + <outputDirectory>lib</outputDirectory> + <useTransitiveFiltering>true</useTransitiveFiltering> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.build.directory}/generated-resources/bin</directory> + <outputDirectory>bin</outputDirectory> + <directoryMode>0750</directoryMode> + <fileMode>0750</fileMode> + <filtered>false</filtered> + </fileSet> + <fileSet> + <directory>${project.build.directory}/generated-resources/conf</directory> + <outputDirectory>conf</outputDirectory> + <excludes> + <exclude>nifi.properties</exclude> + </excludes> + <directoryMode>0750</directoryMode> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </fileSet> + <fileSet> + <directory>${project.build.directory}/generated-resources/lib</directory> + <outputDirectory>lib</outputDirectory> + <directoryMode>0750</directoryMode> + <fileMode>0640</fileMode> + </fileSet> + <fileSet> + <directory>${project.build.directory}/generated-resources/logs</directory> + <outputDirectory>logs</outputDirectory> + <directoryMode>0750</directoryMode> + <fileMode>0640</fileMode> + </fileSet> + <fileSet> + <directory>${project.build.directory}/generated-resources/docs</directory> + <outputDirectory>docs</outputDirectory> + <directoryMode>0750</directoryMode> + <fileMode>0640</fileMode> + <filtered>false</filtered> + </fileSet> + </fileSets> + + <files> + <file> + <source>${project.build.directory}/generated-resources/conf/nifi.properties</source> + <outputDirectory>conf</outputDirectory> + <destName>${project.artifactId}.properties</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/bin/nifi.sh</source> + <outputDirectory>bin</outputDirectory> + <destName>nifi.sh</destName> + <fileMode>0750</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/docs/README.md</source> + <outputDirectory>docs</outputDirectory> + <destName>README.md</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/DISCLAIMER</source> + <outputDirectory>./</outputDirectory> + <destName>DISCLAIMER</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/DISCLAIMER</source> + <outputDirectory>./</outputDirectory> + <destName>DISCLAIMER</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/LICENSE</source> + <outputDirectory>./</outputDirectory> + <destName>LICENSE</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + <file> + <source>${project.build.directory}/generated-resources/NOTICE</source> + <outputDirectory>./</outputDirectory> + <destName>NOTICE</destName> + <fileMode>0640</fileMode> + <filtered>true</filtered> + </file> + </files> +</assembly> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/core-flowfile-attributes/pom.xml ---------------------------------------------------------------------- diff --git a/commons/core-flowfile-attributes/pom.xml b/commons/core-flowfile-attributes/pom.xml new file mode 100644 index 0000000..d611134 --- /dev/null +++ b/commons/core-flowfile-attributes/pom.xml @@ -0,0 +1,32 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-parent</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <artifactId>nifi-core-flowfile-attributes</artifactId> + <version>0.0.1-SNAPSHOT</version> + <packaging>jar</packaging> + + <name>NiFi: Core FlowFile Attributes</name> + + <dependencies> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java ---------------------------------------------------------------------- diff --git a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java new file mode 100644 index 0000000..24f43ca --- /dev/null +++ b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/CoreAttributes.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowfile.attributes; + +public enum CoreAttributes implements FlowFileAttributeKey { + /** + * The flowfile's path indicates the relative directory to which a FlowFile belongs and does not + * contain the filename + */ + PATH("path"), + + /** + * The flowfile's absolute path indicates the absolute directory to which a FlowFile belongs and does not + * contain the filename + */ + ABSOLUTE_PATH("absolute.path"), + + /** + * The filename of the FlowFile. The filename should not contain any directory structure. + */ + FILENAME("filename"), + + /** + * A unique UUID assigned to this FlowFile + */ + UUID("uuid"), + + /** + * A numeric value indicating the FlowFile priority + */ + PRIORITY("priority"), + + /** + * The MIME Type of this FlowFile + */ + MIME_TYPE("mime.type"), + + /** + * Specifies the reason that a FlowFile is being discarded + */ + DISCARD_REASON("discard.reason"), + + /** + * Indicates an identifier other than the FlowFile's UUID that is known to refer to this FlowFile. + */ + ALTERNATE_IDENTIFIER("alternate.identifier"); + + private final String key; + private CoreAttributes(final String key) { + this.key = key; + } + + @Override + public String key() { + return key; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java ---------------------------------------------------------------------- diff --git a/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java new file mode 100644 index 0000000..cc6c28e --- /dev/null +++ b/commons/core-flowfile-attributes/src/main/java/org/apache/nifi/flowfile/attributes/FlowFileAttributeKey.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowfile.attributes; + +public interface FlowFileAttributeKey { + String key(); +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/.gitignore ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/.gitignore b/commons/data-provenance-utils/.gitignore new file mode 100755 index 0000000..19f2e00 --- /dev/null +++ b/commons/data-provenance-utils/.gitignore @@ -0,0 +1,2 @@ +/target +/target http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/pom.xml ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/pom.xml b/commons/data-provenance-utils/pom.xml new file mode 100644 index 0000000..0f33c06 --- /dev/null +++ b/commons/data-provenance-utils/pom.xml @@ -0,0 +1,42 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-parent</artifactId> + <version>0.0.1-SNAPSHOT</version> + </parent> + + <artifactId>data-provenance-utils</artifactId> + <version>0.0.1-SNAPSHOT</version> + <packaging>jar</packaging> + + <name>data-provenance-utils</name> + + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + <version>[0.0.1-SNAPSHOT,1.0.0-SNAPSHOT)</version> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-core-flowfile-attributes</artifactId> + <version>0.0.1-SNAPSHOT</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java new file mode 100644 index 0000000..dc24a93 --- /dev/null +++ b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.util.Collection; +import java.util.Date; +import java.util.UUID; + +import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; +import org.apache.nifi.provenance.lineage.LineageComputationType; + +/** + * + */ +public class AsyncLineageSubmission implements ComputeLineageSubmission { + private final String lineageIdentifier = UUID.randomUUID().toString(); + private final Date submissionTime = new Date(); + + private final LineageComputationType computationType; + private final Long eventId; + private final Collection<String> lineageFlowFileUuids; + + private volatile boolean canceled = false; + + private final StandardLineageResult result; + + public AsyncLineageSubmission(final LineageComputationType computationType, final Long eventId, final Collection<String> lineageFlowFileUuids, final int numSteps) { + this.computationType = computationType; + this.eventId = eventId; + this.lineageFlowFileUuids = lineageFlowFileUuids; + this.result = new StandardLineageResult(numSteps, lineageFlowFileUuids); + } + + @Override + public StandardLineageResult getResult() { + return result; + } + + @Override + public Date getSubmissionTime() { + return submissionTime; + } + + @Override + public String getLineageIdentifier() { + return lineageIdentifier; + } + + @Override + public void cancel() { + this.canceled = true; + } + + @Override + public boolean isCanceled() { + return canceled; + } + + @Override + public LineageComputationType getLineageComputationType() { + return computationType; + } + + @Override + public Long getExpandedEventId() { + return eventId; + } + + @Override + public Collection<String> getLineageFlowFileUuids() { + return lineageFlowFileUuids; + } +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java new file mode 100644 index 0000000..4244476 --- /dev/null +++ b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.util.Date; +import java.util.concurrent.TimeUnit; + +import org.apache.nifi.provenance.search.Query; +import org.apache.nifi.provenance.search.QuerySubmission; + +/** + * + */ +public class AsyncQuerySubmission implements QuerySubmission { + + public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS); + + private final Date submissionTime = new Date(); + private final Query query; + + private volatile boolean canceled = false; + private final StandardQueryResult queryResult; + + /** + * Constructs an AsyncQuerySubmission with the given query and the given + * number of steps, indicating how many results must be added to this + * AsyncQuerySubmission before it is considered finished + * + * @param query + * @param numSteps + */ + public AsyncQuerySubmission(final Query query, final int numSteps) { + this.query = query; + queryResult = new StandardQueryResult(query, numSteps); + } + + @Override + public Date getSubmissionTime() { + return submissionTime; + } + + @Override + public String getQueryIdentifier() { + return query.getIdentifier(); + } + + @Override + public void cancel() { + this.canceled = true; + queryResult.cancel(); + } + + @Override + public boolean isCanceled() { + return canceled; + } + + @Override + public Query getQuery() { + return query; + } + + @Override + public StandardQueryResult getResult() { + return queryResult; + } +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java new file mode 100644 index 0000000..dc2903f --- /dev/null +++ b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import org.apache.nifi.provenance.search.SearchableField; +import org.apache.nifi.provenance.search.SearchableFieldType; + +import static java.util.Objects.requireNonNull; + +/** + * + */ +public class NamedSearchableField implements SearchableField { + + private final String identifier; + private final String searchableName; + private final SearchableFieldType fieldType; + private final String friendlyName; + private final boolean attribute; + + NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute) { + this(identifier, searchableName, friendlyName, attribute, SearchableFieldType.STRING); + } + + NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute, final SearchableFieldType fieldType) { + this.identifier = requireNonNull(identifier); + this.searchableName = requireNonNull(searchableName); + this.friendlyName = requireNonNull(friendlyName); + this.attribute = requireNonNull(attribute); + this.fieldType = requireNonNull(fieldType); + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public String getSearchableFieldName() { + return searchableName; + } + + @Override + public String getFriendlyName() { + return friendlyName; + } + + @Override + public boolean isAttribute() { + return attribute; + } + + @Override + public SearchableFieldType getFieldType() { + return fieldType; + } + + @Override + public String toString() { + return friendlyName; + } + + @Override + public int hashCode() { + return 298347 + searchableName.hashCode() + (attribute ? 1 : 0); + } + + @Override + public boolean equals(final Object obj) { + if (obj == null) { + return false; + } + + if (!(obj instanceof SearchableField)) { + return false; + } + + final SearchableField other = (SearchableField) obj; + return (this.searchableName.equals(other.getSearchableFieldName()) && attribute == other.isAttribute()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java new file mode 100644 index 0000000..6a934b1 --- /dev/null +++ b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.nifi.provenance.search.SearchableField; + +public class SearchableFieldParser { + + public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) { + final List<SearchableField> searchableFields = new ArrayList<>(); + if (indexedFieldString != null) { + final String[] split = indexedFieldString.split(","); + for (String fieldName : split) { + fieldName = fieldName.trim(); + if (fieldName.isEmpty()) { + continue; + } + + final SearchableField searchableField; + if (predefinedField) { + searchableField = SearchableFields.getSearchableField(fieldName); + } else { + searchableField = SearchableFields.newSearchableAttribute(fieldName); + } + + if (searchableField == null) { + throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field"); + } + searchableFields.add(searchableField); + } + } + + return searchableFields; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/4d998c12/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java new file mode 100644 index 0000000..97c9880 --- /dev/null +++ b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.provenance; + +import org.apache.nifi.provenance.search.SearchableField; +import org.apache.nifi.provenance.search.SearchableFieldType; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +public class SearchableFields { + + public static final SearchableField Identifier = new NamedSearchableField("Identifier", "identifier", "Identifier", false); + public static final SearchableField EventTime = new NamedSearchableField("EventTime", "time", "Event Time", false, SearchableFieldType.DATE); + public static final SearchableField FlowFileUUID = new NamedSearchableField("FlowFileUUID", "uuid", "FlowFile UUID", false); + public static final SearchableField Filename = new NamedSearchableField("Filename", "filename", "Filename", false); + public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false); + public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false); + public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false); + public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false); + public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE); + public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING); + public static final SearchableField Relationship = new NamedSearchableField("Relationship", "relationship", "Relationship", false, SearchableFieldType.STRING); + + public static final SearchableField LineageStartDate = new NamedSearchableField("LineageStartDate", "lineageStartDate", "Lineage Start Date", false, SearchableFieldType.DATE); + public static final SearchableField LineageIdentifier = new NamedSearchableField("LineageIdentifiers", "lineageIdentifier", "Lineage Identifier", false, SearchableFieldType.STRING); + + public static final SearchableField ContentClaimSection = new NamedSearchableField("ContentClaimSection", "contentClaimSection", "Content Claim Section", false, SearchableFieldType.STRING); + public static final SearchableField ContentClaimContainer = new NamedSearchableField("ContentClaimContainer", "contentClaimContainer", "Content Claim Container", false, SearchableFieldType.STRING); + public static final SearchableField ContentClaimIdentifier = new NamedSearchableField("ContentClaimIdentifier", "contentClaimIdentifier", "Content Claim Identifier", false, SearchableFieldType.STRING); + public static final SearchableField ContentClaimOffset = new NamedSearchableField("ContentClaimOffset", "contentClaimOffset", "Content Claim Offset", false, SearchableFieldType.LONG); + public static final SearchableField SourceQueueIdentifier = new NamedSearchableField("SourceQueueIdentifier", "sourceQueueIdentifier", "Source Queue Identifier", false, SearchableFieldType.STRING); + + private static final Map<String, SearchableField> standardFields; + + static { + final SearchableField[] searchableFields = new SearchableField[]{ + EventTime, FlowFileUUID, Filename, EventType, TransitURI, + ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details, + LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier, + ContentClaimOffset, SourceQueueIdentifier}; + + final Map<String, SearchableField> fields = new HashMap<>(); + for (final SearchableField field : searchableFields) { + fields.put(field.getIdentifier(), field); + } + + standardFields = Collections.unmodifiableMap(fields); + } + + private SearchableFields() { + } + + public static Collection<SearchableField> getStandardFields() { + return standardFields.values(); + } + + public static SearchableField getSearchableField(final String fieldIdentifier) { + return standardFields.get(fieldIdentifier); + } + + public static SearchableField newSearchableAttribute(final String attributeName) { + return new NamedSearchableField(attributeName, attributeName, attributeName, true); + } +}
