http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml deleted file mode 100644 index e785c45..0000000 --- a/assembly/pom.xml +++ /dev/null @@ -1,458 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-parent</artifactId> - <version>0.0.1-SNAPSHOT</version> - </parent> - <artifactId>nifi</artifactId> - <version>0.0.1-SNAPSHOT</version> - <packaging>pom</packaging> - <name>NiFi Release</name> - <description>This is the assembly Apache NiFi (incubating)</description> - <build> - <plugins> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <attach>true</attach> - </configuration> - <executions> - <execution> - <id>make shared resource</id> - <goals> - <goal>single</goal> - </goals> - <phase>package</phase> - <configuration> - <descriptors> - <descriptor>src/main/assembly/dependencies.xml</descriptor> - </descriptors> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - <dependencies> - <dependency> - <groupId>ch.qos.logback</groupId> - <artifactId>logback-classic</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jcl-over-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>jul-to-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>log4j-over-slf4j</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-runtime</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-bootstrap</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-resources</artifactId> - <classifier>resources</classifier> - <scope>runtime</scope> - <type>zip</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-docs</artifactId> - <classifier>resources</classifier> - <scope>runtime</scope> - <type>zip</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-framework-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>volatile-provenance-repository-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>persistent-provenance-repository-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>standard-services-api-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>ssl-context-service-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>distributed-cache-services-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-standard-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-jetty-bundle</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>update-attribute-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>monitor-threshold-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>hadoop-libraries-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>hadoop-nar</artifactId> - <type>nar</type> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>kafka-nar</artifactId> - <type>nar</type> - </dependency> - </dependencies> - - <properties> - <!--Wrapper Properties--> - <nifi.wrapper.jvm.heap.initial.mb>256</nifi.wrapper.jvm.heap.initial.mb> - <nifi.wrapper.jvm.heap.max.mb>512</nifi.wrapper.jvm.heap.max.mb> - <nifi.initial.permgen.size.mb>128</nifi.initial.permgen.size.mb> - <nifi.max.permgen.size.mb>128</nifi.max.permgen.size.mb> - <nifi.wrapper.logfile.maxsize>10m</nifi.wrapper.logfile.maxsize> - <nifi.wrapper.logfile.maxfiles>10</nifi.wrapper.logfile.maxfiles> - - <!-- nifi.properties: core properties --> - <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> - <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> - <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> - <nifi.administrative.yield.duration>30 sec</nifi.administrative.yield.duration> - - <nifi.flow.configuration.file>./conf/flow.xml.gz</nifi.flow.configuration.file> - <nifi.flow.configuration.archive.dir>./conf/archive/</nifi.flow.configuration.archive.dir> - <nifi.reporting.task.configuration.file>./conf/reporting-tasks.xml</nifi.reporting.task.configuration.file> - <nifi.controller.service.configuration.file>./conf/controller-services.xml</nifi.controller.service.configuration.file> - <nifi.authority.provider.configuration.file>./conf/authority-providers.xml</nifi.authority.provider.configuration.file> - <nifi.templates.directory>./conf/templates</nifi.templates.directory> - <nifi.database.directory>./database_repository</nifi.database.directory> - - <nifi.flowfile.repository.implementation>org.apache.nifi.controller.repository.WriteAheadFlowFileRepository</nifi.flowfile.repository.implementation> - <nifi.flowfile.repository.directory>./flowfile_repository</nifi.flowfile.repository.directory> - <nifi.flowfile.repository.partitions>256</nifi.flowfile.repository.partitions> - <nifi.flowfile.repository.checkpoint.interval>2 mins</nifi.flowfile.repository.checkpoint.interval> - <nifi.flowfile.repository.always.sync>false</nifi.flowfile.repository.always.sync> - <nifi.swap.manager.implementation>org.apache.nifi.controller.FileSystemSwapManager</nifi.swap.manager.implementation> - <nifi.queue.swap.threshold>20000</nifi.queue.swap.threshold> - <nifi.swap.in.period>5 sec</nifi.swap.in.period> - <nifi.swap.in.threads>1</nifi.swap.in.threads> - <nifi.swap.out.period>5 sec</nifi.swap.out.period> - <nifi.swap.out.threads>4</nifi.swap.out.threads> - - <nifi.content.repository.implementation>org.apache.nifi.controller.repository.FileSystemRepository</nifi.content.repository.implementation> - <nifi.content.claim.max.appendable.size>10 MB</nifi.content.claim.max.appendable.size> - <nifi.content.claim.max.flow.files>100</nifi.content.claim.max.flow.files> - <nifi.content.repository.directory.default>./content_repository</nifi.content.repository.directory.default> - <nifi.content.repository.archive.max.retention.period /> - <nifi.content.repository.archive.max.usage.percentage /> - <nifi.content.repository.archive.enabled>false</nifi.content.repository.archive.enabled> - <nifi.content.repository.always.sync>false</nifi.content.repository.always.sync> - <nifi.content.viewer.url /> - - - <nifi.restore.directory /> - <nifi.ui.banner.text></nifi.ui.banner.text> - <nifi.ui.autorefresh.interval>30 sec</nifi.ui.autorefresh.interval> - <nifi.nar.library.directory>./lib</nifi.nar.library.directory> - <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> - <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> - - <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> - <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> - <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> - - <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> - - <!-- persistent provenance repository properties --> - <nifi.provenance.repository.implementation>org.apache.nifi.provenance.PersistentProvenanceRepository</nifi.provenance.repository.implementation> - <nifi.provenance.repository.directory.default>./provenance_repository</nifi.provenance.repository.directory.default> - <nifi.provenance.repository.max.storage.time>24 hours</nifi.provenance.repository.max.storage.time> - <nifi.provenance.repository.max.storage.size>1 GB</nifi.provenance.repository.max.storage.size> - <nifi.provenance.repository.rollover.time>5 mins</nifi.provenance.repository.rollover.time> - <nifi.provenance.repository.rollover.size>100 MB</nifi.provenance.repository.rollover.size> - <nifi.provenance.repository.query.threads>2</nifi.provenance.repository.query.threads> - <nifi.provenance.repository.compress.on.rollover>true</nifi.provenance.repository.compress.on.rollover> - <nifi.provenance.repository.indexed.fields>EventType, FlowFileUUID, Filename, ProcessorID</nifi.provenance.repository.indexed.fields> - <nifi.provenance.repository.indexed.attributes /> - <nifi.provenance.repository.index.shard.size>500 MB</nifi.provenance.repository.index.shard.size> - <nifi.provenance.repository.always.sync>false</nifi.provenance.repository.always.sync> - <nifi.provenance.repository.journal.count>16</nifi.provenance.repository.journal.count> - - <!-- volatile provenance repository properties --> - <nifi.provenance.repository.buffer.size>100000</nifi.provenance.repository.buffer.size> - - <!-- Component status repository properties --> - <nifi.components.status.repository.implementation>org.apache.nifi.controller.status.history.VolatileComponentStatusRepository</nifi.components.status.repository.implementation> - <nifi.components.status.repository.buffer.size>288</nifi.components.status.repository.buffer.size> - <nifi.components.status.snapshot.frequency>5 mins</nifi.components.status.snapshot.frequency> - - <!-- nifi.properties: web properties --> - <nifi.web.war.directory>./lib</nifi.web.war.directory> - <nifi.web.http.host /> - <nifi.web.http.port>8080</nifi.web.http.port> - <nifi.web.https.host /> - <nifi.web.https.port /> - <nifi.jetty.work.dir>./work/jetty</nifi.jetty.work.dir> - <nifi.web.jetty.threads>200</nifi.web.jetty.threads> - - <!-- nifi.properties: security properties --> - <nifi.security.keystore /> - <nifi.security.keystoreType /> - <nifi.security.keystorePasswd /> - <nifi.security.keyPasswd /> - <nifi.security.truststore /> - <nifi.security.truststoreType /> - <nifi.security.truststorePasswd /> - <nifi.security.needClientAuth /> - <nifi.security.authorizedUsers.file>./conf/authorized-users.xml</nifi.security.authorizedUsers.file> - <nifi.security.user.credential.cache.duration>24 hours</nifi.security.user.credential.cache.duration> - <nifi.security.user.authority.provider>file-provider</nifi.security.user.authority.provider> - <nifi.security.x509.principal.extractor /> - <nifi.security.support.new.account.requests /> - <nifi.security.ocsp.responder.url /> - <nifi.security.ocsp.responder.certificate /> - - <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) --> - <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> - <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> - <nifi.cluster.protocol.socket.timeout>30 sec</nifi.cluster.protocol.socket.timeout> - <nifi.cluster.protocol.connection.handshake.timeout>45 sec</nifi.cluster.protocol.connection.handshake.timeout> - <nifi.cluster.protocol.use.multicast>false</nifi.cluster.protocol.use.multicast> - <nifi.cluster.protocol.multicast.address /> - <nifi.cluster.protocol.multicast.port /> - <nifi.cluster.protocol.multicast.service.broadcast.delay>500 ms</nifi.cluster.protocol.multicast.service.broadcast.delay> - <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> - <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> - - <!-- nifi.properties: cluster node properties (only configure for cluster nodes) --> - <nifi.cluster.is.node>false</nifi.cluster.is.node> - <nifi.cluster.node.address /> - <nifi.cluster.node.protocol.port /> - <nifi.cluster.node.protocol.threads>2</nifi.cluster.node.protocol.threads> - <nifi.cluster.node.unicast.manager.address /> - <nifi.cluster.node.unicast.manager.protocol.port /> - - <!-- nifi.properties: cluster manager properties (only configure for cluster manager) --> - <nifi.cluster.is.manager>false</nifi.cluster.is.manager> - <nifi.cluster.manager.address /> - <nifi.cluster.manager.protocol.port /> - <nifi.cluster.manager.node.firewall.file /> - <nifi.cluster.manager.node.event.history.size>10</nifi.cluster.manager.node.event.history.size> - <nifi.cluster.manager.node.api.connection.timeout>30 sec</nifi.cluster.manager.node.api.connection.timeout> - <nifi.cluster.manager.node.api.read.timeout>30 sec</nifi.cluster.manager.node.api.read.timeout> - <nifi.cluster.manager.node.api.request.threads>10</nifi.cluster.manager.node.api.request.threads> - <nifi.cluster.manager.flow.retrieval.delay>5 sec</nifi.cluster.manager.flow.retrieval.delay> - <nifi.cluster.manager.protocol.threads>10</nifi.cluster.manager.protocol.threads> - <nifi.cluster.manager.safemode.duration>0 sec</nifi.cluster.manager.safemode.duration> - </properties> - <profiles> - <profile> - <id>rpm</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <build> - <plugins> - <plugin> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>unpack-shared-resources</id> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <phase>generate-resources</phase> - <configuration> - <outputDirectory>${project.build.directory}/generated-resources</outputDirectory> - <includeArtifactIds>nifi-resources</includeArtifactIds> - <includeGroupIds>org.apache.nifi</includeGroupIds> - <excludeTransitive>false</excludeTransitive> - </configuration> - </execution> - <execution> - <id>unpack-docs</id> - <goals> - <goal>unpack-dependencies</goal> - </goals> - <phase>generate-resources</phase> - <configuration> - <outputDirectory>${project.build.directory}/generated-docs</outputDirectory> - <includeArtifactIds>nifi-docs</includeArtifactIds> - <includeGroupIds>org.apache.nifi</includeGroupIds> - <excludeTransitive>false</excludeTransitive> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>rpm-maven-plugin</artifactId> - <configuration> - <summary>Apache NiFi (incubating)</summary> - <description>Apache Nifi (incubating) is dataflow system based on the Flow-Based Programming concepts.</description> - <license>Apache License, Version 2.0 and others (see included LICENSE file)</license> - <url>http://nifi.incubator.apache.org</url> - <group>Utilities</group> - <prefix>/opt/nifi</prefix> - <defineStatements> - <defineStatement>_use_internal_dependency_generator 0</defineStatement> - </defineStatements> - <defaultDirmode>750</defaultDirmode> - <defaultFilemode>640</defaultFilemode> - <defaultUsername>root</defaultUsername> - <defaultGroupname>root</defaultGroupname> - </configuration> - <executions> - <execution> - <id>build-bin-rpm</id> - <goals> - <goal>attached-rpm</goal> - </goals> - <configuration> - <classifier>bin</classifier> - <provides> - <provide>nifi</provide> - </provides> - <mappings> - <mapping> - <directory>/opt/nifi/nifi-${project.version}</directory> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}</directory> - <sources> - <source> - <location>../LICENSE</location> - </source> - <source> - <location>../NOTICE</location> - </source> - <source> - <location>../DISCLAIMER</location> - </source> - <source> - <location>../README.md</location> - <destination>README</destination> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/bin</directory> - <filemode>750</filemode> - <sources> - <source> - <location>${project.build.directory}/generated-resources/bin/nifi.sh</location> - <destination>nifi.sh</destination> - <filter>true</filter> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/conf</directory> - <configuration>true</configuration> - <sources> - <source> - <location>${project.build.directory}/generated-resources/conf</location> - <filter>true</filter> - </source> - </sources> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/lib</directory> - <dependency> - <excludes> - <exclude>org.apache.nifi:nifi-bootstrap</exclude> - <exclude>org.apache.nifi:nifi-resources</exclude> - <exclude>org.apache.nifi:nifi-docs</exclude> - </excludes> - </dependency> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/lib/bootstrap</directory> - <dependency> - <includes> - <include>org.apache.nifi:nifi-bootstrap</include> - </includes> - </dependency> - </mapping> - <mapping> - <directory>/opt/nifi/nifi-${project.version}/docs</directory> - <sources> - <source> - <location>${project.build.directory}/generated-docs</location> - </source> - </sources> - </mapping> - </mappings> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project>
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/assembly/src/main/assembly/dependencies.xml ---------------------------------------------------------------------- diff --git a/assembly/src/main/assembly/dependencies.xml b/assembly/src/main/assembly/dependencies.xml deleted file mode 100644 index 3481b0a..0000000 --- a/assembly/src/main/assembly/dependencies.xml +++ /dev/null @@ -1,140 +0,0 @@ -<?xml version="1.0"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<assembly> - <id>bin</id> - <formats> - <format>dir</format> - <format>zip</format> - <format>tar.gz</format> - </formats> - <includeBaseDirectory>true</includeBaseDirectory> - <baseDirectory>${project.artifactId}-${project.version}</baseDirectory> - - <dependencySets> - <!-- Write out all dependency artifacts to lib directory --> - <dependencySet> - <scope>runtime</scope> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>lib</outputDirectory> - <directoryMode>0750</directoryMode> - <fileMode>0640</fileMode> - <useTransitiveFiltering>true</useTransitiveFiltering> - <excludes> - <exclude>nifi-bootstrap</exclude> - <exclude>nifi-resources</exclude> - <exclude>nifi-docs</exclude> - </excludes> - </dependencySet> - - <!-- Write out the bootstrap lib component to its own dir --> - <dependencySet> - <scope>runtime</scope> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>lib/bootstrap</outputDirectory> - <directoryMode>0750</directoryMode> - <fileMode>0640</fileMode> - <useTransitiveFiltering>true</useTransitiveFiltering> - <includes> - <include>nifi-bootstrap</include> - </includes> - </dependencySet> - - <!-- Write out the conf directory contents --> - <dependencySet> - <scope>runtime</scope> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>./</outputDirectory> - <directoryMode>0750</directoryMode> - <fileMode>0640</fileMode> - <useTransitiveFiltering>true</useTransitiveFiltering> - <includes> - <include>nifi-resources</include> - </includes> - <unpack>true</unpack> - <unpackOptions> - <filtered>true</filtered> - <includes> - <include>conf/*</include> - </includes> - </unpackOptions> - </dependencySet> - - <!-- Write out the bin directory contents --> - <dependencySet> - <scope>runtime</scope> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>./</outputDirectory> - <directoryMode>0750</directoryMode> - <fileMode>0750</fileMode> - <useTransitiveFiltering>true</useTransitiveFiltering> - <includes> - <include>nifi-resources</include> - </includes> - <unpack>true</unpack> - <unpackOptions> - <filtered>true</filtered> - <includes> - <include>bin/*</include> - </includes> - </unpackOptions> - </dependencySet> - - <!-- Writes out the docs directory contents --> - <dependencySet> - <scope>runtime</scope> - <useProjectArtifact>false</useProjectArtifact> - <outputDirectory>docs/</outputDirectory> - <useTransitiveFiltering>true</useTransitiveFiltering> - <includes> - <include>nifi-docs</include> - </includes> - <unpack>true</unpack> - <unpackOptions> - <filtered>false</filtered> - </unpackOptions> - </dependencySet> - </dependencySets> - <files> - <file> - <source>../README.md</source> - <outputDirectory>./</outputDirectory> - <destName>README</destName> - <fileMode>0644</fileMode> - <filtered>true</filtered> - </file> - <file> - <source>../DISCLAIMER</source> - <outputDirectory>./</outputDirectory> - <destName>DISCLAIMER</destName> - <fileMode>0644</fileMode> - <filtered>true</filtered> - </file> - <file> - <source>../LICENSE</source> - <outputDirectory>./</outputDirectory> - <destName>LICENSE</destName> - <fileMode>0644</fileMode> - <filtered>true</filtered> - </file> - <file> - <source>../NOTICE</source> - <outputDirectory>./</outputDirectory> - <destName>NOTICE</destName> - <fileMode>0644</fileMode> - <filtered>true</filtered> - </file> - </files> -</assembly> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/.gitignore ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/.gitignore b/commons/data-provenance-utils/.gitignore deleted file mode 100755 index 19f2e00..0000000 --- a/commons/data-provenance-utils/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -/target http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/pom.xml ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/pom.xml b/commons/data-provenance-utils/pom.xml deleted file mode 100644 index 0024b70..0000000 --- a/commons/data-provenance-utils/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-commons-parent</artifactId> - <version>0.0.1-SNAPSHOT</version> - </parent> - - <artifactId>data-provenance-utils</artifactId> - <version>0.0.1-SNAPSHOT</version> - <packaging>jar</packaging> - - <name>data-provenance-utils</name> - - <dependencies> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-utils</artifactId> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java deleted file mode 100644 index dc24a93..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncLineageSubmission.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import java.util.Collection; -import java.util.Date; -import java.util.UUID; - -import org.apache.nifi.provenance.lineage.ComputeLineageSubmission; -import org.apache.nifi.provenance.lineage.LineageComputationType; - -/** - * - */ -public class AsyncLineageSubmission implements ComputeLineageSubmission { - private final String lineageIdentifier = UUID.randomUUID().toString(); - private final Date submissionTime = new Date(); - - private final LineageComputationType computationType; - private final Long eventId; - private final Collection<String> lineageFlowFileUuids; - - private volatile boolean canceled = false; - - private final StandardLineageResult result; - - public AsyncLineageSubmission(final LineageComputationType computationType, final Long eventId, final Collection<String> lineageFlowFileUuids, final int numSteps) { - this.computationType = computationType; - this.eventId = eventId; - this.lineageFlowFileUuids = lineageFlowFileUuids; - this.result = new StandardLineageResult(numSteps, lineageFlowFileUuids); - } - - @Override - public StandardLineageResult getResult() { - return result; - } - - @Override - public Date getSubmissionTime() { - return submissionTime; - } - - @Override - public String getLineageIdentifier() { - return lineageIdentifier; - } - - @Override - public void cancel() { - this.canceled = true; - } - - @Override - public boolean isCanceled() { - return canceled; - } - - @Override - public LineageComputationType getLineageComputationType() { - return computationType; - } - - @Override - public Long getExpandedEventId() { - return eventId; - } - - @Override - public Collection<String> getLineageFlowFileUuids() { - return lineageFlowFileUuids; - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java deleted file mode 100644 index 4244476..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/AsyncQuerySubmission.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import java.util.Date; -import java.util.concurrent.TimeUnit; - -import org.apache.nifi.provenance.search.Query; -import org.apache.nifi.provenance.search.QuerySubmission; - -/** - * - */ -public class AsyncQuerySubmission implements QuerySubmission { - - public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS); - - private final Date submissionTime = new Date(); - private final Query query; - - private volatile boolean canceled = false; - private final StandardQueryResult queryResult; - - /** - * Constructs an AsyncQuerySubmission with the given query and the given - * number of steps, indicating how many results must be added to this - * AsyncQuerySubmission before it is considered finished - * - * @param query - * @param numSteps - */ - public AsyncQuerySubmission(final Query query, final int numSteps) { - this.query = query; - queryResult = new StandardQueryResult(query, numSteps); - } - - @Override - public Date getSubmissionTime() { - return submissionTime; - } - - @Override - public String getQueryIdentifier() { - return query.getIdentifier(); - } - - @Override - public void cancel() { - this.canceled = true; - queryResult.cancel(); - } - - @Override - public boolean isCanceled() { - return canceled; - } - - @Override - public Query getQuery() { - return query; - } - - @Override - public StandardQueryResult getResult() { - return queryResult; - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java deleted file mode 100644 index dc2903f..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/NamedSearchableField.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import org.apache.nifi.provenance.search.SearchableField; -import org.apache.nifi.provenance.search.SearchableFieldType; - -import static java.util.Objects.requireNonNull; - -/** - * - */ -public class NamedSearchableField implements SearchableField { - - private final String identifier; - private final String searchableName; - private final SearchableFieldType fieldType; - private final String friendlyName; - private final boolean attribute; - - NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute) { - this(identifier, searchableName, friendlyName, attribute, SearchableFieldType.STRING); - } - - NamedSearchableField(final String identifier, final String searchableName, final String friendlyName, final boolean attribute, final SearchableFieldType fieldType) { - this.identifier = requireNonNull(identifier); - this.searchableName = requireNonNull(searchableName); - this.friendlyName = requireNonNull(friendlyName); - this.attribute = requireNonNull(attribute); - this.fieldType = requireNonNull(fieldType); - } - - @Override - public String getIdentifier() { - return identifier; - } - - @Override - public String getSearchableFieldName() { - return searchableName; - } - - @Override - public String getFriendlyName() { - return friendlyName; - } - - @Override - public boolean isAttribute() { - return attribute; - } - - @Override - public SearchableFieldType getFieldType() { - return fieldType; - } - - @Override - public String toString() { - return friendlyName; - } - - @Override - public int hashCode() { - return 298347 + searchableName.hashCode() + (attribute ? 1 : 0); - } - - @Override - public boolean equals(final Object obj) { - if (obj == null) { - return false; - } - - if (!(obj instanceof SearchableField)) { - return false; - } - - final SearchableField other = (SearchableField) obj; - return (this.searchableName.equals(other.getSearchableFieldName()) && attribute == other.isAttribute()); - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java deleted file mode 100644 index 6a934b1..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFieldParser.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.nifi.provenance.search.SearchableField; - -public class SearchableFieldParser { - - public static List<SearchableField> extractSearchableFields(final String indexedFieldString, final boolean predefinedField) { - final List<SearchableField> searchableFields = new ArrayList<>(); - if (indexedFieldString != null) { - final String[] split = indexedFieldString.split(","); - for (String fieldName : split) { - fieldName = fieldName.trim(); - if (fieldName.isEmpty()) { - continue; - } - - final SearchableField searchableField; - if (predefinedField) { - searchableField = SearchableFields.getSearchableField(fieldName); - } else { - searchableField = SearchableFields.newSearchableAttribute(fieldName); - } - - if (searchableField == null) { - throw new RuntimeException("Invalid Configuration: Provenance Repository configured to Index field '" + fieldName + "', but this is not a valid field"); - } - searchableFields.add(searchableField); - } - } - - return searchableFields; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java deleted file mode 100644 index 97c9880..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/SearchableFields.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import org.apache.nifi.provenance.search.SearchableField; -import org.apache.nifi.provenance.search.SearchableFieldType; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * - */ -public class SearchableFields { - - public static final SearchableField Identifier = new NamedSearchableField("Identifier", "identifier", "Identifier", false); - public static final SearchableField EventTime = new NamedSearchableField("EventTime", "time", "Event Time", false, SearchableFieldType.DATE); - public static final SearchableField FlowFileUUID = new NamedSearchableField("FlowFileUUID", "uuid", "FlowFile UUID", false); - public static final SearchableField Filename = new NamedSearchableField("Filename", "filename", "Filename", false); - public static final SearchableField EventType = new NamedSearchableField("EventType", "eventType", "Event Type", false); - public static final SearchableField TransitURI = new NamedSearchableField("TransitURI", "transitUri", "Transit URI", false); - public static final SearchableField ComponentID = new NamedSearchableField("ProcessorID", "processorId", "Component ID", false); - public static final SearchableField AlternateIdentifierURI = new NamedSearchableField("AlternateIdentifierURI", "alternateIdentifierUri", "Alternate Identifier URI", false); - public static final SearchableField FileSize = new NamedSearchableField("FileSize", "fileSize", "File Size", false, SearchableFieldType.DATA_SIZE); - public static final SearchableField Details = new NamedSearchableField("Details", "details", "Details", false, SearchableFieldType.STRING); - public static final SearchableField Relationship = new NamedSearchableField("Relationship", "relationship", "Relationship", false, SearchableFieldType.STRING); - - public static final SearchableField LineageStartDate = new NamedSearchableField("LineageStartDate", "lineageStartDate", "Lineage Start Date", false, SearchableFieldType.DATE); - public static final SearchableField LineageIdentifier = new NamedSearchableField("LineageIdentifiers", "lineageIdentifier", "Lineage Identifier", false, SearchableFieldType.STRING); - - public static final SearchableField ContentClaimSection = new NamedSearchableField("ContentClaimSection", "contentClaimSection", "Content Claim Section", false, SearchableFieldType.STRING); - public static final SearchableField ContentClaimContainer = new NamedSearchableField("ContentClaimContainer", "contentClaimContainer", "Content Claim Container", false, SearchableFieldType.STRING); - public static final SearchableField ContentClaimIdentifier = new NamedSearchableField("ContentClaimIdentifier", "contentClaimIdentifier", "Content Claim Identifier", false, SearchableFieldType.STRING); - public static final SearchableField ContentClaimOffset = new NamedSearchableField("ContentClaimOffset", "contentClaimOffset", "Content Claim Offset", false, SearchableFieldType.LONG); - public static final SearchableField SourceQueueIdentifier = new NamedSearchableField("SourceQueueIdentifier", "sourceQueueIdentifier", "Source Queue Identifier", false, SearchableFieldType.STRING); - - private static final Map<String, SearchableField> standardFields; - - static { - final SearchableField[] searchableFields = new SearchableField[]{ - EventTime, FlowFileUUID, Filename, EventType, TransitURI, - ComponentID, AlternateIdentifierURI, FileSize, Relationship, Details, - LineageStartDate, LineageIdentifier, ContentClaimSection, ContentClaimContainer, ContentClaimIdentifier, - ContentClaimOffset, SourceQueueIdentifier}; - - final Map<String, SearchableField> fields = new HashMap<>(); - for (final SearchableField field : searchableFields) { - fields.put(field.getIdentifier(), field); - } - - standardFields = Collections.unmodifiableMap(fields); - } - - private SearchableFields() { - } - - public static Collection<SearchableField> getStandardFields() { - return standardFields.values(); - } - - public static SearchableField getSearchableField(final String fieldIdentifier) { - return standardFields.get(fieldIdentifier); - } - - public static SearchableField newSearchableAttribute(final String attributeName) { - return new NamedSearchableField(attributeName, attributeName, attributeName, true); - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java ---------------------------------------------------------------------- diff --git a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java b/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java deleted file mode 100644 index afb56e8..0000000 --- a/commons/data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardLineageResult.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.provenance; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.nifi.provenance.lineage.ComputeLineageResult; -import org.apache.nifi.provenance.lineage.EdgeNode; -import org.apache.nifi.provenance.lineage.EventNode; -import org.apache.nifi.provenance.lineage.FlowFileNode; -import org.apache.nifi.provenance.lineage.LineageEdge; -import org.apache.nifi.provenance.lineage.LineageNode; - -/** - * - */ -public class StandardLineageResult implements ComputeLineageResult { - - public static final int TTL = (int) TimeUnit.MILLISECONDS.convert(30, TimeUnit.MINUTES); - private static final Logger logger = LoggerFactory.getLogger(StandardLineageResult.class); - - private final Collection<String> flowFileUuids; - private final Collection<ProvenanceEventRecord> relevantRecords = new ArrayList<>(); - private final Set<LineageNode> nodes = new HashSet<>(); - private final Set<LineageEdge> edges = new HashSet<>(); - private final int numSteps; - private final long creationNanos; - private long computationNanos; - - private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); - private final Lock readLock = rwLock.readLock(); - private final Lock writeLock = rwLock.writeLock(); - - private Date expirationDate = null; - private String error = null; - private int numCompletedSteps = 0; - - private volatile boolean canceled = false; - - public StandardLineageResult(final int numSteps, final Collection<String> flowFileUuids) { - this.numSteps = numSteps; - this.creationNanos = System.nanoTime(); - this.flowFileUuids = flowFileUuids; - - updateExpiration(); - } - - @Override - public List<LineageNode> getNodes() { - readLock.lock(); - try { - return new ArrayList<>(nodes); - } finally { - readLock.unlock(); - } - } - - @Override - public List<LineageEdge> getEdges() { - readLock.lock(); - try { - return new ArrayList<>(edges); - } finally { - readLock.unlock(); - } - } - - public int getNumberOfEdges() { - readLock.lock(); - try { - return edges.size(); - } finally { - readLock.unlock(); - } - } - - public int getNumberOfNodes() { - readLock.lock(); - try { - return nodes.size(); - } finally { - readLock.unlock(); - } - } - - public long getComputationTime(final TimeUnit timeUnit) { - readLock.lock(); - try { - return timeUnit.convert(computationNanos, TimeUnit.NANOSECONDS); - } finally { - readLock.unlock(); - } - } - - @Override - public Date getExpiration() { - readLock.lock(); - try { - return expirationDate; - } finally { - readLock.unlock(); - } - } - - @Override - public String getError() { - readLock.lock(); - try { - return error; - } finally { - readLock.unlock(); - } - } - - @Override - public int getPercentComplete() { - readLock.lock(); - try { - return (numSteps < 1) ? 100 : (int) (((float) numCompletedSteps / (float) numSteps) * 100.0F); - } finally { - readLock.unlock(); - } - } - - @Override - public boolean isFinished() { - readLock.lock(); - try { - return numCompletedSteps >= numSteps || canceled; - } finally { - readLock.unlock(); - } - } - - public void setError(final String error) { - writeLock.lock(); - try { - this.error = error; - numCompletedSteps++; - - updateExpiration(); - - if (numCompletedSteps >= numSteps) { - computationNanos = System.nanoTime() - creationNanos; - } - } finally { - writeLock.unlock(); - } - } - - public void update(final Collection<ProvenanceEventRecord> records) { - writeLock.lock(); - try { - relevantRecords.addAll(records); - - numCompletedSteps++; - updateExpiration(); - - if (numCompletedSteps >= numSteps && error == null) { - computeLineage(); - computationNanos = System.nanoTime() - creationNanos; - } - } finally { - writeLock.unlock(); - } - } - - /** - * Computes the lineage from the relevant Provenance Event Records. This - * method must be called with the write lock held and is only going to be - * useful after all of the records have been successfully obtained - */ - private void computeLineage() { - final long startNanos = System.nanoTime(); - - nodes.clear(); - edges.clear(); - - Map<String, LineageNode> lastEventMap = new HashMap<>(); // maps FlowFile UUID to last event for that FlowFile - final List<ProvenanceEventRecord> sortedRecords = new ArrayList<>(relevantRecords); - Collections.sort(sortedRecords, new Comparator<ProvenanceEventRecord>() { - @Override - public int compare(final ProvenanceEventRecord o1, final ProvenanceEventRecord o2) { - // Sort on Event Time, then Event ID. - final int eventTimeComparison = Long.compare(o1.getEventTime(), o2.getEventTime()); - if (eventTimeComparison == 0) { - return Long.compare(o1.getEventId(), o2.getEventId()); - } else { - return eventTimeComparison; - } - } - }); - - // convert the StandardProvenanceRecord objects into Lineage nodes (FlowFileNode, EventNodes). - for (final ProvenanceEventRecord record : sortedRecords) { - final LineageNode lineageNode = new EventNode(record); - final boolean added = nodes.add(lineageNode); - if (!added) { - logger.debug("Did not add {} because it already exists in the 'nodes' set", lineageNode); - } - - // Create an edge that connects this node to the previous node for the same FlowFile UUID. - final LineageNode lastNode = lastEventMap.get(record.getFlowFileUuid()); - if (lastNode != null) { - // We calculate the Edge UUID based on whether or not this event is a SPAWN. - // If this event is a SPAWN, then we want to use the previous node's UUID because a - // SPAWN Event's UUID is not necessarily what we want, since a SPAWN Event's UUID pertains to - // only one of (potentially) many UUIDs associated with the event. Otherwise, we know that - // the UUID of this record is appropriate, so we just use it. - final String edgeUuid; - - switch (record.getEventType()) { - case JOIN: - case CLONE: - case REPLAY: - edgeUuid = lastNode.getFlowFileUuid(); - break; - default: - edgeUuid = record.getFlowFileUuid(); - break; - } - - edges.add(new EdgeNode(edgeUuid, lastNode, lineageNode)); - } - - lastEventMap.put(record.getFlowFileUuid(), lineageNode); - - switch (record.getEventType()) { - case FORK: - case JOIN: - case REPLAY: - case CLONE: { - // For events that create FlowFile nodes, we need to create the FlowFile Nodes and associated Edges, as appropriate - for (final String childUuid : record.getChildUuids()) { - if (flowFileUuids.contains(childUuid)) { - final FlowFileNode childNode = new FlowFileNode(childUuid, record.getEventTime()); - final boolean isNewFlowFile = nodes.add(childNode); - if (!isNewFlowFile) { - final String msg = "Unable to generate Lineage Graph because multiple events were registered claiming to have generated the same FlowFile (UUID = " + childNode.getFlowFileUuid() + ")"; - logger.error(msg); - setError(msg); - return; - } - - edges.add(new EdgeNode(childNode.getFlowFileUuid(), lineageNode, childNode)); - lastEventMap.put(childUuid, childNode); - } - } - for (final String parentUuid : record.getParentUuids()) { - LineageNode lastNodeForParent = lastEventMap.get(parentUuid); - if (lastNodeForParent != null && !lastNodeForParent.equals(lineageNode)) { - edges.add(new EdgeNode(parentUuid, lastNodeForParent, lineageNode)); - } - - lastEventMap.put(parentUuid, lineageNode); - } - } - break; - case RECEIVE: - case CREATE: { - // for a receive event, we want to create a FlowFile Node that represents the FlowFile received - // and create an edge from the Receive Event to the FlowFile Node - final LineageNode flowFileNode = new FlowFileNode(record.getFlowFileUuid(), record.getEventTime()); - final boolean isNewFlowFile = nodes.add(flowFileNode); - if (!isNewFlowFile) { - final String msg = "Found cycle in graph. This indicates that multiple events were registered claiming to have generated the same FlowFile (UUID = " + flowFileNode.getFlowFileUuid() + ")"; - setError(msg); - logger.error(msg); - return; - } - edges.add(new EdgeNode(record.getFlowFileUuid(), lineageNode, flowFileNode)); - lastEventMap.put(record.getFlowFileUuid(), flowFileNode); - } - break; - default: - break; - } - } - - final long nanos = System.nanoTime() - startNanos; - logger.debug("Finished building lineage with {} nodes and {} edges in {} millis", nodes.size(), edges.size(), TimeUnit.NANOSECONDS.toMillis(nanos)); - } - - void cancel() { - this.canceled = true; - } - - /** - * Must be called with write lock! - */ - private void updateExpiration() { - expirationDate = new Date(System.currentTimeMillis() + TTL); - } -}
