http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties index 4ad5784..1e308bf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties @@ -14,7 +14,6 @@ # limitations under the License. # Core Properties # -nifi.version=nifi-test 3.0.0 nifi.flow.configuration.file=./target/flow.xml.gz nifi.flow.configuration.archive.dir=./target/archive/ nifi.flowcontroller.autoResumeState=true
http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index cecfabf..b8d3ad9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -29,7 +29,6 @@ <nifi.jvm.permgen.mb>128</nifi.jvm.permgen.mb> <nifi.run.as /> <!-- nifi.properties: core properties --> - <nifi.version>${project.version}</nifi.version> <nifi.flowcontroller.autoResumeState>true</nifi.flowcontroller.autoResumeState> <nifi.flowcontroller.graceful.shutdown.period>10 sec</nifi.flowcontroller.graceful.shutdown.period> <nifi.flowservice.writedelay.interval>500 ms</nifi.flowservice.writedelay.interval> @@ -174,13 +173,6 @@ <nifi.kerberos.spnego.principal /> <nifi.kerberos.spnego.keytab.location /> <nifi.kerberos.spnego.authentication.expiration>12 hours</nifi.kerberos.spnego.authentication.expiration> - - <!-- nifi.properties: build info --> - <nifi.build.tag>${project.scm.tag}</nifi.build.tag> - <nifi.build.timestamp>${maven.build.timestamp}</nifi.build.timestamp> - <!-- buildRevision and buildBranch provided by buildnumber-maven-plugin or build-info-read-properties profile --> - <nifi.build.branch>${buildBranch}</nifi.build.branch> - <nifi.build.revision>${buildRevision}</nifi.build.revision> </properties> <build> <plugins> http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 4a93828..62b4c8f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -14,7 +14,6 @@ # limitations under the License. # Core Properties # -nifi.version=${nifi.version} nifi.flow.configuration.file=${nifi.flow.configuration.file} nifi.flow.configuration.archive.enabled=${nifi.flow.configuration.archive.enabled} nifi.flow.configuration.archive.dir=${nifi.flow.configuration.archive.dir} @@ -198,10 +197,4 @@ nifi.kerberos.spnego.authentication.expiration=${nifi.kerberos.spnego.authentica # external properties files for variable registry # supports a comma delimited list of file locations -nifi.variable.registry.properties= - -# Build info -nifi.build.tag=${nifi.build.tag} -nifi.build.branch=${nifi.build.branch} -nifi.build.revision=${nifi.build.revision} -nifi.build.timestamp=${nifi.build.timestamp} \ No newline at end of file +nifi.variable.registry.properties= \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml index 76c0131..44ec7ea 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml @@ -24,6 +24,11 @@ <dependencies> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-nar-utils</artifactId> <scope>compile</scope> </dependency> http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java index 0488823..32cbeba 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java @@ -16,6 +16,18 @@ */ package org.apache.nifi; +import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.documentation.DocGenerator; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.nar.ExtensionMapping; +import org.apache.nifi.nar.NarClassLoaders; +import org.apache.nifi.nar.NarUnpacker; +import org.apache.nifi.util.FileUtils; +import org.apache.nifi.util.NiFiProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.slf4j.bridge.SLF4JBridgeHandler; + import java.io.File; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; @@ -30,6 +42,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.Executors; @@ -40,17 +53,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import org.apache.nifi.documentation.DocGenerator; -import org.apache.nifi.nar.ExtensionManager; -import org.apache.nifi.nar.ExtensionMapping; -import org.apache.nifi.nar.NarClassLoaders; -import org.apache.nifi.nar.NarUnpacker; -import org.apache.nifi.util.FileUtils; -import org.apache.nifi.util.NiFiProperties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.bridge.SLF4JBridgeHandler; - public class NiFi { private static final Logger LOGGER = LoggerFactory.getLogger(NiFi.class); @@ -124,31 +126,34 @@ public class NiFi { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); + final Bundle systemBundle = ExtensionManager.createSystemBundle(properties); + // expand the nars - final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties); + final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties, systemBundle); // load the extensions classloaders NarClassLoaders.getInstance().init(properties.getFrameworkWorkingDirectory(), properties.getExtensionsWorkingDirectory()); // load the framework classloader - final ClassLoader frameworkClassLoader = NarClassLoaders.getInstance().getFrameworkClassLoader(); + final ClassLoader frameworkClassLoader = NarClassLoaders.getInstance().getFrameworkBundle().getClassLoader(); if (frameworkClassLoader == null) { throw new IllegalStateException("Unable to find the framework NAR ClassLoader."); } // discover the extensions - ExtensionManager.discoverExtensions(NarClassLoaders.getInstance().getExtensionClassLoaders()); + final Set<Bundle> narBundles = NarClassLoaders.getInstance().getBundles(); + ExtensionManager.discoverExtensions(systemBundle, narBundles); ExtensionManager.logClassLoaderMapping(); - DocGenerator.generate(properties); + DocGenerator.generate(properties, extensionMapping); // load the server from the framework classloader Thread.currentThread().setContextClassLoader(frameworkClassLoader); Class<?> jettyServer = Class.forName("org.apache.nifi.web.server.JettyServer", true, frameworkClassLoader); - Constructor<?> jettyConstructor = jettyServer.getConstructor(NiFiProperties.class); + Constructor<?> jettyConstructor = jettyServer.getConstructor(NiFiProperties.class, Set.class); final long startTime = System.nanoTime(); - nifiServer = (NiFiServer) jettyConstructor.newInstance(properties); + nifiServer = (NiFiServer) jettyConstructor.newInstance(properties, narBundles); nifiServer.setExtensionMapping(extensionMapping); if (shutdown) { http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties index f341c01..ff190d4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties @@ -14,7 +14,6 @@ # limitations under the License. # Core Properties # -nifi.version=1.0.0-SNAPSHOT nifi.flow.configuration.file=./target/conf/flow.xml.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./target/conf/archive/ http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties index cbc9382..6dd949b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties @@ -14,7 +14,6 @@ # limitations under the License. # Core Properties # -nifi.version=1.0.0-SNAPSHOT nifi.flow.configuration.file=./target/conf/flow.xml.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./target/conf/archive/ http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties index f170d7e..9c52fe6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties @@ -14,7 +14,6 @@ # limitations under the License. # Core Properties # -nifi.version=1.0.0-SNAPSHOT nifi.flow.configuration.file=./target/conf/flow.xml.gz nifi.flow.configuration.archive.enabled=true nifi.flow.configuration.archive.dir=./target/conf/archive/ http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties index 5b1134b..4709c21 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/test/resources/nifi.properties @@ -25,7 +25,6 @@ nifi.flowfile.repository.checkpoint.millis=120000 nifi.content.repository.directory.default=./content_repository nifi.provenance.repository.capacity=25000 nifi.templates.directory=./conf/templates -nifi.version=nifi 0.2.1-SNAPSHOT nifi.ui.banner.text=DEFAULT BANNER nifi.ui.autorefresh.interval.seconds=30 nifi.flowcontroller.autoStartProcessors=true http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/pom.xml new file mode 100644 index 0000000..fd3e5a4 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/pom.xml @@ -0,0 +1,40 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-bundle</artifactId> + <version>1.2.0-SNAPSHOT</version> + </parent> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-standard-prioritizers</artifactId> + <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-utils</artifactId> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-mock</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/FirstInFirstOutPrioritizer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/FirstInFirstOutPrioritizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/FirstInFirstOutPrioritizer.java new file mode 100644 index 0000000..08437c7 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/FirstInFirstOutPrioritizer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.FlowFilePrioritizer; + +public class FirstInFirstOutPrioritizer implements FlowFilePrioritizer { + + @Override + public int compare(final FlowFile o1, final FlowFile o2) { + if (o1 == null && o2 == null) { + return 0; + } else if (o2 == null) { + return -1; + } else if (o1 == null) { + return 1; + } + + final int dateComparison = o1.getLastQueueDate().compareTo(o2.getLastQueueDate()); + if (dateComparison != 0) { + return dateComparison; + } + + return Long.compare(o1.getQueueDateIndex(), o2.getQueueDateIndex()); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/NewestFlowFileFirstPrioritizer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/NewestFlowFileFirstPrioritizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/NewestFlowFileFirstPrioritizer.java new file mode 100644 index 0000000..4893cf0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/NewestFlowFileFirstPrioritizer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.FlowFilePrioritizer; + +public class NewestFlowFileFirstPrioritizer implements FlowFilePrioritizer { + + @Override + public int compare(final FlowFile o1, final FlowFile o2) { + if (o1 == null && o2 == null) { + return 0; + } else if (o2 == null) { + return -1; + } else if (o1 == null) { + return 1; + } + + final int lineageDateCompare = Long.compare(o2.getLineageStartDate(), o1.getLineageStartDate()); + if (lineageDateCompare != 0) { + return lineageDateCompare; + } + + return Long.compare(o2.getLineageStartIndex(), o1.getLineageStartIndex()); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/OldestFlowFileFirstPrioritizer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/OldestFlowFileFirstPrioritizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/OldestFlowFileFirstPrioritizer.java new file mode 100644 index 0000000..386d912 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/OldestFlowFileFirstPrioritizer.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.FlowFilePrioritizer; + +public class OldestFlowFileFirstPrioritizer implements FlowFilePrioritizer { + + @Override + public int compare(final FlowFile o1, final FlowFile o2) { + if (o1 == null && o2 == null) { + return 0; + } else if (o2 == null) { + return -1; + } else if (o1 == null) { + return 1; + } + + final int lineageDateCompare = Long.compare(o1.getLineageStartDate(), o2.getLineageStartDate()); + if (lineageDateCompare != 0) { + return lineageDateCompare; + } + + return Long.compare(o1.getLineageStartIndex(), o2.getLineageStartIndex()); + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizer.java new file mode 100644 index 0000000..3d27930 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizer.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import java.util.regex.Pattern; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.FlowFilePrioritizer; +import org.apache.nifi.flowfile.attributes.CoreAttributes; + +/** + * This prioritizer checks each FlowFile for a "priority" attribute and lets + * that attribute determine the priority. + * + * 1. if neither FlowFile has a "priority" attribute then order will be + * FirstInFirstOut 2. if one FlowFile has a "priority" attribute and the other + * does not, then the one with the attribute wins 3. if one or both "priority" + * attributes is an integer, then the lowest number wins 4. the "priority" + * attributes are compared lexicographically and the lowest wins + */ +public class PriorityAttributePrioritizer implements FlowFilePrioritizer { + + private static final Pattern intPattern = Pattern.compile("-?\\d+"); + + @Override + public int compare(FlowFile o1, FlowFile o2) { + if (o1 == null && o2 == null) { + return 0; + } else if (o2 == null) { + return -1; + } else if (o1 == null) { + return 1; + } + + String o1Priority = o1.getAttribute(CoreAttributes.PRIORITY.key()); + String o2Priority = o2.getAttribute(CoreAttributes.PRIORITY.key()); + if (o1Priority == null && o2Priority == null) { + return -1; // this is not 0 to match FirstInFirstOut + } else if (o2Priority == null) { + return -1; + } else if (o1Priority == null) { + return 1; + } + + // priority exists on both FlowFiles + if (intPattern.matcher(o1Priority.trim()).matches()) { + if (intPattern.matcher(o2Priority.trim()).matches()) { + try { + // both o1Priority and o2Priority are numbers + long o1num = Long.parseLong(o1Priority.trim()); + long o2num = Long.parseLong(o2Priority.trim()); + return o1num < o2num ? -1 : (o1num > o2num ? 1 : 0); + } catch (NumberFormatException e) { + // not a long after regex matched + return 0; + } + } else { + // o1Priority is a number, o2Priority is not, o1 wins + return -1; + } + } else { + if (intPattern.matcher(o2Priority.trim()).matches()) { + // o2Priority is a number, o1Priority is not, o2 wins + return 1; + } else { + // neither o1Priority nor o2Priority are numbers + return o1Priority.compareTo(o2Priority); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/resources/META-INF/services/org.apache.nifi.flowfile.FlowFilePrioritizer ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/resources/META-INF/services/org.apache.nifi.flowfile.FlowFilePrioritizer b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/resources/META-INF/services/org.apache.nifi.flowfile.FlowFilePrioritizer new file mode 100644 index 0000000..1c3cd25 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/main/resources/META-INF/services/org.apache.nifi.flowfile.FlowFilePrioritizer @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer +org.apache.nifi.prioritizer.OldestFlowFileFirstPrioritizer +org.apache.nifi.prioritizer.PriorityAttributePrioritizer \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/NewestFirstPrioritizerTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/NewestFirstPrioritizerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/NewestFirstPrioritizerTest.java new file mode 100644 index 0000000..37364e5 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/NewestFirstPrioritizerTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockProcessSession; +import org.apache.nifi.util.SharedSessionState; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class NewestFirstPrioritizerTest { + + @Test + public void testPrioritizer() throws InstantiationException, IllegalAccessException { + final Processor processor = new SimpleProcessor(); + final AtomicLong idGenerator = new AtomicLong(0L); + final MockProcessSession session = new MockProcessSession(new SharedSessionState(processor, idGenerator), Mockito.mock(Processor.class)); + + final MockFlowFile flowFile1 = session.create(); + try { + Thread.sleep(2); // guarantee the FlowFile entryDate for flowFile2 is different than flowFile1 + } catch (final InterruptedException e) { + } + final MockFlowFile flowFile2 = session.create(); + + final NewestFlowFileFirstPrioritizer prioritizer = new NewestFlowFileFirstPrioritizer(); + Assert.assertEquals(0, prioritizer.compare(null, null)); + Assert.assertEquals(-1, prioritizer.compare(flowFile1, null)); + Assert.assertEquals(1, prioritizer.compare(null, flowFile1)); + Assert.assertEquals(0, prioritizer.compare(flowFile1, flowFile1)); + Assert.assertEquals(0, prioritizer.compare(flowFile2, flowFile2)); + Assert.assertEquals(1, prioritizer.compare(flowFile1, flowFile2)); + Assert.assertEquals(-1, prioritizer.compare(flowFile2, flowFile1)); + } + + public class SimpleProcessor extends AbstractProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + } + + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/OldestFirstPrioritizerTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/OldestFirstPrioritizerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/OldestFirstPrioritizerTest.java new file mode 100644 index 0000000..68a8d7d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/OldestFirstPrioritizerTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockProcessSession; +import org.apache.nifi.util.SharedSessionState; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class OldestFirstPrioritizerTest { + + @Test + public void testPrioritizer() throws InstantiationException, IllegalAccessException { + final Processor processor = new SimpleProcessor(); + final AtomicLong idGenerator = new AtomicLong(0L); + final MockProcessSession session = new MockProcessSession(new SharedSessionState(processor, idGenerator), Mockito.mock(Processor.class)); + + final MockFlowFile flowFile1 = session.create(); + try { + Thread.sleep(2); // guarantee the FlowFile entryDate for flowFile2 is different than flowFile1 + } catch (final InterruptedException e) { + } + final MockFlowFile flowFile2 = session.create(); + + final OldestFlowFileFirstPrioritizer prioritizer = new OldestFlowFileFirstPrioritizer(); + Assert.assertEquals(0, prioritizer.compare(null, null)); + Assert.assertEquals(-1, prioritizer.compare(flowFile1, null)); + Assert.assertEquals(1, prioritizer.compare(null, flowFile1)); + Assert.assertEquals(0, prioritizer.compare(flowFile1, flowFile1)); + Assert.assertEquals(0, prioritizer.compare(flowFile2, flowFile2)); + Assert.assertEquals(-1, prioritizer.compare(flowFile1, flowFile2)); + Assert.assertEquals(1, prioritizer.compare(flowFile2, flowFile1)); + } + + public class SimpleProcessor extends AbstractProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + } + + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizerTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizerTest.java new file mode 100644 index 0000000..7098551 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-standard-prioritizers/src/test/java/org/apache/nifi/prioritizer/PriorityAttributePrioritizerTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.prioritizer; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Processor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockProcessSession; +import org.apache.nifi.util.SharedSessionState; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +public class PriorityAttributePrioritizerTest { + + static Map<String, String> attrsPri1 = new HashMap<String, String>(); + static Map<String, String> attrsPri2 = new HashMap<String, String>(); + static Map<String, String> attrsPrin1 = new HashMap<String, String>(); + static Map<String, String> attrsPriA = new HashMap<String, String>(); + static Map<String, String> attrsPriB = new HashMap<String, String>(); + static Map<String, String> attrsPriLP = new HashMap<String, String>(); + static Map<String, String> attrsPriLN = new HashMap<String, String>(); + + @BeforeClass + public static void init() { + attrsPri1.put(CoreAttributes.PRIORITY.key(), "1"); + attrsPri2.put(CoreAttributes.PRIORITY.key(), "2"); + attrsPrin1.put(CoreAttributes.PRIORITY.key(), "-1"); + attrsPriA.put(CoreAttributes.PRIORITY.key(), "A"); + attrsPriB.put(CoreAttributes.PRIORITY.key(), "B"); + attrsPriLP.put(CoreAttributes.PRIORITY.key(), "5432123456789"); + attrsPriLN.put(CoreAttributes.PRIORITY.key(), "-5432123456789"); + } + + @Test + public void testPrioritizer() throws InstantiationException, IllegalAccessException { + final Processor processor = new SimpleProcessor(); + final AtomicLong idGenerator = new AtomicLong(0L); + final MockProcessSession session = new MockProcessSession(new SharedSessionState(processor, idGenerator), Mockito.mock(Processor.class)); + + final MockFlowFile ffNoPriority = session.create(); + final MockFlowFile ffPri1 = session.create(); + ffPri1.putAttributes(attrsPri1); + final MockFlowFile ffPri2 = session.create(); + ffPri2.putAttributes(attrsPri2); + final MockFlowFile ffPrin1 = session.create(); + ffPrin1.putAttributes(attrsPrin1); + final MockFlowFile ffPriA = session.create(); + ffPriA.putAttributes(attrsPriA); + final MockFlowFile ffPriB = session.create(); + ffPriB.putAttributes(attrsPriB); + final MockFlowFile ffPriLP = session.create(); + ffPriLP.putAttributes(attrsPriLP); + final MockFlowFile ffPriLN = session.create(); + ffPriLN.putAttributes(attrsPriLN); + + final PriorityAttributePrioritizer prioritizer = new PriorityAttributePrioritizer(); + assertEquals(0, prioritizer.compare(null, null)); + assertEquals(-1, prioritizer.compare(ffNoPriority, null)); + assertEquals(1, prioritizer.compare(null, ffNoPriority)); + + assertEquals(-1, prioritizer.compare(ffNoPriority, ffNoPriority)); + assertEquals(-1, prioritizer.compare(ffPri1, ffNoPriority)); + assertEquals(1, prioritizer.compare(ffNoPriority, ffPri1)); + + assertEquals(0, prioritizer.compare(ffPri1, ffPri1)); + assertEquals(-1, prioritizer.compare(ffPri1, ffPri2)); + assertEquals(1, prioritizer.compare(ffPri2, ffPri1)); + assertEquals(-1, prioritizer.compare(ffPrin1, ffPri1)); + assertEquals(1, prioritizer.compare(ffPri1, ffPrin1)); + + assertEquals(-1, prioritizer.compare(ffPri1, ffPriA)); + assertEquals(1, prioritizer.compare(ffPriA, ffPri1)); + + assertEquals(0, prioritizer.compare(ffPriA, ffPriA)); + assertEquals(-1, prioritizer.compare(ffPriA, ffPriB)); + assertEquals(1, prioritizer.compare(ffPriB, ffPriA)); + + assertEquals(1, prioritizer.compare(ffPriLP, ffPri1)); + assertEquals(-1, prioritizer.compare(ffPri1, ffPriLP)); + assertEquals(-1, prioritizer.compare(ffPriLN, ffPri1)); + assertEquals(1, prioritizer.compare(ffPri1, ffPriLN)); + } + + public class SimpleProcessor extends AbstractProcessor { + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + } + + } + +} http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java index d2def9f..84c9671 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java @@ -19,15 +19,15 @@ package org.apache.nifi.web.server; import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.NiFiServer; +import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.bundle.BundleDetails; import org.apache.nifi.controller.UninheritableFlowException; import org.apache.nifi.controller.serialization.FlowSerializationException; import org.apache.nifi.controller.serialization.FlowSynchronizationException; import org.apache.nifi.lifecycle.LifeCycleStartException; import org.apache.nifi.nar.ExtensionMapping; -import org.apache.nifi.nar.NarClassLoaders; import org.apache.nifi.security.util.KeyStoreUtils; import org.apache.nifi.services.FlowService; import org.apache.nifi.ui.extension.UiExtension; @@ -125,12 +125,7 @@ public class JettyServer implements NiFiServer { private UiExtensionMapping componentUiExtensions; private Collection<WebAppContext> componentUiExtensionWebContexts; - /** - * Creates and configures a new Jetty instance. - * - * @param props the configuration - */ - public JettyServer(final NiFiProperties props) { + public JettyServer(final NiFiProperties props, final Set<Bundle> bundles) { final QueuedThreadPool threadPool = new QueuedThreadPool(props.getWebThreads()); threadPool.setName("NiFi Web Server"); @@ -145,41 +140,14 @@ public class JettyServer implements NiFiServer { // configure server configureConnectors(server); - // load wars from the nar working directories - loadWars(locateNarWorkingDirectories()); - } - - private Set<File> locateNarWorkingDirectories() { - final File frameworkWorkingDir = props.getFrameworkWorkingDirectory(); - final File extensionsWorkingDir = props.getExtensionsWorkingDirectory(); - - final File[] frameworkDir = frameworkWorkingDir.listFiles(); - if (frameworkDir == null) { - throw new IllegalStateException(String.format("Unable to access framework working directory: %s", frameworkWorkingDir.getAbsolutePath())); - } - - final File[] extensionDirs = extensionsWorkingDir.listFiles(); - if (extensionDirs == null) { - throw new IllegalStateException(String.format("Unable to access extensions working directory: %s", extensionsWorkingDir.getAbsolutePath())); - } - - // we want to consider the framework and all extension NARs - final Set<File> narWorkingDirectories = new HashSet<>(Arrays.asList(frameworkDir)); - narWorkingDirectories.addAll(Arrays.asList(extensionDirs)); - - return narWorkingDirectories; + // load wars from the bundle + loadWars(bundles); } - /** - * Loads the WARs in the specified NAR working directories. A WAR file must - * have a ".war" extension. - * - * @param narWorkingDirectories dirs - */ - private void loadWars(final Set<File> narWorkingDirectories) { + private void loadWars(final Set<Bundle> bundles) { // load WARs - Map<File, File> warToNarWorkingDirectoryLookup = findWars(narWorkingDirectories); + final Map<File, Bundle> warToBundleLookup = findWars(bundles); // locate each war being deployed File webUiWar = null; @@ -188,7 +156,7 @@ public class JettyServer implements NiFiServer { File webDocsWar = null; File webContentViewerWar = null; List<File> otherWars = new ArrayList<>(); - for (File war : warToNarWorkingDirectoryLookup.keySet()) { + for (File war : warToBundleLookup.keySet()) { if (war.getName().toLowerCase().startsWith("nifi-web-api")) { webApiWar = war; } else if (war.getName().toLowerCase().startsWith("nifi-web-error")) { @@ -242,8 +210,8 @@ public class JettyServer implements NiFiServer { String warName = StringUtils.substringBeforeLast(war.getName(), "."); String warContextPath = String.format("/%s", warName); - // attempt to locate the nar class loader for this war - ClassLoader narClassLoaderForWar = NarClassLoaders.getInstance().getExtensionClassLoader(warToNarWorkingDirectoryLookup.get(war)); + // get the classloader for this war + ClassLoader narClassLoaderForWar = warToBundleLookup.get(war).getClassLoader(); // this should never be null if (narClassLoaderForWar == null) { @@ -269,17 +237,22 @@ public class JettyServer implements NiFiServer { contentViewerWebContexts.add(extensionUiContext); } else { // consider each component type identified - for (final String componentType : types) { - logger.info(String.format("Loading UI extension [%s, %s] for %s", extensionType, warContextPath, types)); + for (final String componentTypeCoordinates : types) { + logger.info(String.format("Loading UI extension [%s, %s] for %s", extensionType, warContextPath, componentTypeCoordinates)); // record the extension definition final UiExtension uiExtension = new UiExtension(extensionType, warContextPath); // create if this is the first extension for this component type - List<UiExtension> componentUiExtensionsForType = componentUiExtensionsByType.get(componentType); + List<UiExtension> componentUiExtensionsForType = componentUiExtensionsByType.get(componentTypeCoordinates); if (componentUiExtensionsForType == null) { componentUiExtensionsForType = new ArrayList<>(); - componentUiExtensionsByType.put(componentType, componentUiExtensionsForType); + componentUiExtensionsByType.put(componentTypeCoordinates, componentUiExtensionsForType); + } + + // see if there is already a ui extension of this same time + if (containsUiExtensionType(componentUiExtensionsForType, extensionType)) { + throw new IllegalStateException(String.format("Encountered duplicate UI for %s", componentTypeCoordinates)); } // record this extension @@ -335,6 +308,23 @@ public class JettyServer implements NiFiServer { } /** + * Returns whether or not the specified ui extensions already contains an extension of the specified type. + * + * @param componentUiExtensionsForType ui extensions for the type + * @param extensionType type of ui extension + * @return whether or not the specified ui extensions already contains an extension of the specified type + */ + private boolean containsUiExtensionType(final List<UiExtension> componentUiExtensionsForType, final UiExtensionType extensionType) { + for (final UiExtension uiExtension : componentUiExtensionsForType) { + if (extensionType.equals(uiExtension.getExtensionType())) { + return true; + } + } + + return false; + } + + /** * Enables compression for the specified handler. * * @param handler handler to enable compression for @@ -347,12 +337,13 @@ public class JettyServer implements NiFiServer { return gzip; } - private Map<File, File> findWars(final Set<File> narWorkingDirectories) { - final Map<File, File> wars = new HashMap<>(); + private Map<File, Bundle> findWars(final Set<Bundle> bundles) { + final Map<File, Bundle> wars = new HashMap<>(); // consider each nar working directory - for (final File narWorkingDirectory : narWorkingDirectories) { - final File narDependencies = new File(narWorkingDirectory, "META-INF/bundled-dependencies"); + bundles.forEach(bundle -> { + final BundleDetails details = bundle.getBundleDetails(); + final File narDependencies = new File(details.getWorkingDirectory(), "META-INF/bundled-dependencies"); if (narDependencies.isDirectory()) { // list the wars from this nar final File[] narDependencyDirs = narDependencies.listFiles(WAR_FILTER); @@ -362,10 +353,10 @@ public class JettyServer implements NiFiServer { // add each war for (final File war : narDependencyDirs) { - wars.put(war, narWorkingDirectory); + wars.put(war, bundle); } } - } + }); return wars; } @@ -433,48 +424,6 @@ public class JettyServer implements NiFiServer { return null; } - /** - * Returns the extension in the specified WAR using the specified path. - * - * @param war war - * @param path path - * @return extensions - */ - private List<String> getWarExtensions(final File war, final String path) { - List<String> processorTypes = new ArrayList<>(); - - // load the jar file and attempt to find the nifi-processor entry - JarFile jarFile = null; - try { - jarFile = new JarFile(war); - JarEntry jarEntry = jarFile.getJarEntry(path); - - // ensure the nifi-processor entry was found - if (jarEntry != null) { - // get an input stream for the nifi-processor configuration file - try (final BufferedReader in = new BufferedReader( - new InputStreamReader(jarFile.getInputStream(jarEntry)))) { - - // read in each configured type - String rawProcessorType; - while ((rawProcessorType = in.readLine()) != null) { - // extract the processor type - final String processorType = extractComponentType(rawProcessorType); - if (processorType != null) { - processorTypes.add(processorType); - } - } - } - } - } catch (IOException ioe) { - logger.warn("Unable to inspect {} for a custom processor UI.", new Object[]{war, ioe}); - } finally { - IOUtils.closeQuietly(jarFile); - } - - return processorTypes; - } - private WebAppContext loadWar(final File warFile, final String contextPath, final ClassLoader parentClassLoader) { final WebAppContext webappContext = new WebAppContext(warFile.getPath(), contextPath); webappContext.setContextPath(contextPath); @@ -530,8 +479,8 @@ public class JettyServer implements NiFiServer { final Resource docsResource = Resource.newResource(docsDir); // load the component documentation working directory - final String componentDocsDirPath = props.getProperty(NiFiProperties.COMPONENT_DOCS_DIRECTORY, "work/docs/components"); - final File workingDocsDirectory = Paths.get(componentDocsDirPath).toRealPath().getParent().toFile(); + final File componentDocsDirPath = props.getComponentDocumentationWorkingDirectory(); + final File workingDocsDirectory = componentDocsDirPath.toPath().toRealPath().getParent().toFile(); final Resource workingDocsResource = Resource.newResource(workingDocsDirectory); // load the rest documentation @@ -917,4 +866,5 @@ public class JettyServer implements NiFiServer { logger.warn("Failed to stop web server", ex); } } + } http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/pom.xml index 3fcbfe9..a79ec78 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/pom.xml @@ -31,9 +31,5 @@ <groupId>org.apache.nifi</groupId> <artifactId>nifi-framework-api</artifactId> </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-user-actions</artifactId> - </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/src/main/java/org/apache/nifi/ui/extension/UiExtensionMapping.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/src/main/java/org/apache/nifi/ui/extension/UiExtensionMapping.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/src/main/java/org/apache/nifi/ui/extension/UiExtensionMapping.java index 73e39f8..4202830 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/src/main/java/org/apache/nifi/ui/extension/UiExtensionMapping.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-ui-extension/src/main/java/org/apache/nifi/ui/extension/UiExtensionMapping.java @@ -30,21 +30,45 @@ public class UiExtensionMapping { this.uiExtensions = uiExtensions; } + private String getBundleSpecificKey(final String type, final String bundleGroup, final String bundleArtifact, final String bundleVersion) { + return type + ":" + bundleGroup + ":" + bundleArtifact + ":" + bundleVersion; + } + /** * @param type type + * @param bundleGroup bundle group + * @param bundleArtifact bundle artifact + * @param bundleVersion bundle version * @return whether there are any UI extensions for the specified component * type */ - public boolean hasUiExtension(final String type) { + public boolean hasUiExtension(final String type, final String bundleGroup, final String bundleArtifact, final String bundleVersion) { + // if there is an extension registered with these bundle coordinates use it + final String bundleSpecificKey = getBundleSpecificKey(type, bundleGroup, bundleArtifact, bundleVersion); + if (uiExtensions.containsKey(bundleSpecificKey)) { + return true; + } + + // otherwise fall back to the component type return uiExtensions.containsKey(type); } /** * @param type type + * @param bundleGroup bundle group + * @param bundleArtifact bundle artifact + * @param bundleVersion bundle version * @return the listing of all discovered UI extensions for the specified * component type */ - public List<UiExtension> getUiExtension(final String type) { + public List<UiExtension> getUiExtension(final String type, final String bundleGroup, final String bundleArtifact, final String bundleVersion) { + // if there is an extension registered with these bundle coordinates use it + final String bundleSpecificKey = getBundleSpecificKey(type, bundleGroup, bundleArtifact, bundleVersion); + if (uiExtensions.containsKey(bundleSpecificKey)) { + return uiExtensions.get(bundleSpecificKey); + } + + // otherwise fall back to the component type return uiExtensions.get(type); } http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java index b4af48f..efb11a0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ControllerServiceAuditor.java @@ -25,6 +25,7 @@ import org.apache.nifi.action.details.ActionDetails; import org.apache.nifi.action.details.FlowChangeConfigureDetails; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; +import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ConfiguredComponent; import org.apache.nifi.controller.ProcessorNode; @@ -60,6 +61,7 @@ public class ControllerServiceAuditor extends NiFiAuditor { private static final String COMMENTS = "Comments"; private static final String NAME = "Name"; private static final String ANNOTATION_DATA = "Annotation Data"; + private static final String EXTENSION_VERSION = "Extension Version"; /** * Audits the creation of controller service via createControllerService(). @@ -414,6 +416,10 @@ public class ControllerServiceAuditor extends NiFiAuditor { if (controllerServiceDTO.getAnnotationData() != null) { values.put(ANNOTATION_DATA, controllerService.getAnnotationData()); } + if (controllerServiceDTO.getBundle() != null) { + final BundleCoordinate bundle = controllerService.getBundleCoordinate(); + values.put(EXTENSION_VERSION, formatExtensionVersion(controllerService.getComponentType(), bundle)); + } if (controllerServiceDTO.getProperties() != null) { // for each property specified, extract its configured value Map<String, String> properties = controllerServiceDTO.getProperties(); http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/NiFiAuditor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/NiFiAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/NiFiAuditor.java index 14c4b64..8dd7c85 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/NiFiAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/NiFiAuditor.java @@ -16,19 +16,20 @@ */ package org.apache.nifi.audit; -import java.util.ArrayList; -import java.util.Collection; - import org.apache.commons.lang3.StringUtils; import org.apache.nifi.action.Action; import org.apache.nifi.action.details.FlowChangeMoveDetails; import org.apache.nifi.action.details.MoveDetails; import org.apache.nifi.admin.service.AuditService; +import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.web.NiFiServiceFacade; import org.apache.nifi.web.dao.ProcessGroupDAO; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.Collection; + /** * A NiFi audit service. */ @@ -91,6 +92,24 @@ public abstract class NiFiAuditor { return moveDetails; } + protected String formatExtensionVersion(final String type, final BundleCoordinate bundle) { + final String formattedType; + if (BundleCoordinate.DEFAULT_VERSION.equals(bundle.getVersion())) { + formattedType = type; + } else { + formattedType = type + " " + bundle.getVersion(); + } + + final String formattedBundle; + if (BundleCoordinate.DEFAULT_GROUP.equals(bundle.getGroup())) { + formattedBundle = bundle.getId(); + } else { + formattedBundle = bundle.getGroup() + " - " + bundle.getId(); + } + + return String.format("%s from %s", formattedType, formattedBundle); + } + /* setters / getters */ public void setAuditService(AuditService auditService) { this.auditService = auditService; http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java index 915ee79..c40c087 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ProcessorAuditor.java @@ -26,6 +26,7 @@ import org.apache.nifi.action.details.ActionDetails; import org.apache.nifi.action.details.FlowChangeConfigureDetails; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; +import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ScheduledState; @@ -69,6 +70,7 @@ public class ProcessorAuditor extends NiFiAuditor { private static final String SCHEDULING_PERIOD = "Run Schedule"; private static final String SCHEDULING_STRATEGY = "Scheduling Strategy"; private static final String EXECUTION_NODE = "Execution Node"; + private static final String EXTENSION_VERSION = "Extension Version"; /** * Audits the creation of processors via createProcessor(). @@ -315,6 +317,10 @@ public class ProcessorAuditor extends NiFiAuditor { if (processorDTO.getName() != null) { values.put(NAME, processor.getName()); } + if (processorDTO.getBundle() != null) { + final BundleCoordinate bundle = processor.getBundleCoordinate(); + values.put(EXTENSION_VERSION, formatExtensionVersion(processor.getComponentType(), bundle)); + } if (processorDTO.getConfig() != null) { ProcessorConfigDTO newConfig = processorDTO.getConfig(); if (newConfig.getConcurrentlySchedulableTaskCount() != null) { http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java index e198d5e..c8e2682 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ReportingTaskAuditor.java @@ -25,6 +25,7 @@ import org.apache.nifi.action.details.ActionDetails; import org.apache.nifi.action.details.FlowChangeConfigureDetails; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; +import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.ScheduledState; @@ -53,6 +54,7 @@ public class ReportingTaskAuditor extends NiFiAuditor { private static final String NAME = "Name"; private static final String ANNOTATION_DATA = "Annotation Data"; + private static final String EXTENSION_VERSION = "Extension Version"; /** * Audits the creation of reporting task via createReportingTask(). @@ -306,6 +308,10 @@ public class ReportingTaskAuditor extends NiFiAuditor { if (reportingTaskDTO.getAnnotationData() != null) { values.put(ANNOTATION_DATA, reportingTask.getAnnotationData()); } + if (reportingTaskDTO.getBundle() != null) { + final BundleCoordinate bundle = reportingTask.getBundleCoordinate(); + values.put(EXTENSION_VERSION, formatExtensionVersion(reportingTask.getComponentType(), bundle)); + } if (reportingTaskDTO.getProperties() != null) { // for each property specified, extract its configured value Map<String, String> properties = reportingTaskDTO.getProperties(); http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java index ce7313e..af3d19b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/SnippetAuditor.java @@ -94,7 +94,7 @@ public class SnippetAuditor extends NiFiAuditor { * @throws Throwable ex */ @Around("within(org.apache.nifi.web.dao.SnippetDAO+) && " - + "execution(org.apache.nifi.web.api.dto.FlowSnippetDTO copySnippet(java.lang.String, java.lang.String, java.lang.Double, java.lang.Double))") + + "execution(org.apache.nifi.web.api.dto.FlowSnippetDTO copySnippet(java.lang.String, java.lang.String, java.lang.Double, java.lang.Double, java.lang.String))") public FlowSnippetDTO copySnippetAdvice(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { // perform the underlying operation FlowSnippetDTO snippet = (FlowSnippetDTO) proceedingJoinPoint.proceed(); @@ -110,7 +110,8 @@ public class SnippetAuditor extends NiFiAuditor { * @throws Throwable ex */ @Around("within(org.apache.nifi.web.dao.TemplateDAO+) && " - + "execution(org.apache.nifi.web.api.dto.FlowSnippetDTO instantiateTemplate(java.lang.String, java.lang.Double, java.lang.Double, java.lang.String))") + + "execution(org.apache.nifi.web.api.dto.FlowSnippetDTO instantiateTemplate(" + + "java.lang.String, java.lang.Double, java.lang.Double, java.lang.String, org.apache.nifi.web.api.dto.FlowSnippetDTO, java.lang.String))") public FlowSnippetDTO instantiateTemplateAdvice(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { // perform the underlying operation FlowSnippetDTO snippet = (FlowSnippetDTO) proceedingJoinPoint.proceed(); http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java index b1d0844..db5bd17 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java @@ -17,6 +17,8 @@ package org.apache.nifi.authorization; import org.apache.nifi.authorization.resource.Authorizable; +import org.apache.nifi.web.api.dto.BundleDTO; +import org.apache.nifi.web.api.dto.FlowSnippetDTO; public interface AuthorizableLookup { @@ -43,7 +45,7 @@ public interface AuthorizableLookup { * @param type processor type * @return authorizable */ - ConfigurableComponentAuthorizable getProcessorByType(String type); + ConfigurableComponentAuthorizable getProcessorByType(String type, BundleDTO bundle); /** * Get the authorizable for querying Provenance. @@ -144,10 +146,11 @@ public interface AuthorizableLookup { * controller service. The intent of this method is to provide access to the PropertyDescriptors * prior to the component being created. * - * @param type processor type + * @param type controller service type + * @param bundle bundle * @return authorizable */ - ConfigurableComponentAuthorizable getControllerServiceByType(String type); + ConfigurableComponentAuthorizable getControllerServiceByType(String type, BundleDTO bundle); /** * Get the authorizable referencing component. @@ -171,10 +174,11 @@ public interface AuthorizableLookup { * reporting task. The intent of this method is to provide access to the PropertyDescriptors * prior to the component being created. * - * @param type processor type + * @param type reporting task type + * @param bundle bundle * @return authorizable */ - ConfigurableComponentAuthorizable getReportingTaskByType(String type); + ConfigurableComponentAuthorizable getReportingTaskByType(String type, BundleDTO bundle); /** * Get the authorizable Template. @@ -182,7 +186,15 @@ public interface AuthorizableLookup { * @param id template id * @return authorizable */ - TemplateAuthorizable getTemplate(String id); + Authorizable getTemplate(String id); + + /** + * Get the authorizable Template contents. + * + * @param snippet the template contents + * @return authorizable + */ + TemplateContentsAuthorizable getTemplateContents(FlowSnippetDTO snippet); /** * Get the authorizable connectable. Note this does not include RemoteGroupPorts. http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConfigurableComponentAuthorizable.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConfigurableComponentAuthorizable.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConfigurableComponentAuthorizable.java index f352211..aaf1d3d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConfigurableComponentAuthorizable.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConfigurableComponentAuthorizable.java @@ -61,4 +61,9 @@ public interface ConfigurableComponentAuthorizable { * @return value */ String getValue(PropertyDescriptor propertyDescriptor); + + /** + * Cleans up any resources resulting from the creation of these temporary components. + */ + void cleanUpResources(); } http://git-wip-us.apache.org/repos/asf/nifi/blob/d90cf846/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java index 28092cd..ebc98de 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java @@ -26,6 +26,7 @@ import org.apache.nifi.authorization.resource.ResourceType; import org.apache.nifi.authorization.resource.RestrictedComponentsAuthorizable; import org.apache.nifi.authorization.resource.TenantAuthorizable; import org.apache.nifi.authorization.user.NiFiUser; +import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; @@ -34,15 +35,16 @@ import org.apache.nifi.controller.ConfiguredComponent; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.Snippet; -import org.apache.nifi.controller.Template; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceReference; import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.remote.PortAuthorizationResult; import org.apache.nifi.remote.RootGroupPort; +import org.apache.nifi.util.BundleUtils; import org.apache.nifi.web.ResourceNotFoundException; +import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; -import org.apache.nifi.web.api.dto.TemplateDTO; import org.apache.nifi.web.controller.ControllerFacade; import org.apache.nifi.web.dao.AccessPolicyDAO; import org.apache.nifi.web.dao.ConnectionDAO; @@ -146,9 +148,9 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } @Override - public ConfigurableComponentAuthorizable getProcessorByType(String type) { + public ConfigurableComponentAuthorizable getProcessorByType(String type, BundleDTO bundle) { try { - final ProcessorNode processorNode = controllerFacade.createTemporaryProcessor(type); + final ProcessorNode processorNode = controllerFacade.createTemporaryProcessor(type, bundle); return new ProcessorConfigurableComponentAuthorizable(processorNode); } catch (final Exception e) { throw new AccessDeniedException("Unable to create processor to verify if it references any Controller Services."); @@ -255,9 +257,9 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } @Override - public ConfigurableComponentAuthorizable getControllerServiceByType(String type) { + public ConfigurableComponentAuthorizable getControllerServiceByType(String type, BundleDTO bundle) { try { - final ControllerServiceNode controllerService = controllerFacade.createTemporaryControllerService(type); + final ControllerServiceNode controllerService = controllerFacade.createTemporaryControllerService(type, bundle); return new ControllerServiceConfigurableComponentAuthorizable(controllerService); } catch (final Exception e) { throw new AccessDeniedException("Unable to create controller service to verify if it references any Controller Services."); @@ -314,9 +316,9 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } @Override - public ConfigurableComponentAuthorizable getReportingTaskByType(String type) { + public ConfigurableComponentAuthorizable getReportingTaskByType(String type, BundleDTO bundle) { try { - final ReportingTaskNode reportingTask = controllerFacade.createTemporaryReportingTask(type); + final ReportingTaskNode reportingTask = controllerFacade.createTemporaryReportingTask(type, bundle); return new ReportingTaskConfigurableComponentAuthorizable(reportingTask); } catch (final Exception e) { throw new AccessDeniedException("Unable to create reporting to verify if it references any Controller Services."); @@ -510,7 +512,7 @@ class StandardAuthorizableLookup implements AuthorizableLookup { authorizable = getReportingTask(componentId).getAuthorizable(); break; case Template: - authorizable = getTemplate(componentId).getAuthorizable(); + authorizable = getTemplate(componentId); break; } @@ -622,11 +624,25 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } if (snippet.getProcessors() != null) { - processors.addAll(snippet.getProcessors().stream().map(processor -> getProcessorByType(processor.getType())).collect(Collectors.toSet())); + snippet.getProcessors().forEach(processor -> { + try { + final BundleCoordinate bundle = BundleUtils.getCompatibleBundle(processor.getType(), processor.getBundle()); + processors.add(getProcessorByType(processor.getType(), new BundleDTO(bundle.getGroup(), bundle.getId(), bundle.getVersion()))); + } catch (final IllegalStateException e) { + // no compatible bundles... no additional auth checks necessary... if created, will be ghosted + } + }); } if (snippet.getControllerServices() != null) { - controllerServices.addAll(snippet.getControllerServices().stream().map(controllerService -> getControllerServiceByType(controllerService.getType())).collect(Collectors.toSet())); + snippet.getControllerServices().forEach(controllerService -> { + try { + final BundleCoordinate bundle = BundleUtils.getCompatibleBundle(controllerService.getType(), controllerService.getBundle()); + controllerServices.add(getControllerServiceByType(controllerService.getType(), new BundleDTO(bundle.getGroup(), bundle.getId(), bundle.getVersion()))); + } catch (final IllegalStateException e) { + // no compatible bundles... no additional auth checks necessary... if created, will be ghosted + } + }); } if (snippet.getProcessGroups() != null) { @@ -635,23 +651,20 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } @Override - public TemplateAuthorizable getTemplate(final String id) { - final Template template = templateDAO.getTemplate(id); - final TemplateDTO contents = template.getDetails(); + public Authorizable getTemplate(String id) { + return templateDAO.getTemplate(id); + } + @Override + public TemplateContentsAuthorizable getTemplateContents(final FlowSnippetDTO snippet) { // templates are immutable so we can pre-compute all encapsulated processors and controller services final Set<ConfigurableComponentAuthorizable> processors = new HashSet<>(); final Set<ConfigurableComponentAuthorizable> controllerServices = new HashSet<>(); // find all processors and controller services - createTemporaryProcessorsAndControllerServices(contents.getSnippet(), processors, controllerServices); - - return new TemplateAuthorizable() { - @Override - public Authorizable getAuthorizable() { - return template; - } + createTemporaryProcessorsAndControllerServices(snippet, processors, controllerServices); + return new TemplateContentsAuthorizable() { @Override public Set<ConfigurableComponentAuthorizable> getEncapsulatedProcessors() { return processors; @@ -720,6 +733,11 @@ class StandardAuthorizableLookup implements AuthorizableLookup { public List<PropertyDescriptor> getPropertyDescriptors() { return processorNode.getPropertyDescriptors(); } + + @Override + public void cleanUpResources() { + ExtensionManager.removeInstanceClassLoaderIfExists(processorNode.getIdentifier()); + } } /** @@ -756,6 +774,11 @@ class StandardAuthorizableLookup implements AuthorizableLookup { public List<PropertyDescriptor> getPropertyDescriptors() { return controllerServiceNode.getControllerServiceImplementation().getPropertyDescriptors(); } + + @Override + public void cleanUpResources() { + ExtensionManager.removeInstanceClassLoaderIfExists(controllerServiceNode.getIdentifier()); + } } /** @@ -792,6 +815,11 @@ class StandardAuthorizableLookup implements AuthorizableLookup { public List<PropertyDescriptor> getPropertyDescriptors() { return reportingTaskNode.getReportingTask().getPropertyDescriptors(); } + + @Override + public void cleanUpResources() { + ExtensionManager.removeInstanceClassLoaderIfExists(reportingTaskNode.getIdentifier()); + } } private static class StandardProcessGroupAuthorizable implements ProcessGroupAuthorizable {
