http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/maven-plugins/nar-maven-plugin/src/main/java/nifi/NarMojo.java ---------------------------------------------------------------------- diff --git a/maven-plugins/nar-maven-plugin/src/main/java/nifi/NarMojo.java b/maven-plugins/nar-maven-plugin/src/main/java/nifi/NarMojo.java new file mode 100644 index 0000000..9b70ec0 --- /dev/null +++ b/maven-plugins/nar-maven-plugin/src/main/java/nifi/NarMojo.java @@ -0,0 +1,613 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nifi; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.maven.archiver.MavenArchiveConfiguration; +import org.apache.maven.archiver.MavenArchiver; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.DependencyResolutionRequiredException; +import org.apache.maven.artifact.factory.ArtifactFactory; +import org.apache.maven.artifact.installer.ArtifactInstaller; +import org.apache.maven.artifact.metadata.ArtifactMetadataSource; +import org.apache.maven.artifact.repository.ArtifactRepository; +import org.apache.maven.artifact.repository.ArtifactRepositoryFactory; +import org.apache.maven.artifact.resolver.ArtifactCollector; +import org.apache.maven.artifact.resolver.ArtifactNotFoundException; +import org.apache.maven.artifact.resolver.ArtifactResolutionException; +import org.apache.maven.artifact.resolver.ArtifactResolver; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugin.MojoFailureException; +import org.apache.maven.plugin.dependency.utils.DependencyStatusSets; +import org.apache.maven.plugin.dependency.utils.DependencyUtil; +import org.apache.maven.plugin.dependency.utils.filters.DestFileFilter; +import org.apache.maven.plugin.dependency.utils.resolvers.ArtifactsResolver; +import org.apache.maven.plugin.dependency.utils.resolvers.DefaultArtifactsResolver; +import org.apache.maven.plugin.dependency.utils.translators.ArtifactTranslator; +import org.apache.maven.plugin.dependency.utils.translators.ClassifierTypeTranslator; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugins.annotations.Component; +import org.apache.maven.project.MavenProjectHelper; +import org.apache.maven.shared.artifact.filter.collection.ArtifactFilterException; +import org.apache.maven.shared.artifact.filter.collection.ArtifactIdFilter; +import org.apache.maven.shared.artifact.filter.collection.ArtifactsFilter; +import org.apache.maven.shared.artifact.filter.collection.ClassifierFilter; +import org.apache.maven.shared.artifact.filter.collection.FilterArtifacts; +import org.apache.maven.shared.artifact.filter.collection.GroupIdFilter; +import org.apache.maven.shared.artifact.filter.collection.ScopeFilter; +import org.apache.maven.shared.artifact.filter.collection.ProjectTransitivityFilter; +import org.apache.maven.shared.artifact.filter.collection.TypeFilter; +import org.codehaus.plexus.archiver.ArchiverException; +import org.codehaus.plexus.archiver.jar.JarArchiver; +import org.codehaus.plexus.archiver.jar.ManifestException; +import org.codehaus.plexus.archiver.manager.ArchiverManager; +import org.codehaus.plexus.util.FileUtils; +import org.codehaus.plexus.util.StringUtils; + +/** + * Packages the current project as an Apache NiFi Archive (NAR). + * + * The following code is derived from maven-dependencies-plugin and + * maven-jar-plugin. The functionality of CopyDependenciesMojo and JarMojo was + * simplified to the use case of NarMojo. + * + */ +@Mojo(name = "nar", defaultPhase = LifecyclePhase.PACKAGE, threadSafe = false, requiresDependencyResolution = ResolutionScope.RUNTIME) +public class NarMojo extends AbstractMojo { + + private static final String[] DEFAULT_EXCLUDES = new String[]{"**/package.html"}; + private static final String[] DEFAULT_INCLUDES = new String[]{"**/**"}; + + /** + * POM + * + */ + @Parameter(defaultValue = "${project}", readonly = true, required = true) + protected MavenProject project; + + @Parameter(defaultValue = "${session}", readonly = true, required = true) + protected MavenSession session; + + /** + * List of files to include. Specified as fileset patterns. + */ + @Parameter(property = "includes") + protected String[] includes; + /** + * List of files to exclude. Specified as fileset patterns. + */ + @Parameter(property = "excludes") + protected String[] excludes; + /** + * Name of the generated NAR. + * + */ + @Parameter(alias = "narName", property = "nar.finalName", defaultValue = "${project.build.finalName}", required = true) + protected String finalName; + + /** + * The Jar archiver. + * + * \@\component role="org.codehaus.plexus.archiver.Archiver" roleHint="jar" + */ + @Component(role = org.codehaus.plexus.archiver.Archiver.class, hint = "jar") + private JarArchiver jarArchiver; + /** + * The archive configuration to use. + * + * See <a + * href="http://maven.apache.org/shared/maven-archiver/index.html";>the + * documentation for Maven Archiver</a>. + * + */ + @Parameter(property = "archive") + protected final MavenArchiveConfiguration archive = new MavenArchiveConfiguration(); + /** + * Path to the default MANIFEST file to use. It will be used if + * <code>useDefaultManifestFile</code> is set to <code>true</code>. + * + */ + @Parameter(property = "defaultManifestFiles", defaultValue = "${project.build.outputDirectory}/META-INF/MANIFEST.MF", readonly = true, required = true) + protected File defaultManifestFile; + + /** + * Set this to <code>true</code> to enable the use of the + * <code>defaultManifestFile</code>. + * + * @since 2.2 + */ + @Parameter(property = "nar.useDefaultManifestFile", defaultValue = "false") + protected boolean useDefaultManifestFile; + + @Component + protected MavenProjectHelper projectHelper; + + /** + * Whether creating the archive should be forced. + * + */ + @Parameter(property = "nar.forceCreation", defaultValue = "false") + protected boolean forceCreation; + + /** + * Classifier to add to the artifact generated. If given, the artifact will + * be an attachment instead. + * + */ + @Parameter(property = "classifier") + protected String classifier; + + @Component + protected ArtifactInstaller installer; + + @Component + protected ArtifactRepositoryFactory repositoryFactory; + + /** + * This only applies if the classifier parameter is used. + * + */ + @Parameter(property = "mdep.failOnMissingClassifierArtifact", defaultValue = "true", required = false) + protected boolean failOnMissingClassifierArtifact = true; + + /** + * Comma Separated list of Types to include. Empty String indicates include + * everything (default). + * + */ + @Parameter(property = "includeTypes", required = false) + protected String includeTypes; + + /** + * Comma Separated list of Types to exclude. Empty String indicates don't + * exclude anything (default). + * + */ + @Parameter(property = "excludeTypes", required = false) + protected String excludeTypes; + + /** + * Scope to include. An Empty string indicates all scopes (default). + * + */ + @Parameter(property = "includeScope", required = false) + protected String includeScope; + + /** + * Scope to exclude. An Empty string indicates no scopes (default). + * + */ + @Parameter(property = "excludeScope", required = false) + protected String excludeScope; + + /** + * Comma Separated list of Classifiers to include. Empty String indicates + * include everything (default). + * + */ + @Parameter(property = "includeClassifiers", required = false) + protected String includeClassifiers; + + /** + * Comma Separated list of Classifiers to exclude. Empty String indicates + * don't exclude anything (default). + * + */ + @Parameter(property = "excludeClassifiers", required = false) + protected String excludeClassifiers; + + /** + * Specify classifier to look for. Example: sources + * + */ + @Parameter(property = "classifier", required = false) + protected String copyDepClassifier; + + /** + * Specify type to look for when constructing artifact based on classifier. + * Example: java-source,jar,war, nar + * + */ + @Parameter(property = "type", required = false, defaultValue = "nar") + protected String type; + + /** + * Comma separated list of Artifact names too exclude. + * + */ + @Parameter(property = "excludeArtifacts", required = false) + protected String excludeArtifactIds; + + /** + * Comma separated list of Artifact names to include. + * + */ + @Parameter(property = "includeArtifacts", required = false) + protected String includeArtifactIds; + + /** + * Comma separated list of GroupId Names to exclude. + * + */ + @Parameter(property = "excludeArtifacts", required = false) + protected String excludeGroupIds; + + /** + * Comma separated list of GroupIds to include. + * + */ + @Parameter(property = "includeGroupIds", required = false) + protected String includeGroupIds; + + /** + * Directory to store flag files + * + */ + @Parameter(property = "markersDirectory", required = false, defaultValue = "${project.build.directory}/dependency-maven-plugin-markers") + protected File markersDirectory; + + /** + * Overwrite release artifacts + * + */ + @Parameter(property = "overWriteReleases", required = false) + protected boolean overWriteReleases; + + /** + * Overwrite snapshot artifacts + * + */ + @Parameter(property = "overWriteSnapshots", required = false) + protected boolean overWriteSnapshots; + + /** + * Overwrite artifacts that don't exist or are older than the source. + * + */ + @Parameter(property = "overWriteIfNewer", required = false, defaultValue = "true") + protected boolean overWriteIfNewer; + + @Parameter( property = "projectBuildDirectory", required = false, defaultValue = "${project.build.directory}") + protected File projectBuildDirectory; + + /** + * Used to look up Artifacts in the remote repository. + */ + @Component + protected ArtifactFactory factory; + + /** + * Used to look up Artifacts in the remote repository. + * + */ + @Component + protected ArtifactResolver resolver; + + /** + * Artifact collector, needed to resolve dependencies. + * + */ + @Component(role = org.apache.maven.artifact.resolver.ArtifactCollector.class) + protected ArtifactCollector artifactCollector; + + @Component(role = org.apache.maven.artifact.metadata.ArtifactMetadataSource.class) + protected ArtifactMetadataSource artifactMetadataSource; + + /** + * Location of the local repository. + * + */ + @Parameter(property = "localRepository", required = true, readonly = true) + protected ArtifactRepository local; + + /** + * List of Remote Repositories used by the resolver + * + */ + @Parameter(property = "project.remoteArtifactRepositories", required = true, readonly = true) + protected List remoteRepos; + + /** + * To look up Archiver/UnArchiver implementations + * + */ + @Component + protected ArchiverManager archiverManager; + + /** + * Contains the full list of projects in the reactor. + * + */ + @Parameter(property = "reactorProjects", required = true, readonly = true) + protected List reactorProjects; + + /** + * If the plugin should be silent. + * + */ + @Parameter(property = "silent", required = false, defaultValue = "false") + public boolean silent; + + /** + * Output absolute filename for resolved artifacts + * + */ + @Parameter(property = "outputAbsoluteArtifactFilename", defaultValue = "false", required = false) + protected boolean outputAbsoluteArtifactFilename; + + @Override + public void execute() throws MojoExecutionException, MojoFailureException { + copyDependencies(); + makeNar(); + } + + private void copyDependencies() throws MojoExecutionException { + DependencyStatusSets dss = getDependencySets(this.failOnMissingClassifierArtifact); + Set artifacts = dss.getResolvedDependencies(); + + for (Object artifactObj : artifacts) { + copyArtifact((Artifact) artifactObj); + } + + artifacts = dss.getSkippedDependencies(); + for (Object artifactOjb : artifacts) { + Artifact artifact = (Artifact) artifactOjb; + getLog().info(artifact.getFile().getName() + " already exists in destination."); + } + } + + protected void copyArtifact(Artifact artifact) throws MojoExecutionException { + String destFileName = DependencyUtil.getFormattedFileName(artifact, false); + final File destDir = DependencyUtil.getFormattedOutputDirectory(false, false, false, false, false, getDependenciesDirectory(), artifact); + final File destFile = new File(destDir, destFileName); + copyFile(artifact.getFile(), destFile); + } + + protected Artifact getResolvedPomArtifact(Artifact artifact) { + Artifact pomArtifact = this.factory.createArtifact(artifact.getGroupId(), artifact.getArtifactId(), artifact.getVersion(), "", "pom"); + // Resolve the pom artifact using repos + try { + this.resolver.resolve(pomArtifact, this.remoteRepos, this.local); + } catch (ArtifactResolutionException | ArtifactNotFoundException e) { + getLog().info(e.getMessage()); + } + return pomArtifact; + } + + protected ArtifactsFilter getMarkedArtifactFilter() { + return new DestFileFilter(this.overWriteReleases, this.overWriteSnapshots, this.overWriteIfNewer, false, false, false, false, false, getDependenciesDirectory()); + } + + protected DependencyStatusSets getDependencySets(boolean stopOnFailure) throws MojoExecutionException { + // add filters in well known order, least specific to most specific + FilterArtifacts filter = new FilterArtifacts(); + + filter.addFilter(new ProjectTransitivityFilter(project.getDependencyArtifacts(), false)); + filter.addFilter(new ScopeFilter(this.includeScope, this.excludeScope)); + filter.addFilter(new TypeFilter(this.includeTypes, this.excludeTypes)); + filter.addFilter(new ClassifierFilter(this.includeClassifiers, this.excludeClassifiers)); + filter.addFilter(new GroupIdFilter(this.includeGroupIds, this.excludeGroupIds)); + filter.addFilter(new ArtifactIdFilter(this.includeArtifactIds, this.excludeArtifactIds)); + + // explicitly filter our nar dependencies + filter.addFilter(new TypeFilter("", "nar")); + + // start with all artifacts. + Set artifacts = project.getArtifacts(); + + // perform filtering + try { + artifacts = filter.filter(artifacts); + } catch (ArtifactFilterException e) { + throw new MojoExecutionException(e.getMessage(), e); + } + + // transform artifacts if classifier is set + final DependencyStatusSets status; + if (StringUtils.isNotEmpty(copyDepClassifier)) { + status = getClassifierTranslatedDependencies(artifacts, stopOnFailure); + } else { + status = filterMarkedDependencies(artifacts); + } + + return status; + } + + protected DependencyStatusSets getClassifierTranslatedDependencies(Set artifacts, boolean stopOnFailure) throws MojoExecutionException { + Set unResolvedArtifacts = new HashSet(); + Set resolvedArtifacts = artifacts; + DependencyStatusSets status = new DependencyStatusSets(); + + // possibly translate artifacts into a new set of artifacts based on the + // classifier and type + // if this did something, we need to resolve the new artifacts + if (StringUtils.isNotEmpty(copyDepClassifier)) { + ArtifactTranslator translator = new ClassifierTypeTranslator(this.copyDepClassifier, this.type, this.factory); + artifacts = translator.translate(artifacts, getLog()); + + status = filterMarkedDependencies(artifacts); + + // the unskipped artifacts are in the resolved set. + artifacts = status.getResolvedDependencies(); + + // resolve the rest of the artifacts + ArtifactsResolver artifactsResolver = new DefaultArtifactsResolver(this.resolver, this.local, + this.remoteRepos, stopOnFailure); + resolvedArtifacts = artifactsResolver.resolve(artifacts, getLog()); + + // calculate the artifacts not resolved. + unResolvedArtifacts.addAll(artifacts); + unResolvedArtifacts.removeAll(resolvedArtifacts); + } + + // return a bean of all 3 sets. + status.setResolvedDependencies(resolvedArtifacts); + status.setUnResolvedDependencies(unResolvedArtifacts); + + return status; + } + + protected DependencyStatusSets filterMarkedDependencies(Set artifacts) throws MojoExecutionException { + // remove files that have markers already + FilterArtifacts filter = new FilterArtifacts(); + filter.clearFilters(); + filter.addFilter(getMarkedArtifactFilter()); + + Set unMarkedArtifacts; + try { + unMarkedArtifacts = filter.filter(artifacts); + } catch (ArtifactFilterException e) { + throw new MojoExecutionException(e.getMessage(), e); + } + + // calculate the skipped artifacts + Set skippedArtifacts = new HashSet(); + skippedArtifacts.addAll(artifacts); + skippedArtifacts.removeAll(unMarkedArtifacts); + + return new DependencyStatusSets(unMarkedArtifacts, null, skippedArtifacts); + } + + protected void copyFile(File artifact, File destFile) throws MojoExecutionException { + try { + getLog().info("Copying " + (this.outputAbsoluteArtifactFilename ? artifact.getAbsolutePath() : artifact.getName()) + " to " + destFile); + FileUtils.copyFile(artifact, destFile); + } catch (Exception e) { + throw new MojoExecutionException("Error copying artifact from " + artifact + " to " + destFile, e); + } + } + + private File getClassesDirectory() { + final File outputDirectory = projectBuildDirectory; + return new File(outputDirectory, "classes"); + } + + private File getDependenciesDirectory() { + return new File(getClassesDirectory(), "META-INF/bundled-dependencies"); + } + + private void makeNar() throws MojoExecutionException { + File narFile = createArchive(); + + if (classifier != null) { + projectHelper.attachArtifact(project, "nar", classifier, narFile); + } else { + project.getArtifact().setFile(narFile); + } + } + + public File createArchive() throws MojoExecutionException { + final File outputDirectory = projectBuildDirectory; + File narFile = getNarFile(outputDirectory, finalName, classifier); + MavenArchiver archiver = new MavenArchiver(); + archiver.setArchiver(jarArchiver); + archiver.setOutputFile(narFile); + archive.setForced(forceCreation); + + try { + File contentDirectory = getClassesDirectory(); + if (!contentDirectory.exists()) { + getLog().warn("NAR will be empty - no content was marked for inclusion!"); + } else { + archiver.getArchiver().addDirectory(contentDirectory, getIncludes(), getExcludes()); + } + + File existingManifest = defaultManifestFile; + if (useDefaultManifestFile && existingManifest.exists() && archive.getManifestFile() == null) { + getLog().info("Adding existing MANIFEST to archive. Found under: " + existingManifest.getPath()); + archive.setManifestFile(existingManifest); + } + + // automatically add the artifact id to the manifest + archive.addManifestEntry("Nar-Id", project.getArtifactId()); + + // look for a nar dependency + String narDependency = getNarDependency(); + if (narDependency != null) { + archive.addManifestEntry("Nar-Dependency-Id", narDependency); + } + + archiver.createArchive(session, project, archive); + return narFile; + } catch (ArchiverException | MojoExecutionException | ManifestException | IOException | DependencyResolutionRequiredException e) { + throw new MojoExecutionException("Error assembling NAR", e); + } + } + + private String[] getIncludes() { + if (includes != null && includes.length > 0) { + return includes; + } + return DEFAULT_INCLUDES; + } + + private String[] getExcludes() { + if (excludes != null && excludes.length > 0) { + return excludes; + } + return DEFAULT_EXCLUDES; + } + + protected File getNarFile(File basedir, String finalName, String classifier) { + if (classifier == null) { + classifier = ""; + } else if (classifier.trim().length() > 0 && !classifier.startsWith("-")) { + classifier = "-" + classifier; + } + + return new File(basedir, finalName + classifier + ".nar"); + } + + private String getNarDependency() throws MojoExecutionException { + String narDependency = null; + + // get nar dependencies + FilterArtifacts filter = new FilterArtifacts(); + filter.addFilter(new TypeFilter("nar", "")); + + // start with all artifacts. + Set artifacts = project.getArtifacts(); + + // perform filtering + try { + artifacts = filter.filter(artifacts); + } catch (ArtifactFilterException e) { + throw new MojoExecutionException(e.getMessage(), e); + } + + // ensure there is a single nar dependency + if (artifacts.size() > 1) { + throw new MojoExecutionException("Each NAR represents a ClassLoader. A NAR dependency allows that NAR's ClassLoader to be " + + "used as the parent of this NAR's ClassLoader. As a result, only a single NAR dependency is allowed."); + } else if (artifacts.size() == 1) { + final Artifact artifact = (Artifact) artifacts.iterator().next(); + narDependency = artifact.getArtifactId(); + } + + return narDependency; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/maven-plugins/nar-maven-plugin/src/main/resources/META-INF/plexus/components.xml ---------------------------------------------------------------------- diff --git a/maven-plugins/nar-maven-plugin/src/main/resources/META-INF/plexus/components.xml b/maven-plugins/nar-maven-plugin/src/main/resources/META-INF/plexus/components.xml new file mode 100644 index 0000000..0680d18 --- /dev/null +++ b/maven-plugins/nar-maven-plugin/src/main/resources/META-INF/plexus/components.xml @@ -0,0 +1,52 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<component-set> + <components> + <component> + <role>org.apache.maven.lifecycle.mapping.LifecycleMapping</role> + <role-hint>nar</role-hint> + <implementation>org.apache.maven.lifecycle.mapping.DefaultLifecycleMapping</implementation> + <configuration> + <lifecycles> + <lifecycle> + <id>default</id> + <phases> + <process-resources>org.apache.maven.plugins:maven-resources-plugin:resources</process-resources> + <compile>org.apache.maven.plugins:maven-compiler-plugin:compile</compile> + <process-test-resources>org.apache.maven.plugins:maven-resources-plugin:testResources</process-test-resources> + <test-compile>org.apache.maven.plugins:maven-compiler-plugin:testCompile</test-compile> + <test>org.apache.maven.plugins:maven-surefire-plugin:test</test> + <package>org.apache.nifi:nar-maven-plugin:nar</package> + <install>org.apache.maven.plugins:maven-install-plugin:install</install> + <deploy>org.apache.maven.plugins:maven-deploy-plugin:deploy</deploy> + </phases> + </lifecycle> + </lifecycles> + </configuration> + </component> + <component> + <role>org.apache.maven.artifact.handler.ArtifactHandler</role> + <role-hint>nar</role-hint> + <implementation>org.apache.maven.artifact.handler.DefaultArtifactHandler</implementation> + <configuration> + <type>nar</type> + <language>java</language> + <addedToClasspath>false</addedToClasspath> + <includesDependencies>true</includesDependencies> + </configuration> + </component> + </components> +</component-set> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/maven-plugins/pom.xml ---------------------------------------------------------------------- diff --git a/maven-plugins/pom.xml b/maven-plugins/pom.xml new file mode 100644 index 0000000..31d2bdc --- /dev/null +++ b/maven-plugins/pom.xml @@ -0,0 +1,346 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>16</version> + <relativePath/> + </parent> + <groupId>org.apache.nifi</groupId> + <artifactId>maven-plugins</artifactId> + <version>0.0.1-SNAPSHOT</version> + <packaging>pom</packaging> + <name>Apache NiFi Maven Plugins</name> + <description>Apache NiFi Maven Plugins. It is currently a part of the Apache Incubator.</description> + <url>http://nifi.incubator.apache.org/maven-site/</url> + <organization> + <name>Apache NiFi (incubating) Project</name> + <url>http://nifi.incubating.apache.org/</url> + </organization> + <licenses> + <license> + <name>Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0</url> + </license> + </licenses> + <mailingLists> + <mailingList> + <name>Dev</name> + <subscribe>[email protected]</subscribe> + <unsubscribe>[email protected]</unsubscribe> + <post>[email protected]</post> + <archive>http://mail-archives.apache.org/mod_mbox/incubator-nifi-dev</archive> + </mailingList> + <mailingList> + <name>Commits</name> + <subscribe>[email protected]</subscribe> + <unsubscribe>[email protected]</unsubscribe> + <post>[email protected]</post> + <archive>http://mail-archives.apache.org/mod_mbox/incubator-nifi-commits</archive> + </mailingList> + </mailingLists> + <prerequisites> + <maven>${maven.min-version}</maven> + </prerequisites> + <modules> + <module>nar-maven-plugin</module> + </modules> + <scm> + <connection>scm:git:git://git.apache.org/incubator-nifi.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-nifi.git</developerConnection> + <url>https://git-wip-us.apache.org/repos/asf?p=incubator-nifi.git</url> + </scm> + <issueManagement> + <system>JIRA</system> + <url>https://issues.apache.org/jira/browse/NIFI</url> + </issueManagement> + <properties> + <maven.compiler.source>1.7</maven.compiler.source> + <maven.compiler.target>1.7</maven.compiler.target> + <maven.version>3.0.5</maven.version> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <sealJars>false</sealJars> + </properties> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.2</version> + <configuration> + <fork>true</fork> + <optimize>true</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + </configuration> + </plugin> + <plugin> + <artifactId>maven-war-plugin</artifactId> + <version>2.5</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-plugin-plugin</artifactId> + <version>3.3</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.9</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-resources-plugin</artifactId> + <version>2.7</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.18</version> + <configuration> + <argLine>-Xmx1G</argLine> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>2.5.2</version> + <configuration> + <attach>false</attach> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>jaxb2-maven-plugin</artifactId> + <version>1.6</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>2.4</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-site-plugin</artifactId> + <version>3.4</version> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.3.2</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.10.1</version> + <configuration> + <failOnError>false</failOnError> + <quiet>true</quiet> + <show>private</show> + <encoding>UTF-8</encoding> + <quiet>true</quiet> + <javadocVersion>1.7</javadocVersion> + <additionalJOption>-J-Xmx512m</additionalJOption> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + <version>2.5.1</version> + <executions> + <execution> + <id>default</id> + <goals> + <goal>prepare</goal> + <goal>perform</goal> + </goals> + <configuration> + <pomFileName>platform/pom.xml</pomFileName> + <arguments>-P apache-release,check-licenses</arguments> + <autoVersionSubmodules>true</autoVersionSubmodules> + <releaseProfiles>apache-release</releaseProfiles> + <goals>deploy</goals> + <tagNameFormat>@{project.artifactId}-@{project.version}</tagNameFormat> + <pushChanges>false</pushChanges> + <localCheckout>true</localCheckout> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <configuration> + <archive> + <manifestEntries> + <Sealed>${sealJars}</Sealed> + <Implementation-Build>${mvngit.commit.id}</Implementation-Build> + </manifestEntries> + </archive> + </configuration> + </plugin> + </plugins> + </pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <executions> + <execution> + <id>enforce-maven</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireSameVersions> + <plugins> + <plugin>org.apache.maven.plugins:maven-surefire-plugin</plugin> + <plugin>org.apache.maven.plugins:maven-failsafe-plugin</plugin> + <plugin>org.apache.maven.plugins:maven-surefire-report-plugin</plugin> + </plugins> + </requireSameVersions> + <requireMavenVersion> + <version>${maven.version}</version> + </requireMavenVersion> + </rules> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-plugin-api</artifactId> + <version>2.0.11</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.9</version> + <type>maven-plugin</type> + </dependency> + <dependency> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.5</version> + </dependency> + <dependency> + <groupId>org.apache.maven.plugin-tools</groupId> + <artifactId>maven-plugin-annotations</artifactId> + <version>3.3</version> + </dependency> + </dependencies> + </dependencyManagement> + <profiles> + <profile> + <id>apache-release</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.apache.resources</groupId> + <artifactId>apache-source-release-assembly-descriptor</artifactId> + <version>1.0.4</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>source-release-assembly</id> + <goals> + <goal>single</goal> + </goals> + <phase>validate</phase> + <configuration> + <runOnlyAtExecutionRoot>true</runOnlyAtExecutionRoot> + <finalName>nifi-${project.artifactId}-${project.version}</finalName> + <descriptorRefs> + <descriptorRef>source-release-zip-tar</descriptorRef> + </descriptorRefs> + <tarLongFileFormat>gnu</tarLongFileFormat> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>rename-source-release-assembly</id> + <goals> + <goal>exec</goal> + </goals> + <phase>validate</phase> + <configuration> + <executable>mv</executable> + <workingDirectory>${project.build.directory}</workingDirectory> + <commandlineArgs>-n nifi-${project.artifactId}-${project.version}-source-release.tar.gz nifi-${project.artifactId}-${project.version}-src.tar.gz</commandlineArgs> + <successCodes> + <successCode>0</successCode> + <successCode>1</successCode> + </successCodes> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <!-- Seal jars and skip tests when the + apache-release profile is activated. --> + <id>seal-jars</id> + <properties> + <sealJars>true</sealJars> + <skipTests>true</skipTests> + </properties> + </profile> + <profile> + <!-- Automatically check for licenses. + Activate with -P check-licenses --> + <id>check-licenses</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>check</goal> + </goals> + <phase>verify</phase> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/pom.xml ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/pom.xml b/nar-bundles/execute-script-bundle/execute-script-processors/pom.xml deleted file mode 100644 index a5d3d11..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/pom.xml +++ /dev/null @@ -1,81 +0,0 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.nifi</groupId> - <artifactId>execute-script-bundle</artifactId> - <version>0.0.1-SNAPSHOT</version> - </parent> - <artifactId>execute-script-processors</artifactId> - <description>NiFi Processors to Run Scripts</description> - <name>NiFi Script Execution Processors</name> - <dependencies> - - <dependency> - <groupId>org.jruby</groupId> - <artifactId>jruby</artifactId> - <exclusions> - <exclusion> - <artifactId>jnr-netdb</artifactId> - <groupId>com.github.jnr</groupId> - </exclusion> - <exclusion> - <artifactId>jnr-posix</artifactId> - <groupId>com.github.jnr</groupId> - </exclusion> - <exclusion> - <artifactId>jffi</artifactId> - <groupId>com.github.jnr</groupId> - </exclusion> - <exclusion> - <artifactId>nailgun-server</artifactId> - <groupId>com.martiansoftware</groupId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.python</groupId> - <artifactId>jython-standalone</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-mock</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-processor-utils</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-core-flowfile-attributes</artifactId> - </dependency> - <dependency> - <groupId>org.apache.nifi</groupId> - <artifactId>nifi-stream-utils</artifactId> - </dependency> - </dependencies> -</project> - - http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java b/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java deleted file mode 100644 index 9058cf4..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java +++ /dev/null @@ -1,566 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.processors.script; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import javax.script.ScriptException; - -import org.apache.nifi.components.PropertyDescriptor; -import org.apache.nifi.components.ValidationContext; -import org.apache.nifi.components.ValidationResult; -import org.apache.nifi.components.Validator; -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.io.BufferedInputStream; -import org.apache.nifi.io.BufferedOutputStream; -import org.apache.nifi.processor.AbstractProcessor; -import org.apache.nifi.processor.ProcessContext; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.ProcessorInitializationContext; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.annotation.CapabilityDescription; -import org.apache.nifi.processor.annotation.EventDriven; -import org.apache.nifi.processor.annotation.Tags; -import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.io.StreamCallback; -import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.scripting.ConverterScript; -import org.apache.nifi.scripting.ReaderScript; -import org.apache.nifi.scripting.Script; -import org.apache.nifi.scripting.ScriptFactory; -import org.apache.nifi.scripting.WriterScript; - -/** - * <!-- Processor Documentation ================================================== --> - * <h2>Description:</h2> - * <p> - * This processor provides the capability to execute scripts in various - * scripting languages, and passes into the scripts the input stream and output - * stream(s) representing an incoming flow file and any created flow files. The - * processor is designed to be thread safe, so multiple concurrent tasks may - * execute against a single script. The processor provides a framework which - * enables script writers to implement 3 different types of scripts: - * <ul> - * ReaderScript - which enables stream-based reading of a FlowFile's - * content</br> WriterScript - which enables stream-based reading and - * writing/modifying of a FlowFile's content</br> ConverterScript - which - * enables stream-based reading a FlowFile's content and stream-based writing to - * newly created FlowFiles</br> - * </ul> - * Presently, the processor supports 3 scripting languages: Ruby, Python, and - * JavaScript. The processor is built on the javax.script API which enables - * ScriptEngine discovery, thread management, and encapsulates much of the low - * level bridging-code that enables Java to Script language integration. Thus, - * it is designed to be easily extended to other scripting languages. </br> The - * attributes of a FlowFile and properties of the Processor are exposed to the - * script by either a variable in the base class or a getter method. A script - * may declare new Processor Properties and different Relationships via - * overriding the getPropertyDescriptors and getRelationships methods, - * respectively. - * </p> - * <p> - * <strong>Properties:</strong> - * </p> - * <p> - * In the list below, the names of required properties appear in bold. Any other - * properties (not in bold) are considered optional. If a property has a default - * value, it is indicated. If a property supports the use of the NiFi Expression - * Language (or simply, "expression language"), that is also indicated. Of - * particular note: This processor allows scripts to define additional Processor - * properties, which will not be initially visible. Once the processor's - * configuration is validated, script defined properties will become visible, - * and may affect the validity of the processor. - * </p> - * <ul> - * <li> - * <strong>Script File Name</strong> - * <ul> - * <li>Script location, can be relative or absolute path.</li> - * <li>Default value: no default</li> - * <li>Supports expression language: false</li> - * </ul> - * </li> - * <li> - * <strong>Script Check Interval</strong> - * <ul> - * <li>The time period between checking for updates to a script.</li> - * <li>Default value: 15 sec</li> - * <li>Supports expression language: false</li> - * </ul> - * </li> - * </ul> - * - * <p> - * <strong>Relationships:</strong> - * </p> - * <p> - * The initial 'out of the box' relationships are below. Of particular note is - * the ability of a script to change the set of relationships. However, any - * relationships defined by the script will not be visible until the processor's - * configuration has been validated. Once done, new relationships will become - * visible. - * </p> - * <ul> - * <li> - * success - * <ul> - * <li>Used when a file is successfully processed by a script.</li> - * </ul> - * </li> - * <li> - * failure - * <ul> - * <li>Used when an error occurs while processing a file with a script.</li> - * </ul> - * </li> - * </ul> - * - * <p> - * <strong>Example Scripts:</strong> - * </p> - * <ul> - * JavaScript example - the 'with' statement imports packages defined in the - * framework. Since the 'instance' variable is intended to be local scope (not - * global), it must be named 'instance' as it it not passed back to the - * processor upon script evaluation and must be fetched. If you make it global, - * you can name it whatever you'd like...but this is intended to be - * multi-threaded so do so at your own risk. Presently, there are issues with - * the JavaScript scripting engine that prevent sub-classing the base classes in - * the Processor's Java framework. So, what is actually happening is an instance - * of the ReaderScript is created with a provided callback object. When we are - * able to move to a more competent scripting engine, the code below will remain - * the same, but the 'instance' variable will actually be a sub-class of - * ReaderScript. - * - * <pre> - * with (Scripting) { - * var instance = new ReaderScript({ - * route : function(input) { - * var str = IOUtils.toString(input); - * var expr = instance.getProperty("expr"); - * filename = instance.attributes.get("filename"); - * instance.setAttribute("filename", filename + ".modified"); - * if (str.match(expr)) { - * return Script.FAIL_RELATIONSHIP; - * } else { - * return Script.SUCCESS_RELATIONSHIP; - * } - * } - * }); - * } - * </pre> - * - * Ruby example - the 'OutputStreamHandler' is an interface which is called when - * creating flow files. - * - * <pre> - * java_import 'org.apache.nifi.scripting.OutputStreamHandler' - * class SimpleConverter < ConverterScript - * field_reader :FAIL_RELATIONSHIP, :SUCCESS_RELATIONSHIP, :logger, :attributes - * - * def convert(input) - * in_io = input.to_io - * createFlowFile("firstLine", FAIL_RELATIONSHIP, OutputStreamHandler.impl do |method, out| - * out_io = out.to_io - * out_io << in_io.readline.to_java_bytes - * out_io.close - * logger.debug("Wrote data to failure...this message logged with logger from super class") - * end) - * - * createFlowFile("otherLines", SUCCESS_RELATIONSHIP, OutputStreamHandler.impl do |method, out| - * out_io = out.to_io - * in_io.each_line { |line| - * out_io << line - * } - * out_io.close - * logger.debug("Wrote data to success...this message logged with logger from super class") - * end) - * in_io.close - * end - * - * end - * - * $logger.debug("Creating SimpleConverter...this message logged with logger from shared variables") - * SimpleConverter.new - * </pre> - * - * Python example - The difficulty with Python is that it does not return - * objects upon script evaluation, so the instance of the Script class must be - * fetched by name. Thus, you must define a variable called 'instance'. - * - * <pre> - * import re - * - * class RoutingReader(ReaderScript): - * A = Relationship.Builder().name("a").description("some good stuff").build() - * B = Relationship.Builder().name("b").description("some other stuff").build() - * C = Relationship.Builder().name("c").description("some bad stuff").build() - * - * def getRelationships(self): - * return [self.A,self.B,self.C] - * - * def getExceptionRoute(self): - * return self.C - * - * def route( self, input ): - * for line in FileUtil.wrap(input): - * if re.match("^bad", line, re.IGNORECASE): - * return self.B - * if re.match("^sed", line): - * raise RuntimeError("That's no good!") - * - * return self.A - * - * instance = RoutingReader() - * </pre> - * - * </ul> - * <p> - * <strong>Shared Variables</strong> - * </p> - * <ul> - * <li>logger : global scope</li> - * <li>properties : local/instance scope</li> - * </ul> - * <p> - * <strong>Script API:</strong> - * </p> - * <ul> - * <li>getAttribute(String) : String</li> - * <li>getAttributes() : Map(String,String)</li> - * <li>getExceptionRoute() : Relationship</li> - * <li>getFileName() : String</li> - * <li>getFlowFileEntryDate() : Calendar</li> - * <li>getFlowFileSize() : long</li> - * <li>getProperties() : Map(String, String)</li> - * <li>getProperty(String) : String</li> - * <li>getPropertyDescriptors() : List(PropertyDescriptor)</li> - * <li>getRelationships() : Collection(Relationship)</li> - * <li>getRoute() : Relationship</li> - * <li>setRoute(Relationship)</li> - * <li>setAttribute(String, String)</li> - * <li>validate() : Collection(String)</li> - * </ul> - * <p> - * <strong>ReaderScript API:</strong> - * </p> - * <ul> - * <li>route(InputStream) : Relationship</li> - * </ul> - * <p> - * <strong>WriterScript API:</strong> - * </p> - * <ul> - * <li>process(InputStream, OutputStream)</li> - * </ul> - * <p> - * <strong>ConverterScript API:</strong> - * </p> - * <ul> - * <li>convert(InputStream)</li> - * <li>createFlowFile(String, Relationship, OutputStreamHandler)</li> - * </ul> - * <p> - * <strong>OutputStreamHandler API:</strong> - * </p> - * <ul> - * <li>write(OutputStream)</li> - * </ul> - */ -@EventDriven -@Tags({"script", "ruby", "python", "javascript", "execute"}) -@CapabilityDescription("Execute scripts in various scripting languages, and passes into the scripts the input stream and output stream(s) " - + "representing an incoming flow file and any created flow files.") -public class ExecuteScript extends AbstractProcessor { - - private final AtomicBoolean doCustomValidate = new AtomicBoolean(true); - private final AtomicReference<Set<Relationship>> relationships = new AtomicReference<>(); - private final AtomicReference<List<PropertyDescriptor>> propertyDescriptors = new AtomicReference<>(); - private volatile ScriptFactory scriptFactory; - private volatile Relationship exceptionRoute; - - /** - * Script location, can be relative or absolute path -- passed as-is to - * {@link File#File(String) File constructor} - */ - public static final PropertyDescriptor SCRIPT_FILE_NAME = new PropertyDescriptor.Builder() - .name("Script File Name") - .description("Script location, can be relative or absolute path") - .required(true) - .addValidator(new Validator() { - - @Override - public ValidationResult validate(String subject, String input, ValidationContext context) { - ValidationResult result = StandardValidators.FILE_EXISTS_VALIDATOR.validate(subject, input, context); - if (result.isValid()) { - int dotPos = input.lastIndexOf('.'); - if (dotPos < 1) { - result = new ValidationResult.Builder() - .subject(subject) - .valid(false) - .explanation("Filename must have an extension") - .input(input) - .build(); - } - } - return result; - } - }) - .build(); - - static final PropertyDescriptor SCRIPT_CHECK_INTERVAL = new PropertyDescriptor.Builder() - .name("Script Check Interval") - .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) - .description("The time period between checking for updates to a script") - .required(true) - .defaultValue("15 sec") - .build(); - - @Override - protected void init(ProcessorInitializationContext context) { - Set<Relationship> empty = Collections.emptySet(); - relationships.set(empty); - ArrayList<PropertyDescriptor> propDescs = new ArrayList<>(); - propDescs.add(SCRIPT_FILE_NAME); - propDescs.add(SCRIPT_CHECK_INTERVAL); - propertyDescriptors.set(Collections.unmodifiableList(propDescs)); - scriptFactory = new ScriptFactory(getLogger()); - } - - @Override - public List<PropertyDescriptor> getSupportedPropertyDescriptors() { - return propertyDescriptors.get(); - } - - @Override - protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName) { - return new PropertyDescriptor.Builder() - .name(propertyDescriptorName) - .dynamic(true) - .addValidator(Validator.VALID) - .build(); - } - - @Override - public void onPropertyModified(PropertyDescriptor descriptor, String oldValue, String newValue) { - doCustomValidate.set(true); - } - - @Override - public Set<Relationship> getRelationships() { - return relationships.get(); - } - - /** - * Called by framework. - * - * Returns a list of reasons why this processor cannot be run. - * @return - */ - @Override - protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { - if (doCustomValidate.getAndSet(false)) { - long interval = validationContext.getProperty(SCRIPT_CHECK_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS); - scriptFactory.setScriptCheckIntervalMS(interval); - List<ValidationResult> results = new ArrayList<>(); - String file = validationContext.getProperty(SCRIPT_FILE_NAME).getValue(); - try { - Script s = scriptFactory.getScript(file); - - // set the relationships of the processor - relationships.set(new HashSet<>(s.getRelationships())); - - // need to get script's prop. descs. and validate. May, or may not, have dynamic - // props already...depends if this is the first time the processor is being configured. - Map<PropertyDescriptor, String> properties = validationContext.getProperties(); - - // need to compare props, if any, against script-expected props that are required. - // script may be expecting required props that are not known, or some props may have invalid - // values. - // processor may be configured with dynamic props that the script will use...but does not declare which would - // be a bad thing - List<PropertyDescriptor> scriptPropDescs = s.getPropertyDescriptors(); - getLogger().debug("Script is {}", new Object[]{s}); - getLogger().debug("Script file name is {}", new Object[]{s.getFileName()}); - getLogger().debug("Script Prop Descs are: {}", new Object[]{scriptPropDescs.toString()}); - getLogger().debug("Thread is: {}", new Object[]{Thread.currentThread().toString()}); - for (PropertyDescriptor propDesc : scriptPropDescs) { - // need to check for missing props - if (propDesc.isRequired() && !properties.containsKey(propDesc)) { - results.add(new ValidationResult.Builder() - .subject("Script Properties") - .valid(false) - .explanation("Missing Property " + propDesc.getName()) - .build()); - - // need to validate current value against script provided validator - } else if (properties.containsKey(propDesc)) { - String value = properties.get(propDesc); - ValidationResult result = propDesc.validate(value, validationContext); - if (!result.isValid()) { - results.add(result); - } - } // else it is an optional prop according to the script and it is not specified by - // the configuration of the processor - } - - // need to update the known prop desc's with what we just got from the script - List<PropertyDescriptor> pds = new ArrayList<>(propertyDescriptors.get()); - pds.addAll(scriptPropDescs); - propertyDescriptors.set(Collections.unmodifiableList(pds)); - - if (results.isEmpty()) { - // so needed props are supplied and individually validated, now validate script - Collection<String> reasons; - reasons = s.validate(); - if (null == reasons) { - getLogger().warn("Script had invalid return value for validate(), ignoring."); - } else { - for (String reason : reasons) { - ValidationResult result = new ValidationResult.Builder() - .subject("ScriptValidation") - .valid(false) - .explanation(reason) - .build(); - results.add(result); - } - } - } - - // get the exception route - exceptionRoute = s.getExceptionRoute(); - - return results; - } catch (ScriptException | IOException | NoSuchMethodException e) { - doCustomValidate.set(true); - results.add(new ValidationResult.Builder() - .subject("ScriptValidation") - .valid(false) - .explanation("Cannot create script due to " + e.getMessage()) - .input(file) - .build()); - getLogger().error("Cannot create script due to " + e, e); - return results; - } - } - return null; - } - - @Override - public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { - FlowFile flowFile = session.get(); - if (flowFile == null) { - return; // fail-fast if there is no work to do - } - - final String scriptFileName = context.getProperty(SCRIPT_FILE_NAME).getValue(); - // doing this cloning because getProperties does not initialize props that have only their default values - // must do a getProperty for that value to be initialized - Map<String, String> props = new HashMap<>(); - for (PropertyDescriptor propDesc : context.getProperties().keySet()) { - if (propDesc.isExpressionLanguageSupported()) { - props.put(propDesc.getName(), context.getProperty(propDesc).evaluateAttributeExpressions(flowFile).getValue()); - } else { - props.put(propDesc.getName(), context.getProperty(propDesc).getValue()); - } - } - Script script = null; - try { - final Script finalScript = scriptFactory.getScript(scriptFileName, props, flowFile); - script = finalScript; - if (finalScript instanceof ReaderScript) { - session.read(flowFile, new InputStreamCallback() { - - @Override - public void process(InputStream in) throws IOException { - try { - ((ReaderScript) finalScript).process(new BufferedInputStream(in)); - } catch (NoSuchMethodException | ScriptException e) { - getLogger().error("Failed to execute ReaderScript", e); - throw new IOException(e); - } - } - }); - } else if (finalScript instanceof WriterScript) { - flowFile = session.write(flowFile, new StreamCallback() { - - @Override - public void process(InputStream in, OutputStream out) throws IOException { - try { - ((WriterScript) finalScript).process(new BufferedInputStream(in), new BufferedOutputStream(out)); - out.flush(); - } catch (NoSuchMethodException | ScriptException e) { - getLogger().error("Failed to execute WriterScript", e); - throw new IOException(e); - } - } - }); - } else if (finalScript instanceof ConverterScript) { - ((ConverterScript) finalScript).process(session); - - // Note that these scripts don't pass the incoming FF through, - // they always create new outputs - session.remove(flowFile); - return; - } else { - // only thing we can do is assume script has already run and done it's thing, so just transfer the incoming - // flowfile - getLogger().debug("Successfully executed script from {}", new Object[]{scriptFileName}); - } - - // update flow file attributes - flowFile = session.putAllAttributes(flowFile, finalScript.getAttributes()); - Relationship route = finalScript.getRoute(); - if (null == route) { - session.remove(flowFile); - getLogger().info("Removing flowfile {}", new Object[]{flowFile}); - } else { - session.transfer(flowFile, route); - getLogger().info("Transferring flowfile {} to {}", new Object[]{flowFile, route}); - } - } catch (ScriptException | IOException e) { - getLogger().error("Failed to create script from {} with flowFile {}. Rolling back session.", - new Object[]{scriptFileName, flowFile}, e); - throw new ProcessException(e); - } catch (Exception e) { - if (null != script) { - getLogger().error("Failed to execute script from {}. Transferring flow file {} to {}", - new Object[]{scriptFileName, flowFile, exceptionRoute}, e); - session.transfer(flowFile, exceptionRoute); - } else { - getLogger().error("Failed to execute script from {} with flowFile {}. Rolling back session", - new Object[]{scriptFileName, flowFile}, e); - throw new ProcessException(e); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/ConverterScript.java ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/ConverterScript.java b/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/ConverterScript.java deleted file mode 100644 index 7be47a8..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/ConverterScript.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.scripting; - -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Map; - -import javax.script.Invocable; -import javax.script.ScriptException; - -import org.apache.nifi.flowfile.FlowFile; -import org.apache.nifi.flowfile.attributes.CoreAttributes; -import org.apache.nifi.io.BufferedInputStream; -import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processor.Relationship; -import org.apache.nifi.processor.io.InputStreamCallback; -import org.apache.nifi.processor.io.OutputStreamCallback; - -/** - * <p> - * Script authors should extend this class if they want to perform complex - * conversions in a NiFi processor. - * </p> - * - * <p> - * Scripts must implement {@link #convert(FileInputStream)}. This method may - * create new FlowFiles and pass them to one or more routes. The input FlowFile - * will be removed from the repository after execution of this method completes. - * </p> - * - * <p> - * In general, the {@link #convert(FileInputStream)} will read from the supplied - * stream, then create one or more output sinks and route the result to the - * relationship of choice using - * {@link #routeStream(ByteArrayOutputStream, String, String)} or - * {@link #routeBytes(byte[], String, String)}. - * - * <p> - * Implement {@link #getProcessorRelationships()} to allow writing to - * relationships other than <code>success</code> and <code>failure</code>. The - * {@link #getRoute()} superclass method is *not* used by Converter Scripts. - * </p> - * - */ -public class ConverterScript extends Script { - - private ProcessSession session; // used to create files - private Object convertCallback; - - public ConverterScript() { - - } - - public ConverterScript(Object... callbacks) { - super(callbacks); - for (Object callback : callbacks) { - if (callback instanceof Map<?, ?>) { - convertCallback = convertCallback == null && ((Map<?, ?>) callback).containsKey("convert") ? callback : convertCallback; - } - } - } - - // Subclasses should implement this to define basic logic - protected void convert(InputStream stream) throws NoSuchMethodException, ScriptException { - if (convertCallback != null) { - ((Invocable) engine).invokeMethod(convertCallback, "convert", stream); - } - } - - /** - * Owning processor uses this method to kick off handling of a single file - * - * @param aSession the owning processor's Repository (needed to make new - * files) - */ - public void process(ProcessSession aSession) { - this.session = aSession; - this.session.read(this.flowFile, new InputStreamCallback() { - - @Override - public void process(InputStream in) throws IOException { - BufferedInputStream stream = new BufferedInputStream(in); - try { - convert(stream); - } catch (NoSuchMethodException | ScriptException e) { - logger.error("Failed to execute 'convert' function in script", e); - throw new IOException(e); - } - } - }); - } - - // this should go back to protected once we get Nashorn - public void createFlowFile(final String flowFileName, final Relationship relationship, final OutputStreamHandler handler) { - FlowFile result = session.create(this.flowFile); - result = session.putAttribute(result, CoreAttributes.FILENAME.key(), flowFileName); - try { - result = session.write(result, new OutputStreamCallback() { - - @Override - public void process(OutputStream out) throws IOException { - handler.write(out); - } - }); - this.logger.info("Transfer flow file {} to {}", new Object[]{result, relationship}); - session.transfer(result, relationship); - } catch (Exception e) { - this.logger.error("Could not create new flow file from script", e); - session.remove(result); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JRubyScriptFactory.java ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JRubyScriptFactory.java b/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JRubyScriptFactory.java deleted file mode 100644 index 883b688..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JRubyScriptFactory.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.scripting; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; - -public enum JRubyScriptFactory { - - INSTANCE; - - private static final String PRELOADS = "include Java\n" - + "\n" - + "java_import 'org.apache.nifi.components.PropertyDescriptor'\n" - + "java_import 'org.apache.nifi.components.Validator'\n" - + "java_import 'org.apache.nifi.processor.util.StandardValidators'\n" - + "java_import 'org.apache.nifi.processor.Relationship'\n" - + "java_import 'org.apache.nifi.logging.ProcessorLog'\n" - + "java_import 'org.apache.nifi.scripting.ReaderScript'\n" - + "java_import 'org.apache.nifi.scripting.WriterScript'\n" - + "java_import 'org.apache.nifi.scripting.ConverterScript'\n" - + "\n"; - - public String getScript(File scriptFile) throws IOException { - StringBuilder sb = new StringBuilder(); - sb.append(PRELOADS) - .append(FileUtils.readFileToString(scriptFile, "UTF-8")); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JavaScriptScriptFactory.java ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JavaScriptScriptFactory.java b/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JavaScriptScriptFactory.java deleted file mode 100644 index 774fb1f..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JavaScriptScriptFactory.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.scripting; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - -public enum JavaScriptScriptFactory { - - INSTANCE; - - private static final String PRELOADS = "var Scripting = JavaImporter(\n" - + " Packages.org.apache.nifi.components,\n" - + " Packages.org.apache.nifi.processor.util,\n" - + " Packages.org.apache.nifi.processor,\n" - + " Packages.org.apache.nifi.logging,\n" - + " Packages.org.apache.nifi.scripting,\n" - + " Packages.org.apache.commons.io);\n" - + "var readFile = function (file) {\n" - + " var script = Packages.org.apache.commons.io.FileUtils.readFileToString(" - + " new java.io.File($PATH, file)" - + " );\n" - + " return \"\" + script;\n" - + "}\n" - + "var require = function (file){\n" - + " var exports={}, module={};\n" - + " module.__defineGetter__('id', function(){return file;});" - + " eval(readFile(file));\n" - + " return exports;\n" - + "}\n"; - - public String getScript(File scriptFile) throws IOException { - StringBuilder sb = new StringBuilder(); - final String parent = StringUtils.replace(scriptFile.getParent(), "\\", "/"); - sb.append(PRELOADS).append("var $PATH = \"").append(parent).append("\"\n") - .append(FileUtils.readFileToString(scriptFile, "UTF-8")); - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/300952a9/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JythonScriptFactory.java ---------------------------------------------------------------------- diff --git a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JythonScriptFactory.java b/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JythonScriptFactory.java deleted file mode 100644 index 6b40b5e..0000000 --- a/nar-bundles/execute-script-bundle/execute-script-processors/src/main/java/org/apache/nifi/scripting/JythonScriptFactory.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.scripting; - -import java.io.File; -import java.io.IOException; - -import org.apache.commons.io.FileUtils; - -public enum JythonScriptFactory { - - INSTANCE; - - private final static String PRELOADS = "from org.python.core.util import FileUtil\n" - + "from org.apache.nifi.components import PropertyDescriptor\n" - + "from org.apache.nifi.components import Validator\n" - + "from org.apache.nifi.processor.util import StandardValidators\n" - + "from org.apache.nifi.processor import Relationship\n" - + "from org.apache.nifi.logging import ProcessorLog\n" - + "from org.apache.nifi.scripting import ReaderScript\n" - + "from org.apache.nifi.scripting import WriterScript\n" - + "from org.apache.nifi.scripting import ConverterScript\n"; - - public String getScript(File scriptFile) throws IOException { - StringBuilder sb = new StringBuilder(); - sb.append(PRELOADS) - .append(FileUtils.readFileToString(scriptFile, "UTF-8")); - - return sb.toString(); - } -}
