MINIFI-242 - Initial integration tests focused around backwards compatibility
This closes #76. Signed-off-by: Aldrin Piri <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/25298e90 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/25298e90 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/25298e90 Branch: refs/heads/master Commit: 25298e90b0024779316fd55d13510613acfa6b90 Parents: f89f415 Author: Bryan Rosander <[email protected]> Authored: Tue Mar 21 12:37:05 2017 -0400 Committer: Aldrin Piri <[email protected]> Committed: Fri Mar 31 13:10:46 2017 -0400 ---------------------------------------------------------------------- README.md | 9 + minifi-integration-tests/pom.xml | 122 +++ .../standalone/test/StandaloneXmlTest.java | 53 ++ .../standalone/test/StandaloneYamlTest.java | 131 ++++ .../src/test/resources/bootstrap.conf | 99 +++ .../resources/docker-compose-v1-standalone.yml | 33 + .../src/test/resources/logback.xml | 100 +++ .../standalone/v1/CsvToJson/xml/CsvToJson.xml | 400 ++++++++++ .../standalone/v1/CsvToJson/xml/expected.json | 4 + .../standalone/v1/CsvToJson/yml/CsvToJson.yml | 178 +++++ .../standalone/v1/CsvToJson/yml/expected.json | 4 + .../xml/DecompressionCircularFlow.xml | 757 +++++++++++++++++++ .../DecompressionCircularFlow/xml/expected.json | 4 + .../yml/DecompressionCircularFlow.yml | 293 +++++++ .../DecompressionCircularFlow/yml/expected.json | 4 + .../xml/MiNiFiTailLogAttribute.xml | 134 ++++ .../v1/MiNiFiTailLogAttribute/xml/expected.json | 4 + .../yml/MiNiFiTailLogAttribute.yml | 98 +++ .../v1/MiNiFiTailLogAttribute/yml/expected.json | 4 + ...aceTextExpressionLanguageCSVReformatting.xml | 270 +++++++ .../xml/expected.json | 4 + ...aceTextExpressionLanguageCSVReformatting.yml | 146 ++++ .../yml/expected.json | 4 + .../xml/MultipleRelationships.xml | 340 +++++++++ .../v2/MultipleRelationships/xml/expected.json | 4 + .../yml/MultipleRelationships.yml | 153 ++++ .../v2/MultipleRelationships/yml/expected.json | 4 + .../v2/ProcessGroups/xml/ProcessGroups.xml | 370 +++++++++ .../v2/ProcessGroups/xml/expected.json | 4 + .../v2/ProcessGroups/yml/ProcessGroups.yml | 163 ++++ .../v2/ProcessGroups/yml/expected.json | 4 + .../xml/StressTestFramework.xml | 539 +++++++++++++ .../v2/StressTestFramework/xml/expected.json | 4 + .../yml/StressTestFramework.yml | 211 ++++++ .../v2/StressTestFramework/yml/expected.json | 4 + .../src/test/resources/tailFileServer.py | 61 ++ pom.xml | 1 + 37 files changed, 4717 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index ec6916f..ad29baa 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ MiNiFi is a child project effort of Apache NiFi - [Features](#features) - [Requirements](#requirements) - [Getting Started](#getting-started) +- [Docker Build](#docker-build) - [Getting Help](#getting-help) - [Documentation](#documentation) - [License](#license) @@ -97,6 +98,14 @@ To run MiNiFi: $ cd ~/example-minifi-deploy/minifi-* $ ./bin/minifi.sh stop +## Docker Build + +To build: +- Execute mvn -P docker clean install. This will run the full build, create a docker image based on it, and run docker-compose integration tests. After it completes successfully, you should have an apacheminifi:${minifi.version} image that can be started with the following command (replacing ${minifi.version} with the current maven version of your branch): +``` +docker run -d -v YOUR_CONFIG.YML:/opt/minifi/minifi-${minifi.version}/conf/config.yml apacheminifi:${minifi.version} +``` + ## Getting Help If you have questions, you can reach out to our mailing list: [email protected] ([archive](https://mail-archives.apache.org/mod_mbox/nifi-dev)). http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/pom.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/pom.xml b/minifi-integration-tests/pom.xml new file mode 100644 index 0000000..6356434 --- /dev/null +++ b/minifi-integration-tests/pom.xml @@ -0,0 +1,122 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>minifi</artifactId> + <groupId>org.apache.nifi.minifi</groupId> + <version>0.2.0-SNAPSHOT</version> + </parent> + <artifactId>minifi-integration-tests</artifactId> + <packaging>jar</packaging> + + <properties> + <minifi.version>${project.version}</minifi.version> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-commons-schema</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.palantir.docker.compose</groupId> + <artifactId>docker-compose-rule-junit4</artifactId> + <version>0.31.1</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi.minifi</groupId> + <artifactId>minifi-toolkit-configuration</artifactId> + <version>${project.version}</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <testResources> + <testResource> + <directory>src/test/resources</directory> + <filtering>true</filtering> + </testResource> + </testResources> + + <plugins> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes combine.children="append"> + <exclude>**/expected.json</exclude> + </excludes> + </configuration> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>docker</id> + <properties> + <name>docker</name> + </properties> + <build> + <plugins> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <skip>false</skip> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <repositories> + <repository> + <id>palantir.bintray</id> + <name>Palantir Bintray</name> + <url>https://dl.bintray.com/palantir/releases</url> + </repository> + </repositories> +</project> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java new file mode 100644 index 0000000..9672928 --- /dev/null +++ b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneXmlTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.integration.standalone.test; + +import org.apache.nifi.minifi.commons.schema.ConfigSchema; +import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; +import org.apache.nifi.minifi.toolkit.configuration.ConfigMain; + +import javax.xml.bind.JAXBException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; + +public class StandaloneXmlTest extends StandaloneYamlTest { + public StandaloneXmlTest(String version, String name) throws IOException, JAXBException { + super(version, name); + ConfigSchema configSchema; + try (InputStream inputStream = StandaloneXmlTest.class.getClassLoader().getResourceAsStream("./standalone/" + version + "/" + name + "/xml/" + name + ".xml")) { + configSchema = ConfigMain.transformTemplateToSchema(inputStream); + } + try (OutputStream outputStream = Files.newOutputStream(Paths.get(StandaloneXmlTest.class.getClassLoader().getResource("docker-compose-v1-standalone.yml").getFile()) + .getParent().toAbsolutePath().resolve(getConfigYml()))) { + SchemaSaver.saveConfigSchema(configSchema, outputStream); + } + } + + @Override + protected String getConfigYml() { + return "./standalone/" + version + "/" + name + "/xml/" + name + ".yml"; + } + + @Override + protected String getExpectedJson() { + return "standalone/" + version + "/" + name + "/xml/expected.json"; + } +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java new file mode 100644 index 0000000..bf9cd96 --- /dev/null +++ b/minifi-integration-tests/src/test/java/org/apache/nifi/minifi/integration/standalone/test/StandaloneYamlTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.integration.standalone.test; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.palantir.docker.compose.DockerComposeRule; +import com.palantir.docker.compose.connection.DockerPort; +import com.palantir.docker.compose.connection.waiting.HealthChecks; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; + +import static org.junit.Assert.fail; + +@RunWith(Parameterized.class) +public class StandaloneYamlTest { + private static final Logger logger = LoggerFactory.getLogger(StandaloneYamlTest.class); + + protected final String version; + protected final String name; + + @Parameterized.Parameters(name = "{index}: Schema Version: {0} Name: {1}") + public static Collection<Object[]> data() { + return Arrays.asList(new Object[][]{ + {"v1", "CsvToJson"}, + {"v1", "DecompressionCircularFlow"}, + {"v1", "MiNiFiTailLogAttribute"}, + {"v1", "ReplaceTextExpressionLanguageCSVReformatting"}, + {"v2", "MultipleRelationships"}, + {"v2", "ProcessGroups"}, + {"v2", "StressTestFramework"} + }); + } + + @Rule + public DockerComposeRule dockerComposeRule; + + public StandaloneYamlTest(String version, String name) throws IOException { + this.version = version; + this.name = name; + String dockerComposeYmlFile = "target/test-classes/docker-compose-" + version + "-" + name + "Test-yml.yml"; + try (InputStream inputStream = StandaloneYamlTest.class.getClassLoader().getResourceAsStream("docker-compose-v1-standalone.yml"); + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + FileWriter fileWriter = new FileWriter(dockerComposeYmlFile); + BufferedWriter bufferedWriter = new BufferedWriter(fileWriter)) { + String line; + while ((line = bufferedReader.readLine()) != null) { + bufferedWriter.write(line.replace("REPLACED_WITH_CONFIG_FILE", getConfigYml())); + bufferedWriter.newLine(); + } + } + dockerComposeRule = DockerComposeRule.builder() + .file(dockerComposeYmlFile) + .waitingForService("minifi", HealthChecks.toRespond2xxOverHttp(8000, dockerPort -> "http://"; + dockerPort.getIp() + ":" + dockerPort.getExternalPort())) + .build(); + } + + protected String getConfigYml() { + return "./standalone/" + version + "/" + name + "/yml/" + name + ".yml"; + } + + protected String getExpectedJson() { + return "standalone/" + version + "/" + name + "/yml/expected.json"; + } + + @Test(timeout = 60_000) + public void verifyLogEntries() throws IOException, InterruptedException, ExecutionException { + Pattern expectedLine; + int expectedOccurences; + try (InputStream inputStream = StandaloneYamlTest.class.getClassLoader().getResourceAsStream(getExpectedJson())) { + Map<String, Object> map = new ObjectMapper().readValue(inputStream, Map.class); + expectedLine = Pattern.compile((String) map.get("pattern")); + expectedOccurences = (int) map.getOrDefault("occurrences", 1); + } + DockerPort dockerPort = dockerComposeRule.containers().container("minifi").port(8000); + URL url = new URL("http://"; + dockerPort.getIp() + ":" + dockerPort.getExternalPort()); + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + try (InputStream inputStream = urlConnection.getInputStream(); + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + String line; + int occurrences = 0; + while ((line = bufferedReader.readLine()) != null) { + if (expectedLine.matcher(line).find()) { + logger.info("Found expected: " + line); + if (++occurrences >= expectedOccurences) { + logger.info("Found target " + occurrences + " times"); + return; + } + } + } + fail("End of log reached without " + expectedOccurences + " match(es)"); + } finally { + urlConnection.disconnect(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/bootstrap.conf ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/bootstrap.conf b/minifi-integration-tests/src/test/resources/bootstrap.conf new file mode 100644 index 0000000..28e9d1b --- /dev/null +++ b/minifi-integration-tests/src/test/resources/bootstrap.conf @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running MiNiFi +java=java + +# Username to use when running MiNiFi. This value will be ignored on Windows. +run.as= + +# Configure where MiNiFi's lib and conf directories live +lib.dir=./lib +conf.dir=./conf + +# How long to wait after telling MiNiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# The location for the configuration file +nifi.minifi.config=./conf/config.yml + +# Notifiers to use for the associated agent, comma separated list of class names +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.FileChangeIngestor +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.RestChangeIngestor +#nifi.minifi.notifier.ingestors=org.apache.nifi.minifi.bootstrap.configuration.ingestors.PullHttpChangeIngestor + +# File change notifier configuration + +# Path of the file to monitor for changes. When these occur, the FileChangeNotifier, if configured, will begin the configuration reloading process +#nifi.minifi.notifier.ingestors.file.config.path= +# How frequently the file specified by 'nifi.minifi.notifier.file.config.path' should be evaluated for changes. +#nifi.minifi.notifier.ingestors.file.polling.period.seconds=5 + +# Rest change notifier configuration + +# Port on which the Jetty server will bind to, keep commented for a random open port +#nifi.minifi.notifier.ingestors.receive.http.port=8338 + +#Pull HTTP change notifier configuration + +# Hostname on which to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.hostname=localhost +# Port on which to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.port=4567 +# Path to pull configurations from +#nifi.minifi.notifier.ingestors.pull.http.path=/c2/config +# Query string to pull configurations with +#nifi.minifi.notifier.ingestors.pull.http.query=class=raspi3 +# Period on which to pull configurations from, defaults to 5 minutes if commented out +#nifi.minifi.notifier.ingestors.pull.http.period.ms=300000 + +# Periodic Status Reporters to use for the associated agent, comma separated list of class names +#nifi.minifi.status.reporter.components=org.apache.nifi.minifi.bootstrap.status.reporters.StatusLogger + +# Periodic Status Logger configuration + +# The FlowStatus query to submit to the MiNiFi instance +#nifi.minifi.status.reporter.log.query=instance:health,bulletins +# The log level at which the status will be logged +#nifi.minifi.status.reporter.log.level=INFO +# The period (in milliseconds) at which to log the status +#nifi.minifi.status.reporter.log.period=60000 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms256m +java.arg.3=-Xmx256m + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +#java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +java.arg.15=-Djava.security.egd=file:/dev/./urandom \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml b/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml new file mode 100644 index 0000000..1cd18ae --- /dev/null +++ b/minifi-integration-tests/src/test/resources/docker-compose-v1-standalone.yml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" + +services: + minifi: + image: apacheminifi:${minifi.version} + ports: + - "8000" + hostname: minifi + volumes: + - ./tailFileServer.py:/home/minifi/tailFileServer.py + + - ./bootstrap.conf:/opt/minifi/minifi-${minifi.version}/conf/bootstrap.conf + - ./logback.xml:/opt/minifi/minifi-${minifi.version}/conf/logback.xml + - REPLACED_WITH_CONFIG_FILE:/opt/minifi/minifi-${minifi.version}/conf/config.yml + entrypoint: + - bash + - -c + - /opt/minifi/minifi-${minifi.version}/bin/minifi.sh start && python /home/minifi/tailFileServer.py --file /opt/minifi/minifi-${minifi.version}/logs/minifi-app.log \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/logback.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/logback.xml b/minifi-integration-tests/src/test/resources/logback.xml new file mode 100644 index 0000000..168ff9e --- /dev/null +++ b/minifi-integration-tests/src/test/resources/logback.xml @@ -0,0 +1,100 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<configuration scan="true" scanPeriod="30 seconds"> + <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"> + <resetJUL>true</resetJUL> + </contextListener> + + <appender name="APP_FILE" class="ch.qos.logback.core.FileAppender"> + <file>${org.apache.nifi.minifi.bootstrap.config.log.dir}/minifi-app.log</file> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + <immediateFlush>true</immediateFlush> + </encoder> + </appender> + + <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.minifi.bootstrap.config.log.dir}/minifi-bootstrap.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- + For daily rollover, use 'user_%d.log'. + For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'. + To GZIP rolled files, replace '.log' with '.log.gz'. + To ZIP rolled files, replace '.log' with '.log.zip'. + --> + <fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/minifi-bootstrap_%d.log.gz</fileNamePattern> + <!-- Keep 5 rolling periods worth of logs--> + <maxHistory>5</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + </encoder> + </appender> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + </encoder> + </appender> + + <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR --> + + <logger name="org.apache.nifi" level="INFO"/> + <logger name="org.apache.nifi.processors" level="WARN"/> + <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/> + <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="INFO" /> + + <!-- Logger for managing logging statements for jetty --> + <logger name="org.eclipse.jetty" level="INFO"/> + + <!-- Suppress non-error messages due to excessive logging by class or library --> + <logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/> + <logger name="com.sun.jersey.spi.spring" level="ERROR"/> + <logger name="org.springframework" level="ERROR"/> + + <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) --> + <logger name="com.sun.jersey.spi.inject.Errors" level="ERROR"/> + + <!-- + Logger for capturing Bootstrap logs and MiNiFi's standard error and standard out. + --> + <logger name="org.apache.nifi.minifi.bootstrap" level="INFO" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + <logger name="org.apache.nifi.minifi.bootstrap.Command" level="INFO" additivity="false"> + <appender-ref ref="CONSOLE" /> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + <!-- Everything written to MiNiFi's Standard Out will be logged with the logger org.apache.nifi.minifi.StdOut at INFO level --> + <logger name="org.apache.nifi.minifi.StdOut" level="INFO" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + <!-- Everything written to MiNiFi's Standard Error will be logged with the logger org.apache.nifi.minifi.StdErr at ERROR level --> + <logger name="org.apache.nifi.minifi.StdErr" level="ERROR" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + + <root level="INFO"> + <appender-ref ref="APP_FILE"/> + </root> + +</configuration> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml new file mode 100644 index 0000000..60f5296 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/CsvToJson.xml @@ -0,0 +1,400 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<template encoding-version="1.0"> + <description></description> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <name>CsvToJsonWorking</name> + <snippet> + <connections> + <id>0cc34e0a-0ae9-44ba-838f-792ed393a301</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>0</backPressureObjectThreshold> + <destination> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>cdcc2028-238b-42b7-bf95-dd4f301b91fc</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <prioritizers>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</prioritizers> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>ffb6d4e9-9d6c-4fbe-ab8a-52bad90ce2e5</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>e32e19cb-6c4c-4c6c-a8db-408447ba2ca2</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>0</backPressureObjectThreshold> + <destination> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>5f0ec33c-0a07-4305-a3a9-9a3ba0548ac6</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <prioritizers>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</prioritizers> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>cdcc2028-238b-42b7-bf95-dd4f301b91fc</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>b061c170-19da-4e84-9709-327303fef579</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>0</backPressureObjectThreshold> + <destination> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>1d00089c-78cd-467f-9aa6-31e3bdf90cb0</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <prioritizers>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</prioritizers> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>8db2ebc8-12c8-4396-89b6-479d907fbc6b</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>56ef3e2e-ee35-4598-9fbe-ae86050960b0</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>0</backPressureObjectThreshold> + <destination> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>8db2ebc8-12c8-4396-89b6-479d907fbc6b</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <prioritizers>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</prioritizers> + <selectedRelationships>matched</selectedRelationships> + <source> + <groupId>3d665027-7897-4322-96a1-cabef6b2bcf7</groupId> + <id>5f0ec33c-0a07-4305-a3a9-9a3ba0548ac6</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <labels> + <id>a1854f35-469c-41fa-80a2-70ae90cf1ed9</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>514.0</x> + <y>431.6599933547974</y> + </position> + <height>281.4172668457031</height> + <label></label> + <style/> + <width>388.0479431152344</width> + </labels> + <processors> + <id>cdcc2028-238b-42b7-bf95-dd4f301b91fc</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>569.3500686645507</x> + <y>239.77198108673096</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Regular Expression</key> + <value>(?s:^.*$)</value> + </entry> + <entry> + <key>Replacement Value</key> + <value>a,b,c,d</value> + </entry> + <entry> + <key>Character Set</key> + <value>UTF-8</value> + </entry> + <entry> + <key>Maximum Buffer Size</key> + <value>1 MB</value> + </entry> + <entry> + <key>Replacement Strategy</key> + </entry> + <entry> + <key>Evaluation Mode</key> + <value>Entire text</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>ReplaceText</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>failure</name> + </relationships> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style/> + <type>org.apache.nifi.processors.standard.ReplaceText</type> + </processors> + <processors> + <id>ffb6d4e9-9d6c-4fbe-ab8a-52bad90ce2e5</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>558.2500778198241</x> + <y>31.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>1 b</value> + </entry> + <entry> + <key>Batch Size</key> + <value>1</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>1 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style/> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>1d00089c-78cd-467f-9aa6-31e3bdf90cb0</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>585.2500320434569</x> + <y>866.0879933547974</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Delete Attributes Expression</key> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>UpdateAttribute</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>success</name> + </relationships> + <style/> + <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> + </processors> + <processors> + <id>8db2ebc8-12c8-4396-89b6-479d907fbc6b</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>590.3500228881835</x> + <y>658.9239851760865</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Regular Expression</key> + <value>(?s:^.*$)</value> + </entry> + <entry> + <key>Replacement Value</key> + <value>{ "field1" : "${csv.1}", "field2" : "${csv.2}", + "field3" : "${csv.3}", "field4" : "${csv.4}" } + </value> + </entry> + <entry> + <key>Character Set</key> + <value>UTF-8</value> + </entry> + <entry> + <key>Maximum Buffer Size</key> + <value>1 MB</value> + </entry> + <entry> + <key>Replacement Strategy</key> + </entry> + <entry> + <key>Evaluation Mode</key> + <value>Entire text</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>ReplaceText2</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>failure</name> + </relationships> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style/> + <type>org.apache.nifi.processors.standard.ReplaceText</type> + </processors> + <processors> + <id>5f0ec33c-0a07-4305-a3a9-9a3ba0548ac6</id> + <parentGroupId>3d665027-7897-4322-96a1-cabef6b2bcf7</parentGroupId> + <position> + <x>580.7500320434569</x> + <y>450.15196063995364</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Character Set</key> + <value>UTF-8</value> + </entry> + <entry> + <key>Maximum Buffer Size</key> + <value>1 MB</value> + </entry> + <entry> + <key>Maximum Capture Group Length</key> + <value>1024</value> + </entry> + <entry> + <key>Enable Canonical Equivalence</key> + <value>false</value> + </entry> + <entry> + <key>Enable Case-insensitive Matching</key> + <value>false</value> + </entry> + <entry> + <key>Permit Whitespace and Comments in Pattern</key> + <value>false</value> + </entry> + <entry> + <key>Enable DOTALL Mode</key> + <value>false</value> + </entry> + <entry> + <key>Enable Literal Parsing of the Pattern</key> + <value>false</value> + </entry> + <entry> + <key>Enable Multiline Mode</key> + <value>false</value> + </entry> + <entry> + <key>Enable Unicode-aware Case Folding</key> + <value>false</value> + </entry> + <entry> + <key>Enable Unicode Predefined Character Classes</key> + <value>false</value> + </entry> + <entry> + <key>Enable Unix Lines Mode</key> + <value>false</value> + </entry> + <entry> + <key>Include Capture Group 0</key> + <value>false</value> + </entry> + <entry> + <key>csv</key> + <value>(.+),(.+),(.+),(.+)</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>ExtractText</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>matched</name> + </relationships> + <relationships> + <autoTerminate>true</autoTerminate> + <name>unmatched</name> + </relationships> + <style/> + <type>org.apache.nifi.processors.standard.ExtractText</type> + </processors> + </snippet> + <timestamp>07/04/2016 20:20:45 UTC</timestamp> +</template> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json new file mode 100644 index 0000000..ebc43fc --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/xml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=1d00089c-78cd-467f-9aa6-31e3bdf90cb0\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/CsvToJson.yml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/CsvToJson.yml b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/CsvToJson.yml new file mode 100644 index 0000000..1237ac8 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/CsvToJson.yml @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: CsvToJsonWorking + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: ExtractText + class: org.apache.nifi.processors.standard.ExtractText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - unmatched + Properties: + Character Set: UTF-8 + Enable Canonical Equivalence: 'false' + Enable Case-insensitive Matching: 'false' + Enable DOTALL Mode: 'false' + Enable Literal Parsing of the Pattern: 'false' + Enable Multiline Mode: 'false' + Enable Unicode Predefined Character Classes: 'false' + Enable Unicode-aware Case Folding: 'false' + Enable Unix Lines Mode: 'false' + Include Capture Group 0: 'false' + Maximum Buffer Size: 1 MB + Maximum Capture Group Length: '1024' + Permit Whitespace and Comments in Pattern: 'false' + csv: (.+),(.+),(.+),(.+) +- name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 1 b + Unique FlowFiles: 'false' +- name: ReplaceText + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: UTF-8 + Evaluation Mode: Entire text + Maximum Buffer Size: 1 MB + Regular Expression: (?s:^.*$) + Replacement Strategy: + Replacement Value: a,b,c,d +- name: ReplaceText2 + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: UTF-8 + Evaluation Mode: Entire text + Maximum Buffer Size: 1 MB + Regular Expression: (?s:^.*$) + Replacement Strategy: + Replacement Value: "{ \"field1\" : \"${csv.1}\", \"field2\" : \"${csv.2}\",\n\ + \ \"field3\" : \"${csv.3}\", \"field4\" : \"${csv.4}\"\ + \ }\n " +- name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Delete Attributes Expression: +Connections: +- name: ExtractText/matched/ReplaceText2 + source name: ExtractText + source relationship name: matched + destination name: ReplaceText2 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GenerateFlowFile/success/ReplaceText + source name: GenerateFlowFile + source relationship name: success + destination name: ReplaceText + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: ReplaceText/success/ExtractText + source name: ReplaceText + source relationship name: success + destination name: ExtractText + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: ReplaceText2/success/UpdateAttribute + source name: ReplaceText2 + source relationship name: success + destination name: UpdateAttribute + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json new file mode 100644 index 0000000..1f89457 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v1/CsvToJson/yml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for UpdateAttribute\\[id=2aac227f-e8e9-370f-87c8-4f970e0b260e\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 2 +}
