Repository: nifi-minifi Updated Branches: refs/heads/master 8913a9026 -> 1bbeedf6f
http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index 74e1f7f..251e9a3 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -18,10 +18,11 @@ package org.apache.nifi.minifi.toolkit.configuration; import org.apache.nifi.minifi.commons.schema.ConfigSchema; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; +import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.apache.nifi.minifi.commons.schema.serialization.SchemaSaver; -import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.apache.nifi.minifi.toolkit.configuration.dto.ConfigSchemaFunction; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; @@ -46,6 +47,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -57,12 +59,16 @@ public class ConfigMain { public static final int ERR_UNABLE_TO_TRANSFORM_TEMPLATE = 5; public static final int ERR_UNABLE_TO_PARSE_CONFIG = 6; public static final int ERR_INVALID_CONFIG = 7; + public static final int ERR_UNABLE_TO_CLOSE_CONFIG = 8; + public static final int ERR_UNABLE_TO_SAVE_CONFIG = 9; public static final int SUCCESS = 0; public static final String TRANSFORM = "transform"; public static final String VALIDATE = "validate"; - public static final String NO_VALIDATION_ERRORS_FOUND_IN_TEMPLATE = "No validation errors found in template."; + public static final String UPGRADE = "upgrade"; + public static final String THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED = + "There are validation errors with the template, still outputting YAML but it will need to be edited."; private final Map<String, Command> commandMap; private final PathInputStreamFactory pathInputStreamFactory; @@ -96,31 +102,22 @@ public class ConfigMain { } try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { try { - ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(inputStream); - if (!configSchema.isValid()) { - configSchema.getValidationIssues().forEach(s -> System.out.println(s)); - System.out.println(); - return ERR_INVALID_CONFIG; - } else { - System.out.println(NO_VALIDATION_ERRORS_FOUND_IN_TEMPLATE); - } + return loadAndPrintValidationErrors(inputStream, (configSchema, valid) -> { + if (valid) { + return SUCCESS; + } else { + return ERR_INVALID_CONFIG; + } + }); } catch (IOException|SchemaLoaderException e) { - System.out.println("Unable to load configuration. (" + e + ")"); - System.out.println(); - printValidateUsage(); - return ERR_UNABLE_TO_PARSE_CONFIG; + return handleErrorLoadingConfiguration(e, ConfigMain::printValidateUsage); } } catch (FileNotFoundException e) { - System.out.println("Unable to open file " + args[1] + " for reading. (" + e + ")"); - System.out.println(); - printValidateUsage(); - return ERR_UNABLE_TO_OPEN_INPUT; + return handleErrorOpeningInput(args[1], ConfigMain::printValidateUsage, e); } catch (IOException e) { - System.out.println("Error closing input. (" + e + ")"); - System.out.println(); + handleErrorClosingInput(e); + return ERR_UNABLE_TO_CLOSE_CONFIG; } - - return SUCCESS; } public static void printTransformUsage() { @@ -130,6 +127,13 @@ public class ConfigMain { System.out.println(); } + public static void printUpgradeUsage() { + System.out.println("Upgrade Usage:"); + System.out.println(); + System.out.println(" upgrade INPUT_FILE OUTPUT_FILE"); + System.out.println(); + } + private static void enrichTemplateDTO(TemplateDTO templateDTO) { FlowSnippetDTO flowSnippetDTO = templateDTO.getSnippet(); @@ -182,32 +186,28 @@ public class ConfigMain { } } - public static ConfigSchema transformTemplateToSchema(InputStream source) throws JAXBException, IOException, SchemaLoaderException { - try { - TemplateDTO templateDTO = (TemplateDTO) JAXBContext.newInstance(TemplateDTO.class).createUnmarshaller().unmarshal(source); + public static ConfigSchema transformTemplateToSchema(InputStream source) throws JAXBException, SchemaLoaderException { + TemplateDTO templateDTO = (TemplateDTO) JAXBContext.newInstance(TemplateDTO.class).createUnmarshaller().unmarshal(source); - if (templateDTO.getSnippet().getProcessGroups().size() != 0){ - throw new SchemaLoaderException("Process Groups are not currently supported in MiNiFi. Please remove any from the template and try again."); - } - - if (templateDTO.getSnippet().getOutputPorts().size() != 0){ - throw new SchemaLoaderException("Output Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); - } + if (templateDTO.getSnippet().getProcessGroups().size() != 0){ + throw new SchemaLoaderException("Process Groups are not currently supported in MiNiFi. Please remove any from the template and try again."); + } - if (templateDTO.getSnippet().getInputPorts().size() != 0){ - throw new SchemaLoaderException("Input Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); - } + if (templateDTO.getSnippet().getOutputPorts().size() != 0){ + throw new SchemaLoaderException("Output Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); + } - if (templateDTO.getSnippet().getFunnels().size() != 0){ - throw new SchemaLoaderException("Funnels are not currently supported in MiNiFi. Please remove any from the template and try again."); - } + if (templateDTO.getSnippet().getInputPorts().size() != 0){ + throw new SchemaLoaderException("Input Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); + } - enrichTemplateDTO(templateDTO); - ConfigSchema configSchema = new ConfigSchemaFunction().apply(templateDTO); - return configSchema; - } finally { - source.close(); + if (templateDTO.getSnippet().getFunnels().size() != 0){ + throw new SchemaLoaderException("Funnels are not currently supported in MiNiFi. Please remove any from the template and try again."); } + + enrichTemplateDTO(templateDTO); + ConfigSchema configSchema = new ConfigSchemaFunction().apply(templateDTO); + return configSchema; } private static void setName(Map<String, String> connectableNameMap, ConnectableDTO connectableDTO) { @@ -235,62 +235,158 @@ public class ConfigMain { } } - public int transform(String[] args) { + public int upgrade(String[] args) { if (args.length != 3) { - printTransformUsage(); + printUpgradeUsage(); return ERR_INVALID_ARGS; } + + ConfigSchema currentSchema = null; try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { - try (OutputStream fileOutputStream = pathOutputStreamFactory.create(args[2])) { - try { - ConfigSchema configSchema = transformTemplateToSchema(inputStream); - if (!configSchema.isValid()) { - System.out.println("There are validation errors with the template, still outputting YAML but it will need to be edited."); - for (String s : configSchema.getValidationIssues()) { - System.out.println(s); - } + try { + currentSchema = loadAndPrintValidationErrors(inputStream, (configSchema, valid) -> { + if (!valid) { + System.out.println(THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED); System.out.println(); - } else { - System.out.println(NO_VALIDATION_ERRORS_FOUND_IN_TEMPLATE); } - SchemaSaver.saveConfigSchema(configSchema, fileOutputStream); - } catch (JAXBException e) { - System.out.println("Error reading template. (" + e + ")"); - System.out.println(); - printTransformUsage(); - return ERR_UNABLE_TO_READ_TEMPLATE; - } catch (IOException e) { - System.out.println("Error transforming template to YAML. (" + e + ")"); - System.out.println(); - printTransformUsage(); - return ERR_UNABLE_TO_TRANSFORM_TEMPLATE; - } catch (SchemaLoaderException e) { - System.out.println("Error transforming template to YAML. (" + e.getMessage() + ")"); + return configSchema; + }); + } catch (IOException|SchemaLoaderException e) { + return handleErrorLoadingConfiguration(e, ConfigMain::printUpgradeUsage); + } + } catch (FileNotFoundException e) { + return handleErrorOpeningInput(args[1], ConfigMain::printUpgradeUsage, e); + } catch (IOException e) { + handleErrorClosingInput(e); + } + + try (OutputStream fileOutputStream = pathOutputStreamFactory.create(args[2])) { + try { + SchemaSaver.saveConfigSchema(currentSchema, fileOutputStream); + } catch (IOException e) { + return handleErrorSavingCofiguration(e); + } + } catch (FileNotFoundException e) { + return handleErrorOpeningOutput(args[2], ConfigMain::printUpgradeUsage, e); + } catch (IOException e) { + handleErrorClosingOutput(e); + } + + return SUCCESS; + } + + public <T> T loadAndPrintValidationErrors(InputStream inputStream, BiFunction<ConfigSchema, Boolean, T> resultHandler) throws IOException, SchemaLoaderException { + ConvertableSchema<ConfigSchema> configSchema = SchemaLoader.loadConvertableSchemaFromYaml(inputStream); + boolean valid = true; + if (!configSchema.isValid()) { + System.out.println("Found the following errors when parsing the configuration according to its version. (" + configSchema.getVersion() + ")"); + configSchema.getValidationIssues().forEach(s -> System.out.println(s)); + System.out.println(); + valid = false; + configSchema.clearValidationIssues(); + } else { + System.out.println("No errors found when parsing configuration according to its version. (" + configSchema.getVersion() + ")"); + } + + ConfigSchema currentSchema = configSchema.convert(); + if (!currentSchema.isValid()) { + System.out.println("Found the following errors when converting configuration to latest version. (" + ConfigSchema.CONFIG_VERSION + ")"); + currentSchema.getValidationIssues().forEach(s -> System.out.println(s)); + System.out.println(); + valid = false; + } else if (configSchema.getVersion() == currentSchema.getVersion()) { + System.out.println("Configuration was already latest version (" + ConfigSchema.CONFIG_VERSION + ") so no conversion was needed."); + } else { + System.out.println("No errors found when converting configuration to latest version. (" + ConfigSchema.CONFIG_VERSION + ")"); + } + return resultHandler.apply(currentSchema, valid); + } + + public int transform(String[] args) { + if (args.length != 3) { + printTransformUsage(); + return ERR_INVALID_ARGS; + } + + ConfigSchema configSchema = null; + try (InputStream inputStream = pathInputStreamFactory.create(args[1])) { + try { + configSchema = transformTemplateToSchema(inputStream); + if (!configSchema.isValid()) { + System.out.println(THERE_ARE_VALIDATION_ERRORS_WITH_THE_TEMPLATE_STILL_OUTPUTTING_YAML_BUT_IT_WILL_NEED_TO_BE_EDITED); + configSchema.getValidationIssues().forEach(System.out::println); System.out.println(); - return ERR_UNABLE_TO_TRANSFORM_TEMPLATE; + } else { + System.out.println("No validation errors found in converted configuration."); } - } catch (FileNotFoundException e) { - System.out.println("Unable to open file " + args[2] + " for writing. (" + e + ")"); + } catch (JAXBException e) { + System.out.println("Error reading template. (" + e + ")"); System.out.println(); printTransformUsage(); - return ERR_UNABLE_TO_OPEN_OUTPUT; - } catch (IOException e) { - System.out.println("Error closing output. (" + e + ")"); + return ERR_UNABLE_TO_READ_TEMPLATE; + } catch (SchemaLoaderException e) { + System.out.println("Error transforming template to YAML. (" + e.getMessage() + ")"); System.out.println(); + return ERR_UNABLE_TO_TRANSFORM_TEMPLATE; } } catch (FileNotFoundException e) { - System.out.println("Unable to open file " + args[1] + " for reading. (" + e + ")"); - System.out.println(); - printTransformUsage(); - return ERR_UNABLE_TO_OPEN_INPUT; + return handleErrorOpeningInput(args[1], ConfigMain::printTransformUsage, e); } catch (IOException e) { - System.out.println("Error closing input. (" + e + ")"); - System.out.println(); + handleErrorClosingInput(e); + } + + try (OutputStream fileOutputStream = pathOutputStreamFactory.create(args[2])) { + try { + SchemaSaver.saveConfigSchema(configSchema, fileOutputStream); + } catch (IOException e) { + return handleErrorSavingCofiguration(e); + } + } catch (FileNotFoundException e) { + return handleErrorOpeningOutput(args[2], ConfigMain::printTransformUsage, e); + } catch (IOException e) { + handleErrorClosingOutput(e); } return SUCCESS; } + protected void handleErrorClosingOutput(IOException e) { + System.out.println("Error closing output. (" + e + ")"); + System.out.println(); + } + + protected void handleErrorClosingInput(IOException e) { + System.out.println("Error closing input. (" + e + ")"); + System.out.println(); + } + + protected int handleErrorOpeningInput(String fileName, Runnable usagePrinter, FileNotFoundException e) { + System.out.println("Unable to open file " + fileName + " for reading. (" + e + ")"); + System.out.println(); + usagePrinter.run(); + return ERR_UNABLE_TO_OPEN_INPUT; + } + + protected int handleErrorOpeningOutput(String fileName, Runnable usagePrinter, FileNotFoundException e) { + System.out.println("Unable to open file " + fileName + " for writing. (" + e + ")"); + System.out.println(); + usagePrinter.run(); + return ERR_UNABLE_TO_OPEN_OUTPUT; + } + + protected int handleErrorLoadingConfiguration(Exception e, Runnable usagePrinter) { + System.out.println("Unable to load configuration. (" + e + ")"); + System.out.println(); + usagePrinter.run(); + return ERR_UNABLE_TO_PARSE_CONFIG; + } + + protected int handleErrorSavingCofiguration(IOException e) { + System.out.println("Unable to save configuration: " + e); + System.out.println(); + return ERR_UNABLE_TO_SAVE_CONFIG; + } + public int execute(String[] args) { if (args.length < 1 || !commandMap.containsKey(args[0].toLowerCase())) { printUsage(); @@ -303,6 +399,7 @@ public class ConfigMain { Map<String, Command> result = new TreeMap<>(); result.put(TRANSFORM, new Command(this::transform, "Transform template xml into MiNiFi config YAML")); result.put(VALIDATE, new Command(this::validate, "Validate config YAML")); + result.put(UPGRADE, new Command(this::upgrade, "Upgrade config YAML to current version (" + ConfigSchema.CONFIG_VERSION + ")")); return result; } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java index 5ca4933..cfdc48f 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java @@ -18,6 +18,11 @@ package org.apache.nifi.minifi.toolkit.configuration; import org.apache.commons.io.Charsets; +import org.apache.nifi.minifi.commons.schema.ConfigSchema; +import org.apache.nifi.minifi.commons.schema.ConnectionSchema; +import org.apache.nifi.minifi.commons.schema.ProcessorSchema; +import org.apache.nifi.minifi.commons.schema.RemoteInputPortSchema; +import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.junit.Before; @@ -31,14 +36,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import static org.apache.nifi.minifi.toolkit.configuration.ConfigMain.SUCCESS; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.when; @@ -106,7 +115,14 @@ public class ConfigMainTest { public void testValidateSuccess() throws FileNotFoundException { when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> ConfigMainTest.class.getClassLoader().getResourceAsStream("config.yml")); - assertEquals(ConfigMain.SUCCESS, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); + assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); + } + + @Test + public void testValidateV1Success() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> + ConfigMainTest.class.getClassLoader().getResourceAsStream("config-v1.yml")); + assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.VALIDATE, testInput})); } @Test @@ -117,6 +133,8 @@ public class ConfigMainTest { @Test public void testTransformErrorOpeningOutput() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> + ConfigMainTest.class.getClassLoader().getResourceAsStream("CsvToJson.xml")); when(pathOutputStreamFactory.create(testOutput)).thenThrow(new FileNotFoundException()); assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_OUTPUT, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); } @@ -130,13 +148,7 @@ public class ConfigMainTest { @Test public void testTransformErrorTransformingTemplate() throws FileNotFoundException { when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> - ConfigMainTest.class.getClassLoader().getResourceAsStream("CsvToJson.xml")); - when(pathOutputStreamFactory.create(testOutput)).thenAnswer(invocation -> new OutputStream() { - @Override - public void write(int b) throws IOException { - throw new IOException(); - } - }); + ConfigMainTest.class.getClassLoader().getResourceAsStream("TemplateWithInputPort.xml")); assertEquals(ConfigMain.ERR_UNABLE_TO_TRANSFORM_TEMPLATE, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); } @@ -145,7 +157,7 @@ public class ConfigMainTest { when(pathInputStreamFactory.create(testInput)).thenAnswer(invocation -> ConfigMainTest.class.getClassLoader().getResourceAsStream("CsvToJson.xml")); when(pathOutputStreamFactory.create(testOutput)).thenAnswer(invocation -> new ByteArrayOutputStream()); - assertEquals(ConfigMain.SUCCESS, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); + assertEquals(SUCCESS, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); } @Test @@ -198,10 +210,106 @@ public class ConfigMainTest { ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithFunnel.xml")).toMap(); } + @Test + public void testUpgradeInputFileNotFoundException() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenThrow(new FileNotFoundException()); + assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_INPUT, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); + } + + @Test + public void testUpgradeCantLoadSchema() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenReturn(new InputStream() { + @Override + public int read() throws IOException { + throw new IOException(); + } + }); + assertEquals(ConfigMain.ERR_UNABLE_TO_PARSE_CONFIG, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); + } + + @Test + public void testUpgradeOutputFileNotFoundException() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenReturn(getClass().getClassLoader().getResourceAsStream("CsvToJson-v1.yml")); + when(pathOutputStreamFactory.create(testOutput)).thenThrow(new FileNotFoundException()); + assertEquals(ConfigMain.ERR_UNABLE_TO_OPEN_OUTPUT, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); + } + + @Test + public void testUpgradeCantSaveSchema() throws FileNotFoundException { + when(pathInputStreamFactory.create(testInput)).thenReturn(getClass().getClassLoader().getResourceAsStream("CsvToJson-v1.yml")); + when(pathOutputStreamFactory.create(testOutput)).thenReturn(new OutputStream() { + @Override + public void write(int b) throws IOException { + throw new IOException(); + } + }); + assertEquals(ConfigMain.ERR_UNABLE_TO_SAVE_CONFIG, configMain.execute(new String[]{ConfigMain.UPGRADE, testInput, testOutput})); + } + + @Test + public void testUpgradeInvalidArgs() { + assertEquals(ConfigMain.ERR_INVALID_ARGS, configMain.execute(new String[]{ConfigMain.UPGRADE})); + } + private void transformRoundTrip(String name) throws JAXBException, IOException, SchemaLoaderException { Map<String, Object> templateMap = ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream(name + ".xml")).toMap(); Map<String, Object> yamlMap = SchemaLoader.loadYamlAsMap(getClass().getClassLoader().getResourceAsStream(name + ".yml")); assertNoMapDifferences(templateMap, yamlMap); + testV1YmlIfPresent(name, yamlMap); + } + + private InputStream upgradeAndReturn(String name) throws FileNotFoundException { + InputStream yamlV1Stream = getClass().getClassLoader().getResourceAsStream(name + "-v1.yml"); + if (yamlV1Stream == null) { + return null; + } + when(pathInputStreamFactory.create(testInput)).thenReturn(yamlV1Stream); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + when(pathOutputStreamFactory.create(testOutput)).thenReturn(outputStream); + assertEquals(SUCCESS, configMain.execute(new String[]{"upgrade", testInput, testOutput})); + return new ByteArrayInputStream(outputStream.toByteArray()); + } + + private void testV1YmlIfPresent(String name, Map<String, Object> yamlMap) throws IOException, SchemaLoaderException { + InputStream upgradedInputStream = upgradeAndReturn(name); + if (upgradedInputStream != null) { + ConvertableSchema<ConfigSchema> configSchemaConvertableSchema = SchemaLoader.loadConvertableSchemaFromYaml(upgradedInputStream); + ConfigSchema configSchemaUpgradedFromV1 = configSchemaConvertableSchema.convert(); + assertTrue(configSchemaUpgradedFromV1.isValid()); + assertEquals(configSchemaConvertableSchema, configSchemaUpgradedFromV1); + ConfigSchema configSchemaFromCurrent = new ConfigSchema(yamlMap); + List<ProcessorSchema> currentProcessors = configSchemaFromCurrent.getProcessors(); + List<ProcessorSchema> v1Processors = configSchemaUpgradedFromV1.getProcessors(); + assertEquals(currentProcessors.size(), v1Processors.size()); + + // V1 doesn't have ids so we need to map the autogenerated ones to the ones from the template + Map<String, String> v1IdToCurrentIdMap = new HashMap<>(); + for (int i = 0; i < currentProcessors.size(); i++) { + ProcessorSchema currentProcessor = currentProcessors.get(i); + ProcessorSchema v1Processor = v1Processors.get(i); + assertEquals(currentProcessor.getName(), v1Processor.getName()); + v1IdToCurrentIdMap.put(v1Processor.getId(), currentProcessor.getId()); + v1Processor.setId(currentProcessor.getId()); + } + configSchemaUpgradedFromV1.getRemoteProcessingGroups().stream().flatMap(g -> g.getInputPorts().stream()).map(RemoteInputPortSchema::getId).sequential() + .forEach(id -> v1IdToCurrentIdMap.put(id, id)); + + List<ConnectionSchema> currentConnections = configSchemaFromCurrent.getConnections(); + List<ConnectionSchema> v1Connections = configSchemaUpgradedFromV1.getConnections(); + + // Update source and dest ids, can set connection id equal because it isn't referenced elsewhere + assertEquals(currentConnections.size(), v1Connections.size()); + for (int i = 0; i < currentConnections.size(); i++) { + ConnectionSchema currentConnection = currentConnections.get(i); + ConnectionSchema v1Connection = v1Connections.get(i); + assertEquals(currentConnection.getName(), v1Connection.getName()); + v1Connection.setId(currentConnection.getId()); + v1Connection.setSourceId(v1IdToCurrentIdMap.get(v1Connection.getSourceId())); + v1Connection.setDestinationId(v1IdToCurrentIdMap.get(v1Connection.getDestinationId())); + } + Map<String, Object> v1YamlMap = configSchemaUpgradedFromV1.toMap(); + assertNoMapDifferences(v1YamlMap, configSchemaFromCurrent.toMap()); + } } private void assertNoMapDifferences(Map<String, Object> templateMap, Map<String, Object> yamlMap) { http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java index a8c1908..6ad7e03 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/BaseSchemaTester.java @@ -17,7 +17,7 @@ package org.apache.nifi.minifi.toolkit.configuration.dto; -import org.apache.nifi.minifi.commons.schema.common.BaseSchema; +import org.apache.nifi.minifi.commons.schema.common.WritableSchema; import org.junit.Test; import java.util.Map; @@ -25,7 +25,7 @@ import java.util.function.Function; import static org.junit.Assert.assertEquals; -public abstract class BaseSchemaTester<Schema extends BaseSchema, DTO> { +public abstract class BaseSchemaTester<Schema extends WritableSchema, DTO> { protected final Function<DTO, Schema> dtoSchemaFunction; protected final Function<Map, Schema> mapSchemaFunction; protected DTO dto; http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java index b4efa76..6aa77fb 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/ConnectionSchemaTest.java @@ -98,7 +98,7 @@ public class ConnectionSchemaTest extends BaseSchemaTester<ConnectionSchema, Con public void testNoSourceId() { dto.setSource(new ConnectableDTO()); map.remove(ConnectionSchema.SOURCE_ID_KEY); - assertDtoAndMapConstructorAreSame(2); + assertDtoAndMapConstructorAreSame(1); } @Test @@ -123,7 +123,7 @@ public class ConnectionSchemaTest extends BaseSchemaTester<ConnectionSchema, Con public void testNoDestinationName() { dto.setDestination(new ConnectableDTO()); map.remove(ConnectionSchema.DESTINATION_ID_KEY); - assertDtoAndMapConstructorAreSame(2); + assertDtoAndMapConstructorAreSame(1); } @Test http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml new file mode 100644 index 0000000..cee775a --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson-v1.yml @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: CsvToJsonWorking + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: ExtractText + class: org.apache.nifi.processors.standard.ExtractText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - unmatched + Properties: + Character Set: UTF-8 + Enable Canonical Equivalence: 'false' + Enable Case-insensitive Matching: 'false' + Enable DOTALL Mode: 'false' + Enable Literal Parsing of the Pattern: 'false' + Enable Multiline Mode: 'false' + Enable Unicode Predefined Character Classes: 'false' + Enable Unicode-aware Case Folding: 'false' + Enable Unix Lines Mode: 'false' + Include Capture Group 0: 'false' + Maximum Buffer Size: 1 MB + Maximum Capture Group Length: '1024' + Permit Whitespace and Comments in Pattern: 'false' + csv: (.+),(.+),(.+),(.+) +- name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 10 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 1 b + Unique FlowFiles: 'false' +- name: ReplaceText + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: UTF-8 + Evaluation Mode: Entire text + Maximum Buffer Size: 1 MB + Regular Expression: (?s:^.*$) + Replacement Strategy: + Replacement Value: a,b,c,d +- name: ReplaceText2 + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: UTF-8 + Evaluation Mode: Entire text + Maximum Buffer Size: 1 MB + Regular Expression: (?s:^.*$) + Replacement Strategy: + Replacement Value: "{ \"field1\" : \"${csv.1}\", \"field2\" : \"${csv.2}\",\n\ + \ \"field3\" : \"${csv.3}\", \"field4\" : \"${csv.4}\"\ + \ }\n " +- name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Delete Attributes Expression: +Connections: +- name: ExtractText/matched/ReplaceText2 + source name: ExtractText + source relationship name: matched + destination name: ReplaceText2 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GenerateFlowFile/success/ReplaceText + source name: GenerateFlowFile + source relationship name: success + destination name: ReplaceText + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: ReplaceText/success/ExtractText + source name: ReplaceText + source relationship name: success + destination name: ExtractText + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: ReplaceText2/success/UpdateAttribute + source name: ReplaceText2 + source relationship name: success + destination name: UpdateAttribute + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow-v1.yml new file mode 100644 index 0000000..743fdf6 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow-v1.yml @@ -0,0 +1,293 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: DecompressionCircularFlow2 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: Compressed? + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Routing Strategy: + gzip: ${mime.type:toUpper():contains("GZIP")} +- name: GZIP CompressContent + class: org.apache.nifi.processors.standard.CompressContent + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Compression Format: gzip + Compression Level: + Mode: + Update Filename: +- name: GZIP CompressContent2 + class: org.apache.nifi.processors.standard.CompressContent + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Compression Format: gzip + Compression Level: + Mode: + Update Filename: +- name: GZIP CompressContent3 + class: org.apache.nifi.processors.standard.CompressContent + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Compression Format: gzip + Compression Level: + Mode: + Update Filename: +- name: GZIP CompressContent4 + class: org.apache.nifi.processors.standard.CompressContent + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Compression Format: gzip + Compression Level: + Mode: + Update Filename: +- name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 min + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: + Data Format: + File Size: 10kb + Unique FlowFiles: +- name: IdentifyMimeType + class: org.apache.nifi.processors.standard.IdentifyMimeType + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: {} +- name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: + Log Payload: + Log prefix: +- name: Uncompress GZIP + class: org.apache.nifi.processors.standard.CompressContent + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Compression Format: gzip + Compression Level: + Mode: decompress + Update Filename: +Connections: +- name: Compressed?/gzip/Uncompress GZIP + source name: Compressed? + source relationship name: gzip + destination name: Uncompress GZIP + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Compressed?/unmatched/LogAttribute + source name: Compressed? + source relationship name: unmatched + destination name: LogAttribute + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent/failure/GZIP CompressContent + source name: GZIP CompressContent + source relationship name: failure + destination name: GZIP CompressContent + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent/success/GZIP CompressContent2 + source name: GZIP CompressContent + source relationship name: success + destination name: GZIP CompressContent2 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent2/failure/GZIP CompressContent2 + source name: GZIP CompressContent2 + source relationship name: failure + destination name: GZIP CompressContent2 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent2/success/GZIP CompressContent3 + source name: GZIP CompressContent2 + source relationship name: success + destination name: GZIP CompressContent3 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent3/failure/GZIP CompressContent3 + source name: GZIP CompressContent3 + source relationship name: failure + destination name: GZIP CompressContent3 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent3/success/GZIP CompressContent4 + source name: GZIP CompressContent3 + source relationship name: success + destination name: GZIP CompressContent4 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent4/failure/GZIP CompressContent4 + source name: GZIP CompressContent4 + source relationship name: failure + destination name: GZIP CompressContent4 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GZIP CompressContent4/success/Uncompress GZIP + source name: GZIP CompressContent4 + source relationship name: success + destination name: Uncompress GZIP + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: GenerateFlowFile/success/GZIP CompressContent + source name: GenerateFlowFile + source relationship name: success + destination name: GZIP CompressContent + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: IdentifyMimeType/success/Compressed? + source name: IdentifyMimeType + source relationship name: success + destination name: Compressed? + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Uncompress GZIP/failure/Uncompress GZIP + source name: Uncompress GZIP + source relationship name: failure + destination name: Uncompress GZIP + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 5 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Uncompress GZIP/success/IdentifyMimeType + source name: Uncompress GZIP + source relationship name: success + destination name: IdentifyMimeType + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml new file mode 100644 index 0000000..6a07ae4 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest-v1.yml @@ -0,0 +1,262 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: InvokeHttpMiNiFiTemplateTest2 + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- name: LogAttribute2 + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- name: Route On Status Code + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + '200': ${invokehttp.status.code:equals(200)} + Routing Strategy: Route to Property name +- name: Search Google + class: org.apache.nifi.processors.standard.InvokeHTTP + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Add Response Headers to Request: 'false' + Always Output Response: 'false' + Attributes to Send: + Basic Authentication Password: + Basic Authentication Username: + Connection Timeout: 5 secs + Content-Type: ${mime.type} + Digest Authentication: 'false' + Follow Redirects: 'True' + HTTP Method: GET + Include Date Header: 'True' + Max Length To Put In Attribute: '256' + Penalize on "No Retry": 'false' + Proxy Host: + Proxy Port: + Put Response Body In Attribute: + Read Timeout: 15 secs + Remote URL: http://www.google.com/search?q=${q}&rct=j + SSL Context Service: + Trusted Hostname: + Use Chunked Encoding: 'false' + invokehttp-proxy-password: + invokehttp-proxy-user: + send-message-body: 'true' +- name: Source + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 10 b + Unique FlowFiles: 'false' +- name: q=nifi + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + q: nifi +Connections: +- name: Route On Status Code/200/LogAttribute + source name: Route On Status Code + source relationship name: '200' + destination name: LogAttribute + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/200/b23a4621-cf19-42e6-967c-ffd3716e6a24 + source name: Route On Status Code + source relationship name: '200' + destination name: b23a4621-cf19-42e6-967c-ffd3716e6a24 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/unmatched/LogAttribute2 + source name: Route On Status Code + source relationship name: unmatched + destination name: LogAttribute2 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Route On Status Code/unmatched/b23a4621-cf19-42e6-967c-ffd3716e6a24 + source name: Route On Status Code + source relationship name: unmatched + destination name: b23a4621-cf19-42e6-967c-ffd3716e6a24 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Failure/Route On Status Code + source name: Search Google + source relationship name: Failure + destination name: Route On Status Code + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/No Retry/Route On Status Code + source name: Search Google + source relationship name: No Retry + destination name: Route On Status Code + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Original/Route On Status Code + source name: Search Google + source relationship name: Original + destination name: Route On Status Code + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Response/Route On Status Code + source name: Search Google + source relationship name: Response + destination name: Route On Status Code + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Search Google/Retry/Route On Status Code + source name: Search Google + source relationship name: Retry + destination name: Route On Status Code + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Source/success/q=nifi + source name: Source + source relationship name: success + destination name: q=nifi + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: q=nifi/success/Search Google + source name: q=nifi + source relationship name: success + destination name: Search Google + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: +- name: http://nifi:8080/nifi + url: http://nifi:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + Input Ports: + - id: b23a4621-cf19-42e6-967c-ffd3716e6a24 + name: response + comment: '' + max concurrent tasks: 1 + use compression: false http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml new file mode 100644 index 0000000..d0f1d66 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting-v1.yml @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: ReplaceTextExpressionLanguageCSVReformatting + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: Generate Empty File + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 3 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: + Data Format: + File Size: 0b + Unique FlowFiles: +- name: No-Op Termination + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Delete Attributes Expression: +- name: Reformat Date Column + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: + Evaluation Mode: Line-by-Line + Maximum Buffer Size: + Regular Expression: (.*?),(.*?),(\d+.*) + Replacement Strategy: + Replacement Value: $1,$2,${ '$3':toDate('ddMMMyyyy'):format('yyyy/MM/dd') } +- name: Set CSV Content + class: org.apache.nifi.processors.standard.ReplaceText + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - failure + Properties: + Character Set: + Evaluation Mode: + Maximum Buffer Size: + Regular Expression: + Replacement Strategy: + Replacement Value: |- + 2006,10-01-2004,10may2004 + 2007,15-05-2006,10jun2005 + 2009,8-8-2008,10aug2008 +Connections: +- name: Generate Empty File/success/Set CSV Content + source name: Generate Empty File + source relationship name: success + destination name: Set CSV Content + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Reformat Date Column/success/No-Op Termination + source name: Reformat Date Column + source relationship name: success + destination name: No-Op Termination + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: Set CSV Content/success/Reformat Date Column + source name: Set CSV Content + source relationship name: success + destination name: Reformat Date Column + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml new file mode 100644 index 0000000..970552b --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework-v1.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: StressTestFramework + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '100' + Data Format: + File Size: 0 KB + Unique FlowFiles: +- name: RouteOnAttribute + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 2 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 50000 + auto-terminated relationships list: + - unmatched + Properties: + Routing Strategy: +- name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 2 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 100000 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + property 1: value 1 + property 2: value 2 ${nextInt()} +Connections: +- name: GenerateFlowFile/success/UpdateAttribute + source name: GenerateFlowFile + source relationship name: success + destination name: UpdateAttribute + max work queue size: 200 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +- name: UpdateAttribute/success/RouteOnAttribute + source name: UpdateAttribute + source relationship name: success + destination name: RouteOnAttribute + max work queue size: 2000 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Processing Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-v1.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-v1.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-v1.yml new file mode 100644 index 0000000..2af6b9b --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config-v1.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Flow Controller: + name: MiNiFi Flow + comment: + +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 + +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 + +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false + +Provenance Repository: + provenance rollover time: 1 min + +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min + +Security Properties: + keystore: /tmp/ssl/localhost-ks.jks + keystore type: JKS + keystore password: localtest + key password: localtest + truststore: /tmp/ssl/localhost-ts.jks + truststore type: JKS + truststore password: localtest + ssl protocol: TLS + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC + +Processors: + - name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + Properties: + File to Tail: logs/minifi-app.log + Rolling Filename Pattern: minifi-app* + Initial Start Position: Beginning of File + +Connections: + - name: TailToS2S + source name: TailFile + source relationship name: success + destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer + +Remote Processing Groups: + - name: NiFi Flow + comment: + url: https://localhost:8090/nifi + timeout: 30 secs + yield period: 10 sec + Input Ports: + - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + name: tailed log + comments: + max concurrent tasks: 1 + use compression: false + +Provenance Reporting: + comment: + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + destination url: https://localhost:8090/ + port name: provenance + originating url: http://${hostname(true)}:8081/nifi + use compression: true + timeout: 30 secs + batch size: 1000 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/1bbeedf6/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml index 2af6b9b..64a28fe 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/config.yml @@ -13,99 +13,93 @@ # See the License for the specific language governing permissions and # limitations under the License. +MiNiFi Config Version: 2 Flow Controller: - name: MiNiFi Flow - comment: - + name: MiNiFi Flow + comment: '' Core Properties: - flow controller graceful shutdown period: 10 sec - flow service write delay interval: 500 ms - administrative yield duration: 30 sec - bored yield duration: 10 millis - max concurrent threads: 1 - + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 FlowFile Repository: - partitions: 256 - checkpoint interval: 2 mins - always sync: false - Swap: - threshold: 20000 - in period: 5 sec - in threads: 1 - out period: 5 sec - out threads: 4 - + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 Content Repository: - content claim max appendable size: 10 MB - content claim max flow files: 100 - always sync: false - + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false Provenance Repository: - provenance rollover time: 1 min - + provenance rollover time: 1 min Component Status Repository: - buffer size: 1440 - snapshot frequency: 1 min - + buffer size: 1440 + snapshot frequency: 1 min Security Properties: - keystore: /tmp/ssl/localhost-ks.jks - keystore type: JKS - keystore password: localtest - key password: localtest - truststore: /tmp/ssl/localhost-ts.jks - truststore type: JKS - truststore password: localtest - ssl protocol: TLS - Sensitive Props: - key: - algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL - provider: BC - + keystore: /tmp/ssl/localhost-ks.jks + keystore type: JKS + keystore password: localtest + key password: localtest + truststore: /tmp/ssl/localhost-ts.jks + truststore type: JKS + truststore password: localtest + ssl protocol: TLS + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC Processors: - - name: TailFile - class: org.apache.nifi.processors.standard.TailFile - max concurrent tasks: 1 - scheduling strategy: TIMER_DRIVEN - scheduling period: 1 sec - penalization period: 30 sec - yield period: 1 sec - run duration nanos: 0 - auto-terminated relationships list: - Properties: - File to Tail: logs/minifi-app.log - Rolling Filename Pattern: minifi-app* - Initial Start Position: Beginning of File - +- id: TailFile + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File to Tail: logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: minifi-app* Connections: - - name: TailToS2S - source name: TailFile - source relationship name: success - destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61 - max work queue size: 0 - max work queue data size: 1 MB - flowfile expiration: 60 sec - queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer - +- id: TailToS2S + name: TailToS2S + source id: TailFile + source relationship names: + - success + destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer Remote Processing Groups: - - name: NiFi Flow - comment: - url: https://localhost:8090/nifi - timeout: 30 secs - yield period: 10 sec - Input Ports: - - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 - name: tailed log - comments: - max concurrent tasks: 1 - use compression: false - +- name: NiFi Flow + url: https://localhost:8090/nifi + comment: '' + timeout: 30 secs + yield period: 10 sec + Input Ports: + - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + name: tailed log + comment: '' + max concurrent tasks: 1 + use compression: false Provenance Reporting: - comment: - scheduling strategy: TIMER_DRIVEN - scheduling period: 30 sec - destination url: https://localhost:8090/ - port name: provenance - originating url: http://${hostname(true)}:8081/nifi - use compression: true - timeout: 30 secs - batch size: 1000 \ No newline at end of file + comment: '' + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + destination url: https://localhost:8090/ + port name: provenance + originating url: http://${hostname(true)}:8081/nifi + use compression: true + timeout: 30 secs + batch size: 1000
