Repository: nifi-minifi Updated Branches: refs/heads/master ed801f32a -> 7b57cc816
MINIFI-216 - Allowing override of nifi.properties values This closes #72. Signed-off-by: Bryan Rosander <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/7b57cc81 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/7b57cc81 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/7b57cc81 Branch: refs/heads/master Commit: 7b57cc816c0a12eb64c8d6bed902c56626712373 Parents: ed801f3 Author: Bryan Rosander <[email protected]> Authored: Tue Feb 21 11:15:09 2017 -0500 Committer: Bryan Rosander <[email protected]> Committed: Tue Feb 21 14:38:15 2017 -0500 ---------------------------------------------------------------------- .../bootstrap/util/ConfigTransformer.java | 231 +++++++++---------- .../bootstrap/util/OrderedProperties.java | 78 +++++++ .../bootstrap/util/ConfigTransformerTest.java | 50 ++++ .../bootstrap/util/OrderedPropertiesTest.java | 55 +++++ .../src/test/resources/MINIFI-216/config.yml | 106 +++++++++ .../resources/MINIFI-216/configOverrides.yml | 110 +++++++++ .../resources/MINIFI-216/nifi.properties.before | 121 ++++++++++ .../orderedPropertiesExpected.properties | 21 ++ .../minifi/commons/schema/ConfigSchema.java | 14 ++ .../schema/common/CommonPropertyKeys.java | 1 + .../minifi/commons/schema/ConfigSchemaTest.java | 30 +++ .../src/main/markdown/System_Admin_Guide.md | 14 ++ .../src/main/resources/conf/config.yml | 3 +- .../src/test/resources/CsvToJson.yml | 1 + .../resources/DecompressionCircularFlow.yml | 1 + .../resources/InvokeHttpMiNiFiTemplateTest.yml | 1 + .../test/resources/MultipleRelationships.yml | 1 + .../test/resources/NestedControllerServices.yml | 1 + .../ProcessGroupsAndRemoteProcessGroups.yml | 1 + ...aceTextExpressionLanguageCSVReformatting.yml | 1 + .../src/test/resources/SimpleTailFileToRPG.yml | 1 + .../src/test/resources/StressTestFramework.yml | 1 + .../resources/StressTestFrameworkFunnel.yml | 3 +- 23 files changed, 722 insertions(+), 124 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 9c963f5..836214d 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -38,13 +38,13 @@ import org.apache.nifi.minifi.commons.schema.ProvenanceReportingSchema; import org.apache.nifi.minifi.commons.schema.ProvenanceRepositorySchema; import org.apache.nifi.minifi.commons.schema.RemoteInputPortSchema; import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; +import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; +import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; +import org.apache.nifi.minifi.commons.schema.SwapSchema; import org.apache.nifi.minifi.commons.schema.common.ConvertableSchema; import org.apache.nifi.minifi.commons.schema.common.Schema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; -import org.apache.nifi.minifi.commons.schema.SecurityPropertiesSchema; -import org.apache.nifi.minifi.commons.schema.SensitivePropsSchema; -import org.apache.nifi.minifi.commons.schema.SwapSchema; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -61,13 +61,10 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PrintWriter; -import java.io.UnsupportedEncodingException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -83,6 +80,7 @@ public final class ConfigTransformer { public static final String NIFI_VERSION = "1.1.0"; public static final String ROOT_GROUP = "Root-Group"; public static final String DEFAULT_PROV_REPORTING_TASK_CLASS = "org.apache.nifi.reporting.SiteToSiteProvenanceReportingTask"; + public static final String NIFI_VERSION_KEY = "nifi.version"; // Final util classes should have private constructor private ConfigTransformer() { @@ -150,11 +148,8 @@ public final class ConfigTransformer { } } - protected static void writeNiFiProperties(ConfigSchema configSchema, OutputStream outputStream) throws FileNotFoundException, UnsupportedEncodingException, ConfigurationChangeException { - PrintWriter writer = null; + protected static void writeNiFiProperties(ConfigSchema configSchema, OutputStream outputStream) throws IOException, ConfigurationChangeException { try { - writer = new PrintWriter(outputStream, true); - CorePropertiesSchema coreProperties = configSchema.getCoreProperties(); FlowFileRepositorySchema flowfileRepoSchema = configSchema.getFlowfileRepositoryProperties(); SwapSchema swapProperties = flowfileRepoSchema.getSwapProperties(); @@ -164,120 +159,112 @@ public final class ConfigTransformer { SensitivePropsSchema sensitiveProperties = securityProperties.getSensitiveProps(); ProvenanceRepositorySchema provenanceRepositorySchema = configSchema.getProvenanceRepositorySchema(); - writer.print(PROPERTIES_FILE_APACHE_2_0_LICENSE); - writer.println("# Core Properties #"); - writer.println(); - writer.println("nifi.version=" + NIFI_VERSION); - writer.println("nifi.flow.configuration.file=./conf/flow.xml.gz"); - writer.println("nifi.flow.configuration.archive.enabled=false"); - writer.println("nifi.flow.configuration.archive.dir=./conf/archive/"); - writer.println("nifi.flowcontroller.autoResumeState=true"); - writer.println("nifi.flowcontroller.graceful.shutdown.period=" + coreProperties.getFlowControllerGracefulShutdownPeriod()); - writer.println("nifi.flowservice.writedelay.interval=" + coreProperties.getFlowServiceWriteDelayInterval()); - writer.println("nifi.administrative.yield.duration=" + coreProperties.getAdministrativeYieldDuration()); - writer.println("# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?"); - writer.println("nifi.bored.yield.duration=" + coreProperties.getBoredYieldDuration()); - writer.println(); - writer.println("nifi.authority.provider.configuration.file=./conf/authority-providers.xml"); - writer.println("nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml"); - writer.println("nifi.templates.directory=./conf/templates"); - writer.println("nifi.ui.banner.text= "); - writer.println("nifi.ui.autorefresh.interval=30 sec"); - writer.println("nifi.nar.library.directory=./lib"); - writer.println("nifi.nar.working.directory=./work/nar/"); - writer.println("nifi.documentation.working.directory=./work/docs/components"); - writer.println(); - writer.println("####################"); - writer.println("# State Management #"); - writer.println("####################"); - writer.println("nifi.state.management.configuration.file=./conf/state-management.xml"); - writer.println("# The ID of the local state provider"); - writer.println("nifi.state.management.provider.local=local-provider"); - writer.println(); - writer.println("# H2 Settings"); - writer.println("nifi.database.directory=./database_repository"); - writer.println("nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE"); - writer.println(); - writer.println("# FlowFile Repository"); - writer.println("nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository"); - writer.println("nifi.flowfile.repository.directory=./flowfile_repository"); - writer.println("nifi.flowfile.repository.partitions=" + flowfileRepoSchema.getPartitions()); - writer.println("nifi.flowfile.repository.checkpoint.interval=" + flowfileRepoSchema.getCheckpointInterval()); - writer.println("nifi.flowfile.repository.always.sync=" + flowfileRepoSchema.getAlwaysSync()); - writer.println(); - writer.println("nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager"); - writer.println("nifi.queue.swap.threshold=" + swapProperties.getThreshold()); - writer.println("nifi.swap.in.period=" + swapProperties.getInPeriod()); - writer.println("nifi.swap.in.threads=" + swapProperties.getInThreads()); - writer.println("nifi.swap.out.period=" + swapProperties.getOutPeriod()); - writer.println("nifi.swap.out.threads=" + swapProperties.getOutThreads()); - writer.println(); - writer.println("# Content Repository"); - writer.println("nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository"); - writer.println("nifi.content.claim.max.appendable.size=" + contentRepoProperties.getContentClaimMaxAppendableSize()); - writer.println("nifi.content.claim.max.flow.files=" + contentRepoProperties.getContentClaimMaxFlowFiles()); - writer.println("nifi.content.repository.archive.max.retention.period="); - writer.println("nifi.content.repository.archive.max.usage.percentage="); - writer.println("nifi.content.repository.archive.enabled=false"); - writer.println("nifi.content.repository.directory.default=./content_repository"); - writer.println("nifi.content.repository.always.sync=" + contentRepoProperties.getAlwaysSync()); - writer.println(); - writer.println("# Provenance Repository Properties"); - writer.println("nifi.provenance.repository.implementation=org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository"); - writer.println("nifi.provenance.repository.rollover.time=" + provenanceRepositorySchema.getProvenanceRepoRolloverTimeKey()); - writer.println(); - writer.println("# Volatile Provenance Respository Properties"); - writer.println("nifi.provenance.repository.buffer.size=10000"); - writer.println(); - writer.println("# Component Status Repository"); - writer.println("nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository"); - writer.println("nifi.components.status.repository.buffer.size=" + componentStatusRepoProperties.getBufferSize()); - writer.println("nifi.components.status.snapshot.frequency=" + componentStatusRepoProperties.getSnapshotFrequency()); - writer.println(); - writer.println("# web properties #"); - writer.println("nifi.web.war.directory=./lib"); - writer.println("nifi.web.http.host="); - writer.println("nifi.web.http.port=8081"); - writer.println("nifi.web.https.host="); - writer.println("nifi.web.https.port="); - writer.println("nifi.web.jetty.working.directory=./work/jetty"); - writer.println("nifi.web.jetty.threads=200"); - writer.println(); - writer.println("# security properties #"); - writer.println("nifi.sensitive.props.key=" + sensitiveProperties.getKey()); - writer.println("nifi.sensitive.props.algorithm=" + sensitiveProperties.getAlgorithm()); - writer.println("nifi.sensitive.props.provider=" + sensitiveProperties.getProvider()); - writer.println(); - writer.println("nifi.security.keystore=" + securityProperties.getKeystore()); - writer.println("nifi.security.keystoreType=" + securityProperties.getKeystoreType()); - writer.println("nifi.security.keystorePasswd=" + securityProperties.getKeystorePassword()); - writer.println("nifi.security.keyPasswd=" + securityProperties.getKeyPassword()); - writer.println("nifi.security.truststore=" + securityProperties.getTruststore()); - writer.println("nifi.security.truststoreType=" + securityProperties.getTruststoreType()); - writer.println("nifi.security.truststorePasswd=" + securityProperties.getTruststorePassword()); - writer.println("nifi.security.needClientAuth="); - writer.println("nifi.security.user.credential.cache.duration=24 hours"); - writer.println("nifi.security.user.authority.provider=file-provider"); - writer.println("nifi.security.user.login.identity.provider="); - writer.println("nifi.security.support.new.account.requests="); - writer.println("# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI"); - writer.println("nifi.security.anonymous.authorities="); - writer.println("nifi.security.ocsp.responder.url="); - writer.println("nifi.security.ocsp.responder.certificate="); - writer.println(); - writer.println(); - writer.println("# cluster node properties (only configure for cluster nodes) #"); - writer.println("nifi.cluster.is.node=false"); - writer.println(); - writer.println("# cluster manager properties (only configure for cluster manager) #"); - writer.println("nifi.cluster.is.manager=false"); + OrderedProperties orderedProperties = new OrderedProperties(); + orderedProperties.setProperty(NIFI_VERSION_KEY, NIFI_VERSION,"# Core Properties #" + System.lineSeparator()); + orderedProperties.setProperty("nifi.flow.configuration.file", "./conf/flow.xml.gz"); + orderedProperties.setProperty("nifi.flow.configuration.archive.enabled", "false"); + orderedProperties.setProperty("nifi.flow.configuration.archive.dir", "./conf/archive/"); + orderedProperties.setProperty("nifi.flowcontroller.autoResumeState", "true"); + orderedProperties.setProperty("nifi.flowcontroller.graceful.shutdown.period", coreProperties.getFlowControllerGracefulShutdownPeriod()); + orderedProperties.setProperty("nifi.flowservice.writedelay.interval", coreProperties.getFlowServiceWriteDelayInterval()); + orderedProperties.setProperty("nifi.administrative.yield.duration", coreProperties.getAdministrativeYieldDuration()); + + orderedProperties.setProperty("nifi.bored.yield.duration", coreProperties.getBoredYieldDuration(), + "# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?"); + + orderedProperties.setProperty("nifi.authority.provider.configuration.file", "./conf/authority-providers.xml", ""); + orderedProperties.setProperty("nifi.login.identity.provider.configuration.file", "./conf/login-identity-providers.xml"); + orderedProperties.setProperty("nifi.templates.directory", "./conf/templates"); + orderedProperties.setProperty("nifi.ui.banner.text", ""); + orderedProperties.setProperty("nifi.ui.autorefresh.interval", "30 sec"); + orderedProperties.setProperty("nifi.nar.library.directory", "./lib"); + orderedProperties.setProperty("nifi.nar.working.directory", "./work/nar/"); + orderedProperties.setProperty("nifi.documentation.working.directory", "./work/docs/components"); + + orderedProperties.setProperty("nifi.state.management.configuration.file", "./conf/state-management.xml", System.lineSeparator() + + "####################" + + "# State Management #" + + "####################"); + + orderedProperties.setProperty("nifi.state.management.provider.local", "local-provider", "# The ID of the local state provider"); + + orderedProperties.setProperty("nifi.database.directory", "./database_repository", System.lineSeparator() + "# H2 Settings"); + orderedProperties.setProperty("nifi.h2.url.append", ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE"); + orderedProperties.setProperty("nifi.flowfile.repository.implementation", "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository", + System.lineSeparator() + "# FlowFile Repository"); + orderedProperties.setProperty("nifi.flowfile.repository.directory", "./flowfile_repository"); + orderedProperties.setProperty("nifi.flowfile.repository.partitions", String.valueOf(flowfileRepoSchema.getPartitions())); + orderedProperties.setProperty("nifi.flowfile.repository.checkpoint.interval", flowfileRepoSchema.getCheckpointInterval()); + orderedProperties.setProperty("nifi.flowfile.repository.always.sync", Boolean.toString(flowfileRepoSchema.getAlwaysSync())); + + orderedProperties.setProperty("nifi.swap.manager.implementation", "org.apache.nifi.controller.FileSystemSwapManager", ""); + orderedProperties.setProperty("nifi.queue.swap.threshold", String.valueOf(swapProperties.getThreshold())); + orderedProperties.setProperty("nifi.swap.in.period", swapProperties.getInPeriod()); + orderedProperties.setProperty("nifi.swap.in.threads", String.valueOf(swapProperties.getInThreads())); + orderedProperties.setProperty("nifi.swap.out.period", swapProperties.getOutPeriod()); + orderedProperties.setProperty("nifi.swap.out.threads", String.valueOf(swapProperties.getOutThreads())); + + orderedProperties.setProperty("nifi.content.repository.implementation", "org.apache.nifi.controller.repository.FileSystemRepository", System.lineSeparator() + "# Content Repository"); + orderedProperties.setProperty("nifi.content.claim.max.appendable.size", contentRepoProperties.getContentClaimMaxAppendableSize()); + orderedProperties.setProperty("nifi.content.claim.max.flow.files", String.valueOf(contentRepoProperties.getContentClaimMaxFlowFiles())); + orderedProperties.setProperty("nifi.content.repository.archive.max.retention.period", ""); + orderedProperties.setProperty("nifi.content.repository.archive.max.usage.percentage", ""); + orderedProperties.setProperty("nifi.content.repository.archive.enabled", "false"); + orderedProperties.setProperty("nifi.content.repository.directory.default", "./content_repository"); + orderedProperties.setProperty("nifi.content.repository.always.sync", Boolean.toString(contentRepoProperties.getAlwaysSync())); + + orderedProperties.setProperty("nifi.provenance.repository.implementation", "org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository", + System.lineSeparator() + "# Provenance Repository Properties"); + orderedProperties.setProperty("nifi.provenance.repository.rollover.time", provenanceRepositorySchema.getProvenanceRepoRolloverTimeKey()); + + orderedProperties.setProperty("nifi.provenance.repository.buffer.size", "10000", System.lineSeparator() + "# Volatile Provenance Respository Properties"); + + orderedProperties.setProperty("nifi.components.status.repository.implementation", "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository", + System.lineSeparator() + "# Component Status Repository"); + orderedProperties.setProperty("nifi.components.status.repository.buffer.size", String.valueOf(componentStatusRepoProperties.getBufferSize())); + orderedProperties.setProperty("nifi.components.status.snapshot.frequency", componentStatusRepoProperties.getSnapshotFrequency()); + + orderedProperties.setProperty("nifi.web.war.directory", "./lib", System.lineSeparator() + "# web properties #"); + orderedProperties.setProperty("nifi.web.http.host", ""); + orderedProperties.setProperty("nifi.web.http.port", "8081"); + orderedProperties.setProperty("nifi.web.https.host", ""); + orderedProperties.setProperty("nifi.web.https.port", ""); + orderedProperties.setProperty("nifi.web.jetty.working.directory", "./work/jetty"); + orderedProperties.setProperty("nifi.web.jetty.threads", "200"); + + orderedProperties.setProperty("nifi.sensitive.props.key", sensitiveProperties.getKey(), System.lineSeparator() + "# security properties #"); + orderedProperties.setProperty("nifi.sensitive.props.algorithm", sensitiveProperties.getAlgorithm()); + orderedProperties.setProperty("nifi.sensitive.props.provider", sensitiveProperties.getProvider()); + + orderedProperties.setProperty("nifi.security.keystore", securityProperties.getKeystore(), ""); + orderedProperties.setProperty("nifi.security.keystoreType", securityProperties.getKeystoreType()); + orderedProperties.setProperty("nifi.security.keystorePasswd", securityProperties.getKeystorePassword()); + orderedProperties.setProperty("nifi.security.keyPasswd", securityProperties.getKeyPassword()); + orderedProperties.setProperty("nifi.security.truststore", securityProperties.getTruststore()); + orderedProperties.setProperty("nifi.security.truststoreType", securityProperties.getTruststoreType()); + orderedProperties.setProperty("nifi.security.truststorePasswd", securityProperties.getTruststorePassword()); + orderedProperties.setProperty("nifi.security.needClientAuth", ""); + orderedProperties.setProperty("nifi.security.user.credential.cache.duration", "24 hours"); + orderedProperties.setProperty("nifi.security.user.authority.provider", "file-provider"); + orderedProperties.setProperty("nifi.security.user.login.identity.provider", ""); + orderedProperties.setProperty("nifi.security.support.new.account.requests", ""); + + orderedProperties.setProperty("nifi.security.anonymous.authorities", "", "# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI"); + orderedProperties.setProperty("nifi.security.ocsp.responder.url", ""); + orderedProperties.setProperty("nifi.security.ocsp.responder.certificate", ""); + + orderedProperties.setProperty("nifi.cluster.is.node", "false", System.lineSeparator() + System.lineSeparator() + "# cluster node properties (only configure for cluster nodes) #"); + orderedProperties.setProperty("nifi.cluster.is.manager", "false", System.lineSeparator() + "# cluster manager properties (only configure for cluster manager) #"); + + for (Map.Entry<String, String> entry : configSchema.getNifiPropertiesOverrides().entrySet()) { + orderedProperties.setProperty(entry.getKey(), entry.getValue()); + } + + orderedProperties.store(outputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); } catch (NullPointerException e) { throw new ConfigurationChangeException("Failed to parse the config YAML while creating the nifi.properties", e); } finally { - if (writer != null) { - writer.flush(); - writer.close(); - } + outputStream.close(); } } @@ -704,7 +691,7 @@ public final class ConfigTransformer { } public static final String PROPERTIES_FILE_APACHE_2_0_LICENSE = - "# Licensed to the Apache Software Foundation (ASF) under one or more\n" + + " Licensed to the Apache Software Foundation (ASF) under one or more\n" + "# contributor license agreements. See the NOTICE file distributed with\n" + "# this work for additional information regarding copyright ownership.\n" + "# The ASF licenses this file to You under the Apache License, Version 2.0\n" + http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java new file mode 100644 index 0000000..d7496fa --- /dev/null +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/OrderedProperties.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.util; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +public class OrderedProperties extends Properties { + private final Map<String, String> textBeforeMap = new HashMap<>(); + private final LinkedHashSet<Object> linkedHashSet = new LinkedHashSet<>(); + + @Override + public synchronized Object put(Object key, Object value) { + linkedHashSet.add(key); + return super.put(key, value); + } + + @Override + public synchronized Enumeration<Object> keys() { + return Collections.enumeration(linkedHashSet.stream().filter(this::containsKey).collect(Collectors.toList())); + } + + public synchronized Object setProperty(String key, String value, String textBefore) { + textBeforeMap.put(key, textBefore); + return setProperty(key, value); + } + + @Override + public synchronized void store(OutputStream out, String comments) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + super.store(byteArrayOutputStream, comments); + + try(BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()), "8859_1")); + BufferedWriter bufferedWriter = new BufferedWriter(new OutputStreamWriter(out, "8859_1"))) { + String line; + while ((line = bufferedReader.readLine()) != null) { + int equalsIndex = line.indexOf('='); + if (equalsIndex != -1) { + String textBefore = textBeforeMap.get(line.substring(0, equalsIndex)); + if (textBefore != null) { + bufferedWriter.write(textBefore); + bufferedWriter.newLine(); + } + } + bufferedWriter.write(line); + bufferedWriter.newLine(); + } + } + } +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index 2cb3a9c..38fc673 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -28,6 +28,7 @@ import org.apache.nifi.minifi.commons.schema.ProcessorSchema; import org.apache.nifi.minifi.commons.schema.RemoteInputPortSchema; import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; import org.apache.nifi.minifi.commons.schema.common.StringUtil; +import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException; import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader; import org.junit.Before; import org.junit.Test; @@ -44,6 +45,8 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -51,11 +54,13 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class ConfigTransformerTest { @@ -134,6 +139,51 @@ public class ConfigTransformerTest { testConfigFileTransform("InvokeHttpMiNiFiProxyPasswordTemplateTest.yml"); } + @Test + public void testNifiPropertiesNoOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { + Properties pre216Properties = new Properties(); + try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { + pre216Properties.load(pre216PropertiesStream); + } + pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/config.yml")) { + ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); + } + Properties properties = new Properties(); + properties.load(new ByteArrayInputStream(outputStream.toByteArray())); + + for (String name : pre216Properties.stringPropertyNames()) { + assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name)); + } + } + + @Test + public void testNifiPropertiesOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { + Properties pre216Properties = new Properties(); + try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { + pre216Properties.load(pre216PropertiesStream); + } + pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/configOverrides.yml")) { + ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); + assertTrue(configSchema.getNifiPropertiesOverrides().size() > 0); + for (Map.Entry<String, String> entry : configSchema.getNifiPropertiesOverrides().entrySet()) { + pre216Properties.setProperty(entry.getKey(), entry.getValue()); + } + ConfigTransformer.writeNiFiProperties(configSchema, outputStream); + } + Properties properties = new Properties(); + properties.load(new ByteArrayInputStream(outputStream.toByteArray())); + + for (String name : pre216Properties.stringPropertyNames()) { + assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name)); + } + } + public void testConfigFileTransform(String configFile) throws Exception { ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(ConfigTransformerTest.class.getClassLoader().getResourceAsStream(configFile)); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java new file mode 100644 index 0000000..bc9c4bc --- /dev/null +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/OrderedPropertiesTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.minifi.bootstrap.util; + +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; + +import static org.apache.nifi.minifi.bootstrap.util.ConfigTransformer.PROPERTIES_FILE_APACHE_2_0_LICENSE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class OrderedPropertiesTest { + @Test + public void testOrderedProperties() throws IOException { + OrderedProperties orderedProperties = new OrderedProperties(); + orderedProperties.setProperty("prop1", "origVal1"); + orderedProperties.setProperty("prop2", "val2", "#this is property 2"); + orderedProperties.setProperty("prop3", "val3"); + orderedProperties.setProperty("prop1", "newVal1"); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + orderedProperties.store(byteArrayOutputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); + + try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(byteArrayOutputStream.toByteArray()))); + BufferedReader expectedReader = new BufferedReader(new InputStreamReader(OrderedPropertiesTest.class.getClassLoader().getResourceAsStream("orderedPropertiesExpected.properties")))) { + String expectedLine; + while((expectedLine = expectedReader.readLine()) != null) { + String actualLine = bufferedReader.readLine(); + if (!"#Tue Feb 21 11:03:08 EST 2017".equals(expectedLine)) { + assertEquals(expectedLine, actualLine); + } + } + assertNull(bufferedReader.readLine()); + } + } +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/resources/MINIFI-216/config.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/MINIFI-216/config.yml b/minifi-bootstrap/src/test/resources/MINIFI-216/config.yml new file mode 100644 index 0000000..8886205 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/MINIFI-216/config.yml @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: MiNiFi Flow + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: /tmp/ssl/localhost-ks.jks + keystore type: JKS + keystore password: localtest + key password: localtest + truststore: /tmp/ssl/localhost-ts.jks + truststore type: JKS + truststore password: localtest + ssl protocol: TLS + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File to Tail: logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: minifi-app* +Connections: +- id: f6cef9b0-8982-391c-8d70-76f33917ac12 + name: TailToS2S + source id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + source relationship names: + - success + destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer +Remote Process Groups: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e3 + name: NiFi Flow + url: https://localhost:8090/nifi + comment: '' + timeout: 30 secs + yield period: 10 sec + Input Ports: + - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + name: tailed log + comment: '' + max concurrent tasks: 1 + use compression: false +Provenance Reporting: + comment: '' + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + destination url: https://localhost:8090/ + port name: provenance + originating url: http://${hostname(true)}:8081/nifi + use compression: true + timeout: 30 secs + batch size: 1000 http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/resources/MINIFI-216/configOverrides.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/MINIFI-216/configOverrides.yml b/minifi-bootstrap/src/test/resources/MINIFI-216/configOverrides.yml new file mode 100644 index 0000000..45673b0 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/MINIFI-216/configOverrides.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: MiNiFi Flow + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: /tmp/ssl/localhost-ks.jks + keystore type: JKS + keystore password: localtest + key password: localtest + truststore: /tmp/ssl/localhost-ts.jks + truststore type: JKS + truststore password: localtest + ssl protocol: TLS + Sensitive Props: + key: '' + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File to Tail: logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: minifi-app* +Connections: +- id: f6cef9b0-8982-391c-8d70-76f33917ac12 + name: TailToS2S + source id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e2 + source relationship names: + - success + destination id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + max work queue size: 0 + max work queue data size: 1 MB + flowfile expiration: 60 sec + queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer +Remote Process Groups: +- id: 94b8e610-b4ed-3ec9-b26f-c839931bf3e3 + name: NiFi Flow + url: https://localhost:8090/nifi + comment: '' + timeout: 30 secs + yield period: 10 sec + Input Ports: + - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61 + name: tailed log + comment: '' + max concurrent tasks: 1 + use compression: false +Provenance Reporting: + comment: '' + scheduling strategy: TIMER_DRIVEN + scheduling period: 30 sec + destination url: https://localhost:8090/ + port name: provenance + originating url: http://${hostname(true)}:8081/nifi + use compression: true + timeout: 30 secs + batch size: 1000 +NiFi Properties Overrides: + nifi.flowfile.repository.directory: ./flowfile_repository_override + nifi.content.repository.directory.default: ./content_repository_override + nifi.database.directory: ./database_repository_override http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before b/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before new file mode 100644 index 0000000..22b02ad --- /dev/null +++ b/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # + +nifi.version=1.1.0 +nifi.flow.configuration.file=./conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=false +nifi.flow.configuration.archive.dir=./conf/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authority.provider.configuration.file=./conf/authority-providers.xml +nifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml +nifi.templates.directory=./conf/templates +nifi.ui.banner.text= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./lib +nifi.nar.working.directory=./work/nar/ +nifi.documentation.working.directory=./work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider + +# H2 Settings +nifi.database.directory=./database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.archive.max.retention.period= +nifi.content.repository.archive.max.usage.percentage= +nifi.content.repository.archive.enabled=false +nifi.content.repository.directory.default=./content_repository +nifi.content.repository.always.sync=false + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository +nifi.provenance.repository.rollover.time=1 min + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=10000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# web properties # +nifi.web.war.directory=./lib +nifi.web.http.host= +nifi.web.http.port=8081 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore=/tmp/ssl/localhost-ks.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=localtest +nifi.security.keyPasswd=localtest +nifi.security.truststore=/tmp/ssl/localhost-ts.jks +nifi.security.truststoreType=JKS +nifi.security.truststorePasswd=localtest +nifi.security.needClientAuth= +nifi.security.user.credential.cache.duration=24 hours +nifi.security.user.authority.provider=file-provider +nifi.security.user.login.identity.provider= +nifi.security.support.new.account.requests= +# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI +nifi.security.anonymous.authorities= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-bootstrap/src/test/resources/orderedPropertiesExpected.properties ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/orderedPropertiesExpected.properties b/minifi-bootstrap/src/test/resources/orderedPropertiesExpected.properties new file mode 100644 index 0000000..9e5dca1 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/orderedPropertiesExpected.properties @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +#Tue Feb 21 11:03:08 EST 2017 +prop1=newVal1 +#this is property 2 +prop2=val2 +prop3=val3 http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java index 39e5a2f..1d06974 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/ConfigSchema.java @@ -25,6 +25,7 @@ import org.apache.nifi.minifi.commons.schema.common.WritableSchema; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -35,6 +36,7 @@ import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CO import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.CORE_PROPS_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOWFILE_REPO_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.FLOW_CONTROLLER_PROPS_KEY; +import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPORTING_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.PROVENANCE_REPO_KEY; import static org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys.SECURITY_PROPS_KEY; @@ -66,6 +68,8 @@ public class ConfigSchema extends BaseSchema implements WritableSchema, Converta private ProvenanceRepositorySchema provenanceRepositorySchema; + private Map<String, String> nifiPropertiesOverrides; + public ConfigSchema(Map map) { this(map, Collections.emptyList()); } @@ -85,6 +89,11 @@ public class ConfigSchema extends BaseSchema implements WritableSchema, Converta provenanceReportingProperties = getMapAsType(map, PROVENANCE_REPORTING_KEY, ProvenanceReportingSchema.class, TOP_LEVEL_NAME, false, false); + nifiPropertiesOverrides = (Map<String, String>) map.get(NIFI_PROPERTIES_OVERRIDES_KEY); + if (nifiPropertiesOverrides == null) { + nifiPropertiesOverrides = new HashMap<>(); + } + addIssuesIfNotNull(flowControllerProperties); addIssuesIfNotNull(coreProperties); addIssuesIfNotNull(flowfileRepositoryProperties); @@ -160,6 +169,7 @@ public class ConfigSchema extends BaseSchema implements WritableSchema, Converta putIfNotNull(result, SECURITY_PROPS_KEY, securityProperties); result.putAll(processGroupSchema.toMap()); putIfNotNull(result, PROVENANCE_REPORTING_KEY, provenanceReportingProperties); + result.put(NIFI_PROPERTIES_OVERRIDES_KEY, nifiPropertiesOverrides); return result; } @@ -199,6 +209,10 @@ public class ConfigSchema extends BaseSchema implements WritableSchema, Converta return provenanceRepositorySchema; } + public Map<String, String> getNifiPropertiesOverrides() { + return nifiPropertiesOverrides; + } + @Override public int getVersion() { return CONFIG_VERSION; http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CommonPropertyKeys.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CommonPropertyKeys.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CommonPropertyKeys.java index 05ad607..c52f7fe 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CommonPropertyKeys.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/common/CommonPropertyKeys.java @@ -38,6 +38,7 @@ public class CommonPropertyKeys { public static final String CONTROLLER_SERVICES_KEY = "Controller Services"; public static final String FUNNELS_KEY = "Funnels"; public static final String PROVENANCE_REPO_KEY = "Provenance Repository"; + public static final String NIFI_PROPERTIES_OVERRIDES_KEY = "NiFi Properties Overrides"; public static final String NAME_KEY = "name"; http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java index 0037bce..dfb1fac 100644 --- a/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java +++ b/minifi-commons/minifi-commons-schema/src/test/java/org/apache/nifi/minifi/commons/schema/ConfigSchemaTest.java @@ -27,6 +27,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -110,6 +111,35 @@ public class ConfigSchemaTest { ConfigSchema.CONNECTION_WITH_ID + connection.get(ID_KEY) + ConfigSchema.HAS_INVALID_SOURCE_ID + fakeSource)), validationIssues); } + @Test + public void testNullNifiPropertyOverrides() { + ConfigSchema configSchema = new ConfigSchema(new HashMap<>()); + assertEquals(Collections.emptyMap(), configSchema.getNifiPropertiesOverrides()); + assertEquals(Collections.emptyMap(), configSchema.toMap().get(CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY)); + } + + @Test + public void testEmptyNifiPropertyOverrides() { + Map<Object, Object> map = new HashMap<>(); + map.put(CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY, new HashMap<>()); + ConfigSchema configSchema = new ConfigSchema(map); + assertEquals(Collections.emptyMap(), configSchema.getNifiPropertiesOverrides()); + assertEquals(Collections.emptyMap(), configSchema.toMap().get(CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY)); + } + + @Test + public void testNifiPropertyOverrides() { + Map<Object, Object> map = new HashMap<>(); + HashMap<Object, Object> overrides = new HashMap<>(); + overrides.put("nifi.flowfile.repository.directory", "./flowfile_repository_override"); + overrides.put("nifi.content.repository.directory.default", "./content_repository_override"); + overrides.put("nifi.database.directory", "./database_repository_override"); + map.put(CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY, new HashMap<>(overrides)); + ConfigSchema configSchema = new ConfigSchema(map); + assertEquals(overrides, configSchema.getNifiPropertiesOverrides()); + assertEquals(overrides, configSchema.toMap().get(CommonPropertyKeys.NIFI_PROPERTIES_OVERRIDES_KEY)); + } + public static List<Map<String, Object>> getListWithNames(String... names) { return getListWithKeyValues(CommonPropertyKeys.NAME_KEY, names); } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-docs/src/main/markdown/System_Admin_Guide.md ---------------------------------------------------------------------- diff --git a/minifi-docs/src/main/markdown/System_Admin_Guide.md b/minifi-docs/src/main/markdown/System_Admin_Guide.md index f2bcd40..6d7f14e 100644 --- a/minifi-docs/src/main/markdown/System_Admin_Guide.md +++ b/minifi-docs/src/main/markdown/System_Admin_Guide.md @@ -349,6 +349,7 @@ parses and upconverts to the current version without issue. ### Version 2 -> Version 3 changes 1. Added support for Controller Services. 2. Added support for Site-To-Site over proxy. +3. Added support for overriding nifi.properties values ## Flow Controller @@ -628,6 +629,19 @@ batch size | Specifies how many records to send in a single batch, at **Note:** In order to send via HTTPS, the "Security Properties" must be fully configured. A StandardSSLContextService will be made automatically with the ID "SSL-Context-Service" and used by the Provenance Reporting. +## NiFi Properties Overrides + +This is a yaml map that contains values to be put into nifi.properties. This will supercede any hardcoded or other schema values that are substituted into nifi.properties file. + +### Example NiFi Properties Overrides + +```yaml +NiFi Properties Overrides: + nifi.flowfile.repository.directory: ./flowfile_repository_override + nifi.content.repository.directory.default: ./content_repository_override + nifi.database.directory: ./database_repository_override +``` + # Example Config File Below are two example config YAML files. The first tails the minifi-app.log, send the tailed log and provenance data back to a secure instance of NiFi. The second uses a series of processors to tail the app log, routes off only lines that contain "WriteAheadFlowFileRepository" and puts it as a file in the "./" directory. http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml ---------------------------------------------------------------------- diff --git a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml index 5bcf11d..1a7f872 100644 --- a/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml +++ b/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/config.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -MiNiFi Config Version: 2 +MiNiFi Config Version: 3 Flow Controller: name: MiNiFi Flow comment: '' @@ -60,3 +60,4 @@ Process Groups: [] Funnels: [] Connections: [] Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml index ad0d0ac..1d6a8a2 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/CsvToJson.yml @@ -195,3 +195,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml index f93877e..8e0f8dc 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/DecompressionCircularFlow.yml @@ -334,3 +334,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml index 5fcaa46..d7def45 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/InvokeHttpMiNiFiTemplateTest.yml @@ -300,3 +300,4 @@ Remote Process Groups: comment: '' max concurrent tasks: 1 use compression: false +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml index 264b68d..8c6455a 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleRelationships.yml @@ -152,3 +152,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: '' Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml index 964b784..526e55d 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/NestedControllerServices.yml @@ -288,3 +288,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: '' Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml index 2a071ae..8203cac 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ProcessGroupsAndRemoteProcessGroups.yml @@ -292,3 +292,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: '' Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml index ec14cff..9985966 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/ReplaceTextExpressionLanguageCSVReformatting.yml @@ -160,3 +160,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml index 6946a8b..09859b7 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/SimpleTailFileToRPG.yml @@ -116,3 +116,4 @@ Remote Process Groups: comment: '' max concurrent tasks: 1 use compression: false +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml index 8bf6ef9..87fa189 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFramework.yml @@ -125,3 +125,4 @@ Connections: flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer Remote Process Groups: [] +NiFi Properties Overrides: {} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7b57cc81/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml index 1106a85..2502cdb 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/StressTestFrameworkFunnel.yml @@ -209,4 +209,5 @@ Connections: max work queue data size: 0 MB flowfile expiration: 0 sec queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer -Remote Process Groups: [] \ No newline at end of file +Remote Process Groups: [] +NiFi Properties Overrides: {}
