AMBARI-18385: Add HDF management pack (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/37e71db7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/37e71db7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/37e71db7 Branch: refs/heads/branch-dev-patch-upgrade Commit: 37e71db741cacb5acc4113131a27d2c1b7ac5791 Parents: 532b689 Author: Jayush Luniya <[email protected]> Authored: Tue Sep 13 22:26:38 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Tue Sep 13 22:26:38 2016 -0700 ---------------------------------------------------------------------- .../hdf-ambari-mpack/.gitignore | 1 + .../management-packs/hdf-ambari-mpack/pom.xml | 150 ++ .../src/main/assemblies/hdf-ambari-mpack.xml | 79 + .../common-services/NIFI/1.0.0/alerts.json | 18 + .../1.0.0/configuration/nifi-ambari-config.xml | 189 ++ .../configuration/nifi-ambari-ssl-config.xml | 226 +++ .../configuration/nifi-authorizers-env.xml | 103 + .../1.0.0/configuration/nifi-bootstrap-env.xml | 116 ++ ...nifi-bootstrap-notification-services-env.xml | 80 + .../NIFI/1.0.0/configuration/nifi-env.xml | 116 ++ .../NIFI/1.0.0/configuration/nifi-flow-env.xml | 69 + .../nifi-login-identity-providers-env.xml | 147 ++ .../configuration/nifi-node-logback-env.xml | 201 ++ .../1.0.0/configuration/nifi-properties.xml | 848 ++++++++ .../configuration/nifi-state-management-env.xml | 96 + .../1.0.0/configuration/ranger-nifi-audit.xml | 91 + .../ranger-nifi-plugin-properties.xml | 170 ++ .../configuration/ranger-nifi-policymgr-ssl.xml | 84 + .../configuration/ranger-nifi-security.xml | 90 + .../common-services/NIFI/1.0.0/kerberos.json | 72 + .../common-services/NIFI/1.0.0/metainfo.xml | 149 ++ .../common-services/NIFI/1.0.0/metrics.json | 131 ++ .../1.0.0/package/scripts/alert_check_nifi.py | 49 + .../NIFI/1.0.0/package/scripts/nifi.py | 231 +++ .../NIFI/1.0.0/package/scripts/nifi_ca.py | 114 ++ .../NIFI/1.0.0/package/scripts/nifi_ca_util.py | 110 ++ .../1.0.0/package/scripts/nifi_constants.py | 25 + .../NIFI/1.0.0/package/scripts/params.py | 445 +++++ .../NIFI/1.0.0/package/scripts/run_ca.sh | 63 + .../NIFI/1.0.0/package/scripts/service_check.py | 75 + .../1.0.0/package/scripts/setup_ranger_nifi.py | 68 + .../NIFI/1.0.0/package/scripts/status_params.py | 29 + .../NIFI/1.0.0/package/templates/nifi.conf.j2 | 36 + .../NIFI/1.0.0/quicklinks/quicklinks.json | 35 + .../common-services/NIFI/1.0.0/widgets.json | 463 +++++ .../src/main/resources/hooks/after_install.py | 49 + .../src/main/resources/mpack.json | 30 + .../HDF/2.0/configuration/cluster-env.xml | 210 ++ .../HDF/2.0/hooks/after-INSTALL/scripts/hook.py | 37 + .../2.0/hooks/after-INSTALL/scripts/params.py | 101 + .../scripts/shared_initialization.py | 108 ++ .../hooks/before-ANY/files/changeToSecureUid.sh | 53 + .../HDF/2.0/hooks/before-ANY/scripts/hook.py | 36 + .../HDF/2.0/hooks/before-ANY/scripts/params.py | 235 +++ .../before-ANY/scripts/shared_initialization.py | 224 +++ .../2.0/hooks/before-INSTALL/scripts/hook.py | 37 + .../2.0/hooks/before-INSTALL/scripts/params.py | 113 ++ .../scripts/repo_initialization.py | 68 + .../scripts/shared_initialization.py | 37 + .../2.0/hooks/before-RESTART/scripts/hook.py | 29 + .../hooks/before-START/files/checkForFormat.sh | 65 + .../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 19285850 bytes .../before-START/files/task-log4j.properties | 134 ++ .../hooks/before-START/files/topology_script.py | 66 + .../HDF/2.0/hooks/before-START/scripts/hook.py | 39 + .../2.0/hooks/before-START/scripts/params.py | 318 +++ .../before-START/scripts/rack_awareness.py | 47 + .../scripts/shared_initialization.py | 175 ++ .../templates/commons-logging.properties.j2 | 43 + .../templates/exclude_hosts_list.j2 | 21 + .../templates/hadoop-metrics2.properties.j2 | 104 + .../before-START/templates/health_check.j2 | 81 + .../templates/include_hosts_list.j2 | 21 + .../templates/topology_mappings.data.j2 | 24 + .../main/resources/stacks/HDF/2.0/kerberos.json | 64 + .../main/resources/stacks/HDF/2.0/metainfo.xml | 23 + .../HDF/2.0/properties/stack_features.json | 281 +++ .../stacks/HDF/2.0/properties/stack_tools.json | 4 + .../resources/stacks/HDF/2.0/repos/repoinfo.xml | 104 + .../stacks/HDF/2.0/role_command_order.json | 55 + .../HDF/2.0/services/AMBARI_INFRA/metainfo.xml | 26 + .../AMBARI_INFRA/role_command_order.json | 7 + .../2.0/services/AMBARI_METRICS/metainfo.xml | 26 + .../KAFKA/configuration/ranger-kafka-audit.xml | 77 + .../ranger-kafka-policymgr-ssl.xml | 37 + .../stacks/HDF/2.0/services/KAFKA/kerberos.json | 69 + .../stacks/HDF/2.0/services/KAFKA/metainfo.xml | 27 + .../HDF/2.0/services/KERBEROS/metainfo.xml | 26 + .../HDF/2.0/services/LOGSEARCH/metainfo.xml | 26 + .../services/LOGSEARCH/role_command_order.json | 10 + .../stacks/HDF/2.0/services/NIFI/metainfo.xml | 27 + .../RANGER/configuration/ranger-admin-site.xml | 45 + .../RANGER/configuration/ranger-env.xml | 66 + .../configuration/ranger-tagsync-site.xml | 72 + .../RANGER/configuration/ranger-ugsync-site.xml | 59 + .../tagsync-application-properties.xml | 39 + .../stacks/HDF/2.0/services/RANGER/metainfo.xml | 42 + .../services/RANGER/themes/theme_version_4.json | 480 +++++ .../STORM/configuration/ranger-storm-audit.xml | 36 + .../ranger-storm-policymgr-ssl.xml | 37 + .../configuration/ranger-storm-security.xml | 30 + .../services/STORM/configuration/storm-site.xml | 62 + .../stacks/HDF/2.0/services/STORM/metainfo.xml | 28 + .../HDF/2.0/services/ZOOKEEPER/metainfo.xml | 51 + .../stacks/HDF/2.0/services/stack_advisor.py | 1828 ++++++++++++++++++ .../main/resources/stacks/HDF/2.0/widgets.json | 95 + contrib/management-packs/pom.xml | 3 + pom.xml | 1 + 98 files changed, 11337 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/.gitignore ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/.gitignore b/contrib/management-packs/hdf-ambari-mpack/.gitignore new file mode 100644 index 0000000..1377554 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/.gitignore @@ -0,0 +1 @@ +*.swp http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/pom.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/pom.xml b/contrib/management-packs/hdf-ambari-mpack/pom.xml new file mode 100644 index 0000000..4764fa2 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/pom.xml @@ -0,0 +1,150 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>org.apache.ambari</groupId> + <artifactId>hdf-ambari-mpack</artifactId> + <packaging>pom</packaging> + <version>0.1.0.0-SNAPSHOT</version> + <name>HDF Ambari Management Pack</name> + <url>http://ambari.apache.org/</url> + <properties> + <minAmbariVersion>2.4.0.0</minAmbariVersion> + <maxAmbariVersion></maxAmbariVersion> + <nifiversion>1.0.0</nifiversion> + </properties> + <parent> + <groupId>org.apache.ambari.contrib.mpacks</groupId> + <artifactId>ambari-contrib-mpacks</artifactId> + <version>2.0.0.0-SNAPSHOT</version> + </parent> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>parse-version</id> + <phase>validate</phase> + <goals> + <goal>parse-version</goal> + </goals> + </execution> + <execution> + <id>regex-property</id> + <goals> + <goal>regex-property</goal> + </goals> + <configuration> + <name>mpackVersion</name> + <value>${project.version}</value> + <regex>^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).*</regex> + <replacement>$1.$2.$3.$4</replacement> + <failIfNoMatch>false</failIfNoMatch> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>unpack</id> + <phase>process-resources</phase> + <goals> + <goal>unpack</goal> + </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-toolkit-assembly</artifactId> + <classifier>bin</classifier> + <type>zip</type> + <overWrite>true</overWrite> + <outputDirectory>${project.build.directory}/nifi-toolkit</outputDirectory> + </artifactItem> + </artifactItems> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>2.6</version> + <executions> + <execution> + <id>copy-resources</id> + <phase>compile</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${basedir}/target/</outputDirectory> + <resources> + <resource> + <directory>${basedir}/src/main/resources</directory> + <includes> + <include>mpack.json</include> + </includes> + <filtering>true</filtering> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <tarLongFileMode>gnu</tarLongFileMode> + <descriptors> + <descriptor>src/main/assemblies/hdf-ambari-mpack.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>build-tarball</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencies> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-toolkit-assembly</artifactId> + <version>${nifiversion}</version> + <classifier>bin</classifier> + <type>zip</type> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>3.8.1</version> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml new file mode 100644 index 0000000..2df8075 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/assemblies/hdf-ambari-mpack.xml @@ -0,0 +1,79 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<assembly> + <id></id> + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>true</includeBaseDirectory> + <!-- File sets. Syntax: + <fileSets> + <fileSet> + <useDefaultExcludes/> + <outputDirectory/> + <includes/> + <excludes/> + <fileMode/> + <directoryMode/> + <directory/> + <lineEnding/> + <filtered/> + </fileSet> + </fileSets> + --> + <fileSets> + <fileSet> + <directory>src/main/resources/hooks</directory> + <outputDirectory>hooks</outputDirectory> + </fileSet> + <fileSet> + <directory>src/main/resources/common-services</directory> + <outputDirectory>common-services</outputDirectory> + </fileSet> + <fileSet> + <directory>src/main/resources/stacks</directory> + <outputDirectory>stacks</outputDirectory> + </fileSet> + <fileSet> + <directory>${project.build.directory}/nifi-toolkit</directory> + <outputDirectory>common-services/NIFI/1.0.0/package/files</outputDirectory> + </fileSet> + </fileSets> + <!-- Single files. Syntax: + <files> + <file> + <source/> + <outputDirectory/> + <destName/> + <fileMode/> + <lineEnding/> + <filtered/> + </file> + </files> + --> + <files> + <file> + <source>target/mpack.json</source> + </file> + </files> + <dependencySets> + </dependencySets> +</assembly> + http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json new file mode 100644 index 0000000..9d065a0 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/alerts.json @@ -0,0 +1,18 @@ +{ + "NIFI": { + "service": [], + "NIFI_MASTER": [ + { + "name": "nifi_status", + "label": "Nifi Status", + "description": "This host-level alert is triggered if the Nifi service cannot be determined to be up and responding to client requests.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "NIFI/1.0.0/package/scripts/alert_check_nifi.py" + } + } + ] + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml new file mode 100644 index 0000000..743fda0 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-config.xml @@ -0,0 +1,189 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + + + <!-- Note: these two properties must be in same xml file for quicklinks to correctly work --> + <property> + <name>nifi.node.port</name> + <value>9090</value> + <display-name>Nifi HTTP port (non-SSL)</display-name> + <description>HTTP port NiFi node runs on when SSL is not enabled</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.node.ssl.port</name> + <value>9091</value> + <display-name>Nifi HTTP port (SSL)</display-name> + <description>HTTP port NiFi node runs on when SSL is enabled</description> + <on-ambari-upgrade add="true"/> + </property> + + <!-- End Note --> + + <property> + <name>nifi.node.protocol.port</name> + <value>9088</value> + <display-name>Nifi protocol port</display-name> + <description>HTTP port NiFi node protocol runs on</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.initial_mem</name> + <value>512m</value> + <display-name>Initial memory allocation</display-name> + <description>Initial memory allocation for NiFi JVM (Xms)</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.max_mem</name> + <value>512m</value> + <display-name>Max memory allocation</display-name> + <description>Max memory allocation for Nifi JVM (Xmx)</description> + <on-ambari-upgrade add="true"/> + </property> + + + <property> + <name>nifi.ambari_reporting_frequency</name> + <value>1 mins</value> + <display-name>Metrics reporting frequency</display-name> + <description>Frequency with which Nifi metrics will be reported to Ambari Metrics. Only used during initial install of service. The existence of nifi_znode is used to determine whether initial install or not. Post-install this can be reconfigured from Nifi UI under Controller Settings</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.ambari_reporting_enabled</name> + <value>true</value> + <display-name>Enable metrics reporting to Ambari?</display-name> + <description>Whether to setup reporting of Nifi metrics to Ambari Metrics. Only used during initial install of service. The existence of nifi_znode is used to determine whether initial install or not. Post-install this can be reconfigured from Nifi UI under Controller Settings</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.nifi_znode</name> + <value>/nifi</value> + <display-name>Zookeeper znode</display-name> + <description>Zookeeper znode for Nifi</description> + <on-ambari-upgrade add="true"/> + </property> + + + <property> + <name>nifi.internal.dir</name> + <value>/var/lib/nifi</value> + <display-name>Nifi internal dir</display-name> + <description>Internal directory for Nifi (used to store archive, templates, work dirs). This directory will be created (if not already existing)</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.config.dir</name> + <value>{nifi_install_dir}/conf</value> + <display-name>Nifi config dir</display-name> + <description>The directory to store config files (nifi.properties, bootstrap.conf etc). This directory will be created (if not already existing)</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.flow.config.dir</name> + <value>{nifi_internal_dir}/conf</value> + <display-name>Nifi internal config dir</display-name> + <description>The directory to store the state, flow.xml.gz and authorizations.xml files. This directory will be created (if not already existing)</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.state.dir</name> + <value>{nifi_internal_dir}/state/local</value> + <display-name>Nifi internal state dir</display-name> + <description>The directory to store the state. This directory will be created (if not already existing)</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.database.dir</name> + <value>/var/lib/nifi/database_repository</value> + <display-name>Nifi H2 database dir</display-name> + <description>The directory for the H2 database repository. The H2 database keeps track of user access and flow controller history. This directory will be created (if not already existing)</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.flowfile.repository.dir</name> + <value>/var/lib/nifi/flowfile_repository</value> + <display-name>Nifi flowfile repository dir</display-name> + <description>The directory for the flowfile repository. This directory will be created (if not already existing)</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.content.repository.dir.default</name> + <value>/var/lib/nifi/content_repository</value> + <display-name>Nifi content repository default dir</display-name> + <description>The default directory for the content repository. This directory will be created (if not already existing). NOTE: Multiple content repositories can be specified by using the nifi.content.repository.directory. prefix with unique suffixes and separate paths as values. For example, to provide two additional locations to act as part of the content repository, a user could also specify additional properties with keys of: nifi.content.repository.directory.content1=/repos/content1</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi.provenance.repository.dir.default</name> + <value>/var/lib/nifi/provenance_repository</value> + <display-name>Nifi provenance repository default dir</display-name> + <description>The default directory for the provenance repository. This directory will be created (if not already existing). NOTE: Multiple provenance repositories can be specified by using the nifi.provenance.repository.directory. prefix with unique suffixes and separate paths as values. For example, to provide two additional locations to act as part of the provenance repository, a user could also specify additional properties with keys of: nifi.provenance.repository.directory.content1=/repos/provenance1</description> + <value-attributes> + <type>directory</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + <property require-input="true"> + <name>nifi.sensitive.props.key</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>Sensitive property values encryption password</display-name> + <description> + This is the password used to encrypt any sensitive property values that are configured in processors. + It can be a string of any length, although the recommended minimum length is 10 characters. + Be aware that once this password is set and one or more sensitive processor properties have been configured, this password should not be changed. + </description> + <value-attributes> + <overridable>false</overridable> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml new file mode 100644 index 0000000..272535f --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-ambari-ssl-config.xml @@ -0,0 +1,226 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + + <property> + <name>nifi.node.ssl.isenabled</name> + <value>false</value> + <display-name>Enable SSL?</display-name> + <description> + Check flag to enable SSL. A few additional properties are also required (depending on your setup). + Assuming NIFI's Certificate Authority (CA) was installed: If Ranger auth will be used, only 'Nifi CA Token' is required - otherwise 'Nifi CA Token', 'Initial Admin Identity', 'Node identities' are all required. + Assuming CA is not installed: If Ranger auth will be used, only Truststore/Keystore paths/type/passwords should be set - otherwise Truststore/Keystore paths/type/passwords as well as 'Initial Admin Identity','Node identities' are required. + </description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nifi.toolkit.tls.regenerate</name> + <value>false</value> + <display-name>NiFi CA Force Regenerate?</display-name> + <description>Will force regeneration of Tls Certificates and Configuration</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property require-input="true"> + <name>nifi.toolkit.tls.token</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>NiFi CA Token</display-name> + <description> + This is a token that will be used by the NiFi Certificate Authority to verify the identity of NiFi nodes before issuing them certificates and by the NiFi nodes + to verify the identity of the NiFi Certificate Authority. + + If relying on NiFi Certificate Authority, set this to a long, random value. + </description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nifi.toolkit.tls.helper.days</name> + <value>1095</value> + <on-ambari-upgrade add="true"/> + <display-name>NiFi CA Certificate Duration</display-name> + <description>The number of days that a certificate from the NiFi Certificate Authority should be valid for.</description> + </property> + <property> + <name>nifi.toolkit.tls.port</name> + <value>10443</value> + <on-ambari-upgrade add="true"/> + <display-name>NiFi Certificate Authority port</display-name> + <description>Port for NiFi Certificate Authority to listen on.</description> + </property> + <property> + <name>nifi.toolkit.dn.prefix</name> + <value>CN=</value> + <on-ambari-upgrade add="true"/> + <display-name>NiFi CA DN prefix</display-name> + <description>Portion of the distinguished name to put before the hostname in TLS certificates (Note, NiFi will sort the DN attributes according to the order they appear in https://www.ietf.org/rfc/rfc2253.txt)</description> + </property> + <property> + <name>nifi.toolkit.dn.suffix</name> + <value>, OU=NIFI</value> + <on-ambari-upgrade add="true"/> + <display-name>NiFi CA DN suffix</display-name> + <description>Portion of the distinguished name to put after the hostname in TLS certificates (Note, NiFi will sort the DN attributes according to the order they appear in https://www.ietf.org/rfc/rfc2253.txt)</description> + </property> + <property> + <name>nifi.security.keystore</name> + <value>{{nifi_config_dir}}/keystore.jks</value> + <on-ambari-upgrade add="true"/> + <display-name>Keystore path</display-name> + <description>The full path and name of the keystore. To reference paths that include hostnames you can use format such as: /etc/security/nifi-certs/{nifi_node_ssl_host}/{nifi_node_ssl_host}.jks</description> + </property> + <property> + <name>nifi.security.keystoreType</name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + <display-name>Keystore type</display-name> + <description>The keystore type. It is blank by default but required if nifi.node.ssl.isenabled=true. e.g. jks</description> + </property> + <property> + <name>nifi.security.keystorePasswd</name> + <value></value> + <property-type>PASSWORD</property-type> + <display-name>Keystore password</display-name> + <description>The keystore password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nifi.security.keyPasswd</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>Key password</display-name> + <description>The key password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nifi.security.truststore</name> + <value>{{nifi_config_dir}}/truststore.jks</value> + <on-ambari-upgrade add="true"/> + <display-name>Truststore path</display-name> + <description>The full path and name of the truststore. To reference paths that include hostnames you can use format such as: /etc/security/nifi-certs/{nifi_node_ssl_host}/truststore.jks</description> + </property> + <property> + <name>nifi.security.truststoreType</name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + <display-name>Truststore type</display-name> + <description>The truststore type. It is blank by default but required if nifi.node.ssl.isenabled=true. e.g. jks</description> + </property> + <property> + <name>nifi.security.truststorePasswd</name> + <value/> + <property-type>PASSWORD</property-type> + <display-name>Truststore password</display-name> + <description>The truststore password. It is blank by default but required if nifi.node.ssl.isenabled=true.</description> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + <type>password</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>nifi.security.needClientAuth</name> + <value>false</value> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + <value-attributes> + <type>boolean</type> + </value-attributes> + <display-name>Clients need to authenticate?</display-name> + <description>Specifies whether or not connecting clients must authenticate themselves. Specifically this property is used by the NiFi cluster protocol. If the Truststore properties are not set, this must be false. Otherwise, a value of true indicates that nodes in the cluster will be authenticated and must have certificates that are trusted by the Truststores.</description> + </property> + + <property> + <name>nifi.initial.admin.identity</name> + <value/> + <value-attributes> + <empty-value-valid>true</empty-value-valid> + </value-attributes> + <on-ambari-upgrade add="true"/> + <display-name>Initial Admin Identity</display-name> + <description> + The identity of an initial admin user that will be granted access to the UI and + given the ability to create additional users, groups, and policies. Must be specified when Ranger plugin for + Nifi will not be used for authorization. + Before changing value of this field, note that it will only take effect if authorizations.xml does not contain any existing policies. + The value of this property could be + a DN when using certificates or LDAP, or a Kerberos principal. This property will only be used when there + are no other users, groups, and policies defined. If this property is specified then a Legacy Authorized + Users File can not be specified. + </description> + </property> + + <property> + <name>content</name> + <display-name>Node Identities</display-name> + <description> + Node Identity [unique key] - The identity of a NiFi cluster node. When clustered, a property for each node + should be defined, so that every node knows about every other node. If not clustered these properties can be ignored. + Also must be specified when Ranger Nifi plugin will not be used for authorization. + Before changing value of this field, note that it will only take effect if authorizations.xml does not contain any existing policies. + The name of each property must be unique, for example for a three node cluster: + "Node Identity A", "Node Identity B", "Node Identity C" or "Node Identity 1", "Node Identity 2", "Node Identity 3" + NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities, + so the values should be the unmapped identities (i.e. full DN from a certificate). + </description> + <value> +<!-- Provide the identity (typically a DN) of each node when clustered (see tool tip for detailed description of Node Identity). Must be specified when Ranger Nifi plugin will not be used for authorization. --> + +<!-- +<property name="Node Identity 1"></property> +<property name="Node Identity 2"></property> +<property name="Node Identity 3"></property> +<property name="Node Identity 4"></property> +--> + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml new file mode 100644 index 0000000..e6505aa --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-authorizers-env.xml @@ -0,0 +1,103 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>content</name> + <display-name>Template for authorizers.xml</display-name> + <description>This is the jinja template for authorizers.xml</description> + <value> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + <!-- + This file lists the authority providers to use when running securely. In order + to use a specific provider it must be configured here and it's identifier + must be specified in the nifi.properties file. + --> + <authorizers> + + <!-- + The FileAuthorizer is NiFi"s provided authorizer and has the following properties: + - Authorizations File - The file where the FileAuthorizer will store policies. + - Users File - The file where the FileAuthorizer will store users and groups. + - Initial Admin Identity - The identity of an initial admin user that will be granted access to the UI and + given the ability to create additional users, groups, and policies. The value of this property could be + a DN when using certificates or LDAP, or a Kerberos principal. This property will only be used when there + are no other users, groups, and policies defined. If this property is specified then a Legacy Authorized + Users File can not be specified. + NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the initial admin identity, + so the value should be the unmapped identity. + - Legacy Authorized Users File - The full path to an existing authorized-users.xml that will be automatically + converted to the new authorizations model. If this property is specified then an Initial Admin Identity can + not be specified, and this property will only be used when there are no other users, groups, and policies defined. + - Node Identity [unique key] - The identity of a NiFi cluster node. When clustered, a property for each node + should be defined, so that every node knows about every other node. If not clustered these properties can be ignored. + The name of each property must be unique, for example for a three node cluster: + "Node Identity A", "Node Identity B", "Node Identity C" or "Node Identity 1", "Node Identity 2", "Node Identity 3" + NOTE: Any identity mapping rules specified in nifi.properties will also be applied to the node identities, + so the values should be the unmapped identities (i.e. full DN from a certificate). + --> + <authorizer> + <identifier>{{nifi_authorizer}}</identifier> + {% if has_ranger_admin and enable_ranger_nifi %} + <class>org.apache.nifi.ranger.authorization.RangerNiFiAuthorizer</class> + <property name="Ranger Audit Config Path">{{nifi_config_dir}}/ranger-nifi-audit.xml</property> + <property name="Ranger Security Config Path">{{nifi_config_dir}}/ranger-nifi-security.xml</property> + <property name="Ranger Service Type">nifi</property> + <property name="Ranger Application Id">nifi</property> + <property name="Allow Anonymous">true</property> + <property name="Ranger Admin Identity">{{ranger_admin_identity}}</property> + {% if security_enabled %} + <property name="Ranger Kerberos Enabled">true</property> + {% else %} + <property name="Ranger Kerberos Enabled">false</property> + {% endif %} + {% else %} + <class>org.apache.nifi.authorization.FileAuthorizer</class> + <property name="Authorizations File">{{nifi_flow_config_dir}}/authorizations.xml</property> + <property name="Users File">{{nifi_flow_config_dir}}/users.xml</property> + <property name="Initial Admin Identity">{{nifi_initial_admin_id}}</property> + <property name="Legacy Authorized Users File"></property> + {% endif %} + + {{nifi_ssl_config_content}} + + </authorizer> + </authorizers> + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml new file mode 100644 index 0000000..ea4e919 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-env.xml @@ -0,0 +1,116 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + + <property> + <name>content</name> + <display-name>Template for bootstrap.conf</display-name> + <value> +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Java command to use when running NiFi +java=java + +# Username to use when running NiFi. This value will be ignored on Windows. +run.as={{nifi_user}} +##run.as=root + +# Configure where NiFi's lib and conf directories live +lib.dir={{nifi_install_dir}}/lib +conf.dir={{nifi_config_dir}} + +# How long to wait after telling NiFi to shutdown before explicitly killing the Process +graceful.shutdown.seconds=20 + +# Disable JSR 199 so that we can use JSP's without running a JDK +java.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true + +# JVM memory settings +java.arg.2=-Xms{{nifi_initial_mem}} +java.arg.3=-Xmx{{nifi_max_mem}} + +# Enable Remote Debugging +#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000 + +java.arg.4=-Djava.net.preferIPv4Stack=true + +# allowRestrictedHeaders is required for Cluster/Node communications to work properly +java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true +java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol + +# The G1GC is still considered experimental but has proven to be very advantageous in providing great +# performance without significant "stop-the-world" delays. +java.arg.13=-XX:+UseG1GC + +#Set headless mode by default +java.arg.14=-Djava.awt.headless=true + +#Ambari Metrics Collector URL - passed in to flow.xml for AmbariReportingTask +java.arg.15=-Dambari.metrics.collector.url=http://{{metrics_collector_host}}:{{metrics_collector_port}}/ws/v1/timeline/metrics + +#Application ID - used in flow.xml - passed into flow.xml for AmbariReportingTask +java.arg.16=-Dambari.application.id=nifi + + +### +# Notification Services for notifying interested parties when NiFi is stopped, started, dies +### + +# XML File that contains the definitions of the notification services +notification.services.file={{nifi_config_dir}}/bootstrap-notification-services.xml + +# In the case that we are unable to send a notification for an event, how many times should we retry? +notification.max.attempts=5 + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started? +#nifi.start.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped? +#nifi.stop.notification.services=email-notification + +# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies? +#nifi.dead.notification.services=email-notification + + </value> + <description>Template for bootstrap.conf</description> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> + http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml new file mode 100644 index 0000000..3c5b535 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-bootstrap-notification-services-env.xml @@ -0,0 +1,80 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>content</name> + <display-name>Template for bootstrap-notification-services.xml</display-name> + <description>This is the jinja template for bootstrap-notification-services.xml</description> + <value> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<services> + <!-- This file is used to define how interested parties are notified when events in NiFi's lifecycle occur. --> + <!-- The format of this file is: + <services> + <service> + <id>service-identifier</id> + <class>org.apache.nifi.notifications.DesiredNotificationService</class> + <property name="property name">property value</property> + <property name="another property">another property value</property> + </service> + </services> + + This file can contain 0 to many different service definitions. + The id can then be referenced from the bootstrap.conf file in order to configure the notification service + to be used when particular lifecycle events occur. + --> + +<!-- + <service> + <id>email-notification</id> + <class>org.apache.nifi.bootstrap.notification.email.EmailNotificationService</class> + <property name="SMTP Hostname"></property> + <property name="SMTP Port"></property> + <property name="SMTP Username"></property> + <property name="SMTP Password"></property> + <property name="SMTP TLS"></property> + <property name="From"></property> + <property name="To"></property> + </service> +--> +</services> + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml new file mode 100644 index 0000000..76de1c0 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-env.xml @@ -0,0 +1,116 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + + <property> + <name>nifi_node_log_dir</name> + <value>/var/log/nifi</value> + <display-name>NiFi Log dir</display-name> + <description>NiFi Log dir</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi_pid_dir</name> + <value>/var/run/nifi</value> + <display-name>NiFi PID dir</display-name> + <description>Dir containing process ID file</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi_user</name> + <value>nifi</value> + <property-type>USER</property-type> + <display-name>Nifi User</display-name> + <description>User nifi daemon runs as</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi_group</name> + <value>nifi</value> + <property-type>GROUP</property-type> + <display-name>Nifi Group</display-name> + <description>nifi group</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi_user_nofile_limit</name> + <value>50000</value> + <display-name>Open files limit for Nifi</display-name> + <description>Max open files (nofile) limit setting for Nifi user.</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>nifi_user_nproc_limit</name> + <value>10000</value> + <display-name>Process limit for Nifi</display-name> + <description>Max number of processes (nproc) limit setting for Nifi user.</description> + <on-ambari-upgrade add="true"/> + </property> + + <property> + <name>content</name> + <display-name>Template for nifi-env.sh</display-name> + <value> +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The java implementation to use. +export JAVA_HOME={{jdk64_home}} + +export NIFI_HOME={{nifi_install_dir}} + +#The directory for the NiFi pid file +export NIFI_PID_DIR="{{nifi_pid_dir}}" + +#The directory for NiFi log files +export NIFI_LOG_DIR="{{nifi_node_log_dir}}" + + + </value> + <description>Template for nifi-env.sh</description> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> + http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml new file mode 100644 index 0000000..f01164a --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-flow-env.xml @@ -0,0 +1,69 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>content</name> + <display-name>Template for flow.xml</display-name> + <value> +<flowController> + <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount> + <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount> + <rootGroup> + <id>7c84501d-d10c-407c-b9f3-1d80e38fe36a</id> + <name>NiFi Flow</name> + <position x="0.0" y="0.0"/> + <comment/> + </rootGroup> + <controllerServices/> + <reportingTasks> + <reportingTask> + <id>3b80ba0f-a6c0-48db-b721-4dbc04cef28e</id> + <name>AmbariReportingTask</name> + <comment/> + <class>org.apache.nifi.reporting.ambari.AmbariReportingTask</class> + <schedulingPeriod>{{nifi_ambari_reporting_frequency}}</schedulingPeriod> + <scheduledState>RUNNING</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <property> + <name>Metrics Collector URL</name> + <value>${ambari.metrics.collector.url}</value> + </property> + <property> + <name>Application ID</name> + <value>${ambari.application.id}</value> + </property> + <property> + <name>Hostname</name> + <value>${hostname(true)}</value> + </property> + </reportingTask> + </reportingTasks> +</flowController> + </value> + <description>Initial template for flow.xml. Only used during initial install of Nifi service. Post-install this can be reconfigured from Nifi UI under Controller Settings</description> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> + http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml new file mode 100644 index 0000000..636ecdb --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-login-identity-providers-env.xml @@ -0,0 +1,147 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + <property> + <name>content</name> + <description>This is the jinja template for login-identity-providers.xml</description> + <display-name>Template for login-identity-providers.xml</display-name> + <value> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!-- + This file lists the login identity providers to use when running securely. In order + to use a specific provider it must be configured here and it's identifier + must be specified in the nifi.properties file. +--> +<loginIdentityProviders> + <!-- + Identity Provider for users logging in with username/password against an LDAP server. + + 'Authentication Strategy' - How the connection to the LDAP server is authenticated. Possible + values are ANONYMOUS, SIMPLE, or START_TLS. + + 'Manager DN' - The DN of the manager that is used to bind to the LDAP server to search for users. + 'Manager Password' - The password of the manager that is used to bind to the LDAP server to + search for users. + + 'TLS - Keystore' - Path to the Keystore that is used when connecting to LDAP using START_TLS. + 'TLS - Keystore Password' - Password for the Keystore that is used when connecting to LDAP + using START_TLS. + 'TLS - Keystore Type' - Type of the Keystore that is used when connecting to LDAP using + START_TLS (i.e. JKS or PKCS12). + 'TLS - Truststore' - Path to the Truststore that is used when connecting to LDAP using START_TLS. + 'TLS - Truststore Password' - Password for the Truststore that is used when connecting to + LDAP using START_TLS. + 'TLS - Truststore Type' - Type of the Truststore that is used when connecting to LDAP using + START_TLS (i.e. JKS or PKCS12). + 'TLS - Client Auth' - Client authentication policy when connecting to LDAP using START_TLS. + Possible values are REQUIRED, WANT, NONE. + 'TLS - Protocol' - Protocol to use when connecting to LDAP using START_TLS. (i.e. TLS, + TLSv1.1, TLSv1.2, etc). + 'TLS - Shutdown Gracefully' - Specifies whether the TLS should be shut down gracefully + before the target context is closed. Defaults to false. + + 'Referral Strategy' - Strategy for handling referrals. Possible values are FOLLOW, IGNORE, THROW. + 'Connect Timeout' - Duration of connect timeout. (i.e. 10 secs). + 'Read Timeout' - Duration of read timeout. (i.e. 10 secs). + + 'Url' - Url of the LDAP servier (i.e. ldap://<hostname>:<port>). + 'User Search Base' - Base DN for searching for users (i.e. CN=Users,DC=example,DC=com). + 'User Search Filter' - Filter for searching for users against the 'User Search Base'. + (i.e. sAMAccountName={0}). The user specified name is inserted into '{0}'. + + 'Authentication Expiration' - The duration of how long the user authentication is valid + for. If the user never logs out, they will be required to log back in following + this duration. + --> + <!-- To enable the ldap-provider remove 2 lines. This is 1 of 2. + <provider> + <identifier>ldap-provider</identifier> + <class>org.apache.nifi.ldap.LdapProvider</class> + <property name="Authentication Strategy">START_TLS</property> + + <property name="Manager DN"></property> + <property name="Manager Password"></property> + + <property name="TLS - Keystore"></property> + <property name="TLS - Keystore Password"></property> + <property name="TLS - Keystore Type"></property> + <property name="TLS - Truststore"></property> + <property name="TLS - Truststore Password"></property> + <property name="TLS - Truststore Type"></property> + <property name="TLS - Client Auth"></property> + <property name="TLS - Protocol"></property> + <property name="TLS - Shutdown Gracefully"></property> + + <property name="Referral Strategy">FOLLOW</property> + <property name="Connect Timeout">10 secs</property> + <property name="Read Timeout">10 secs</property> + + <property name="Url"></property> + <property name="User Search Base"></property> + <property name="User Search Filter"></property> + + <property name="Authentication Expiration">12 hours</property> + </provider> + To enable the ldap-provider remove 2 lines. This is 2 of 2. --> + + <!-- + Identity Provider for users logging in with username/password against a Kerberos KDC server. + + 'Default Realm' - Default realm to provide when user enters incomplete user principal (i.e. NIFI.APACHE.ORG). + 'Authentication Expiration' - The duration of how long the user authentication is valid for. If the user never logs out, they will be required to log back in following this duration. + --> + {% if not security_enabled %} + <!-- To enable the kerberos-provider remove 2 lines. This is 1 of 2. + {% endif %} + <provider> + <identifier>kerberos-provider</identifier> + <class>org.apache.nifi.kerberos.KerberosProvider</class> + <property name="Default Realm">{{nifi_kerberos_realm}}</property> + <property name="Authentication Expiration">{{nifi_kerberos_authentication_expiration}}</property> + </provider> + {% if not security_enabled %} + To enable the kerberos-provider remove 2 lines. This is 2 of 2. --> + {% endif %} + +</loginIdentityProviders> + + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml new file mode 100644 index 0000000..7989c05 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/configuration/nifi-node-logback-env.xml @@ -0,0 +1,201 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<configuration> + + + <property> + <name>content</name> + <display-name>Template for logback.xml</display-name> + <value> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration scan="true" scanPeriod="30 seconds"> + <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator"> + <resetJUL>true</resetJUL> + </contextListener> + + <appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- + For daily rollover, use 'app_%d.log'. + For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'. + To GZIP rolled files, replace '.log' with '.log.gz'. + To ZIP rolled files, replace '.log' with '.log.zip'. + --> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log.gz</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!-- keep 30 log files worth of history --> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + <immediateFlush>true</immediateFlush> + </encoder> + </appender> + + <appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- + For daily rollover, use 'user_%d.log'. + For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'. + To GZIP rolled files, replace '.log' with '.log.gz'. + To ZIP rolled files, replace '.log' with '.log.zip'. + --> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log.gz</fileNamePattern> + <!-- keep 30 log files worth of history --> + <maxHistory>30</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + </encoder> + </appender> + + <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- + For daily rollover, use 'user_%d.log'. + For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'. + To GZIP rolled files, replace '.log' with '.log.gz'. + To ZIP rolled files, replace '.log' with '.log.zip'. + --> + <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log.gz</fileNamePattern> + <!-- keep 5 log files worth of history --> + <maxHistory>5</maxHistory> + </rollingPolicy> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + </encoder> + </appender> + + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%date %level [%thread] %logger{40} %msg%n</pattern> + </encoder> + </appender> + + <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR --> + + <logger name="org.apache.nifi" level="INFO"/> + <logger name="org.apache.nifi.processors" level="WARN"/> + <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/> + <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" /> + + + <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" /> + <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" /> + <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" /> + <logger name="org.apache.zookeeper.server.quorum" level="ERROR" /> + <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" /> + + <!-- Logger for managing logging statements for nifi clusters. --> + <logger name="org.apache.nifi.cluster" level="INFO"/> + + <!-- Logger for logging HTTP requests received by the web server. --> + <logger name="org.apache.nifi.server.JettyServer" level="INFO"/> + + <!-- Logger for managing logging statements for jetty --> + <logger name="org.eclipse.jetty" level="INFO"/> + + <!-- Suppress non-error messages due to excessive logging by class or library --> + <logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/> + <logger name="com.sun.jersey.spi.spring" level="ERROR"/> + <logger name="org.springframework" level="ERROR"/> + + <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) --> + <logger name="com.sun.jersey.spi.inject.Errors" level="ERROR"/> + + <!-- + Logger for capturing user events. We do not want to propagate these + log events to the root logger. These messages are only sent to the + user-log appender. + --> + <logger name="org.apache.nifi.web.security" level="INFO" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.apache.nifi.authorization" level="INFO" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + <logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false"> + <appender-ref ref="USER_FILE"/> + </logger> + + + <!-- + Logger for capturing Bootstrap logs and NiFi's standard error and standard out. + --> + <logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + <logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false"> + <appender-ref ref="CONSOLE" /> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level --> + <logger name="org.apache.nifi.StdOut" level="INFO" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level --> + <logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false"> + <appender-ref ref="BOOTSTRAP_FILE" /> + </logger> + + + <root level="INFO"> + <appender-ref ref="APP_FILE"/> + </root> + +</configuration> + + </value> + <description>Template for logback.xml</description> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + +</configuration> +
