http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py new file mode 100644 index 0000000..362a40b --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys, nifi_ca_util, os, pwd, grp, signal, time, glob, socket, json +from resource_management.core import sudo +from resource_management import * +from subprocess import call +from setup_ranger_nifi import setup_ranger_nifi + +reload(sys) +sys.setdefaultencoding('utf8') + +class Master(Script): + def install(self, env): + + import params + import status_params + + self.install_packages(env) + + Directory([params.nifi_node_dir], + owner=params.nifi_user, + group=params.nifi_group, + create_parents=True, + recursive_ownership=True + ) + + #update the configs specified by user + self.configure(env, True) + + Execute('touch ' + params.nifi_node_log_file, user=params.nifi_user) + + def configure(self, env, isInstall=False, is_starting = False): + import params + import status_params + env.set_params(params) + env.set_params(status_params) + + #create the log, pid, conf dirs if not already present + Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_internal_dir, params.nifi_database_dir, params.nifi_flowfile_repo_dir, params.nifi_content_repo_dir_default, params.nifi_provenance_repo_dir_default, params.nifi_config_dir, params.nifi_flow_config_dir, params.nifi_state_dir, params.lib_dir], + owner=params.nifi_user, + group=params.nifi_group, + create_parents=True, + recursive_ownership=True + ) + + # On some OS this folder may not exist, so we will create it before pushing files there + Directory(params.limits_conf_dir, + create_parents = True, + owner='root', + group='root' + ) + + File(os.path.join(params.limits_conf_dir, 'nifi.conf'), + owner='root', + group='root', + mode=0644, + content=Template("nifi.conf.j2") + ) + + + ca_client_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh') + File(ca_client_script, mode=0755) + + + if params.nifi_ca_host and params.nifi_ssl_enabled: + ca_client_json = os.path.realpath(os.path.join(params.nifi_config_dir, 'nifi-certificate-authority-client.json')) + File(ca_client_json, + owner = params.nifi_user, + group = params.nifi_group, + mode = 0600) + ca_client_dict = nifi_ca_util.load(ca_client_json) + if is_starting: + if params.nifi_toolkit_tls_regenerate: + nifi_ca_util.move_keystore_truststore(ca_client_dict) + ca_client_dict = {} + else: + nifi_ca_util.move_keystore_truststore_if_necessary(ca_client_dict, params.nifi_ca_client_config) + nifi_ca_util.overlay(ca_client_dict, params.nifi_ca_client_config) + nifi_ca_util.dump(ca_client_json, ca_client_dict) + if is_starting: + Execute('JAVA_HOME='+params.jdk64_home+' '+ca_client_script+' client -F -f '+ca_client_json, user=params.nifi_user) + nifi_ca_util.update_nifi_properties(nifi_ca_util.load(ca_client_json), params.nifi_properties) + + #write out nifi.properties + PropertiesFile(params.nifi_config_dir + '/nifi.properties', + properties = params.nifi_properties, + mode = 0400, + owner = params.nifi_user, + group = params.nifi_group) + + #write out boostrap.conf + bootstrap_content=InlineTemplate(params.nifi_boostrap_content) + File(format("{params.nifi_config_dir}/bootstrap.conf"), content=bootstrap_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + #write out logback.xml + logback_content=InlineTemplate(params.nifi_node_logback_content) + File(format("{params.nifi_config_dir}/logback.xml"), content=logback_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + #write out state-management.xml + statemgmt_content=InlineTemplate(params.nifi_state_management_content) + File(format("{params.nifi_config_dir}/state-management.xml"), content=statemgmt_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + #write out authorizers file + authorizers_content=InlineTemplate(params.nifi_authorizers_content) + File(format("{params.nifi_config_dir}/authorizers.xml"), content=authorizers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + #write out login-identity-providers.xml + login_identity_providers_content=InlineTemplate(params.nifi_login_identity_providers_content) + File(format("{params.nifi_config_dir}/login-identity-providers.xml"), content=login_identity_providers_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + #write out nifi-env in bin as 0755 (see BUG-61769) + env_content=InlineTemplate(params.nifi_env_content) + File(format("{params.bin_dir}/nifi-env.sh"), content=env_content, owner=params.nifi_user, group=params.nifi_group, mode=0755) + + #write out bootstrap-notification-services.xml + boostrap_notification_content=InlineTemplate(params.nifi_boostrap_notification_content) + File(format("{params.nifi_config_dir}/bootstrap-notification-services.xml"), content=boostrap_notification_content, owner=params.nifi_user, group=params.nifi_group, mode=0400) + + def stop(self, env): + import params + import status_params + + Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh stop >> ' + params.nifi_node_log_file, user=params.nifi_user) + if os.path.isfile(status_params.nifi_node_pid_file): + sudo.unlink(status_params.nifi_node_pid_file) + + def start(self, env): + import params + import status_params + self.configure(env, is_starting = True) + setup_ranger_nifi(upgrade_type=None) + + # Write out flow.xml.gz to internal dir only if AMS installed (must be writable by Nifi) + # only during first install. It is used to automate setup of Ambari metrics reporting task in Nifi + if params.metrics_collector_host and params.nifi_ambari_reporting_enabled and self.check_is_fresh_install(self): + Execute('echo "First time setup so generating flow.xml.gz" >> ' + params.nifi_node_log_file, user=params.nifi_user) + flow_content=InlineTemplate(params.nifi_flow_content) + File(format("{params.nifi_flow_config_dir}/flow.xml"), content=flow_content, owner=params.nifi_user, group=params.nifi_group, mode=0600) + Execute(format("cd {params.nifi_flow_config_dir}; mv flow.xml.gz flow_$(date +%d-%m-%Y).xml.gz ;"),user=params.nifi_user,ignore_failures=True) + Execute(format("cd {params.nifi_flow_config_dir}; gzip flow.xml;"), user=params.nifi_user) + + + Execute ('export JAVA_HOME='+params.jdk64_home+';'+params.bin_dir+'/nifi.sh start >> ' + params.nifi_node_log_file, user=params.nifi_user) + #If nifi pid file not created yet, wait a bit + if not os.path.isfile(status_params.nifi_pid_dir+'/nifi.pid'): + Execute ('sleep 5') + + + def status(self, env): + import status_params + check_process_status(status_params.nifi_node_pid_file) + + + def check_is_fresh_install(self, env): + """ + Checks if fresh nifi install by checking if zk dir exists + :return: + """ + import params, re + from resource_management.core import shell + from resource_management.core.exceptions import Fail + from resource_management.core.logger import Logger + + ZK_CONNECT_ERROR = "ConnectionLoss" + ZK_NODE_NOT_EXIST = "Node does not exist" + + zookeeper_queried = False + is_fresh_nifi_install = True + + # For every zk server try to find nifi zk dir + zookeeper_server_list = params.config['clusterHostInfo']['zookeeper_hosts'] + for zookeeper_server in zookeeper_server_list: + # Determine where the zkCli.sh shell script is + zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh") + if params.stack_version_buildnum is not None: + zk_command_location = os.path.join(params.stack_root, params.stack_version_buildnum, "zookeeper", "bin", "zkCli.sh") + + # create the ZooKeeper query command e.g. + # /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181 ls /nifi + command = "{0} -server {1}:{2} ls {3}".format( + zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode) + + # echo 'ls /nifi' | /usr/hdf/current/zookeeper-client/bin/zkCli.sh -server node:2181 + #command = "echo 'ls {3}' | {0} -server {1}:{2}".format( + # zk_command_location, zookeeper_server, params.zookeeper_port, params.nifi_znode) + + Logger.info("Running command: " + command) + + code, out = shell.call(command, logoutput=True, quiet=False, timeout=20) + if not out or re.search(ZK_CONNECT_ERROR, out): + Logger.info("Unable to query Zookeeper: " + zookeeper_server + ". Skipping and trying next ZK server") + continue + elif re.search(ZK_NODE_NOT_EXIST, out): + Logger.info("Nifi ZNode does not exist, so must be fresh install of Nifi: " + params.nifi_znode) + zookeeper_queried = True + is_fresh_nifi_install = True + break + else: + Logger.info("Nifi ZNode already exists, so must not be a fresh install of Nifi: " + params.nifi_znode) + zookeeper_queried = True + is_fresh_nifi_install = False + break + + # fail if the ZK data could not be queried + if not zookeeper_queried: + raise Fail("Unable to query for znode on on any of the following ZooKeeper hosts: {0}. Please ensure Zookeepers are started and retry".format( + zookeeper_server_list)) + else: + return is_fresh_nifi_install + + +if __name__ == "__main__": + Master().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py new file mode 100644 index 0000000..bc343e4 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import nifi_ca_util, os, time + +from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.resources.system import Directory, Execute +from resource_management.core.sudo import kill, read_file, path_isfile, unlink +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.script.script import Script +from resource_management.core.resources import File +from signal import SIGTERM, SIGKILL + +class CertificateAuthority(Script): + def install(self, env): + import params + import status_params + + self.install_packages(env) + + #Be sure ca script is in cache + nifi_ca_util.get_toolkit_script('tls-toolkit.sh') + + def configure(self, env): + import params + import status_params + env.set_params(params) + env.set_params(status_params) + + #create the log, pid, conf dirs if not already present + Directory([status_params.nifi_pid_dir, params.nifi_node_log_dir, params.nifi_config_dir], + owner=params.nifi_user, + group=params.nifi_group, + create_parents=True + ) + + ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json') + ca_dict = nifi_ca_util.load(ca_json) + nifi_ca_util.overlay(ca_dict, params.nifi_ca_config) + nifi_ca_util.dump(ca_json, ca_dict) + + Directory([params.nifi_config_dir], + owner=params.nifi_user, + group=params.nifi_group, + create_parents=True, + recursive_ownership=True + ) + + def invalidate_ca_server(self, env): + import params + ca_json = os.path.join(params.nifi_config_dir, 'nifi-certificate-authority.json') + nifi_ca_util.move_store(nifi_ca_util.load(ca_json), 'keyStore') + unlink(ca_json) + + def status(self, env): + import status_params + check_process_status(status_params.nifi_ca_pid_file) + + def start(self, env): + import params + import status_params + + self.configure(env) + ca_server_script = nifi_ca_util.get_toolkit_script('tls-toolkit.sh') + run_ca_script = os.path.join(os.path.dirname(__file__), 'run_ca.sh') + Directory([params.nifi_config_dir], + owner=params.nifi_user, + group=params.nifi_group, + create_parents=True, + recursive_ownership=True + ) + + File(ca_server_script, mode=0755) + File(run_ca_script, mode=0755) + Execute((run_ca_script, params.jdk64_home, ca_server_script, params.nifi_config_dir + '/nifi-certificate-authority.json', params.nifi_ca_log_file_stdout, params.nifi_ca_log_file_stderr, status_params.nifi_ca_pid_file), user=params.nifi_user) + if not os.path.isfile(status_params.nifi_ca_pid_file): + raise Exception('Expected pid file to exist') + + def stop(self, env): + import status_params + + if path_isfile(status_params.nifi_ca_pid_file): + try: + self.status(env) + pid = int(read_file(status_params.nifi_ca_pid_file)) + for i in range(25): + kill(pid, SIGTERM) + time.sleep(1) + self.status(env) + kill(pid, SIGKILL) + time.sleep(5) + self.status(env) + except ComponentIsNotRunning: + unlink(status_params.nifi_ca_pid_file) + +if __name__ == "__main__": + CertificateAuthority().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py new file mode 100644 index 0000000..6807491 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_ca_util.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import json, nifi_constants, os +from resource_management.core import sudo +from resource_management.core.resources.system import File + +script_dir = os.path.dirname(__file__) +files_dir = os.path.realpath(os.path.join(os.path.dirname(script_dir), 'files')) + +def load(config_json): + if sudo.path_isfile(config_json): + contents = sudo.read_file(config_json) + if len(contents) > 0: + return json.loads(contents) + return {} + +def dump(config_json, config_dict): + import params + File(config_json, + owner=params.nifi_user, + group=params.nifi_group, + mode=0600, + content=json.dumps(config_dict, sort_keys=True, indent=4) + ) + +def overlay(config_dict, overlay_dict): + for k, v in overlay_dict.iteritems(): + if v or k not in config_dict: + config_dict[k] = v + +def get_toolkit_script(scriptName, scriptDir = files_dir): + nifiToolkitDir = None + for dir in os.listdir(scriptDir): + if dir.startswith('nifi-toolkit-'): + nifiToolkitDir = os.path.join(scriptDir, dir) + + if nifiToolkitDir is None: + raise Exception("Couldn't find nifi toolkit directory in " + scriptDir) + result = nifiToolkitDir + '/bin/' + scriptName + if not sudo.path_isfile(result): + raise Exception("Couldn't find file " + result) + return result + +def update_nifi_properties(client_dict, nifi_properties): + nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_TYPE] = client_dict['keyStoreType'] + nifi_properties[nifi_constants.NIFI_SECURITY_KEYSTORE_PASSWD] = client_dict['keyStorePassword'] + nifi_properties[nifi_constants.NIFI_SECURITY_KEY_PASSWD] = client_dict['keyPassword'] + nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_TYPE] = client_dict['trustStoreType'] + nifi_properties[nifi_constants.NIFI_SECURITY_TRUSTSTORE_PASSWD] = client_dict['trustStorePassword'] + +def store_exists(client_dict, key): + if key not in client_dict: + return False + return sudo.path_isfile(client_dict[key]) + +def different(one, two, key): + if key not in one: + return False + if len(one[key]) == 0: + return False + if key not in two: + return False + if len(two[key]) == 0: + return False + return one[key] != two[key] + +def move_keystore_truststore_if_necessary(orig_client_dict, new_client_dict): + if not (store_exists(new_client_dict, 'keyStore') or store_exists(new_client_dict, 'trustStore')): + return + if different(orig_client_dict, new_client_dict, 'keyStoreType'): + move_keystore_truststore(new_client_dict) + elif different(orig_client_dict, new_client_dict, 'keyStorePassword'): + move_keystore_truststore(new_client_dict) + elif different(orig_client_dict, new_client_dict, 'keyPassword'): + move_keystore_truststore(new_client_dict) + elif different(orig_client_dict, new_client_dict, 'trustStoreType'): + move_keystore_truststore(new_client_dict) + elif different(orig_client_dict, new_client_dict, 'trustStorePassword'): + move_keystore_truststore(new_client_dict) + +def move_keystore_truststore(client_dict): + move_store(client_dict, 'keyStore') + move_store(client_dict, 'trustStore') + +def move_store(client_dict, key): + if store_exists(client_dict, key): + num = 0 + name = client_dict[key] + while sudo.path_isfile(name + '.bak.' + str(num)): + num += 1 + sudo.copy(name, name + '.bak.' + str(num)) + sudo.unlink(name) http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py new file mode 100644 index 0000000..632a473 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/nifi_constants.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +NIFI_SECURITY_KEY_PASSWD = 'nifi.security.keyPasswd' +NIFI_SECURITY_KEYSTORE_PASSWD = 'nifi.security.keystorePasswd' +NIFI_SECURITY_KEYSTORE_TYPE = 'nifi.security.keystoreType' +NIFI_SECURITY_TRUSTSTORE_PASSWD = 'nifi.security.truststorePasswd' +NIFI_SECURITY_TRUSTSTORE_TYPE = 'nifi.security.truststoreType' http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py new file mode 100644 index 0000000..087cef4 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/params.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.libraries.script.script import Script +import sys, os, glob, socket, re +from resource_management.libraries.functions import format +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.version import format_stack_version +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import stack_select +from resource_management.libraries.functions import conf_select +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources +import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set + +# server configurations +config = Script.get_config() +stack_root = Script.get_stack_root() +stack_version_buildnum = default("/commandParams/version", None) + +#nifi_install_dir = '/usr/hdf/current/nifi' +nifi_install_dir = os.path.join(stack_root, "current", "nifi") +if stack_version_buildnum is not None: + nifi_install_dir = os.path.join(stack_root, stack_version_buildnum, "nifi") + +# params from nifi-ambari-config +nifi_initial_mem = config['configurations']['nifi-ambari-config']['nifi.initial_mem'] +nifi_max_mem = config['configurations']['nifi-ambari-config']['nifi.max_mem'] +nifi_ambari_reporting_frequency = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_frequency'] +nifi_ambari_reporting_enabled = config['configurations']['nifi-ambari-config']['nifi.ambari_reporting_enabled'] + +# note: nifi.node.port and nifi.node.ssl.port must be defined in same xml file for quicklinks to work +nifi_node_port = config['configurations']['nifi-ambari-config']['nifi.node.port'] +nifi_node_ssl_port = config['configurations']['nifi-ambari-config']['nifi.node.ssl.port'] +nifi_node_protocol_port = config['configurations']['nifi-ambari-config']['nifi.node.protocol.port'] + +nifi_znode = config['configurations']['nifi-ambari-config']['nifi.nifi_znode'] + +nifi_internal_dir=config['configurations']['nifi-ambari-config']['nifi.internal.dir'] +nifi_state_dir=config['configurations']['nifi-ambari-config']['nifi.state.dir'] +nifi_database_dir=config['configurations']['nifi-ambari-config']['nifi.database.dir'] +nifi_flowfile_repo_dir=config['configurations']['nifi-ambari-config']['nifi.flowfile.repository.dir'] +nifi_content_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.content.repository.dir.default'] +nifi_provenance_repo_dir_default=config['configurations']['nifi-ambari-config']['nifi.provenance.repository.dir.default'] +nifi_config_dir = config['configurations']['nifi-ambari-config']['nifi.config.dir'] +nifi_flow_config_dir = config['configurations']['nifi-ambari-config']['nifi.flow.config.dir'] +nifi_sensitive_props_key = config['configurations']['nifi-ambari-config']['nifi.sensitive.props.key'] + + +nifi_flow_config_dir = nifi_flow_config_dir.replace('{nifi_internal_dir}',nifi_internal_dir) +nifi_state_dir = nifi_state_dir.replace('{nifi_internal_dir}',nifi_internal_dir) +nifi_config_dir = nifi_config_dir.replace('{nifi_install_dir}',nifi_install_dir) + +master_configs = config['clusterHostInfo'] +nifi_master_hosts = master_configs['nifi_master_hosts'] + +# detect if running in single (sandbox) box +nifi_num_nodes = len(master_configs['nifi_master_hosts']) +#if nifi_num_nodes > 1: +# nifi_is_node='true' +#else: +# nifi_is_node='false' +#nifi_node_hosts = ",".join(master_configs['nifi_master_hosts']) + +# In sandbox scenario, Ambari should still setup nifi in clustered mode for now +nifi_is_node='true' + +nifi_node_dir=nifi_install_dir +bin_dir = os.path.join(*[nifi_node_dir,'bin']) +lib_dir = os.path.join(*[nifi_node_dir,'lib']) + +nifi_ca_host = None +if 'nifi_ca_hosts' in master_configs: + nifi_ca_hosts = master_configs['nifi_ca_hosts'] + if len(nifi_ca_hosts) > 0: + nifi_ca_host = nifi_ca_hosts[0] + +# params from nifi-ambari-ssl-config + +nifi_ssl_enabled = config['configurations']['nifi-ambari-ssl-config']['nifi.node.ssl.isenabled'] +nifi_keystore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystore'] +nifi_keystoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystoreType'] +nifi_keystorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keystorePasswd'] +nifi_keyPasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.keyPasswd'] +nifi_truststore = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststore'] +nifi_truststoreType = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststoreType'] +nifi_truststorePasswd = config['configurations']['nifi-ambari-ssl-config']['nifi.security.truststorePasswd'] +nifi_needClientAuth = config['configurations']['nifi-ambari-ssl-config']['nifi.security.needClientAuth'] +nifi_initial_admin_id = config['configurations']['nifi-ambari-ssl-config']['nifi.initial.admin.identity'] +nifi_ssl_config_content = config['configurations']['nifi-ambari-ssl-config']['content'] + +#property that is set to hostname regardless of whether SSL enabled +nifi_node_host = socket.getfqdn() + +nifi_truststore = nifi_truststore.replace('{nifi_node_ssl_host}',nifi_node_host) +nifi_keystore = nifi_keystore.replace('{nifi_node_ssl_host}',nifi_node_host) + +#populate properties whose values depend on whether SSL enabled +nifi_keystore = nifi_keystore.replace('{{nifi_config_dir}}',nifi_config_dir) +nifi_truststore = nifi_truststore.replace('{{nifi_config_dir}}',nifi_config_dir) + +if nifi_ssl_enabled: + nifi_node_ssl_host = nifi_node_host + nifi_node_port = "" +else: + nifi_node_nonssl_host = nifi_node_host + nifi_node_ssl_port = "" + +nifi_ca_parent_config = config['configurations']['nifi-ambari-ssl-config'] +nifi_use_ca = nifi_ca_parent_config['nifi.toolkit.tls.token'] +nifi_toolkit_dn_prefix = nifi_ca_parent_config['nifi.toolkit.dn.prefix'] +nifi_toolkit_dn_suffix = nifi_ca_parent_config['nifi.toolkit.dn.suffix'] +nifi_toolkit_tls_regenerate = nifi_ca_parent_config['nifi.toolkit.tls.regenerate'] +nifi_ca_log_file_stdout = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stdout' +nifi_ca_log_file_stderr = config['configurations']['nifi-env']['nifi_node_log_dir'] + '/nifi-ca.stderr' + +nifi_ca_config = { + "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']), + "keyStore" : nifi_config_dir + '/nifi-certificate-authority-keystore.jks', + "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'], + "caHostname" : nifi_ca_host, + "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port']) +} + +if nifi_ca_host: + nifi_ca_config['dn'] = nifi_toolkit_dn_prefix + nifi_ca_host + nifi_toolkit_dn_suffix + +nifi_ca_client_config = { + "days" : int(nifi_ca_parent_config['nifi.toolkit.tls.helper.days']), + "keyStore" : nifi_keystore, + "keyStoreType" : nifi_keystoreType, + "keyStorePassword" : nifi_keystorePasswd, + "keyPassword" : nifi_keyPasswd, + "token" : nifi_ca_parent_config['nifi.toolkit.tls.token'], + "dn" : nifi_toolkit_dn_prefix + nifi_node_host + nifi_toolkit_dn_suffix, + "port" : int(nifi_ca_parent_config['nifi.toolkit.tls.port']), + "caHostname" : nifi_ca_host, + "trustStore" : nifi_truststore, + "trustStoreType" : nifi_truststoreType, + "trustStorePassword": nifi_truststorePasswd +} + +# params from nifi-env +nifi_user = config['configurations']['nifi-env']['nifi_user'] +nifi_group = config['configurations']['nifi-env']['nifi_group'] + +nifi_node_log_dir = config['configurations']['nifi-env']['nifi_node_log_dir'] +nifi_node_log_file = os.path.join(nifi_node_log_dir,'nifi-setup.log') + +# limits related params +limits_conf_dir = '/etc/security/limits.d' +nifi_user_nofile_limit = config['configurations']['nifi-env']['nifi_user_nofile_limit'] +nifi_user_nproc_limit = config['configurations']['nifi-env']['nifi_user_nproc_limit'] + +# params from nifi-boostrap +nifi_env_content = config['configurations']['nifi-env']['content'] + + +# params from nifi-logback +nifi_master_logback_content = config['configurations']['nifi-master-logback-env']['content'] +nifi_node_logback_content = config['configurations']['nifi-node-logback-env']['content'] + +# params from nifi-properties-env +nifi_master_properties_content = config['configurations']['nifi-master-properties-env']['content'] +nifi_properties = config['configurations']['nifi-properties'].copy() + +#kerberos params +nifi_kerberos_authentication_expiration = config['configurations']['nifi-properties']['nifi.kerberos.spnego.authentication.expiration'] +nifi_kerberos_realm = default("/configurations/kerberos-env/realm", None) + +# params from nifi-flow +nifi_flow_content = config['configurations']['nifi-flow-env']['content'] + +# params from nifi-state-management-env +nifi_state_management_content = config['configurations']['nifi-state-management-env']['content'] + +# params from nifi-authorizers-env +nifi_authorizers_content = config['configurations']['nifi-authorizers-env']['content'] + +# params from nifi-login-identity-providers-env +nifi_login_identity_providers_content = config['configurations']['nifi-login-identity-providers-env']['content'] + +# params from nifi-boostrap +nifi_boostrap_content = config['configurations']['nifi-bootstrap-env']['content'] + +# params from nifi-bootstrap-notification-services-env +nifi_boostrap_notification_content = config['configurations']['nifi-bootstrap-notification-services-env']['content'] + +#autodetect jdk home +jdk64_home=config['hostLevelParams']['java_home'] + +#autodetect ambari server for metrics +if 'metrics_collector_hosts' in config['clusterHostInfo']: + metrics_collector_host = str(config['clusterHostInfo']['metrics_collector_hosts'][0]) + metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) +else: + metrics_collector_host = '' + metrics_collector_port = '' + + +#detect zookeeper_quorum +zookeeper_port=default('/configurations/zoo.cfg/clientPort', None) +#get comma separated list of zookeeper hosts from clusterHostInfo +index = 0 +zookeeper_quorum="" +for host in config['clusterHostInfo']['zookeeper_hosts']: + zookeeper_quorum += host + ":"+str(zookeeper_port) + index += 1 + if index < len(config['clusterHostInfo']['zookeeper_hosts']): + zookeeper_quorum += "," + + +#setup ranger configuration + +retryAble = default("/commandParams/command_retry_enabled", False) +version = default("/commandParams/version", None) +namenode_hosts = default("/clusterHostInfo/namenode_host", None) + +if type(namenode_hosts) is list: + namenode_host = namenode_hosts[0] +else: + namenode_host = namenode_hosts + +has_namenode = not namenode_host == None + + +nifi_authorizer = 'file-provider' + +nifi_host_name = config['hostname'] +nifi_host_port = config['configurations']['nifi-ambari-config']['nifi.node.port'] +java_home = config['hostLevelParams']['java_home'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) + +if security_enabled: + _hostname_lowercase = nifi_host_name.lower() + nifi_properties['nifi.kerberos.service.principal'] = nifi_properties['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase) + nifi_properties['nifi.kerberos.spnego.principal'] = nifi_properties['nifi.kerberos.spnego.principal'].replace('_HOST',_hostname_lowercase) + +# ranger host +# E.g., 2.3 +stack_version_unformatted = config['hostLevelParams']['stack_version'] +stack_version_formatted = format_stack_version(stack_version_unformatted) +stack_supports_ranger_kerberos = stack_version_formatted and check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, stack_version_formatted) +stack_supports_ranger_audit_db = stack_version_formatted and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted) + +ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) +has_ranger_admin = not len(ranger_admin_hosts) == 0 +xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported'] + +ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] + +# ranger nifi properties +policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url'] + +if 'admin-properties' in config['configurations'] and 'policymgr_external_url' in config['configurations']['admin-properties'] and policymgr_mgr_url.endswith('/'): + policymgr_mgr_url = policymgr_mgr_url.rstrip('/') + +xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name'] +xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user'] +xa_db_host = config['configurations']['admin-properties']['db_host'] +repo_name = str(config['clusterName']) + '_nifi' + +repo_config_username = config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] + +ranger_env = config['configurations']['ranger-env'] +ranger_plugin_properties = config['configurations']['ranger-nifi-plugin-properties'] +policy_user = config['configurations']['ranger-nifi-plugin-properties']['policy_user'] + +#For curl command in ranger plugin to get db connector +jdk_location = config['hostLevelParams']['jdk_location'] +java_share_dir = '/usr/share/java' + +if has_ranger_admin: + enable_ranger_nifi = (config['configurations']['ranger-nifi-plugin-properties']['ranger-nifi-plugin-enabled'].lower() == 'yes') + xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password']) if stack_supports_ranger_audit_db else None + repo_config_password = unicode(config['configurations']['ranger-nifi-plugin-properties']['REPOSITORY_CONFIG_PASSWORD']) + xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower() + previous_jdbc_jar_name= None + + if stack_supports_ranger_audit_db: + if xa_audit_db_flavor == 'mysql': + jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None) + audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "com.mysql.jdbc.Driver" + elif xa_audit_db_flavor == 'oracle': + jdbc_jar_name = default("/hostLevelParams/custom_oracle_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None) + colon_count = xa_db_host.count(':') + if colon_count == 2 or colon_count == 0: + audit_jdbc_url = format('jdbc:oracle:thin:@{xa_db_host}') + else: + audit_jdbc_url = format('jdbc:oracle:thin:@//{xa_db_host}') + jdbc_driver = "oracle.jdbc.OracleDriver" + elif xa_audit_db_flavor == 'postgres': + jdbc_jar_name = default("/hostLevelParams/custom_postgres_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None) + audit_jdbc_url = format('jdbc:postgresql://{xa_db_host}/{xa_audit_db_name}') + jdbc_driver = "org.postgresql.Driver" + elif xa_audit_db_flavor == 'mssql': + jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlserver://{xa_db_host};databaseName={xa_audit_db_name}') + jdbc_driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + elif xa_audit_db_flavor == 'sqla': + jdbc_jar_name = default("/hostLevelParams/custom_sqlanywhere_jdbc_name", None) + previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_sqlanywhere_jdbc_name", None) + audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') + jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" + + downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + driver_curl_source = format("{jdk_location}/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + + driver_curl_target = format("{stack_root}/current/nifi/ext/{jdbc_jar_name}") if stack_supports_ranger_audit_db else None + previous_jdbc_jar = format("{stack_root}/current/nifi/ext/{previous_jdbc_jar_name}") if stack_supports_ranger_audit_db else None + sql_connector_jar = '' + + ssl_keystore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None + ssl_truststore_password = unicode(config['configurations']['ranger-nifi-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None + credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None + credential_file_type = 'jceks' + ranger_admin_username = config['configurations']['ranger-env']['ranger_admin_username'] + ranger_admin_password = config['configurations']['ranger-env']['ranger_admin_password'] + + #create ranger service's nifi client properties + nifi_authentication = config['configurations']['ranger-nifi-plugin-properties']['nifi.authentication'] + ranger_id_owner_for_certificate = config['configurations']['ranger-nifi-plugin-properties']['owner.for.certificate'] + nifi_id_owner_for_certificate = config['configurations']['ranger-nifi-policymgr-ssl']['owner.for.certificate'] + regex = r"(CN)=([a-zA-Z0-9\.\-\* ]*)" + match = re.search(regex, nifi_id_owner_for_certificate) + common_name_for_certificate = match.group(2) if match else 'NONE' + + if nifi_authentication == 'SSL': + + nifi_ranger_plugin_config = { + 'nifi.authentication': nifi_authentication, + 'nifi.url': format("https://{nifi_host_name}:{nifi_node_ssl_port}/nifi-api/resources";), + 'nifi.ssl.keystore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystore'], + 'nifi.ssl.keystoreType':config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystoreType'], + 'nifi.ssl.keystorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.keystorePassword'], + 'nifi.ssl.truststore': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststore'], + 'nifi.ssl.truststoreType': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststoreType'], + 'nifi.ssl.truststorePassword': config['configurations']['ranger-nifi-plugin-properties']['nifi.ssl.truststorePassword'], + 'commonNameForCertificate': common_name_for_certificate + } + else: + nifi_ranger_plugin_config = { + 'nifi.authentication': nifi_authentication, + 'nifi.url': format("https://{nifi_host_name}:{nifi_host_port}/nifi-api/resources";), + 'commonNameForCertificate': common_name_for_certificate + } + + nifi_ranger_plugin_repo = { + 'isActive': 'true', + 'config': json.dumps(nifi_ranger_plugin_config), + 'description': 'nifi repo', + 'name': repo_name, + 'repositoryType': 'nifi', + 'assetType': '5' + } + + # used in nifi authorizers + ranger_admin_identity = ranger_id_owner_for_certificate + + if stack_supports_ranger_kerberos and security_enabled: + nifi_ranger_plugin_config['policy.download.auth.users'] = nifi_user + nifi_ranger_plugin_config['tag.download.auth.users'] = nifi_user + ranger_nifi_principal = config['configurations']['nifi-properties']['nifi.kerberos.service.principal'].replace('_HOST',_hostname_lowercase) + ranger_nifi_keytab = config['configurations']['nifi-properties']['nifi.kerberos.service.keytab.location'] + + if stack_supports_ranger_kerberos: + nifi_ranger_plugin_config['ambari.service.check.user'] = policy_user + + nifi_ranger_plugin_repo = { + 'isEnabled': 'true', + 'configs': nifi_ranger_plugin_config, + 'description': 'nifi repo', + 'name': repo_name, + 'type': 'nifi' + } + + xa_audit_db_is_enabled = False + ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] + + if xml_configurations_supported and stack_supports_ranger_audit_db: + xa_audit_db_is_enabled = config['configurations']['ranger-nifi-audit']['xasecure.audit.destination.db'] + + xa_audit_hdfs_is_enabled = default('/configurations/ranger-nifi-audit/xasecure.audit.destination.hdfs', False) + + + #For SQLA explicitly disable audit to DB for Ranger + if xa_audit_db_flavor == 'sqla': + xa_audit_db_is_enabled = False + + nifi_authorizer = 'ranger-provider' + +hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None +hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None +hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None +hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None +default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None +hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None +hadoop_conf_dir = conf_select.get_hadoop_conf_dir() if has_namenode else None + +import functools +#create partial functions with common arguments for every HdfsResource call +#to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code +HdfsResource = functools.partial( + HdfsResource, + user=hdfs_user, + hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", + security_enabled = security_enabled, + keytab = hdfs_user_keytab, + kinit_path_local = kinit_path_local, + hadoop_bin_dir = hadoop_bin_dir, + hadoop_conf_dir = hadoop_conf_dir, + principal_name = hdfs_principal_name, + hdfs_site = hdfs_site, + default_fs = default_fs, + immutable_paths = get_not_managed_resources() +) http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh new file mode 100644 index 0000000..c5863fa --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/run_ca.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Launches NiFi CA server +# $1 -> JAVA_HOME +# $2 -> tls-toolkit.sh path +# $3 -> config json +# $4 -> stdout log +# $5 -> stderr log +# $6 -> pid file + +die() { + echo "$1" + exit 1 +} + +read -r -d '' WAIT_FOR_LOG << 'EOF' + STDOUT_LOG="$0" + until [ -f "$STDOUT_LOG" ]; do + echo "Waiting for $STDOUT_LOG to exist" + sleep 1; + done +EOF + +read -r -d '' WAIT_FOR_CHILD << 'EOF' + SHELL_PID="$0" + PID_FILE="$1" + CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)" + while [ -z "$CHILD_PROCESS" ]; do + echo "Waiting for child java process to exist" + sleep 1; + CHILD_PROCESS="$(pgrep -P "$SHELL_PID" java)" + done + echo "$CHILD_PROCESS" > "$PID_FILE" +EOF + +JAVA_HOME="$1" nohup "$2" server -F -f "$3" > "$4" 2> "$5" < /dev/null & +SHELL_PID="$!" + +timeout 30 bash -c "$WAIT_FOR_LOG" "$4" || die "Timed out while waiting for $4" + +timeout 30 bash -c "$WAIT_FOR_CHILD" "$SHELL_PID" "$6" || die "Timed out while waiting for child java process to exist" + +#Want to wait until Jetty starts +#See http://superuser.com/questions/270529/monitoring-a-file-until-a-string-is-found#answer-900134 +( tail -f -n +1 "$4" & ) | timeout 180 grep -q "Server Started" || die "Timed out while waiting for CA server to start" http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py new file mode 100644 index 0000000..535592f --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/service_check.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +import urllib2 +import httplib + +from resource_management.core.logger import Logger +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script + +from ambari_commons.inet_utils import openurl +from ambari_commons.exceptions import TimeoutError +from resource_management.core.exceptions import Fail +from resource_management.libraries.functions.decorator import retry + +class NifiServiceCheck(Script): + def service_check(self, env): + import params + Logger.info("Running Nifi service check") + for nifi_master_host in params.nifi_master_hosts: + url = "" + if params.nifi_ssl_enabled: + url = "https://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_ssl_port) + else: + url = "http://{0}:{1}/nifi".format(nifi_master_host, params.nifi_node_port) + Logger.info("Checking Nifi portal {0} status".format(url)) + NifiServiceCheck.check_nifi_portal(url) + + @staticmethod + @retry(times=15, sleep_time=5, max_sleep_time=20, backoff_factor=2, err_class=Fail) + def check_nifi_portal(url): + try: + request = urllib2.Request(url) + result = openurl(request, timeout=20) + response_code = result.getcode() + if response_code == 200 or response_code == 401: + Logger.info("Nifi portal {0} is up. Response code {1}".format(url, response_code)) + else: + raise Fail("Error connecting to {0}. Response code {1}".format(url, response_code)) + except urllib2.URLError, e: + if isinstance(e, urllib2.HTTPError): + if e.code == 401: + Logger.info("Nifi portal {0} is up. Response code {1}".format(url, e.code)) + else: + raise Fail("Error connecting to {0}. Http status code - {1}. \n {2}".format(url, e.code, e.read())) + elif e.reason and "violation of protocol" in str(e.reason): + Logger.info("Ignore certificate validation error - {0}".format(e.reason)) + pass + else: + raise Fail("Error connecting to {0}. Reason - {1}.".format(url, e.reason)) + except httplib.BadStatusLine: + raise Fail("Error connecting to {0}. Reason - Not Reachable".format(url)) + except TimeoutError: + raise Fail("Error connecting to {0}. Reason - Timeout".format(url)) + +if __name__ == "__main__": + NifiServiceCheck().execute() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py new file mode 100644 index 0000000..91a6885 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/setup_ranger_nifi.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from resource_management.core.logger import Logger +from resource_management.core.resources import File + +def setup_ranger_nifi(upgrade_type=None): + import params, os + + if params.has_ranger_admin and params.enable_ranger_nifi: + + stack_version = None + if upgrade_type is not None: + stack_version = params.version + + if params.retryAble: + Logger.info("nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !") + else: + Logger.info("nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !") + + + api_version=None + if params.stack_supports_ranger_kerberos: + api_version='v2' + from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin + setup_ranger_plugin('nifi', 'nifi', params.previous_jdbc_jar, + params.downloaded_custom_connector, params.driver_curl_source, + params.driver_curl_target, params.java_home, + params.repo_name, params.nifi_ranger_plugin_repo, + params.ranger_env, params.ranger_plugin_properties, + params.policy_user, params.policymgr_mgr_url, + params.enable_ranger_nifi, conf_dict=params.nifi_config_dir, + component_user=params.nifi_user, component_group=params.nifi_group, cache_service_list=['nifi'], + plugin_audit_properties=params.config['configurations']['ranger-nifi-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-nifi-audit'], + plugin_security_properties=params.config['configurations']['ranger-nifi-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-nifi-security'], + plugin_policymgr_ssl_properties=params.config['configurations']['ranger-nifi-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-nifi-policymgr-ssl'], + component_list=[], audit_db_is_enabled=params.xa_audit_db_is_enabled, + credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password, + ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password, + stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version, + is_security_enabled = params.security_enabled, + is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos, + component_user_principal=params.ranger_nifi_principal if params.security_enabled else None, + component_user_keytab=params.ranger_nifi_keytab if params.security_enabled else None) + + #change permissions of ranger xml that were written to 0400 + File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400) + File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400) + File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400) + + else: + Logger.info('Ranger admin not installed') \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py new file mode 100755 index 0000000..9748661 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/scripts/status_params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +import sys, os + +config = Script.get_config() + +nifi_pid_dir=config['configurations']['nifi-env']['nifi_pid_dir'] +#nifi_master_pid_file=nifi_pid_dir + '/nifi-master.pid' +nifi_node_pid_file=nifi_pid_dir + '/nifi.pid' +nifi_ca_pid_file=nifi_pid_dir + '/nifi-ca.pid' http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 new file mode 100644 index 0000000..97f5711 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/package/templates/nifi.conf.j2 @@ -0,0 +1,36 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{nifi_user}} - nofile {{nifi_user_nofile_limit}} +{{nifi_user}} - nproc {{nifi_user_nproc_limit}} + http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json new file mode 100644 index 0000000..bd27705 --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/quicklinks/quicklinks.json @@ -0,0 +1,35 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": + { + "type":"https", + "checks":[ + { + "property":"nifi.node.ssl.isenabled", + "desired":"true", + "site":"nifi-ambari-ssl-config" + } + ] + }, + + "links": [ + { + "name": "nifi_ui", + "label": "Nifi UI", + "requires_user_name": "false", + "component_name": "NIFI_MASTER", + "url":"%@://%@:%@/nifi", + "port":{ + "http_property": "nifi.node.port", + "http_default_port": "9090", + "https_property": "nifi.node.ssl.port", + "https_default_port": "9091", + "regex": "^(\\d+)$", + "site": "nifi-ambari-config" + } + } + ] + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json new file mode 100644 index 0000000..cd6791b --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/common-services/NIFI/1.0.0/widgets.json @@ -0,0 +1,463 @@ +{ + "layouts": [ + { + "layout_name": "default_nifi_dashboard", + "display_name": "Standard NiFi Dashboard", + "section_name": "NIFI_SUMMARY", + "widgetLayoutInfo": [ + { + "widget_name": "FlowFiles Received Last 5 mins", + "description": "The number of FlowFiles received in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "FlowFilesReceivedLast5Minutes", + "metric_path": "metrics/nifi/FlowFilesReceivedLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "FlowFiles Received", + "value": "${FlowFilesReceivedLast5Minutes}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "MBs Received Last 5 mins", + "description": "The number of MBs received in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "BytesReceivedLast5Minutes", + "metric_path": "metrics/nifi/BytesReceivedLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "MBs Received", + "value": "${BytesReceivedLast5Minutes/1024000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "FlowFiles Sent Last 5 mins", + "description": "The number of FlowFiles sent in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "FlowFilesSentLast5Minutes", + "metric_path": "metrics/nifi/FlowFilesSentLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "FlowFiles Sent", + "value": "${FlowFilesSentLast5Minutes}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "MBs Sent Last 5 mins", + "description": "The number of MBs sent in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "BytesSentLast5Minutes", + "metric_path": "metrics/nifi/BytesSentLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "MBs Sent", + "value": "${BytesSentLast5Minutes/1024000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "FlowFiles Queued", + "description": "The number of FlowFiles queued.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "FlowFilesQueued", + "metric_path": "metrics/nifi/FlowFilesQueued", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "FlowFiles Queued", + "value": "${FlowFilesQueued}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "MBs Queued", + "description": "The number of MBs queued.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "BytesQueued", + "metric_path": "metrics/nifi/BytesQueued", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "MBs Queued", + "value": "${BytesQueued/1024000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "MBs Read Last 5 mins", + "description": "The number of MBs read in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "BytesReadLast5Minutes", + "metric_path": "metrics/nifi/BytesReadLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "MBs Read", + "value": "${BytesReadLast5Minutes/1024000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "MBs Written Last 5 mins", + "description": "The number of MBs written in the last 5 minutes.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "BytesWrittenLast5Minutes", + "metric_path": "metrics/nifi/BytesWrittenLast5Minutes", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "MBs Written", + "value": "${BytesWrittenLast5Minutes/1024000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Active Threads", + "description": "The number of active threads.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "ActiveThreads", + "metric_path": "metrics/nifi/ActiveThreads", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Active Threads", + "value": "${ActiveThreads}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Total Task Duration Milliseconds", + "description": "The total task duration in Milliseconds.", + "widget_type": "GRAPH", + "is_visible": true, + "metrics": [ + { + "name": "TotalTaskDurationNanoSeconds", + "metric_path": "metrics/nifi/TotalTaskDurationNanoSeconds", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Total Task Duration Milliseconds", + "value": "${TotalTaskDurationNanoSeconds/1000}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "JVM Heap Used (MBs)", + "description": "Number of MBs being used on the JVM Heap.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "jvm.heap_used", + "metric_path": "metrics/jvm/heap_used", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "JVM Heap Used", + "value": "${jvm.heap_used/1024/1024}" + } + ], + "properties": { + "warning_threshold": "100000", + "error_threshold": "200000" + } + }, + { + "widget_name": "JVM Heap Usage", + "description": "Precentage of JVM heap being used.", + "widget_type": "GAUGE", + "is_visible": true, + "metrics": [ + { + "name": "jvm.heap_usage", + "metric_path": "metrics/jvm/heap_usage", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "JVM Heap Usage", + "value": "${jvm.heap_usage}" + } + ], + "properties": { + "warning_threshold": "0.8", + "error_threshold": "0.9" + } + }, + { + "widget_name": "JVM File Descriptor Usage", + "description": "Precentage of file descriptors being used.", + "widget_type": "GAUGE", + "is_visible": true, + "metrics": [ + { + "name": "jvm.file_descriptor_usage", + "metric_path": "metrics/jvm/file_descriptor_usage", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "JVM File Descriptor Usage", + "value": "${jvm.file_descriptor_usage}" + } + ], + "properties": { + "warning_threshold": "0.8", + "error_threshold": "0.9" + } + }, + { + "widget_name": "Thread Count", + "description": "Number of threads in the JVM.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "jvm.thread_count", + "metric_path": "metrics/jvm/thread_count", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Thread Count", + "value": "${jvm.thread_count}" + } + ], + "properties": { + } + }, + { + "widget_name": "Daemon Thread Count", + "description": "Number of daemon threads in the JVM.", + "widget_type": "NUMBER", + "is_visible": true, + "metrics": [ + { + "name": "jvm.daemon_thread_count", + "metric_path": "metrics/jvm/daemon_thread_count", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Daemon Thread Count", + "value": "${jvm.daemon_thread_count}" + } + ], + "properties": { + } + }, + { + "widget_name": "Runnable Threads", + "description": "Number of runnable threads in the JVM.", + "widget_type": "GRAPH", + "is_visible": false, + "metrics": [ + { + "name": "jvm.thread_states.runnable", + "metric_path": "metrics/jvm/thread_states/runnable", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Runnable Threads", + "value": "${jvm.thread_states.runnable}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Blocked Threads", + "description": "Number of blocked threads in the JVM.", + "widget_type": "GRAPH", + "is_visible": false, + "metrics": [ + { + "name": "jvm.thread_states.blocked", + "metric_path": "metrics/jvm/thread_states/blocked", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Blocked Threads", + "value": "${jvm.thread_states.blocked}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Timed-Waiting Threads", + "description": "Number of timed-waiting threads in the JVM.", + "widget_type": "GRAPH", + "is_visible": false, + "metrics": [ + { + "name": "jvm.thread_states.timed_waiting", + "metric_path": "metrics/jvm/thread_states/timed_waiting", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Timed-Waiting Threads", + "value": "${jvm.thread_states.timed_waiting}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + }, + { + "widget_name": "Terminated Threads", + "description": "Number of terminated threads in the JVM.", + "widget_type": "GRAPH", + "is_visible": false, + "metrics": [ + { + "name": "jvm.thread_states.terminated", + "metric_path": "metrics/jvm/thread_states/terminated", + "service_name": "NIFI", + "component_name": "NIFI_MASTER" + } + ], + "values": [ + { + "name": "Terminated Threads", + "value": "${jvm.thread_states.terminated}" + } + ], + "properties": { + "graph_type": "LINE", + "time_range": "1" + } + } + ] + } + ] +} http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py new file mode 100644 index 0000000..8f0743b --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/hooks/after_install.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import sys +import os +from ambari_server.serverConfiguration import get_ambari_properties, get_resources_location +from resource_management.core import sudo + +def main(): + properties = get_ambari_properties() + if properties == -1: + print >> sys.stderr, "Error getting ambari properties" + return -1 + + resources_location = get_resources_location(properties) + views_dir = os.path.join(resources_location, "views") + + for file in os.listdir(views_dir): + path = os.path.join(views_dir, file) + if os.path.isfile(path): + if "ambari-admin" in path or "storm-view" in path: + print "Keeping views jar : " + path + else: + print "Deleting views jar : " + path + sudo.unlink(path) + else: + print "Deleting views directory : " + path + sudo.rmtree(path) + return 0 + +if __name__ == "__main__": + exit (main()) http://git-wip-us.apache.org/repos/asf/ambari/blob/37e71db7/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json ---------------------------------------------------------------------- diff --git a/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json new file mode 100644 index 0000000..865561e --- /dev/null +++ b/contrib/management-packs/hdf-ambari-mpack/src/main/resources/mpack.json @@ -0,0 +1,30 @@ +{ + "type" : "full-release", + "name" : "hdf-ambari-mpack", + "version": "${project.version}", + "description" : "HDF Ambari Management Pack", + "prerequisites": { + "min-ambari-version" : "${minAmbariVersion}", + "max-ambari-version" : "${maxAmbariVersion}" + }, + "hooks": [ + { + "name": "after-install", + "type": "python", + "script": "hooks/after_install.py" + } + ], + "artifacts": [ + { + "name" : "hdf-service-definitions", + "type" : "service-definitions", + "source_dir": "common-services" + }, + { + "name" : "hdf-stack-definitions", + "type" : "stack-definitions", + "source_dir": "stacks" + } + ] +} +
