AMBARI-7162. Refactor ambari-web code and ambari-agent code to use cluster-env configuration. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e7131fa3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e7131fa3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e7131fa3 Branch: refs/heads/branch-alerts-dev Commit: e7131fa370540945fae6e14858f66226f04c7f41 Parents: 981adc6 Author: Jaimin Jetly <jai...@hortonworks.com> Authored: Fri Sep 5 11:09:14 2014 -0700 Committer: Jaimin Jetly <jai...@hortonworks.com> Committed: Fri Sep 5 11:09:14 2014 -0700 ---------------------------------------------------------------------- .../HDP/1.3.2/configuration/cluster-env.xml | 5 + .../1.3.2/hooks/after-INSTALL/scripts/params.py | 5 +- .../1.3.2/hooks/before-ANY/scripts/params.py | 3 +- .../hooks/before-INSTALL/scripts/params.py | 11 +- .../1.3.2/hooks/before-START/scripts/params.py | 11 +- .../scripts/shared_initialization.py | 121 +++++++++--------- .../services/GANGLIA/package/scripts/params.py | 2 +- .../services/HBASE/package/scripts/params.py | 11 +- .../services/HDFS/configuration/hadoop-env.xml | 32 +---- .../services/HDFS/package/scripts/params.py | 9 +- .../services/HIVE/package/scripts/params.py | 9 +- .../MAPREDUCE/package/scripts/params.py | 9 +- .../services/NAGIOS/package/scripts/params.py | 5 +- .../services/OOZIE/package/scripts/params.py | 9 +- .../services/PIG/package/scripts/params.py | 9 +- .../services/SQOOP/package/scripts/params.py | 11 +- .../services/WEBHCAT/package/scripts/params.py | 9 +- .../ZOOKEEPER/package/scripts/params.py | 9 +- .../HDP/2.0.6/configuration/cluster-env.xml | 5 + .../2.0.6/hooks/after-INSTALL/scripts/params.py | 8 +- .../scripts/shared_initialization.py | 48 +++---- .../2.0.6/hooks/before-ANY/scripts/params.py | 3 +- .../hooks/before-INSTALL/scripts/params.py | 11 +- .../2.0.6/hooks/before-START/scripts/params.py | 12 +- .../scripts/shared_initialization.py | 124 ++++++++++--------- .../services/FLUME/package/scripts/params.py | 2 +- .../services/GANGLIA/package/scripts/params.py | 2 +- .../services/HBASE/package/scripts/params.py | 10 +- .../services/HDFS/configuration/hadoop-env.xml | 27 ---- .../services/HDFS/package/scripts/params.py | 9 +- .../services/HIVE/package/scripts/params.py | 9 +- .../services/NAGIOS/package/scripts/params.py | 6 +- .../services/OOZIE/package/scripts/params.py | 9 +- .../services/PIG/package/scripts/params.py | 9 +- .../services/SQOOP/package/scripts/params.py | 10 +- .../services/WEBHCAT/package/scripts/params.py | 9 +- .../services/YARN/package/scripts/params.py | 9 +- .../ZOOKEEPER/package/scripts/params.py | 9 +- .../services/FALCON/package/scripts/params.py | 9 +- .../services/STORM/package/scripts/params.py | 7 +- .../services/TEZ/package/scripts/params.py | 2 +- .../services/YARN/package/scripts/params.py | 9 +- .../services/FALCON/package/scripts/params.py | 9 +- .../services/STORM/package/scripts/params.py | 7 +- .../2.1/services/TEZ/package/scripts/params.py | 2 +- .../1.3.2/configs/default.hbasedecom.json | 24 ++-- .../python/stacks/1.3.2/configs/default.json | 25 ++-- .../1.3.2/configs/default.non_gmetad_host.json | 22 ++-- .../stacks/1.3.2/configs/default_client.json | 24 ++-- .../python/stacks/1.3.2/configs/secured.json | 24 ++-- .../stacks/1.3.2/configs/secured_client.json | 25 ++-- .../2.0.6/configs/default.hbasedecom.json | 24 ++-- .../python/stacks/2.0.6/configs/default.json | 23 ++-- .../2.0.6/configs/default.non_gmetad_host.json | 24 ++-- .../stacks/2.0.6/configs/default_client.json | 24 ++-- .../python/stacks/2.0.6/configs/ha_default.json | 24 ++-- .../python/stacks/2.0.6/configs/ha_secured.json | 35 +++--- .../2.0.6/configs/rebalancehdfs_default.json | 24 ++-- .../python/stacks/2.0.6/configs/secured.json | 25 ++-- .../stacks/2.0.6/configs/secured_client.json | 25 ++-- .../test/python/stacks/2.1/configs/default.json | 22 ++-- .../test/python/stacks/2.1/configs/secured.json | 25 ++-- .../app/controllers/main/admin/security.js | 42 ++++--- .../main/admin/security/add/step3.js | 3 +- .../controllers/main/admin/security/disable.js | 4 +- .../security/security_progress_controller.js | 8 ++ .../main/service/reassign_controller.js | 6 +- ambari-web/app/controllers/wizard.js | 66 +++++----- .../app/controllers/wizard/step8_controller.js | 16 ++- ambari-web/app/data/HDP2/secure_configs.js | 12 +- ambari-web/app/data/HDP2/secure_properties.js | 20 +-- ambari-web/app/data/HDP2/site_properties.js | 29 +++-- ambari-web/app/data/secure_configs.js | 10 +- ambari-web/app/data/secure_properties.js | 23 +--- ambari-web/app/data/service_configs.js | 1 + ambari-web/app/data/site_properties.js | 31 +++-- ambari-web/app/utils/ajax/ajax.js | 4 + ambari-web/app/utils/config.js | 83 ++++++++++--- .../main/admin/security/disable_test.js | 6 +- .../controllers/main/admin/security_test.js | 18 +-- 80 files changed, 754 insertions(+), 703 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/configuration/cluster-env.xml index 6b97346..d41ff98 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/configuration/cluster-env.xml @@ -43,6 +43,11 @@ <description>User executing service checks</description> </property> <property> + <name>smokeuser_keytab</name> + <value>/etc/security/keytabs/smokeuser.headless.keytab</value> + <description>Path to smoke test user keytab file</description> + </property> + <property> <name>user_group</name> <value>hadoop</value> <property-type>GROUP</property-type> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py index 9fc0322..4ba0ea7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py @@ -24,8 +24,7 @@ import os config = Script.get_config() #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] #hadoop params @@ -60,4 +59,4 @@ hadoop_libexec_dir = "/usr/lib/hadoop/libexec" #users and groups hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['hadoop-env']['user_group'] \ No newline at end of file +user_group = config['configurations']['cluster-env']['user_group'] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py index 0c6a415..af67cb6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py @@ -22,8 +22,7 @@ from resource_management import * config = Script.get_config() tmp_dir = Script.get_tmp_dir() -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py index dc60927..c7380f4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py @@ -38,8 +38,7 @@ if System.get_instance().os_family == "suse": else: jsvc_path = "/usr/libexec/bigtop-utils" #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #hadoop params hadoop_conf_dir = "/etc/hadoop/conf" @@ -72,12 +71,12 @@ hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_p #users and groups hbase_user = config['configurations']['hbase-env']['hbase_user'] nagios_user = config['configurations']['nagios-env']['nagios_user'] -smoke_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] gmond_user = config['configurations']['ganglia-env']["gmond_user"] -user_group = config['configurations']['hadoop-env']['user_group'] -proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] +user_group = config['configurations']['cluster-env']['user_group'] +proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") nagios_group = config['configurations']['nagios-env']['nagios_group'] #hosts @@ -119,7 +118,7 @@ if has_ganglia_server: ganglia_server_host = ganglia_server_hosts[0] hbase_tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] -ignore_groupsusers_create = default("/configurations/hadoop-env/ignore_groupsusers_create", False) +ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") if has_hbase_masters: http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py index e68f7c4..afbacbb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py @@ -24,13 +24,12 @@ import os config = Script.get_config() #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #users and groups hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] #hosts hostname = config["hostname"] @@ -48,6 +47,7 @@ namenode_host = default("/clusterHostInfo/namenode_host", []) zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) +has_namenode = not len(namenode_host) == 0 has_resourcemanager = not len(rm_host) == 0 has_slaves = not len(slave_hosts) == 0 has_nagios = not len(hagios_server_hosts) == 0 @@ -67,7 +67,8 @@ is_slave = hostname in slave_hosts if has_ganglia_server: ganglia_server_host = ganglia_server_hosts[0] #hadoop params -hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") +if has_namenode: + hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") hadoop_lib_home = "/usr/lib/hadoop/lib" hadoop_conf_dir = "/etc/hadoop/conf" hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] @@ -93,7 +94,7 @@ ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'] ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'] ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'] -if 'rca_enabled' in config['configurations']['mapred-env']: +if has_namenode and 'rca_enabled' in config['configurations']['mapred-env']: rca_enabled = config['configurations']['mapred-env']['rca_enabled'] else: rca_enabled = False http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py index e545b02..669414c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/shared_initialization.py @@ -33,55 +33,57 @@ def setup_hadoop(): install_snappy() - #directories - Directory(params.hdfs_log_dir_prefix, - recursive=True, - owner='root', - group='root' - ) - Directory(params.hadoop_pid_dir_prefix, - recursive=True, - owner='root', - group='root' - ) - - #files - if params.security_enabled: - tc_owner = "root" - else: - tc_owner = params.hdfs_user - File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'), - owner=tc_owner, - content=Template("commons-logging.properties.j2") + if params.has_namenode: + #directories + Directory(params.hdfs_log_dir_prefix, + recursive=True, + owner='root', + group='root' + ) + Directory(params.hadoop_pid_dir_prefix, + recursive=True, + owner='root', + group='root' ) - health_check_template = "health_check" #for stack 1 use 'health_check' - File(os.path.join(params.hadoop_conf_dir, "health_check"), - owner=tc_owner, - content=Template(health_check_template + ".j2") - ) + #files + if params.security_enabled: + tc_owner = "root" + else: + tc_owner = params.hdfs_user - log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties") - if (params.log4j_props != None): - File(log4j_filename, - mode=0644, - group=params.user_group, - owner=params.hdfs_user, - content=params.log4j_props + File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'), + owner=tc_owner, + content=Template("commons-logging.properties.j2") + ) + + health_check_template = "health_check" #for stack 1 use 'health_check' + File(os.path.join(params.hadoop_conf_dir, "health_check"), + owner=tc_owner, + content=Template(health_check_template + ".j2") ) - elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): - File(log4j_filename, - mode=0644, - group=params.user_group, + + log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties") + if (params.log4j_props != None): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + ) + + File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), owner=params.hdfs_user, + content=Template("hadoop-metrics2.properties.j2") ) - File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), - owner=params.hdfs_user, - content=Template("hadoop-metrics2.properties.j2") - ) - def setup_database(): """ Load DB @@ -113,33 +115,34 @@ def setup_configs(): """ import params - File(params.task_log4j_properties_location, - content=StaticFile("task-log4j.properties"), - mode=0755 - ) - - Link('/usr/lib/hadoop/lib/hadoop-tools.jar', - to = '/usr/lib/hadoop/hadoop-tools.jar' - ) - - if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')): - File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'), - owner=params.hdfs_user, - group=params.user_group + if params.has_namenode: + File(params.task_log4j_properties_location, + content=StaticFile("task-log4j.properties"), + mode=0755 ) - if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')): - File(os.path.join(params.hadoop_conf_dir, 'masters'), - owner=params.hdfs_user, - group=params.user_group + Link('/usr/lib/hadoop/lib/hadoop-tools.jar', + to = '/usr/lib/hadoop/hadoop-tools.jar' ) + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')): + File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'), + owner=params.hdfs_user, + group=params.user_group + ) + + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')): + File(os.path.join(params.hadoop_conf_dir, 'masters'), + owner=params.hdfs_user, + group=params.user_group + ) + # generate_include_file() def generate_include_file(): import params - if params.dfs_hosts and params.has_slaves: + if params.has_namenode and params.dfs_hosts and params.has_slaves: include_hosts_list = params.slave_hosts File(params.dfs_hosts, content=Template("include_hosts_list.j2"), http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/GANGLIA/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/GANGLIA/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/GANGLIA/package/scripts/params.py index 50df082..0417980 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/GANGLIA/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/GANGLIA/package/scripts/params.py @@ -22,7 +22,7 @@ import os config = Script.get_config() -user_group = config['configurations']['hadoop-env']["user_group"] +user_group = config['configurations']['cluster-env']["user_group"] ganglia_conf_dir = default("/configurations/ganglia-env/ganglia_conf_dir","/etc/ganglia/hdp") ganglia_dir = "/etc/ganglia" ganglia_runtime_dir = config['configurations']['ganglia-env']["ganglia_runtime_dir"] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py index 65ee7f3..ae85c8b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py @@ -36,10 +36,9 @@ hbase_drain_only = config['commandParams']['mark_draining_only'] hbase_included_hosts = config['commandParams']['included_hosts'] hbase_user = status_params.hbase_user -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -user_group = config['configurations']['hadoop-env']['user_group'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +user_group = config['configurations']['cluster-env']['user_group'] # this is "hadoop-metrics2-hbase.properties" for 2.x stacks metric_prop_file_name = "hadoop-metrics.properties" @@ -68,7 +67,7 @@ ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_h rs_hosts = config['clusterHostInfo']['slave_hosts'] #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves -smoke_test_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_test_user = config['configurations']['cluster-env']['smokeuser'] smokeuser_permissions = "RWXCA" service_check_data = functions.get_unique_id_and_date() @@ -79,7 +78,7 @@ if security_enabled: master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) if security_enabled: http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml index 3e42d2f..1de475e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/configuration/hadoop-env.xml @@ -62,47 +62,17 @@ <description>DataNode maximum Java heap size</description> </property> <property> + <property> <name>proxyuser_group</name> <value>users</value> <property-type>GROUP</property-type> <description>Proxy user group.</description> </property> - - <property> - <name>security_enabled</name> - <value>false</value> - <description>Hadoop Security</description> - </property> - <property> - <name>kerberos_domain</name> - <value>EXAMPLE.COM</value> - <description>Kerberos realm.</description> - </property> - - <property> <name>hdfs_user</name> <value>hdfs</value> <property-type>USER</property-type> <description>User to run HDFS as</description> </property> - <property> - <name>ignore_groupsusers_create</name> - <value>false</value> - <description>Whether to ignores failures on users and group creation</description> - </property> - <property> - <name>smokeuser</name> - <value>ambari-qa</value> - <property-type>USER</property-type> - <description>User executing service checks</description> - </property> - <property> - <name>user_group</name> - <value>hadoop</value> - <property-type>GROUP</property-type> - <description>Proxy user group.</description> - </property> - <!-- hadoop-env.sh --> <property> <name>content</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py index e1a8b8f..a7dc196 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py @@ -27,9 +27,8 @@ tmp_dir = Script.get_tmp_dir() ulimit_cmd = "ulimit -c unlimited; " #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] #exclude file @@ -87,11 +86,11 @@ oozie_user = config['configurations']['oozie-env']['oozie_user'] webhcat_user = config['configurations']['hive-env']['hcat_user'] hcat_user = config['configurations']['hive-env']['hcat_user'] hive_user = config['configurations']['hive-env']['hive_user'] -smoke_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] mapred_user = config['configurations']['mapred-env']['mapred_user'] hdfs_user = status_params.hdfs_user -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] nagios_group = config['configurations']['nagios-env']['nagios_group'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py index 3e3b3cc..e38f059 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py @@ -57,13 +57,12 @@ hive_server_host = config['clusterHostInfo']['hive_server_host'][0] hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000") hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}") -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] smoke_test_sql = format("{tmp_dir}/hiveserver2.sql") smoke_test_path = format("{tmp_dir}/hiveserver2Smoke.sh") -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file'] @@ -94,7 +93,7 @@ java_share_dir = '/usr/share/java' driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") target = format("{hive_lib}/{jdbc_jar_name}") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py index ddcb48b..5b44cca 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py @@ -36,17 +36,16 @@ tasktracker_pid_file = status_params.tasktracker_pid_file hadoop_libexec_dir = '/usr/lib/hadoop/libexec' hadoop_bin = "/usr/lib/hadoop/bin" -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] mapred_log_dir_prefix = hdfs_log_dir_prefix mapred_local_dir = config['configurations']['mapred-site']['mapred.local.dir'] update_exclude_file_only = config['commandParams']['update_exclude_file_only'] hadoop_jar_location = "/usr/lib/hadoop/" -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) #exclude file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py index c721454..2384e3a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py @@ -79,8 +79,7 @@ clientPort = config['configurations']['zookeeper-env']['clientPort'] #ZK java64_home = config['hostLevelParams']['java_home'] check_cpu_on = is_jdk_greater_6(java64_home) -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] nagios_keytab_path = default("/configurations/nagios-env/nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab") kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) @@ -109,7 +108,7 @@ nagios_user = config['configurations']['nagios-env']['nagios_user'] nagios_group = config['configurations']['nagios-env']['nagios_group'] nagios_web_login = config['configurations']['nagios-env']['nagios_web_login'] nagios_web_password = config['configurations']['nagios-env']['nagios_web_password'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] nagios_contact = config['configurations']['nagios-env']['nagios_contact'] namenode_host = default("/clusterHostInfo/namenode_host", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py index 70aee3b..4c53c4d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/params.py @@ -27,10 +27,10 @@ tmp_dir = Script.get_tmp_dir() ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] oozie_user = config['configurations']['oozie-env']['oozie_user'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] conf_dir = "/etc/oozie/conf" hadoop_conf_dir = "/etc/hadoop/conf" -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] jdk_location = config['hostLevelParams']['jdk_location'] check_db_connection_jar_name = "DBConnectionVerification.jar" check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") @@ -44,13 +44,12 @@ hadoop_jar_location = "/usr/lib/hadoop/" ext_js_path = "/usr/share/HDP-oozie/ext.zip" oozie_libext_dir = "/usr/lib/oozie/libext" lzo_enabled = config['configurations']['mapred-env']['lzo_enabled'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'] oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'] -smokeuser_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] oozie_keytab = config['configurations']['oozie-env']['oozie_keytab'] oracle_driver_jar_name = "ojdbc6.jar" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py index e84070c..111e158 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/PIG/package/scripts/params.py @@ -28,11 +28,10 @@ tmp_dir = Script.get_tmp_dir() pig_conf_dir = "/etc/pig/conf" hadoop_conf_dir = "/etc/hadoop/conf" hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -user_group = config['configurations']['hadoop-env']['user_group'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +user_group = config['configurations']['cluster-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) pig_env_sh_template = config['configurations']['pig-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/SQOOP/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/SQOOP/package/scripts/params.py index c76a95c..5c31aae 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/SQOOP/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/SQOOP/package/scripts/params.py @@ -20,10 +20,9 @@ from resource_management import * config = Script.get_config() -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -user_group = config['configurations']['hadoop-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +user_group = config['configurations']['cluster-env']['user_group'] sqoop_env_sh_template = config['configurations']['sqoop-env']['content'] sqoop_conf_dir = "/usr/lib/sqoop/conf" @@ -33,6 +32,6 @@ zoo_conf_dir = "/etc/zookeeper" sqoop_lib = "/usr/lib/sqoop/lib" sqoop_user = config['configurations']['sqoop-env']['sqoop_user'] -keytab_path = config['configurations']['hadoop-env']['keytab_path'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +keytab_path = config['configurations']['cluster-env']['keytab_path'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py index 94d905c..174c92c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/params.py @@ -41,15 +41,14 @@ hadoop_conf_dir = config['configurations']['webhcat-site']['templeton.hadoop.con templeton_jar = config['configurations']['webhcat-site']['templeton.jar'] hadoop_home = '/usr' -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] webhcat_server_host = config['clusterHostInfo']['webhcat_server_host'] webhcat_apps_dir = "/apps/webhcat" -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) #hdfs directories http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/ZOOKEEPER/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/ZOOKEEPER/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/ZOOKEEPER/package/scripts/params.py index 6874c19..2dc3792 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/ZOOKEEPER/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/ZOOKEEPER/package/scripts/params.py @@ -30,7 +30,7 @@ config_dir = "/etc/zookeeper/conf" zk_user = config['configurations']['zookeeper-env']['zk_user'] hostname = config['hostname'] zk_bin = '/usr/lib/zookeeper/bin' -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] zk_env_sh_template = config['configurations']['zookeeper-env']['content'] smoke_script = "/usr/lib/zookeeper/bin/zkCli.sh" @@ -63,11 +63,10 @@ zookeeper_hosts.sort() zk_keytab_path = config['configurations']['zookeeper-env']['zookeeper_keytab_path'] zk_server_jaas_file = format("{config_dir}/zookeeper_jaas.conf") zk_client_jaas_file = format("{config_dir}/zookeeper_client_jaas.conf") -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) #log4j.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml index 6b97346..d41ff98 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml @@ -43,6 +43,11 @@ <description>User executing service checks</description> </property> <property> + <name>smokeuser_keytab</name> + <value>/etc/security/keytabs/smokeuser.headless.keytab</value> + <description>Path to smoke test user keytab file</description> + </property> + <property> <name>user_group</name> <value>hadoop</value> <property-type>GROUP</property-type> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py index e741247..d537199 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py @@ -24,8 +24,7 @@ import os config = Script.get_config() #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] #hadoop params @@ -63,4 +62,7 @@ mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefi #users and groups hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['hadoop-env']['user_group'] \ No newline at end of file +user_group = config['configurations']['cluster-env']['user_group'] + +namenode_host = default("/clusterHostInfo/namenode_host", []) +has_namenode = not len(namenode_host) == 0 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py index 0b40ea6..1ba0ae7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py @@ -21,30 +21,32 @@ from resource_management import * def setup_hadoop_env(): import params - if params.security_enabled: - tc_owner = "root" - else: - tc_owner = params.hdfs_user - Directory(params.hadoop_conf_empty_dir, - recursive=True, - owner='root', - group='root' - ) - Link(params.hadoop_conf_dir, - to=params.hadoop_conf_empty_dir, - not_if=format("ls {hadoop_conf_dir}") - ) - File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), - owner=tc_owner, - content=InlineTemplate(params.hadoop_env_sh_template) - ) + if params.has_namenode: + if params.security_enabled: + tc_owner = "root" + else: + tc_owner = params.hdfs_user + Directory(params.hadoop_conf_empty_dir, + recursive=True, + owner='root', + group='root' + ) + Link(params.hadoop_conf_dir, + to=params.hadoop_conf_empty_dir, + not_if=format("ls {hadoop_conf_dir}") + ) + File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'), + owner=tc_owner, + content=InlineTemplate(params.hadoop_env_sh_template) + ) def setup_config(): import params - XmlConfig("core-site.xml", - conf_dir=params.hadoop_conf_dir, - configurations=params.config['configurations']['core-site'], - configuration_attributes=params.config['configuration_attributes']['core-site'], - owner=params.hdfs_user, - group=params.user_group + if params.has_namenode: + XmlConfig("core-site.xml", + conf_dir=params.hadoop_conf_dir, + configurations=params.config['configurations']['core-site'], + configuration_attributes=params.config['configuration_attributes']['core-site'], + owner=params.hdfs_user, + group=params.user_group ) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py index 0c6a415..af67cb6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py @@ -22,8 +22,7 @@ from resource_management import * config = Script.get_config() tmp_dir = Script.get_tmp_dir() -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py index 8bb1565..01789a7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py @@ -29,13 +29,13 @@ tmp_dir = Script.get_tmp_dir() #users and groups hbase_user = config['configurations']['hbase-env']['hbase_user'] nagios_user = config['configurations']['nagios-env']['nagios_user'] -smoke_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] gmond_user = config['configurations']['ganglia-env']["gmond_user"] tez_user = config['configurations']['tez-env']["tez_user"] -user_group = config['configurations']['hadoop-env']['user_group'] -proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] +user_group = config['configurations']['cluster-env']['user_group'] +proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") nagios_group = config['configurations']['nagios-env']['nagios_group'] #hosts @@ -84,8 +84,7 @@ if has_ganglia_server: hbase_tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] @@ -94,7 +93,7 @@ jdk_name = default("/hostLevelParams/jdk_name", None) # None when jdk is already jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user jce_location = config['hostLevelParams']['jdk_location'] jdk_location = config['hostLevelParams']['jdk_location'] -ignore_groupsusers_create = default("/configurations/hadoop-env/ignore_groupsusers_create", False) +ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") if has_hbase_masters: http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py index 151303c..fc525a6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py @@ -24,15 +24,14 @@ import os config = Script.get_config() #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] #users and groups mapred_user = config['configurations']['mapred-env']['mapred_user'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] yarn_user = config['configurations']['yarn-env']['yarn_user'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] #hosts hostname = config["hostname"] @@ -50,6 +49,7 @@ namenode_host = default("/clusterHostInfo/namenode_host", []) zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) +has_namenode = not len(namenode_host) == 0 has_resourcemanager = not len(rm_host) == 0 has_slaves = not len(slave_hosts) == 0 has_nagios = not len(hagios_server_hosts) == 0 @@ -69,7 +69,9 @@ is_slave = hostname in slave_hosts if has_ganglia_server: ganglia_server_host = ganglia_server_hosts[0] #hadoop params -hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") + +if has_namenode: + hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}") hadoop_lib_home = "/usr/lib/hadoop/lib" hadoop_conf_dir = "/etc/hadoop/conf" hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] @@ -94,7 +96,7 @@ ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0] ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0] ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0] -if 'rca_enabled' in config['configurations']['hadoop-env']: +if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']: rca_enabled = config['configurations']['hadoop-env']['rca_enabled'] else: rca_enabled = False http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py index 4c08afb..0d00aca 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py @@ -34,58 +34,59 @@ def setup_hadoop(): install_snappy() #directories - Directory(params.hdfs_log_dir_prefix, - recursive=True, - owner='root', - group='root' - ) - Directory(params.hadoop_pid_dir_prefix, - recursive=True, - owner='root', - group='root' - ) + if params.has_namenode: + Directory(params.hdfs_log_dir_prefix, + recursive=True, + owner='root', + group='root' + ) + Directory(params.hadoop_pid_dir_prefix, + recursive=True, + owner='root', + group='root' + ) #this doesn't needed with stack 1 - Directory(params.hadoop_tmp_dir, - recursive=True, - owner=params.hdfs_user, - ) + Directory(params.hadoop_tmp_dir, + recursive=True, + owner=params.hdfs_user, + ) #files - if params.security_enabled: - tc_owner = "root" - else: - tc_owner = params.hdfs_user - - File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'), - owner=tc_owner, - content=Template('commons-logging.properties.j2') - ) - - health_check_template = "health_check-v2" #for stack 1 use 'health_check' - File(os.path.join(params.hadoop_conf_dir, "health_check"), - owner=tc_owner, - content=Template(health_check_template + ".j2") - ) + if params.security_enabled: + tc_owner = "root" + else: + tc_owner = params.hdfs_user + + File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'), + owner=tc_owner, + content=Template('commons-logging.properties.j2') + ) - log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties") - if (params.log4j_props != None): - File(log4j_filename, - mode=0644, - group=params.user_group, - owner=params.hdfs_user, - content=params.log4j_props + health_check_template = "health_check-v2" #for stack 1 use 'health_check' + File(os.path.join(params.hadoop_conf_dir, "health_check"), + owner=tc_owner, + content=Template(health_check_template + ".j2") ) - elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): - File(log4j_filename, - mode=0644, - group=params.user_group, + + log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties") + if (params.log4j_props != None): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + content=params.log4j_props + ) + elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))): + File(log4j_filename, + mode=0644, + group=params.user_group, + owner=params.hdfs_user, + ) + + File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), owner=params.hdfs_user, + content=Template("hadoop-metrics2.properties.j2") ) - File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"), - owner=params.hdfs_user, - content=Template("hadoop-metrics2.properties.j2") - ) - def setup_database(): """ Load DB @@ -113,33 +114,34 @@ def setup_database(): def setup_configs(): """ - Creates configs for services DHFS mapred + Creates configs for services HDFS mapred """ import params - File(params.task_log4j_properties_location, - content=StaticFile("task-log4j.properties"), - mode=0755 - ) - - if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')): - File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'), - owner=params.hdfs_user, - group=params.user_group - ) - if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')): - File(os.path.join(params.hadoop_conf_dir, 'masters'), - owner=params.hdfs_user, - group=params.user_group + if params.has_namenode: + File(params.task_log4j_properties_location, + content=StaticFile("task-log4j.properties"), + mode=0755 ) + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')): + File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'), + owner=params.hdfs_user, + group=params.user_group + ) + if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')): + File(os.path.join(params.hadoop_conf_dir, 'masters'), + owner=params.hdfs_user, + group=params.user_group + ) + generate_include_file() def generate_include_file(): import params - if params.dfs_hosts and params.has_slaves: + if params.has_namenode and params.dfs_hosts and params.has_slaves: include_hosts_list = params.slave_hosts File(params.dfs_hosts, content=Template("include_hosts_list.j2"), http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py index bcf4d0d..128eed4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/FLUME/package/scripts/params.py @@ -21,7 +21,7 @@ from resource_management import * config = Script.get_config() -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] security_enabled = False http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/GANGLIA/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/GANGLIA/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/GANGLIA/package/scripts/params.py index 5ae90e1..e155122 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/GANGLIA/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/GANGLIA/package/scripts/params.py @@ -22,7 +22,7 @@ import os config = Script.get_config() -user_group = config['configurations']['hadoop-env']["user_group"] +user_group = config['configurations']['cluster-env']["user_group"] ganglia_conf_dir = default("/configurations/ganglia-env/ganglia_conf_dir", "/etc/ganglia/hdp") ganglia_dir = "/etc/ganglia" ganglia_runtime_dir = config['configurations']['ganglia-env']["ganglia_runtime_dir"] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py index c11b4d4..364649c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py @@ -36,10 +36,9 @@ hbase_drain_only = config['commandParams']['mark_draining_only'] hbase_included_hosts = config['commandParams']['included_hosts'] hbase_user = status_params.hbase_user -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] _authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -user_group = config['configurations']['hadoop-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] # this is "hadoop-metrics.properties" for 1.x stacks metric_prop_file_name = "hadoop-metrics2-hbase.properties" @@ -74,9 +73,10 @@ if 'slave_hosts' in config['clusterHostInfo']: else: rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') -smoke_test_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_test_user = config['configurations']['cluster-env']['smokeuser'] smokeuser_permissions = "RWXCA" service_check_data = functions.get_unique_id_and_date() +user_group = config['configurations']['cluster-env']["user_group"] if security_enabled: _hostname_lowercase = config['hostname'].lower() @@ -85,7 +85,7 @@ if security_enabled: master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) if security_enabled: http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml index 1bd08d5..b3935d7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/configuration/hadoop-env.xml @@ -68,38 +68,11 @@ <description>Proxy user group.</description> </property> <property> - <name>security_enabled</name> - <value>false</value> - <description>Hadoop Security</description> - </property> - <property> - <name>kerberos_domain</name> - <value>EXAMPLE.COM</value> - <description>Kerberos realm.</description> - </property> - <property> <name>hdfs_user</name> <value>hdfs</value> <property-type>USER</property-type> <description>User to run HDFS as</description> </property> - <property> - <name>ignore_groupsusers_create</name> - <value>false</value> - <description>Whether to ignores failures on users and group creation</description> - </property> - <property> - <name>smokeuser</name> - <value>ambari-qa</value> - <property-type>USER</property-type> - <description>User executing service checks</description> - </property> - <property> - <name>user_group</name> - <value>hadoop</value> - <property-type>GROUP</property-type> - <description>Proxy user group.</description> - </property> <!-- hadoop-env.sh --> <property> http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py index f6905e8..47ee8ca 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py @@ -27,9 +27,8 @@ tmp_dir = Script.get_tmp_dir() ulimit_cmd = "ulimit -c unlimited; " #security params -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] falcon_user = config['configurations']['falcon-env']['falcon_user'] @@ -91,12 +90,12 @@ oozie_user = config['configurations']['oozie-env']['oozie_user'] webhcat_user = config['configurations']['hive-env']['hcat_user'] hcat_user = config['configurations']['hive-env']['hcat_user'] hive_user = config['configurations']['hive-env']['hive_user'] -smoke_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] mapred_user = config['configurations']['mapred-env']['mapred_user'] hdfs_user = status_params.hdfs_user hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] nagios_group = config['configurations']['nagios-env']['nagios_group'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py index 42f52c6..5db3970 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py @@ -61,14 +61,13 @@ hive_server_host = config['clusterHostInfo']['hive_server_host'][0] hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000") hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}") -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] smoke_test_sql = format("{tmp_dir}/hiveserver2.sql") smoke_test_path = format("{tmp_dir}/hiveserver2Smoke.sh") -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] fs_root = config['configurations']['core-site']['fs.defaultFS'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) hive_metastore_keytab_path = config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file'] @@ -99,7 +98,7 @@ java_share_dir = '/usr/share/java' driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}") hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] artifact_dir = format("{tmp_dir}/AMBARI-artifacts/") target = format("{hive_lib}/{jdbc_jar_name}") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py index d2683f8..87c6d3e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py @@ -141,9 +141,7 @@ clientPort = config['configurations']['zookeeper-env']['clientPort'] #ZK java64_home = config['hostLevelParams']['java_home'] check_cpu_on = is_jdk_greater_6(java64_home) -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') - +security_enabled = config['configurations']['cluster-env']['security_enabled'] nagios_keytab_path = default("/configurations/nagios-env/nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab") kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) @@ -204,7 +202,7 @@ nagios_user = config['configurations']['nagios-env']['nagios_user'] nagios_group = config['configurations']['nagios-env']['nagios_group'] nagios_web_login = config['configurations']['nagios-env']['nagios_web_login'] nagios_web_password = config['configurations']['nagios-env']['nagios_web_password'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] nagios_contact = config['configurations']['nagios-env']['nagios_contact'] # - test for HDFS or HCFS (glusterfs) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py index ebcc984..260063f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/params.py @@ -26,10 +26,10 @@ config = Script.get_config() tmp_dir = Script.get_tmp_dir() oozie_user = config['configurations']['oozie-env']['oozie_user'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] conf_dir = "/etc/oozie/conf" hadoop_conf_dir = "/etc/hadoop/conf" -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] jdk_location = config['hostLevelParams']['jdk_location'] check_db_connection_jar_name = "DBConnectionVerification.jar" check_db_connection_jar = format("/usr/lib/ambari-agent/{check_db_connection_jar_name}") @@ -42,13 +42,12 @@ hadoop_jar_location = "/usr/lib/hadoop/" # for HDP1 it's "/usr/share/HDP-oozie/ext.zip" ext_js_path = "/usr/share/HDP-oozie/ext-2.2.zip" oozie_libext_dir = "/usr/lib/oozie/libext" -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file'] oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal'] -smokeuser_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] oozie_keytab = config['configurations']['oozie-env']['oozie_keytab'] oozie_env_sh_template = config['configurations']['oozie-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py index 570401d..1b522b8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py @@ -29,11 +29,10 @@ pig_conf_dir = "/etc/pig/conf" hadoop_conf_dir = "/etc/hadoop/conf" hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -user_group = config['configurations']['hadoop-env']['user_group'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +user_group = config['configurations']['cluster-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) pig_env_sh_template = config['configurations']['pig-env']['content'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py index 3f50bef..144a587 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py @@ -21,10 +21,9 @@ from resource_management import * config = Script.get_config() -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -user_group = config['configurations']['hadoop-env']['user_group'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +user_group = config['configurations']['cluster-env']['user_group'] sqoop_env_sh_template = config['configurations']['sqoop-env']['content'] sqoop_conf_dir = "/usr/lib/sqoop/conf" @@ -34,6 +33,5 @@ zoo_conf_dir = "/etc/zookeeper" sqoop_lib = "/usr/lib/sqoop/lib" sqoop_user = config['configurations']['sqoop-env']['sqoop_user'] -keytab_path = config['configurations']['hadoop-env']['keytab_path'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py index fa9460b..a7959f0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/params.py @@ -47,15 +47,14 @@ hadoop_conf_dir = config['configurations']['webhcat-site']['templeton.hadoop.con templeton_jar = config['configurations']['webhcat-site']['templeton.jar'] hadoop_home = '/usr' -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] webhcat_server_host = config['clusterHostInfo']['webhcat_server_host'] webhcat_apps_dir = "/apps/webhcat" -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) hcat_hdfs_user_dir = format("/user/{hcat_user}") http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py index e3be1f4..8c5fa9b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py @@ -32,10 +32,9 @@ mapred_user = status_params.mapred_user yarn_user = status_params.yarn_user hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] +security_enabled = config['configurations']['cluster-env']['security_enabled'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) rm_hosts = config['clusterHostInfo']['rm_host'] @@ -92,7 +91,7 @@ yarn_job_summary_log = format("{yarn_log_dir_prefix}/{yarn_user}/hadoop-mapreduc mapred_bin = "/usr/lib/hadoop-mapreduce/sbin" yarn_bin = "/usr/lib/hadoop-yarn/sbin" -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] limits_conf_dir = "/etc/security/limits.d" hadoop_conf_dir = "/etc/hadoop/conf" yarn_container_bin = "/usr/lib/hadoop-yarn/bin" http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py index 6874c19..2dc3792 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/ZOOKEEPER/package/scripts/params.py @@ -30,7 +30,7 @@ config_dir = "/etc/zookeeper/conf" zk_user = config['configurations']['zookeeper-env']['zk_user'] hostname = config['hostname'] zk_bin = '/usr/lib/zookeeper/bin' -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] zk_env_sh_template = config['configurations']['zookeeper-env']['content'] smoke_script = "/usr/lib/zookeeper/bin/zkCli.sh" @@ -63,11 +63,10 @@ zookeeper_hosts.sort() zk_keytab_path = config['configurations']['zookeeper-env']['zookeeper_keytab_path'] zk_server_jaas_file = format("{config_dir}/zookeeper_jaas.conf") zk_client_jaas_file = format("{config_dir}/zookeeper_client_jaas.conf") -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] -smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] -smokeuser = config['configurations']['hadoop-env']['smokeuser'] +smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] +smokeuser = config['configurations']['cluster-env']['smokeuser'] kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"]) #log4j.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/package/scripts/params.py index 93e292d..056380b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/FALCON/package/scripts/params.py @@ -25,9 +25,9 @@ config = Script.get_config() oozie_user = config['configurations']['oozie-env']['oozie_user'] falcon_user = config['configurations']['falcon-env']['falcon_user'] -smoke_user = config['configurations']['hadoop-env']['smokeuser'] +smoke_user = config['configurations']['cluster-env']['smokeuser'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] java_home = config['hostLevelParams']['java_home'] @@ -45,14 +45,13 @@ falcon_host = config['clusterHostInfo']['falcon_server_hosts'][0] falcon_port = config['configurations']['falcon-env']['falcon_port'] falcon_runtime_properties = config['configurations']['falcon-runtime.properties'] falcon_startup_properties = config['configurations']['falcon-startup.properties'] -smokeuser_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] +smokeuser_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] falcon_env_sh_template = config['configurations']['falcon-env']['content'] falcon_webapp_dir = '/var/lib/falcon/webapp' flacon_apps_dir = '/apps/falcon' #for create_hdfs_directory -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] hostname = config["hostname"] hadoop_conf_dir = "/etc/hadoop/conf" hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] http://git-wip-us.apache.org/repos/asf/ambari/blob/e7131fa3/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/params.py index def5120..3f5c790 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/params.py @@ -29,7 +29,7 @@ log_dir = config['configurations']['storm-env']['storm_log_dir'] pid_dir = status_params.pid_dir conf_dir = "/etc/storm/conf" local_dir = config['configurations']['storm-site']['storm.local.dir'] -user_group = config['configurations']['hadoop-env']['user_group'] +user_group = config['configurations']['cluster-env']['user_group'] java64_home = config['hostLevelParams']['java_home'] nimbus_host = config['configurations']['storm-site']['nimbus.host'] nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port'] @@ -48,12 +48,11 @@ if 'ganglia_server_host' in config['clusterHostInfo'] and \ else: ganglia_installed = False -_authentication = config['configurations']['core-site']['hadoop.security.authentication'] -security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') +security_enabled = config['configurations']['cluster-env']['security_enabled'] if security_enabled: _hostname_lowercase = config['hostname'].lower() - _kerberos_domain = config['configurations']['hadoop-env']['kerberos_domain'] + _kerberos_domain = config['configurations']['cluster-env']['kerberos_domain'] _storm_principal_name = config['configurations']['storm-env']['storm_principal_name'] storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase) storm_keytab_path = config['configurations']['storm-env']['storm_keytab']
