Updated Branches: refs/heads/trunk 0d30fb95e -> 236d94ed2
AMBARI-4168. 2.0._ fails deploy due to non existant user/group (Arsen Babych via aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/236d94ed Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/236d94ed Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/236d94ed Branch: refs/heads/trunk Commit: 236d94ed2b33015fde5d74cff13ff655c3c2b227 Parents: 0d30fb9 Author: Andrew Onischuk <[email protected]> Authored: Tue Dec 24 11:49:05 2013 -0800 Committer: Andrew Onischuk <[email protected]> Committed: Tue Dec 24 11:49:05 2013 -0800 ---------------------------------------------------------------------- .../before-INSTALL/files/changeToSecureUid.sh | 50 +++++++++ .../2.0.8/hooks/before-INSTALL/scripts/hook.py | 35 +++++++ .../hooks/before-INSTALL/scripts/params.py | 81 +++++++++++++++ .../scripts/shared_initialization.py | 102 +++++++++++++++++++ .../before-START/files/changeToSecureUid.sh | 50 --------- .../2.0.8/hooks/before-START/scripts/hook.py | 1 - .../2.0.8/hooks/before-START/scripts/params.py | 15 +-- .../scripts/shared_initialization.py | 83 --------------- .../HDFS/package/scripts/hdfs_namenode.py | 2 + 9 files changed, 271 insertions(+), 148 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/files/changeToSecureUid.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/files/changeToSecureUid.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/files/changeToSecureUid.sh new file mode 100644 index 0000000..4872a10 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/files/changeToSecureUid.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +username=$1 +directories=$2 + +function find_available_uid() { + for ((i=1001; i<=2000; i++)) + do + grep -q $i /etc/passwd + if [ "$?" -ne 0 ] + then + newUid=$i + break + fi + done +} + +find_available_uid + +if [ $newUid -eq 0 ] +then + echo "Failed to find Uid between 1000 and 2000" + exit 1 +fi + +dir_array=($(echo $directories | sed 's/,/\n/g')) +old_uid=$(id -u $username) +echo "Changing uid of $username from $old_uid to $newUid" +echo "Changing directory permisions for ${dir_array[@]}" +usermod -u $newUid $username && for dir in ${dir_array[@]} ; do chown -Rh $newUid $dir ; done +exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/hook.py new file mode 100644 index 0000000..0708c15 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/hook.py @@ -0,0 +1,35 @@ +##!/usr/bin/env python2.6 +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * +from shared_initialization import * + +#TODO this must be "CONFIGURE" hook when CONFIGURE command will be implemented +class BeforeConfigureHook(Hook): + + def hook(self, env): + import params + + env.set_params(params) + setup_users() + +if __name__ == "__main__": + BeforeConfigureHook().execute() http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/params.py new file mode 100644 index 0000000..fa19ca3 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/params.py @@ -0,0 +1,81 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * +from resource_management.core.system import System +import os + +config = Script.get_config() + +#users and groups +yarn_user = config['configurations']['global']['yarn_user'] +hbase_user = config['configurations']['global']['hbase_user'] +nagios_user = config['configurations']['global']['nagios_user'] +oozie_user = config['configurations']['global']['oozie_user'] +webhcat_user = config['configurations']['global']['hcat_user'] +hcat_user = config['configurations']['global']['hcat_user'] +hive_user = config['configurations']['global']['hive_user'] +smoke_user = config['configurations']['global']['smokeuser'] +mapred_user = config['configurations']['global']['mapred_user'] +hdfs_user = config['configurations']['global']['hdfs_user'] +zk_user = config['configurations']['global']['zk_user'] +gmetad_user = config['configurations']['global']["gmetad_user"] +gmond_user = config['configurations']['global']["gmond_user"] + +user_group = config['configurations']['global']['user_group'] +proxyuser_group = config['configurations']['global']['proxyuser_group'] +nagios_group = config['configurations']['global']['nagios_group'] +smoke_user_group = "users" +mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) + +#hosts +hostname = config["hostname"] +rm_host = default("/clusterHostInfo/rm_host", []) +slave_hosts = default("/clusterHostInfo/slave_hosts", []) +hagios_server_hosts = default("/clusterHostInfo/nagios_server_host", []) +oozie_servers = default("/clusterHostInfo/oozie_server", []) +hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", []) +hive_server_host = default("/clusterHostInfo/hive_server_host", []) +hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) +hs_host = default("/clusterHostInfo/hs_host", []) +jtnode_host = default("/clusterHostInfo/jtnode_host", []) +namenode_host = default("/clusterHostInfo/namenode_host", []) +zk_hosts = default("/clusterHostInfo/zookeeper_hosts", []) +ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) + +has_resourcemanager = not len(rm_host) == 0 +has_slaves = not len(slave_hosts) == 0 +has_nagios = not len(hagios_server_hosts) == 0 +has_oozie_server = not len(oozie_servers) == 0 +has_hcat_server_host = not len(hcat_server_hosts) == 0 +has_hive_server_host = not len(hive_server_host) == 0 +has_hbase_masters = not len(hbase_master_hosts) == 0 +has_zk_host = not len(zk_hosts) == 0 +has_ganglia_server = not len(ganglia_server_hosts) == 0 + +is_namenode_master = hostname in namenode_host +is_jtnode_master = hostname in jtnode_host +is_rmnode_master = hostname in rm_host +is_hsnode_master = hostname in hs_host +is_hbase_master = hostname in hbase_master_hosts +is_slave = hostname in slave_hosts +if has_ganglia_server: + ganglia_server_host = ganglia_server_hosts[0] + +hbase_tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/shared_initialization.py new file mode 100644 index 0000000..a127678 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-INSTALL/scripts/shared_initialization.py @@ -0,0 +1,102 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os + +from resource_management import * + +def setup_users(): + """ + Creates users before cluster installation + """ + import params + + Group(params.user_group) + Group(params.smoke_user_group) + Group(params.proxyuser_group) + User(params.smoke_user, + gid=params.user_group, + groups=[params.proxyuser_group] + ) + smoke_user_dirs = format( + "/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") + set_uid(params.smoke_user, smoke_user_dirs) + + if params.has_hbase_masters: + User(params.hbase_user, + gid = params.user_group, + groups=[params.user_group]) + hbase_user_dirs = format( + "/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}") + set_uid(params.hbase_user, hbase_user_dirs) + + if params.has_nagios: + Group(params.nagios_group) + User(params.nagios_user, + gid=params.nagios_group) + + if params.has_oozie_server: + User(params.oozie_user, + gid = params.user_group) + + if params.has_hcat_server_host: + User(params.webhcat_user, + gid = params.user_group) + User(params.hcat_user, + gid = params.user_group) + + if params.has_hive_server_host: + User(params.hive_user, + gid = params.user_group) + + if params.has_resourcemanager: + User(params.yarn_user, + gid = params.user_group) + + if params.has_ganglia_server: + Group(params.gmetad_user) + Group(params.gmond_user) + User(params.gmond_user, + gid=params.user_group, + groups=[params.gmond_user]) + User(params.gmetad_user, + gid=params.user_group, + groups=[params.gmetad_user]) + + User(params.hdfs_user, + gid=params.user_group, + groups=[params.user_group] + ) + User(params.mapred_user, + gid=params.user_group, + groups=[params.user_group] + ) + if params.has_zk_host: + User(params.zk_user, + gid=params.user_group) + +def set_uid(user, user_dirs): + """ + user_dirs - comma separated directories + """ + File("/tmp/changeUid.sh", + content=StaticFile("changeToSecureUid.sh"), + mode=0555) + Execute(format("/tmp/changeUid.sh {user} {user_dirs} 2>/dev/null"), + not_if = format("test $(id -u {user}) -gt 1000")) http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/files/changeToSecureUid.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/files/changeToSecureUid.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/files/changeToSecureUid.sh deleted file mode 100644 index 4872a10..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/files/changeToSecureUid.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -username=$1 -directories=$2 - -function find_available_uid() { - for ((i=1001; i<=2000; i++)) - do - grep -q $i /etc/passwd - if [ "$?" -ne 0 ] - then - newUid=$i - break - fi - done -} - -find_available_uid - -if [ $newUid -eq 0 ] -then - echo "Failed to find Uid between 1000 and 2000" - exit 1 -fi - -dir_array=($(echo $directories | sed 's/,/\n/g')) -old_uid=$(id -u $username) -echo "Changing uid of $username from $old_uid to $newUid" -echo "Changing directory permisions for ${dir_array[@]}" -usermod -u $newUid $username && for dir in ${dir_array[@]} ; do chown -Rh $newUid $dir ; done -exit 0 http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/hook.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/hook.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/hook.py index 61e3720..e11bfac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/hook.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/hook.py @@ -30,7 +30,6 @@ class BeforeConfigureHook(Hook): env.set_params(params) setup_java() - setup_users() setup_hadoop() setup_configs() http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/params.py index 75203ea..a5c5c35 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/params.py @@ -47,24 +47,11 @@ dfs_secondary_namenode_kerberos_principal = config['configurations']['hdfs-site' dfs_journalnode_kerberos_internal_spnego_principal = config['configurations']['hdfs-site']['dfs.journalnode.kerberos.internal.spnego.principal'] #users and groups -yarn_user = config['configurations']['global']['yarn_user'] -hbase_user = config['configurations']['global']['hbase_user'] -nagios_user = config['configurations']['global']['nagios_user'] -oozie_user = config['configurations']['global']['oozie_user'] -webhcat_user = config['configurations']['global']['hcat_user'] -hcat_user = config['configurations']['global']['hcat_user'] -hive_user = config['configurations']['global']['hive_user'] -smoke_user = config['configurations']['global']['smokeuser'] mapred_user = config['configurations']['global']['mapred_user'] hdfs_user = config['configurations']['global']['hdfs_user'] -zk_user = config['configurations']['global']['zk_user'] -gmetad_user = config['configurations']['global']["gmetad_user"] -gmond_user = config['configurations']['global']["gmond_user"] +yarn_user = config['configurations']['global']['yarn_user'] user_group = config['configurations']['global']['user_group'] -proxyuser_group = config['configurations']['global']['proxyuser_group'] -nagios_group = config['configurations']['global']['nagios_group'] -smoke_user_group = "users" mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group) #snmp http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/shared_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/shared_initialization.py index cb064f3..736cf21 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/shared_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/hooks/before-START/scripts/shared_initialization.py @@ -52,77 +52,6 @@ def setup_java(): path = ['/bin/','/usr/bin'] ) -def setup_users(): - """ - Creates users before cluster installation - """ - import params - - Group(params.user_group) - Group(params.smoke_user_group) - Group(params.proxyuser_group) - User(params.smoke_user, - gid=params.user_group, - groups=[params.proxyuser_group] - ) - smoke_user_dirs = format( - "/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") - set_uid(params.smoke_user, smoke_user_dirs) - - if params.has_hbase_masters: - User(params.hbase_user, - gid = params.user_group, - groups=[params.user_group]) - hbase_user_dirs = format( - "/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}") - set_uid(params.hbase_user, hbase_user_dirs) - - if params.has_nagios: - Group(params.nagios_group) - User(params.nagios_user, - gid=params.nagios_group) - - if params.has_oozie_server: - User(params.oozie_user, - gid = params.user_group) - - if params.has_hcat_server_host: - User(params.webhcat_user, - gid = params.user_group) - User(params.hcat_user, - gid = params.user_group) - - if params.has_hive_server_host: - User(params.hive_user, - gid = params.user_group) - - if params.has_resourcemanager: - User(params.yarn_user, - gid = params.user_group) - - if params.has_ganglia_server: - Group(params.gmetad_user) - Group(params.gmond_user) - User(params.gmond_user, - gid=params.user_group, - groups=[params.gmond_user]) - User(params.gmetad_user, - gid=params.user_group, - groups=[params.gmetad_user]) - - User(params.hdfs_user, - gid=params.user_group, - groups=[params.user_group] - ) - User(params.mapred_user, - gid=params.user_group, - groups=[params.user_group] - ) - if params.has_zk_host: - User(params.zk_user, - gid=params.user_group) - - def setup_hadoop(): """ Setup hadoop files and directories @@ -331,18 +260,6 @@ def setup_configs(): generate_exlude_file() generate_include_file() - -def set_uid(user, user_dirs): - """ - user_dirs - comma separated directories - """ - File("/tmp/changeUid.sh", - content=StaticFile("changeToSecureUid.sh"), - mode=0555) - Execute(format("/tmp/changeUid.sh {user} {user_dirs} 2>/dev/null"), - not_if = format("test $(id -u {user}) -gt 1000")) - - def update_log4j_props(file): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/236d94ed/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py index a4772cb..1f2abc8 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.8/services/HDFS/package/scripts/hdfs_namenode.py @@ -36,6 +36,8 @@ def namenode(action=None, format=True): service( action="start", name="namenode", user=params.hdfs_user, keytab=params.dfs_namenode_keytab_file, + create_pid_dir=True, + create_log_dir=True, principal=params.dfs_namenode_kerberos_principal )
