Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/153706
Change subject: Add script and class to manage HDFS user directories
......................................................................
Add script and class to manage HDFS user directories
This will allow automatic creation of HDFS user directories
based on posix group membership.
Change-Id: I7208dc722a3ccf7a6af884471b212a467191602b
---
A files/hadoop/manage_hdfs_user_directories.sh
A manifests/hadoop/users.pp
2 files changed, 157 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/cdh
refs/changes/06/153706/1
diff --git a/files/hadoop/manage_hdfs_user_directories.sh
b/files/hadoop/manage_hdfs_user_directories.sh
new file mode 100755
index 0000000..796b207
--- /dev/null
+++ b/files/hadoop/manage_hdfs_user_directories.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+set -e
+
+usage_message="Usage $(basename "$0") [--dry-run|-n] [-v|--verbose] <group>"
+
+
+help_message="${usage_message}
+
+Manages HDFS user directories.
+
+This will ensure that users in the specified group all have
+HDFS user (home) directories at hdfs:///user/<username>.
+This script must be run by the HDFS super user 'hdfs'.
+
+Options:
+ -n, --dry-run Don't actually create directories, just
+ output what would have been done. If
+ There are changes needed, this script
+ will exit 3.
+ -v, --verbose Print out hdfs commands that would be run.
+"
+
+dry_run='false'
+verbose='false'
+changes_needed='false'
+
+
+if [ $(whoami) != 'hdfs' ]; then
+ echo "Error: $(basename "$0") must be run as the 'hdfs' user." >&2
+ echo "${usage_message}"
+ exit 1
+fi
+
+
+function validate_group {
+ local group="${1}"
+
+ set +e
+ if [ -z "${group}" ]; then
+ echo "Error: Must specify group" >&2
+ echo "${usage_message}"
+ exit 1
+ fi
+
+ /usr/bin/getent group "${group}" > /dev/null
+ if [ $? -ne 0 ]; then
+ echo "Error: Invalid group. '${group}' does not exist." >&2
+ echo "${usage_message}"
+ exit 1
+ fi
+
+ set -e
+}
+
+function create_hdfs_user_directory {
+ local user="${1}"
+
+ command="hdfs dfs -mkdir /user/${user} && hdfs dfs -chown ${user}:${user}
/user/${user}"
+
+ if [ "${verbose}" == 'true' ]; then
+ echo "${command}"
+ fi
+
+ if [ "${dry_run}" == 'false' ]; then
+ $command
+ fi
+}
+
+
+# Parse CLI args
+while [ $# != 0 ]; do
+
+ case "$1" in
+ -n|--dry-run)
+ dry_run='true'
+ ;;
+ -v|--verbose)
+ verbose='true'
+ ;;
+ -h|--help)
+ echo "${help_message}"
+ exit 0
+ ;;
+ *)
+ if [ $# -gt 1 ]; then
+ exit_unknown "Invalid argument '${1}'"
+ else
+ group="${1}"
+ fi
+ ;;
+ esac
+ shift
+done
+
+
+
+# Make sure group was specified and exists.
+validate_group "${group}"
+
+# Get a list of usernames in $group
+group_members=$(/usr/bin/getent group "${group}" | /usr/bin/awk -F ':' '{print
$NF}' | tr ',' ' ')
+
+# Get a list of existant HDFS user directories.
+hdfs_user_directories=$(/usr/bin/hdfs dfs -ls /user | /usr/bin/awk '{print
$NF}' | /bin/sed 's@/user/@@g')
+
+
+if [ "${dry_run}" == 'true' -a "${verbose}" == 'true' ]; then
+ echo "--dry-run mode on; not actually doing anything."
+fi
+
+# For each user in $group, make sure that
+# user has an hdfs user directory. If they don't,
+# go ahead and create it now
+for user in $group_members; do
+ (echo "${hdfs_user_directories}" | grep -q "${user}") ||
+ (create_hdfs_user_directory "${user}"; changes_needed='true')
+done
+
+if [ "${changes_needed}" == 'true' -a "${dry_run}" == 'true' ]; then
+ if [ "${verbose}" == 'true' ];
+ echo "--dry-run mode on and there were needed changes. Exiting 3."
+ fi
+ exit 3
+fi
diff --git a/manifests/hadoop/users.pp b/manifests/hadoop/users.pp
new file mode 100644
index 0000000..97edd08
--- /dev/null
+++ b/manifests/hadoop/users.pp
@@ -0,0 +1,32 @@
+# == Class cdh::hadoop::users
+# Ensures that all users in the posix group $group
+# have HDFS user directories at /user/<username>
+#
+# == Parameters
+# $group - Group name in which all users should have
+# access to Hadoop. Default: hadoop
+#
+# == Usage
+# The following will ensure that any users in the
+# posix group 'my-analytics-group' have HDFS user
+# directories.
+#
+# class { 'cdh::hadoop::users':
+# group => 'my-analytics-group',
+# }
+#
+class cdh::hadoop::users($group = 'hadoop') {
+ Class['cdh::hadoop'] -> Class['cdh::hadoop::users']
+
+ file { '/usr/local/bin/manage_hdfs_user_directories.sh':
+ source =>
'puppet://modules/cdh/hadoop/manage_hdfs_user_directories.sh',
+ mode => '0755',
+ }
+
+ exec { 'manage_hdfs_user_directories':
+ command => "/usr/local/bin/manage_hdfs_user_directories.sh --verbose
${group}",
+ unless => "/usr/local/bin/manage_hdfs_user_directories.sh --dry-run
${group}"
+ user => 'hdfs',
+ logoutput => true,
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/153706
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7208dc722a3ccf7a6af884471b212a467191602b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/cdh
Gerrit-Branch: master
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
