Repository: ambari Updated Branches: refs/heads/trunk e4cadbb15 -> 9e93c476d
AMBARI-22040. configs.py does not work properly when dealing with files (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9e93c476 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9e93c476 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9e93c476 Branch: refs/heads/trunk Commit: 9e93c476ddd8d4397f550062fd1645ac5422ed2e Parents: e4cadbb Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Fri Sep 22 14:06:17 2017 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Fri Sep 22 14:06:17 2017 +0300 ---------------------------------------------------------------------- .../src/main/resources/scripts/configs.py | 219 +++++++++++---- .../src/main/resources/scripts/configs.sh | 272 +------------------ 2 files changed, 168 insertions(+), 323 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9e93c476/ambari-server/src/main/resources/scripts/configs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/configs.py b/ambari-server/src/main/resources/scripts/configs.py index b524461..8d4de1c 100644 --- a/ambari-server/src/main/resources/scripts/configs.py +++ b/ambari-server/src/main/resources/scripts/configs.py @@ -18,11 +18,19 @@ See the License for the specific language governing permissions and limitations under the License. ''' +import optparse +from optparse import OptionGroup import sys import urllib2 import time import json import base64 +import xml +import xml.etree.ElementTree as ET +import os +import logging + +logger = logging.getLogger('AmbariConfig') HTTP_PROTOCOL = 'http' HTTPS_PROTOCOL = 'https' @@ -106,14 +114,14 @@ def create_new_desired_config(cluster, config_type, properties, attributes, acce new_config[CLUSTERS][DESIRED_CONFIGS][ATTRIBUTES] = attributes request_body = json.dumps(new_config) new_file = 'doSet_{0}.json'.format(new_tag) - print '### PUTting json into: {0}'.format(new_file) + logger.info('### PUTting json into: {0}'.format(new_file)) output_to_file(new_file)(new_config) accessor(CLUSTERS_URL.format(cluster), PUT_REQUEST_TYPE, request_body) - print '### NEW Site:{0}, Tag:{1}'.format(config_type, new_tag) + logger.info('### NEW Site:{0}, Tag:{1}'.format(config_type, new_tag)) def get_current_config(cluster, config_type, accessor): config_tag = get_config_tag(cluster, config_type, accessor) - print "### on (Site:{0}, Tag:{1})".format(config_type, config_tag) + logger.info("### on (Site:{0}, Tag:{1})".format(config_type, config_tag)) response = accessor(CONFIGURATION_URL.format(cluster, config_type, config_tag)) config_by_tag = json.loads(response) current_config = config_by_tag[ITEMS][0] @@ -130,6 +138,41 @@ def update_specific_property(config_name, config_value): return properties, attributes return update +def update_from_xml(config_file): + def update(cluster, config_type, accessor): + return read_xml_data_to_map(config_file) + return update + +# Used DOM parser to read data into a map +def read_xml_data_to_map(path): + configurations = {} + properties_attributes = {} + tree = ET.parse(path) + root = tree.getroot() + for properties in root.getiterator('property'): + name = properties.find('name') + value = properties.find('value') + final = properties.find('final') + + if name != None: + name_text = name.text if name.text else "" + else: + logger.warn("No name is found for one of the properties in {0}, ignoring it".format(path)) + continue + + if value != None: + value_text = value.text if value.text else "" + else: + logger.warn("No value is found for \"{0}\" in {1}, using empty string for it".format(name_text, path)) + value_text = "" + + if final != None: + final_text = final.text if final.text else "" + properties_attributes[name_text] = final_text + + configurations[name_text] = value_text + return configurations, {"final" : properties_attributes} + def update_from_file(config_file): def update(cluster, config_type, accessor): try: @@ -138,12 +181,12 @@ def update_from_file(config_file): except Exception as e: raise Exception('Cannot find file "{0}" to PUT'.format(config_file)) try: - file_properties = json.loads('{' + file_content + '}') + file_properties = json.loads(file_content) except Exception as e: raise Exception('File "{0}" should be in the following JSON format ("properties_attributes" is optional):\n{1}'.format(config_file, FILE_FORMAT)) new_properties = file_properties.get(PROPERTIES, {}) new_attributes = file_properties.get(ATTRIBUTES, {}) - print '### PUTting file: "{0}"'.format(config_file) + logger.info('### PUTting file: "{0}"'.format(config_file)) return new_properties, new_attributes return update @@ -156,26 +199,14 @@ def delete_specific_property(config_name): return properties, attributes return update -def format_json(dictionary, tab_level=0): - output = '' - tab = ' ' * 2 * tab_level - for key, value in dictionary.iteritems(): - output += ',\n{0}"{1}": '.format(tab, key) - if isinstance(value, dict): - output += '{\n' + format_json(value, tab_level + 1) + tab + '}' - else: - output += '"{0}"'.format(value) - output += '\n' - return output[2:] - def output_to_file(filename): def output(config): with open(filename, 'w') as out_file: - out_file.write(format_json(config)) + json.dump(config, out_file, indent=2) return output def output_to_console(config): - print format_json(config) + print json.dumps(config, indent=2) def get_config(cluster, config_type, accessor, output): properties, attributes = get_current_config(cluster, config_type, accessor) @@ -185,70 +216,152 @@ def get_config(cluster, config_type, accessor, output): output(config) def set_properties(cluster, config_type, args, accessor): - print '### Performing "set" content:' - if len(args) == 0: - raise UsageException("Not enough arguments. Expected config key and value or filename.") + logger.info('### Performing "set":') if len(args) == 1: config_file = args[0] - updater = update_from_file(config_file) - print '### from file "{0}"'.format(config_file) + root, ext = os.path.splitext(config_file) + if ext == ".xml": + updater = update_from_xml(config_file) + elif ext == ".json": + updater = update_from_file(config_file) + else: + logger.error("File extension {0} doesn't supported".format(ext)) + return -1 + logger.info('### from file {0}'.format(config_file)) else: config_name = args[0] config_value = args[1] updater = update_specific_property(config_name, config_value) - print '### new property - "{0}":"{1}"'.format(config_name, config_value) + logger.info('### new property - "{0}":"{1}"'.format(config_name, config_value)) update_config(cluster, config_type, updater, accessor) + return 0 def delete_properties(cluster, config_type, args, accessor): - print '### Performing "delete":' + logger.info('### Performing "delete":') if len(args) == 0: - raise UsageException("Not enough arguments. Expected config key.") + logger.error("Not enough arguments. Expected config key.") + return -1 config_name = args[0] - print '### on property "{0}"'.format(config_name) + logger.info('### on property "{0}"'.format(config_name)) update_config(cluster, config_type, delete_specific_property(config_name), accessor) + return 0 + def get_properties(cluster, config_type, args, accessor): - print '### Performing "get" content:' + logger.info("### Performing \"get\" content:") if len(args) > 0: filename = args[0] output = output_to_file(filename) - print '### to file "{0}"'.format(filename) + logger.info('### to file "{0}"'.format(filename)) else: output = output_to_console get_config(cluster, config_type, accessor, output) + return 0 def main(): - if len(sys.argv) < 9: - raise UsageException('Not enough arguments.') - args = sys.argv[1:] - user = args[0] - password = args[1] - port = args[2] - protocol = args[3] - action = args[4] - host = args[5] - cluster = args[6] - config_type = args[7] - action_args = args[8:] + + parser = optparse.OptionParser(usage="usage: %prog [options]") + + login_options_group = OptionGroup(parser, "To specify credentials please use \"-e\" OR \"-u\" and \"-p'\"") + login_options_group.add_option("-u", "--user", dest="user", default="admin", help="Optional user ID to use for authentication. Default is 'admin'") + login_options_group.add_option("-p", "--password", dest="password", default="admin", help="Optional password to use for authentication. Default is 'admin'") + login_options_group.add_option("-e", "--credentials-file", dest="credentials_file", help="Optional file with user credentials separated by new line.") + parser.add_option_group(login_options_group) + + parser.add_option("-t", "--port", dest="port", default="8080", help="Optional port number for Ambari server. Default is '8080'. Provide empty string to not use port.") + parser.add_option("-s", "--protocol", dest="protocol", default="http", help="Optional support of SSL. Default protocol is 'http'") + parser.add_option("-a", "--action", dest="action", help="Script action: <get>, <set>, <delete>") + parser.add_option("-l", "--host", dest="host", help="Server external host name") + parser.add_option("-n", "--cluster", dest="cluster", help="Name given to cluster. Ex: 'c1'") + parser.add_option("-c", "--config-type", dest="config_type", help="One of the various configuration types in Ambari. Ex: core-site, hdfs-site, mapred-queue-acls, etc.") + + config_options_group = OptionGroup(parser, "To specify property(s) please use \"-f\" OR \"-k\" and \"-v'\"") + config_options_group.add_option("-f", "--file", dest="file", help="File where entire configurations are saved to, or read from. Supported extensions (.xml, .json>)") + config_options_group.add_option("-k", "--key", dest="key", help="Key that has to be set or deleted. Not necessary for 'get' action.") + config_options_group.add_option("-v", "--value", dest="value", help="Optional value to be set. Not necessary for 'get' or 'delete' actions.") + parser.add_option_group(config_options_group) + + (options, args) = parser.parse_args() + + logger.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.INFO) + stdout_handler.setFormatter(formatter) + logger.addHandler(stdout_handler) + + # options with default value + + if not options.credentials_file and (not options.user or not options.password): + parser.error("You should use option (-e) to set file with Ambari user credentials OR use (-u) username and (-p) password") + + if options.credentials_file: + if os.path.isfile(options.credentials_file): + try: + with open(options.credentials_file) as credentials_file: + file_content = credentials_file.read() + login_lines = filter(None, file_content.splitlines()) + if len(login_lines) == 2: + user = login_lines[0] + password = login_lines[1] + else: + logger.error("Incorrect content of {0} file. File should contain Ambari username and password separated by new line.".format(options.credentials_file)) + return -1 + except Exception as e: + logger.error("You don't have permissions to {0} file".format(options.credentials_file)) + return -1 + else: + logger.error("File {0} doesn't exist or you don't have permissions.".format(options.credentials_file)) + return -1 + else: + user = options.user + password = options.password + + port = options.port + protocol = options.protocol + + #options without default value + if None in [options.action, options.host, options.cluster, options.config_type]: + parser.error("One of required options is not passed") + + action = options.action + host = options.host + cluster = options.cluster + config_type = options.config_type + accessor = api_accessor(host, user, password, protocol, port) if action == SET_ACTION: - set_properties(cluster, config_type, action_args, accessor) + + if not options.file and (not options.key or not options.value): + parser.error("You should use option (-f) to set file where entire configurations are saved OR (-k) key and (-v) value for one property") + if options.file: + action_args = [options.file] + else: + action_args = [options.key, options.value] + return set_properties(cluster, config_type, action_args, accessor) + elif action == GET_ACTION: - get_properties(cluster, config_type, action_args, accessor) + if options.file: + action_args = [options.file] + else: + action_args = [] + return get_properties(cluster, config_type, action_args, accessor) + elif action == DELETE_ACTION: - delete_properties(cluster, config_type, action_args, accessor) + if not options.key: + parser.error("You should use option (-k) to set property name witch will be deleted") + else: + action_args = [options.key] + return delete_properties(cluster, config_type, action_args, accessor) else: - raise UsageException('Action "{0}" is not supported. Supported actions: "get", "set", "delete".'.format(action)) + logger.error('Action "{0}" is not supported. Supported actions: "get", "set", "delete".'.format(action)) + return -1 if __name__ == "__main__": try: - main() - except UsageException as usage_exc: - print '[ERROR] {0}'.format(usage_exc) - sys.exit(2) - except Exception as exc: - for line in str(exc).split('\n'): - print '[ERROR] {0}'.format(line) + sys.exit(main()) + except (KeyboardInterrupt, EOFError): + print("\nAborting ... Keyboard Interrupt.") sys.exit(1) http://git-wip-us.apache.org/repos/asf/ambari/blob/9e93c476/ambari-server/src/main/resources/scripts/configs.sh ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/scripts/configs.sh b/ambari-server/src/main/resources/scripts/configs.sh index 7364d0e..4141fed 100755 --- a/ambari-server/src/main/resources/scripts/configs.sh +++ b/ambari-server/src/main/resources/scripts/configs.sh @@ -18,273 +18,5 @@ # under the License. # -usage () { - echo ""; - echo "WARNING: THIS SCRIPT IS DEPRECATED AND DOESN'T SUPPORT NEW FEATURES. PLEASE USE configs.py" - echo ""; - echo "Usage: configs.sh [-u userId] [-p password] [-port port] [-s] <ACTION> <AMBARI_HOST> <CLUSTER_NAME> <CONFIG_TYPE> [CONFIG_FILENAME | CONFIG_KEY [CONFIG_VALUE]]"; - echo ""; - echo " [-u userId]: Optional user ID to use for authentication. Default is 'admin'."; - echo " [-p password]: Optional password to use for authentication. Default is 'admin'."; - echo " [-port port]: Optional port number for Ambari server. Default is '8080'. Provide empty string to not use port."; - echo " [-s]: Optional support of SSL. Default is 'false'. Provide empty string to not use SSL."; - echo " <ACTION>: One of 'get', 'set', 'delete'. 'Set' adds/updates as necessary."; - echo " <AMBARI_HOST>: Server external host name"; - echo " <CLUSTER_NAME>: Name given to cluster. Ex: 'c1'" - echo " <CONFIG_TYPE>: One of the various configuration types in Ambari. Ex:global, core-site, hdfs-site, mapred-queue-acls, etc."; - echo " [CONFIG_FILENAME]: File where entire configurations are saved to, or read from. Only applicable to 'get' and 'set' actions"; - echo " [CONFIG_KEY]: Key that has to be set or deleted. Not necessary for 'get' action."; - echo " [CONFIG_VALUE]: Optional value to be set. Not necessary for 'get' or 'delete' actions."; - exit 1; -} - -USERID="admin" -PASSWD="admin" -PORT=":8080" -SSL_URL_PREFIX="" - -if [ "$1" == "-u" ] ; then - USERID=$2; - shift 2; - echo "USERID=$USERID"; -fi - -if [ "$1" == "-p" ] ; then - PASSWD=$2; - shift 2; - echo "PASSWORD=$PASSWD"; -fi - -if [ "$1" == "-port" ] ; then - if [ -z $2 ]; then - PORT=""; - else - PORT=":$2"; - fi - shift 2; - echo "PORT=$PORT"; -fi - -if [ "$1" == "-s" ] ; then - SSL_URL_PREFIX="s" - shift; - echo "SSL is enabled"; -fi - -AMBARIURL="http$SSL_URL_PREFIX://$2$PORT" -CLUSTER=$3 -SITE=$4 -SITETAG='' -CONFIGKEY=$5 -CONFIGVALUE=$6 - -################### -## currentSiteTag() -################### -currentSiteTag () { - currentSiteTag='' - found='' - - #currentSite=`cat ds.json | grep -E "$SITE|tag"`; - currentSite=`curl -k -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$CLUSTER?fields=Clusters/desired_configs" | grep -E "$SITE|tag"`; - for line in $currentSite; do - if [ $line != "{" -a $line != ":" -a $line != '"tag"' ] ; then - if [ -n "$found" -a -z "$currentSiteTag" ]; then - currentSiteTag=$line; - fi - if [ $line == "\"$SITE\"" ]; then - found=$SITE; - fi - fi - done; - if [ -z $currentSiteTag ]; then - errOutput=`curl -k -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$CLUSTER?fields=Clusters/desired_configs"`; - echo "[ERROR] \"$SITE\" not found in server response."; - echo "[ERROR] Output of \`curl -k -s -u $USERID:$PASSWD \"$AMBARIURL/api/v1/clusters/$CLUSTER?fields=Clusters/desired_configs\"\` is:"; - echo $errOutput | while read -r line; do - echo "[ERROR] $line"; - done; - exit 1; - fi - currentSiteTag=`echo $currentSiteTag|cut -d \" -f 2` - SITETAG=$currentSiteTag; -} - -############################################# -## doConfigUpdate() -## @param MODE of update. Either 'set' or 'delete' -############################################# -doConfigUpdate () { - MODE=$1 - currentSiteTag - echo "########## Performing '$MODE' $CONFIGKEY:$CONFIGVALUE on (Site:$SITE, Tag:$SITETAG)"; - propertiesStarted=0 - attributesStarted=0 - currentLevel=0 - curl -k -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$CLUSTER/configurations?type=$SITE&tag=$SITETAG" | while read -r line; do - if [ "$propertiesStarted" -eq 0 -a "$attributesStarted" -eq 0 ]; then - if [ "$line" = "\"properties_attributes\" : {" ]; then - attributesStarted=$currentLevel - elif [ "$line" = "\"properties\" : {" ]; then - propertiesStarted=$currentLevel - fi - fi - if [ "$propertiesStarted" -gt 0 ]; then - if [ "`echo $line | grep -E "},?$"`" ]; then - ## Properties ended - ## Add property - propLen=${#newProperties} - lastChar=${newProperties:$propLen-1:1} - if [ "$MODE" == "delete" ]; then - # Remove the last , - if [ "$lastChar" == "," ]; then - newProperties=${newProperties:0:$propLen-1} - fi - elif [ "$MODE" == "set" ]; then - # Add comma if required - if [ "$lastChar" != "," -a "$lastChar" != "{" ]; then - newProperties="$newProperties," - fi - newProperties="$newProperties \"$CONFIGKEY\" : \"$CONFIGVALUE\"" - fi - newProperties=$newProperties$line - propertiesStarted=0 - elif [ "`echo $line | grep "\\\"$CONFIGKEY\\\""`" ]; then - echo "########## Config found. Skipping origin value" - else - newProperties=$newProperties$line - fi - elif [ "$attributesStarted" -gt 0 ]; then - newProperties=$newProperties$line - fi - if [ "`echo $line | grep -E "{$"`" ]; then - currentLevel=$((currentLevel+1)) - elif [ "`echo $line | grep -E "},?$"`" ]; then - currentLevel=$((currentLevel-1)) - if [ "$currentLevel" == 1 ]; then - # if no properties in current config - if [ "$MODE" == "set" -a -z "$newProperties" ]; then - newProperties="\"properties\" : { \"$CONFIGKEY\" : \"$CONFIGVALUE\"}" - fi - newTag=`date "+%s%N"` - newTag="version${newTag}" - finalJson="{ \"Clusters\": { \"desired_config\": {\"type\": \"$SITE\", \"tag\":\"$newTag\", $newProperties}}}" - newFile="doSet_$newTag.json" - echo "########## PUTting json into: $newFile" - echo "$finalJson" > $newFile - curl -k -u $USERID:$PASSWD -X PUT -H "X-Requested-By: ambari" "$AMBARIURL/api/v1/clusters/$CLUSTER" --data @$newFile - currentSiteTag - echo "########## NEW Site:$SITE, Tag:$SITETAG"; - fi - fi - if [ "$attributesStarted" -eq "$currentLevel" ]; then - attributesStarted=0 - fi - done -} - -############################################# -## doConfigFileUpdate() -## @param File name to PUT on server -############################################# -doConfigFileUpdate () { - FILENAME=$1 - if [ -f $FILENAME ]; then - if [ "1" == "$(grep -En ^\"properties\" $FILENAME | cut -d : -f 1)" ]; then - newTag=`date "+%s%N"` - newTag="version${newTag}" - newProperties=`cat $FILENAME`; - finalJson="{ \"Clusters\": { \"desired_config\": {\"type\": \"$SITE\", \"tag\":\"$newTag\", $newProperties}}}" - newFile="doSet_$newTag.json" - echo "$finalJson" > $newFile - echo "########## PUTting file:\"$FILENAME\" into config(type:\"$SITE\", tag:$newTag) via $newFile" - curl -k -u $USERID:$PASSWD -X PUT -H "X-Requested-By: ambari" "$AMBARIURL/api/v1/clusters/$CLUSTER" --data @$newFile - currentSiteTag - echo "########## NEW Site:$SITE, Tag:$SITETAG"; - else - echo "[ERROR] File \"$FILENAME\" should be in the following JSON format (\"properties_attributes\" is optional):"; - echo "[ERROR] \"properties\": {"; - echo "[ERROR] \"key1\": \"value1\","; - echo "[ERROR] \"key2\": \"value2\","; - echo "[ERROR] },"; - echo "[ERROR] \"properties_attributes\": {"; - echo "[ERROR] \"final\": {"; - echo "[ERROR] \"key1\": \"value1\","; - echo "[ERROR] \"key2\": \"value2\","; - echo "[ERROR] }"; - echo "[ERROR] }"; - exit 1; - fi - else - echo "[ERROR] Cannot find file \"$1\"to PUT"; - exit 1; - fi -} - - -############################################# -## doGet() -## @param Optional filename to save to -############################################# -doGet () { - FILENAME=$1 - if [ -n $FILENAME -a -f $FILENAME ]; then - rm -f $FILENAME - fi - currentSiteTag - echo "########## Performing 'GET' on (Site:$SITE, Tag:$SITETAG)"; - propertiesStarted=0 - curl -k -s -u $USERID:$PASSWD "$AMBARIURL/api/v1/clusters/$CLUSTER/configurations?type=$SITE&tag=$SITETAG" | while read -r line; do - # echo ">>> $line"; - if [ "$propertiesStarted" -eq 0 ]; then - if [ "`echo $line | grep "\"properties\""`" -o "`echo $line | grep "\"properties_attributes\""`" ]; then - propertiesStarted=$currentLevel - fi - fi - if [ "$propertiesStarted" -gt "0" ]; then - if [ -z $FILENAME ]; then - echo "$line" - else - echo "$line" >> $FILENAME - fi - fi - if [ "`echo $line | grep -E "{$"`" ]; then - currentLevel=$((currentLevel+1)) - elif [ "`echo $line | grep -E "},?$"`" ]; then - currentLevel=$((currentLevel-1)) - fi - if [ "$propertiesStarted" -eq "$currentLevel" ]; then - propertiesStarted=0 - fi - done -} - -case "$1" in - set) - if (($# == 6)); then - doConfigUpdate "set" # Individual key - elif (($# == 5)); then - doConfigFileUpdate $5 # File based - else - usage - fi - ;; - get) - if (($# == 4)); then - doGet - elif (($# == 5)); then - doGet $5 - else - usage - fi - ;; - delete) - if (($# != 5)); then - usage - fi - doConfigUpdate "delete" - ;; - *) - usage - ;; -esac +echo "ERROR: THIS SCRIPT IS NO LONGER SUPPORTED. PLEASE USE configs.py INSTEAD" +exit 1 \ No newline at end of file