AMBARI-12602. [PluggableStackDefinition] Create tool for generating pluggable stack definition (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0d400379 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0d400379 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0d400379 Branch: refs/heads/branch-2.1 Commit: 0d4003790ef423b81ba3fb38922ce2b63297eb4a Parents: 3d1b2fe Author: Andrew Onishuk <[email protected]> Authored: Fri Jul 31 14:07:57 2015 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Jul 31 14:07:57 2015 +0300 ---------------------------------------------------------------------- .../GenerateStackDefinition.py | 489 +++++++++++++++++++ .../pluggable_stack_definition/__init__.py | 18 + .../python/pluggable_stack_definition/test.json | 137 ++++++ .../white_labeling/WhiteLabelStackDefinition.py | 274 ----------- .../src/main/python/white_labeling/__init__.py | 18 - .../src/main/python/white_labeling/test.json | 61 --- 6 files changed, 644 insertions(+), 353 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py new file mode 100644 index 0000000..100e453 --- /dev/null +++ b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +import getopt +import json +import os +import shutil +import xml.etree.ElementTree as ET +from xml.dom import minidom +import re +from os.path import join + + +class _named_dict(dict): + """ + Allow to get dict items using attribute notation, eg dict.attr == dict['attr'] + """ + + def __init__(self, _dict): + + def repl_list(_list): + for i, e in enumerate(_list): + if isinstance(e, list): + _list[i] = repl_list(e) + if isinstance(e, dict): + _list[i] = _named_dict(e) + return _list + + dict.__init__(self, _dict) + for key, value in self.iteritems(): + if isinstance(value, dict): + self[key] = _named_dict(value) + if isinstance(value, list): + self[key] = repl_list(value) + + def __getattr__(self, item): + if item in self: + return self[item] + else: + dict.__getattr__(self, item) + + +def copy_tree(src, dest, exclude=None, post_copy=None): + """ + Copy files form src to dest. + + :param src: source folder + :param dest: destination folder + :param exclude: list for excluding, eg [".xml"] will exclude all xml files + :param post_copy: callable that accepts source and target paths and will be called after copying + """ + if not os.path.exists(src): + return + for item in os.listdir(src): + if exclude: + skip = False + for ex in exclude: + if item.endswith(ex): + skip = True + break + if skip: + continue + + _src = os.path.join(src, item) + _dest = os.path.join(dest, item) + + if os.path.isdir(_src): + if not os.path.exists(_dest): + os.makedirs(_dest) + copy_tree(_src, _dest, exclude, post_copy) + else: + _dest_dirname = os.path.dirname(_dest) + if not os.path.exists(_dest_dirname): + os.makedirs(_dest_dirname) + shutil.copy(_src, _dest) + + if post_copy: + post_copy(_src, _dest) + +# TODO: Add support for default text replacements based on stack version mapping +def process_replacements(file_path, config_data, stack_version_changes): + file_data = open(file_path, 'r').read().decode(encoding='utf-8') + # replace user defined values + if 'textReplacements' in config_data: + for _from, _to in config_data['textReplacements']: + file_data = file_data.replace(_from, _to) + # replace stack version changes + # it can be dangerous to replace versions in xml files, because it can be a part of version of some package or service + # eg 2.1.2.1 with stack version change 2.1->3.0 will result in 3.0.3.0 + if not file_path.endswith(".xml"): + for _from, _to in stack_version_changes.iteritems(): + file_data = file_data.replace(_from, _to) + # preform common replacements + if 'performCommonReplacements' in config_data and config_data.performCommonReplacements: + for from_version, to_version in stack_version_changes.iteritems(): + file_data = file_data.replace('HDP-'+from_version, config_data.stackName+"-"+to_version) + file_data = file_data.replace('HDP '+from_version, config_data.stackName+" "+to_version) + file_data = file_data.replace('hdp', config_data.stackName.lower()) + file_data = file_data.replace('HDP', config_data.stackName) + with open(file_path, "w") as target: + target.write(file_data.encode(encoding='utf-8')) + return file_path + + +def process_metainfo(file_path, config_data, stack_version_changes, common_services = []): + tree = ET.parse(file_path) + root = tree.getroot() + + if root.find('versions') is not None or root.find('services') is None: + # process stack metainfo.xml + extends_tag = root.find('extends') + if extends_tag is not None: + version = extends_tag.text + if version in stack_version_changes: + extends_tag.text = stack_version_changes[version] + tree.write(file_path) + + current_version = file_path.split(os.sep)[-2] + modify_active_tag = False + active_tag_value = None + for stack in config_data.versions: + if stack.version == current_version and 'active' in stack: + modify_active_tag = True + active_tag_value = stack.active + break + + if modify_active_tag: + versions_tag = root.find('versions') + if versions_tag is None: + versions_tag = ET.SubElement(root, 'versions') + active_tag = versions_tag.find('active') + if active_tag is None: + active_tag = ET.SubElement(versions_tag, 'active') + active_tag.text = active_tag_value + tree.write(file_path) + else: + # Process service metainfo.xml + services_tag = root.find('services') + if services_tag is not None: + for service_tag in services_tag.findall('service'): + name = service_tag.find('name').text + #################################################################################################### + # Add common service to be copied. + #################################################################################################### + extends_tag = service_tag.find('extends') + if extends_tag is not None: + common_services.append(extends_tag.text) + service_version_tag = service_tag.find('version') + # file_path <resource_dir>/stacks/<stack_name>/<stack_version>/services/<service_name>/metainfo.xml + split_path = file_path.split(os.path.sep) + split_path_len = len(split_path) + path_stack_version = split_path[split_path_len - 4] + for stack in config_data.versions: + if stack.version == path_stack_version: + for service in stack.services: + if service.name == name: + ###################################################################################################### + # Update service version + ###################################################################################################### + if 'version' in service: + #################################################################################################### + # If explicit service version is provided in the config, override the service version + #################################################################################################### + if service_version_tag is None: + service_version_tag = ET.SubElement(service_tag, 'version') + service_version_tag.text = service.version + else: + #################################################################################################### + # Default: Update service version by replacing the stack version in the service version string + # Example (ex: HDFS 2.7.1.2.3 -> HDFS 2.7.1.3.1) + #################################################################################################### + if service_version_tag is not None: + service_version_split = service_version_tag.text.split(".") + if len(stack.baseVersion) < len(service_version_tag.text): + version_suffix = service_version_tag.text[-len(stack.baseVersion):] + if version_suffix == stack.baseVersion: + version_prefix = service_version_tag.text[0:-len(stack.baseVersion)] + service_version_tag.text = version_prefix + stack.version + ###################################################################################################### + # Update service version + ###################################################################################################### + osSpecifics_tag = service_tag.find('osSpecifics') + if 'packages' in service: + if osSpecifics_tag is not None: + service_tag.remove(osSpecifics_tag) + osSpecifics_tag = ET.SubElement(service_tag, 'osSpecifics') + for item in service['packages']: + osSpecific_tag = ET.SubElement(osSpecifics_tag, 'osSpecific') + family = item['family'] + osFamily_tag = ET.SubElement(osSpecific_tag, 'osFamily') + osFamily_tag.text = family + packages_tag = ET.SubElement(osSpecific_tag, 'packages') + for package in item['packages']: + package_tag = ET.SubElement(packages_tag, 'package') + name_tag = ET.SubElement(package_tag, 'name') + if isinstance(package, basestring): + name_tag.text = package + else: + name_tag.text = package['name'] + if 'skipUpgrade' in package: + skipUpgrade_tag = ET.SubElement(package_tag, 'skipUpgrade') + skipUpgrade_tag.text = package['skipUpgrade'] + else: + #################################################################################################### + # Default: Update package version by replacing stack version in the package name + # Example (ex: falcon_2_2_* -> falcon_3_0_*, falcon-2-3-* -> falcon-3-1-*) + #################################################################################################### + for packages_tag in service_tag.getiterator('packages'): + for package_tag in packages_tag.getiterator('package'): + name_tag = package_tag.find('name') + for base_version in stack_version_changes: + version = stack_version_changes[base_version] + dash_base_version = base_version.replace('.', '-') + dash_version = version.replace('.', '-') + underscore_base_version = base_version.replace('.', '_') + underscore_version = version.replace('.', '_') + if dash_base_version in name_tag.text: + name_tag.text = name_tag.text.replace(dash_base_version, dash_version) + break + elif underscore_base_version in name_tag.text: + name_tag.text = name_tag.text.replace(underscore_base_version, underscore_version) + break + tree.write(file_path) + return file_path + +def process_upgrade_xml(file_path, target_version, config_data, stack_version_changes): + # change versions in xml + tree = ET.parse(file_path) + root = tree.getroot() + + for target_tag in root.findall('target'): + version = '.'.join([el for el in target_tag.text.split('.') if el != '*']) + if version in stack_version_changes: + target_tag.text = target_tag.text.replace(version, stack_version_changes[version]) + tree.write(file_path) + + for target_tag in root.findall('target-stack'): + base_stack_name, base_stack_version = target_tag.text.split('-') + new_target_stack_text = target_tag.text.replace(base_stack_name, config_data.stackName) + if base_stack_version in stack_version_changes: + new_target_stack_text = new_target_stack_text.replace(base_stack_version, + stack_version_changes[base_stack_version]) + target_tag.text = new_target_stack_text + tree.write(file_path) + + # rename upgrade files + new_file_path = file_path + if target_version in stack_version_changes: + new_file_path = os.path.join(os.path.dirname(file_path), + 'upgrade-{0}.xml'.format(stack_version_changes[target_version])) + os.rename(file_path, new_file_path) + return new_file_path + +def process_stack_advisor(file_path, config_data, stack_version_changes): + CLASS_NAME_REGEXP = r'([A-Za-z]+)(\d+)StackAdvisor' + + stack_advisor_content = open(file_path, 'r').read() + + for stack_name, stack_version in re.findall(CLASS_NAME_REGEXP, stack_advisor_content): + what = stack_name + stack_version + 'StackAdvisor' + stack_version_dotted = '.'.join(list(stack_version)) + if stack_version_dotted in stack_version_changes: + to = config_data.stackName + stack_version_changes[stack_version_dotted].replace('.', '') + 'StackAdvisor' + else: + to = config_data.stackName + stack_version + 'StackAdvisor' + stack_advisor_content = stack_advisor_content.replace(what, to) + + with open(file_path, 'w') as f: + f.write(stack_advisor_content) + return file_path + +def process_repoinfo_xml(file_path, config_data, stack_version_changes, stack): + if 'repoinfo' in stack: + ######################################################################################### + # Update repo info from explicitly defined repo info from config + # Assumption: All elements in repo info are configured + ######################################################################################### + root = ET.Element("reposinfo") + if 'latest' in stack.repoinfo: + latest_tag = ET.SubElement(root, 'latest') + latest_tag.text = stack.repoinfo.latest + if 'os' in stack.repoinfo: + for family, repos in stack.repoinfo.os.iteritems(): + os_tag = ET.SubElement(root, 'os') + os_tag.set('family', family) + for repo in repos: + repo_tag = ET.SubElement(os_tag, 'repo') + baseurl_tag = ET.SubElement(repo_tag, 'baseurl') + baseurl_tag.text = repo.baseurl + repoid_tag = ET.SubElement(repo_tag, 'repoid') + repoid_tag.text = repo.repoid + reponame_tag= ET.SubElement(repo_tag, 'reponame') + reponame_tag.text = repo.reponame + open(file_path,"w").write(minidom.parseString(ET.tostring(root, 'utf-8')).toprettyxml(indent=" ")) + else: + ######################################################################################### + # Update repo info with defaults if repo info is not defined in config + ######################################################################################### + tree = ET.parse(file_path) + root = tree.getroot() + # Update all base urls + for baseurl_tag in root.getiterator('baseurl'): + baseurl_tag.text = 'http://SET_REPO_URL' + # Update latest url + for latest_tag in root.getiterator('latest'): + latest_tag.text = 'http://SET_LATEST_REPO_URL_INFO' + # Update repo ids + for repoid_tag in root.getiterator('repoid'): + repoid_tag.text = repoid_tag.text.replace(config_data.baseStackName, config_data.stackName) + for baseVersion in stack_version_changes: + repoid_tag.text = repoid_tag.text.replace(baseVersion, stack_version_changes[baseVersion]) + # Update repo name + for reponame_tag in root.getiterator('reponame'): + reponame_tag.text = reponame_tag.text.replace(config_data.baseStackName, config_data.stackName) + tree.write(file_path) + return file_path + +def process_py_files(file_path, config_data, stack_version_changes): + new_file_path = process_replacements(file_path, config_data, stack_version_changes) + if config_data.baseStackName.lower() in file_path: + new_file_path = file_path.replace(config_data.baseStackName.lower(), config_data.stackName.lower()) + os.rename(file_path, new_file_path) + return new_file_path + +def process_xml_files(file_path, config_data, stack_version_changes): + return process_replacements(file_path, config_data, stack_version_changes) + +class GeneratorHelper(object): + def __init__(self, config_data, resources_folder, output_folder): + self.config_data = config_data + self.resources_folder = resources_folder + self.output_folder = output_folder + + stack_version_changes = {} + + for stack in config_data.versions: + if stack.version != stack.baseVersion: + stack_version_changes[stack.baseVersion] = stack.version + + self.stack_version_changes = stack_version_changes + self.common_services = [] + + def copy_stacks(self): + original_folder = os.path.join(self.resources_folder, 'stacks', self.config_data.baseStackName) + target_folder = os.path.join(self.output_folder, 'stacks', self.config_data.stackName) + + for stack in self.config_data.versions: + original_stack = os.path.join(original_folder, stack.baseVersion) + target_stack = os.path.join(target_folder, stack.version) + + desired_services = [service.name for service in stack.services] + desired_services.append('stack_advisor.py') # stack_advisor.py placed in stacks folder + base_stack_services = os.listdir(os.path.join(original_stack, 'services')) + ignored_files = [service for service in base_stack_services if service not in desired_services] + ignored_files.append('.pyc') + + def post_copy(src, target): + if target.endswith('.xml'): + #################################################################### + # Add special case handling for specific xml files + ################################################################### + # process metainfo.xml + if target.endswith('metainfo.xml'): + target = process_metainfo(target, self.config_data, self.stack_version_changes, self.common_services) + # process repoinfo.xml + if target.endswith('repoinfo.xml'): + target = process_repoinfo_xml(target, self.config_data, self.stack_version_changes, stack) + # process upgrade-x.x.xml + _upgrade_re = re.compile('upgrade-(.*)\.xml') + result = re.search(_upgrade_re, target) + if result: + target_version = result.group(1) + target = process_upgrade_xml(target, target_version, self.config_data, self.stack_version_changes) + #################################################################### + # Generic processing for xml files + ################################################################### + process_xml_files(target, self.config_data, self.stack_version_changes) + return + if target.endswith('.py'): + #################################################################### + # Add special case handling for specific py files + ################################################################### + # process stack_advisor.py + if target.endswith('stack_advisor.py'): + target = process_stack_advisor(target, self.config_data, self.stack_version_changes) + #################################################################### + # Generic processing for py files + ################################################################### + target = process_py_files(target, self.config_data, self.stack_version_changes) + return + + copy_tree(original_stack, target_stack, ignored_files, post_copy=post_copy) + # copy default stack advisor + shutil.copy(os.path.join(self.resources_folder, 'stacks', 'stack_advisor.py'), os.path.join(target_folder, '../stack_advisor.py')) + + def copy_common_services(self, common_services = []): + ignored_files = ['.pyc'] + if not common_services: + common_services = self.common_services + for original_folder in common_services: + source_folder = os.path.join(self.resources_folder, original_folder) + target_folder = os.path.join(self.output_folder, original_folder) + parent_services = [] + def post_copy(src, target): + if target.endswith('.xml'): + # process metainfo.xml + if target.endswith('metainfo.xml'): + process_metainfo(target, self.config_data, self.stack_version_changes, parent_services) + # process generic xml + if target.endswith('.xml'): + process_xml_files(target, self.config_data, self.stack_version_changes) + # process python files + if target.endswith('.py'): + process_py_files(target, self.config_data, self.stack_version_changes) + return + + copy_tree(source_folder, target_folder, ignored_files, post_copy=post_copy) + if parent_services: + self.copy_common_services(parent_services) + pass + + + def copy_resource_management(self): + source_folder = join(os.path.abspath(join(self.resources_folder, "..", "..", "..", "..")), + 'ambari-common', 'src', 'main', 'python', 'resource_management') + target_folder = join(self.output_folder, 'python', 'resource_management') + ignored_files = ['.pyc'] + + def post_copy(src, target): + # process python files + if target.endswith('.py'): + # process script.py + process_py_files(target, self.config_data, self.stack_version_changes) + return + + copy_tree(source_folder, target_folder, ignored_files, post_copy=post_copy) + + +def main(argv): + HELP_STRING = 'GenerateStackDefinition.py -c <config> -r <resources_folder> -o <output_folder>' + config = '' + resources_folder = '' + output_folder = '' + try: + opts, args = getopt.getopt(argv, "hc:o:r:", ["config=", "out=", "resources="]) + except getopt.GetoptError: + print HELP_STRING + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print HELP_STRING + sys.exit() + elif opt in ("-c", "--config"): + config = arg + elif opt in ("-r", "--resources"): + resources_folder = arg + elif opt in ("-o", "--out"): + output_folder = arg + if not config or not resources_folder or not output_folder: + print HELP_STRING + sys.exit(2) + + config_data = _named_dict(json.load(open(config, "r"))) + gen_helper = GeneratorHelper(config_data, resources_folder, output_folder) + gen_helper.copy_stacks() + gen_helper.copy_resource_management() + gen_helper.copy_common_services() + + +if __name__ == "__main__": + main(sys.argv[1:]) http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/pluggable_stack_definition/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/__init__.py b/ambari-common/src/main/python/pluggable_stack_definition/__init__.py new file mode 100644 index 0000000..0a0e1ca --- /dev/null +++ b/ambari-common/src/main/python/pluggable_stack_definition/__init__.py @@ -0,0 +1,18 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/pluggable_stack_definition/test.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/test.json b/ambari-common/src/main/python/pluggable_stack_definition/test.json new file mode 100644 index 0000000..07b1048 --- /dev/null +++ b/ambari-common/src/main/python/pluggable_stack_definition/test.json @@ -0,0 +1,137 @@ +{ + "stackName": "PHD", + "baseStackName": "HDP", + "performCommonReplacements": true, + "textReplacements": [ + ["hdp-select", "distro-select"], + ["conf-select", "phd-conf-select"] + ], + "versions": [ + { + "version": "2.0.6", + "baseVersion": "2.0.6", + "active": "false", + "repoinfo": { + "latest": "http://test.com/test/test_urlinfo.json";, + "os" : { + "redhat6":[ + { + "baseurl":"http://test.com/repo";, + "repoid": "hello", + "reponame": "HELLO" + }, + { + "baseurl":"http://test.com/repo2";, + "repoid": "world", + "reponame": "WORLD" + } + ] + } + }, + "services": [ + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + } + ] + }, + { + "version": "2.1", + "baseVersion": "2.1", + "active": "false", + "services": [ + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "TEZ", + "packages":[ + { + "family": "redhat6,suse11", + "packages": [ + "hello", + { + "name": "wolrd", + "skipUpgrade": "true" + } + ] + } + ] + } + ] + }, + { + "version": "3.0", + "baseVersion": "2.2", + "active": "true", + "services": [ + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "TEZ" + } + ] + }, + { + "version": "3.1", + "baseVersion": "2.3", + "active": "true", + "services": [ + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "TEZ" + } + ] + } + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/white_labeling/WhiteLabelStackDefinition.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/white_labeling/WhiteLabelStackDefinition.py b/ambari-common/src/main/python/white_labeling/WhiteLabelStackDefinition.py deleted file mode 100644 index c85020f..0000000 --- a/ambari-common/src/main/python/white_labeling/WhiteLabelStackDefinition.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import getopt -import json -import os -import shutil -import xml.etree.ElementTree as ET -import re - - -class _named_dict(dict): - """ - Allow to get dict items using attribute notation, eg dict.attr == dict['attr'] - """ - - def __init__(self, _dict): - - def repl_list(_list): - for i, e in enumerate(_list): - if isinstance(e, list): - _list[i] = repl_list(e) - if isinstance(e, dict): - _list[i] = _named_dict(e) - return _list - - dict.__init__(self, _dict) - for key, value in self.iteritems(): - if isinstance(value, dict): - self[key] = _named_dict(value) - if isinstance(value, list): - self[key] = repl_list(value) - - def __getattr__(self, item): - if item in self: - return self[item] - else: - dict.__getattr__(self, item) - - -def copy_tree(src, dest, exclude=None, post_copy=None): - """ - Copy files form src to dest. - - :param src: source folder - :param dest: destination folder - :param exclude: list for excluding, eg [".xml"] will exclude all xml files - :param post_copy: callable that accepts source and target paths and will be called after copying - """ - for item in os.listdir(src): - if exclude and item in exclude: - continue - - _src = os.path.join(src, item) - _dest = os.path.join(dest, item) - - if os.path.isdir(_src): - if not os.path.exists(_dest): - os.makedirs(_dest) - copy_tree(_src, _dest, exclude, post_copy) - else: - _dest_dirname = os.path.dirname(_dest) - if not os.path.exists(_dest_dirname): - os.makedirs(_dest_dirname) - shutil.copy(_src, _dest) - - if post_copy: - post_copy(_src, _dest) - - -def process_metainfo(file_path, config_data, stack_version_changes): - tree = ET.parse(file_path) - root = tree.getroot() - - if root.find('versions') is not None or root.find('services') is None: - # process stack metainfo.xml - extends_tag = root.find('extends') - if extends_tag is not None: - version = extends_tag.text - if version in stack_version_changes: - extends_tag.text = stack_version_changes[version] - tree.write(file_path) - - current_version = file_path.split(os.sep)[-2] - modify_active_tag = False - active_tag_value = None - for stack in config_data.versions: - if stack.version == current_version and 'active' in stack: - modify_active_tag = True - active_tag_value = stack.active - break - - if modify_active_tag: - versions_tag = root.find('versions') - if versions_tag is None: - versions_tag = ET.SubElement(root, 'versions') - active_tag = versions_tag.find('active') - if active_tag is None: - active_tag = ET.SubElement(versions_tag, 'active') - active_tag.text = active_tag_value - tree.write(file_path) - else: - # process service metainfo.xml - services_tag = root.find('services') - if services_tag is not None: - for service_tag in services_tag.findall('service'): - name = service_tag.find('name').text - for stack in config_data.versions: - for service in stack.services: - if service.name == name: - if 'version' in service: - service_version_tag = service_tag.find('version') - if service_version_tag is None: - service_version_tag = ET.SubElement(service_tag, 'version') - service_version_tag.text = service.version - tree.write(file_path) - - -def process_upgrade_xml(file_path, target_version, config_data, stack_version_changes): - # change versions in xml - tree = ET.parse(file_path) - root = tree.getroot() - - for target_tag in root.findall('target'): - version = '.'.join([el for el in target_tag.text.split('.') if el != '*']) - if version in stack_version_changes: - target_tag.text = target_tag.text.replace(version, stack_version_changes[version]) - tree.write(file_path) - - for target_tag in root.findall('target-stack'): - base_stack_name, base_stack_version = target_tag.text.split('-') - new_target_stack_text = target_tag.text.replace(base_stack_name, config_data.stackName) - if base_stack_version in stack_version_changes: - new_target_stack_text = new_target_stack_text.replace(base_stack_version, - stack_version_changes[base_stack_version]) - target_tag.text = new_target_stack_text - tree.write(file_path) - - # rename upgrade files - if target_version in stack_version_changes: - new_name = os.path.join(os.path.dirname(file_path), - 'upgrade-{0}.xml'.format(stack_version_changes[target_version])) - os.rename(file_path, new_name) - pass - - -def process_stack_advisor(file_path, config_data, stack_version_changes): - CLASS_NAME_REGEXP = '([A-Za-z]+)(\d+)StackAdvisor' - - stack_advisor_content = open(file_path, 'r').read() - - for stack_name, stack_version in re.findall(CLASS_NAME_REGEXP, stack_advisor_content): - what = stack_name + stack_version + 'StackAdvisor' - stack_version_dotted = '.'.join(list(stack_version)) - if stack_version_dotted in stack_version_changes: - to = config_data.stackName + stack_version_changes[stack_version_dotted].replace('.','') + 'StackAdvisor' - else: - to = config_data.stackName + stack_version + 'StackAdvisor' - stack_advisor_content = stack_advisor_content.replace(what, to) - - with open(file_path, 'w') as f: - f.write(stack_advisor_content) - -def process_repoinfo_xml(file_path, config_data, stack_version_changes): - tree = ET.parse(file_path) - root = tree.getroot() - for baseurl_tag in root.iter('baseurl'): - baseurl_tag.text = 'http://examplerepo.com' - - tree.write(file_path) - -def process_py_files(file_path, config_data, stack_version_changes): - file_content = open(file_path, 'r').read() - # replace select tools - file_content = file_content.replace('hdp-select', config_data.selectTool) - file_content = file_content.replace('conf-select', config_data.confSelectTool) - - with open(file_path, 'w') as f: - f.write(file_content) - -def copy_stacks(resources_folder, output_folder, config_data): - original_folder = os.path.join(resources_folder, 'stacks', config_data.baseStackName) - target_folder = os.path.join(output_folder, 'stacks', config_data.stackName) - stack_version_changes = {} - - for stack in config_data.versions: - if stack.version != stack.baseVersion: - stack_version_changes[stack.baseVersion] = stack.version - - for stack in config_data.versions: - original_stack = os.path.join(original_folder, stack.baseVersion) - target_stack = os.path.join(target_folder, stack.version) - - desired_services = [service.name for service in stack.services] - desired_services.append('stack_advisor.py') # stack_advisor.py placed in stacks folder - base_stack_services = os.listdir(os.path.join(original_stack, 'services')) - ignored_services = [service for service in base_stack_services if service not in desired_services] - - def post_copy(src, target): - # process metainfo.xml - if target.endswith('metainfo.xml'): - process_metainfo(target, config_data, stack_version_changes) - return - # process upgrade-x.x.xml - _upgrade_re = re.compile('upgrade-(.*)\.xml') - result = re.search(_upgrade_re, target) - if result: - target_version = result.group(1) - process_upgrade_xml(target, target_version, config_data, stack_version_changes) - return - # process stack_advisor.py - if target.endswith('stack_advisor.py'): - process_stack_advisor(target, config_data, stack_version_changes) - return - # process repoinfo.xml - if target.endswith('repoinfo.xml'): - process_repoinfo_xml(target, config_data, stack_version_changes) - return - # process python files - if target.endswith('.py'): - process_py_files(target, config_data, stack_version_changes) - return - - # TODO add more processing here for *.py files and others - - copy_tree(original_stack, target_stack, ignored_services, post_copy=post_copy) - - -def main(argv): - HELP_STRING = 'WhiteLabelStackDefinition.py -c <config> -r <resources_folder> -o <output_folder>' - config = '' - resources_folder = '' - output_folder = '' - try: - opts, args = getopt.getopt(argv, "hc:o:r:", ["config=", "out=", "resources="]) - except getopt.GetoptError: - print HELP_STRING - sys.exit(2) - for opt, arg in opts: - if opt == '-h': - print HELP_STRING - sys.exit() - elif opt in ("-c", "--config"): - config = arg - elif opt in ("-r", "--resources"): - resources_folder = arg - elif opt in ("-o", "--out"): - output_folder = arg - if not config or not resources_folder or not output_folder: - print HELP_STRING - sys.exit(2) - copy_stacks(resources_folder, output_folder, _named_dict(json.load(open(config, "r")))) - - -if __name__ == "__main__": - main(sys.argv[1:]) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/white_labeling/__init__.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/white_labeling/__init__.py b/ambari-common/src/main/python/white_labeling/__init__.py deleted file mode 100644 index 0a0e1ca..0000000 --- a/ambari-common/src/main/python/white_labeling/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/0d400379/ambari-common/src/main/python/white_labeling/test.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/white_labeling/test.json b/ambari-common/src/main/python/white_labeling/test.json deleted file mode 100644 index 0df0dda..0000000 --- a/ambari-common/src/main/python/white_labeling/test.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "stackName": "PHD", - "baseStackName": "HDP", - "selectTool": "phd-select", - "confSelectTool": "phd-conf-select", - "versions": [ - { - "version": "2.0.6", - "baseVersion": "2.0.6", - "active": "false", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - } - ] - }, - { - "version": "2.1", - "baseVersion": "2.1", - "active": "false", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - } - ] - }, - { - "version": "3.0", - "baseVersion": "2.2", - "active": "true", - "services": [ - { - "name": "HDFS", - "version": "2.6.0.2.2.0.0" - }, - { - "name": "ZOOKEEPER" - } - ] - }, - { - "version": "3.1", - "baseVersion": "2.3", - "active": "true", - "services": [ - { - "name": "HDFS" - }, - { - "name": "ZOOKEEPER" - } - ] - } - ] -} \ No newline at end of file
