Repository: ambari Updated Branches: refs/heads/branch-2.1 ae56a56f8 -> 2d9612e9f
AMBARI-12907: [PluggableStackDefinition] Add support for overriding stack config properties (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2d9612e9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2d9612e9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2d9612e9 Branch: refs/heads/branch-2.1 Commit: 2d9612e9f446c8df97bcc79369328566b14e264a Parents: ae56a56 Author: Jayush Luniya <[email protected]> Authored: Thu Aug 27 20:14:16 2015 -0700 Committer: Jayush Luniya <[email protected]> Committed: Thu Aug 27 20:15:13 2015 -0700 ---------------------------------------------------------------------- .../GenerateStackDefinition.py | 91 +++++ .../configs/SAPHD.json | 386 +++++++++++++++++++ 2 files changed, 477 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2d9612e9/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py index c09c788..275501d 100644 --- a/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py +++ b/ambari-common/src/main/python/pluggable_stack_definition/GenerateStackDefinition.py @@ -375,6 +375,91 @@ def process_py_files(file_path, config_data, stack_version_changes): def process_xml_files(file_path, config_data, stack_version_changes): return process_replacements(file_path, config_data, stack_version_changes) +def process_config_xml(file_path, config_data): + tree = ET.parse(file_path) + root = tree.getroot() + ############################################################################################# + # <resource_dir>/common-services/<service_name>/<service_version>/configuration/<config>.xml + ############################################################################################# + COMMON_SERVICES_CONFIG_PATH_REGEX = r'common-services/([A-Za-z_-]+)/([0-9\.]+)/configuration/([A-Za-z0-9_-]+).xml' + ############################################################################################# + # <resource_dir>/stacks/<stack_name>/<stack_version>/services/<service_name>/configuration/<config>.xml + ############################################################################################# + STACK_SERVICE_CONFIG_PATH_REGEX = r'stacks/([A-Za-z_-]+)/([0-9\.]+)/services/([A-Za-z_-]+)/configuration/([A-Za-z0-9_-]+).xml' + ############################################################################################# + # <resource_dir>/stacks/<stack_name>/<stack_version>/configuration/<config>.xml + ############################################################################################# + STACK_CONFIG_PATH_REGEX = r'stacks/([A-Za-z_-]+)/([0-9\.]+)/configuration/([A-Za-z0-9_-]+).xml' + + ######################################################################################### + # Override stack config properties + ######################################################################################### + match = re.search(COMMON_SERVICES_CONFIG_PATH_REGEX, file_path) + if match: + ############################################################################################# + # Config file path in common services + ############################################################################################# + path_service_name = match.group(1) + path_service_version = match.group(2) + path_config_name = match.group(3) + if 'common-services' in config_data: + for service in config_data['common-services']: + if service.name == path_service_name: + for serviceVersion in service.versions: + if serviceVersion.version == path_service_version: + if 'configurations' in serviceVersion: + for conf in serviceVersion['configurations']: + if conf.name == path_config_name: + for property_tag in root.findall('property'): + property_name = property_tag.find('name').text + if property_name in conf.properties: + value_tag = property_tag.find('value') + value_tag.text = conf.properties[property_name] + else: + match = re.search(STACK_SERVICE_CONFIG_PATH_REGEX, file_path) + if match: + ############################################################################################# + # Config file path for a service in stack + ############################################################################################# + path_stack_name = match.group(1) + path_stack_version = match.group(2) + path_service_name = match.group(3) + path_config_name = match.group(4) + for stack in config_data.versions: + if stack.version == path_stack_version: + for service in stack.services: + if service.name == path_service_name: + if 'configurations' in service: + for conf in service['configurations']: + if conf.name == path_config_name: + for property_tag in root.findall('property'): + property_name = property_tag.find('name').text + if property_name in conf.properties: + value_tag = property_tag.find('value') + value_tag.text = conf.properties[property_name] + else: + match = re.search(STACK_CONFIG_PATH_REGEX, file_path) + if match: + ############################################################################################# + # Config file path for global stack configs + ############################################################################################# + path_stack_name = match.group(1) + path_stack_version = match.group(2) + path_config_name = match.group(3) + for stack in config_data.versions: + if stack.version == path_stack_version: + if 'configurations' in stack: + for conf in stack['configurations']: + if conf.name == path_config_name: + for property_tag in root.findall('property'): + property_name = property_tag.find('name').text + if property_name in conf.properties: + value_tag = property_tag.find('value') + value_tag.text = conf.properties[property_name] + + tree.write(file_path) + return file_path + class GeneratorHelper(object): def __init__(self, config_data, resources_folder, output_folder): self.config_data = config_data @@ -415,6 +500,9 @@ class GeneratorHelper(object): # process repoinfo.xml if target.endswith('repoinfo.xml'): target = process_repoinfo_xml(target, self.config_data, self.stack_version_changes, stack) + if os.path.basename(os.path.dirname(target)) == 'configuration': + # process configuration xml + target = process_config_xml(target, self.config_data) # process upgrade-x.x.xml _upgrade_re = re.compile('upgrade-(.*)\.xml') result = re.search(_upgrade_re, target) @@ -456,6 +544,9 @@ class GeneratorHelper(object): # process metainfo.xml if target.endswith('metainfo.xml'): process_metainfo(target, self.config_data, self.stack_version_changes, parent_services) + if os.path.basename(os.path.dirname(target)) == 'configuration': + # process configuration xml + target = process_config_xml(target, self.config_data) # process generic xml if target.endswith('.xml'): process_xml_files(target, self.config_data, self.stack_version_changes) http://git-wip-us.apache.org/repos/asf/ambari/blob/2d9612e9/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json b/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json new file mode 100644 index 0000000..2ffa54d --- /dev/null +++ b/ambari-common/src/main/python/pluggable_stack_definition/configs/SAPHD.json @@ -0,0 +1,386 @@ +{ + "stackName": "SAPHD", + "baseStackName": "HDP", + "performCommonReplacements": true, + "textReplacements": [ + ["hdp-select", "distro-select"] + ], + "preservedText": ["ext-2.2", "HDP-oozie", "hdp.version", "HDP_VERSION"], + "uiMapping": [ + { + "stackVersionNumber": "2.3", + "sign": "<", + "baseStackFolder": "HDP2" + } + ], + "ambariProperties": { + "jdk.download.supported" : "false", + "jce.download.supported" : "false", + "jdk1.7.url" : "http://DOWNLOAD_NOT_SUPPORTED";, + "jdk1.8.url" : "http://DOWNLOAD_NOT_SUPPORTED";, + "jdk1.7.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED";, + "jdk1.8.jcpol-url" : "http://DOWNLOAD_NOT_SUPPORTED";, + "rolling.upgrade.min.stack" : "SAPHD-2.2" + }, + "common-services": [ + { + "name": "HIVE", + "versions": [ + { + "version": "0.12.0.2.0", + "configurations": [ + { + "name": "hive-env", + "properties": { + "hive_database_type": "sqlanywhere", + "hive_database": "Existing SQLA Database" + } + }, + { + "name": "hive-site", + "properties": { + "javax.jdo.option.ConnectionURL": "jdbc:sqlanywhere:host=localhost;database=hive", + "javax.jdo.option.ConnectionDriverName": "sap.jdbc4.sqlanywhere.IDriver" + } + } + ] + } + ] + }, + { + "name": "OOZIE", + "versions": [ + { + "version": "4.0.0.2.0", + "configurations": [ + { + "name": "oozie-env", + "properties": { + "oozie_database": "Existing SQLA Database" + } + }, + { + "name": "oozie-site", + "properties": { + "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie", + "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver" + } + } + ] + }, + { + "version": "4.2.0.2.3", + "configurations": [ + { + "name": "oozie-site", + "properties": { + "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie", + "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver" + } + } + ] + } + ] + }, + { + "name": "RANGER", + "versions": [ + { + "version": "0.4.0", + "configurations": [ + { + "name": "admin-properties", + "properties": { + "DB_FLAVOR": "SQLA" + } + } + ] + } + ] + } + ], + "versions": [ + { + "version": "2.0.6", + "baseVersion": "2.0.6", + "family": "redhat6,suse11", + "active": "false", + "services": [ + { + "name": "KERBEROS" + }, + { + "name": "AMBARI_METRICS" + }, + { + "name": "HDFS" + + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "HIVE" + }, + { + "name": "TEZ" + }, + { + "name": "OOZIE" + }, + { + "name": "KNOX" + }, + { + "name": "PIG" + }, + { + "name": "FLUME" + }, + { + "name": "SQOOP" + } + ] + }, + { + "version": "2.1", + "baseVersion": "2.1", + "active": "false", + "family": "redhat6,suse11", + "services": [ + { + "name": "KERBEROS" + }, + { + "name": "AMBARI_METRICS" + }, + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "HIVE" + }, + { + "name": "TEZ" + }, + { + "name": "OOZIE", + "configurations": [ + { + "name": "oozie-site", + "properties": { + "oozie.service.JPAService.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=oozie", + "oozie.service.JPAService.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver" + } + } + ] + }, + { + "name": "KNOX" + }, + { + "name": "PIG" + }, + { + "name": "FLUME" + }, + { + "name": "SQOOP" + }, + { + "name": "FALCON" + }, + { + "name": "STORM" + } + ] + }, + { + "version": "2.2", + "baseVersion": "2.2", + "active": "false", + "family": "redhat6,suse11", + "services": [ + { + "name": "KERBEROS" + }, + { + "name": "AMBARI_METRICS" + }, + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "HIVE", + "configurations": [ + { + "name": "hive-site", + "properties": { + "javax.jdo.option.ConnectionURL": "jdbc:sqlanywhere:host=localhost;database=hive", + "javax.jdo.option.ConnectionDriverName": "sap.jdbc4.sqlanywhere.IDriver" + } + } + ] + }, + { + "name": "TEZ" + }, + { + "name": "OOZIE" + }, + { + "name": "KNOX" + }, + { + "name": "PIG" + }, + { + "name": "FLUME" + }, + { + "name": "SQOOP" + }, + { + "name": "FALCON" + }, + { + "name": "STORM" + }, + { + "name": "KAFKA" + }, + { + "name": "RANGER" + }, + { + "name": "SLIDER" + }, + { + "name": "SPARK" + } + ] + }, + { + "version": "2.3", + "baseVersion": "2.3", + "active": "true", + "family": "redhat6,redhat7,suse11", + "services": [ + { + "name": "KERBEROS" + }, + { + "name": "AMBARI_METRICS" + }, + { + "name": "HDFS" + }, + { + "name": "ZOOKEEPER" + }, + { + "name": "HBASE" + }, + { + "name": "YARN" + }, + { + "name": "MAPREDUCE2" + }, + { + "name": "HIVE" + }, + { + "name": "TEZ" + }, + { + "name": "OOZIE" + }, + { + "name": "KNOX" + }, + { + "name": "PIG" + }, + { + "name": "FLUME" + }, + { + "name": "SQOOP" + }, + { + "name": "FALCON" + }, + { + "name": "STORM" + }, + { + "name": "KAFKA" + }, + { + "name": "RANGER", + "configurations":[ + { + "name": "ranger-admin-site", + "properties": { + "ranger.jpa.jdbc.driver": "sap.jdbc4.sqlanywhere.IDriver", + "ranger.jpa.jdbc.url": "jdbc:sqlanywhere:host=localhost;database=ranger" + } + } + ] + }, + { + "name": "SLIDER" + }, + { + "name": "SPARK" + }, + { + "name": "RANGER_KMS" + }, + { + "name": "ACCUMULO" + }, + { + "name": "ATLAS" + }, + { + "name": "MAHOUT" + } + ] + } + ] +} \ No newline at end of file
