This is an automated email from the ASF dual-hosted git repository. mugdha pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push: new d930ad4 AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha) (#1878) d930ad4 is described below commit d930ad4faa5898d992fb0055e86a261d31195924 Author: Mugdha Varadkar <fimug...@users.noreply.github.com> AuthorDate: Fri Jul 27 10:24:25 2018 +0530 AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha) (#1878) * AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade (mugdha) * AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Minor UT fix. (mugdha) * AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Adding Doc Comment. (mugdha) * AMBARI-24352 Updating Ranger and Ranger KMS configs during Ambari upgrade. Adding logic for fetching port from existing db conn string. (mugdha) --- .../ambari/server/upgrade/SchemaUpgradeHelper.java | 1 + .../ambari/server/upgrade/UpgradeCatalog271.java | 198 ++++++++++++++++++ .../KAFKA/0.8.1/package/scripts/params.py | 2 - .../RANGER/0.4.0/package/scripts/params.py | 6 +- .../package/templates/input.config-ranger.json.j2 | 6 +- .../0.5.0/configuration/ranger-ugsync-site.xml | 4 +- .../0.7.0/configuration/ranger-admin-site.xml | 11 + .../RANGER/0.7.0/configuration/ranger-env.xml | 10 + .../RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml | 20 ++ .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 2 - .../RANGER_KMS/themes/theme_version_1.json | 49 ++++- .../stacks/HDP/2.3/services/stack_advisor.py | 19 ++ .../server/upgrade/UpgradeCatalog271Test.java | 229 +++++++++++++++++++++ .../python/stacks/2.3/common/test_stack_advisor.py | 10 + .../stacks/2.5/configs/ranger-admin-default.json | 2 +- .../stacks/2.5/configs/ranger-admin-secured.json | 2 +- .../stacks/2.6/configs/ranger-admin-default.json | 2 +- .../stacks/2.6/configs/ranger-admin-secured.json | 2 +- 18 files changed, 552 insertions(+), 23 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java index de80052..4a05175 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java @@ -189,6 +189,7 @@ public class SchemaUpgradeHelper { catalogBinder.addBinding().to(UpgradeCatalog261.class); catalogBinder.addBinding().to(UpgradeCatalog262.class); catalogBinder.addBinding().to(UpgradeCatalog270.class); + catalogBinder.addBinding().to(UpgradeCatalog271.class); catalogBinder.addBinding().to(UpdateAlertScriptPaths.class); catalogBinder.addBinding().to(FinalUpgradeCatalog.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java new file mode 100644 index 0000000..c8a4d99 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog271.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.upgrade; + +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.orm.dao.DaoUtils; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; +import com.google.inject.Inject; +import com.google.inject.Injector; + +public class UpgradeCatalog271 extends AbstractUpgradeCatalog { + + /** + * Logger + */ + private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog271.class); + + @Inject + DaoUtils daoUtils; + + /** + * Constructor + * + * @param injector + */ + @Inject + public UpgradeCatalog271(Injector injector) { + super(injector); + daoUtils = injector.getInstance(DaoUtils.class); + } + + /** + * {@inheritDoc} + */ + @Override + public String getTargetVersion() { + return "2.7.1"; + } + + /** + * {@inheritDoc} + */ + @Override + public String getSourceVersion() { + return "2.7.0"; + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDDLUpdates() throws AmbariException, SQLException { + } + + /** + * {@inheritDoc} + */ + @Override + protected void executePreDMLUpdates() throws AmbariException, SQLException { + } + + /** + * {@inheritDoc} + */ + @Override + protected void executeDMLUpdates() throws AmbariException, SQLException { + addNewConfigurationsFromXml(); + updateRangerLogDirConfigs(); + updateRangerKmsDbUrl(); + } + + /** + * Updating Ranger Admin and Ranger Usersync log directory configs + * @throws AmbariException + */ + protected void updateRangerLogDirConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Set<String> installedServices = cluster.getServices().keySet(); + if (installedServices.contains("RANGER")) { + Config rangerEnvConfig = cluster.getDesiredConfigByType("ranger-env"); + Config rangerAdminSiteConfig = cluster.getDesiredConfigByType("ranger-admin-site"); + Config rangerUgsyncSiteConfig = cluster.getDesiredConfigByType("ranger-ugsync-site"); + if (rangerEnvConfig != null) { + String rangerAdminLogDir = rangerEnvConfig.getProperties().get("ranger_admin_log_dir"); + String rangerUsersyncLogDir = rangerEnvConfig.getProperties().get("ranger_usersync_log_dir"); + if (rangerAdminLogDir != null && rangerAdminSiteConfig != null) { + Map<String, String> newProperty = new HashMap<String, String>(); + newProperty.put("ranger.logs.base.dir", rangerAdminLogDir); + updateConfigurationPropertiesForCluster(cluster, "ranger-admin-site", newProperty, true, false); + } + if (rangerUsersyncLogDir != null && rangerUgsyncSiteConfig != null && rangerUgsyncSiteConfig.getProperties().containsKey("ranger.usersync.logdir")) { + Map<String, String> updateProperty = new HashMap<String, String>(); + updateProperty.put("ranger.usersync.logdir", rangerUsersyncLogDir); + updateConfigurationPropertiesForCluster(cluster, "ranger-ugsync-site", updateProperty, true, false); + } + Set<String> removeProperties = Sets.newHashSet("ranger_admin_log_dir", "ranger_usersync_log_dir"); + removeConfigurationPropertiesFromCluster(cluster, "ranger-env", removeProperties); + } + } + } + } + } + + } + + /** + * Updating JDBC connection url in Ranger KMS for verifying communication to database + * using database root user credentials + * @throws AmbariException + */ + protected void updateRangerKmsDbUrl() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + if (clusters != null) { + Map<String, Cluster> clusterMap = clusters.getClusters(); + if (clusterMap != null && !clusterMap.isEmpty()) { + for (final Cluster cluster : clusterMap.values()) { + Set<String> installedServices = cluster.getServices().keySet(); + if (installedServices.contains("RANGER_KMS")) { + Config rangerKmsPropertiesConfig = cluster.getDesiredConfigByType("kms-properties"); + Config rangerKmsEnvConfig = cluster.getDesiredConfigByType("kms-env"); + Config rangerKmsDbksConfig = cluster.getDesiredConfigByType("dbks-site"); + if (rangerKmsPropertiesConfig != null) { + String dbFlavor = rangerKmsPropertiesConfig.getProperties().get("DB_FLAVOR"); + String dbHost = rangerKmsPropertiesConfig.getProperties().get("db_host"); + String rangerKmsRootDbUrl = ""; + if (dbFlavor != null && dbHost != null) { + String port = ""; + if (rangerKmsDbksConfig != null) { + String rangerKmsDbUrl = rangerKmsDbksConfig.getProperties().get("ranger.ks.jpa.jdbc.url"); + if (rangerKmsDbUrl != null) { + Pattern pattern = Pattern.compile("(:[0-9]+)"); + Matcher matcher = pattern.matcher(rangerKmsDbUrl); + if (matcher.find()) { + port = matcher.group(); + } + } + } + if ("MYSQL".equalsIgnoreCase(dbFlavor)) { + rangerKmsRootDbUrl = "jdbc:mysql://" + dbHost + (!port.equalsIgnoreCase("")?port:":3306"); + } else if ("ORACLE".equalsIgnoreCase(dbFlavor)) { + rangerKmsRootDbUrl = "jdbc:oracle:thin:@//" + dbHost + (!port.equalsIgnoreCase("")?port:":1521"); + } else if ("POSTGRES".equalsIgnoreCase(dbFlavor)) { + rangerKmsRootDbUrl = "jdbc:postgresql://" + dbHost + (!port.equalsIgnoreCase("")?port:":5432") + "/postgres"; + } else if ("MSSQL".equalsIgnoreCase(dbFlavor)) { + rangerKmsRootDbUrl = "jdbc:sqlserver://" + dbHost + (!port.equalsIgnoreCase("")?port:":1433"); + } else if ("SQLA".equalsIgnoreCase(dbFlavor)) { + rangerKmsRootDbUrl = "jdbc:sqlanywhere:host=" + dbHost + (!port.equalsIgnoreCase("")?port:":2638") + ";"; + } + Map<String, String> newProperty = new HashMap<String, String>(); + newProperty.put("ranger_kms_privelege_user_jdbc_url", rangerKmsRootDbUrl); + if (rangerKmsEnvConfig != null) { + updateConfigurationPropertiesForCluster(cluster, "kms-env", newProperty, true, false); + } + } + } + } + } + } + } + } + +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py index 7b2de27..5c2b60e 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py @@ -209,8 +209,6 @@ xml_configurations_supported = check_stack_feature(StackFeature.RANGER_XML_CONFI # ambari-server hostname ambari_server_hostname = config['ambariLevelParams']['ambari_server_host'] -ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin") - # ranger kafka plugin enabled property enable_ranger_kafka = default("configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled", "No") enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py index a65a53c..206d820 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/params.py @@ -128,8 +128,10 @@ ambari_java_home = default("/ambariLevelParams/ambari_java_home", java_home) unix_user = config['configurations']['ranger-env']['ranger_user'] unix_group = config['configurations']['ranger-env']['ranger_group'] ranger_pid_dir = default("/configurations/ranger-env/ranger_pid_dir", "/var/run/ranger") -usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync") -admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") +old_usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync") +old_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin") +usersync_log_dir = default("/configurations/ranger-ugsync-site/ranger.usersync.logdir", old_usersync_log_dir) +admin_log_dir = default("/configurations/ranger-admin-site/ranger.logs.base.dir", old_admin_log_dir) ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml') security_app_context_file = format('{ranger_conf}/security-applicationContext.xml') ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml') diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 index 6c5bb1f..9724cb6 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/templates/input.config-ranger.json.j2 @@ -20,17 +20,17 @@ { "type":"ranger_admin", "rowtype":"service", - "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log" + "path":"{{default('/configurations/ranger-admin-site/ranger.logs.base.dir', '/var/log/ranger/admin')}}/xa_portal.log" }, { "type":"ranger_dbpatch", "is_enabled":"true", - "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" + "path":"{{default('/configurations/ranger-admin-site/ranger.logs.base.dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" }, { "type":"ranger_usersync", "rowtype":"service", - "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log" + "path":"{{default('/configurations/ranger-ugsync-site/ranger.usersync.logdir', '/var/log/ranger/usersync')}}/usersync.log" } ], "filter":[ diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml index 331d07b..cbe98a9 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.5.0/configuration/ranger-ugsync-site.xml @@ -308,10 +308,10 @@ </property> <property> <name>ranger.usersync.logdir</name> - <value>{{usersync_log_dir}}</value> + <display-name>Ranger Usersync Log Dir</display-name> + <value>/var/log/ranger/usersync</value> <description>User sync log directory</description> <value-attributes> - <visible>false</visible> <overridable>false</overridable> </value-attributes> <on-ambari-upgrade add="true"/> diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml index ebf8517..94eb2f0 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-admin-site.xml @@ -28,4 +28,15 @@ <description></description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>ranger.logs.base.dir</name> + <display-name>Ranger Admin Log Dir</display-name> + <value>/var/log/ranger/admin</value> + <description>Ranger Admin Log Dir</description> + <value-attributes> + <type>directory</type> + <overridable>false</overridable> + </value-attributes> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml index dfc9ca9..86e2204 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml +++ b/ambari-server/src/main/resources/common-services/RANGER/0.7.0/configuration/ranger-env.xml @@ -70,4 +70,14 @@ </value-attributes> <on-ambari-upgrade add="false"/> </property> + <property> + <name>ranger_usersync_log_dir</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> + <property> + <name>ranger_admin_log_dir</name> + <deleted>true</deleted> + <on-ambari-upgrade add="false"/> + </property> </configuration> \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml index fbb5574..6bb53c1 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/kms-env.xml @@ -101,4 +101,24 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <property> + <name>ranger_kms_privelege_user_jdbc_url</name> + <display-name>JDBC connect string for root user</display-name> + <description>JDBC connect string - auto populated based on other values. This is to be used by root user</description> + <value>jdbc:mysql://localhost</value> + <value-attributes> + <overridable>false</overridable> + </value-attributes> + <depends-on> + <property> + <type>kms-properties</type> + <name>DB_FLAVOR</name> + </property> + <property> + <type>kms-properties</type> + <name>db_host</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 3fbb1e4..5b4177c 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -453,8 +453,6 @@ if dfs_ha_namenode_active is not None: else: namenode_hostname = config['clusterHostInfo']['namenode_hosts'][0] -ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin") - scheme = 'http' if not yarn_https_on else 'https' yarn_rm_address = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'] if not yarn_https_on else config['configurations']['yarn-site']['yarn.resourcemanager.webapp.https.address'] rm_active_port = rm_https_port if yarn_https_on else rm_port diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json index c08a56c..34f9167 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/themes/theme_version_1.json @@ -167,14 +167,6 @@ "subsection-name": "subsection-kms-db-row1-col2" }, { - "config": "kms-properties/db_root_user", - "subsection-name": "subsection-kms-db-root-user-row3-col1" - }, - { - "config": "kms-properties/db_root_password", - "subsection-name": "subsection-kms-db-root-user-row3-col2" - }, - { "config": "kms-properties/KMS_MASTER_KEY_PASSWD", "subsection-name": "subsection-kms-master-row4-col" }, @@ -206,6 +198,25 @@ } } ] + }, + { + "config": "kms-properties/db_root_user", + "subsection-name": "subsection-kms-db-root-user-row3-col1" + }, + { + "config": "kms-env/ranger_kms_privelege_user_jdbc_url", + "subsection-name": "subsection-kms-db-root-user-row3-col1" + }, + { + "config": "kms-properties/db_root_password", + "subsection-name": "subsection-kms-db-root-user-row3-col2" + }, + { + "config": "kms-env/test_root_db_kms_connection", + "subsection-name": "subsection-kms-db-root-user-row3-col1", + "property_value_attributes": { + "ui_only_property": true + } } ] }, @@ -287,6 +298,28 @@ } }, { + "config": "kms-env/test_root_db_kms_connection", + "widget": { + "type": "test-db-connection", + "display-name": "Test Connection", + "required-properties": { + "jdbc.driver.class": "dbks-site/ranger.ks.jpa.jdbc.driver", + "jdbc.driver.url": "kms-env/ranger_kms_privelege_user_jdbc_url", + "db.connection.source.host": "ranger_kms-site/ranger_kms_server_hosts", + "db.type": "kms-properties/DB_FLAVOR", + "db.connection.destination.host": "kms-properties/db_host", + "db.connection.user": "kms-properties/db_root_user", + "db.connection.password": "kms-properties/db_root_password" + } + } + }, + { + "config": "kms-env/ranger_kms_privelege_user_jdbc_url", + "widget": { + "type": "text-field" + } + }, + { "config": "dbks-site/ranger.ks.jpa.jdbc.driver", "widget" : { "type": "text-field" diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py index c4c493d..ac4632b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py @@ -443,6 +443,7 @@ class HDP23StackAdvisor(HDP22StackAdvisor): security_enabled = self.isSecurityEnabled(services) putRangerKmsSiteProperty = self.putProperty(configurations, "kms-site", services) putRangerKmsSitePropertyAttribute = self.putPropertyAttribute(configurations, "kms-site") + putRangerKmsEnvProperty = self.putProperty(configurations, "kms-env", services) if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']): @@ -470,6 +471,24 @@ class HDP23StackAdvisor(HDP22StackAdvisor): for key in rangerKmsDbProperties: putRangerKmsDbksProperty(key, rangerKmsDbProperties.get(key)) + if 'kms-properties' in services['configurations'] and ('DB_FLAVOR' in services['configurations']['kms-properties']['properties']) \ + and ('db_host' in services['configurations']['kms-properties']['properties']): + + rangerKmsDbFlavor = services['configurations']["kms-properties"]["properties"]["DB_FLAVOR"] + rangerKmsDbHost = services['configurations']["kms-properties"]["properties"]["db_host"] + + ranger_kms_db_privelege_url_dict = { + 'MYSQL': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:mysql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost)}, + 'ORACLE': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@' + self.getOracleDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost, None)}, + 'POSTGRES': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:postgresql://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + '/postgres'}, + 'MSSQL': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:sqlserver://' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';'}, + 'SQLA': {'ranger_kms_privelege_user_jdbc_url': 'jdbc:sqlanywhere:host=' + self.getDBConnectionHostPort(rangerKmsDbFlavor, rangerKmsDbHost) + ';'} + } + + rangerKmsPrivelegeDbProperties = ranger_kms_db_privelege_url_dict.get(rangerKmsDbFlavor, ranger_kms_db_privelege_url_dict['MYSQL']) + for key in rangerKmsPrivelegeDbProperties: + putRangerKmsEnvProperty(key, rangerKmsPrivelegeDbProperties.get(key)) + if kmsEnvProperties and self.checkSiteProperties(kmsEnvProperties, 'kms_user') and 'KERBEROS' in servicesList: kmsUser = kmsEnvProperties['kms_user'] kmsUserOld = self.getOldValue(services, 'kms-env', 'kms_user') diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java new file mode 100644 index 0000000..d7a1069 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog271Test.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.upgrade; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMockBuilder; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariManagementControllerImpl; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.StackId; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; +import org.junit.Assert; +import org.junit.Test; + +import com.google.inject.Injector; + +public class UpgradeCatalog271Test { + + @Test + public void testExecuteDMLUpdates() throws Exception { + Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); + Method updateRangerLogDirConfigs = UpgradeCatalog271.class.getDeclaredMethod("updateRangerLogDirConfigs"); + Method updateRangerKmsDbUrl = UpgradeCatalog271.class.getDeclaredMethod("updateRangerKmsDbUrl"); + + UpgradeCatalog271 upgradeCatalog271 = createMockBuilder(UpgradeCatalog271.class) + .addMockedMethod(updateRangerKmsDbUrl) + .addMockedMethod(updateRangerLogDirConfigs) + .addMockedMethod(addNewConfigurationsFromXml) + .createMock(); + + upgradeCatalog271.addNewConfigurationsFromXml(); + expectLastCall().once(); + + upgradeCatalog271.updateRangerLogDirConfigs(); + expectLastCall().once(); + + upgradeCatalog271.updateRangerKmsDbUrl(); + expectLastCall().once(); + + replay(upgradeCatalog271); + upgradeCatalog271.executeDMLUpdates(); + verify(upgradeCatalog271); + } + + @Test + public void testUpdateRangerLogDirConfigs() throws Exception { + + Map<String, Service> installedServices = new HashMap<String, Service>() { + { + put("RANGER", null); + } + }; + + Map<String, String> rangerEnvConfig = new HashMap<String, String>() { + { + put("ranger_admin_log_dir", "/var/log/ranger/admin"); + put("ranger_usersync_log_dir", "/var/log/ranger/usersync"); + } + }; + + Map<String, String> oldRangerUgsyncSiteConfig = new HashMap<String, String>() { + { + put("ranger.usersync.logdir", "{{usersync_log_dir}}"); + } + }; + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .createNiceMock(); + + expect(injector.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getClusterName()).andReturn("cl1").anyTimes(); + expect(cluster.getServices()).andReturn(installedServices).atLeastOnce(); + + Config mockRangerEnvConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("ranger-env")).andReturn(mockRangerEnvConfig).atLeastOnce(); + expect(mockRangerEnvConfig.getProperties()).andReturn(rangerEnvConfig).anyTimes(); + + Config mockRangerAdminSiteConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("ranger-admin-site")).andReturn(mockRangerAdminSiteConfig).atLeastOnce(); + expect(mockRangerAdminSiteConfig.getProperties()).andReturn(Collections.emptyMap()).anyTimes(); + + Config mockRangerUgsyncSiteConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("ranger-ugsync-site")).andReturn(mockRangerUgsyncSiteConfig).atLeastOnce(); + expect(mockRangerUgsyncSiteConfig.getProperties()).andReturn(oldRangerUgsyncSiteConfig).anyTimes(); + + Capture<Map> rangerAdminpropertiesCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerAdminpropertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + Capture<Map> rangerUgsyncPropertiesCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerUgsyncPropertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + Capture<Map> rangerEnvPropertiesCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(rangerEnvPropertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector, clusters, mockRangerEnvConfig, mockRangerAdminSiteConfig, mockRangerUgsyncSiteConfig, cluster); + new UpgradeCatalog271(injector).updateRangerLogDirConfigs(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedRangerAdminConfig = rangerAdminpropertiesCapture.getValue(); + Assert.assertEquals(updatedRangerAdminConfig.get("ranger.logs.base.dir"), "/var/log/ranger/admin"); + + Map<String, String> updatedRangerUgsyncSiteConfig = rangerUgsyncPropertiesCapture.getValue(); + Assert.assertEquals(updatedRangerUgsyncSiteConfig.get("ranger.usersync.logdir"), "/var/log/ranger/usersync"); + + Map<String, String> updatedRangerEnvConfig = rangerEnvPropertiesCapture.getValue(); + Assert.assertFalse(updatedRangerEnvConfig.containsKey("ranger_admin_log_dir")); + Assert.assertFalse(updatedRangerEnvConfig.containsKey("ranger_usersync_log_dir")); + } + + @Test + public void testUpdateRangerKmsDbUrl() throws Exception { + + Map<String, Service> installedServices = new HashMap<String, Service>() { + { + put("RANGER_KMS", null); + } + }; + + Map<String, String> rangerKmsPropertiesConfig = new HashMap<String, String>() { + { + put("DB_FLAVOR", "MYSQL"); + put("db_host", "c6401.ambari.apache.org"); + } + }; + + Map<String, String> rangerKmsDbksPropertiesConfig = new HashMap<String, String>() { + { + put("ranger.ks.jpa.jdbc.url", "jdbc:mysql://c6401.ambari.apache.org:3546"); + } + }; + + EasyMockSupport easyMockSupport = new EasyMockSupport(); + + Clusters clusters = easyMockSupport.createNiceMock(Clusters.class); + final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class); + + Injector injector = easyMockSupport.createNiceMock(Injector.class); + AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class) + .addMockedMethod("createConfiguration") + .addMockedMethod("getClusters", new Class[] { }) + .addMockedMethod("createConfig") + .createNiceMock(); + + expect(injector.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + + expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", cluster); + }}).once(); + expect(cluster.getClusterName()).andReturn("cl1").once(); + expect(cluster.getServices()).andReturn(installedServices).atLeastOnce(); + + Config mockRangerKmsPropertiesConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("kms-properties")).andReturn(mockRangerKmsPropertiesConfig).atLeastOnce(); + + Config mockRangerKmsEnvConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("kms-env")).andReturn(mockRangerKmsEnvConfig).atLeastOnce(); + + Config mockRangerKmsDbksConfig = easyMockSupport.createNiceMock(Config.class); + expect(cluster.getDesiredConfigByType("dbks-site")).andReturn(mockRangerKmsDbksConfig).atLeastOnce(); + + expect(mockRangerKmsPropertiesConfig.getProperties()).andReturn(rangerKmsPropertiesConfig).anyTimes(); + expect(mockRangerKmsEnvConfig.getProperties()).andReturn(Collections.emptyMap()).anyTimes(); + expect(mockRangerKmsDbksConfig.getProperties()).andReturn(rangerKmsDbksPropertiesConfig).anyTimes(); + + Capture<Map> propertiesCapture = EasyMock.newCapture(); + expect(controller.createConfig(anyObject(Cluster.class), anyObject(StackId.class), anyString(), capture(propertiesCapture), anyString(), + anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once(); + + replay(controller, injector, clusters, mockRangerKmsPropertiesConfig, mockRangerKmsEnvConfig, mockRangerKmsDbksConfig, cluster); + new UpgradeCatalog271(injector).updateRangerKmsDbUrl(); + easyMockSupport.verifyAll(); + + Map<String, String> updatedRangerKmsEnvConfig = propertiesCapture.getValue(); + Assert.assertEquals(updatedRangerKmsEnvConfig.get("ranger_kms_privelege_user_jdbc_url"), "jdbc:mysql://c6401.ambari.apache.org:3546"); + } + +} diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py index 6539302..6dd94d9 100644 --- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py @@ -1698,6 +1698,11 @@ class TestHDP23StackAdvisor(TestCase): 'hadoop.kms.proxyuser.root.hosts': {'delete': 'true'}, 'hadoop.kms.proxyuser.root.users': {'delete': 'true'} } + }, + 'kms-env': { + 'properties': { + 'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE' + } } } @@ -1741,6 +1746,11 @@ class TestHDP23StackAdvisor(TestCase): 'hadoop.kms.proxyuser.ambari-cl1.hosts': '*', 'hadoop.kms.proxyuser.ambari-cl1.users': '*' } + }, + 'kms-env': { + 'properties': { + 'ranger_kms_privelege_user_jdbc_url': 'jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE' + } } } diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json index 91a9b79..53e66ed 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-default.json @@ -534,7 +534,7 @@ "ranger.usersync.ldap.referral": "ignore", "ranger.usersync.group.searchfilter": "", "ranger.usersync.ldap.user.objectclass": "person", - "ranger.usersync.logdir": "{{usersync_log_dir}}", + "ranger.usersync.logdir": "/var/log/ranger/usersync", "ranger.usersync.ldap.user.searchfilter": "", "ranger.usersync.ldap.groupname.caseconversion": "none", "ranger.usersync.ldap.ldapbindpassword": "", diff --git a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json index 05edf27..3719a8d 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/ranger-admin-secured.json @@ -586,7 +586,7 @@ "ranger.usersync.ldap.referral": "ignore", "ranger.usersync.group.searchfilter": "", "ranger.usersync.ldap.user.objectclass": "person", - "ranger.usersync.logdir": "{{usersync_log_dir}}", + "ranger.usersync.logdir": "/var/log/ranger/usersync", "ranger.usersync.ldap.user.searchfilter": "", "ranger.usersync.ldap.groupname.caseconversion": "none", "ranger.usersync.ldap.ldapbindpassword": "", diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json index 43d65a0..7888054 100644 --- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json +++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-default.json @@ -565,7 +565,7 @@ "ranger.usersync.ldap.referral": "ignore", "ranger.usersync.group.searchfilter": "", "ranger.usersync.ldap.user.objectclass": "person", - "ranger.usersync.logdir": "{{usersync_log_dir}}", + "ranger.usersync.logdir": "/var/log/ranger/usersync", "ranger.usersync.ldap.user.searchfilter": "", "ranger.usersync.ldap.groupname.caseconversion": "none", "ranger.usersync.ldap.ldapbindpassword": "", diff --git a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json index 3fc0316..8228401 100644 --- a/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json +++ b/ambari-server/src/test/python/stacks/2.6/configs/ranger-admin-secured.json @@ -615,7 +615,7 @@ "ranger.usersync.ldap.referral": "ignore", "ranger.usersync.group.searchfilter": "", "ranger.usersync.ldap.user.objectclass": "person", - "ranger.usersync.logdir": "{{usersync_log_dir}}", + "ranger.usersync.logdir": "/var/log/ranger/usersync", "ranger.usersync.ldap.user.searchfilter": "", "ranger.usersync.ldap.groupname.caseconversion": "none", "ranger.usersync.ldap.ldapbindpassword": "",