Repository: ambari Updated Branches: refs/heads/trunk 1faa5acd4 -> 1042f1cd8
AMBARI-15912. Handle database connection properties upgrade for Hive and Oozie. (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1042f1cd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1042f1cd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1042f1cd Branch: refs/heads/trunk Commit: 1042f1cd8050874a36be98c12c44f23e5a8abbd5 Parents: 1faa5ac Author: Andrew Onishuk <[email protected]> Authored: Tue Apr 19 13:07:29 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Tue Apr 19 13:07:29 2016 +0300 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 2 - .../server/upgrade/UpgradeCatalog240.java | 37 +++++++++++ .../BlueprintConfigurationProcessorTest.java | 37 ----------- .../server/upgrade/UpgradeCatalog240Test.java | 68 ++++++++++++++++++++ 4 files changed, 105 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/1042f1cd/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 60a9263..5e8241b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2361,7 +2361,6 @@ public class BlueprintConfigurationProcessor { multiCoreSiteMap.put("hadoop.security.key.provider.path", new MultipleHostTopologyUpdater("RANGER_KMS_SERVER", ';', false, false, true)); multiWebhcatSiteMap.put("templeton.hive.properties", new TempletonHivePropertyUpdater()); multiWebhcatSiteMap.put("templeton.kerberos.principal", new MultipleHostTopologyUpdater("WEBHCAT_SERVER")); - hiveEnvMap.put("hive_hostname", new SingleHostTopologyUpdater("HIVE_SERVER")); multiHiveSiteMap.put("hive.zookeeper.quorum", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); multiHiveSiteMap.put("hive.cluster.delegation.token.store.zookeeper.connectString", new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); @@ -2454,7 +2453,6 @@ public class BlueprintConfigurationProcessor { oozieSiteMap.put("oozie.base.url", new SingleHostTopologyUpdater("OOZIE_SERVER")); oozieSiteMap.put("oozie.authentication.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); oozieSiteMap.put("oozie.service.HadoopAccessorService.kerberos.principal", new SingleHostTopologyUpdater("OOZIE_SERVER")); - oozieEnvMap.put("oozie_hostname", new SingleHostTopologyUpdater("OOZIE_SERVER")); multiCoreSiteMap.put("hadoop.proxyuser.oozie.hosts", new MultipleHostTopologyUpdater("OOZIE_SERVER")); // register updaters for Oozie properties that may point to an external DB http://git-wip-us.apache.org/repos/asf/ambari/blob/1042f1cd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index b66ad66..097a079 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -102,6 +102,9 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { public static final String VIEWINSTANCE_TABLE = "viewinstance"; public static final String SHORT_URL_COLUMN = "short_url"; + private static final String OOZIE_ENV_CONFIG = "oozie-env"; + private static final String HIVE_ENV_CONFIG = "hive-env"; + @Inject PermissionDAO permissionDAO; @@ -199,6 +202,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { updateHostRoleCommandTableDML(); updateKerberosConfigs(); updateYarnEnv(); + removeHiveOozieDBConnectionConfigs(); } private void createSettingTable() throws SQLException { @@ -283,6 +287,39 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { } + protected void removeHiveOozieDBConnectionConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters()); + + for (final Cluster cluster : clusterMap.values()) { + Config oozieEnv = cluster.getDesiredConfigByType(OOZIE_ENV_CONFIG); + if(oozieEnv != null) { + Map<String, String> oozieEnvProperties = oozieEnv.getProperties(); + Set<String> removePropertiesSet = new HashSet<>(); + if (oozieEnvProperties.containsKey("oozie_derby_database")) { + LOG.info("Removing property oozie_derby_database from " + OOZIE_ENV_CONFIG); + removePropertiesSet.add("oozie_derby_database"); + } + if (oozieEnvProperties.containsKey("oozie_hostname")) { + LOG.info("Removing property oozie_hostname from " + OOZIE_ENV_CONFIG); + removePropertiesSet.add("oozie_hostname"); + } + if (!removePropertiesSet.isEmpty()) { + removeConfigurationPropertiesFromCluster(cluster, OOZIE_ENV_CONFIG, removePropertiesSet); + } + } + + Config hiveEnv = cluster.getDesiredConfigByType(HIVE_ENV_CONFIG); + if(hiveEnv != null) { + Map<String, String> hiveEnvProperties = hiveEnv.getProperties(); + if (hiveEnvProperties.containsKey("hive_hostname")) { + LOG.info("Removing property hive_hostname from " + HIVE_ENV_CONFIG); + removeConfigurationPropertiesFromCluster(cluster, HIVE_ENV_CONFIG, Collections.singleton("hive_hostname")); + } + } + } + } + protected void updateAlerts() { // map of alert_name -> property_name -> visibility_value final Map<String, String> hdfsVisibilityMap = new HashMap<String, String>(){{ http://git-wip-us.apache.org/repos/asf/ambari/blob/1042f1cd/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 0bf01b3..dd26c75 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -1388,7 +1388,6 @@ public class BlueprintConfigurationProcessorTest { hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); hiveSiteProperties.put("hive.zookeeper.quorum", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); hiveSiteProperties.put("hive.cluster.delegation.token.store.zookeeper.connectString", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); - hiveEnvProperties.put("hive_hostname", expectedHostName); webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName); @@ -1432,9 +1431,6 @@ public class BlueprintConfigurationProcessorTest { assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", - createExportedHostName(expectedHostGroupName), hiveEnvProperties.get("hive_hostname")); - - assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", @@ -1489,7 +1485,6 @@ public class BlueprintConfigurationProcessorTest { hiveSiteProperties.put("javax.jdo.option.ConnectionURL", expectedHostName + ":" + expectedPortNum); hiveSiteProperties.put("hive.zookeeper.quorum", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); hiveSiteProperties.put("hive.cluster.delegation.token.store.zookeeper.connectString", expectedHostName + ":" + expectedPortNum + "," + expectedHostNameTwo + ":" + expectedPortNum); - hiveEnvProperties.put("hive_hostname", expectedHostName); webHCatSiteProperties.put("templeton.hive.properties", expectedHostName + "," + expectedHostNameTwo); webHCatSiteProperties.put("templeton.kerberos.principal", expectedHostName); @@ -1532,9 +1527,6 @@ public class BlueprintConfigurationProcessorTest { assertEquals("hive property not properly exported", createExportedAddress(expectedPortNum, expectedHostGroupName), hiveSiteProperties.get("javax.jdo.option.ConnectionURL")); assertEquals("hive property not properly exported", - createExportedHostName(expectedHostGroupName), hiveEnvProperties.get("hive_hostname")); - - assertEquals("hive property not properly exported", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), webHCatSiteProperties.get("templeton.hive.properties")); assertEquals("hive property not properly exported", @@ -1581,7 +1573,6 @@ public class BlueprintConfigurationProcessorTest { oozieSiteProperties.put("oozie.service.HadoopAccessorService.kerberos.principal", expectedHostName); oozieSiteProperties.put("oozie.service.JPAService.jdbc.url", "jdbc:mysql://" + expectedExternalHost + "/ooziedb"); - oozieEnvProperties.put("oozie_hostname", expectedHostName); oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); oozieEnvProperties.put("oozie_heapsize", "1024m"); oozieEnvProperties.put("oozie_permsize", "2048m"); @@ -1625,8 +1616,6 @@ public class BlueprintConfigurationProcessorTest { assertEquals("oozie property not exported correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); assertEquals("oozie property not exported correctly", - createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname")); - assertEquals("oozie property not exported correctly", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); // verify that the oozie properties that can refer to an external DB are not included in the export @@ -2712,9 +2701,6 @@ public class BlueprintConfigurationProcessorTest { public void testHiveConfigClusterUpdateSpecifyingHostNamesHiveServer2HA() throws Exception { final String expectedHostGroupName = "host_group_1"; - final String expectedPropertyValue = - "c6401.ambari.apache.org"; - final String expectedMetaStoreURIs = "thrift://c6401.ambari.apache.org:9083,thrift://c6402.ambari.apache.org:9083"; Map<String, Map<String, String>> configProperties = @@ -2728,10 +2714,6 @@ public class BlueprintConfigurationProcessorTest { configProperties.put("hive-env", hiveEnvProperties); configProperties.put("hive-site", hiveSiteProperties); - // setup properties that include host information - hiveEnvProperties.put("hive_hostname", - expectedPropertyValue); - // simulate HA mode, since this property must be present in HiveServer2 HA hiveSiteProperties.put("hive.server2.support.dynamic.service.discovery", "true"); @@ -2759,10 +2741,6 @@ public class BlueprintConfigurationProcessorTest { BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); - assertEquals("Unexpected config update for hive_hostname", - expectedPropertyValue, - hiveEnvProperties.get("hive_hostname")); - assertEquals("Unexpected config update for hive.metastore.uris", expectedMetaStoreURIs, hiveSiteProperties.get("hive.metastore.uris")); @@ -2798,10 +2776,6 @@ public class BlueprintConfigurationProcessorTest { configProperties.put("hive-env", hiveEnvProperties); configProperties.put("hive-site", hiveSiteProperties); - // setup properties that include host information - hiveEnvProperties.put("hive_hostname", - expectedHostNameOne); - // simulate HA mode, since this property must be present in HiveServer2 HA hiveSiteProperties.put("hive.server2.support.dynamic.service.discovery", "true"); @@ -2831,10 +2805,6 @@ public class BlueprintConfigurationProcessorTest { BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); - assertEquals("Unexpected config update for hive_hostname", - expectedHostNameOne, - hiveEnvProperties.get("hive_hostname")); - assertEquals("Unexpected config update for hive.metastore.uris", expectedMetaStoreURIs, hiveSiteProperties.get("hive.metastore.uris")); @@ -2920,8 +2890,6 @@ public class BlueprintConfigurationProcessorTest { oozieSiteProperties.put("oozie.services.ext", "org.apache.oozie.service.ZKLocksService,org.apache.oozie.service.ZKXLogStreamingService,org.apache.oozie.service.ZKJobsConcurrencyService,org.apache.oozie.service.ZKUUIDService"); - - oozieEnvProperties.put("oozie_hostname", expectedHostName); oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); @@ -2952,8 +2920,6 @@ public class BlueprintConfigurationProcessorTest { assertEquals("oozie property not updated correctly", expectedHostName, oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); assertEquals("oozie property not updated correctly", - expectedHostName, oozieEnvProperties.get("oozie_hostname")); - assertEquals("oozie property not updated correctly", expectedHostName + "," + expectedHostNameTwo, coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); } @@ -2988,7 +2954,6 @@ public class BlueprintConfigurationProcessorTest { oozieSiteProperties.put("oozie.zookeeper.connection.string", createHostAddress(expectedHostName, "2181") + "," + createHostAddress(expectedHostNameTwo, "2181")); - oozieEnvProperties.put("oozie_hostname", expectedHostName); oozieEnvProperties.put("oozie_existing_mysql_host", expectedExternalHost); coreSiteProperties.put("hadoop.proxyuser.oozie.hosts", expectedHostName + "," + expectedHostNameTwo); @@ -3021,8 +2986,6 @@ public class BlueprintConfigurationProcessorTest { assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName), oozieSiteProperties.get("oozie.service.HadoopAccessorService.kerberos.principal")); assertEquals("oozie property not updated correctly", - createExportedHostName(expectedHostGroupName), oozieEnvProperties.get("oozie_hostname")); - assertEquals("oozie property not updated correctly", createExportedHostName(expectedHostGroupName) + "," + createExportedHostName(expectedHostGroupNameTwo), coreSiteProperties.get("hadoop.proxyuser.oozie.hosts")); assertEquals("oozie property not updated correctly", createExportedAddress("2181", expectedHostGroupName) + "," + createExportedAddress("2181", expectedHostGroupNameTwo), oozieSiteProperties.get("oozie.zookeeper.connection.string")); http://git-wip-us.apache.org/repos/asf/ambari/blob/1042f1cd/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index cfb57ab..e3e8d35 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -19,6 +19,9 @@ package org.apache.ambari.server.upgrade; +import javax.persistence.EntityManager; + +import junit.framework.Assert; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; @@ -33,6 +36,7 @@ import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertNull; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -68,6 +72,7 @@ import org.apache.ambari.server.state.AlertFirmness; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.stack.OsFamily; import org.easymock.Capture; @@ -372,6 +377,7 @@ public class UpgradeCatalog240Test { Method updateHostRoleCommandTableDML = UpgradeCatalog240.class.getDeclaredMethod("updateHostRoleCommandTableDML"); Method updateKerberosEnv = UpgradeCatalog240.class.getDeclaredMethod("updateKerberosConfigs"); Method updateYarnEnv = UpgradeCatalog240.class.getDeclaredMethod("updateYarnEnv"); + Method removeHiveOozieDBConnectionConfigs = UpgradeCatalog240.class.getDeclaredMethod("removeHiveOozieDBConnectionConfigs"); Capture<String> capturedStatements = newCapture(CaptureType.ALL); @@ -389,6 +395,7 @@ public class UpgradeCatalog240Test { .addMockedMethod(updateHostRoleCommandTableDML) .addMockedMethod(updateKerberosEnv) .addMockedMethod(updateYarnEnv) + .addMockedMethod(removeHiveOozieDBConnectionConfigs) .createMock(); Field field = AbstractUpgradeCatalog.class.getDeclaredField("dbAccessor"); @@ -404,6 +411,7 @@ public class UpgradeCatalog240Test { upgradeCatalog240.updateHostRoleCommandTableDML(); upgradeCatalog240.updateKerberosConfigs(); upgradeCatalog240.updateYarnEnv(); + upgradeCatalog240.removeHiveOozieDBConnectionConfigs(); replay(upgradeCatalog240, dbAccessor); @@ -424,6 +432,66 @@ public class UpgradeCatalog240Test { } @Test + public void testRemoveHiveOozieDBConnectionConfigs() throws Exception{ + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock( + AmbariManagementController.class); + final ConfigHelper mockConfigHelper = easyMockSupport.createMock(ConfigHelper.class); + + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + + final Config mockOozieEnv = easyMockSupport.createNiceMock(Config.class); + final Config mockHiveEnv = easyMockSupport.createNiceMock(Config.class); + + final Map<String, String> propertiesExpectedOozieEnv = new HashMap<String, String>(); + propertiesExpectedOozieEnv.put("oozie_derby_database", "Derby"); + propertiesExpectedOozieEnv.put("property", "value"); + // Imitate missing property + // propertiesExpectedOozieEnv.put("oozie_hostname", "hostname"); + final Map<String, String> propertiesExpectedHiveEnv = new HashMap<String, String>(); + propertiesExpectedHiveEnv.put("hive_hostname", "hostname"); + + final Injector mockInjector = Guice.createInjector(new Module() { + @Override + public void configure(Binder binder) { + binder.bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + binder.bind(ConfigHelper.class).toInstance(mockConfigHelper); + binder.bind(Clusters.class).toInstance(mockClusters); + binder.bind(EntityManager.class).toInstance(entityManager); + binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", mockClusterExpected); + }}).once(); + + expect(mockClusterExpected.getDesiredConfigByType("oozie-env")).andReturn(mockOozieEnv).atLeastOnce(); + expect(mockClusterExpected.getDesiredConfigByType("hive-env")).andReturn(mockHiveEnv).atLeastOnce(); + expect(mockOozieEnv.getProperties()).andReturn(propertiesExpectedOozieEnv).anyTimes(); + expect(mockHiveEnv.getProperties()).andReturn(propertiesExpectedHiveEnv).anyTimes(); + + Capture<Map<String, String>> oozieCapture = newCapture(); + Capture<Map<String, String>> hiveCapture = newCapture(); + expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("oozie-env"), + capture(oozieCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); + expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("hive-env"), + capture(hiveCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); + + easyMockSupport.replayAll(); + mockInjector.getInstance(UpgradeCatalog240.class).removeHiveOozieDBConnectionConfigs(); + easyMockSupport.verifyAll(); + + assertEquals("value", oozieCapture.getValue().get("property")); + assertNull(oozieCapture.getValue().get("oozie_derby_database")); + assertNull(oozieCapture.getValue().get("oozie_hostname")); + assertNull(hiveCapture.getValue().get("hive_hostname")); + } + + @Test public void test_addParam_ParamsNotAvailable() { UpgradeCatalog240 upgradeCatalog240 = new UpgradeCatalog240(injector);
