AMBARI-16959. Redundant info under config tab for falcon (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/23f71f24 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/23f71f24 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/23f71f24 Branch: refs/heads/branch-2.4 Commit: 23f71f24ea4de437396e75fa784c92aa950f91d5 Parents: 2e689f3 Author: Andrew Onishuk <[email protected]> Authored: Thu Jun 2 15:52:16 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Jun 2 15:52:16 2016 +0300 ---------------------------------------------------------------------- .../server/upgrade/UpgradeCatalog240.java | 24 +++++++++ .../0.5.0.2.1/configuration/falcon-env.xml | 6 --- .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 11 ----- .../0.5.0.2.1/package/scripts/params_linux.py | 1 - .../server/upgrade/UpgradeCatalog240Test.java | 51 ++++++++++++++++++++ .../stacks/2.1/FALCON/test_falcon_server.py | 8 --- ambari-web/app/data/HDP2/site_properties.js | 6 --- 7 files changed, 75 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java index 949e4d7..19d370c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog240.java @@ -353,6 +353,7 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { updateKerberosConfigs(); updateYarnEnv(); updatePhoenixConfigs(); + updateFalconConfigs(); updateKerberosDescriptorArtifacts(); removeHiveOozieDBConnectionConfigs(); updateClustersAndHostsVersionStateTableDML(); @@ -1893,6 +1894,29 @@ public class UpgradeCatalog240 extends AbstractUpgradeCatalog { } /** + * Updates the Falcon-related configurations for the clusters managed by this Ambari + * Removes falcon_store_uri from falcon-env. + * + * @throws AmbariException if an error occurs while updating the configurations + */ + protected void updateFalconConfigs() throws AmbariException { + AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class); + Clusters clusters = ambariManagementController.getClusters(); + Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters); + + for (final Cluster cluster : clusterMap.values()) { + Config falconEnvConfig = cluster.getDesiredConfigByType("falcon-env"); + if (falconEnvConfig != null) { + Map<String, String> falconEnvEnvProperties = falconEnvConfig.getProperties(); + if (falconEnvEnvProperties.containsKey("falcon_store_uri")) { + LOG.info("Removing property falcon_store_uri from falcon-env"); + removeConfigurationPropertiesFromCluster(cluster, "falcon-env", Collections.singleton("falcon_store_uri")); + } + } + } + } + + /** * Removes the HDFS/AMS alert definitions for the standard deviation alerts, * including all history, notifications and groupings. * <p/> http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml index 45cd9a7..194999a 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml @@ -69,12 +69,6 @@ <type>directory</type> </value-attributes> </property> - <property> - <name>falcon_store_uri</name> - <value>file:///hadoop/falcon/store</value> - <display-name>Falcon store URI</display-name> - <description>Directory where entity definitions are stored.</description> - </property> <!--embeddedmq properties--> <property> <name>falcon.embeddedmq.data</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 6e2e678..5e25325 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -130,17 +130,6 @@ def falcon(type, action = None, upgrade_type=None): owner = params.falcon_user, mode = 0777) - if params.falcon_store_uri[0:4] == "hdfs": - params.HdfsResource(params.falcon_store_uri, - type = "directory", - action = "create_on_execute", - owner = params.falcon_user, - mode = 0755) - elif params.falcon_store_uri[0:4] == "file": - Directory(params.falcon_store_uri[7:], - owner = params.falcon_user, - create_parents = True) - if params.supports_hive_dr: params.HdfsResource(params.dfs_data_mirroring_dir, type = "directory", http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 8e666ac..8c2ad8e 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -80,7 +80,6 @@ proxyuser_group = config['configurations']['hadoop-env']['proxyuser_group'] java_home = config['hostLevelParams']['java_home'] falcon_local_dir = config['configurations']['falcon-env']['falcon_local_dir'] -falcon_store_uri = config['configurations']['falcon-env']['falcon_store_uri'] falcon_log_dir = config['configurations']['falcon-env']['falcon_log_dir'] # falcon-startup.properties http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java index f8510c2..9b1c16c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog240Test.java @@ -491,6 +491,7 @@ public class UpgradeCatalog240Test { Method upgradeCapSchedulerView = UpgradeCatalog240.class.getDeclaredMethod("upgradeCapSchedulerView"); Method updatePhoenixConfigs = UpgradeCatalog240.class.getDeclaredMethod("updatePhoenixConfigs"); Method updateKerberosDescriptorArtifacts = AbstractUpgradeCatalog.class.getDeclaredMethod("updateKerberosDescriptorArtifacts"); + Method updateFalconConfigs = UpgradeCatalog240.class.getDeclaredMethod("updateFalconConfigs"); Capture<String> capturedStatements = newCapture(CaptureType.ALL); @@ -524,6 +525,7 @@ public class UpgradeCatalog240Test { .addMockedMethod(upgradeCapSchedulerView) .addMockedMethod(updatePhoenixConfigs) .addMockedMethod(updateKerberosDescriptorArtifacts) + .addMockedMethod(updateFalconConfigs) .createMock(); Field field = AbstractUpgradeCatalog.class.getDeclaredField("dbAccessor"); @@ -552,6 +554,7 @@ public class UpgradeCatalog240Test { upgradeCatalog240.upgradeCapSchedulerView(); upgradeCatalog240.updatePhoenixConfigs(); upgradeCatalog240.updateKerberosDescriptorArtifacts(); + upgradeCatalog240.updateFalconConfigs(); replay(upgradeCatalog240, dbAccessor); @@ -636,6 +639,54 @@ public class UpgradeCatalog240Test { } @Test + public void testUpdateFalconConfigs() throws Exception{ + EasyMockSupport easyMockSupport = new EasyMockSupport(); + final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock( + AmbariManagementController.class); + final ConfigHelper mockConfigHelper = easyMockSupport.createMock(ConfigHelper.class); + + final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class); + final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class); + + final Config mockFalconEnv = easyMockSupport.createNiceMock(Config.class); + + final Map<String, String> propertiesExpectedFalconEnv = new HashMap<String, String>(); + propertiesExpectedFalconEnv.put("falcon_store_uri", "file:///hadoop/falcon/store"); + propertiesExpectedFalconEnv.put("property", "value"); + + final Injector mockInjector = Guice.createInjector(new Module() { + @Override + public void configure(Binder binder) { + binder.bind(AmbariManagementController.class).toInstance(mockAmbariManagementController); + binder.bind(ConfigHelper.class).toInstance(mockConfigHelper); + binder.bind(Clusters.class).toInstance(mockClusters); + binder.bind(EntityManager.class).toInstance(entityManager); + binder.bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class)); + binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + } + }); + + expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once(); + expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{ + put("normal", mockClusterExpected); + }}).once(); + + expect(mockClusterExpected.getDesiredConfigByType("falcon-env")).andReturn(mockFalconEnv).atLeastOnce(); + expect(mockFalconEnv.getProperties()).andReturn(propertiesExpectedFalconEnv).anyTimes(); + + Capture<Map<String, String>> falconCapture = newCapture(); + expect(mockAmbariManagementController.createConfig(eq(mockClusterExpected), eq("falcon-env"), + capture(falconCapture), anyString(), (Map<String, Map<String, String>>)anyObject())).andReturn(null).once(); + + easyMockSupport.replayAll(); + mockInjector.getInstance(UpgradeCatalog240.class).updateFalconConfigs(); + easyMockSupport.verifyAll(); + + assertEquals("value", falconCapture.getValue().get("property")); + assertNull(falconCapture.getValue().get("falcon_store_uri")); + } + + @Test public void test_addParam_ParamsNotAvailable() { UpgradeCatalog240 upgradeCatalog240 = new UpgradeCatalog240(injector); http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py index 298480c..68dc7c9 100644 --- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py +++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py @@ -146,10 +146,6 @@ class TestFalconServer(RMFTestCase): action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, ) - self.assertResourceCalled('Directory', '/hadoop/falcon/store', - owner = 'falcon', - create_parents = True, - ) self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, @@ -322,10 +318,6 @@ class TestFalconServer(RMFTestCase): action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', mode = 0777, ) - self.assertResourceCalled('Directory', '/hadoop/falcon/store', - owner = 'falcon', - create_parents = True, - ) self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, http://git-wip-us.apache.org/repos/asf/ambari/blob/23f71f24/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 2634785..794da25 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -1614,12 +1614,6 @@ var hdp2properties = [ "filename": "falcon-env.xml", "category": "FALCON_SERVER" }, - { - "name": "falcon_store_uri", - "serviceName": "FALCON", - "filename": "falcon-env.xml", - "category": "FALCON_SERVER" - }, /************************************************AMBARI_METRICS******************************************/ { "name": "timeline.metrics.service.operation.mode",
