This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new a072610 AMBARI-24283 - DB consistency warning due to config group
without service name (#1876)
a072610 is described below
commit a07261021bd3001cd33748c2203c2479b9df8c32
Author: kasakrisz <33458261+kasakr...@users.noreply.github.com>
AuthorDate: Fri Jul 27 14:26:51 2018 +0200
AMBARI-24283 - DB consistency warning due to config group without service
name (#1876)
* AMBARI-24283 - DB consistency warning due to config group without service
name
* AMBARI-24283 - DB consistency warning due to config group without service
name - output message
* AMBARI-24283 - DB consistency warning due to config group without service
name - doc
* AMBARI-24283 - DB consistency warning due to config group without service
name - out message fix
* AMBARI-24283 - DB consistency warning due to config group without service
name - check service name is valid
* AMBARI-24283 - DB consistency warning due to config group without service
name - check service name is valid by service map
* AMBARI-24283 - DB consistency warning due to config group without service
name - do not delete config groups without service name
---
.../checks/DatabaseConsistencyCheckHelper.java | 109 ++++++++++++++---
.../checks/DatabaseConsistencyCheckHelperTest.java | 130 +++++++++++++++++++--
2 files changed, 218 insertions(+), 21 deletions(-)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index 1f94bae..fac79ac 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -51,7 +51,6 @@ import
org.apache.ambari.server.orm.dao.HostComponentDesiredStateDAO;
import org.apache.ambari.server.orm.dao.HostComponentStateDAO;
import org.apache.ambari.server.orm.dao.MetainfoDAO;
import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
-import org.apache.ambari.server.orm.entities.ConfigGroupConfigMappingEntity;
import org.apache.ambari.server.orm.entities.HostComponentDesiredStateEntity;
import org.apache.ambari.server.orm.entities.HostComponentStateEntity;
import org.apache.ambari.server.orm.entities.MetainfoEntity;
@@ -60,8 +59,6 @@ import
org.apache.ambari.server.state.ClientConfigFileDefinition;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ComponentInfo;
-import org.apache.ambari.server.state.Config;
-import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.SecurityState;
import org.apache.ambari.server.state.Service;
@@ -185,6 +182,7 @@ public class DatabaseConsistencyCheckHelper {
if (fixIssues) {
fixHostComponentStatesCountEqualsHostComponentsDesiredStates();
fixClusterConfigsNotMappedToAnyService();
+ fixConfigGroupServiceNames();
fixConfigGroupHostMappings();
fixConfigGroupsForDeletedServices();
fixConfigsSelectedMoreThanOnce();
@@ -198,6 +196,7 @@ public class DatabaseConsistencyCheckHelper {
checkServiceConfigs();
checkTopologyTables();
checkForLargeTables();
+ checkConfigGroupsHasServiceName();
checkConfigGroupHostMapping(true);
checkConfigGroupsForDeletedServices(true);
LOG.info("******************************* Check database completed
*******************************");
@@ -1191,6 +1190,82 @@ public class DatabaseConsistencyCheckHelper {
}
/**
+ * This method collects the ConfigGroups with empty or null service name
field
+ */
+ static Map<Long, ConfigGroup> collectConfigGroupsWithoutServiceName() {
+ Map<Long, ConfigGroup> configGroupMap = new HashMap<>();
+ Clusters clusters = injector.getInstance(Clusters.class);
+ Map<String, Cluster> clusterMap = clusters.getClusters();
+
+ if (MapUtils.isEmpty(clusterMap))
+ return configGroupMap;
+
+ for (Cluster cluster : clusterMap.values()) {
+ Map<Long, ConfigGroup> configGroups = cluster.getConfigGroups();
+
+ if (MapUtils.isEmpty(configGroups))
+ continue;
+
+ for (ConfigGroup configGroup : configGroups.values()) {
+ if (StringUtils.isEmpty(configGroup.getServiceName())) {
+ configGroupMap.put(configGroup.getId(), configGroup);
+ }
+ }
+ }
+
+ return configGroupMap;
+ }
+
+ /**
+ * This method checks if there are any ConfigGroup with empty or null
service name field
+ */
+ static void checkConfigGroupsHasServiceName() {
+ Map<Long, ConfigGroup> configGroupMap =
collectConfigGroupsWithoutServiceName();
+ if (MapUtils.isEmpty(configGroupMap))
+ return;
+
+ StringBuilder output = new StringBuilder("[(ConfigGroup) => ");
+
+ for (ConfigGroup configGroup : configGroupMap.values()) {
+ output.append("( ");
+ output.append(configGroup.getName());
+ output.append(" ), ");
+ }
+
+ output.replace(output.lastIndexOf(","), output.length(), "]");
+ warning("You have config groups present in the database with no " +
+ "service name, {}. Run --auto-fix-database to fix " +
+ "this automatically. Please backup Ambari Server database before
running --auto-fix-database.", output.toString());
+ }
+
+ /**
+ * Fix inconsistencies found by @collectConfigGroupsWithoutServiceName
+ */
+ @Transactional
+ static void fixConfigGroupServiceNames() {
+ Map<Long, ConfigGroup> configGroupMap =
collectConfigGroupsWithoutServiceName();
+ if (MapUtils.isEmpty(configGroupMap))
+ return;
+
+ Clusters clusters = injector.getInstance(Clusters.class);
+
+ for (Map.Entry<Long, ConfigGroup> configGroupEntry :
configGroupMap.entrySet()) {
+ ConfigGroup configGroup = configGroupEntry.getValue();
+ try {
+ Cluster cluster = clusters.getCluster(configGroup.getClusterName());
+ Map<String, Service> serviceMap = cluster.getServices();
+ if (serviceMap.containsKey(configGroup.getTag())) {
+ LOG.info("Setting service name of config group {} with id {} to {}",
+ configGroup.getName(), configGroupEntry.getKey(),
configGroup.getTag());
+ configGroup.setServiceName(configGroup.getTag());
+ }
+ } catch (AmbariException e) {
+ // Ignore if cluster not found
+ }
+ }
+ }
+
+ /**
* This method checks if there are any ConfigGroup host mappings with hosts
* that are not longer a part of the cluster.
*/
@@ -1253,7 +1328,7 @@ public class DatabaseConsistencyCheckHelper {
warning("You have config group host mappings with hosts that are no " +
"longer associated with the cluster, {}. Run --auto-fix-database to " +
"fix this automatically. Alternatively, you can remove this mapping " +
- "from the UI.", output.toString());
+ "from the UI. Please backup Ambari Server database before running
--auto-fix-database.", output.toString());
}
return nonMappedHostIds;
@@ -1289,7 +1364,7 @@ public class DatabaseConsistencyCheckHelper {
output.replace(output.lastIndexOf(","), output.length(), "]");
warning("You have config groups present in the database with no " +
"corresponding service found, {}. Run --auto-fix-database to fix " +
- "this automatically.", output.toString());
+ "this automatically. Please backup Ambari Server database before
running --auto-fix-database.", output.toString());
}
return configGroupMap;
@@ -1304,15 +1379,21 @@ public class DatabaseConsistencyCheckHelper {
for (Map.Entry<Long, ConfigGroup> configGroupEntry :
configGroupMap.entrySet()) {
Long id = configGroupEntry.getKey();
ConfigGroup configGroup = configGroupEntry.getValue();
- LOG.info("Deleting config group {} with id {} for deleted service {}",
- configGroup.getName(), id, configGroup.getServiceName());
- try {
- Cluster cluster = clusters.getCluster(configGroup.getClusterName());
- cluster.deleteConfigGroup(id);
- } catch (AuthorizationException e) {
- // This call does not thrown Authorization Exception
- } catch (AmbariException e) {
- // Ignore if cluster not found
+ if (!StringUtils.isEmpty(configGroup.getServiceName())) {
+ LOG.info("Deleting config group {} with id {} for deleted service
{}",
+ configGroup.getName(), id, configGroup.getServiceName());
+ try {
+ Cluster cluster =
clusters.getCluster(configGroup.getClusterName());
+ cluster.deleteConfigGroup(id);
+ } catch (AuthorizationException e) {
+ // This call does not thrown Authorization Exception
+ } catch (AmbariException e) {
+ // Ignore if cluster not found
+ }
+ }
+ else {
+ warning("The config group {} with id {} can not be fixed
automatically because service name is missing.",
+ configGroup.getName(), id);
}
}
}
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
index 23dd1e6..73c4aac 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelperTest.java
@@ -713,23 +713,26 @@ public class DatabaseConsistencyCheckHelperTest {
});
Map<String, Cluster> clusters = new HashMap<>();
- Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+ Cluster cluster = easyMockSupport.createStrictMock(Cluster.class);
clusters.put("c1", cluster);
expect(mockClusters.getClusters()).andReturn(clusters).anyTimes();
Map<Long, ConfigGroup> configGroupMap = new HashMap<>();
ConfigGroup cg1 = easyMockSupport.createNiceMock(ConfigGroup.class);
ConfigGroup cg2 = easyMockSupport.createNiceMock(ConfigGroup.class);
+ ConfigGroup cgWithoutServiceName =
easyMockSupport.createNiceMock(ConfigGroup.class);
configGroupMap.put(1L, cg1);
configGroupMap.put(2L, cg2);
+ configGroupMap.put(3L, cgWithoutServiceName);
- expect(cluster.getConfigGroups()).andReturn(configGroupMap).anyTimes();
+ expect(cluster.getConfigGroups()).andStubReturn(configGroupMap);
expect(cg1.getName()).andReturn("cg1").anyTimes();
expect(cg1.getId()).andReturn(1L).anyTimes();
expect(cg1.getServiceName()).andReturn("YARN").anyTimes();
expect(cg2.getServiceName()).andReturn("HDFS").anyTimes();
-
- expect(cluster.getClusterName()).andReturn("c1").anyTimes();
+ expect(cgWithoutServiceName.getName()).andReturn("cg3").anyTimes();
+ expect(cgWithoutServiceName.getId()).andReturn(3L).anyTimes();
+ expect(cgWithoutServiceName.getServiceName()).andReturn(null).anyTimes();
Service service = easyMockSupport.createNiceMock(Service.class);
Map<String, Service> services = new HashMap<>();
@@ -737,7 +740,7 @@ public class DatabaseConsistencyCheckHelperTest {
expect(cluster.getServices()).andReturn(services).anyTimes();
expect(cg1.getClusterName()).andReturn("c1");
- expect(mockClusters.getCluster("c1")).andReturn(cluster);
+ expect(mockClusters.getCluster("c1")).andReturn(cluster).anyTimes();
cluster.deleteConfigGroup(1L);
expectLastCall();
@@ -751,8 +754,121 @@ public class DatabaseConsistencyCheckHelperTest {
easyMockSupport.verifyAll();
Assert.assertFalse(MapUtils.isEmpty(configGroups));
- Assert.assertEquals(1, configGroups.size());
- Assert.assertEquals(1L,
configGroups.values().iterator().next().getId().longValue());
+ Assert.assertEquals(2, configGroups.size());
+ Assert.assertTrue(configGroups.containsKey(1L));
+ Assert.assertFalse(configGroups.containsKey(2L));
+ Assert.assertTrue(configGroups.containsKey(3L));
+ }
+
+ @Test
+ public void testCollectConfigGroupsWithoutServiceName() throws Exception {
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ final DBAccessor mockDBDbAccessor =
easyMockSupport.createNiceMock(DBAccessor.class);
+
+ final StackManagerFactory mockStackManagerFactory =
easyMockSupport.createNiceMock(StackManagerFactory.class);
+ final EntityManager mockEntityManager =
easyMockSupport.createNiceMock(EntityManager.class);
+ final Clusters mockClusters =
easyMockSupport.createNiceMock(Clusters.class);
+ final OsFamily mockOSFamily =
easyMockSupport.createNiceMock(OsFamily.class);
+ final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(StackManagerFactory.class).toInstance(mockStackManagerFactory);
+ bind(EntityManager.class).toInstance(mockEntityManager);
+ bind(DBAccessor.class).toInstance(mockDBDbAccessor);
+ bind(Clusters.class).toInstance(mockClusters);
+ bind(OsFamily.class).toInstance(mockOSFamily);
+ }
+ });
+
+ Map<String, Cluster> clusters = new HashMap<>();
+ Cluster cluster1 = easyMockSupport.createNiceMock(Cluster.class);
+ clusters.put("c1", cluster1);
+ Cluster cluster2 = easyMockSupport.createNiceMock(Cluster.class);
+ clusters.put("c2", cluster2);
+ expect(cluster2.getConfigGroups()).andReturn(new HashMap<Long,
ConfigGroup>(0)).anyTimes();
+ expect(mockClusters.getClusters()).andReturn(clusters).anyTimes();
+ expect(mockClusters.getCluster("c1")).andReturn(cluster1).anyTimes();
+ expect(mockClusters.getCluster("c2")).andReturn(cluster2).anyTimes();
+
+ Map<Long, ConfigGroup> configGroupMap = new HashMap<>();
+ ConfigGroup cgWithoutServiceName =
easyMockSupport.createNiceMock(ConfigGroup.class);
+ ConfigGroup cgWithServiceName =
easyMockSupport.createNiceMock(ConfigGroup.class);
+ ConfigGroup cgForNonExistentService =
easyMockSupport.createNiceMock(ConfigGroup.class);
+ configGroupMap.put(1L, cgWithoutServiceName);
+ configGroupMap.put(2L, cgWithServiceName);
+ configGroupMap.put(3L, cgForNonExistentService);
+
+ expect(cluster1.getConfigGroups()).andReturn(configGroupMap).anyTimes();
+ expect(cgWithoutServiceName.getId()).andReturn(1L).anyTimes();
+ expect(cgWithoutServiceName.getClusterName()).andReturn("c1").anyTimes();
+ expect(cgWithoutServiceName.getServiceName()).andReturn(null).anyTimes();
+ expect(cgWithoutServiceName.getTag()).andReturn("YARN").anyTimes();
+ cgWithoutServiceName.setServiceName("YARN"); expectLastCall();
+ expect(cgWithServiceName.getId()).andReturn(2L).anyTimes();
+ expect(cgWithServiceName.getClusterName()).andReturn("c1").anyTimes();
+ expect(cgWithServiceName.getServiceName()).andReturn("HDFS").anyTimes();
+ expect(cgForNonExistentService.getId()).andReturn(3L).anyTimes();
+
expect(cgForNonExistentService.getClusterName()).andReturn("c1").anyTimes();
+
expect(cgForNonExistentService.getServiceName()).andReturn(null).anyTimes();
+
expect(cgForNonExistentService.getTag()).andReturn("NOT_EXISTS").anyTimes();
+
+ Service hdfsService = easyMockSupport.createNiceMock(Service.class);
+ Service yarnService = easyMockSupport.createNiceMock(Service.class);
+ Map<String, Service> services = new HashMap<>();
+ services.put("HDFS", hdfsService);
+ services.put("YARN", yarnService);
+ expect(cluster1.getServices()).andReturn(services).anyTimes();
+
+ DatabaseConsistencyCheckHelper.setInjector(mockInjector);
+
+ easyMockSupport.replayAll();
+
+ Map<Long, ConfigGroup> configGroups =
DatabaseConsistencyCheckHelper.collectConfigGroupsWithoutServiceName();
+ DatabaseConsistencyCheckHelper.fixConfigGroupServiceNames();
+
+ easyMockSupport.verifyAll();
+
+ Assert.assertFalse(MapUtils.isEmpty(configGroups));
+ Assert.assertEquals(2, configGroups.size());
+ Assert.assertTrue(configGroups.containsKey(1L));
+ Assert.assertFalse(configGroups.containsKey(2L));
+ Assert.assertTrue(configGroups.containsKey(3L));
+ }
+
+ @Test
+ public void
testCollectConfigGroupsWithoutServiceNameReturnsEmptyMapWhenNoClusters() throws
Exception {
+ EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+ final DBAccessor mockDBDbAccessor =
easyMockSupport.createNiceMock(DBAccessor.class);
+
+ final StackManagerFactory mockStackManagerFactory =
easyMockSupport.createNiceMock(StackManagerFactory.class);
+ final EntityManager mockEntityManager =
easyMockSupport.createNiceMock(EntityManager.class);
+ final Clusters mockClusters =
easyMockSupport.createNiceMock(Clusters.class);
+ final OsFamily mockOSFamily =
easyMockSupport.createNiceMock(OsFamily.class);
+ final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+ @Override
+ protected void configure() {
+ bind(StackManagerFactory.class).toInstance(mockStackManagerFactory);
+ bind(EntityManager.class).toInstance(mockEntityManager);
+ bind(DBAccessor.class).toInstance(mockDBDbAccessor);
+ bind(Clusters.class).toInstance(mockClusters);
+ bind(OsFamily.class).toInstance(mockOSFamily);
+ }
+ });
+
+ Map<String, Cluster> clusters = new HashMap<>();
+ expect(mockClusters.getClusters()).andReturn(clusters).anyTimes();
+
+ DatabaseConsistencyCheckHelper.setInjector(mockInjector);
+
+ easyMockSupport.replayAll();
+
+ Map<Long, ConfigGroup> configGroups =
DatabaseConsistencyCheckHelper.collectConfigGroupsWithoutServiceName();
+
+ easyMockSupport.verifyAll();
+
+ Assert.assertTrue(MapUtils.isEmpty(configGroups));
}
@Test
