AMBARI-14519. GET calls to /requests become very slow after installing packages on 900 node cluster. (mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8a748ffd Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8a748ffd Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8a748ffd Branch: refs/heads/branch-dev-patch-upgrade Commit: 8a748ffd9829cc25350ea79ed9f44abe0a98b012 Parents: 9022bfb Author: Myroslav Papirkovskyy <[email protected]> Authored: Tue Dec 29 16:45:15 2015 +0200 Committer: Myroslav Papirkovskyy <[email protected]> Committed: Tue Dec 29 19:45:27 2015 +0200 ---------------------------------------------------------------------- .../ambari/server/topology/AmbariContext.java | 4 +++ .../ambari/server/topology/HostRequest.java | 4 +++ .../ambari/server/topology/LogicalRequest.java | 35 +++++++++++++++++--- .../server/upgrade/UpgradeCatalog221.java | 6 +++- .../main/resources/Ambari-DDL-Derby-CREATE.sql | 5 +++ .../main/resources/Ambari-DDL-MySQL-CREATE.sql | 5 +++ .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 5 +++ .../resources/Ambari-DDL-Postgres-CREATE.sql | 5 +++ .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 5 +++ .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 5 +++ .../resources/Ambari-DDL-SQLServer-CREATE.sql | 5 +++ .../server/upgrade/UpgradeCatalog221Test.java | 31 +++++++++++++++++ 12 files changed, 110 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 0a2bd18..6bfee93 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -145,6 +145,10 @@ public class AmbariContext { return getController().getActionManager().getTaskById(id); } + public Collection<HostRoleCommand> getPhysicalTasks(Collection<Long> ids) { + return getController().getActionManager().getTasks(ids); + } + public void createAmbariResources(ClusterTopology topology, String clusterName, SecurityType securityType) { Stack stack = topology.getBlueprint().getStack(); createAmbariClusterResource(clusterName, stack.getName(), stack.getVersion(), securityType); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java index 7f178f4..440638c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java @@ -387,6 +387,10 @@ public class HostRequest implements Comparable<HostRequest> { return physicalTasks.get(logicalTaskId); } + public Map<Long, Long> getPhysicalTaskMapping() { + return new HashMap<>(physicalTasks); + } + //todo: since this is used to determine equality, using hashCode() isn't safe as it can return the same //todo: value for 2 unequal requests @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java index fb6b774..bd9f2e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java @@ -195,6 +195,7 @@ public class LogicalRequest extends Request { } // currently we are just returning all stages for all requests + //TODO technically StageEntity is simply a container for HostRequest info with additional redundant transformations public Collection<StageEntity> getStageEntities() { Collection<StageEntity> stages = new ArrayList<StageEntity>(); for (HostRequest hostRequest : allHostRequests) { @@ -232,7 +233,32 @@ public class LogicalRequest extends Request { public Map<Long, HostRoleCommandStatusSummaryDTO> getStageSummaries() { Map<Long, HostRoleCommandStatusSummaryDTO> summaryMap = new HashMap<Long, HostRoleCommandStatusSummaryDTO>(); - for (StageEntity stage : getStageEntities()) { + Map<Long, Collection<HostRoleCommand>> stageTasksMap = new HashMap<>(); + + Map<Long, Long> taskToStageMap = new HashMap<>(); + + + for (HostRequest hostRequest : getHostRequests()) { + Map<Long, Long> physicalTaskMapping = hostRequest.getPhysicalTaskMapping(); + Collection<Long> stageTasks = physicalTaskMapping.values(); + for (Long stageTask : stageTasks) { + taskToStageMap.put(stageTask, hostRequest.getStageId()); + } + } + + Collection<HostRoleCommand> physicalTasks = topology.getAmbariContext().getPhysicalTasks(taskToStageMap.keySet()); + + for (HostRoleCommand physicalTask : physicalTasks) { + Long stageId = taskToStageMap.get(physicalTask.getTaskId()); + Collection<HostRoleCommand> stageTasks = stageTasksMap.get(stageId); + if (stageTasks == null) { + stageTasks = new ArrayList<>(); + stageTasksMap.put(stageId, stageTasks); + } + stageTasks.add(physicalTask); + } + + for (Long stageId : stageTasksMap.keySet()) { //Number minStartTime = 0; //Number maxEndTime = 0; int aborted = 0; @@ -248,7 +274,7 @@ public class LogicalRequest extends Request { int skippedFailed = 0; //todo: where does this logic belong? - for (HostRoleCommandEntity task : stage.getHostRoleCommands()) { + for (HostRoleCommand task : stageTasksMap.get(stageId)) { HostRoleStatus taskStatus = task.getStatus(); switch (taskStatus) { @@ -291,12 +317,13 @@ public class LogicalRequest extends Request { } HostRoleCommandStatusSummaryDTO stageSummary = new HostRoleCommandStatusSummaryDTO( - stage.isSkippable() ? 1 : 0, 0, 0, stage.getStageId(), aborted, completed, failed, + 0, 0, 0, stageId, aborted, completed, failed, holding, holdingFailed, holdingTimedout, inProgress, pending, queued, timedout, skippedFailed); - summaryMap.put(stage.getStageId(), stageSummary); + summaryMap.put(stageId, stageSummary); } + return summaryMap; } http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java index b33d44e..c4216bd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java @@ -109,7 +109,11 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog { @Override protected void executeDDLUpdates() throws AmbariException, SQLException { - //To change body of implemented methods use File | Settings | File Templates. + // indices to improve request status calc performance + dbAccessor.createIndex("idx_stage_request_id", "stage", "request_id"); + dbAccessor.createIndex("idx_hrc_request_id", "host_role_command", "request_id"); + dbAccessor.createIndex("idx_rsc_request_id", "role_success_criteria", "request_id"); + } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 0e534b7..d93a3c2 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + --------altering tables by creating unique constraints---------- ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (version_tag, type_name, cluster_id); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index fcfcb93..aa8ced1 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -660,6 +660,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + -- altering tables by creating unique constraints---------- ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 4305a4c..b534344 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -649,6 +649,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + --------altering tables by creating unique constraints---------- ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 32fd2a6..941fc6e 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -653,6 +653,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + --------altering tables by creating unique constraints---------- ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name, version_tag); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql index 7894c43..dd517f8 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql @@ -734,6 +734,11 @@ CREATE TABLE ambari.topology_logical_task ( ); GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username; +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON ambari.stage (request_id); +CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON ambari.role_success_criteria (request_id); + --------altering tables by creating unique constraints---------- ALTER TABLE ambari.users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE ambari.clusterconfig ADD CONSTRAINT UQ_config_type_tag UNIQUE (cluster_id, type_name, version_tag); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 6df1569..f837f9e 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -650,6 +650,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + -- altering tables by creating unique constraints---------- ALTER TABLE users ADD CONSTRAINT UNQ_users_0 UNIQUE (user_name, user_type); ALTER TABLE groups ADD CONSTRAINT UNQ_groups_0 UNIQUE (group_name, ldap_group); http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index 83b2a7e..239d27e 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -758,6 +758,11 @@ CREATE TABLE topology_logical_task ( PRIMARY KEY CLUSTERED (id) ); +-- tasks indices -- +CREATE INDEX idx_stage_request_id ON stage (request_id); +CREATE INDEX idx_hrc_request_id ON host_role_command (request_id); +CREATE INDEX idx_rsc_request_id ON role_success_criteria (request_id); + -- altering tables by creating unique constraints---------- --------altering tables to add constraints---------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8a748ffd/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java index bbae7a7..5bd8f2a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java @@ -22,8 +22,10 @@ package org.apache.ambari.server.upgrade; import com.google.inject.AbstractModule; import com.google.common.collect.Maps; import com.google.gson.Gson; +import com.google.inject.Binder; import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.Module; import com.google.inject.Provider; import com.google.inject.persist.PersistService; import junit.framework.Assert; @@ -60,6 +62,7 @@ import java.util.Map; import static org.easymock.EasyMock.createMockBuilder; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.createStrictMock; +import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.replay; @@ -101,6 +104,34 @@ public class UpgradeCatalog221Test { } @Test + public void testExecuteDDLUpdates() throws Exception{ + final DBAccessor dbAccessor = createNiceMock(DBAccessor.class); + + dbAccessor.createIndex(eq("idx_stage_request_id"), eq("stage"), eq("request_id")); + expectLastCall().once(); + dbAccessor.createIndex(eq("idx_hrc_request_id"), eq("host_role_command"), eq("request_id")); + expectLastCall().once(); + dbAccessor.createIndex(eq("idx_rsc_request_id"), eq("role_success_criteria"), eq("request_id")); + expectLastCall().once(); + + + replay(dbAccessor); + Module module = new Module() { + @Override + public void configure(Binder binder) { + binder.bind(DBAccessor.class).toInstance(dbAccessor); + binder.bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class)); + binder.bind(EntityManager.class).toInstance(entityManager); + } + }; + + Injector injector = Guice.createInjector(module); + UpgradeCatalog221 upgradeCatalog221 = injector.getInstance(UpgradeCatalog221.class); + upgradeCatalog221.executeDDLUpdates(); + verify(dbAccessor); + } + + @Test public void testExecuteDMLUpdates() throws Exception { Method addNewConfigurationsFromXml = AbstractUpgradeCatalog.class.getDeclaredMethod("addNewConfigurationsFromXml"); Method updateAlerts = UpgradeCatalog221.class.getDeclaredMethod("updateAlerts");
