Repository: hadoop Updated Branches: refs/heads/HDFS-6581 222bf0fe6 -> 09dab88d3
YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless explicitly set. Contributed by Zhijie Shen Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/444acf8e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/444acf8e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/444acf8e Branch: refs/heads/HDFS-6581 Commit: 444acf8ea795e4bc782f1ce3b5ef7a1a47d1d27d Parents: 25fd69a Author: Jian He <jia...@apache.org> Authored: Fri Sep 19 11:26:29 2014 -0700 Committer: Jian He <jia...@apache.org> Committed: Fri Sep 19 11:26:29 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ApplicationHistoryServer.java | 4 +- .../ahs/RMApplicationHistoryWriter.java | 49 +++++++++++--------- .../ahs/TestRMApplicationHistoryWriter.java | 18 +++++++ 4 files changed, 50 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 8f717ae..7175b14 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -400,6 +400,9 @@ Release 2.6.0 - UNRELEASED YARN-2568. Fixed the potential test failures due to race conditions when RM work-preserving recovery is enabled. (Jian He via zjshen) + YARN-2565. Fixed RM to not use FileSystemApplicationHistoryStore unless + explicitly set. (Zhijie Shen via jianhe) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 204e118..f52ab07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -182,7 +182,9 @@ public class ApplicationHistoryServer extends CompositeService { // APPLICATION_HISTORY_STORE is neither null nor empty, it means that the // user has enabled it explicitly. if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null || - conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) { + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals( + NullApplicationHistoryStore.class.getName())) { return new ApplicationHistoryManagerOnTimelineStore( timelineDataManager, aclsManager); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index 58d2e3d..bd328ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -52,6 +52,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import com.google.common.annotations.VisibleForTesting; + /** * <p> * {@link ResourceManager} uses this class to write the information of @@ -71,8 +73,10 @@ public class RMApplicationHistoryWriter extends CompositeService { .getLog(RMApplicationHistoryWriter.class); private Dispatcher dispatcher; - private ApplicationHistoryWriter writer; - private boolean historyServiceEnabled; + @VisibleForTesting + ApplicationHistoryWriter writer; + @VisibleForTesting + boolean historyServiceEnabled; public RMApplicationHistoryWriter() { super(RMApplicationHistoryWriter.class.getName()); @@ -80,13 +84,18 @@ public class RMApplicationHistoryWriter extends CompositeService { @Override protected synchronized void serviceInit(Configuration conf) throws Exception { - historyServiceEnabled = conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED); + if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0 || + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).equals( + NullApplicationHistoryStore.class.getName())) { + historyServiceEnabled = false; + } - // Only create the services when the history service is enabled, preventing - // wasting the system resources. + // Only create the services when the history service is enabled and not + // using the null store, preventing wasting the system resources. if (historyServiceEnabled) { writer = createApplicationHistoryStore(conf); addIfService(writer); @@ -112,25 +121,19 @@ public class RMApplicationHistoryWriter extends CompositeService { protected ApplicationHistoryStore createApplicationHistoryStore( Configuration conf) { - // If the history writer is not enabled, a dummy store will be used to - // write nothing - if (historyServiceEnabled) { - try { - Class<? extends ApplicationHistoryStore> storeClass = - conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE, - FileSystemApplicationHistoryStore.class, + try { + Class<? extends ApplicationHistoryStore> storeClass = + conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE, + NullApplicationHistoryStore.class, ApplicationHistoryStore.class); - return storeClass.newInstance(); - } catch (Exception e) { - String msg = - "Could not instantiate ApplicationHistoryWriter: " - + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE, - FileSystemApplicationHistoryStore.class.getName()); - LOG.error(msg, e); - throw new YarnRuntimeException(msg, e); - } - } else { - return new NullApplicationHistoryStore(); + return storeClass.newInstance(); + } catch (Exception e) { + String msg = + "Could not instantiate ApplicationHistoryWriter: " + + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE, + NullApplicationHistoryStore.class.getName()); + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/444acf8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java index e83a6b9..1c90ad2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java @@ -78,6 +78,8 @@ public class TestRMApplicationHistoryWriter { store = new MemoryApplicationHistoryStore(); Configuration conf = new Configuration(); conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + conf.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, + MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class); writer = new RMApplicationHistoryWriter() { @Override @@ -175,6 +177,22 @@ public class TestRMApplicationHistoryWriter { } @Test + public void testDefaultStoreSetup() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true); + RMApplicationHistoryWriter writer = new RMApplicationHistoryWriter(); + writer.init(conf); + writer.start(); + try { + Assert.assertFalse(writer.historyServiceEnabled); + Assert.assertNull(writer.writer); + } finally { + writer.stop(); + writer.close(); + } + } + + @Test public void testWriteApplication() throws Exception { RMApp app = createRMApp(ApplicationId.newInstance(0, 1));
