[18/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-12 Thread sunilg 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74ba6ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74ba6ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74ba6ffa

Branch: refs/heads/YARN-5881
Commit: 74ba6ffa0b601badc68366685dcf993a0844520a
Parents: 04f9e80
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:11:23 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[32/51] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-10 Thread aengineer 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74ba6ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74ba6ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74ba6ffa

Branch: refs/heads/HDFS-7240
Commit: 74ba6ffa0b601badc68366685dcf993a0844520a
Parents: 04f9e80
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:11:23 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[10/18] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread xgong 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/793476e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/793476e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/793476e4

Branch: refs/heads/branch-3.0
Commit: 793476e4b703b26421362e6af7eba2076ee032f2
Parents: 09b7c87
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 14:54:55 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/793476e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 60fdfb5..7da4772 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -648,8 +648,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/793476e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b261a5f..0291840 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3363,4 +3363,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[13/18] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread xgong 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74ba6ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74ba6ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74ba6ffa

Branch: refs/heads/trunk
Commit: 74ba6ffa0b601badc68366685dcf993a0844520a
Parents: 04f9e80
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:11:23 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[10/17] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/793476e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/793476e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/793476e4

Branch: refs/heads/YARN-5734-branch-3.0
Commit: 793476e4b703b26421362e6af7eba2076ee032f2
Parents: 09b7c87
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 14:54:55 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/793476e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 60fdfb5..7da4772 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -648,8 +648,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/793476e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b261a5f..0291840 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3363,4 +3363,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[12/18] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread xgong 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd58f5da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd58f5da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd58f5da

Branch: refs/heads/branch-2
Commit: cd58f5da5a87f9d1d1f585ad639c06aac85dd775
Parents: 16b2757
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:12:07 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd58f5da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cef3c74..84465eb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -612,8 +612,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd58f5da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 80984d9..c4a761d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3215,4 +3215,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[39/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74ba6ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74ba6ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74ba6ffa

Branch: refs/heads/YARN-5734
Commit: 74ba6ffa0b601badc68366685dcf993a0844520a
Parents: 04f9e80
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:11:23 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/74ba6ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[36/43] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-09 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd58f5da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd58f5da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd58f5da

Branch: refs/heads/YARN-5734-branch-2
Commit: cd58f5da5a87f9d1d1f585ad639c06aac85dd775
Parents: 16b2757
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Oct 9 11:12:07 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd58f5da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cef3c74..84465eb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -612,8 +612,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd58f5da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 80984d9..c4a761d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3215,4 +3215,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[44/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-06 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f127c549
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f127c549
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f127c549

Branch: refs/heads/YARN-5734
Commit: f127c549c3973db4d6d3df21d5e562f9e459ca32
Parents: 376ab6b
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Fri Oct 6 16:54:53 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b603135..ea8652d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f127c549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e124c5b..5afec1b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[09/17] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-10-05 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/341de82d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/341de82d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/341de82d

Branch: refs/heads/YARN-5734-branch-2
Commit: 341de82dd0facdda1f45c1e5512931bfed8836e6
Parents: 0c3e7ec
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Thu Oct 5 13:40:50 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/341de82d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cef3c74..84465eb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -612,8 +612,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/341de82d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index df2ae1f..7afa513 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3215,4 +3215,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[43/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-09-27 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44475c4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44475c4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44475c4f

Branch: refs/heads/YARN-5734
Commit: 44475c4fef3685189177a72bd9c1886e2a93c493
Parents: a210907
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Wed Sep 27 19:52:58 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/44475c4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 609f689..612e3d1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/44475c4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5fcb27c..63c6f09 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[28/30] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-09-22 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00eda9b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00eda9b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00eda9b0

Branch: refs/heads/YARN-5734
Commit: 00eda9b033f239b7a71dc54066d960461538d03d
Parents: 6053efd
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Fri Sep 22 11:26:30 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/00eda9b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 609f689..612e3d1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/00eda9b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5fcb27c..63c6f09 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3390,4 +3390,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[48/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-09-20 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b06711cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b06711cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b06711cb

Branch: refs/heads/YARN-5734
Commit: b06711cb1f6cdbe5f60fbd1f0cbf7890e1ef779d
Parents: e462f10
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Wed Sep 20 17:40:54 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06711cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e1062d7..a33d85d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -677,8 +677,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b06711cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 86aa15e..4529f20 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3371,4 +3371,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[35/36] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-08-22 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d74672dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d74672dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d74672dd

Branch: refs/heads/YARN-5734
Commit: d74672dd788df15258d8ebb0437454b33cd5604f
Parents: a88c3da
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Tue Aug 22 19:56:58 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d74672dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 3c001ff..6ffe1f7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -642,8 +642,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d74672dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a15dbe9..2d6574f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3249,4 +3249,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[50/50] [abbrv] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-08-17 Thread jhung 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/279456ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/279456ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/279456ec

Branch: refs/heads/YARN-5734
Commit: 279456ecedd4a94ec0ee477160bb082176f1cc3f
Parents: b44f573
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Jonathan Hung 
Committed: Thu Aug 17 13:50:11 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/279456ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e7cbcfa..73f4d16 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -638,8 +638,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/279456ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index dae3a86..bad0492 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3241,4 +3241,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

[20/20] hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-08-01 Thread xgong 
YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3e2b6fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3e2b6fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3e2b6fd

Branch: refs/heads/YARN-5734
Commit: d3e2b6fd5b7701a7d5f2ac33b09c72f520da8d6e
Parents: 79701d9
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Xuan 
Committed: Tue Aug 1 08:46:44 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e2b6fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 01db626..c3644cd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -624,8 +624,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3e2b6fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a0bed5f..6de9ab6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3159,4 +3159,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+

hadoop git commit: YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing store. Contributed by Jonathan Hung

2017-07-31 Thread xgong 
Repository: hadoop
Updated Branches:
  refs/heads/YARN-5734 7418ef399 -> fa2c5451f


YARN-5947: Create LeveldbConfigurationStore class using Leveldb as backing 
store. Contributed by Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa2c5451
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa2c5451
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa2c5451

Branch: refs/heads/YARN-5734
Commit: fa2c5451fd3841da79a03409ebe30838d47f3659
Parents: 7418ef3
Author: Xuan 
Authored: Mon Jul 31 16:48:40 2017 -0700
Committer: Xuan 
Committed: Mon Jul 31 16:48:40 2017 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  13 +
 .../src/main/resources/yarn-default.xml |  29 ++
 .../scheduler/MutableConfigurationProvider.java |   6 +
 .../scheduler/capacity/CapacityScheduler.java   |   3 +
 .../conf/LeveldbConfigurationStore.java | 314 +++
 .../conf/MutableCSConfigurationProvider.java|  38 ++-
 .../capacity/conf/YarnConfigurationStore.java   |  14 +-
 .../conf/TestYarnConfigurationStore.java|   3 +-
 8 files changed, 414 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa2c5451/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 01db626..c3644cd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -624,8 +624,21 @@ public class YarnConfiguration extends Configuration {
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX + "scheduler.configuration.store.class";
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
   public static final String DEFAULT_CONFIGURATION_STORE =
   MEMORY_CONFIGURATION_STORE;
+  public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.path";
+
+  public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
+  YARN_PREFIX
+  + "scheduler.configuration.leveldb-store.compaction-interval-secs";
+  public static final long
+  DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
+
+  public static final String RM_SCHEDCONF_LEVELDB_MAX_LOGS =
+  YARN_PREFIX + "scheduler.configuration.leveldb-store.max-logs";
+  public static final int DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
 
   public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
   YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa2c5451/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a0bed5f..6de9ab6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3159,4 +3159,33 @@
 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy
   
 
+  
+
+  The storage path for LevelDB implementation of configuration store,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb".
+
+yarn.scheduler.configuration.leveldb-store.path
+${hadoop.tmp.dir}/yarn/system/confstore
+  
+
+  
+
+  The compaction interval for LevelDB configuration store in secs,
+  when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is one day.
+
+
yarn.scheduler.configuration.leveldb-store.compaction-interval-secs
+86400
+  
+
+  
+
+  The max number of configuration change log entries kept in LevelDB config
+  store, when yarn.scheduler.configuration.store.class is configured to be
+  "leveldb". Default is 1000.
+
+yarn.scheduler.configuration.leveldb-store.max-logs
+1000
+