This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 2f01a91 YARN-10002. Code cleanup and improvements in
ConfigurationStoreBaseTest. Contributed by Benjamin Teke
2f01a91 is described below
commit 2f01a91428341248620374271f7d362e4229a924
Author: Szilard Nemeth <snem...@apache.org>
AuthorDate: Wed Apr 15 08:24:15 2020 +0200
YARN-10002. Code cleanup and improvements in ConfigurationStoreBaseTest.
Contributed by Benjamin Teke
---
.../conf/FSSchedulerConfigurationStore.java | 7 ++
.../capacity/conf/InMemoryConfigurationStore.java | 7 ++
.../capacity/conf/LeveldbConfigurationStore.java | 1 +
.../capacity/conf/YarnConfigurationStore.java | 8 ++
.../capacity/conf/ZKConfigurationStore.java | 1 +
.../capacity/conf/ConfigurationStoreBaseTest.java | 40 +++---
.../conf/PersistentConfigurationStoreBaseTest.java | 136 ++++++++++++++++++++
.../conf/TestFSSchedulerConfigurationStore.java | 115 +++++++++++++----
.../conf/TestLeveldbConfigurationStore.java | 112 ++---------------
.../capacity/conf/TestZKConfigurationStore.java | 140 ++++-----------------
10 files changed, 299 insertions(+), 268 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 464ef14..690a60f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -21,6 +21,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -327,6 +328,12 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
}
@Override
+ protected LinkedList<LogMutation> getLogs() {
+ // Unimplemented.
+ return null;
+ }
+
+ @Override
protected Version getConfStoreVersion() throws Exception {
return null;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 47dd6bd..8bd4ad0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -85,6 +86,12 @@ public class InMemoryConfigurationStore extends
YarnConfigurationStore {
}
@Override
+ protected LinkedList<LogMutation> getLogs() {
+ // Unimplemented.
+ return null;
+ }
+
+ @Override
public Version getConfStoreVersion() throws Exception {
// Does nothing.
return null;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 5da002e..f921a9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -333,6 +333,7 @@ public class LeveldbConfigurationStore extends
YarnConfigurationStore {
}
@VisibleForTesting
+ @Override
protected LinkedList<LogMutation> getLogs() throws Exception {
return deserLogMutations(db.get(bytes(LOG_KEY)));
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 597f67a..4c64449 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -27,6 +27,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import java.io.IOException;
import java.io.Serializable;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -154,6 +155,13 @@ public abstract class YarnConfigurationStore {
protected abstract Version getConfStoreVersion() throws Exception;
/**
+ * Get a list of configuration mutations.
+ * @return list of configuration mutations.
+ * @throws Exception On mutation fetch failure
+ */
+ protected abstract LinkedList<LogMutation> getLogs() throws Exception;
+
+ /**
* Persist the hard-coded schema version to the conf store.
* @throws Exception On storage failure
*/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 1aee415..934f70b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -120,6 +120,7 @@ public class ZKConfigurationStore extends
YarnConfigurationStore {
}
@VisibleForTesting
+ @Override
protected LinkedList<LogMutation> getLogs() throws Exception {
return (LinkedList<LogMutation>)
deserializeObject(zkManager.getData(logsPath));
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
index 0f50b53..ed1159c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationStoreBaseTest.java
@@ -34,16 +34,13 @@ import static org.junit.Assert.assertNull;
* Base class for {@link YarnConfigurationStore} implementations.
*/
public abstract class ConfigurationStoreBaseTest {
+ static final String TEST_USER = "testUser";
+ YarnConfigurationStore confStore = createConfStore();
+ Configuration conf;
+ Configuration schedConf;
+ RMContext rmContext;
- protected YarnConfigurationStore confStore = createConfStore();
-
- protected abstract YarnConfigurationStore createConfStore();
-
- protected Configuration conf;
- protected Configuration schedConf;
- protected RMContext rmContext;
-
- protected static final String TEST_USER = "testUser";
+ abstract YarnConfigurationStore createConfStore();
@Before
public void setUp() throws Exception {
@@ -59,19 +56,11 @@ public abstract class ConfigurationStoreBaseTest {
confStore.initialize(conf, schedConf, rmContext);
assertEquals("val1", confStore.retrieve().get("key1"));
- Map<String, String> update1 = new HashMap<>();
- update1.put("keyUpdate1", "valUpdate1");
- YarnConfigurationStore.LogMutation mutation1 =
- new YarnConfigurationStore.LogMutation(update1, TEST_USER);
- confStore.logMutation(mutation1);
+ prepareLogMutation("keyUpdate1", "valUpdate1");
confStore.confirmMutation(true);
assertEquals("valUpdate1", confStore.retrieve().get("keyUpdate1"));
- Map<String, String> update2 = new HashMap<>();
- update2.put("keyUpdate2", "valUpdate2");
- YarnConfigurationStore.LogMutation mutation2 =
- new YarnConfigurationStore.LogMutation(update2, TEST_USER);
- confStore.logMutation(mutation2);
+ prepareLogMutation("keyUpdate2", "valUpdate2");
confStore.confirmMutation(false);
assertNull("Configuration should not be updated",
confStore.retrieve().get("keyUpdate2"));
@@ -84,13 +73,18 @@ public abstract class ConfigurationStoreBaseTest {
confStore.initialize(conf, schedConf, rmContext);
assertEquals("val", confStore.retrieve().get("key"));
+ prepareLogMutation("key", null);
+ confStore.confirmMutation(true);
+ assertNull(confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ void prepareLogMutation(String key, String value)
+ throws Exception {
Map<String, String> update = new HashMap<>();
- update.put("key", null);
+ update.put(key, value);
YarnConfigurationStore.LogMutation mutation =
new YarnConfigurationStore.LogMutation(update, TEST_USER);
confStore.logMutation(mutation);
- confStore.confirmMutation(true);
- assertNull(confStore.retrieve().get("key"));
- confStore.close();
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
new file mode 100644
index 0000000..dfb5a29
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/PersistentConfigurationStoreBaseTest.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.junit.Test;
+
+import java.util.LinkedList;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Base class for the persistent {@link YarnConfigurationStore}
+ * implementations, namely {@link TestLeveldbConfigurationStore} and
+ * {@link TestZKConfigurationStore}.
+ */
+public abstract class PersistentConfigurationStoreBaseTest extends
+ ConfigurationStoreBaseTest {
+
+ abstract Version getVersion();
+
+ @Test
+ public void testGetConfigurationVersion() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ long v1 = confStore.getConfigVersion();
+ assertEquals(1, v1);
+ prepareLogMutation("keyver", "valver");
+ confStore.confirmMutation(true);
+ long v2 = confStore.getConfigVersion();
+ assertEquals(2, v2);
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistConfiguration() throws Exception {
+ schedConf.set("key", "val");
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for old configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+ @Test
+ public void testPersistUpdatedConfiguration() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.retrieve().get("key"));
+
+ prepareLogMutation("key", "val");
+ confStore.confirmMutation(true);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+
+ // Create a new configuration store, and check for updated configuration
+ confStore = createConfStore();
+ schedConf.set("key", "badVal");
+ // Should ignore passed-in scheduler configuration.
+ confStore.initialize(conf, schedConf, rmContext);
+ assertEquals("val", confStore.retrieve().get("key"));
+ confStore.close();
+ }
+
+
+ @Test
+ public void testVersion() throws Exception {
+ confStore.initialize(conf, schedConf, rmContext);
+ assertNull(confStore.getConfStoreVersion());
+ confStore.checkVersion();
+ assertEquals(getVersion(),
+ confStore.getConfStoreVersion());
+ confStore.close();
+ }
+
+ @Test
+ public void testMaxLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
+ confStore.initialize(conf, schedConf, rmContext);
+ LinkedList<YarnConfigurationStore.LogMutation> logs = confStore.getLogs();
+ assertEquals(0, logs.size());
+
+ prepareLogMutation("key1", "val1");
+ logs = confStore.getLogs();
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ confStore.confirmMutation(true);
+ assertEquals(1, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+
+ prepareLogMutation("key2", "val2");
+ logs = confStore.getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val1", logs.get(0).getUpdates().get("key1"));
+ assertEquals("val2", logs.get(1).getUpdates().get("key2"));
+
+ // Next update should purge first update from logs.
+ prepareLogMutation("key3", "val3");
+ logs = confStore.getLogs();
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.confirmMutation(true);
+ assertEquals(2, logs.size());
+ assertEquals("val2", logs.get(0).getUpdates().get("key2"));
+ assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ confStore.close();
+ }
+
+
+}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
index 33596c3..c3ef164 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
@@ -28,21 +28,26 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
/**
* Tests {@link FSSchedulerConfigurationStore}.
*/
public class TestFSSchedulerConfigurationStore {
+ private static final String TEST_USER = "test";
private FSSchedulerConfigurationStore configurationStore;
private Configuration conf;
private File testSchedulerConfigurationDir;
@@ -88,35 +93,29 @@ public class TestFSSchedulerConfigurationStore {
Configuration storeConf = configurationStore.retrieve();
compareConfig(conf, storeConf);
- Map<String, String> updates = new HashMap<>();
- updates.put("a", null);
- updates.put("b", "bb");
-
Configuration expectConfig = new Configuration(conf);
expectConfig.unset("a");
expectConfig.set("b", "bb");
- LogMutation logMutation = new LogMutation(updates, "test");
- configurationStore.logMutation(logMutation);
- configurationStore.confirmMutation(true);
+ prepareParameterizedLogMutation(configurationStore, true,
+ "a", null, "b", "bb");
storeConf = configurationStore.retrieve();
- assertEquals(null, storeConf.get("a"));
+ assertNull(storeConf.get("a"));
assertEquals("bb", storeConf.get("b"));
assertEquals("c", storeConf.get("c"));
compareConfig(expectConfig, storeConf);
- updates.put("b", "bbb");
- configurationStore.logMutation(logMutation);
- configurationStore.confirmMutation(true);
+ prepareParameterizedLogMutation(configurationStore, true,
+ "a", null, "b", "bbb");
storeConf = configurationStore.retrieve();
- assertEquals(null, storeConf.get("a"));
+ assertNull(storeConf.get("a"));
assertEquals("bbb", storeConf.get("b"));
assertEquals("c", storeConf.get("c"));
}
@Test
- public void confirmMutationWithInValid() throws Exception {
+ public void confirmMutationWithInvalid() throws Exception {
conf.set("a", "a");
conf.set("b", "b");
conf.set("c", "c");
@@ -125,19 +124,59 @@ public class TestFSSchedulerConfigurationStore {
Configuration storeConf = configurationStore.retrieve();
compareConfig(conf, storeConf);
- Map<String, String> updates = new HashMap<>();
- updates.put("a", null);
- updates.put("b", "bb");
-
- LogMutation logMutation = new LogMutation(updates, "test");
- configurationStore.logMutation(logMutation);
- configurationStore.confirmMutation(false);
+ prepareParameterizedLogMutation(configurationStore, false,
+ "a", null, "b", "bb");
storeConf = configurationStore.retrieve();
compareConfig(conf, storeConf);
}
@Test
+ public void testFileSystemClose() throws Exception {
+ MiniDFSCluster hdfsCluster = null;
+ FileSystem fs;
+ Path path = new Path("/tmp/confstore");
+ try {
+ HdfsConfiguration hdfsConfig = new HdfsConfiguration();
+ hdfsCluster = new MiniDFSCluster.Builder(hdfsConfig)
+ .numDataNodes(1).build();
+
+ fs = hdfsCluster.getFileSystem();
+ if (!fs.exists(path)) {
+ fs.mkdirs(path);
+ }
+
+ FSSchedulerConfigurationStore configStore =
+ new FSSchedulerConfigurationStore();
+ hdfsConfig.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH,
+ path.toString());
+ configStore.initialize(hdfsConfig, hdfsConfig, null);
+
+ // Close the FileSystem object and validate
+ fs.close();
+
+ try {
+ prepareParameterizedLogMutation(configStore, true,
+ "testkey", "testvalue");
+ } catch (IOException e) {
+ if (e.getMessage().contains("Filesystem closed")) {
+ fail("FSSchedulerConfigurationStore failed to handle " +
+ "FileSystem close");
+ } else {
+ fail("Should not get any exceptions");
+ }
+ }
+ } finally {
+ assert hdfsCluster != null;
+ fs = hdfsCluster.getFileSystem();
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ hdfsCluster.shutdown();
+ }
+ }
+
+ @Test
public void testFormatConfiguration() throws Exception {
Configuration schedulerConf = new Configuration();
schedulerConf.set("a", "a");
@@ -146,15 +185,14 @@ public class TestFSSchedulerConfigurationStore {
Configuration storedConfig = configurationStore.retrieve();
assertEquals("a", storedConfig.get("a"));
configurationStore.format();
- boolean exceptionCaught = false;
try {
- storedConfig = configurationStore.retrieve();
+ configurationStore.retrieve();
+ fail("Expected an IOException with message containing \"no capacity " +
+ "scheduler file in\" to be thrown");
} catch (IOException e) {
- if (e.getMessage().contains("no capacity scheduler file in")) {
- exceptionCaught = true;
- }
+ assertThat(e.getMessage(),
+ CoreMatchers.containsString("no capacity scheduler file in"));
}
- assertTrue(exceptionCaught);
}
@Test
@@ -192,4 +230,27 @@ public class TestFSSchedulerConfigurationStore {
schedulerConf.get(entry.getKey()));
}
}
+
+ private void prepareParameterizedLogMutation(
+ FSSchedulerConfigurationStore configStore,
+ boolean validityFlag, String... values) throws Exception {
+ Map<String, String> updates = new HashMap<>();
+ String key;
+ String value;
+
+ if (values.length % 2 != 0) {
+ throw new IllegalArgumentException("The number of parameters should be "
+
+ "even.");
+ }
+
+ for (int i = 1; i <= values.length; i += 2) {
+ key = values[i - 1];
+ value = values[i];
+ updates.put(key, value);
+ }
+
+ LogMutation logMutation = new LogMutation(updates, TEST_USER);
+ configStore.logMutation(logMutation);
+ configStore.confirmMutation(validityFlag);
+ }
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
index 14e4001..4c92120 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestLeveldbConfigurationStore.java
@@ -34,9 +34,6 @@ import org.junit.Before;
import org.junit.Test;
import java.io.File;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@@ -44,7 +41,8 @@ import static org.junit.Assert.assertNull;
/**
* Tests {@link LeveldbConfigurationStore}.
*/
-public class TestLeveldbConfigurationStore extends ConfigurationStoreBaseTest {
+public class TestLeveldbConfigurationStore extends
+ PersistentConfigurationStoreBaseTest {
public static final Logger LOG =
LoggerFactory.getLogger(TestLeveldbConfigurationStore.class);
@@ -53,8 +51,6 @@ public class TestLeveldbConfigurationStore extends
ConfigurationStoreBaseTest {
System.getProperty("java.io.tmpdir")),
TestLeveldbConfigurationStore.class.getName());
- private ResourceManager rm;
-
@Before
public void setUp() throws Exception {
super.setUp();
@@ -64,16 +60,6 @@ public class TestLeveldbConfigurationStore extends
ConfigurationStoreBaseTest {
conf.set(YarnConfiguration.RM_SCHEDCONF_STORE_PATH, TEST_DIR.toString());
}
- @Test
- public void testVersioning() throws Exception {
- confStore.initialize(conf, schedConf, rmContext);
- assertNull(confStore.getConfStoreVersion());
- confStore.checkVersion();
- assertEquals(LeveldbConfigurationStore.CURRENT_VERSION_INFO,
- confStore.getConfStoreVersion());
- confStore.close();
- }
-
@Test(expected = YarnConfStoreVersionIncompatibleException.class)
public void testIncompatibleVersion() throws Exception {
try {
@@ -91,94 +77,6 @@ public class TestLeveldbConfigurationStore extends
ConfigurationStoreBaseTest {
}
}
- @Test
- public void testPersistConfiguration() throws Exception {
- schedConf.set("key", "val");
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
- confStore.close();
-
- // Create a new configuration store, and check for old configuration
- confStore = createConfStore();
- schedConf.set("key", "badVal");
- // Should ignore passed-in scheduler configuration.
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
- confStore.close();
- }
-
- @Test
- public void testPersistUpdatedConfiguration() throws Exception {
- confStore.initialize(conf, schedConf, rmContext);
- assertNull(confStore.retrieve().get("key"));
-
- Map<String, String> update = new HashMap<>();
- update.put("key", "val");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update, TEST_USER);
- confStore.logMutation(mutation);
- confStore.confirmMutation(true);
- assertEquals("val", confStore.retrieve().get("key"));
- confStore.close();
-
- // Create a new configuration store, and check for updated configuration
- confStore = createConfStore();
- schedConf.set("key", "badVal");
- // Should ignore passed-in scheduler configuration.
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
- confStore.close();
- }
-
- @Test
- public void testMaxLogs() throws Exception {
- conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
- confStore.initialize(conf, schedConf, rmContext);
- LinkedList<YarnConfigurationStore.LogMutation> logs =
- ((LeveldbConfigurationStore) confStore).getLogs();
- assertEquals(0, logs.size());
-
- Map<String, String> update1 = new HashMap<>();
- update1.put("key1", "val1");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update1, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((LeveldbConfigurationStore) confStore).getLogs();
- assertEquals(1, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- confStore.confirmMutation(true);
- assertEquals(1, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
-
- Map<String, String> update2 = new HashMap<>();
- update2.put("key2", "val2");
- mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((LeveldbConfigurationStore) confStore).getLogs();
- assertEquals(2, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- assertEquals("val2", logs.get(1).getUpdates().get("key2"));
- confStore.confirmMutation(true);
- assertEquals(2, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- assertEquals("val2", logs.get(1).getUpdates().get("key2"));
-
- // Next update should purge first update from logs.
- Map<String, String> update3 = new HashMap<>();
- update3.put("key3", "val3");
- mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((LeveldbConfigurationStore) confStore).getLogs();
- assertEquals(2, logs.size());
- assertEquals("val2", logs.get(0).getUpdates().get("key2"));
- assertEquals("val3", logs.get(1).getUpdates().get("key3"));
- confStore.confirmMutation(true);
- assertEquals(2, logs.size());
- assertEquals("val2", logs.get(0).getUpdates().get("key2"));
- assertEquals("val3", logs.get(1).getUpdates().get("key3"));
- confStore.close();
- }
-
/**
* When restarting, RM should read from current state of store, including
* any updates from the previous RM instance.
@@ -225,4 +123,10 @@ public class TestLeveldbConfigurationStore extends
ConfigurationStoreBaseTest {
public YarnConfigurationStore createConfStore() {
return new LeveldbConfigurationStore();
}
+
+ @Override
+ Version getVersion() {
+ return LeveldbConfigurationStore.CURRENT_VERSION_INFO;
+ }
+
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 7d8b3c1..e50f84e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -18,13 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
-import org.apache.hadoop.util.curator.ZKCuratorManager;
-import org.apache.hadoop.yarn.server.records.Version;
-import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.data.ACL;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryNTimes;
@@ -34,8 +27,11 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore;
@@ -44,14 +40,17 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurat
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -62,8 +61,8 @@ import static org.junit.Assert.assertTrue;
/**
* Tests {@link ZKConfigurationStore}.
*/
-public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
-
+public class TestZKConfigurationStore extends
+ PersistentConfigurationStoreBaseTest {
public static final Logger LOG =
LoggerFactory.getLogger(TestZKConfigurationStore.class);
@@ -89,6 +88,7 @@ public class TestZKConfigurationStore extends
ConfigurationStoreBaseTest {
}
@Before
+ @Override
public void setUp() throws Exception {
super.setUp();
curatorTestingServer = setupCuratorServer();
@@ -108,15 +108,6 @@ public class TestZKConfigurationStore extends
ConfigurationStoreBaseTest {
curatorTestingServer.stop();
}
- @Test
- public void testVersioning() throws Exception {
- confStore.initialize(conf, schedConf, rmContext);
- assertNull(confStore.getConfStoreVersion());
- confStore.checkVersion();
- assertEquals(ZKConfigurationStore.CURRENT_VERSION_INFO,
- confStore.getConfStoreVersion());
- }
-
@Test(expected = YarnConfStoreVersionIncompatibleException.class)
public void testIncompatibleVersion() throws Exception {
confStore.initialize(conf, schedConf, rmContext);
@@ -141,23 +132,6 @@ public class TestZKConfigurationStore extends
ConfigurationStoreBaseTest {
}
@Test
- public void testPersistConfiguration() throws Exception {
- schedConf.set("key", "val");
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
-
- assertNull(confStore.retrieve().get(YarnConfiguration.RM_HOSTNAME));
-
- // Create a new configuration store, and check for old configuration
- confStore = createConfStore();
- schedConf.set("key", "badVal");
- // Should ignore passed-in scheduler configuration.
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
- }
-
-
- @Test
public void testFormatConfiguration() throws Exception {
schedConf.set("key", "val");
confStore.initialize(conf, schedConf, rmContext);
@@ -167,87 +141,20 @@ public class TestZKConfigurationStore extends
ConfigurationStoreBaseTest {
}
@Test
- public void testGetConfigurationVersion() throws Exception {
- confStore.initialize(conf, schedConf, rmContext);
- long v1 = confStore.getConfigVersion();
- assertEquals(1, v1);
- Map<String, String> update = new HashMap<>();
- update.put("keyver", "valver");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update, TEST_USER);
- confStore.logMutation(mutation);
- confStore.confirmMutation(true);
- long v2 = confStore.getConfigVersion();
- assertEquals(2, v2);
- }
-
- @Test
- public void testPersistUpdatedConfiguration() throws Exception {
+ public void testDisableAuditLogs() throws Exception {
+ conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 0);
confStore.initialize(conf, schedConf, rmContext);
- assertNull(confStore.retrieve().get("key"));
-
- Map<String, String> update = new HashMap<>();
- update.put("key", "val");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update, TEST_USER);
- confStore.logMutation(mutation);
- confStore.confirmMutation(true);
- assertEquals("val", confStore.retrieve().get("key"));
+ String znodeParentPath = conf.get(YarnConfiguration.
+ RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH);
+ String logsPath = ZKCuratorManager.getNodePath(znodeParentPath, "LOGS");
+ byte[] data = null;
+ ((ZKConfigurationStore) confStore).zkManager.setData(logsPath, data, -1);
- // Create a new configuration store, and check for updated configuration
- confStore = createConfStore();
- schedConf.set("key", "badVal");
- // Should ignore passed-in scheduler configuration.
- confStore.initialize(conf, schedConf, rmContext);
- assertEquals("val", confStore.retrieve().get("key"));
- }
+ prepareLogMutation("key1", "val1");
- @Test
- public void testMaxLogs() throws Exception {
- conf.setLong(YarnConfiguration.RM_SCHEDCONF_MAX_LOGS, 2);
- confStore.initialize(conf, schedConf, rmContext);
- LinkedList<YarnConfigurationStore.LogMutation> logs =
- ((ZKConfigurationStore) confStore).getLogs();
- assertEquals(0, logs.size());
-
- Map<String, String> update1 = new HashMap<>();
- update1.put("key1", "val1");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update1, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((ZKConfigurationStore) confStore).getLogs();
- assertEquals(1, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- confStore.confirmMutation(true);
- assertEquals(1, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
-
- Map<String, String> update2 = new HashMap<>();
- update2.put("key2", "val2");
- mutation = new YarnConfigurationStore.LogMutation(update2, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((ZKConfigurationStore) confStore).getLogs();
- assertEquals(2, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- assertEquals("val2", logs.get(1).getUpdates().get("key2"));
- confStore.confirmMutation(true);
- assertEquals(2, logs.size());
- assertEquals("val1", logs.get(0).getUpdates().get("key1"));
- assertEquals("val2", logs.get(1).getUpdates().get("key2"));
-
- // Next update should purge first update from logs.
- Map<String, String> update3 = new HashMap<>();
- update3.put("key3", "val3");
- mutation = new YarnConfigurationStore.LogMutation(update3, TEST_USER);
- confStore.logMutation(mutation);
- logs = ((ZKConfigurationStore) confStore).getLogs();
- assertEquals(2, logs.size());
- assertEquals("val2", logs.get(0).getUpdates().get("key2"));
- assertEquals("val3", logs.get(1).getUpdates().get("key3"));
- confStore.confirmMutation(true);
- assertEquals(2, logs.size());
- assertEquals("val2", logs.get(0).getUpdates().get("key2"));
- assertEquals("val3", logs.get(1).getUpdates().get("key3"));
+ data = ((ZKConfigurationStore) confStore).zkManager.getData(logsPath);
+ assertNull("Failed to Disable Audit Logs", data);
}
public Configuration createRMHAConf(String rmIds, String rmId,
@@ -465,4 +372,9 @@ public class TestZKConfigurationStore extends
ConfigurationStoreBaseTest {
public YarnConfigurationStore createConfStore() {
return new ZKConfigurationStore();
}
+
+ @Override
+ Version getVersion() {
+ return ZKConfigurationStore.CURRENT_VERSION_INFO;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
