HDFS-13560. Insufficient system resources exist to complete the requested service for some tests on Windows. Contributed by Anbang Hu.
(cherry picked from commit 53b807a6a8486cefe0b036f7893de9f619bd44a1) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/905c3773 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/905c3773 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/905c3773 Branch: refs/heads/branch-2.9 Commit: 905c37734b8c0132e9015e7deec5874ae02936a8 Parents: 4795efc Author: Inigo Goiri <inigo...@apache.org> Authored: Thu May 17 17:03:23 2018 -0700 Committer: Inigo Goiri <inigo...@apache.org> Committed: Thu May 17 17:05:13 2018 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/hadoop/util/Shell.java | 15 +++++++++++++++ .../hdfs/server/datanode/TestDirectoryScanner.java | 3 ++- .../datanode/fsdataset/impl/LazyPersistTestCase.java | 3 ++- .../hdfs/server/namenode/TestNameNodeMXBean.java | 7 +++++-- 4 files changed, 24 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/905c3773/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 1798570..2a46bd8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -1377,4 +1377,19 @@ public abstract class Shell { return new HashSet<>(CHILD_SHELLS.keySet()); } } + + /** + * Static method to return the memory lock limit for datanode. + * @param ulimit max value at which memory locked should be capped. + * @return long value specifying the memory lock limit. + */ + public static Long getMemlockLimit(Long ulimit) { + if (WINDOWS) { + // HDFS-13560: if ulimit is too large on Windows, Windows will complain + // "1450: Insufficient system resources exist to complete the requested + // service". Thus, cap Windows memory lock limit at Integer.MAX_VALUE. + return Math.min(Integer.MAX_VALUE, ulimit); + } + return ulimit; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/905c3773/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 2ee624b..fdcee6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.server.datanode; +import static org.apache.hadoop.util.Shell.getMemlockLimit; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; @@ -95,7 +96,7 @@ public class TestDirectoryScanner { CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1); CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L); CONF.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, - Long.MAX_VALUE); + getMemlockLimit(Long.MAX_VALUE)); } @Before http://git-wip-us.apache.org/repos/asf/hadoop/blob/905c3773/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java index 4a01e55..3c86e6d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java @@ -26,6 +26,7 @@ import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST; import static org.apache.hadoop.fs.StorageType.DEFAULT; import static org.apache.hadoop.fs.StorageType.RAM_DISK; import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.util.Shell.getMemlockLimit; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -422,7 +423,7 @@ public abstract class LazyPersistTestCase { private StorageType[] storageTypes = null; private int ramDiskReplicaCapacity = -1; private long ramDiskStorageLimit = -1; - private long maxLockedMemory = Long.MAX_VALUE; + private long maxLockedMemory = getMemlockLimit(Long.MAX_VALUE); private boolean hasTransientStorage = true; private boolean useScr = false; private boolean useLegacyBlockReaderLocal = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/905c3773/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 8250ea6..b93cc64 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -63,6 +63,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import static org.apache.hadoop.util.Shell.getMemlockLimit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -91,8 +92,10 @@ public class TestNameNodeMXBean { @Test public void testNameNodeMXBeanInfo() throws Exception { Configuration conf = new Configuration(); + Long maxLockedMemory = getMemlockLimit( + NativeIO.POSIX.getCacheManipulator().getMemlockLimit()); conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY, - NativeIO.POSIX.getCacheManipulator().getMemlockLimit()); + maxLockedMemory); MiniDFSCluster cluster = null; try { @@ -244,7 +247,7 @@ public class TestNameNodeMXBean { assertEquals(1, statusMap.get("active").size()); assertEquals(1, statusMap.get("failed").size()); assertEquals(0L, mbs.getAttribute(mxbeanName, "CacheUsed")); - assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() * + assertEquals(maxLockedMemory * cluster.getDataNodes().size(), mbs.getAttribute(mxbeanName, "CacheCapacity")); assertNull("RollingUpgradeInfo should be null when there is no rolling" --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org