This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new dd0d841 YARN-2194. Fix bug causing CGroups functionality to fail on
RHEL7.
dd0d841 is described below
commit dd0d841ce50a88c844f82559c5d16ecb8f6fa35e
Author: Jonathan Hung <[email protected]>
AuthorDate: Tue May 14 10:27:14 2019 -0700
YARN-2194. Fix bug causing CGroups functionality to fail on RHEL7.
---
.../src/main/java/org/apache/hadoop/util/StringUtils.java | 9 +++++++++
.../test/java/org/apache/hadoop/util/TestStringUtils.java | 4 ++++
.../yarn/server/nodemanager/LinuxContainerExecutor.java | 14 ++++++++++----
.../nodemanager/util/CgroupsLCEResourcesHandler.java | 5 +++--
.../main/native/container-executor/impl/configuration.c | 8 ++++----
.../container-executor/test/test-container-executor.c | 4 ++--
.../nodemanager/TestLinuxContainerExecutorWithMocks.java | 12 ++++++++----
7 files changed, 40 insertions(+), 16 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index fc4b0ab..8496e1b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -872,6 +872,11 @@ public class StringUtils {
return sb.toString();
}
+ public static String join(char separator, Iterable<?> strings) {
+ return join(separator + "", strings);
+ }
+
+
/**
* Concatenates strings, using a separator.
*
@@ -894,6 +899,10 @@ public class StringUtils {
return sb.toString();
}
+ public static String join(char separator, String[] strings) {
+ return join(separator + "", strings);
+ }
+
/**
* Convert SOME_STUFF to SomeStuff
*
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
index 515c3e0..ed92e87 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
@@ -277,8 +277,12 @@ public class TestStringUtils extends UnitTestcaseTimeLimit
{
s.add("c");
assertEquals("", StringUtils.join(":", s.subList(0, 0)));
assertEquals("a", StringUtils.join(":", s.subList(0, 1)));
+ assertEquals("", StringUtils.join(':', s.subList(0, 0)));
+ assertEquals("a", StringUtils.join(':', s.subList(0, 1)));
assertEquals("a:b", StringUtils.join(":", s.subList(0, 2)));
assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3)));
+ assertEquals("a:b", StringUtils.join(':', s.subList(0, 2)));
+ assertEquals("a:b:c", StringUtils.join(':', s.subList(0, 3)));
}
@Test (timeout = 30000)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index e4288fe..0018698 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -53,6 +53,8 @@ public class LinuxContainerExecutor extends ContainerExecutor
{
private static final Log LOG = LogFactory
.getLog(LinuxContainerExecutor.class);
+ public final static char LINUX_FILE_PATH_SEPARATOR = '%';
+
private String nonsecureLocalUser;
private Pattern nonsecureLocalUserPattern;
private String containerExecutorExe;
@@ -214,8 +216,10 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
Integer.toString(Commands.INITIALIZE_CONTAINER.getValue()),
appId,
nmPrivateContainerTokensPath.toUri().getPath().toString(),
- StringUtils.join(",", localDirs),
- StringUtils.join(",", logDirs)));
+ StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+ localDirs),
+ StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+ logDirs)));
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
@@ -286,8 +290,10 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
nmPrivateCotainerScriptPath.toUri().getPath().toString(),
nmPrivateTokensPath.toUri().getPath().toString(),
pidFilePath.toString(),
- StringUtils.join(",", localDirs),
- StringUtils.join(",", logDirs),
+ StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+ localDirs),
+ StringUtils.join(LINUX_FILE_PATH_SEPARATOR,
+ logDirs),
resourcesOptions));
String[] commandArray = command.toArray(new String[command.size()]);
shExec = new ShellCommandExecutor(commandArray, null, // NM's cwd
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index ffa17ac..b451854 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -409,10 +409,11 @@ public class CgroupsLCEResourcesHandler implements
LCEResourcesHandler {
if (isCpuWeightEnabled()) {
sb.append(pathForCgroup(CONTROLLER_CPU, containerName) + "/tasks");
- sb.append(",");
+ sb.append(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR);
}
- if (sb.charAt(sb.length() - 1) == ',') {
+ if (sb.charAt(sb.length() - 1) ==
+ LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR) {
sb.deleteCharAt(sb.length() - 1);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 1667b0d..41ea268 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -286,14 +286,14 @@ char ** extract_values_delim(char *value, const char
*delim) {
//first allocate any array of 10
if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize);
- tempTok = strtok_r((char *)value, ",", &tempstr);
+ tempTok = strtok_r((char *)value, "%", &tempstr);
while (tempTok != NULL) {
toPass[size++] = tempTok;
if(size == toPassSize) {
toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
}
- tempTok = strtok_r(NULL, ",", &tempstr);
+ tempTok = strtok_r(NULL, "%", &tempstr);
}
}
if (toPass != NULL) {
@@ -303,7 +303,7 @@ char ** extract_values_delim(char *value, const char
*delim) {
}
/**
- * Extracts array of values from the comma separated list of values.
+ * Extracts array of values from the '%' separated list of values.
*/
char ** extract_values(char *value) {
return extract_values_delim(value, ",");
@@ -311,7 +311,7 @@ char ** extract_values(char *value) {
/**
* Function to return an array of values for a key.
- * Value delimiter is assumed to be a comma.
+ * Value delimiter is assumed to be a '%'.
*/
char ** get_values(const char * key, struct configuration *cfg) {
char *value = get_value(key, cfg);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 2375bf4..4102779 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -42,8 +42,8 @@
#define TEST_ROOT TMPDIR "/test-container-executor"
#define DONT_TOUCH_FILE "dont-touch-me"
-#define NM_LOCAL_DIRS TEST_ROOT "/local-1," TEST_ROOT "/local-2," \
- TEST_ROOT "/local-3," TEST_ROOT "/local-4," TEST_ROOT "/local-5"
+#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
+ TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
#define NM_LOG_DIRS TEST_ROOT "/logs/userlogs"
#define ARRAY_SIZE 1000
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 81591ca..1755d8c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -137,8 +137,10 @@ public class TestLinuxContainerExecutorWithMocks {
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
- StringUtils.join(",", dirsHandler.getLocalDirs()),
- StringUtils.join(",", dirsHandler.getLogDirs()), "cgroups=none"),
+ StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLocalDirs()),
+ StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLogDirs()), "cgroups=none"),
readMockParams());
}
@@ -283,8 +285,10 @@ public class TestLinuxContainerExecutorWithMocks {
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, appId, containerId,
workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
- StringUtils.join(",", dirsHandler.getLocalDirs()),
- StringUtils.join(",", dirsHandler.getLogDirs()),
+ StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLocalDirs()),
+ StringUtils.join(LinuxContainerExecutor.LINUX_FILE_PATH_SEPARATOR,
+ dirsHandler.getLogDirs()),
"cgroups=none"), readMockParams());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
