[18/44] hadoop git commit: YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang

2018-04-13 Thread xyao 
YARN-7221. Add security check for privileged docker container. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/933477e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/933477e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/933477e9

Branch: refs/heads/HDFS-7240
Commit: 933477e9e0526e2ed81ea454f8806de31981822a
Parents: f7d5bac
Author: Billie Rinaldi 
Authored: Wed Apr 11 08:23:20 2018 -0700
Committer: Billie Rinaldi 
Committed: Wed Apr 11 11:24:23 2018 -0700

--
 .../runtime/DockerLinuxContainerRuntime.java|  10 +-
 .../container-executor/impl/utils/docker-util.c | 100 ++-
 .../test/utils/test_docker_util.cc  |  97 +-
 .../runtime/TestDockerContainerRuntime.java |  11 +-
 4 files changed, 157 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 51abeb6..7106aad 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -767,7 +767,11 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
   throw new ContainerExecutionException(message);
 }
   }
-  dockerRunAsUser = uid + ":" + gid;
+  if (!allowPrivilegedContainerExecution(container)) {
+dockerRunAsUser = uid + ":" + gid;
+  } else {
+dockerRunAsUser = ctx.getExecutionAttribute(USER);
+  }
 }
 
 //List -> stored as List -> fetched/converted to List
@@ -879,7 +883,9 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 }
 
 if(enableUserReMapping) {
-  runCommand.groupAdd(groups);
+  if (!allowPrivilegedContainerExecution(container)) {
+runCommand.groupAdd(groups);
+  }
 }
 
 // use plugins to update docker run command.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 3bd94a1..fdeaeea 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -16,6 +16,9 @@
  * limitations under the License.
  */
 
+#include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -25,6 +28,9 @@
 #include "docker-util.h"
 #include "string-utils.h"
 #include "util.h"
+#include 
+#include 
+#include 
 
 static int read_and_verify_command_file(const char *command_file, const char 
*docker_command,
 struct configuration *command_config) {
@@ -1254,14 +1260,94 @@ static int  add_rw_mounts(const struct configuration 
*command_config, const stru
   return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
 }
 
+static int check_privileges(const char *user) {
+  int ngroups = 0;
+  gid_t *groups = NULL;
+  struct passwd *pw;
+  struct group *gr;
+  int ret = 0;
+  int waitid = -1;
+  int statval = 0;
+
+  pw = getpwnam(user);
+  if (pw == NULL) {
+fprintf(ERRORFILE, "User %s does not exist in host OS.\n", user);
+exit(INITIALIZE_USER_FAILED);
+  }
+
+  int rc = getgrouplist(user, pw->pw_gid, groups, &ngroups);
+  i

hadoop git commit: YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang

2018-04-11 Thread billie 
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 3c1cd08f0 -> 11f1d4982


YARN-7221. Add security check for privileged docker container. Contributed by 
Eric Yang

(cherry picked from commit 933477e9e0526e2ed81ea454f8806de31981822a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11f1d498
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11f1d498
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11f1d498

Branch: refs/heads/branch-3.1
Commit: 11f1d498232f6be50d657009963183159e1dd97a
Parents: 3c1cd08
Author: Billie Rinaldi 
Authored: Wed Apr 11 08:23:20 2018 -0700
Committer: Billie Rinaldi 
Committed: Wed Apr 11 12:15:00 2018 -0700

--
 .../runtime/DockerLinuxContainerRuntime.java|  10 +-
 .../container-executor/impl/utils/docker-util.c | 100 ++-
 .../test/utils/test_docker_util.cc  |  97 +-
 .../runtime/TestDockerContainerRuntime.java |  11 +-
 4 files changed, 157 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/11f1d498/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 0290493..567c4b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -760,7 +760,11 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
   throw new ContainerExecutionException(message);
 }
   }
-  dockerRunAsUser = uid + ":" + gid;
+  if (!allowPrivilegedContainerExecution(container)) {
+dockerRunAsUser = uid + ":" + gid;
+  } else {
+dockerRunAsUser = ctx.getExecutionAttribute(USER);
+  }
 }
 
 //List -> stored as List -> fetched/converted to List
@@ -872,7 +876,9 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 }
 
 if(enableUserReMapping) {
-  runCommand.groupAdd(groups);
+  if (!allowPrivilegedContainerExecution(container)) {
+runCommand.groupAdd(groups);
+  }
 }
 
 // use plugins to update docker run command.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/11f1d498/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index ccc21fa..465dc49 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -16,6 +16,9 @@
  * limitations under the License.
  */
 
+#include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -25,6 +28,9 @@
 #include "docker-util.h"
 #include "string-utils.h"
 #include "util.h"
+#include 
+#include 
+#include 
 
 static int read_and_verify_command_file(const char *command_file, const char 
*docker_command,
 struct configuration *command_config) {
@@ -1214,14 +1220,94 @@ static int  add_rw_mounts(const struct configuration 
*command_config, const stru
   return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
 }
 
+static int check_privileges(const char *user) {
+  int ngroups = 0;
+  gid_t *groups = NULL;
+  struct passwd *pw;
+  struct group *gr;
+  int ret = 0;
+  int waitid = -1;
+  int statval = 0;
+
+  pw = getpwnam(user);
+  if (pw == NULL) {
+fprintf(ERRORFILE

hadoop git commit: YARN-7221. Add security check for privileged docker container. Contributed by Eric Yang

2018-04-11 Thread billie 
Repository: hadoop
Updated Branches:
  refs/heads/trunk f7d5bace4 -> 933477e9e


YARN-7221. Add security check for privileged docker container. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/933477e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/933477e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/933477e9

Branch: refs/heads/trunk
Commit: 933477e9e0526e2ed81ea454f8806de31981822a
Parents: f7d5bac
Author: Billie Rinaldi 
Authored: Wed Apr 11 08:23:20 2018 -0700
Committer: Billie Rinaldi 
Committed: Wed Apr 11 11:24:23 2018 -0700

--
 .../runtime/DockerLinuxContainerRuntime.java|  10 +-
 .../container-executor/impl/utils/docker-util.c | 100 ++-
 .../test/utils/test_docker_util.cc  |  97 +-
 .../runtime/TestDockerContainerRuntime.java |  11 +-
 4 files changed, 157 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 51abeb6..7106aad 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -767,7 +767,11 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
   throw new ContainerExecutionException(message);
 }
   }
-  dockerRunAsUser = uid + ":" + gid;
+  if (!allowPrivilegedContainerExecution(container)) {
+dockerRunAsUser = uid + ":" + gid;
+  } else {
+dockerRunAsUser = ctx.getExecutionAttribute(USER);
+  }
 }
 
 //List -> stored as List -> fetched/converted to List
@@ -879,7 +883,9 @@ public class DockerLinuxContainerRuntime implements 
LinuxContainerRuntime {
 }
 
 if(enableUserReMapping) {
-  runCommand.groupAdd(groups);
+  if (!allowPrivilegedContainerExecution(container)) {
+runCommand.groupAdd(groups);
+  }
 }
 
 // use plugins to update docker run command.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/933477e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
index 3bd94a1..fdeaeea 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c
@@ -16,6 +16,9 @@
  * limitations under the License.
  */
 
+#include 
+#include 
+#include 
 #include 
 #include 
 #include 
@@ -25,6 +28,9 @@
 #include "docker-util.h"
 #include "string-utils.h"
 #include "util.h"
+#include 
+#include 
+#include 
 
 static int read_and_verify_command_file(const char *command_file, const char 
*docker_command,
 struct configuration *command_config) {
@@ -1254,14 +1260,94 @@ static int  add_rw_mounts(const struct configuration 
*command_config, const stru
   return add_mounts(command_config, conf, "rw-mounts", 0, out, outlen);
 }
 
+static int check_privileges(const char *user) {
+  int ngroups = 0;
+  gid_t *groups = NULL;
+  struct passwd *pw;
+  struct group *gr;
+  int ret = 0;
+  int waitid = -1;
+  int statval = 0;
+
+  pw = getpwnam(user);
+  if (pw == NULL) {
+fprintf(ERRORFILE, "User %s does not exist in host OS.\n", user);
+exit(INITIALIZE_USER_FAILE