hadoop git commit: YARN-6726. Fix issues with docker commands executed by container-executor. (Shane Kumpf via wangda)

2017-09-11 Thread wangda 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 87af3f499 -> c093e4622


YARN-6726. Fix issues with docker commands executed by container-executor. 
(Shane Kumpf via wangda)

Change-Id: I8fa7d70526505d4082ac9b51cd6066376a599108


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c093e462
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c093e462
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c093e462

Branch: refs/heads/branch-2
Commit: c093e4622c7e5c6552528a6e7d052406a3086ba7
Parents: 87af3f4
Author: Wangda Tan 
Authored: Mon Sep 11 11:34:51 2017 -0700
Committer: Wangda Tan 
Committed: Mon Sep 11 11:34:51 2017 -0700

--
 .../src/CMakeLists.txt  |   1 +
 .../impl/container-executor.c   |  78 +++-
 .../impl/container-executor.h   |  17 ++-
 .../impl/utils/string-utils.c   |  86 ++
 .../impl/utils/string-utils.h   |  32 +
 .../test/test-container-executor.c  | 119 ++-
 6 files changed, 327 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c093e462/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index fbc794c..6b345d5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -73,6 +73,7 @@ add_library(container
 main/native/container-executor/impl/configuration.c
 main/native/container-executor/impl/container-executor.c
 main/native/container-executor/impl/get_executable.c
+main/native/container-executor/impl/utils/string-utils.c
 )
 
 add_executable(container-executor

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c093e462/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 46217fb..52ce240 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,6 +18,7 @@
 
 #include "configuration.h"
 #include "container-executor.h"
+#include "utils/string-utils.h"
 
 #include 
 #include 
@@ -40,6 +41,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "config.h"
 
@@ -79,6 +81,11 @@ static const char* TC_READ_STATS_OPTS [] = { "-s",  "-b", 
NULL};
 //struct to store the user details
 struct passwd *user_detail = NULL;
 
+//Docker container related constants.
+static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
+static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
+static const char* DOCKER_PULL_COMMAND = "pull";
+
 FILE* LOGFILE = NULL;
 FILE* ERRORFILE = NULL;
 
@@ -1204,6 +1211,27 @@ char** tokenize_docker_command(const char *input, int 
*split_counter) {
   return linesplit;
 }
 
+int execute_regex_match(const char *regex_str, const char *input) {
+  regex_t regex;
+  int regex_match;
+  if (0 != regcomp(, regex_str, REG_EXTENDED|REG_NOSUB)) {
+fprintf(LOGFILE, "Unable to compile regex.");
+fflush(LOGFILE);
+exit(ERROR_COMPILING_REGEX);
+  }
+  regex_match = regexec(, input, (size_t) 0, NULL, 0);
+  regfree();
+  if(0 == regex_match) {
+return 0;
+  }
+  return 1;
+}
+
+int validate_docker_image_name(const char *image_name) {
+  char *regex_str = 
"^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+  return execute_regex_match(regex_str, image_name);
+}
+
 char* sanitize_docker_command(const char *line) {
   static struct option long_options[] = {
 {"name", required_argument, 0, 'n' },
@@ -1218,6 +1246,7 @@ char* sanitize_docker_command(const char *line) {
 {"cap-drop", required_argument, 0, 'o' },
 {"device", required_argument, 0, 'i' },
 {"detach",

[22/50] [abbrv] hadoop git commit: YARN-6726. Fix issues with docker commands executed by container-executor. (Shane Kumpf via wangda)

2017-08-11 Thread wangda 
YARN-6726. Fix issues with docker commands executed by container-executor. 
(Shane Kumpf via wangda)

Change-Id: If1b1827345f98f0a49cc7e39d1ba41fbeed5e911


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1794de3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1794de3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1794de3e

Branch: refs/heads/YARN-5881
Commit: 1794de3ea4bbd6863fb43dbae9f5a46b6e4230a0
Parents: 735fce5
Author: Wangda Tan 
Authored: Tue Aug 8 12:56:29 2017 -0700
Committer: Wangda Tan 
Committed: Tue Aug 8 12:56:29 2017 -0700

--
 .../src/CMakeLists.txt  |   1 +
 .../impl/container-executor.c   |  78 +++-
 .../impl/container-executor.h   |  17 ++-
 .../impl/utils/string-utils.c   |  86 ++
 .../impl/utils/string-utils.h   |  32 +
 .../test/test-container-executor.c  | 119 ++-
 6 files changed, 327 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1794de3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index f7fe83d..5b52536 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -89,6 +89,7 @@ add_library(container
 main/native/container-executor/impl/configuration.c
 main/native/container-executor/impl/container-executor.c
 main/native/container-executor/impl/get_executable.c
+main/native/container-executor/impl/utils/string-utils.c
 )
 
 add_executable(container-executor

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1794de3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 99f7b56..def628e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,6 +18,7 @@
 
 #include "configuration.h"
 #include "container-executor.h"
+#include "utils/string-utils.h"
 
 #include 
 #include 
@@ -40,6 +41,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "config.h"
 
@@ -79,6 +81,11 @@ static const char* TC_READ_STATS_OPTS [] = { "-s",  "-b", 
NULL};
 //struct to store the user details
 struct passwd *user_detail = NULL;
 
+//Docker container related constants.
+static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
+static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
+static const char* DOCKER_PULL_COMMAND = "pull";
+
 FILE* LOGFILE = NULL;
 FILE* ERRORFILE = NULL;
 
@@ -1208,6 +1215,27 @@ char** tokenize_docker_command(const char *input, int 
*split_counter) {
   return linesplit;
 }
 
+int execute_regex_match(const char *regex_str, const char *input) {
+  regex_t regex;
+  int regex_match;
+  if (0 != regcomp(, regex_str, REG_EXTENDED|REG_NOSUB)) {
+fprintf(LOGFILE, "Unable to compile regex.");
+fflush(LOGFILE);
+exit(ERROR_COMPILING_REGEX);
+  }
+  regex_match = regexec(, input, (size_t) 0, NULL, 0);
+  regfree();
+  if(0 == regex_match) {
+return 0;
+  }
+  return 1;
+}
+
+int validate_docker_image_name(const char *image_name) {
+  char *regex_str = 
"^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+  return execute_regex_match(regex_str, image_name);
+}
+
 char* sanitize_docker_command(const char *line) {
   static struct option long_options[] = {
 {"name", required_argument, 0, 'n' },
@@ -1222,6 +1250,7 @@ char* sanitize_docker_command(const char *line) {
 {"cap-drop", required_argument, 0, 'o' },
 {"device", required_argument, 0, 'i' },
 {"detach", required_argument, 0, 't' },
+{"format", required_argument, 0, 'f' },
 {0, 0, 0, 0}
   };

[37/51] [abbrv] hadoop git commit: YARN-6726. Fix issues with docker commands executed by container-executor. (Shane Kumpf via wangda)

2017-08-09 Thread stevel 
YARN-6726. Fix issues with docker commands executed by container-executor. 
(Shane Kumpf via wangda)

Change-Id: If1b1827345f98f0a49cc7e39d1ba41fbeed5e911


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1794de3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1794de3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1794de3e

Branch: refs/heads/HADOOP-13345
Commit: 1794de3ea4bbd6863fb43dbae9f5a46b6e4230a0
Parents: 735fce5
Author: Wangda Tan 
Authored: Tue Aug 8 12:56:29 2017 -0700
Committer: Wangda Tan 
Committed: Tue Aug 8 12:56:29 2017 -0700

--
 .../src/CMakeLists.txt  |   1 +
 .../impl/container-executor.c   |  78 +++-
 .../impl/container-executor.h   |  17 ++-
 .../impl/utils/string-utils.c   |  86 ++
 .../impl/utils/string-utils.h   |  32 +
 .../test/test-container-executor.c  | 119 ++-
 6 files changed, 327 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1794de3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index f7fe83d..5b52536 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -89,6 +89,7 @@ add_library(container
 main/native/container-executor/impl/configuration.c
 main/native/container-executor/impl/container-executor.c
 main/native/container-executor/impl/get_executable.c
+main/native/container-executor/impl/utils/string-utils.c
 )
 
 add_executable(container-executor

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1794de3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 99f7b56..def628e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -18,6 +18,7 @@
 
 #include "configuration.h"
 #include "container-executor.h"
+#include "utils/string-utils.h"
 
 #include 
 #include 
@@ -40,6 +41,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include "config.h"
 
@@ -79,6 +81,11 @@ static const char* TC_READ_STATS_OPTS [] = { "-s",  "-b", 
NULL};
 //struct to store the user details
 struct passwd *user_detail = NULL;
 
+//Docker container related constants.
+static const char* DOCKER_CONTAINER_NAME_PREFIX = "container_";
+static const char* DOCKER_CLIENT_CONFIG_ARG = "--config=";
+static const char* DOCKER_PULL_COMMAND = "pull";
+
 FILE* LOGFILE = NULL;
 FILE* ERRORFILE = NULL;
 
@@ -1208,6 +1215,27 @@ char** tokenize_docker_command(const char *input, int 
*split_counter) {
   return linesplit;
 }
 
+int execute_regex_match(const char *regex_str, const char *input) {
+  regex_t regex;
+  int regex_match;
+  if (0 != regcomp(, regex_str, REG_EXTENDED|REG_NOSUB)) {
+fprintf(LOGFILE, "Unable to compile regex.");
+fflush(LOGFILE);
+exit(ERROR_COMPILING_REGEX);
+  }
+  regex_match = regexec(, input, (size_t) 0, NULL, 0);
+  regfree();
+  if(0 == regex_match) {
+return 0;
+  }
+  return 1;
+}
+
+int validate_docker_image_name(const char *image_name) {
+  char *regex_str = 
"^(([a-zA-Z0-9.-]+)(:[0-9]+)?/)?([a-z0-9_./-]+)(:[a-zA-Z0-9_.-]+)?$";
+  return execute_regex_match(regex_str, image_name);
+}
+
 char* sanitize_docker_command(const char *line) {
   static struct option long_options[] = {
 {"name", required_argument, 0, 'n' },
@@ -1222,6 +1250,7 @@ char* sanitize_docker_command(const char *line) {
 {"cap-drop", required_argument, 0, 'o' },
 {"device", required_argument, 0, 'i' },
 {"detach", required_argument, 0, 't' },
+{"format", required_argument, 0, 'f' },
 {0, 0, 0, 0}