Repository: hadoop
Updated Branches:
refs/heads/trunk a34d83d2f -> a82be7754
YARN-7999. Added file cache initializer for Linux container-executor.
Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a82be775
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a82be775
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a82be775
Branch: refs/heads/trunk
Commit: a82be7754d74f4d16b206427b91e700bb5f44d56
Parents: a34d83d
Author: Eric Yang <[email protected]>
Authored: Wed Mar 14 12:21:57 2018 -0400
Committer: Eric Yang <[email protected]>
Committed: Wed Mar 14 12:21:57 2018 -0400
----------------------------------------------------------------------
.../impl/container-executor.c | 42 ++++++++++++++++++++
.../impl/container-executor.h | 1 +
2 files changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a82be775/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 751949e..14c0da5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -535,6 +535,18 @@ char *get_user_directory(const char *nm_root, const char
*user) {
}
/**
+ * Get the user private filecache directory of a particular user
+ */
+char *get_user_filecache_directory(const char *nm_root, const char *user) {
+ int result = check_nm_local_dir(nm_uid, nm_root);
+ if (result != 0) {
+ return NULL;
+ }
+ return concatenate(USER_FILECACHE_DIR_PATTERN, "user_filecache_dir_path", 2,
+ nm_root, user);
+}
+
+/**
* Check node manager local dir permission.
*/
int check_nm_local_dir(uid_t caller_uid, const char *nm_root) {
@@ -1422,6 +1434,29 @@ int create_local_dirs(const char * user, const char
*app_id,
return exit_code;
}
+// create the user file directory on all disks
+int create_user_filecache_dirs(const char * user, char* const* local_dirs) {
+ int rc = 0;
+ const mode_t permissions = S_IRWXU | S_IXGRP;
+ for(char* const* ldir_p = local_dirs; *ldir_p != 0; ++ldir_p) {
+ char* filecache_dir = get_user_filecache_directory(*ldir_p, user);
+ if (filecache_dir == NULL) {
+ fprintf(LOGFILE, "Couldn't get user filecache directory for %s.\n",
user);
+ rc = INITIALIZE_USER_FAILED;
+ break;
+ }
+ if (0 != mkdir(filecache_dir, permissions) && EEXIST != errno) {
+ fprintf(LOGFILE, "Failed to create directory %s - %s\n", filecache_dir,
+ strerror(errno));
+ free(filecache_dir);
+ rc = INITIALIZE_USER_FAILED;
+ break;
+ }
+ free(filecache_dir);
+ }
+ return rc;
+}
+
int launch_docker_container_as_user(const char * user, const char *app_id,
const char *container_id, const char *work_dir,
const char *script_name, const char *cred_file,
@@ -1476,6 +1511,13 @@ int launch_docker_container_as_user(const char * user,
const char *app_id,
goto cleanup;
}
+ exit_code = create_user_filecache_dirs(user, local_dirs);
+ if (exit_code != 0) {
+ fprintf(ERRORFILE, "Could not create user filecache directory");
+ fflush(ERRORFILE);
+ goto cleanup;
+ }
+
docker_command = construct_docker_command(command_file);
docker_binary = get_docker_binary(&CFG);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a82be775/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index 5835214..7c3ed77 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -52,6 +52,7 @@ enum operations {
#define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group"
#define USER_DIR_PATTERN "%s/usercache/%s"
+#define USER_FILECACHE_DIR_PATTERN "%s/usercache/%s/filecache"
#define NM_APP_DIR_PATTERN USER_DIR_PATTERN "/appcache/%s"
#define CONTAINER_DIR_PATTERN NM_APP_DIR_PATTERN "/%s"
#define CONTAINER_SCRIPT "launch_container.sh"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
