http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-common-project/hadoop-common/src/main/winutils/task.c ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/task.c b/hadoop-common-project/hadoop-common/src/main/winutils/task.c index 783f162..f0fc19a 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/task.c +++ b/hadoop-common-project/hadoop-common/src/main/winutils/task.c @@ -18,29 +18,32 @@ #include "winutils.h" #include <errno.h> #include <psapi.h> -#include <malloc.h> +#include <malloc.h> +#include <authz.h> +#include <sddl.h> #define PSAPI_VERSION 1 #pragma comment(lib, "psapi.lib") -#define ERROR_TASK_NOT_ALIVE 1 +#define NM_WSCE_IMPERSONATE_ALLOWED L"yarn.nodemanager.windows-secure-container-executor.impersonate.allowed" +#define NM_WSCE_IMPERSONATE_DENIED L"yarn.nodemanager.windows-secure-container-executor.impersonate.denied" -// This exit code for killed processes is compatible with Unix, where a killed -// process exits with 128 + signal. For SIGKILL, this would be 128 + 9 = 137. -#define KILLED_PROCESS_EXIT_CODE 137 +// The S4U impersonation access check mask. Arbitrary value (we use 1 for the service access check) +#define SERVICE_IMPERSONATE_MASK 0x00000002 + + +// Name for tracking this logon process when registering with LSA +static const char *LOGON_PROCESS_NAME="Hadoop Container Executor"; +// Name for token source, must be less or eq to TOKEN_SOURCE_LENGTH (currently 8) chars +static const char *TOKEN_SOURCE_NAME = "HadoopEx"; -// Name for tracking this logon process when registering with LSA -static const char *LOGON_PROCESS_NAME="Hadoop Container Executor"; -// Name for token source, must be less or eq to TOKEN_SOURCE_LENGTH (currently 8) chars -static const char *TOKEN_SOURCE_NAME = "HadoopEx"; - // List of different task related command line options supported by // winutils. typedef enum TaskCommandOptionType { TaskInvalid, TaskCreate, - TaskCreateAsUser, + TaskCreateAsUser, TaskIsAlive, TaskKill, TaskProcessList @@ -93,53 +96,524 @@ static BOOL ParseCommandLine(__in int argc, } } - if (argc >= 6) { - if (wcscmp(argv[1], L"createAsUser") == 0) - { - *command = TaskCreateAsUser; - return TRUE; - } - } - + if (argc >= 6) { + if (wcscmp(argv[1], L"createAsUser") == 0) + { + *command = TaskCreateAsUser; + return TRUE; + } + } + return FALSE; } + //---------------------------------------------------------------------------- -// Function: CreateTaskImpl +// Function: BuildImpersonateSecurityDescriptor +// +// Description: +// Builds the security descriptor for the S4U impersonation permissions +// This describes what users can be impersonated and what not +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +// +DWORD BuildImpersonateSecurityDescriptor(__out PSECURITY_DESCRIPTOR* ppSD) { + DWORD dwError = ERROR_SUCCESS; + size_t countAllowed = 0; + PSID* allowedSids = NULL; + size_t countDenied = 0; + PSID* deniedSids = NULL; + LPCWSTR value = NULL; + WCHAR** tokens = NULL; + size_t len = 0; + size_t count = 0; + int crt = 0; + PSECURITY_DESCRIPTOR pSD = NULL; + + dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_ALLOWED, &len, &value); + if (dwError) { + ReportErrorCode(L"GetConfigValue:1", dwError); + goto done; + } + + if (0 == len) { + dwError = ERROR_BAD_CONFIGURATION; + ReportErrorCode(L"GetConfigValue:2", dwError); + goto done; + } + + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + if (dwError) { + ReportErrorCode(L"SplitStringIgnoreSpaceW:1", dwError); + goto done; + } + + allowedSids = LocalAlloc(LPTR, sizeof(PSID) * count); + if (NULL == allowedSids) { + dwError = GetLastError(); + ReportErrorCode(L"LocalAlloc:1", dwError); + goto done; + } + + for(crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &allowedSids[crt]); + if (dwError) { + ReportErrorCode(L"GetSidFromAcctNameW:1", dwError); + goto done; + } + } + countAllowed = count; + + LocalFree(tokens); + tokens = NULL; + + LocalFree(value); + value = NULL; + + dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_IMPERSONATE_DENIED, &len, &value); + if (dwError) { + ReportErrorCode(L"GetConfigValue:3", dwError); + goto done; + } + + if (0 != len) { + dwError = SplitStringIgnoreSpaceW(len, value, L',', &count, &tokens); + if (dwError) { + ReportErrorCode(L"SplitStringIgnoreSpaceW:2", dwError); + goto done; + } + + deniedSids = LocalAlloc(LPTR, sizeof(PSID) * count); + if (NULL == allowedSids) { + dwError = GetLastError(); + ReportErrorCode(L"LocalAlloc:2", dwError); + goto done; + } + + for(crt = 0; crt < count; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &deniedSids[crt]); + if (dwError) { + ReportErrorCode(L"GetSidFromAcctNameW:2", dwError); + goto done; + } + } + countDenied = count; + } + + dwError = BuildServiceSecurityDescriptor( + SERVICE_IMPERSONATE_MASK, + countAllowed, allowedSids, + countDenied, deniedSids, + NULL, + &pSD); + + if (dwError) { + ReportErrorCode(L"BuildServiceSecurityDescriptor", dwError); + goto done; + } + + *ppSD = pSD; + pSD = NULL; + +done: + if (pSD) LocalFree(pSD); + if (tokens) LocalFree(tokens); + if (allowedSids) LocalFree(allowedSids); + if (deniedSids) LocalFree(deniedSids); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: AddNodeManagerAndUserACEsToObject +// +// Description: +// Adds ACEs to grant NM and user the provided access mask over a given handle +// +// Returns: +// ERROR_SUCCESS: on success +// +DWORD AddNodeManagerAndUserACEsToObject( + __in HANDLE hObject, + __in LPWSTR user, + __in ACCESS_MASK accessMask) { + + DWORD dwError = ERROR_SUCCESS; + int countTokens = 0; + size_t len = 0; + LPCWSTR value = NULL; + WCHAR** tokens = NULL; + int crt = 0; + PACL pDacl = NULL; + PSECURITY_DESCRIPTOR psdProcess = NULL; + LPSTR lpszOldDacl = NULL, lpszNewDacl = NULL; + ULONG daclLen = 0; + PACL pNewDacl = NULL; + ACL_SIZE_INFORMATION si; + DWORD dwNewAclSize = 0; + PACE_HEADER pTempAce = NULL; + BYTE sidTemp[SECURITY_MAX_SID_SIZE]; + DWORD cbSid = SECURITY_MAX_SID_SIZE; + PSID tokenSid = NULL; + // These hard-coded SIDs are allways added + WELL_KNOWN_SID_TYPE forcesSidTypes[] = { + WinLocalSystemSid, + WinBuiltinAdministratorsSid}; + BOOL logSDs = IsDebuggerPresent(); // Check only once to avoid attach-while-running + + + dwError = GetSecurityInfo(hObject, + SE_KERNEL_OBJECT, + DACL_SECURITY_INFORMATION, + NULL, + NULL, + &pDacl, + NULL, + &psdProcess); + if (dwError) { + ReportErrorCode(L"GetSecurityInfo", dwError); + goto done; + } + + // This is debug only output for troubleshooting + if (logSDs) { + if (!ConvertSecurityDescriptorToStringSecurityDescriptor( + psdProcess, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &lpszOldDacl, + &daclLen)) { + dwError = GetLastError(); + ReportErrorCode(L"ConvertSecurityDescriptorToStringSecurityDescriptor", dwError); + goto done; + } + } + + ZeroMemory(&si, sizeof(si)); + if (!GetAclInformation(pDacl, &si, sizeof(si), AclSizeInformation)) { + dwError = GetLastError(); + ReportErrorCode(L"GetAclInformation", dwError); + goto done; + } + + dwError = GetConfigValue(wsceConfigRelativePath, NM_WSCE_ALLOWED, &len, &value); + if (ERROR_SUCCESS != dwError) { + ReportErrorCode(L"GetConfigValue", dwError); + goto done; + } + + if (0 == len) { + dwError = ERROR_BAD_CONFIGURATION; + ReportErrorCode(L"GetConfigValue", dwError); + goto done; + } + + dwError = SplitStringIgnoreSpaceW(len, value, L',', &countTokens, &tokens); + if (ERROR_SUCCESS != dwError) { + ReportErrorCode(L"SplitStringIgnoreSpaceW", dwError); + goto done; + } + + // We're gonna add 1 ACE for each token found, +1 for user and +1 for each forcesSidTypes[] + // ACCESS_ALLOWED_ACE struct contains the first DWORD of the SID + // + dwNewAclSize = si.AclBytesInUse + + (countTokens + 1 + sizeof(forcesSidTypes)/sizeof(forcesSidTypes[0])) * + (sizeof(ACCESS_ALLOWED_ACE) + SECURITY_MAX_SID_SIZE - sizeof(DWORD)); + + pNewDacl = (PSID) LocalAlloc(LPTR, dwNewAclSize); + if (!pNewDacl) { + dwError = ERROR_OUTOFMEMORY; + ReportErrorCode(L"LocalAlloc", dwError); + goto done; + } + + if (!InitializeAcl(pNewDacl, dwNewAclSize, ACL_REVISION)) { + dwError = ERROR_OUTOFMEMORY; + ReportErrorCode(L"InitializeAcl", dwError); + goto done; + } + + // Copy over old ACEs + for (crt = 0; crt < si.AceCount; ++crt) { + if (!GetAce(pDacl, crt, &pTempAce)) { + dwError = ERROR_OUTOFMEMORY; + ReportErrorCode(L"InitializeAcl", dwError); + goto done; + } + if (!AddAce(pNewDacl, ACL_REVISION, MAXDWORD, pTempAce, pTempAce->AceSize)) { + dwError = ERROR_OUTOFMEMORY; + ReportErrorCode(L"InitializeAcl", dwError); + goto done; + } + } + + // Add the configured allowed SIDs + for (crt = 0; crt < countTokens; ++crt) { + dwError = GetSidFromAcctNameW(tokens[crt], &tokenSid); + if (ERROR_SUCCESS != dwError) { + ReportErrorCode(L"GetSidFromAcctNameW", dwError); + goto done; + } + if (!AddAccessAllowedAceEx( + pNewDacl, + ACL_REVISION_DS, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + PROCESS_ALL_ACCESS, + tokenSid)) { + dwError = GetLastError(); + ReportErrorCode(L"AddAccessAllowedAceEx:1", dwError); + goto done; + } + LocalFree(tokenSid); + tokenSid = NULL; + } + + // add the forced SIDs ACE + for (crt = 0; crt < sizeof(forcesSidTypes)/sizeof(forcesSidTypes[0]); ++crt) { + cbSid = SECURITY_MAX_SID_SIZE; + if (!CreateWellKnownSid(forcesSidTypes[crt], NULL, &sidTemp, &cbSid)) { + dwError = GetLastError(); + ReportErrorCode(L"CreateWellKnownSid", dwError); + goto done; + } + if (!AddAccessAllowedAceEx( + pNewDacl, + ACL_REVISION_DS, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + accessMask, + (PSID) sidTemp)) { + dwError = GetLastError(); + ReportErrorCode(L"AddAccessAllowedAceEx:2", dwError); + goto done; + } + } + + // add the user ACE + dwError = GetSidFromAcctNameW(user, &tokenSid); + if (ERROR_SUCCESS != dwError) { + ReportErrorCode(L"GetSidFromAcctNameW:user", dwError); + goto done; + } + + if (!AddAccessAllowedAceEx( + pNewDacl, + ACL_REVISION_DS, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + PROCESS_ALL_ACCESS, + tokenSid)) { + dwError = GetLastError(); + ReportErrorCode(L"AddAccessAllowedAceEx:3", dwError); + goto done; + } + + LocalFree(tokenSid); + tokenSid = NULL; + + dwError = SetSecurityInfo(hObject, + SE_KERNEL_OBJECT, + DACL_SECURITY_INFORMATION, + NULL, + NULL, + pNewDacl, + NULL); + if (dwError) { + ReportErrorCode(L"SetSecurityInfo", dwError); + goto done; + } + + // This is debug only output for troubleshooting + if (logSDs) { + dwError = GetSecurityInfo(hObject, + SE_KERNEL_OBJECT, + DACL_SECURITY_INFORMATION, + NULL, + NULL, + &pDacl, + NULL, + &psdProcess); + if (dwError) { + ReportErrorCode(L"GetSecurityInfo:2", dwError); + goto done; + } + + if (!ConvertSecurityDescriptorToStringSecurityDescriptor( + psdProcess, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &lpszNewDacl, + &daclLen)) { + dwError = GetLastError(); + ReportErrorCode(L"ConvertSecurityDescriptorToStringSecurityDescriptor:2", dwError); + goto done; + } + + LogDebugMessage(L"Old DACL: %s\nNew DACL: %s\n", lpszOldDacl, lpszNewDacl); + } + +done: + if (tokenSid) LocalFree(tokenSid); + if (pNewDacl) LocalFree(pNewDacl); + if (lpszOldDacl) LocalFree(lpszOldDacl); + if (lpszNewDacl) LocalFree(lpszNewDacl); + if (psdProcess) LocalFree(psdProcess); + + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: ValidateImpersonateAccessCheck +// +// Description: +// Performs the access check for S4U impersonation +// +// Returns: +// ERROR_SUCCESS: On success +// ERROR_ACCESS_DENIED, GetLastError: otherwise +// +DWORD ValidateImpersonateAccessCheck(__in HANDLE logonHandle) { + DWORD dwError = ERROR_SUCCESS; + PSECURITY_DESCRIPTOR pSD = NULL; + LUID luidUnused; + AUTHZ_ACCESS_REQUEST request; + AUTHZ_ACCESS_REPLY reply; + DWORD authError = ERROR_SUCCESS; + DWORD saclResult = 0; + ACCESS_MASK grantedMask = 0; + AUTHZ_RESOURCE_MANAGER_HANDLE hManager = NULL; + AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzToken = NULL; + + ZeroMemory(&luidUnused, sizeof(luidUnused)); + ZeroMemory(&request, sizeof(request)); + ZeroMemory(&reply, sizeof(reply)); + + dwError = BuildImpersonateSecurityDescriptor(&pSD); + if (dwError) { + ReportErrorCode(L"BuildImpersonateSecurityDescriptor", dwError); + goto done; + } + + request.DesiredAccess = MAXIMUM_ALLOWED; + reply.Error = &authError; + reply.SaclEvaluationResults = &saclResult; + reply.ResultListLength = 1; + reply.GrantedAccessMask = &grantedMask; + + if (!AuthzInitializeResourceManager( + AUTHZ_RM_FLAG_NO_AUDIT, + NULL, // pfnAccessCheck + NULL, // pfnComputeDynamicGroups + NULL, // pfnFreeDynamicGroups + NULL, // szResourceManagerName + &hManager)) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzInitializeResourceManager", dwError); + goto done; + } + + if (!AuthzInitializeContextFromToken( + 0, + logonHandle, + hManager, + NULL, // expiration time + luidUnused, // not used + NULL, // callback args + &hAuthzToken)) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzInitializeContextFromToken", dwError); + goto done; + } + + if (!AuthzAccessCheck( + 0, + hAuthzToken, + &request, + NULL, // AuditEvent + pSD, + NULL, // OptionalSecurityDescriptorArray + 0, // OptionalSecurityDescriptorCount + &reply, + NULL // phAccessCheckResults + )) { + dwError = GetLastError(); + ReportErrorCode(L"AuthzAccessCheck", dwError); + goto done; + } + + LogDebugMessage(L"AutzAccessCheck: Error:%d sacl:%d access:%d\n", + authError, saclResult, grantedMask); + + if (authError != ERROR_SUCCESS) { + ReportErrorCode(L"AuthzAccessCheck:REPLY:1", authError); + dwError = authError; + } + else if (!(grantedMask & SERVICE_IMPERSONATE_MASK)) { + ReportErrorCode(L"AuthzAccessCheck:REPLY:2", ERROR_ACCESS_DENIED); + dwError = ERROR_ACCESS_DENIED; + } + +done: + if (hAuthzToken) AuthzFreeContext(hAuthzToken); + if (hManager) AuthzFreeResourceManager(hManager); + if (pSD) LocalFree(pSD); + return dwError; +} + +//---------------------------------------------------------------------------- +// Function: CreateTaskImpl // // Description: // Creates a task via a jobobject. Outputs the // appropriate information to stdout on success, or stderr on failure. -// logonHandle may be NULL, in this case the current logon will be utilized for the -// created process +// logonHandle may be NULL, in this case the current logon will be utilized for the +// created process // // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PWSTR cmdLine) +DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PCWSTR cmdLine, + __in LPCWSTR userName) { - DWORD dwErrorCode = ERROR_SUCCESS; + DWORD dwErrorCode = ERROR_SUCCESS; DWORD exitCode = EXIT_FAILURE; - DWORD currDirCnt = 0; + DWORD currDirCnt = 0; STARTUPINFO si; PROCESS_INFORMATION pi; HANDLE jobObject = NULL; JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 }; - void * envBlock = NULL; - BOOL createProcessResult = FALSE; - - wchar_t* curr_dir = NULL; - FILE *stream = NULL; + void * envBlock = NULL; + WCHAR secureJobNameBuffer[MAX_PATH]; + LPCWSTR secureJobName = jobObjName; + + wchar_t* curr_dir = NULL; + FILE *stream = NULL; + + if (NULL != logonHandle) { + dwErrorCode = ValidateImpersonateAccessCheck(logonHandle); + if (dwErrorCode) { + ReportErrorCode(L"ValidateImpersonateAccessCheck", dwErrorCode); + return dwErrorCode; + } + + dwErrorCode = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer); + if (dwErrorCode) { + ReportErrorCode(L"GetSecureJobObjectName", dwErrorCode); + return dwErrorCode; + } + secureJobName = secureJobNameBuffer; + } // Create un-inheritable job object handle and set job object to terminate // when last handle is closed. So winutils.exe invocation has the only open // job object handle. Exit of winutils.exe ensures termination of job object. // Either a clean exit of winutils or crash or external termination. - jobObject = CreateJobObject(NULL, jobObjName); - dwErrorCode = GetLastError(); - if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS) + jobObject = CreateJobObject(NULL, secureJobName); + dwErrorCode = GetLastError(); + if(jobObject == NULL || dwErrorCode == ERROR_ALREADY_EXISTS) { - return dwErrorCode; + ReportErrorCode(L"CreateJobObject", dwErrorCode); + return dwErrorCode; } jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; if(SetInformationJobObject(jobObject, @@ -147,102 +621,143 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW &jeli, sizeof(jeli)) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); + ReportErrorCode(L"SetInformationJobObject", dwErrorCode); CloseHandle(jobObject); - return dwErrorCode; - } + return dwErrorCode; + } + + dwErrorCode = AddNodeManagerAndUserACEsToObject(jobObject, userName, JOB_OBJECT_ALL_ACCESS); + if (dwErrorCode) { + ReportErrorCode(L"AddNodeManagerAndUserACEsToObject", dwErrorCode); + CloseHandle(jobObject); + return dwErrorCode; + } if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); + ReportErrorCode(L"AssignProcessToJobObject", dwErrorCode); CloseHandle(jobObject); - return dwErrorCode; + return dwErrorCode; } // the child JVM uses this env var to send the task OS process identifier // to the TaskTracker. We pass the job object name. if(SetEnvironmentVariable(L"JVM_PID", jobObjName) == 0) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; + dwErrorCode = GetLastError(); + ReportErrorCode(L"SetEnvironmentVariable", dwErrorCode); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; } ZeroMemory( &si, sizeof(si) ); si.cb = sizeof(si); ZeroMemory( &pi, sizeof(pi) ); - if( logonHandle != NULL ) { - // create user environment for this logon - if(!CreateEnvironmentBlock(&envBlock, - logonHandle, - TRUE )) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; - } - } - - // Get the required buffer size first - currDirCnt = GetCurrentDirectory(0, NULL); - if (0 < currDirCnt) { - curr_dir = (wchar_t*) alloca(currDirCnt * sizeof(wchar_t)); - assert(curr_dir); - currDirCnt = GetCurrentDirectory(currDirCnt, curr_dir); - } - - if (0 == currDirCnt) { - dwErrorCode = GetLastError(); - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - return dwErrorCode; - } - - if (logonHandle == NULL) { - createProcessResult = CreateProcess( - NULL, // ApplicationName - cmdLine, // command line - NULL, // process security attributes - NULL, // thread security attributes - TRUE, // inherit handles - 0, // creation flags - NULL, // environment - curr_dir, // current directory - &si, // startup info - &pi); // process info - } - else { - createProcessResult = CreateProcessAsUser( - logonHandle, // logon token handle - NULL, // Application handle - cmdLine, // command line - NULL, // process security attributes - NULL, // thread security attributes - FALSE, // inherit handles - CREATE_UNICODE_ENVIRONMENT, // creation flags - envBlock, // environment - curr_dir, // current directory - &si, // startup info - &pi); // process info - } - - if (FALSE == createProcessResult) { - dwErrorCode = GetLastError(); - if( envBlock != NULL ) { - DestroyEnvironmentBlock( envBlock ); - envBlock = NULL; - } - // We have to explictly Terminate, passing in the error code - // simply closing the job would kill our own process with success exit status - TerminateJobObject(jobObject, dwErrorCode); - - // This is tehnically dead code, we cannot reach this condition - return dwErrorCode; + if( logonHandle != NULL ) { + // create user environment for this logon + if(!CreateEnvironmentBlock(&envBlock, + logonHandle, + TRUE )) { + dwErrorCode = GetLastError(); + ReportErrorCode(L"CreateEnvironmentBlock", dwErrorCode); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; + } + } + + // Get the required buffer size first + currDirCnt = GetCurrentDirectory(0, NULL); + if (0 < currDirCnt) { + curr_dir = (wchar_t*) alloca(currDirCnt * sizeof(wchar_t)); + assert(curr_dir); + currDirCnt = GetCurrentDirectory(currDirCnt, curr_dir); + } + + if (0 == currDirCnt) { + dwErrorCode = GetLastError(); + ReportErrorCode(L"GetCurrentDirectory", dwErrorCode); + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + return dwErrorCode; + } + + dwErrorCode = ERROR_SUCCESS; + + if (logonHandle == NULL) { + if (!CreateProcess( + NULL, // ApplicationName + cmdLine, // command line + NULL, // process security attributes + NULL, // thread security attributes + TRUE, // inherit handles + 0, // creation flags + NULL, // environment + curr_dir, // current directory + &si, // startup info + &pi)) { // process info + dwErrorCode = GetLastError(); + ReportErrorCode(L"CreateProcess", dwErrorCode); + } + goto create_process_done; + } + + // From here on is the secure S4U implementation for CreateProcessAsUser + + // We desire to grant process access to NM so that it can interogate process status + // and resource utilization. Passing in a security descriptor though results in the + // S4U privilege checks being done against that SD and CreateProcessAsUser fails. + // So instead we create the process suspended and then we add the desired ACEs. + // + if (!CreateProcessAsUser( + logonHandle, // logon token handle + NULL, // Application handle + cmdLine, // command line + NULL, // process security attributes + NULL, // thread security attributes + FALSE, // inherit handles + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, // creation flags + envBlock, // environment + curr_dir, // current directory + &si, // startup info + &pi)) { // process info + dwErrorCode = GetLastError(); + ReportErrorCode(L"CreateProcessAsUser", dwErrorCode); + goto create_process_done; + } + + dwErrorCode = AddNodeManagerAndUserACEsToObject(pi.hProcess, userName, PROCESS_ALL_ACCESS); + if (dwErrorCode) { + ReportErrorCode(L"AddNodeManagerAndUserACEsToObject", dwErrorCode); + goto create_process_done; + } + + if (-1 == ResumeThread(pi.hThread)) { + dwErrorCode = GetLastError(); + ReportErrorCode(L"ResumeThread", dwErrorCode); + goto create_process_done; + } + +create_process_done: + + if (dwErrorCode) { + if( envBlock != NULL ) { + DestroyEnvironmentBlock( envBlock ); + envBlock = NULL; + } + // We have to explictly Terminate, passing in the error code + // simply closing the job would kill our own process with success exit status + TerminateJobObject(jobObject, dwErrorCode); + + // This is tehnically dead code, we cannot reach this condition + return dwErrorCode; } CloseHandle(pi.hThread); @@ -251,15 +766,15 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW WaitForSingleObject( pi.hProcess, INFINITE ); if(GetExitCodeProcess(pi.hProcess, &exitCode) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); } CloseHandle( pi.hProcess ); - if( envBlock != NULL ) { - DestroyEnvironmentBlock( envBlock ); - envBlock = NULL; - } - + if( envBlock != NULL ) { + DestroyEnvironmentBlock( envBlock ); + envBlock = NULL; + } + // Terminate job object so that all spawned processes are also killed. // This is needed because once this process closes the handle to the job // object and none of the spawned objects have the handle open (via @@ -267,134 +782,133 @@ DWORD CreateTaskImpl(__in_opt HANDLE logonHandle, __in PCWSTR jobObjName,__in PW // program (say winutils task kill) to terminate this job object via its name. if(TerminateJobObject(jobObject, exitCode) == 0) { - dwErrorCode = GetLastError(); + dwErrorCode = GetLastError(); } - // comes here only on failure of TerminateJobObject + // comes here only on failure of TerminateJobObject CloseHandle(jobObject); - if(dwErrorCode != ERROR_SUCCESS) + if(dwErrorCode != ERROR_SUCCESS) { - return dwErrorCode; + return dwErrorCode; } return exitCode; } //---------------------------------------------------------------------------- -// Function: CreateTask -// -// Description: -// Creates a task via a jobobject. Outputs the -// appropriate information to stdout on success, or stderr on failure. -// -// Returns: -// ERROR_SUCCESS: On success -// GetLastError: otherwise -DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine) -{ - // call with null logon in order to create tasks utilizing the current logon - return CreateTaskImpl( NULL, jobObjName, cmdLine ); -} -//---------------------------------------------------------------------------- -// Function: CreateTask -// -// Description: -// Creates a task via a jobobject. Outputs the -// appropriate information to stdout on success, or stderr on failure. -// -// Returns: -// ERROR_SUCCESS: On success -// GetLastError: otherwise -DWORD CreateTaskAsUser(__in PCWSTR jobObjName,__in PWSTR user, __in PWSTR pidFilePath, __in PWSTR cmdLine) -{ - DWORD err = ERROR_SUCCESS; - DWORD exitCode = EXIT_FAILURE; - ULONG authnPkgId; - HANDLE lsaHandle = INVALID_HANDLE_VALUE; - PROFILEINFO pi; - BOOL profileIsLoaded = FALSE; - FILE* pidFile = NULL; - - DWORD retLen = 0; - HANDLE logonHandle = NULL; - - err = EnablePrivilege(SE_TCB_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_TCB_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_ASSIGNPRIMARYTOKEN_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_ASSIGNPRIMARYTOKEN_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_INCREASE_QUOTA_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_INCREASE_QUOTA_NAME.\n"); - goto done; - } - err = EnablePrivilege(SE_RESTORE_NAME); - if( err != ERROR_SUCCESS ) { - fwprintf(stdout, L"INFO: The user does not have SE_RESTORE_NAME.\n"); - goto done; - } - - err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle); - if( err != ERROR_SUCCESS ) goto done; - - err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId ); - if( err != ERROR_SUCCESS ) goto done; - - err = CreateLogonForUser(lsaHandle, - LOGON_PROCESS_NAME, - TOKEN_SOURCE_NAME, - authnPkgId, - user, - &logonHandle); - if( err != ERROR_SUCCESS ) goto done; - - err = LoadUserProfileForLogon(logonHandle, &pi); - if( err != ERROR_SUCCESS ) goto done; - profileIsLoaded = TRUE; - - // Create the PID file - - if (!(pidFile = _wfopen(pidFilePath, "w"))) { - err = GetLastError(); - goto done; - } - - if (0 > fprintf_s(pidFile, "%ls", jobObjName)) { - err = GetLastError(); - } - - fclose(pidFile); - - if (err != ERROR_SUCCESS) { - goto done; - } - - err = CreateTaskImpl(logonHandle, jobObjName, cmdLine); - -done: - if( profileIsLoaded ) { - UnloadProfileForLogon( logonHandle, &pi ); - profileIsLoaded = FALSE; - } - if( logonHandle != NULL ) { - CloseHandle(logonHandle); - } - - if (INVALID_HANDLE_VALUE != lsaHandle) { - UnregisterWithLsa(lsaHandle); - } - - return err; -} - - -//---------------------------------------------------------------------------- -// Function: IsTaskAlive +// Function: CreateTask +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD CreateTask(__in PCWSTR jobObjName,__in PWSTR cmdLine) +{ + // call with null logon in order to create tasks utilizing the current logon + return CreateTaskImpl( NULL, jobObjName, cmdLine, NULL); +} + +//---------------------------------------------------------------------------- +// Function: CreateTaskAsUser +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD CreateTaskAsUser(__in PCWSTR jobObjName, + __in PCWSTR user, __in PCWSTR pidFilePath, __in PCWSTR cmdLine) +{ + DWORD err = ERROR_SUCCESS; + DWORD exitCode = EXIT_FAILURE; + ULONG authnPkgId; + HANDLE lsaHandle = INVALID_HANDLE_VALUE; + PROFILEINFO pi; + BOOL profileIsLoaded = FALSE; + FILE* pidFile = NULL; + DWORD retLen = 0; + HANDLE logonHandle = NULL; + + err = EnableImpersonatePrivileges(); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"EnableImpersonatePrivileges", err); + goto done; + } + + err = RegisterWithLsa(LOGON_PROCESS_NAME ,&lsaHandle); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"RegisterWithLsa", err); + goto done; + } + + err = LookupKerberosAuthenticationPackageId( lsaHandle, &authnPkgId ); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"LookupKerberosAuthenticationPackageId", err); + goto done; + } + + err = CreateLogonTokenForUser(lsaHandle, + LOGON_PROCESS_NAME, + TOKEN_SOURCE_NAME, + authnPkgId, + user, + &logonHandle); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"CreateLogonTokenForUser", err); + goto done; + } + + err = LoadUserProfileForLogon(logonHandle, &pi); + if( err != ERROR_SUCCESS ) { + ReportErrorCode(L"LoadUserProfileForLogon", err); + goto done; + } + profileIsLoaded = TRUE; + + // Create the PID file + + if (!(pidFile = _wfopen(pidFilePath, "w"))) { + err = GetLastError(); + ReportErrorCode(L"_wfopen:pidFilePath", err); + goto done; + } + + if (0 > fprintf_s(pidFile, "%ls", jobObjName)) { + err = GetLastError(); + } + + fclose(pidFile); + + if (err != ERROR_SUCCESS) { + ReportErrorCode(L"fprintf_s:pidFilePath", err); + goto done; + } + + err = CreateTaskImpl(logonHandle, jobObjName, cmdLine, user); + +done: + if( profileIsLoaded ) { + UnloadProfileForLogon( logonHandle, &pi ); + profileIsLoaded = FALSE; + } + if( logonHandle != NULL ) { + CloseHandle(logonHandle); + } + + if (INVALID_HANDLE_VALUE != lsaHandle) { + UnregisterWithLsa(lsaHandle); + } + + return err; +} + +//---------------------------------------------------------------------------- +// Function: IsTaskAlive // // Description: // Checks if a task is alive via a jobobject. Outputs the @@ -403,15 +917,32 @@ done: // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) +DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) { PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; HANDLE jobObject = NULL; int numProcs = 100; + WCHAR secureJobNameBuffer[MAX_PATH]; *isAlive = FALSE; jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName); + if(jobObject == NULL) + { + // Try Global\... + DWORD err = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer); + if (err) { + ReportErrorCode(L"GetSecureJobObjectName", err); + return err; + } + jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, secureJobNameBuffer); + } + + if(jobObject == NULL) + { + DWORD err = GetLastError(); + return err; + } if(jobObject == NULL) { @@ -454,40 +985,7 @@ DWORD IsTaskAlive(const WCHAR* jobObjName, int* isAlive, int* procsInJob) } //---------------------------------------------------------------------------- -// Function: KillTask -// -// Description: -// Kills a task via a jobobject. Outputs the -// appropriate information to stdout on success, or stderr on failure. -// -// Returns: -// ERROR_SUCCESS: On success -// GetLastError: otherwise -DWORD KillTask(PCWSTR jobObjName) -{ - HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName); - if(jobObject == NULL) - { - DWORD err = GetLastError(); - if(err == ERROR_FILE_NOT_FOUND) - { - // job object does not exist. assume its not alive - return ERROR_SUCCESS; - } - return err; - } - - if(TerminateJobObject(jobObject, KILLED_PROCESS_EXIT_CODE) == 0) - { - return GetLastError(); - } - CloseHandle(jobObject); - - return ERROR_SUCCESS; -} - -//---------------------------------------------------------------------------- -// Function: PrintTaskProcessList +// Function: PrintTaskProcessList // // Description: // Prints resource usage of all processes in the task jobobject @@ -495,12 +993,26 @@ DWORD KillTask(PCWSTR jobObjName) // Returns: // ERROR_SUCCESS: On success // GetLastError: otherwise -DWORD PrintTaskProcessList(const WCHAR* jobObjName) +DWORD PrintTaskProcessList(const WCHAR* jobObjName) { DWORD i; PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; int numProcs = 100; - HANDLE jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName); + WCHAR secureJobNameBuffer[MAX_PATH]; + HANDLE jobObject = NULL; + + jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName); + if(jobObject == NULL) + { + // Try Global\... + DWORD err = GetSecureJobObjectName(jobObjName, MAX_PATH, secureJobNameBuffer); + if (err) { + ReportErrorCode(L"GetSecureJobObjectName", err); + return err; + } + jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, secureJobNameBuffer); + } + if(jobObject == NULL) { DWORD err = GetLastError(); @@ -579,21 +1091,21 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) { DWORD dwErrorCode = ERROR_SUCCESS; TaskCommandOption command = TaskInvalid; - wchar_t* cmdLine = NULL; - wchar_t buffer[16*1024] = L""; // 32K max command line - size_t charCountBufferLeft = sizeof (buffer)/sizeof(wchar_t); - int crtArgIndex = 0; - size_t argLen = 0; - size_t wscatErr = 0; - wchar_t* insertHere = NULL; - - enum { - ARGC_JOBOBJECTNAME = 2, - ARGC_USERNAME, - ARGC_PIDFILE, - ARGC_COMMAND, - ARGC_COMMAND_ARGS - }; + wchar_t* cmdLine = NULL; + wchar_t buffer[16*1024] = L""; // 32K max command line + size_t charCountBufferLeft = sizeof(buffer)/sizeof(wchar_t); + int crtArgIndex = 0; + size_t argLen = 0; + size_t wscatErr = 0; + wchar_t* insertHere = NULL; + + enum { + ARGC_JOBOBJECTNAME = 2, + ARGC_USERNAME, + ARGC_PIDFILE, + ARGC_COMMAND, + ARGC_COMMAND_ARGS + }; if (!ParseCommandLine(argc, argv, &command)) { dwErrorCode = ERROR_INVALID_COMMAND_LINE; @@ -607,57 +1119,57 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) { // Create the task jobobject // - dwErrorCode = CreateTask(argv[2], argv[3]); - if (dwErrorCode != ERROR_SUCCESS) - { - ReportErrorCode(L"CreateTask", dwErrorCode); - goto TaskExit; - } - } else if (command == TaskCreateAsUser) - { - // Create the task jobobject as a domain user - // createAsUser accepts an open list of arguments. All arguments after the command are - // to be passed as argumrnts to the command itself.Here we're concatenating all - // arguments after the command into a single arg entry. - // - cmdLine = argv[ARGC_COMMAND]; - if (argc > ARGC_COMMAND_ARGS) { - crtArgIndex = ARGC_COMMAND; - insertHere = buffer; - while (crtArgIndex < argc) { - argLen = wcslen(argv[crtArgIndex]); - wscatErr = wcscat_s(insertHere, charCountBufferLeft, argv[crtArgIndex]); - switch (wscatErr) { - case 0: - // 0 means success; - break; - case EINVAL: - dwErrorCode = ERROR_INVALID_PARAMETER; - goto TaskExit; - case ERANGE: - dwErrorCode = ERROR_INSUFFICIENT_BUFFER; - goto TaskExit; - default: - // This case is not MSDN documented. - dwErrorCode = ERROR_GEN_FAILURE; - goto TaskExit; - } - insertHere += argLen; - charCountBufferLeft -= argLen; - insertHere[0] = L' '; - insertHere += 1; - charCountBufferLeft -= 1; - insertHere[0] = 0; - ++crtArgIndex; - } - cmdLine = buffer; - } - - dwErrorCode = CreateTaskAsUser( - argv[ARGC_JOBOBJECTNAME], argv[ARGC_USERNAME], argv[ARGC_PIDFILE], cmdLine); + dwErrorCode = CreateTask(argv[2], argv[3]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"CreateTask", dwErrorCode); + goto TaskExit; + } + } else if (command == TaskCreateAsUser) + { + // Create the task jobobject as a domain user + // createAsUser accepts an open list of arguments. All arguments after the command are + // to be passed as argumrnts to the command itself.Here we're concatenating all + // arguments after the command into a single arg entry. + // + cmdLine = argv[ARGC_COMMAND]; + if (argc > ARGC_COMMAND_ARGS) { + crtArgIndex = ARGC_COMMAND; + insertHere = buffer; + while (crtArgIndex < argc) { + argLen = wcslen(argv[crtArgIndex]); + wscatErr = wcscat_s(insertHere, charCountBufferLeft, argv[crtArgIndex]); + switch (wscatErr) { + case 0: + // 0 means success; + break; + case EINVAL: + dwErrorCode = ERROR_INVALID_PARAMETER; + goto TaskExit; + case ERANGE: + dwErrorCode = ERROR_INSUFFICIENT_BUFFER; + goto TaskExit; + default: + // This case is not MSDN documented. + dwErrorCode = ERROR_GEN_FAILURE; + goto TaskExit; + } + insertHere += argLen; + charCountBufferLeft -= argLen; + insertHere[0] = L' '; + insertHere += 1; + charCountBufferLeft -= 1; + insertHere[0] = 0; + ++crtArgIndex; + } + cmdLine = buffer; + } + + dwErrorCode = CreateTaskAsUser( + argv[ARGC_JOBOBJECTNAME], argv[ARGC_USERNAME], argv[ARGC_PIDFILE], cmdLine); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"CreateTaskAsUser", dwErrorCode); + ReportErrorCode(L"CreateTaskAsUser", dwErrorCode); goto TaskExit; } } else if (command == TaskIsAlive) @@ -666,10 +1178,10 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) // int isAlive; int numProcs; - dwErrorCode = IsTaskAlive(argv[2], &isAlive, &numProcs); + dwErrorCode = IsTaskAlive(argv[2], &isAlive, &numProcs); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"IsTaskAlive", dwErrorCode); + ReportErrorCode(L"IsTaskAlive", dwErrorCode); goto TaskExit; } @@ -681,27 +1193,27 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) else { dwErrorCode = ERROR_TASK_NOT_ALIVE; - ReportErrorCode(L"IsTaskAlive returned false", dwErrorCode); + ReportErrorCode(L"IsTaskAlive returned false", dwErrorCode); goto TaskExit; } } else if (command == TaskKill) { // Check if task jobobject // - dwErrorCode = KillTask(argv[2]); + dwErrorCode = KillTask(argv[2]); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"KillTask", dwErrorCode); + ReportErrorCode(L"KillTask", dwErrorCode); goto TaskExit; } } else if (command == TaskProcessList) { // Check if task jobobject // - dwErrorCode = PrintTaskProcessList(argv[2]); + dwErrorCode = PrintTaskProcessList(argv[2]); if (dwErrorCode != ERROR_SUCCESS) { - ReportErrorCode(L"PrintTaskProcessList", dwErrorCode); + ReportErrorCode(L"PrintTaskProcessList", dwErrorCode); goto TaskExit; } } else @@ -712,6 +1224,7 @@ int Task(__in int argc, __in_ecount(argc) wchar_t *argv[]) } TaskExit: + ReportErrorCode(L"TaskExit:", dwErrorCode); return dwErrorCode; } @@ -722,12 +1235,12 @@ void TaskUsage() // ProcessTree.isSetsidSupported() fwprintf(stdout, L"\ Usage: task create [TASKNAME] [COMMAND_LINE] |\n\ - task createAsUser [TASKNAME] [USERNAME] [PIDFILE] [COMMAND_LINE] |\n\ + task createAsUser [TASKNAME] [USERNAME] [PIDFILE] [COMMAND_LINE] |\n\ task isAlive [TASKNAME] |\n\ task kill [TASKNAME]\n\ task processList [TASKNAME]\n\ Creates a new task jobobject with taskname\n\ - Creates a new task jobobject with taskname as the user provided\n\ + Creates a new task jobobject with taskname as the user provided\n\ Checks if task jobobject is alive\n\ Kills task jobobject\n\ Prints to stdout a list of processes in the task\n\
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc new file mode 100644 index 0000000..2add064 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.mc @@ -0,0 +1,64 @@ +;/* +; * Licensed to the Apache Software Foundation (ASF) under one +; * or more contributor license agreements. See the NOTICE file +; * distributed with this work for additional information +; * regarding copyright ownership. The ASF licenses this file +; * to you under the Apache License, Version 2.0 (the +; * "License"); you may not use this file except in compliance +; * with the License. You may obtain a copy of the License at +; * +; * http://www.apache.org/licenses/LICENSE-2.0 +; * +; * Unless required by applicable law or agreed to in writing, software +; * distributed under the License is distributed on an "AS IS" BASIS, +; * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +; * See the License for the specific language governing permissions and +; * limitations under the License. +; */ + +; // winutils.mc + +; // EventLog messages for Hadoop winutils service. + + +LanguageNames=(English=0x409:MSG00409) + + +; // The following are the categories of events. + +MessageIdTypedef=WORD + +MessageId=0x1 +SymbolicName=SERVICE_CATEGORY +Language=English +Service Events +. + +MessageId=0x2 +SymbolicName=LOG_CATEGORY +Language=English +Task Events +. + +; // The following are the message definitions. + +MessageIdTypedef=DWORD + +MessageId=0x80 +SymbolicName=MSG_CHECK_ERROR +Language=English +%1. Error %2: %3. +. + +MessageId=0x100 +SymbolicName=MSG_RPC_SERVICE_HAS_STARTED +Language=English +The LPC server is listenning. +. + +MessageId=0x200 +SymbolicName=MSG_RPC_SERVICE_HAS_STOPPED +Language=English +The LPC server has stopped listenning. +. + http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln index d4e019e..d2784b8 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln @@ -26,26 +26,16 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils. EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32 - {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Release|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Release|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64 {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32 - {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Release|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64 {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64 EndGlobalSection http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj index 5b9a195..d736084 100644 --- a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj @@ -19,18 +19,10 @@ <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> <ItemGroup Label="ProjectConfigurations"> - <ProjectConfiguration Include="Debug|Win32"> - <Configuration>Debug</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> <ProjectConfiguration Include="Debug|x64"> <Configuration>Debug</Configuration> <Platform>x64</Platform> </ProjectConfiguration> - <ProjectConfiguration Include="Release|Win32"> - <Configuration>Release</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> <ProjectConfiguration Include="Release|x64"> <Configuration>Release</Configuration> <Platform>x64</Platform> @@ -42,22 +34,11 @@ <RootNamespace>winutils</RootNamespace> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>true</UseDebugLibraries> - <CharacterSet>Unicode</CharacterSet> - </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>true</UseDebugLibraries> <CharacterSet>Unicode</CharacterSet> </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <UseDebugLibraries>false</UseDebugLibraries> - <WholeProgramOptimization>true</WholeProgramOptimization> - <CharacterSet>Unicode</CharacterSet> - </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration"> <ConfigurationType>Application</ConfigurationType> <UseDebugLibraries>false</UseDebugLibraries> @@ -67,15 +48,9 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <ImportGroup Label="ExtensionSettings"> </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets"> <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> </ImportGroup> - <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> - </ImportGroup> <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets"> <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> </ImportGroup> @@ -83,57 +58,32 @@ <PropertyGroup> <IncludePath>include;$(IncludePath)</IncludePath> </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <LinkIncremental>true</LinkIncremental> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> - <LinkIncremental>true</LinkIncremental> - <OutDir /> - <IntDir>..\..\..\target\winutils\$(Configuration)\</IntDir> - </PropertyGroup> - <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <LinkIncremental>false</LinkIncremental> - </PropertyGroup> <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <LinkIncremental>false</LinkIncremental> - <IntDir>..\..\..\target\winutils\$(Platform)\$(Configuration)\</IntDir> - <OutDir>..\..\..\target\bin\</OutDir> </PropertyGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <ClCompile> - <PrecompiledHeader> - </PrecompiledHeader> - <WarningLevel>Level3</WarningLevel> - <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> - </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <GenerateDebugInformation>true</GenerateDebugInformation> - </Link> - </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> <ClCompile> <PrecompiledHeader> </PrecompiledHeader> <WarningLevel>Level4</WarningLevel> <Optimization>Disabled</Optimization> - <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;_DEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <SubSystem>Console</SubSystem> <GenerateDebugInformation>true</GenerateDebugInformation> </Link> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader> </PrecompiledHeader> - <Optimization>MaxSpeed</Optimization> + <!-- <Optimization>MaxSpeed</Optimization> --> + <Optimization>Disabled</Optimization> <FunctionLevelLinking>true</FunctionLevelLinking> <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;_CONSOLE;NDEBUG;_UNICODE;UNICODE;WSCE_CONFIG_DIR=$(WsceConfigDir);WSCE_CONFIG_FILE=$(WsceConfigFile);%(PreprocessorDefinitions)</PreprocessorDefinitions> </ClCompile> <Link> <SubSystem>Console</SubSystem> @@ -142,24 +92,40 @@ <OptimizeReferences>true</OptimizeReferences> </Link> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ItemDefinitionGroup> <ClCompile> - <WarningLevel>Level3</WarningLevel> - <PrecompiledHeader> - </PrecompiledHeader> - <Optimization>MaxSpeed</Optimization> - <FunctionLevelLinking>true</FunctionLevelLinking> - <IntrinsicFunctions>true</IntrinsicFunctions> - <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>$(IntermediateOutputPath)</AdditionalIncludeDirectories> </ClCompile> - <Link> - <SubSystem>Console</SubSystem> - <GenerateDebugInformation>true</GenerateDebugInformation> - <EnableCOMDATFolding>true</EnableCOMDATFolding> - <OptimizeReferences>true</OptimizeReferences> - </Link> + <CustomBuildStep> + <Message>Compiling Messages</Message> + <Command>mc.exe $(TargetName).mc -z $(TargetName)_msg -r $(IntermediateOutputPath) -h $(IntermediateOutputPath) -U</Command> + <Outputs>$(IntermediateOutputPath)$(TargetName)_msg.rc;$(IntermediateOutputPath)$(TargetName)_msg.h</Outputs> + </CustomBuildStep> + <Midl> + <ApplicationConfigurationMode>true</ApplicationConfigurationMode> + <TargetEnvironment>X64</TargetEnvironment> + <OutputDirectory>$(IntermediateOutputPath)</OutputDirectory> + <GenerateStublessProxies>true</GenerateStublessProxies> + <ValidateAllParameters>true</ValidateAllParameters> + <WarnAsError>true</WarnAsError> + <WarningLevel>2</WarningLevel> + </Midl> </ItemDefinitionGroup> + <PropertyGroup> + <CustomBuildAfterTargets>Midl</CustomBuildAfterTargets> + <CustomBuildBeforeTargets>ClCompile,ResourceCompile</CustomBuildBeforeTargets> + </PropertyGroup> + <ItemGroup> + <Midl Include="hadoopwinutilsvc.idl" /> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="$(IntermediateOutputPath)$(TargetName)_msg.rc" /> + </ItemGroup> + <ItemGroup> + <ClCompile Include="$(IntermediateOutputPath)\hadoopwinutilsvc_s.c" /> + </ItemGroup> <ItemGroup> + <ClCompile Include="service.c" /> <ClCompile Include="readlink.c" /> <ClCompile Include="symlink.c" /> <ClCompile Include="systeminfo.c" /> @@ -179,4 +145,4 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> -</Project> \ No newline at end of file +</Project> http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java index 2f8b84d..1e2d16e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java @@ -296,7 +296,7 @@ public class ProcessTree { return false; } catch (IOException ioe) { LOG.warn("Error executing shell command " - + Arrays.toString(shexec.getExecString()) + ioe); + + shexec.toString() + ioe); return false; } return (shexec.getExitCode() == 0 ? true : false); @@ -321,7 +321,7 @@ public class ProcessTree { return false; } catch (IOException ioe) { LOG.warn("Error executing shell command " - + Arrays.toString(shexec.getExecString()) + ioe); + + shexec.toString() + ioe); return false; } return (shexec.getExitCode() == 0 ? true : false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index df22042..d44e1ca 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -117,6 +117,9 @@ Release 2.6.0 - UNRELEASED YARN-1972. Added a secure container-executor for Windows. (Remus Rusanu via vinodkv) + YARN-2198. Remove the need to run NodeManager as privileged account for + Windows Secure Container Executor. (Remus Rusanu via jianhe) + IMPROVEMENTS YARN-2242. Improve exception information on AM launch crashes. (Li Lu http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index f5e63ba..3b866d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -79,19 +79,23 @@ public abstract class ContainerExecutor implements Configurable { public abstract void init() throws IOException; /** - * On Windows the ContainerLaunch creates a temporary empty jar to workaround the CLASSPATH length - * In a secure cluster this jar must be localized so that the container has access to it + * On Windows the ContainerLaunch creates a temporary special jar manifest of + * other jars to workaround the CLASSPATH length. In a secure cluster this + * jar must be localized so that the container has access to it. * This function localizes on-demand the jar. * * @param classPathJar * @param owner * @throws IOException */ - public void localizeClasspathJar(Path classPathJar, String owner) throws IOException { - // For the default container this is a no-op - // The WindowsSecureContainerExecutor overrides this + public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) + throws IOException { + // Non-secure executor simply use the classpath created + // in the NM fprivate folder + return classPathJar; } - + + /** * Prepare the environment for containers in this application to execute. * For $x in local.dirs @@ -105,14 +109,13 @@ public abstract class ContainerExecutor implements Configurable { * @param appId id of the application * @param nmPrivateContainerTokens path to localized credentials, rsrc by NM * @param nmAddr RPC address to contact NM - * @param localDirs nm-local-dirs - * @param logDirs nm-log-dirs + * @param dirsHandler NM local dirs service, for nm-local-dirs and nm-log-dirs * @throws IOException For most application init failures * @throws InterruptedException If application init thread is halted by NM */ public abstract void startLocalizer(Path nmPrivateContainerTokens, InetSocketAddress nmAddr, String user, String appId, String locId, - List<String> localDirs, List<String> logDirs) + LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException; @@ -132,8 +135,8 @@ public abstract class ContainerExecutor implements Configurable { */ public abstract int launchContainer(Container container, Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath, - String user, String appId, Path containerWorkDir, List<String> localDirs, - List<String> logDirs) throws IOException; + String user, String appId, Path containerWorkDir, + List<String> localDirs, List<String> logDirs) throws IOException; public abstract boolean signalContainer(String user, String pid, Signal signal) http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 76603ca..3e212bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager; import com.google.common.base.Optional; + import static org.apache.hadoop.fs.CreateFlag.CREATE; import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; @@ -32,9 +33,11 @@ import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Random; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; @@ -42,6 +45,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; +import org.apache.hadoop.util.Shell.CommandExecutor; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -92,13 +96,16 @@ public class DefaultContainerExecutor extends ContainerExecutor { @Override public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, - List<String> localDirs, List<String> logDirs) + LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { + List<String> localDirs = dirsHandler.getLocalDirs(); + List<String> logDirs = dirsHandler.getLogDirs(); + ContainerLocalizer localizer = new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf())); - + createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); @@ -120,9 +127,9 @@ public class DefaultContainerExecutor extends ContainerExecutor { @Override public int launchContainer(Container container, Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath, - String userName, String appId, Path containerWorkDir, + String user, String appId, Path containerWorkDir, List<String> localDirs, List<String> logDirs) throws IOException { - + FsPermission dirPerm = new FsPermission(APPDIR_PERM); ContainerId containerId = container.getContainerId(); @@ -134,29 +141,30 @@ public class DefaultContainerExecutor extends ContainerExecutor { getApplicationId()); for (String sLocalDir : localDirs) { Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE); - Path userdir = new Path(usersdir, userName); + Path userdir = new Path(usersdir, user); Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE); Path appDir = new Path(appCacheDir, appIdStr); Path containerDir = new Path(appDir, containerIdStr); - createDir(containerDir, dirPerm, true, userName); + createDir(containerDir, dirPerm, true, user); } // Create the container log-dirs on all disks - createContainerLogDirs(appIdStr, containerIdStr, logDirs, userName); + createContainerLogDirs(appIdStr, containerIdStr, logDirs, user); Path tmpDir = new Path(containerWorkDir, YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR); - createDir(tmpDir, dirPerm, false, userName); + createDir(tmpDir, dirPerm, false, user); - // copy launch script to work dir - Path launchDst = - new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT); - copyFile(nmPrivateContainerScriptPath, launchDst, userName); // copy container tokens to work dir Path tokenDst = new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE); - copyFile(nmPrivateTokensPath, tokenDst, userName); + copyFile(nmPrivateTokensPath, tokenDst, user); + + // copy launch script to work dir + Path launchDst = + new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT); + copyFile(nmPrivateContainerScriptPath, launchDst, user); // Create new local launch wrapper script LocalWrapperScriptBuilder sb = getLocalWrapperScriptBuilder( @@ -181,23 +189,19 @@ public class DefaultContainerExecutor extends ContainerExecutor { + " was marked as inactive. Returning terminated error"); return ExitCode.TERMINATED.getExitCode(); } - + // create log dir under app // fork script - ShellCommandExecutor shExec = null; + Shell.CommandExecutor shExec = null; try { - setScriptExecutable(launchDst, userName); - setScriptExecutable(sb.getWrapperScriptPath(), userName); - - // Setup command to run - String[] command = getRunCommand(sb.getWrapperScriptPath().toString(), - containerIdStr, userName, pidFile, this.getConf()); + setScriptExecutable(launchDst, user); + setScriptExecutable(sb.getWrapperScriptPath(), user); - LOG.info("launchContainer: " + Arrays.toString(command)); - shExec = new ShellCommandExecutor( - command, + shExec = buildCommandExecutor(sb.getWrapperScriptPath().toString(), + containerIdStr, user, pidFile, new File(containerWorkDir.toUri().getPath()), - container.getLaunchContext().getEnvironment()); // sanitized env + container.getLaunchContext().getEnvironment()); + if (isContainerActive(containerId)) { shExec.execute(); } @@ -242,11 +246,26 @@ public class DefaultContainerExecutor extends ContainerExecutor { } return exitCode; } finally { - ; // + if (shExec != null) shExec.close(); } return 0; } + protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, + String containerIdStr, String user, Path pidFile, File wordDir, + Map<String, String> environment) + throws IOException { + + String[] command = getRunCommand(wrapperScriptPath, + containerIdStr, user, pidFile, this.getConf()); + + LOG.info("launchContainer: " + Arrays.toString(command)); + return new ShellCommandExecutor( + command, + wordDir, + environment); + } + protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder( String containerIdStr, Path containerWorkDir) { return Shell.WINDOWS ? @@ -421,7 +440,7 @@ public class DefaultContainerExecutor extends ContainerExecutor { * @param signal signal to send * (for logging). */ - private void killContainer(String pid, Signal signal) throws IOException { + protected void killContainer(String pid, Signal signal) throws IOException { new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid)) .execute(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d0164e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index d628b1c..884a16a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -194,9 +194,12 @@ public class LinuxContainerExecutor extends ContainerExecutor { @Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, - List<String> localDirs, List<String> logDirs) + LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { + List<String> localDirs = dirsHandler.getLocalDirs(); + List<String> logDirs = dirsHandler.getLogDirs(); + verifyUsernamePattern(user); String runAsUser = getRunAsUser(user); List<String> command = new ArrayList<String>();
