Repository: hadoop Updated Branches: refs/heads/trunk a36399e09 -> 3b12fd6cf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java index 578a9a2..ec9c65e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java @@ -17,36 +17,272 @@ */ package org.apache.hadoop.yarn.server.nodemanager; +import java.io.BufferedReader; import java.io.File; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; import java.io.PrintStream; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO.Windows; +import org.apache.hadoop.io.nativeio.NativeIOException; +import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; +import org.apache.hadoop.util.Shell.CommandExecutor; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.LocalWrapperScriptBuilder; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; /** - * Windows secure container executor. Uses winutils task createAsUser. - * + * Windows secure container executor (WSCE). + * This class offers a secure container executor on Windows, similar to the + * LinuxContainerExecutor. As the NM does not run on a high privileged context, + * this class delegates elevated operations to the helper hadoopwintuilsvc, + * implemented by the winutils.exe running as a service. + * JNI and LRPC is used to communicate with the privileged service. */ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { - + private static final Log LOG = LogFactory .getLog(WindowsSecureContainerExecutor.class); + + public static final String LOCALIZER_PID_FORMAT = "STAR_LOCALIZER_%s"; + + + /** + * This class is a container for the JNI Win32 native methods used by WSCE. + */ + private static class Native { + + private static boolean nativeLoaded = false; + + static { + if (NativeCodeLoader.isNativeCodeLoaded()) { + try { + initWsceNative(); + nativeLoaded = true; + } catch (Throwable t) { + LOG.info("Unable to initialize WSCE Native libraries", t); + } + } + } + + /** Initialize the JNI method ID and class ID cache */ + private static native void initWsceNative(); + + + /** + * This class contains methods used by the WindowsSecureContainerExecutor + * file system operations. + */ + public static class Elevated { + private static final int MOVE_FILE = 1; + private static final int COPY_FILE = 2; + + public static void mkdir(Path dirName) throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for mkdir"); + } + elevatedMkDirImpl(dirName.toString()); + } + + private static native void elevatedMkDirImpl(String dirName) + throws IOException; + + public static void chown(Path fileName, String user, String group) + throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for chown"); + } + elevatedChownImpl(fileName.toString(), user, group); + } + + private static native void elevatedChownImpl(String fileName, String user, + String group) throws IOException; + + public static void move(Path src, Path dst, boolean replaceExisting) + throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for move"); + } + elevatedCopyImpl(MOVE_FILE, src.toString(), dst.toString(), + replaceExisting); + } + + public static void copy(Path src, Path dst, boolean replaceExisting) + throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for copy"); + } + elevatedCopyImpl(COPY_FILE, src.toString(), dst.toString(), + replaceExisting); + } + + private static native void elevatedCopyImpl(int operation, String src, + String dst, boolean replaceExisting) throws IOException; + + public static void chmod(Path fileName, int mode) throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for chmod"); + } + elevatedChmodImpl(fileName.toString(), mode); + } + + private static native void elevatedChmodImpl(String path, int mode) + throws IOException; + + public static void killTask(String containerName) throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for killTask"); + } + elevatedKillTaskImpl(containerName); + } + + private static native void elevatedKillTaskImpl(String containerName) + throws IOException; + + public static OutputStream create(Path f, boolean append) + throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for create"); + } + + long desiredAccess = Windows.GENERIC_WRITE; + long shareMode = 0L; + long creationDisposition = append ? + Windows.OPEN_ALWAYS : Windows.CREATE_ALWAYS; + long flags = Windows.FILE_ATTRIBUTE_NORMAL; + + String fileName = f.toString(); + fileName = fileName.replace('/', '\\'); + + long hFile = elevatedCreateImpl( + fileName, desiredAccess, shareMode, creationDisposition, flags); + return new FileOutputStream( + WinutilsProcessStub.getFileDescriptorFromHandle(hFile)); + } + + private static native long elevatedCreateImpl(String path, + long desiredAccess, long shareMode, + long creationDisposition, long flags) throws IOException; + + + public static boolean deleteFile(Path path) throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for deleteFile"); + } + + return elevatedDeletePathImpl(path.toString(), false); + } + + public static boolean deleteDirectory(Path path) throws IOException { + if (!nativeLoaded) { + throw new IOException("Native WSCE libraries are required for deleteDirectory"); + } + + return elevatedDeletePathImpl(path.toString(), true); + } + + public native static boolean elevatedDeletePathImpl(String path, + boolean isDir) throws IOException; + } + + /** + * Wraps a process started by the winutils service helper. + * + */ + public static class WinutilsProcessStub extends Process { + + private final long hProcess; + private final long hThread; + private boolean disposed = false; + + private final InputStream stdErr; + private final InputStream stdOut; + private final OutputStream stdIn; + + public WinutilsProcessStub(long hProcess, long hThread, long hStdIn, + long hStdOut, long hStdErr) { + this.hProcess = hProcess; + this.hThread = hThread; + + this.stdIn = new FileOutputStream(getFileDescriptorFromHandle(hStdIn)); + this.stdOut = new FileInputStream(getFileDescriptorFromHandle(hStdOut)); + this.stdErr = new FileInputStream(getFileDescriptorFromHandle(hStdErr)); + } + + public static native FileDescriptor getFileDescriptorFromHandle(long handle); + + @Override + public native void destroy(); + + @Override + public native int exitValue(); + + @Override + public InputStream getErrorStream() { + return stdErr; + } + @Override + public InputStream getInputStream() { + return stdOut; + } + @Override + public OutputStream getOutputStream() { + return stdIn; + } + @Override + public native int waitFor() throws InterruptedException; + + public synchronized native void dispose(); + + public native void resume() throws NativeIOException; + } + + public synchronized static WinutilsProcessStub createTaskAsUser( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws IOException { + if (!nativeLoaded) { + throw new IOException( + "Native WSCE libraries are required for createTaskAsUser"); + } + synchronized(Shell.WindowsProcessLaunchLock) { + return createTaskAsUser0(cwd, jobName, user, pidFile, cmdLine); + } + } + private static native WinutilsProcessStub createTaskAsUser0( + String cwd, String jobName, String user, String pidFile, String cmdLine) + throws NativeIOException; + } + + /** + * A shell script wrapper builder for WSCE. + * Overwrites the default behavior to remove the creation of the PID file in + * the script wrapper. WSCE creates the pid file as part of launching the + * task in winutils. + */ private class WindowsSecureWrapperScriptBuilder extends LocalWrapperScriptBuilder { @@ -60,21 +296,278 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { } } + /** + * This is a skeleton file system used to elevate certain operations. + * WSCE has to create container dirs under local/userchache/$user but + * this dir itself is owned by $user, with chmod 750. As ther NM has no + * write access, it must delegate the write operations to the privileged + * hadoopwintuilsvc. + */ + private static class ElevatedFileSystem extends DelegateToFileSystem { + + /** + * This overwrites certain RawLocalSystem operations to be performed by a + * privileged process. + * + */ + private static class ElevatedRawLocalFilesystem extends RawLocalFileSystem { + + @Override + protected boolean mkOneDir(File p2f) throws IOException { + Path path = new Path(p2f.getAbsolutePath()); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:mkOneDir: %s", path)); + } + boolean ret = false; + + // File.mkdir returns false, does not throw. Must mimic it. + try { + Native.Elevated.mkdir(path); + ret = true; + } + catch(Throwable e) { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:mkOneDir: %s", + org.apache.hadoop.util.StringUtils.stringifyException(e))); + } + } + return ret; + } + + @Override + public void setPermission(Path p, FsPermission permission) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:setPermission: %s %s", p, permission)); + } + Native.Elevated.chmod(p, permission.toShort()); + } + + @Override + public void setOwner(Path p, String username, String groupname) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:setOwner: %s %s %s", + p, username, groupname)); + } + Native.Elevated.chown(p, username, groupname); + } + + @Override + protected OutputStream createOutputStream(Path f, boolean append) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:create: %s %b", f, append)); + } + return Native.Elevated.create(f, append); + } + + @Override + public boolean delete(Path p, boolean recursive) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("EFS:delete: %s %b", p, recursive)); + } + + // The super delete uses the FileUtil.fullyDelete, + // but we cannot rely on that because we need to use the elevated + // operations to remove the files + // + File f = pathToFile(p); + if (!f.exists()) { + //no path, return false "nothing to delete" + return false; + } + else if (f.isFile()) { + return Native.Elevated.deleteFile(p); + } + else if (f.isDirectory()) { + + // This is a best-effort attempt. There are race conditions in that + // child files can be created/deleted after we snapped the list. + // No need to protect against that case. + File[] files = FileUtil.listFiles(f); + int childCount = files.length; + + if (recursive) { + for(File child:files) { + if (delete(new Path(child.getPath()), recursive)) { + --childCount; + } + } + } + if (childCount == 0) { + return Native.Elevated.deleteDirectory(p); + } + else { + throw new IOException("Directory " + f.toString() + " is not empty"); + } + } + else { + // This can happen under race conditions if an external agent + // is messing with the file type between IFs + throw new IOException("Path " + f.toString() + + " exists, but is neither a file nor a directory"); + } + } + } + + protected ElevatedFileSystem() throws IOException, URISyntaxException { + super(FsConstants.LOCAL_FS_URI, + new ElevatedRawLocalFilesystem(), + new Configuration(), + FsConstants.LOCAL_FS_URI.getScheme(), + false); + } + } + + private static class WintuilsProcessStubExecutor + implements Shell.CommandExecutor { + private Native.WinutilsProcessStub processStub; + private StringBuilder output = new StringBuilder(); + private int exitCode; + + private enum State { + INIT, + RUNNING, + COMPLETE + }; + + private State state;; + + private final String cwd; + private final String jobName; + private final String userName; + private final String pidFile; + private final String cmdLine; + + public WintuilsProcessStubExecutor( + String cwd, + String jobName, + String userName, + String pidFile, + String cmdLine) { + this.cwd = cwd; + this.jobName = jobName; + this.userName = userName; + this.pidFile = pidFile; + this.cmdLine = cmdLine; + this.state = State.INIT; + } + + private void assertComplete() throws IOException { + if (state != State.COMPLETE) { + throw new IOException("Process is not complete"); + } + } + + public String getOutput () throws IOException { + assertComplete(); + return output.toString(); + } + + public int getExitCode() throws IOException { + assertComplete(); + return exitCode; + } + + public void validateResult() throws IOException { + assertComplete(); + if (0 != exitCode) { + LOG.warn(output.toString()); + throw new IOException("Processs exit code is:" + exitCode); + } + } + + private Thread startStreamReader(final InputStream stream) + throws IOException { + Thread streamReaderThread = new Thread() { + + @Override + public void run() { + try + { + BufferedReader lines = new BufferedReader( + new InputStreamReader(stream)); + char[] buf = new char[512]; + int nRead; + while ((nRead = lines.read(buf, 0, buf.length)) > 0) { + output.append(buf, 0, nRead); + } + } + catch(Throwable t) { + LOG.error("Error occured reading the process stdout", t); + } + } + }; + streamReaderThread.start(); + return streamReaderThread; + } + + public void execute() throws IOException { + if (state != State.INIT) { + throw new IOException("Process is already started"); + } + processStub = Native.createTaskAsUser(cwd, + jobName, userName, pidFile, cmdLine); + state = State.RUNNING; + + Thread stdOutReader = startStreamReader(processStub.getInputStream()); + Thread stdErrReader = startStreamReader(processStub.getErrorStream()); + + try { + processStub.resume(); + processStub.waitFor(); + stdOutReader.join(); + stdErrReader.join(); + } + catch(InterruptedException ie) { + throw new IOException(ie); + } + + exitCode = processStub.exitValue(); + state = State.COMPLETE; + } + + @Override + public void close() { + if (processStub != null) { + processStub.dispose(); + } + } + } + private String nodeManagerGroup; + + /** + * Permissions for user WSCE dirs. + */ + static final short DIR_PERM = (short)0750; + + public WindowsSecureContainerExecutor() + throws IOException, URISyntaxException { + super(FileContext.getFileContext(new ElevatedFileSystem(), + new Configuration())); + } @Override public void setConf(Configuration conf) { super.setConf(conf); - nodeManagerGroup = conf.get(YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP); + nodeManagerGroup = conf.get( + YarnConfiguration.NM_WINDOWS_SECURE_CONTAINER_GROUP); } - + @Override protected String[] getRunCommand(String command, String groupId, String userName, Path pidFile, Configuration conf) { - return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, userName, - pidFile.toString(), "cmd /c " + command }; + File f = new File(command); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("getRunCommand: %s exists:%b", + command, f.exists())); + } + return new String[] { Shell.WINUTILS, "task", "createAsUser", groupId, + userName, pidFile.toString(), "cmd /c " + command }; } - + @Override protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder( String containerIdStr, Path containerWorkDir) { @@ -83,90 +576,156 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor { @Override protected void copyFile(Path src, Path dst, String owner) throws IOException { - super.copyFile(src, dst, owner); - lfs.setOwner(dst, owner, nodeManagerGroup); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("copyFile: %s -> %s owner:%s", src.toString(), + dst.toString(), owner)); + } + Native.Elevated.copy(src, dst, true); + Native.Elevated.chown(dst, owner, nodeManagerGroup); } @Override protected void createDir(Path dirPath, FsPermission perms, boolean createParent, String owner) throws IOException { + + // WSCE requires dirs to be 750, not 710 as DCE. + // This is similar to how LCE creates dirs + // + perms = new FsPermission(DIR_PERM); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("createDir: %s perm:%s owner:%s", + dirPath.toString(), perms.toString(), owner)); + } + super.createDir(dirPath, perms, createParent, owner); lfs.setOwner(dirPath, owner, nodeManagerGroup); } @Override - protected void setScriptExecutable(Path script, String owner) throws IOException { - super.setScriptExecutable(script, null); - lfs.setOwner(script, owner, nodeManagerGroup); + protected void setScriptExecutable(Path script, String owner) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("setScriptExecutable: %s owner:%s", + script.toString(), owner)); + } + super.setScriptExecutable(script, owner); + Native.Elevated.chown(script, owner, nodeManagerGroup); } @Override - public void localizeClasspathJar(Path classpathJar, String owner) throws IOException { - lfs.setOwner(classpathJar, owner, nodeManagerGroup); + public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("localizeClasspathJar: %s %s o:%s", + classPathJar, pwd, owner)); + } + createDir(pwd, new FsPermission(DIR_PERM), true, owner); + String fileName = classPathJar.getName(); + Path dst = new Path(pwd, fileName); + Native.Elevated.move(classPathJar, dst, true); + Native.Elevated.chown(dst, owner, nodeManagerGroup); + return dst; } @Override public void startLocalizer(Path nmPrivateContainerTokens, InetSocketAddress nmAddr, String user, String appId, String locId, - List<String> localDirs, List<String> logDirs) throws IOException, + LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { - + + List<String> localDirs = dirsHandler.getLocalDirs(); + List<String> logDirs = dirsHandler.getLogDirs(); + + Path classpathJarPrivateDir = dirsHandler.getLocalPathForWrite( + ResourceLocalizationService.NM_PRIVATE_DIR); createUserLocalDirs(localDirs, user); createUserCacheDirs(localDirs, user); createAppDirs(localDirs, user, appId); createAppLogDirs(appId, logDirs, user); Path appStorageDir = getWorkingDir(localDirs, user, appId); - - String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); + + String tokenFn = String.format( + ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); Path tokenDst = new Path(appStorageDir, tokenFn); - LOG.info("Copying from " + nmPrivateContainerTokens + " to " + tokenDst); copyFile(nmPrivateContainerTokens, tokenDst, user); - List<String> command ; - String[] commandArray; - ShellCommandExecutor shExec; - File cwdApp = new File(appStorageDir.toString()); - LOG.info(String.format("cwdApp: %s", cwdApp)); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("cwdApp: %s", cwdApp)); + } + + List<String> command ; command = new ArrayList<String>(); - command.add(Shell.WINUTILS); - command.add("task"); - command.add("createAsUser"); - command.add("START_LOCALIZER_" + locId); - command.add(user); - command.add("nul:"); // PID file - //use same jvm as parent - File jvm = new File(new File(System.getProperty("java.home"), "bin"), "java.exe"); + File jvm = new File( + new File(System.getProperty("java.home"), "bin"), "java.exe"); command.add(jvm.toString()); + Path cwdPath = new Path(cwdApp.getPath()); // Build a temp classpath jar. See ContainerLaunch.sanitizeEnv(). // Passing CLASSPATH explicitly is *way* too long for command line. String classPath = System.getProperty("java.class.path"); Map<String, String> env = new HashMap<String, String>(System.getenv()); - String[] jarCp = FileUtil.createJarWithClassPath(classPath, appStorageDir, env); - String classPathJar = jarCp[0]; - localizeClasspathJar(new Path(classPathJar), user); - String replacementClassPath = classPathJar + jarCp[1]; + String jarCp[] = FileUtil.createJarWithClassPath(classPath, + classpathJarPrivateDir, cwdPath, env); + String classPathJar = localizeClasspathJar( + new Path(jarCp[0]), cwdPath, user).toString(); command.add("-classpath"); - command.add(replacementClassPath); - + command.add(classPathJar + jarCp[1]); + String javaLibPath = System.getProperty("java.library.path"); if (javaLibPath != null) { command.add("-Djava.library.path=" + javaLibPath); } - ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs); - commandArray = command.toArray(new String[command.size()]); - - shExec = new ShellCommandExecutor( - commandArray, cwdApp); + ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, + localDirs); + + String cmdLine = StringUtils.join(command, " "); + + String localizerPid = String.format(LOCALIZER_PID_FORMAT, locId); + + WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor( + cwdApp.getAbsolutePath(), + localizerPid, user, "nul:", cmdLine); + try { + stubExecutor.execute(); + stubExecutor.validateResult(); + } + finally { + stubExecutor.close(); + try + { + killContainer(localizerPid, Signal.KILL); + } + catch(Throwable e) { + LOG.warn(String.format( + "An exception occured during the cleanup of localizer job %s:\n%s", + localizerPid, + org.apache.hadoop.util.StringUtils.stringifyException(e))); + } + } + } + + @Override + protected CommandExecutor buildCommandExecutor(String wrapperScriptPath, + String containerIdStr, + String userName, Path pidFile,File wordDir, Map<String, String> environment) + throws IOException { - shExec.execute(); + return new WintuilsProcessStubExecutor( + wordDir.toString(), + containerIdStr, userName, pidFile.toString(), + "cmd /c " + wrapperScriptPath); + } + + @Override + protected void killContainer(String pid, Signal signal) throws IOException { + Native.Elevated.killTask(pid); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 30abef5..434cb4e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -212,7 +212,9 @@ public class ContainerLaunch implements Callable<Integer> { + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, containerIdStr)); - + Path nmPrivateClasspathJarDir = + dirsHandler.getLocalPathForWrite( + getContainerPrivateDir(appIdStr, containerIdStr)); DataOutputStream containerScriptOutStream = null; DataOutputStream tokensOutStream = null; @@ -263,7 +265,7 @@ public class ContainerLaunch implements Callable<Integer> { FINAL_CONTAINER_TOKENS_FILE).toUri().getPath()); // Sanitize the container's environment sanitizeEnv(environment, containerWorkDir, appDirs, containerLogDirs, - localResources); + localResources, nmPrivateClasspathJarDir); // Write out the environment writeLaunchEnv(containerScriptOutStream, environment, localResources, @@ -658,7 +660,8 @@ public class ContainerLaunch implements Callable<Integer> { public void sanitizeEnv(Map<String, String> environment, Path pwd, List<Path> appDirs, List<String> containerLogDirs, - Map<Path, List<String>> resources) throws IOException { + Map<Path, List<String>> resources, + Path nmPrivateClasspathJarDir) throws IOException { /** * Non-modifiable environment variables */ @@ -722,6 +725,7 @@ public class ContainerLaunch implements Callable<Integer> { // TODO: Remove Windows check and use this approach on all platforms after // additional testing. See YARN-358. if (Shell.WINDOWS) { + String inputClassPath = environment.get(Environment.CLASSPATH.name()); if (inputClassPath != null && !inputClassPath.isEmpty()) { StringBuilder newClassPath = new StringBuilder(inputClassPath); @@ -763,13 +767,13 @@ public class ContainerLaunch implements Callable<Integer> { Map<String, String> mergedEnv = new HashMap<String, String>( System.getenv()); mergedEnv.putAll(environment); - + String[] jarCp = FileUtil.createJarWithClassPath( - newClassPath.toString(), pwd, mergedEnv); - String classPathJar = jarCp[0]; + newClassPath.toString(), nmPrivateClasspathJarDir, pwd, mergedEnv); // In a secure cluster the classpath jar must be localized to grant access - this.exec.localizeClasspathJar(new Path(classPathJar), container.getUser()); - String replacementClassPath = classPathJar + jarCp[1]; + Path localizedClassPathJar = exec.localizeClasspathJar( + new Path(jarCp[0]), pwd, container.getUser()); + String replacementClassPath = localizedClassPathJar.toString() + jarCp[1]; environment.put(Environment.CLASSPATH.name(), replacementClassPath); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 3525170..32e3553 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -369,10 +369,15 @@ public class ContainerLocalizer { new ContainerLocalizer(FileContext.getLocalFSFileContext(), user, appId, locId, localDirs, RecordFactoryProvider.getRecordFactory(null)); - System.exit(localizer.runLocalization(nmAddr)); + int nRet = localizer.runLocalization(nmAddr); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("nRet: %d", nRet)); + } + System.exit(nRet); } catch (Throwable e) { // Print error to stdout so that LCE can use it. e.printStackTrace(System.out); + LOG.error("Exception in main:", e); throw e; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 371684b..a6143a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -1088,7 +1088,8 @@ public class ResourceLocalizationService extends CompositeService ConverterUtils.toString( context.getContainerId(). getApplicationAttemptId().getApplicationId()), - localizerId, localDirs, logDirs); + localizerId, + dirsHandler); } else { throw new IOException("All disks failed. " + dirsHandler.getDisksHealthReport(false)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index 7db74f3..bf01fb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -303,6 +303,7 @@ public class TestDefaultContainerExecutor { public void testStartLocalizer() throws IOException, InterruptedException { InetSocketAddress localizationServerAddress; + final Path firstDir = new Path(BASE_TMP_PATH, "localDir1"); List<String> localDirs = new ArrayList<String>(); final Path secondDir = new Path(BASE_TMP_PATH, "localDir2"); @@ -383,9 +384,14 @@ public class TestDefaultContainerExecutor { String appSubmitter = "nobody"; String appId = "APP_ID"; String locId = "LOC_ID"; + + LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class); + when(dirsHandler.getLocalDirs()).thenReturn(localDirs); + when(dirsHandler.getLogDirs()).thenReturn(logDirs); + try { mockExec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress, - appSubmitter, appId, locId, localDirs, logDirs); + appSubmitter, appId, locId, dirsHandler); } catch (IOException e) { Assert.fail("StartLocalizer failed to copy token file " + e); } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index 1d5b2fb..c9e09fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -266,7 +266,7 @@ public class TestLinuxContainerExecutor { exec.setConf(conf); exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter, - appId, locId, localDirs, logDirs); + appId, locId, dirsHandler); String locId2 = "container_01_02"; Path nmPrivateContainerTokensPath2 = @@ -276,7 +276,7 @@ public class TestLinuxContainerExecutor { + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2)); files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE)); exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter, - appId, locId2, localDirs, logDirs); + appId, locId2, dirsHandler); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 2e9e8b1..d54367a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -185,7 +185,7 @@ public class TestLinuxContainerExecutorWithMocks { Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath"); try { - mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs()); + mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler); List<String> result=readMockParams(); Assert.assertEquals(result.size(), 17); Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index d569fa7..cf1e9fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -850,7 +850,7 @@ public class TestResourceLocalizationService { ArgumentCaptor<Path> tokenPathCaptor = ArgumentCaptor.forClass(Path.class); verify(exec).startLocalizer(tokenPathCaptor.capture(), isA(InetSocketAddress.class), eq("user0"), eq(appStr), eq(ctnrStr), - isA(List.class), isA(List.class)); + isA(LocalDirsHandlerService.class)); Path localizationTokenPath = tokenPathCaptor.getValue(); // heartbeat from localizer http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b12fd6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm index 1f9688a..722870c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm @@ -83,11 +83,15 @@ min.user.id=1000#Prevent other super-users +---+ - ** Windows Secure Container Executor + ** Windows Secure Container Executor (WSCE) The Windows environment secure container executor is the <<<WindowsSecureContainerExecutor>>>. It uses the Windows S4U infrastructure to launch the container as the - YARN application user. + YARN application user. The WSCE requires the presense of the <<<hadoopwinutilsvc>>> service. This services + is hosted by <<<%HADOOP_HOME%\bin\winutils.exe>>> started with the <<<service>>> command line argument. This + service offers some privileged operations that require LocalSystem authority so that the NM is not required + to run the entire JVM and all the NM code in an elevated context. The NM interacts with the <<<hadoopwintulsvc>>> + service by means of Local RPC (LRPC) via calls JNI to the RCP client hosted in <<<hadoop.dll>>>. *** Configuration @@ -102,17 +106,71 @@ min.user.id=1000#Prevent other super-users <property> <name>yarn.nodemanager.windows-secure-container-executor.group</name> - <value>hadoop</value> + <value>yarn</value> </property> +---+ + *** wsce-site.xml + + The hadoopwinutilsvc uses <<<%HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. - The NodeManager must run as a member of the local <<<Administrators>>> group or as - <<<LocalSystem>>>. It is not enough for the NodeManager to simply impersonate such an user. ++---+ + <property> + <name>yarn.nodemanager.windows-secure-container-executor.impersonate.allowed</name> + <value>HadoopUsers</value> + </property> + + <property> + <name>yarn.nodemanager.windows-secure-container-executor.impersonate.denied</name> + <value>HadoopServices,Administrators</value> + </property> + + <property> + <name>yarn.nodemanager.windows-secure-container-executor.allowed</name> + <value>nodemanager</value> + </property> + + <property> + <name>yarn.nodemanager.windows-secure-container-executor.local-dirs</name> + <value>nm-local-dir, nm-log-dirs</value> + </property> + + <property> + <name>yarn.nodemanager.windows-secure-container-executor.job-name</name> + <value>nodemanager-job-name</value> + </property> ++---+ + + <<<yarn.nodemanager.windows-secure-container-executor.allowed>>> should contain the name of the service account running the + nodemanager. This user will be allowed to access the hadoopwintuilsvc functions. + + <<<yarn.nodemanager.windows-secure-container-executor.impersonate.allowed>>> should contain the users that are allowed to create + containers in the cluster. These users will be allowed to be impersonated by hadoopwinutilsvc. + + <<<yarn.nodemanager.windows-secure-container-executor.impersonate.denied>>> should contain users that are explictly forbiden from + creating containers. hadoopwinutilsvc will refuse to impersonate these users. + + <<<yarn.nodemanager.windows-secure-container-executor.local-dirs>>> should contain the nodemanager local dirs. hadoopwinutilsvc will + allow only file operations under these directories. This should contain the same values as <<<${yarn.nodemanager.local-dirs}, ${yarn.nodemanager.log-dirs}>>> + but note that hadoopwinutilsvc XML configuration processing does not do substitutions so the value must be the final value. All paths + must be absolute and no environment variable substitution will be performed. The paths are compared LOCAL_INVARIANT case insensitive string comparison, + the file path validated must start with one of the paths listed in local-dirs configuration. Use comma as path separator:<<<,>>> + + <<<yarn.nodemanager.windows-secure-container-executor.job-name>>> should contain an Windows NT job name that all containers should be added to. + This configuration is optional. If not set, the container is not added to a global NodeManager job. Normally this should be set to the job that the NM is assigned to, + so that killing the NM kills also all containers. Hadoopwinutilsvc will not attempt to create this job, the job must exists when the container is launched. + If the value is set and the job does not exists, container launch will fail with error 2 <<<The system cannot find the file specified>>>. + Note that this global NM job is not related to the container job, which always gets created for each container and is named after the container ID. + This setting controls a global job that spans all containers and the parent NM, and as such it requires nested jobs. + Nested jobs are available only post Windows 8 and Windows Server 2012. *** Useful Links * {{{http://msdn.microsoft.com/en-us/magazine/cc188757.aspx}Exploring S4U Kerberos Extensions in Windows Server 2003}} + * {{{http://msdn.microsoft.com/en-us/library/windows/desktop/hh448388(v=vs.85).aspx}Nested Jobs}} + * {{{https://issues.apache.org/jira/browse/YARN-1063}Winutils needs ability to create task as domain user}} * {{{https://issues.apache.org/jira/browse/YARN-1972}Implement secure Windows Container Executor}} + + * {{{https://issues.apache.org/jira/browse/YARN-2198}Remove the need to run NodeManager as privileged account for Windows Secure Container Executor}} \ No newline at end of file
