Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java Tue Aug 19 23:49:39 2014 @@ -36,6 +36,7 @@ public class NodeInfo { protected String healthReport; protected long totalVmemAllocatedContainersMB; protected long totalPmemAllocatedContainersMB; + protected long totalVCoresAllocatedContainers; protected boolean vmemCheckEnabled; protected boolean pmemCheckEnabled; protected long lastNodeUpdateTime; @@ -62,6 +63,8 @@ public class NodeInfo { this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); + this.totalVCoresAllocatedContainers = resourceView + .getVCoresAllocatedForContainers(); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); @@ -124,6 +127,10 @@ public class NodeInfo { return this.totalVmemAllocatedContainersMB; } + public long getTotalVCoresAllocated() { + return this.totalVCoresAllocatedContainers; + } + public boolean isVmemCheckEnabled() { return this.vmemCheckEnabled; }
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c Tue Aug 19 23:49:39 2014 @@ -33,6 +33,7 @@ #include <limits.h> #include <sys/stat.h> #include <sys/mount.h> +#include <sys/wait.h> static const int DEFAULT_MIN_USERID = 1000; @@ -111,16 +112,16 @@ int check_executor_permissions(char *exe return -1; } - // check others do not have read/write/execute permissions - if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH) - == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) { + // check others do not have write/execute permissions + if ((filestat.st_mode & S_IWOTH) == S_IWOTH || + (filestat.st_mode & S_IXOTH) == S_IXOTH) { fprintf(LOGFILE, - "The container-executor binary should not have read or write or" - " execute for others.\n"); + "The container-executor binary should not have write or execute " + "for others.\n"); return -1; } - // Binary should be setuid/setgid executable + // Binary should be setuid executable if ((filestat.st_mode & S_ISUID) == 0) { fprintf(LOGFILE, "The container-executor binary should be set setuid.\n"); return -1; @@ -245,6 +246,85 @@ static int write_pid_to_file_as_nm(const } /** + * Write the exit code of the container into the exit code file + * exit_code_file: Path to exit code file where exit code needs to be written + */ +static int write_exit_code_file(const char* exit_code_file, int exit_code) { + char *tmp_ecode_file = concatenate("%s.tmp", "exit_code_path", 1, + exit_code_file); + if (tmp_ecode_file == NULL) { + return -1; + } + + // create with 700 + int ecode_fd = open(tmp_ecode_file, O_WRONLY|O_CREAT|O_EXCL, S_IRWXU); + if (ecode_fd == -1) { + fprintf(LOGFILE, "Can't open file %s - %s\n", tmp_ecode_file, + strerror(errno)); + free(tmp_ecode_file); + return -1; + } + + char ecode_buf[21]; + snprintf(ecode_buf, sizeof(ecode_buf), "%d", exit_code); + ssize_t written = write(ecode_fd, ecode_buf, strlen(ecode_buf)); + close(ecode_fd); + if (written == -1) { + fprintf(LOGFILE, "Failed to write exit code to file %s - %s\n", + tmp_ecode_file, strerror(errno)); + free(tmp_ecode_file); + return -1; + } + + // rename temp file to actual exit code file + // use rename as atomic + if (rename(tmp_ecode_file, exit_code_file)) { + fprintf(LOGFILE, "Can't move exit code file from %s to %s - %s\n", + tmp_ecode_file, exit_code_file, strerror(errno)); + unlink(tmp_ecode_file); + free(tmp_ecode_file); + return -1; + } + + free(tmp_ecode_file); + return 0; +} + +/** + * Wait for the container process to exit and write the exit code to + * the exit code file. + * Returns the exit code of the container process. + */ +static int wait_and_write_exit_code(pid_t pid, const char* exit_code_file) { + int child_status = -1; + int exit_code = -1; + int waitpid_result; + + if (change_effective_user(nm_uid, nm_gid) != 0) { + return -1; + } + do { + waitpid_result = waitpid(pid, &child_status, 0); + } while (waitpid_result == -1 && errno == EINTR); + if (waitpid_result < 0) { + fprintf(LOGFILE, "Error waiting for container process %d - %s\n", + pid, strerror(errno)); + return -1; + } + if (WIFEXITED(child_status)) { + exit_code = WEXITSTATUS(child_status); + } else if (WIFSIGNALED(child_status)) { + exit_code = 0x80 + WTERMSIG(child_status); + } else { + fprintf(LOGFILE, "Unable to determine exit status for pid %d\n", pid); + } + if (write_exit_code_file(exit_code_file, exit_code) < 0) { + return -1; + } + return exit_code; +} + +/** * Change the real and effective user and group to abandon the super user * priviledges. */ @@ -337,6 +417,10 @@ char *get_container_work_directory(const nm_root, user, app_id, container_id); } +char *get_exit_code_file(const char* pid_file) { + return concatenate("%s.exitcode", "exit_code_file", 1, pid_file); +} + char *get_container_launcher_file(const char* work_dir) { return concatenate("%s/%s", "container launcher", 2, work_dir, CONTAINER_SCRIPT); } @@ -879,6 +963,8 @@ int launch_container_as_user(const char int exit_code = -1; char *script_file_dest = NULL; char *cred_file_dest = NULL; + char *exit_code_file = NULL; + script_file_dest = get_container_launcher_file(work_dir); if (script_file_dest == NULL) { exit_code = OUT_OF_MEMORY; @@ -889,6 +975,11 @@ int launch_container_as_user(const char exit_code = OUT_OF_MEMORY; goto cleanup; } + exit_code_file = get_exit_code_file(pid_file); + if (NULL == exit_code_file) { + exit_code = OUT_OF_MEMORY; + goto cleanup; + } // open launch script int container_file_source = open_file_as_nm(script_name); @@ -902,6 +993,13 @@ int launch_container_as_user(const char goto cleanup; } + pid_t child_pid = fork(); + if (child_pid != 0) { + // parent + exit_code = wait_and_write_exit_code(child_pid, exit_code_file); + goto cleanup; + } + // setsid pid_t pid = setsid(); if (pid == -1) { @@ -986,6 +1084,7 @@ int launch_container_as_user(const char exit_code = 0; cleanup: + free(exit_code_file); free(script_file_dest); free(cred_file_dest); return exit_code; @@ -1082,6 +1181,7 @@ static int delete_path(const char *full_ FTS* tree = fts_open(paths, FTS_PHYSICAL | FTS_XDEV, NULL); FTSENT* entry = NULL; int ret = 0; + int ret_errno = 0; if (tree == NULL) { fprintf(LOGFILE, @@ -1099,7 +1199,13 @@ static int delete_path(const char *full_ if (rmdir(entry->fts_accpath) != 0) { fprintf(LOGFILE, "Couldn't delete directory %s - %s\n", entry->fts_path, strerror(errno)); - exit_code = -1; + if (errno == EROFS) { + exit_code = -1; + } + // record the first errno + if (errno != ENOENT && ret_errno == 0) { + ret_errno = errno; + } } } break; @@ -1111,7 +1217,13 @@ static int delete_path(const char *full_ if (unlink(entry->fts_accpath) != 0) { fprintf(LOGFILE, "Couldn't delete file %s - %s\n", entry->fts_path, strerror(errno)); - exit_code = -1; + if (errno == EROFS) { + exit_code = -1; + } + // record the first errno + if (errno != ENOENT && ret_errno == 0) { + ret_errno = errno; + } } break; @@ -1154,6 +1266,9 @@ static int delete_path(const char *full_ } } ret = fts_close(tree); + if (ret_errno != 0) { + exit_code = -1; + } if (exit_code == 0 && ret != 0) { fprintf(LOGFILE, "Error in fts_close while deleting %s\n", full_path); exit_code = -1; Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Aug 19 23:49:39 2014 @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; public class DummyContainerManager extends ContainerManagerImpl { @@ -75,7 +76,7 @@ public class DummyContainerManager exten protected ResourceLocalizationService createResourceLocalizationService( ContainerExecutor exec, DeletionService deletionContext) { return new ResourceLocalizationService(super.dispatcher, exec, - deletionContext, super.dirsHandler) { + deletionContext, super.dirsHandler, new NMNullStateStoreService()) { @Override public void handle(LocalizationEvent event) { switch (event.getType()) { Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java Tue Aug 19 23:49:39 2014 @@ -18,16 +18,37 @@ package org.apache.hadoop.yarn.server.nodemanager; +import static org.apache.hadoop.fs.CreateFlag.CREATE; +import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doAnswer; +import static org.junit.Assert.assertTrue; + +import java.io.File; import java.io.FileNotFoundException; +import java.io.FileReader; import java.io.InputStream; import java.io.IOException; +import java.io.LineNumberReader; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Random; -import org.junit.Assert; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; @@ -45,15 +66,13 @@ import org.apache.hadoop.util.Progressab import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream; -import static org.apache.hadoop.fs.CreateFlag.*; - - import org.junit.AfterClass; +import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; -import org.mockito.ArgumentMatcher; -import org.mockito.Matchers; -import static org.mockito.Mockito.*; +import org.junit.After; +import org.junit.Assert; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestDefaultContainerExecutor { @@ -191,6 +210,92 @@ public class TestDefaultContainerExecuto } } + @Test + public void testContainerLaunchError() + throws IOException, InterruptedException { + + Path localDir = new Path(BASE_TMP_PATH, "localDir"); + List<String> localDirs = new ArrayList<String>(); + localDirs.add(localDir.toString()); + List<String> logDirs = new ArrayList<String>(); + Path logDir = new Path(BASE_TMP_PATH, "logDir"); + logDirs.add(logDir.toString()); + + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.toString()); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString()); + + FileContext lfs = FileContext.getLocalFSFileContext(conf); + DefaultContainerExecutor mockExec = spy(new DefaultContainerExecutor(lfs)); + mockExec.setConf(conf); + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) + throws Throwable { + String diagnostics = (String) invocationOnMock.getArguments()[0]; + assertTrue("Invalid Diagnostics message: " + diagnostics, + diagnostics.contains("No such file or directory")); + return null; + } + } + ).when(mockExec).logOutput(any(String.class)); + + String appSubmitter = "nobody"; + String appId = "APP_ID"; + String containerId = "CONTAINER_ID"; + Container container = mock(Container.class); + ContainerId cId = mock(ContainerId.class); + ContainerLaunchContext context = mock(ContainerLaunchContext.class); + HashMap<String, String> env = new HashMap<String, String>(); + + when(container.getContainerId()).thenReturn(cId); + when(container.getLaunchContext()).thenReturn(context); + try { + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) + throws Throwable { + ContainerDiagnosticsUpdateEvent event = + (ContainerDiagnosticsUpdateEvent) invocationOnMock + .getArguments()[0]; + assertTrue("Invalid Diagnostics message: " + + event.getDiagnosticsUpdate(), + event.getDiagnosticsUpdate().contains("No such file or directory") + ); + return null; + } + }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class)); + + when(cId.toString()).thenReturn(containerId); + when(cId.getApplicationAttemptId()).thenReturn( + ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 0)); + + when(context.getEnvironment()).thenReturn(env); + + mockExec.createUserLocalDirs(localDirs, appSubmitter); + mockExec.createUserCacheDirs(localDirs, appSubmitter); + mockExec.createAppDirs(localDirs, appSubmitter, appId); + mockExec.createAppLogDirs(appId, logDirs); + + Path scriptPath = new Path("file:///bin/echo"); + Path tokensPath = new Path("file:///dev/null"); + Path workDir = localDir; + Path pidFile = new Path(workDir, "pid.txt"); + + mockExec.init(); + mockExec.activateContainer(cId, pidFile); + int ret = mockExec + .launchContainer(container, scriptPath, tokensPath, appSubmitter, + appId, workDir, localDirs, localDirs); + Assert.assertNotSame(0, ret); + } finally { + mockExec.deleteAsUser(appSubmitter, localDir); + mockExec.deleteAsUser(appSubmitter, logDir); + } + } + // @Test // public void testInit() throws IOException, InterruptedException { // Configuration conf = new Configuration(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDeletionService.java Tue Aug 19 23:49:39 2014 @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService; import org.junit.AfterClass; import org.junit.Test; import org.mockito.Mockito; @@ -285,4 +286,58 @@ public class TestDeletionService { del.stop(); } } + + @Test + public void testRecovery() throws Exception { + Random r = new Random(); + long seed = r.nextLong(); + r.setSeed(seed); + System.out.println("SEED: " + seed); + List<Path> baseDirs = buildDirs(r, base, 4); + createDirs(new Path("."), baseDirs); + List<Path> content = buildDirs(r, new Path("."), 10); + for (Path b : baseDirs) { + createDirs(b, content); + } + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); + conf.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 1); + NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); + stateStore.init(conf); + stateStore.start(); + DeletionService del = + new DeletionService(new FakeDefaultContainerExecutor(), stateStore); + try { + del.init(conf); + del.start(); + for (Path p : content) { + assertTrue(lfs.util().exists(new Path(baseDirs.get(0), p))); + del.delete((Long.parseLong(p.getName()) % 2) == 0 ? null : "dingo", + p, baseDirs.toArray(new Path[4])); + } + + // restart the deletion service + del.stop(); + del = new DeletionService(new FakeDefaultContainerExecutor(), + stateStore); + del.init(conf); + del.start(); + + // verify paths are still eventually deleted + int msecToWait = 10 * 1000; + for (Path p : baseDirs) { + for (Path q : content) { + Path fp = new Path(p, q); + while (msecToWait > 0 && lfs.util().exists(fp)) { + Thread.sleep(100); + msecToWait -= 100; + } + assertFalse(lfs.util().exists(fp)); + } + } + } finally { + del.close(); + stateStore.close(); + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Aug 19 23:49:39 2014 @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -79,7 +80,8 @@ public class TestEventFlow { YarnConfiguration conf = new YarnConfiguration(); Context context = new NMContext(new NMContainerTokenSecretManager(conf), - new NMTokenSecretManagerInNM(), null, null) { + new NMTokenSecretManagerInNM(), null, null, + new NMNullStateStoreService()) { @Override public int getHttpPort() { return 1234; Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Tue Aug 19 23:49:39 2014 @@ -19,8 +19,12 @@ package org.apache.hadoop.yarn.server.nodemanager; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import java.io.File; @@ -34,8 +38,6 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,9 +48,13 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.junit.Assert; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestLinuxContainerExecutorWithMocks { @@ -216,7 +222,19 @@ public class TestLinuxContainerExecutorW conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo"); conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null"); - mockExec = new LinuxContainerExecutor(); + mockExec = spy(new LinuxContainerExecutor()); + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) + throws Throwable { + String diagnostics = (String) invocationOnMock.getArguments()[0]; + assertTrue("Invalid Diagnostics message: " + diagnostics, + diagnostics.contains("badcommand")); + return null; + } + } + ).when(mockExec).logOutput(any(String.class)); dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); mockExec.setConf(conf); @@ -233,7 +251,22 @@ public class TestLinuxContainerExecutorW when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); - + doAnswer( + new Answer() { + @Override + public Object answer(InvocationOnMock invocationOnMock) + throws Throwable { + ContainerDiagnosticsUpdateEvent event = + (ContainerDiagnosticsUpdateEvent) invocationOnMock + .getArguments()[0]; + assertTrue("Invalid Diagnostics message: " + + event.getDiagnosticsUpdate(), + event.getDiagnosticsUpdate().contains("badcommand")); + return null; + } + } + ).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class)); + when(cId.toString()).thenReturn(containerId); when(context.getEnvironment()).thenReturn(env); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java Tue Aug 19 23:49:39 2014 @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.server.nodemanager; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -27,17 +30,19 @@ import java.util.concurrent.ConcurrentMa import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Assert; - import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; @@ -46,6 +51,7 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.ResourceTracker; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; @@ -57,6 +63,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -77,6 +84,9 @@ public class TestNodeManagerResync { private CyclicBarrier syncBarrier; private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false); + private final NodeManagerEvent resyncEvent = + new NodeManagerEvent(NodeManagerEventType.RESYNC); + @Before public void setup() throws UnsupportedFileSystemException { @@ -94,34 +104,56 @@ public class TestNodeManagerResync { assertionFailedInThread.set(false); } - @SuppressWarnings("unchecked") @Test public void testKillContainersOnResync() throws IOException, InterruptedException, YarnException { - NodeManager nm = new TestNodeManager1(); + TestNodeManager1 nm = new TestNodeManager1(false); + + testContainerPreservationOnResyncImpl(nm, false); + } + + @Test + public void testPreserveContainersOnResyncKeepingContainers() throws + IOException, + InterruptedException, YarnException { + TestNodeManager1 nm = new TestNodeManager1(true); + + testContainerPreservationOnResyncImpl(nm, true); + } + + @SuppressWarnings("unchecked") + protected void testContainerPreservationOnResyncImpl(TestNodeManager1 nm, + boolean isWorkPreservingRestartEnabled) + throws IOException, YarnException, InterruptedException { YarnConfiguration conf = createNMConfig(); - nm.init(conf); - nm.start(); - ContainerId cId = TestNodeManagerShutdown.createContainerId(); - TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir, - processStartFile); + conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, + isWorkPreservingRestartEnabled); - Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount()); - nm.getNMDispatcher().getEventHandler(). - handle( new NodeManagerEvent(NodeManagerEventType.RESYNC)); try { - syncBarrier.await(); - } catch (BrokenBarrierException e) { + nm.init(conf); + nm.start(); + ContainerId cId = TestNodeManagerShutdown.createContainerId(); + TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir, + processStartFile); + + nm.setExistingContainerId(cId); + Assert.assertEquals(1, ((TestNodeManager1) nm).getNMRegistrationCount()); + nm.getNMDispatcher().getEventHandler().handle(resyncEvent); + try { + syncBarrier.await(); + } catch (BrokenBarrierException e) { + } + Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount()); + // Only containers should be killed on resync, apps should lie around. + // That way local resources for apps can be used beyond resync without + // relocalization + Assert.assertTrue(nm.getNMContext().getApplications() + .containsKey(cId.getApplicationAttemptId().getApplicationId())); + Assert.assertFalse(assertionFailedInThread.get()); + } + finally { + nm.stop(); } - Assert.assertEquals(2, ((TestNodeManager1) nm).getNMRegistrationCount()); - // Only containers should be killed on resync, apps should lie around. That - // way local resources for apps can be used beyond resync without - // relocalization - Assert.assertTrue(nm.getNMContext().getApplications() - .containsKey(cId.getApplicationAttemptId().getApplicationId())); - Assert.assertFalse(assertionFailedInThread.get()); - - nm.stop(); } // This test tests new container requests are blocked when NM starts from @@ -149,7 +181,7 @@ public class TestNodeManagerResync { Assert.assertFalse(assertionFailedInThread.get()); nm.stop(); } - + @SuppressWarnings("unchecked") @Test(timeout=10000) public void testNMshutdownWhenResyncThrowException() throws IOException, @@ -161,7 +193,7 @@ public class TestNodeManagerResync { Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount()); nm.getNMDispatcher().getEventHandler() .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC)); - + synchronized (isNMShutdownCalled) { while (isNMShutdownCalled.get() == false) { try { @@ -170,7 +202,7 @@ public class TestNodeManagerResync { } } } - + Assert.assertTrue("NM shutdown not called.",isNMShutdownCalled.get()); nm.stop(); } @@ -185,6 +217,9 @@ public class TestNodeManagerResync { TestNodeStatusUpdater.createContainerStatus(2, ContainerState.COMPLETE); final Container container = TestNodeStatusUpdater.getMockContainer(testCompleteContainer); + NMContainerStatus report = + createNMContainerStatus(2, ContainerState.COMPLETE); + when(container.getNMContainerStatus()).thenReturn(report); NodeManager nm = new NodeManager() { int registerCount = 0; @@ -203,7 +238,7 @@ public class TestNodeManagerResync { if (registerCount == 0) { // first register, no containers info. try { - Assert.assertEquals(0, request.getContainerStatuses() + Assert.assertEquals(0, request.getNMContainerStatuses() .size()); } catch (AssertionError error) { error.printStackTrace(); @@ -214,8 +249,8 @@ public class TestNodeManagerResync { testCompleteContainer.getContainerId(), container); } else { // second register contains the completed container info. - List<ContainerStatus> statuses = - request.getContainerStatuses(); + List<NMContainerStatus> statuses = + request.getNMContainerStatuses(); try { Assert.assertEquals(1, statuses.size()); Assert.assertEquals(testCompleteContainer.getContainerId(), @@ -302,6 +337,16 @@ public class TestNodeManagerResync { class TestNodeManager1 extends NodeManager { private int registrationCount = 0; + private boolean containersShouldBePreserved; + private ContainerId existingCid; + + public TestNodeManager1(boolean containersShouldBePreserved) { + this.containersShouldBePreserved = containersShouldBePreserved; + } + + public void setExistingContainerId(ContainerId cId) { + existingCid = cId; + } @Override protected NodeStatusUpdater createNodeStatusUpdater(Context context, @@ -333,10 +378,23 @@ public class TestNodeManagerResync { .containermanager.container.Container> containers = getNMContext().getContainers(); try { - // ensure that containers are empty before restart nodeStatusUpdater - Assert.assertTrue(containers.isEmpty()); - super.rebootNodeStatusUpdaterAndRegisterWithRM(); - syncBarrier.await(); + try { + if (containersShouldBePreserved) { + Assert.assertFalse(containers.isEmpty()); + Assert.assertTrue(containers.containsKey(existingCid)); + } else { + // ensure that containers are empty before restart nodeStatusUpdater + Assert.assertTrue(containers.isEmpty()); + } + super.rebootNodeStatusUpdaterAndRegisterWithRM(); + } + catch (AssertionError ae) { + ae.printStackTrace(); + assertionFailedInThread.set(true); + } + finally { + syncBarrier.await(); + } } catch (InterruptedException e) { } catch (BrokenBarrierException e) { } catch (AssertionError ae) { @@ -510,4 +568,17 @@ public class TestNodeManagerResync { } } }} + + public static NMContainerStatus createNMContainerStatus(int id, + ContainerState containerState) { + ApplicationId applicationId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId applicationAttemptId = + ApplicationAttemptId.newInstance(applicationId, 1); + ContainerId containerId = ContainerId.newInstance(applicationAttemptId, id); + NMContainerStatus containerReport = + NMContainerStatus.newInstance(containerId, containerState, + Resource.newInstance(1024, 1), "recover container", 0, + Priority.newInstance(10), 0); + return containerReport; + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Tue Aug 19 23:49:39 2014 @@ -109,6 +109,36 @@ public class TestNodeManagerShutdown { } @Test + public void testStateStoreRemovalOnDecommission() throws IOException { + final File recoveryDir = new File(basedir, "nm-recovery"); + nm = new TestNodeManager(); + YarnConfiguration conf = createNMConfig(); + conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); + conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath()); + + // verify state store is not removed on normal shutdown + nm.init(conf); + nm.start(); + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + nm.stop(); + nm = null; + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + + // verify state store is removed on decommissioned shutdown + nm = new TestNodeManager(); + nm.init(conf); + nm.start(); + Assert.assertTrue(recoveryDir.exists()); + Assert.assertTrue(recoveryDir.isDirectory()); + nm.getNMContext().setDecommissioned(true); + nm.stop(); + nm = null; + Assert.assertFalse(recoveryDir.exists()); + } + + @Test public void testKillContainersOnShutdown() throws IOException, YarnException { nm = new TestNodeManager(); @@ -157,7 +187,7 @@ public class TestNodeManagerShutdown { public static void startContainer(NodeManager nm, ContainerId cId, FileContext localFS, File scriptFileDir, File processStartFile) - throws IOException, YarnException { + throws IOException, YarnException { File scriptFile = createUnhaltingScriptFile(cId, scriptFileDir, processStartFile); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Aug 19 23:49:39 2014 @@ -82,6 +82,8 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -199,6 +201,7 @@ public class TestNodeStatusUpdater { Dispatcher mockDispatcher = mock(Dispatcher.class); EventHandler mockEventHandler = mock(EventHandler.class); when(mockDispatcher.getEventHandler()).thenReturn(mockEventHandler); + NMStateStoreService stateStore = new NMNullStateStoreService(); nodeStatus.setResponseId(heartBeatID++); Map<ApplicationId, List<ContainerStatus>> appToContainers = getAppToContainerStatusMap(nodeStatus.getContainersStatuses()); @@ -224,9 +227,8 @@ public class TestNodeStatusUpdater { firstContainerID, InetAddress.getByName("localhost") .getCanonicalHostName(), 1234, user, resource, currentTime + 10000, 123, "password".getBytes(), currentTime)); - Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, null, - mockMetrics, containerToken); + Container container = new ContainerImpl(conf, mockDispatcher, + stateStore, launchContext, null, mockMetrics, containerToken); this.context.getContainers().put(firstContainerID, container); } else if (heartBeatID == 2) { // Checks on the RM end @@ -255,9 +257,8 @@ public class TestNodeStatusUpdater { secondContainerID, InetAddress.getByName("localhost") .getCanonicalHostName(), 1234, user, resource, currentTime + 10000, 123, "password".getBytes(), currentTime)); - Container container = - new ContainerImpl(conf, mockDispatcher, launchContext, null, - mockMetrics, containerToken); + Container container = new ContainerImpl(conf, mockDispatcher, + stateStore, launchContext, null, mockMetrics, containerToken); this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end @@ -782,7 +783,7 @@ public class TestNodeStatusUpdater { ContainerId cId = ContainerId.newInstance(appAttemptId, 0); - nodeStatusUpdater.updateStoppedContainersInCache(cId); + nodeStatusUpdater.addCompletedContainer(cId); Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId)); long time1 = System.currentTimeMillis(); @@ -930,6 +931,7 @@ public class TestNodeStatusUpdater { Thread.sleep(500); } Assert.assertFalse(heartBeatID < 1); + Assert.assertTrue(nm.getNMContext().getDecommissioned()); // NM takes a while to reach the STOPPED state. waitCount = 0; @@ -1158,7 +1160,8 @@ public class TestNodeStatusUpdater { @Override protected NMContext createNMContext( NMContainerTokenSecretManager containerTokenSecretManager, - NMTokenSecretManagerInNM nmTokenSecretManager) { + NMTokenSecretManagerInNM nmTokenSecretManager, + NMStateStoreService store) { return new MyNMContext(containerTokenSecretManager, nmTokenSecretManager); } @@ -1267,7 +1270,8 @@ public class TestNodeStatusUpdater { public MyNMContext( NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager) { - super(containerTokenSecretManager, nmTokenSecretManager, null, null); + super(containerTokenSecretManager, nmTokenSecretManager, null, null, + new NMNullStateStoreService()); } @Override Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Aug 19 23:49:39 2014 @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; @@ -103,7 +104,8 @@ public abstract class BaseContainerManag protected static final int HTTP_PORT = 5412; protected Configuration conf = new YarnConfiguration(); protected Context context = new NMContext(new NMContainerTokenSecretManager( - conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) { + conf), new NMTokenSecretManagerInNM(), null, + new ApplicationACLsManager(conf), new NMNullStateStoreService()) { public int getHttpPort() { return HTTP_PORT; }; @@ -231,7 +233,7 @@ public abstract class BaseContainerManag protected DeletionService createDeletionService() { return new DeletionService(exec) { @Override - public void delete(String user, Path subDir, Path[] baseDirs) { + public void delete(String user, Path subDir, Path... baseDirs) { // Don't do any deletions. LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir + ", baseDirs - " + baseDirs); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Tue Aug 19 23:49:39 2014 @@ -33,8 +33,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Map; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,6 +44,7 @@ import org.apache.hadoop.service.Service import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -54,10 +53,9 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container - .Container; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.container - .ContainerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; +import org.junit.Assert; import org.junit.Test; public class TestAuxServices { @@ -192,8 +190,9 @@ public class TestAuxServices { ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1, 1); ContainerTokenIdentifier cti = new ContainerTokenIdentifier( ContainerId.newInstance(attemptId, 1), "", "", - Resource.newInstance(1, 1), 0,0,0); - Container container = new ContainerImpl(null, null, null, null, null, cti); + Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0); + Container container = new ContainerImpl(null, null, null, null, null, + null, cti); ContainerId containerId = container.getContainerId(); Resource resource = container.getResource(); event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Aug 19 23:49:39 2014 @@ -31,8 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.junit.Assert; - import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; @@ -49,6 +47,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -57,6 +56,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; @@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.security.C import org.apache.hadoop.yarn.security.NMTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceManagerConstants; import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA; @@ -80,6 +79,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -348,8 +348,7 @@ public class TestContainerManager extend GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); - int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : - ExitCode.TERMINATED.getExitCode(); + int expectedExitCode = ContainerExitStatus.KILLED_BY_APPMASTER; Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); // Assert that the process is not alive anymore @@ -799,7 +798,8 @@ public class TestContainerManager extend Resource r = BuilderUtils.newResource(1024, 1); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, nodeId.toString(), user, r, - System.currentTimeMillis() + 100000L, 123, rmIdentifier); + System.currentTimeMillis() + 100000L, 123, rmIdentifier, + Priority.newInstance(0), 0); Token containerToken = BuilderUtils .newContainerToken(nodeId, containerTokenSecretManager Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Aug 19 23:49:39 2014 @@ -32,14 +32,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -66,6 +65,7 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentMatcher; @@ -538,7 +538,8 @@ public class TestApplication { long currentTime = System.currentTimeMillis(); ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.getContainerId(), "", "", - null, currentTime + 2000, masterKey.getKeyId(), currentTime); + null, currentTime + 2000, masterKey.getKeyId(), currentTime, + Priority.newInstance(0), 0); containerTokenIdentifierMap .put(identifier.getContainerID(), identifier); context.getContainerTokenSecretManager().startContainerSuccessful( Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Tue Aug 19 23:49:39 2014 @@ -48,17 +48,17 @@ import java.util.concurrent.ConcurrentHa import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; @@ -88,7 +88,9 @@ import org.apache.hadoop.yarn.server.nod import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentMatcher; @@ -319,7 +321,7 @@ public class TestContainer { assertEquals(ContainerState.NEW, wc.c.getContainerState()); wc.killContainer(); assertEquals(ContainerState.DONE, wc.c.getContainerState()); - assertEquals(ExitCode.TERMINATED.getExitCode(), + assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, wc.c.cloneAndGetContainerStatus().getExitStatus()); assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics() .contains("KillRequest")); @@ -339,7 +341,7 @@ public class TestContainer { assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState()); wc.killContainer(); assertEquals(ContainerState.KILLING, wc.c.getContainerState()); - assertEquals(ExitCode.TERMINATED.getExitCode(), + assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, wc.c.cloneAndGetContainerStatus().getExitStatus()); assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics() .contains("KillRequest")); @@ -721,6 +723,8 @@ public class TestContainer { Context context = mock(Context.class); when(context.getApplications()).thenReturn( new ConcurrentHashMap<ApplicationId, Application>()); + NMNullStateStoreService stateStore = new NMNullStateStoreService(); + when(context.getNMStateStore()).thenReturn(stateStore); ContainerExecutor executor = mock(ContainerExecutor.class); launcher = new ContainersLauncher(context, dispatcher, executor, null, null); @@ -749,7 +753,7 @@ public class TestContainer { long currentTime = System.currentTimeMillis(); ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource, - currentTime + 10000L, 123, currentTime); + currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0); Token token = BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port), "password".getBytes(), identifier); @@ -776,7 +780,8 @@ public class TestContainer { } when(ctxt.getServiceData()).thenReturn(serviceData); - c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier); + c = new ContainerImpl(conf, dispatcher, new NMNullStateStoreService(), + ctxt, null, metrics, identifier); dispatcher.register(ContainerEventType.class, new EventHandler<ContainerEvent>() { @Override @@ -898,12 +903,14 @@ public class TestContainer { } public void killContainer() { - c.handle(new ContainerKillEvent(cId, "KillRequest")); + c.handle(new ContainerKillEvent(cId, + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, + "KillRequest")); drainDispatcherEvents(); } public void containerKilledOnRequest() { - int exitCode = ExitCode.FORCE_KILLED.getExitCode(); + int exitCode = ContainerExitStatus.KILLED_BY_RESOURCEMANAGER; String diagnosticMsg = "Container completed with exit code " + exitCode; c.handle(new ContainerExitEvent(cId, ContainerEventType.CONTAINER_KILLED_ON_REQUEST, exitCode, Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Tue Aug 19 23:49:39 2014 @@ -19,6 +19,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.junit.matchers.JUnitMatchers.containsString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -27,6 +30,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.io.PrintStream; import java.io.PrintWriter; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -36,8 +40,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.junit.Assert; - import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; @@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -62,6 +65,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; @@ -70,12 +74,13 @@ import org.apache.hadoop.yarn.event.Disp import org.apache.hadoop.yarn.event.Event; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.Apps; @@ -83,6 +88,8 @@ import org.apache.hadoop.yarn.util.Auxil import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.junit.Assert; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -475,7 +482,7 @@ public class TestContainerLaunch extends containerLaunchContext.setCommands(commands); StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, - createContainerToken(cId)); + createContainerToken(cId, Priority.newInstance(0), 0)); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = @@ -599,8 +606,7 @@ public class TestContainerLaunch extends GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); - int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : - ExitCode.TERMINATED.getExitCode(); + int expectedExitCode = ContainerExitStatus.KILLED_BY_APPMASTER; Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); // Assert that the process is not alive anymore @@ -675,7 +681,9 @@ public class TestContainerLaunch extends // set up the rest of the container List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - Token containerToken = createContainerToken(cId); + Priority priority = Priority.newInstance(10); + long createTime = 1234; + Token containerToken = createContainerToken(cId, priority, createTime); StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, @@ -694,6 +702,11 @@ public class TestContainerLaunch extends Assert.assertTrue("ProcessStartFile doesn't exist!", processStartFile.exists()); + NMContainerStatus nmContainerStatus = + containerManager.getContext().getContainers().get(cId) + .getNMContainerStatus(); + Assert.assertEquals(priority, nmContainerStatus.getPriority()); + // Now test the stop functionality. List<ContainerId> containerIds = new ArrayList<ContainerId>(); containerIds.add(cId); @@ -712,7 +725,7 @@ public class TestContainerLaunch extends ContainerStatus containerStatus = containerManager.getContainerStatuses(gcsRequest) .getContainerStatuses().get(0); - Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(), + Assert.assertEquals(ContainerExitStatus.KILLED_BY_APPMASTER, containerStatus.getExitStatus()); // Now verify the contents of the file. Script generates a message when it @@ -743,18 +756,18 @@ public class TestContainerLaunch extends } } - @Test + @Test (timeout = 30000) public void testDelayedKill() throws Exception { internalKillTest(true); } - @Test + @Test (timeout = 30000) public void testImmediateKill() throws Exception { internalKillTest(false); } @SuppressWarnings("rawtypes") - @Test + @Test (timeout = 10000) public void testCallFailureWithNullLocalizedResources() { Container container = mock(Container.class); when(container.getContainerId()).thenReturn(ContainerId.newInstance( @@ -779,11 +792,13 @@ public class TestContainerLaunch extends launch.call(); } - protected Token createContainerToken(ContainerId cId) throws InvalidToken { + protected Token createContainerToken(ContainerId cId, Priority priority, + long createTime) throws InvalidToken { Resource r = BuilderUtils.newResource(1024, 1); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, - r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER); + r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER, + priority, createTime); Token containerToken = BuilderUtils.newContainerToken( context.getNodeId(), @@ -792,4 +807,166 @@ public class TestContainerLaunch extends return containerToken; } + /** + * Test that script exists with non-zero exit code when command fails. + * @throws IOException + */ + @Test (timeout = 10000) + public void testShellScriptBuilderNonZeroExitCode() throws IOException { + ShellScriptBuilder builder = ShellScriptBuilder.create(); + builder.command(Arrays.asList(new String[] {"unknownCommand"})); + File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError"); + PrintStream writer = new PrintStream(new FileOutputStream(shellFile)); + builder.write(writer); + writer.close(); + try { + FileUtil.setExecutable(shellFile, true); + + Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor( + new String[]{shellFile.getAbsolutePath()}, tmpDir); + try { + shexc.execute(); + fail("builder shell command was expected to throw"); + } + catch(IOException e) { + // expected + System.out.println("Received an expected exception: " + e.getMessage()); + } + } + finally { + FileUtil.fullyDelete(shellFile); + } + } + + private static final String expectedMessage = "The command line has a length of"; + + @Test (timeout = 10000) + public void testWindowsShellScriptBuilderCommand() throws IOException { + String callCmd = "@call "; + + // Test is only relevant on Windows + Assume.assumeTrue(Shell.WINDOWS); + + // The tests are built on assuming 8191 max command line length + assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); + + ShellScriptBuilder builder = ShellScriptBuilder.create(); + + // Basic tests: less length, exact length, max+1 length + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat("A", 1024))); + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat( + "E", Shell.WINDOWS_MAX_SHELL_LENGHT - callCmd.length()))); + try { + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat( + "X", Shell.WINDOWS_MAX_SHELL_LENGHT -callCmd.length() + 1))); + fail("longCommand was expected to throw"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } + + // Composite tests, from parts: less, exact and + + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat("A", 1024), + org.apache.commons.lang.StringUtils.repeat("A", 1024), + org.apache.commons.lang.StringUtils.repeat("A", 1024))); + + // buildr.command joins the command parts with an extra space + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat("E", 4095), + org.apache.commons.lang.StringUtils.repeat("E", 2047), + org.apache.commons.lang.StringUtils.repeat("E", 2047 - callCmd.length()))); + + try { + builder.command(Arrays.asList( + org.apache.commons.lang.StringUtils.repeat("X", 4095), + org.apache.commons.lang.StringUtils.repeat("X", 2047), + org.apache.commons.lang.StringUtils.repeat("X", 2048 - callCmd.length()))); + fail("long commands was expected to throw"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } + } + + @Test (timeout = 10000) + public void testWindowsShellScriptBuilderEnv() throws IOException { + // Test is only relevant on Windows + Assume.assumeTrue(Shell.WINDOWS); + + // The tests are built on assuming 8191 max command line length + assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); + + ShellScriptBuilder builder = ShellScriptBuilder.create(); + + // test env + builder.env("somekey", org.apache.commons.lang.StringUtils.repeat("A", 1024)); + builder.env("somekey", org.apache.commons.lang.StringUtils.repeat( + "A", Shell.WINDOWS_MAX_SHELL_LENGHT - ("@set somekey=").length())); + try { + builder.env("somekey", org.apache.commons.lang.StringUtils.repeat( + "A", Shell.WINDOWS_MAX_SHELL_LENGHT - ("@set somekey=").length()) + 1); + fail("long env was expected to throw"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } + } + + @Test (timeout = 10000) + public void testWindowsShellScriptBuilderMkdir() throws IOException { + String mkDirCmd = "@if not exist \"\" mkdir \"\""; + + // Test is only relevant on Windows + Assume.assumeTrue(Shell.WINDOWS); + + // The tests are built on assuming 8191 max command line length + assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); + + ShellScriptBuilder builder = ShellScriptBuilder.create(); + + // test mkdir + builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024))); + builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat( + "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - mkDirCmd.length())/2))); + try { + builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat( + "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - mkDirCmd.length())/2 +1))); + fail("long mkdir was expected to throw"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } + } + + @Test (timeout = 10000) + public void testWindowsShellScriptBuilderLink() throws IOException { + // Test is only relevant on Windows + Assume.assumeTrue(Shell.WINDOWS); + + String linkCmd = "@" +Shell.WINUTILS + " symlink \"\" \"\""; + + // The tests are built on assuming 8191 max command line length + assertEquals(8191, Shell.WINDOWS_MAX_SHELL_LENGHT); + + ShellScriptBuilder builder = ShellScriptBuilder.create(); + + // test link + builder.link(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)), + new Path(org.apache.commons.lang.StringUtils.repeat("B", 1024))); + builder.link( + new Path(org.apache.commons.lang.StringUtils.repeat( + "E", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2)), + new Path(org.apache.commons.lang.StringUtils.repeat( + "F", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2))); + try { + builder.link( + new Path(org.apache.commons.lang.StringUtils.repeat( + "X", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2 + 1)), + new Path(org.apache.commons.lang.StringUtils.repeat( + "Y", (Shell.WINDOWS_MAX_SHELL_LENGHT - linkCmd.length())/2) + 1)); + fail("long link was expected to throw"); + } catch(IOException e) { + assertThat(e.getMessage(), containsString(expectedMessage)); + } + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java Tue Aug 19 23:49:39 2014 @@ -23,6 +23,7 @@ import org.junit.Assert; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalCacheDirectoryManager.Directory; import org.junit.Test; public class TestLocalCacheDirectoryManager { @@ -73,7 +74,7 @@ public class TestLocalCacheDirectoryMana conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1"); Exception e = null; ResourceLocalizationService service = - new ResourceLocalizationService(null, null, null, null); + new ResourceLocalizationService(null, null, null, null, null); try { service.init(conf); } catch (Exception e1) { @@ -109,4 +110,49 @@ public class TestLocalCacheDirectoryMana // first sub directory Assert.assertEquals(firstSubDir, dir.getRelativePathForLocalization()); } + + @Test + public void testDirectoryConversion() { + for (int i = 0; i < 10000; ++i) { + String path = Directory.getRelativePath(i); + Assert.assertEquals("Incorrect conversion for " + i, i, + Directory.getDirectoryNumber(path)); + } + } + + @Test + public void testIncrementFileCountForPath() { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, + LocalCacheDirectoryManager.DIRECTORIES_PER_LEVEL + 2); + LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf); + final String rootPath = ""; + mgr.incrementFileCountForPath(rootPath); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + Assert.assertFalse("root dir should be full", + rootPath.equals(mgr.getRelativePathForLocalization())); + // finish filling the other directory + mgr.getRelativePathForLocalization(); + // free up space in the root dir + mgr.decrementFileCountForPath(rootPath); + mgr.decrementFileCountForPath(rootPath); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + Assert.assertEquals(rootPath, mgr.getRelativePathForLocalization()); + String otherDir = mgr.getRelativePathForLocalization(); + Assert.assertFalse("root dir should be full", otherDir.equals(rootPath)); + + final String deepDir0 = "d/e/e/p/0"; + final String deepDir1 = "d/e/e/p/1"; + final String deepDir2 = "d/e/e/p/2"; + final String deepDir3 = "d/e/e/p/3"; + mgr.incrementFileCountForPath(deepDir0); + Assert.assertEquals(otherDir, mgr.getRelativePathForLocalization()); + Assert.assertEquals(deepDir0, mgr.getRelativePathForLocalization()); + Assert.assertEquals("total dir count incorrect after increment", + deepDir1, mgr.getRelativePathForLocalization()); + mgr.incrementFileCountForPath(deepDir2); + mgr.incrementFileCountForPath(deepDir1); + mgr.incrementFileCountForPath(deepDir2); + Assert.assertEquals(deepDir3, mgr.getRelativePathForLocalization()); + } }
