http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index 5f9b883..c1df562 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.nodemanager; import static org.mockito.Matchers.argThat; -import static org.mockito.Matchers.isNull; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -28,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -58,19 +58,17 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.NMTokenIdentifier; -import org.apache.hadoop.yarn.server.nodemanager.DeletionService.FileDeletionTask; import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.mockito.ArgumentMatcher; public class TestNodeManagerReboot { @@ -195,19 +193,18 @@ public class TestNodeManagerReboot { // restart the NodeManager restartNM(MAX_TRIES); checkNumOfLocalDirs(); - - verify(delService, times(1)).delete( - (String) isNull(), - argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR - + "_DEL_"))); - verify(delService, times(1)).delete((String) isNull(), - argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_"))); - verify(delService, times(1)).scheduleFileDeletionTask( - argThat(new FileDeletionInclude(user, null, - new String[] { destinationFile }))); - verify(delService, times(1)).scheduleFileDeletionTask( - argThat(new FileDeletionInclude(null, ContainerLocalizer.USERCACHE - + "_DEL_", new String[] {}))); + + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, + new Path(ResourceLocalizationService.NM_PRIVATE_DIR + "_DEL_"), null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, new Path(ContainerLocalizer.FILECACHE + "_DEL_"), + null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, null, Arrays.asList(new Path(destinationFile))))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, new Path(ContainerLocalizer.USERCACHE + "_DEL_"), + new ArrayList<Path>()))); // restart the NodeManager again // this time usercache directory should be empty @@ -329,72 +326,4 @@ public class TestNodeManagerReboot { return conf; } } - - class PathInclude extends ArgumentMatcher<Path> { - - final String part; - - PathInclude(String part) { - this.part = part; - } - - @Override - public boolean matches(Object o) { - return ((Path) o).getName().indexOf(part) != -1; - } - } - - class FileDeletionInclude extends ArgumentMatcher<FileDeletionTask> { - final String user; - final String subDirIncludes; - final String[] baseDirIncludes; - - public FileDeletionInclude(String user, String subDirIncludes, - String [] baseDirIncludes) { - this.user = user; - this.subDirIncludes = subDirIncludes; - this.baseDirIncludes = baseDirIncludes; - } - - @Override - public boolean matches(Object o) { - FileDeletionTask fd = (FileDeletionTask)o; - if (fd.getUser() == null && user != null) { - return false; - } else if (fd.getUser() != null && user == null) { - return false; - } else if (fd.getUser() != null && user != null) { - return fd.getUser().equals(user); - } - if (!comparePaths(fd.getSubDir(), subDirIncludes)) { - return false; - } - if (baseDirIncludes == null && fd.getBaseDirs() != null) { - return false; - } else if (baseDirIncludes != null && fd.getBaseDirs() == null ) { - return false; - } else if (baseDirIncludes != null && fd.getBaseDirs() != null) { - if (baseDirIncludes.length != fd.getBaseDirs().size()) { - return false; - } - for (int i =0 ; i < baseDirIncludes.length; i++) { - if (!comparePaths(fd.getBaseDirs().get(i), baseDirIncludes[i])) { - return false; - } - } - } - return true; - } - - public boolean comparePaths(Path p1, String p2) { - if (p1 == null && p2 != null){ - return false; - } else if (p1 != null && p2 == null) { - return false; - } else if (p1 != null && p2 != null ){ - return p1.toUri().getPath().contains(p2.toString()); - } - return true; - } - } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java new file mode 100644 index 0000000..69e01bc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/TestNMProtoUtils.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; +import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.recovery.DeletionTaskRecoveryInfo; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTaskType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +/** + * Test conversion to {@link DeletionTask}. + */ +public class TestNMProtoUtils { + + @Test + public void testConvertProtoToDeletionTask() throws Exception { + DeletionService deletionService = mock(DeletionService.class); + DeletionServiceDeleteTaskProto.Builder protoBuilder = + DeletionServiceDeleteTaskProto.newBuilder(); + int id = 0; + protoBuilder.setId(id); + DeletionServiceDeleteTaskProto proto = protoBuilder.build(); + DeletionTask deletionTask = + NMProtoUtils.convertProtoToDeletionTask(proto, deletionService); + assertEquals(DeletionTaskType.FILE, deletionTask.getDeletionTaskType()); + assertEquals(id, deletionTask.getTaskId()); + } + + @Test + public void testConvertProtoToFileDeletionTask() throws Exception { + DeletionService deletionService = mock(DeletionService.class); + int id = 0; + String user = "user"; + Path subdir = new Path("subdir"); + Path basedir = new Path("basedir"); + DeletionServiceDeleteTaskProto.Builder protoBuilder = + DeletionServiceDeleteTaskProto.newBuilder(); + protoBuilder + .setId(id) + .setUser("user") + .setSubdir(subdir.getName()) + .addBasedirs(basedir.getName()); + DeletionServiceDeleteTaskProto proto = protoBuilder.build(); + DeletionTask deletionTask = + NMProtoUtils.convertProtoToFileDeletionTask(proto, deletionService, id); + assertEquals(DeletionTaskType.FILE.name(), + deletionTask.getDeletionTaskType().name()); + assertEquals(id, deletionTask.getTaskId()); + assertEquals(subdir, ((FileDeletionTask) deletionTask).getSubDir()); + assertEquals(basedir, + ((FileDeletionTask) deletionTask).getBaseDirs().get(0)); + } + + @Test + public void testConvertProtoToDeletionTaskRecoveryInfo() throws Exception { + long delTime = System.currentTimeMillis(); + List<Integer> successorTaskIds = Arrays.asList(1); + DeletionTask deletionTask = mock(DeletionTask.class); + DeletionTaskRecoveryInfo info = + new DeletionTaskRecoveryInfo(deletionTask, successorTaskIds, delTime); + assertEquals(deletionTask, info.getTask()); + assertEquals(successorTaskIds, info.getSuccessorTaskIds()); + assertEquals(delTime, info.getDeletionTimestamp()); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index 2991c0c..7980a80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask; import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; @@ -260,10 +261,10 @@ public abstract class BaseContainerManagerTest { protected DeletionService createDeletionService() { return new DeletionService(exec) { @Override - public void delete(String user, Path subDir, Path... baseDirs) { + public void delete(DeletionTask deletionTask) { // Don't do any deletions. - LOG.info("Psuedo delete: user - " + user + ", subDir - " + subDir - + ", baseDirs - " + Arrays.asList(baseDirs)); + LOG.info("Psuedo delete: user - " + user + + ", type - " + deletionTask.getDeletionTaskType()); }; }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java new file mode 100644 index 0000000..faad456 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/FileDeletionMatcher.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.mockito.ArgumentMatcher; + +import java.util.List; + +/** + * ArgumentMatcher to check the arguments of the {@link FileDeletionTask}. + */ +public class FileDeletionMatcher extends ArgumentMatcher<FileDeletionTask> { + + private final DeletionService delService; + private final String user; + private final Path subDirIncludes; + private final List<Path> baseDirIncludes; + + public FileDeletionMatcher(DeletionService delService, String user, + Path subDirIncludes, List<Path> baseDirIncludes) { + this.delService = delService; + this.user = user; + this.subDirIncludes = subDirIncludes; + this.baseDirIncludes = baseDirIncludes; + } + + @Override + public boolean matches(Object o) { + FileDeletionTask fd = (FileDeletionTask) o; + if (fd.getUser() == null && user != null) { + return false; + } else if (fd.getUser() != null && user == null) { + return false; + } else if (fd.getUser() != null && user != null) { + return fd.getUser().equals(user); + } + if (!comparePaths(fd.getSubDir(), subDirIncludes.getName())) { + return false; + } + if (baseDirIncludes == null && fd.getBaseDirs() != null) { + return false; + } else if (baseDirIncludes != null && fd.getBaseDirs() == null) { + return false; + } else if (baseDirIncludes != null && fd.getBaseDirs() != null) { + if (baseDirIncludes.size() != fd.getBaseDirs().size()) { + return false; + } + for (int i = 0; i < baseDirIncludes.size(); i++) { + if (!comparePaths(fd.getBaseDirs().get(i), + baseDirIncludes.get(i).getName())) { + return false; + } + } + } + return true; + } + + public boolean comparePaths(Path p1, String p2) { + if (p1 == null && p2 != null) { + return false; + } else if (p1 != null && p2 == null) { + return false; + } else if (p1 != null && p2 != null) { + return p1.toUri().getPath().contains(p2.toString()); + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java new file mode 100644 index 0000000..fd2e4fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/TestFileDeletionTask.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; +import org.apache.hadoop.yarn.server.nodemanager.DeletionService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * Test the attributes of the {@link FileDeletionTask} class. + */ +public class TestFileDeletionTask { + + private static final int ID = 0; + private static final String USER = "user"; + private static final Path SUBDIR = new Path("subdir"); + private static final Path BASEDIR = new Path("basedir"); + + private List<Path> baseDirs = new ArrayList<>(); + private DeletionService deletionService; + private FileDeletionTask deletionTask; + + @Before + public void setUp() throws Exception { + deletionService = mock(DeletionService.class); + baseDirs.add(BASEDIR); + deletionTask = new FileDeletionTask(ID, deletionService, USER, SUBDIR, + baseDirs); + } + + @After + public void tearDown() throws Exception { + baseDirs.clear(); + } + + @Test + public void testGetUser() throws Exception { + assertEquals(USER, deletionTask.getUser()); + } + + @Test + public void testGetSubDir() throws Exception { + assertEquals(SUBDIR, deletionTask.getSubDir()); + } + + @Test + public void testGetBaseDirs() throws Exception { + assertEquals(1, deletionTask.getBaseDirs().size()); + assertEquals(baseDirs, deletionTask.getBaseDirs()); + } + + @Test + public void testConvertDeletionTaskToProto() throws Exception { + DeletionServiceDeleteTaskProto proto = + deletionTask.convertDeletionTaskToProto(); + assertEquals(ID, proto.getId()); + assertEquals(USER, proto.getUser()); + assertEquals(SUBDIR, new Path(proto.getSubdir())); + assertEquals(BASEDIR, new Path(proto.getBasedirs(0))); + assertEquals(1, proto.getBasedirsCount()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java index 2874acb..6cab593 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; import static org.mockito.Matchers.any; +import static org.mockito.Mockito.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.mock; @@ -54,6 +55,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent; @@ -823,7 +825,8 @@ public class TestLocalResourcesTrackerImpl { Path rPath = tracker.getPathForLocalization(req1, base_path, delService); Assert.assertFalse(lfs.util().exists(rPath)); - verify(delService, times(1)).delete(eq(user), eq(conflictPath)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, conflictPath, null))); } finally { lfs.delete(base_path, true); if (dispatcher != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 89cbeb4..d863c6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -31,7 +31,6 @@ import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; -import static org.mockito.Matchers.isNull; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -68,6 +67,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.Options; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; import org.junit.Assert; import org.apache.commons.io.FileUtils; @@ -1066,7 +1066,8 @@ public class TestResourceLocalizationService { verify(containerBus, times(3)).handle(argThat(matchesContainerLoc)); // Verify deletion of localization token. - verify(delService).delete((String)isNull(), eq(localizationTokenPath)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, localizationTokenPath, null))); } finally { spyService.stop(); dispatcher.stop(); @@ -1340,8 +1341,8 @@ public class TestResourceLocalizationService { Thread.sleep(50); } // Verify if downloading resources were submitted for deletion. - verify(delService).delete(eq(user), (Path) eq(null), - argThat(new DownloadingPathsMatcher(paths))); + verify(delService, times(2)).delete(argThat(new FileDeletionMatcher( + delService, user, null, new ArrayList<>(paths)))); LocalResourcesTracker tracker = spyService.getLocalResourcesTracker( LocalResourceVisibility.PRIVATE, "user0", appId); @@ -2753,15 +2754,19 @@ public class TestResourceLocalizationService { for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 2) { try { - verify(delService).delete(user, containerLocalDirs.get(i)); - verify(delService).delete(null, nmLocalContainerDirs.get(i)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, containerLocalDirs.get(i), null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, nmLocalContainerDirs.get(i), null))); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { - verify(delService).delete(user, containerLocalDirs.get(i)); - verify(delService).delete(null, nmLocalContainerDirs.get(i)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, containerLocalDirs.get(i), null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, nmLocalContainerDirs.get(i), null))); } } @@ -2802,15 +2807,19 @@ public class TestResourceLocalizationService { for (int i = 0; i < containerLocalDirs.size(); ++i) { if (i == 3) { try { - verify(delService).delete(user, containerLocalDirs.get(i)); - verify(delService).delete(null, nmLocalContainerDirs.get(i)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, containerLocalDirs.get(i), null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, nmLocalContainerDirs.get(i), null))); Assert.fail("deletion attempts for invalid dirs"); } catch (Throwable e) { continue; } } else { - verify(delService).delete(user, appLocalDirs.get(i)); - verify(delService).delete(null, nmLocalAppDirs.get(i)); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, containerLocalDirs.get(i), null))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, null, nmLocalContainerDirs.get(i), null))); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java index 097146b..b4bd9d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestAppLogAggregatorImpl.java @@ -42,17 +42,16 @@ import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeManager; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; -import org.mockito.Matchers; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -311,16 +310,18 @@ public class TestAppLogAggregatorImpl { @Override public Void answer(InvocationOnMock invocationOnMock) throws Throwable { Set<String> paths = new HashSet<>(); - Object[] args = invocationOnMock.getArguments(); - for(int i = 2; i < args.length; i++) { - Path path = (Path) args[i]; - paths.add(path.toUri().getRawPath()); + Object[] tasks = invocationOnMock.getArguments(); + for(int i = 0; i < tasks.length; i++) { + FileDeletionTask task = (FileDeletionTask) tasks[i]; + for (Path path: task.getBaseDirs()) { + paths.add(path.toUri().getRawPath()); + } } verifyFilesToDelete(expectedPathsForDeletion, paths); return null; } }).doNothing().when(deletionServiceWithExpectedFiles).delete( - any(String.class), any(Path.class), Matchers.<Path>anyVararg()); + any(FileDeletionTask.class)); return deletionServiceWithExpectedFiles; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index bc1b4b0..37fe77a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -120,6 +120,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.TestNonAggregatingLogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; @@ -218,8 +220,10 @@ public class TestLogAggregationService extends BaseContainerManagerTest { // ensure filesystems were closed verify(logAggregationService).closeFileSystems( any(UserGroupInformation.class)); - verify(delSrvc).delete(eq(user), eq((Path) null), - eq(new Path(app1LogDir.getAbsolutePath()))); + List<Path> dirList = new ArrayList<>(); + dirList.add(new Path(app1LogDir.toURI())); + verify(delSrvc, times(2)).delete(argThat(new FileDeletionMatcher( + delSrvc, user, null, dirList))); String containerIdStr = container11.toString(); File containerLogDir = new File(app1LogDir, containerIdStr); @@ -333,7 +337,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest { logAggregationService.stop(); delSrvc.stop(); // Aggregated logs should not be deleted if not uploaded. - verify(delSrvc, times(0)).delete(user, null); + FileDeletionTask deletionTask = new FileDeletionTask(delSrvc, user, null, + null); + verify(delSrvc, times(0)).delete(deletionTask); } @Test @@ -815,8 +821,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest { assertEquals(0, logAggregationService.getNumAggregators()); // local log dir shouldn't be deleted given log aggregation cannot // continue due to aggregated log dir creation failure on remoteFS. - verify(spyDelSrvc, never()).delete(eq(user), any(Path.class), - Mockito.<Path>anyVararg()); + FileDeletionTask deletionTask = new FileDeletionTask(spyDelSrvc, user, + null, null); + verify(spyDelSrvc, never()).delete(deletionTask); verify(logAggregationService).closeFileSystems( any(UserGroupInformation.class)); // make sure local log dir is not deleted in case log aggregation http://git-wip-us.apache.org/repos/asf/hadoop/blob/547f18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java index ec3757e..7a4ea88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java @@ -22,12 +22,14 @@ import static org.junit.Assert.assertFalse; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import java.io.File; @@ -36,6 +38,7 @@ import java.io.IOException; import java.io.NotSerializableException; import java.io.ObjectInputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.HashMap; @@ -66,6 +69,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionMatcher; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; @@ -531,8 +535,8 @@ public class TestNonAggregatingLogHandler { boolean matched = false; while (!matched && System.currentTimeMillis() < verifyStartTime + timeout) { try { - verify(delService).delete(eq(user), (Path) eq(null), - Mockito.argThat(new DeletePathsMatcher(matchPaths))); + verify(delService, times(1)).delete(argThat(new FileDeletionMatcher( + delService, user, null, Arrays.asList(matchPaths)))); matched = true; } catch (WantedButNotInvoked e) { notInvokedException = e; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
