http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a5a443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index ca0b32a..ece8d9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -31,6 +31,15 @@ import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecord; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.records.AuxServiceRecords; +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +96,11 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task. import org.junit.Assert; import org.junit.Test; +/** + * Test for auxiliary services. Parameter 0 tests the Configuration-based aux + * services and parameter 1 tests manifest-based aux services. + */ +@RunWith(value = Parameterized.class) public class TestAuxServices { private static final Logger LOG = LoggerFactory.getLogger(TestAuxServices.class); @@ -99,6 +113,36 @@ public class TestAuxServices { private final static Context MOCK_CONTEXT = mock(Context.class); private final static DeletionService MOCK_DEL_SERVICE = mock( DeletionService.class); + private final Boolean useManifest; + private File rootDir = GenericTestUtils.getTestDir(getClass() + .getSimpleName()); + private File manifest = new File(rootDir, "manifest.txt"); + private ObjectMapper mapper = new ObjectMapper(); + + @Parameterized.Parameters + public static Collection<Boolean> getParams() { + return Arrays.asList(false, true); + } + + @Before + public void setup() { + if (!rootDir.exists()) { + rootDir.mkdirs(); + } + } + + @After + public void cleanup() { + if (useManifest) { + manifest.delete(); + } + rootDir.delete(); + } + + public TestAuxServices(Boolean useManifest) { + this.useManifest = useManifest; + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + } static class LightService extends AuxiliaryService implements Service { @@ -204,15 +248,27 @@ public class TestAuxServices { } } + private void writeManifestFile(AuxServiceRecords services, Configuration + conf) throws IOException { + conf.set(YarnConfiguration.NM_AUX_SERVICES_MANIFEST, manifest + .getAbsolutePath()); + mapper.writeValue(manifest, services); + } + @SuppressWarnings("resource") @Test public void testRemoteAuxServiceClassPath() throws Exception { Configuration conf = new YarnConfiguration(); FileSystem fs = FileSystem.get(conf); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[] {"ServiceC"}); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, - "ServiceC"), ServiceC.class, Service.class); + AuxServiceRecord serviceC = + AuxServices.newAuxService("ServiceC", ServiceC.class.getName()); + AuxServiceRecords services = new AuxServiceRecords().serviceList(serviceC); + if (!useManifest) { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[]{"ServiceC"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, + "ServiceC"), ServiceC.class, Service.class); + } Context mockContext2 = mock(Context.class); LocalDirsHandlerService mockDirsHandler = mock( @@ -223,11 +279,8 @@ public class TestAuxServices { rootAuxServiceDirPath); when(mockContext2.getLocalDirsHandler()).thenReturn(mockDirsHandler); - File rootDir = GenericTestUtils.getTestDir(getClass() - .getSimpleName()); - if (!rootDir.exists()) { - rootDir.mkdirs(); - } + DeletionService mockDelService2 = mock(DeletionService.class); + AuxServices aux = null; File testJar = null; try { @@ -243,11 +296,16 @@ public class TestAuxServices { perms.add(PosixFilePermission.GROUP_WRITE); Files.setPosixFilePermissions(Paths.get(testJar.getAbsolutePath()), perms); - conf.set(String.format( - YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, "ServiceC"), - testJar.getAbsolutePath()); + if (useManifest) { + AuxServices.setClasspath(serviceC, testJar.getAbsolutePath()); + writeManifestFile(services, conf); + } else { + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, "ServiceC"), + testJar.getAbsolutePath()); + } aux = new AuxServices(MOCK_AUX_PATH_HANDLER, - mockContext2, MOCK_DEL_SERVICE); + mockContext2, mockDelService2); aux.init(conf); Assert.fail("The permission of the jar is wrong." + "Should throw out exception."); @@ -260,11 +318,16 @@ public class TestAuxServices { testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir, "test-runjar.jar", 2048, ServiceC.class.getName()); - conf.set(String.format( - YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, "ServiceC"), - testJar.getAbsolutePath()); + if (useManifest) { + AuxServices.setClasspath(serviceC, testJar.getAbsolutePath()); + writeManifestFile(services, conf); + } else { + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, "ServiceC"), + testJar.getAbsolutePath()); + } aux = new AuxServices(MOCK_AUX_PATH_HANDLER, - mockContext2, MOCK_DEL_SERVICE); + mockContext2, mockDelService2); aux.init(conf); aux.start(); Map<String, ByteBuffer> meta = aux.getMetaData(); @@ -281,7 +344,7 @@ public class TestAuxServices { // initialize the same auxservice again, and make sure that we did not // re-download the jar from remote directory. aux = new AuxServices(MOCK_AUX_PATH_HANDLER, - mockContext2, MOCK_DEL_SERVICE); + mockContext2, mockDelService2); aux.init(conf); aux.start(); meta = aux.getMetaData(); @@ -290,7 +353,7 @@ public class TestAuxServices { auxName = i.getKey(); } Assert.assertEquals("ServiceC", auxName); - verify(MOCK_DEL_SERVICE, times(0)).delete(any(FileDeletionTask.class)); + verify(mockDelService2, times(0)).delete(any(FileDeletionTask.class)); status = fs.listStatus(rootAuxServiceDirPath); Assert.assertTrue(status.length == 1); aux.serviceStop(); @@ -301,22 +364,17 @@ public class TestAuxServices { FileTime fileTime = FileTime.fromMillis(time); Files.setLastModifiedTime(Paths.get(testJar.getAbsolutePath()), fileTime); - conf.set( - String.format(YarnConfiguration.NM_AUX_SERVICE_REMOTE_CLASSPATH, - "ServiceC"), - testJar.getAbsolutePath()); aux = new AuxServices(MOCK_AUX_PATH_HANDLER, - mockContext2, MOCK_DEL_SERVICE); + mockContext2, mockDelService2); aux.init(conf); aux.start(); - verify(MOCK_DEL_SERVICE, times(1)).delete(any(FileDeletionTask.class)); + verify(mockDelService2, times(1)).delete(any(FileDeletionTask.class)); status = fs.listStatus(rootAuxServiceDirPath); Assert.assertTrue(status.length == 2); aux.serviceStop(); } finally { if (testJar != null) { testJar.delete(); - rootDir.delete(); } if (fs.exists(new Path(root))) { fs.delete(new Path(root), true); @@ -333,10 +391,17 @@ public class TestAuxServices { public void testCustomizedAuxServiceClassPath() throws Exception { // verify that we can load AuxService Class from default Class path Configuration conf = new YarnConfiguration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[] {"ServiceC"}); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, - "ServiceC"), ServiceC.class, Service.class); + AuxServiceRecord serviceC = + AuxServices.newAuxService("ServiceC", ServiceC.class.getName()); + AuxServiceRecords services = new AuxServiceRecords().serviceList(serviceC); + if (useManifest) { + writeManifestFile(services, conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[]{"ServiceC"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, + "ServiceC"), ServiceC.class, Service.class); + } @SuppressWarnings("resource") AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); @@ -358,31 +423,41 @@ public class TestAuxServices { // create a new jar file, and configure it as customized class path // for this AuxService, and make sure that we could load the class // from this configured customized class path - File rootDir = GenericTestUtils.getTestDir(getClass() - .getSimpleName()); - if (!rootDir.exists()) { - rootDir.mkdirs(); - } File testJar = null; try { testJar = JarFinder.makeClassLoaderTestJar(this.getClass(), rootDir, - "test-runjar.jar", 2048, ServiceC.class.getName()); + "test-runjar.jar", 2048, ServiceC.class.getName(), LightService + .class.getName()); conf = new YarnConfiguration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[] {"ServiceC"}); - conf.set(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "ServiceC"), - ServiceC.class.getName()); - conf.set(String.format( - YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, "ServiceC"), - testJar.getAbsolutePath()); // remove "-org.apache.hadoop." from system classes String systemClasses = "-org.apache.hadoop." + "," + - ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; - conf.set(String.format( - YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES, - "ServiceC"), systemClasses); + ApplicationClassLoader.SYSTEM_CLASSES_DEFAULT; + if (useManifest) { + AuxServices.setClasspath(serviceC, testJar.getAbsolutePath()); + AuxServices.setSystemClasses(serviceC, systemClasses); + writeManifestFile(services, conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[]{"ServiceC"}); + conf.set(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, + "ServiceC"), ServiceC.class.getName()); + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICES_CLASSPATH, "ServiceC"), + testJar.getAbsolutePath()); + conf.set(String.format( + YarnConfiguration.NM_AUX_SERVICES_SYSTEM_CLASSES, + "ServiceC"), systemClasses); + } + Context mockContext2 = mock(Context.class); + LocalDirsHandlerService mockDirsHandler = mock( + LocalDirsHandlerService.class); + String root = "target/LocalDir"; + Path rootAuxServiceDirPath = new Path(root, "nmAuxService"); + when(mockDirsHandler.getLocalPathForWrite(anyString())).thenReturn( + rootAuxServiceDirPath); + when(mockContext2.getLocalDirsHandler()).thenReturn(mockDirsHandler); aux = new AuxServices(MOCK_AUX_PATH_HANDLER, - MOCK_CONTEXT, MOCK_DEL_SERVICE); + mockContext2, MOCK_DEL_SERVICE); aux.init(conf); aux.start(); meta = aux.getMetaData(); @@ -405,21 +480,32 @@ public class TestAuxServices { } finally { if (testJar != null) { testJar.delete(); - rootDir.delete(); } } } @Test - public void testAuxEventDispatch() { + public void testAuxEventDispatch() throws IOException { Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" }); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), - ServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), - ServiceB.class, Service.class); - conf.setInt("A.expected.init", 1); - conf.setInt("B.expected.stop", 1); + if (useManifest) { + AuxServiceRecord serviceA = + AuxServices.newAuxService("Asrv", ServiceA.class.getName()); + serviceA.getConfiguration().setProperty("A.expected.init", "1"); + AuxServiceRecord serviceB = + AuxServices.newAuxService("Bsrv", ServiceB.class.getName()); + serviceB.getConfiguration().setProperty("B.expected.stop", "1"); + writeManifestFile(new AuxServiceRecords().serviceList(serviceA, + serviceB), conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[]{"Asrv", + "Bsrv"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), + ServiceA.class, Service.class); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), + ServiceB.class, Service.class); + conf.setInt("A.expected.init", 1); + conf.setInt("B.expected.stop", 1); + } final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); aux.init(conf); @@ -477,14 +563,36 @@ public class TestAuxServices { } } - @Test - public void testAuxServices() { + private Configuration getABConf() throws + IOException { + return getABConf("Asrv", "Bsrv", ServiceA.class, ServiceB.class); + } + + private Configuration getABConf(String aName, String bName, + Class<? extends Service> aClass, Class<? extends Service> bClass) throws + IOException { Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" }); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), - ServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), - ServiceB.class, Service.class); + if (useManifest) { + AuxServiceRecord serviceA = + AuxServices.newAuxService(aName, aClass.getName()); + AuxServiceRecord serviceB = + AuxServices.newAuxService(bName, bClass.getName()); + writeManifestFile(new AuxServiceRecords().serviceList(serviceA, serviceB), + conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[]{aName, + bName}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, aName), + aClass, Service.class); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, bName), + bClass, Service.class); + } + return conf; + } + + @Test + public void testAuxServices() throws IOException { + Configuration conf = getABConf(); final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); aux.init(conf); @@ -510,15 +618,9 @@ public class TestAuxServices { } } - @Test - public void testAuxServicesMeta() { - Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" }); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), - ServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), - ServiceB.class, Service.class); + public void testAuxServicesMeta() throws IOException { + Configuration conf = getABConf(); final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); aux.init(conf); @@ -547,16 +649,10 @@ public class TestAuxServices { } } - - @Test - public void testAuxUnexpectedStop() { - Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" }); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), - ServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), - ServiceB.class, Service.class); + public void testAuxUnexpectedStop() throws IOException { + // AuxServices no longer expected to stop when services stop + Configuration conf = getABConf(); final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); aux.init(conf); @@ -564,21 +660,17 @@ public class TestAuxServices { Service s = aux.getServices().iterator().next(); s.stop(); - assertEquals("Auxiliary service stopped, but AuxService unaffected.", - STOPPED, aux.getServiceState()); - assertTrue(aux.getServices().isEmpty()); + assertEquals("Auxiliary service stop caused AuxServices stop", + STARTED, aux.getServiceState()); + assertEquals(2, aux.getServices().size()); } @Test - public void testValidAuxServiceName() { + public void testValidAuxServiceName() throws IOException { + Configuration conf = getABConf("Asrv1", "Bsrv_2", ServiceA.class, + ServiceB.class); final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); - Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"Asrv1", "Bsrv_2"}); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv1"), - ServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv_2"), - ServiceB.class, Service.class); try { aux.init(conf); } catch (Exception ex) { @@ -588,30 +680,33 @@ public class TestAuxServices { //Test bad auxService Name final AuxServices aux1 = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] {"1Asrv1"}); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "1Asrv1"), - ServiceA.class, Service.class); + if (useManifest) { + AuxServiceRecord serviceA = + AuxServices.newAuxService("1Asrv1", ServiceA.class.getName()); + writeManifestFile(new AuxServiceRecords().serviceList(serviceA), conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] + {"1Asrv1"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, + "1Asrv1"), ServiceA.class, Service.class); + } try { aux1.init(conf); Assert.fail("Should receive the exception."); } catch (Exception ex) { - assertTrue(ex.getMessage().contains("The ServiceName: 1Asrv1 set in " + - "yarn.nodemanager.aux-services is invalid.The valid service name " + - "should only contain a-zA-Z0-9_ and can not start with numbers")); + assertTrue("Wrong message: " + ex.getMessage(), + ex.getMessage().contains("The auxiliary service name: 1Asrv1 is " + + "invalid. The valid service name should only contain a-zA-Z0-9_" + + " and cannot start with numbers.")); } } @Test public void testAuxServiceRecoverySetup() throws IOException { - Configuration conf = new YarnConfiguration(); + Configuration conf = getABConf("Asrv", "Bsrv", RecoverableServiceA.class, + RecoverableServiceB.class); conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); conf.set(YarnConfiguration.NM_RECOVERY_DIR, TEST_DIR.toString()); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[] { "Asrv", "Bsrv" }); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"), - RecoverableServiceA.class, Service.class); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"), - RecoverableServiceB.class, Service.class); try { final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); @@ -708,20 +803,33 @@ public class TestAuxServices { } @Test - public void testAuxServicesConfChange() { + public void testAuxServicesConfChange() throws IOException { Configuration conf = new Configuration(); - conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, - new String[]{"ConfChangeAuxService"}); - conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, - "ConfChangeAuxService"), ConfChangeAuxService.class, Service.class); + if (useManifest) { + AuxServiceRecord service = + AuxServices.newAuxService("ConfChangeAuxService", + ConfChangeAuxService.class.getName()); + service.getConfiguration().setProperty("dummyConfig", "testValue"); + writeManifestFile(new AuxServiceRecords().serviceList(service), conf); + } else { + conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, + new String[]{"ConfChangeAuxService"}); + conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, + "ConfChangeAuxService"), ConfChangeAuxService.class, Service.class); + conf.set("dummyConfig", "testValue"); + } AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER, MOCK_CONTEXT, MOCK_DEL_SERVICE); - conf.set("dummyConfig", "testValue"); aux.init(conf); aux.start(); for (AuxiliaryService s : aux.getServices()) { assertEquals(STARTED, s.getServiceState()); - assertEquals(conf.get("dummyConfig"), "testValue"); + if (useManifest) { + assertNull(conf.get("dummyConfig")); + } else { + assertEquals("testValue", conf.get("dummyConfig")); + } + assertEquals("changedTestValue", s.getConfig().get("dummyConfig")); } aux.stop();
--------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
