YARN-8880. Add configurations for pluggable plugin framework. Contributed by Zhankun Tang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c72d7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c72d7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c72d7b Branch: refs/heads/HDDS-4 Commit: f8c72d7b3acca8285bbc3024f491c4586805be1e Parents: c96cbe8 Author: Weiwei Yang <w...@apache.org> Authored: Thu Nov 8 12:23:00 2018 +0800 Committer: Weiwei Yang <w...@apache.org> Committed: Thu Nov 8 12:23:00 2018 +0800 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 24 ++++- .../src/main/resources/yarn-default.xml | 19 ++++ .../resourceplugin/ResourcePluginManager.java | 34 +++++- .../TestResourcePluginManager.java | 107 +++++++++++++++++-- 4 files changed, 170 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index ce38d27..e88d594 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1606,6 +1606,28 @@ public class YarnConfiguration extends Configuration { NM_PREFIX + "resource-plugins"; /** + * This setting controls if pluggable device plugin framework is enabled. + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = + NM_PREFIX + "pluggable-device-framework.enabled"; + + /** + * The pluggable device plugin framework is disabled by default + * */ + @Private + public static final boolean DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED = + false; + + /** + * This setting contains vendor plugin class names for + * device plugin framework to load. Split by comma + * */ + @Private + public static final String NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES = + NM_PREFIX + "pluggable-device-framework.device-classes"; + + /** * Prefix for gpu configurations. Work in progress: This configuration * parameter may be changed/removed in the future. */ @@ -1647,7 +1669,7 @@ public class YarnConfiguration extends Configuration { NVIDIA_DOCKER_V1; /** - * This setting controls end point of nvidia-docker-v1 plugin + * This setting controls end point of nvidia-docker-v1 plugin. */ @Private public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 1360e73..f5493bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3772,6 +3772,25 @@ <property> <description> + This setting controls if pluggable device framework is enabled. + Disabled by default + </description> + <name>yarn.nodemanager.pluggable-device-framework.enabled</name> + <value>false</value> + </property> + + <property> + <description> + Configure vendor device plugin class name here. Comma separated. + The class must be found in CLASSPATH. The pluggable device framework will + load these classes. + </description> + <name>yarn.nodemanager.pluggable-device-framework.device-classes</name> + <value></value> + </property> + + <property> + <description> When yarn.nodemanager.resource.gpu.allowed-gpu-devices=auto specified, YARN NodeManager needs to run GPU discovery binary (now only support nvidia-smi) to get GPU-related information. http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java index f28aad2..e930d96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePluginManager.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaResourcePlugin; import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuResourcePlugin; @@ -52,11 +53,10 @@ public class ResourcePluginManager { public synchronized void initialize(Context context) throws YarnException { Configuration conf = context.getConf(); - String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS); + Map<String, ResourcePlugin> pluginMap = new HashMap<>(); + String[] plugins = conf.getStrings(YarnConfiguration.NM_RESOURCE_PLUGINS); if (plugins != null) { - Map<String, ResourcePlugin> pluginMap = new HashMap<>(); - // Initialize each plugins for (String resourceName : plugins) { resourceName = resourceName.trim(); @@ -92,8 +92,34 @@ public class ResourcePluginManager { plugin.initialize(context); pluginMap.put(resourceName, plugin); } + } + // Try to load pluggable device plugins + boolean puggableDeviceFrameworkEnabled = conf.getBoolean( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + YarnConfiguration.DEFAULT_NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); + + if (puggableDeviceFrameworkEnabled) { + initializePluggableDevicePlugins(context, conf, pluginMap); + } else { + LOG.info("The pluggable device framework is not enabled." + + " If you want, please set true to {}", + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED); + } + configuredPlugins = Collections.unmodifiableMap(pluginMap); + } - configuredPlugins = Collections.unmodifiableMap(pluginMap); + public void initializePluggableDevicePlugins(Context context, + Configuration configuration, + Map<String, ResourcePlugin> pluginMap) + throws YarnRuntimeException { + LOG.info("The pluggable device framework enabled," + + "trying to load the vendor plugins"); + + String[] pluginClassNames = configuration.getStrings( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); + if (null == pluginClassNames) { + throw new YarnRuntimeException("Null value found in configuration: " + + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c72d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java index 6ed7c56..eb8bf54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/TestResourcePluginManager.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; @@ -41,12 +42,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.util.HashMap; @@ -58,10 +57,18 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; public class TestResourcePluginManager extends NodeManagerTestBase { private NodeManager nm; + private YarnConfiguration conf; + + @Before + public void setup() throws Exception { + this.conf = createNMConfig(); + } + ResourcePluginManager stubResourcePluginmanager() { // Stub ResourcePluginManager final ResourcePluginManager rpm = mock(ResourcePluginManager.class); @@ -183,7 +190,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase { final ResourcePluginManager rpm = stubResourcePluginmanager(); nm = new MyMockNM(rpm); - YarnConfiguration conf = createNMConfig(); nm.init(conf); verify(rpm, times(1)).initialize( any(Context.class)); @@ -198,7 +204,6 @@ public class TestResourcePluginManager extends NodeManagerTestBase { nm = new MyMockNM(rpm); - YarnConfiguration conf = createNMConfig(); nm.init(conf); nm.start(); @@ -238,15 +243,14 @@ public class TestResourcePluginManager extends NodeManagerTestBase { } @Override - protected ContainerExecutor createContainerExecutor(Configuration conf) { + protected ContainerExecutor createContainerExecutor( + Configuration configuration) { ((NMContext)this.getNMContext()).setResourcePluginManager(rpm); - lce.setConf(conf); + lce.setConf(configuration); return lce; } }; - YarnConfiguration conf = createNMConfig(); - nm.init(conf); nm.start(); @@ -264,4 +268,89 @@ public class TestResourcePluginManager extends NodeManagerTestBase { } Assert.assertTrue("New ResourceHandler should be added", newHandlerAdded); } + + // Disabled pluggable framework in configuration. + // We use spy object of real rpm to verify "initializePluggableDevicePlugins" + // because use mock rpm will not working + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkDisabled() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + false); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(0)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // No related configuration set. + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkDisabled2() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(0)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable framework and configure pluggable device classes + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled() + throws Exception { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + conf.setStrings( + YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_DEVICE_CLASSES, + "com.cmp1.hdw1plugin"); + nm.init(conf); + nm.start(); + verify(rpmSpy, times(1)).initialize( + any(Context.class)); + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + } + + // Enable pluggable framework, but leave device classes un-configured + // initializePluggableDevicePlugins invoked but it should throw an exception + @Test(timeout = 30000) + public void testInitializationWithPluggableDeviceFrameworkEnabled2() { + ResourcePluginManager rpm = new ResourcePluginManager(); + + ResourcePluginManager rpmSpy = spy(rpm); + nm = new MyMockNM(rpmSpy); + Boolean fail = false; + try { + conf.setBoolean(YarnConfiguration.NM_PLUGGABLE_DEVICE_FRAMEWORK_ENABLED, + true); + + nm.init(conf); + nm.start(); + } catch (YarnRuntimeException e) { + fail = true; + } catch (Exception e) { + + } + verify(rpmSpy, times(1)).initializePluggableDevicePlugins( + any(Context.class), any(Configuration.class), any(Map.class)); + Assert.assertTrue(fail); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
