YARN-8456. Fix a configuration handling bug when user leave FPGA discover executable path configuration default but set OpenCL SDK path environment variable. (Zhankun Tang via wangda)
Change-Id: Iff150ea98ba0c60d448474fd940eb121afce6965 (cherry picked from commit a457a8951a1b35f06811c40443ca44bb9c698c30) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f7ed043 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f7ed043 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f7ed043 Branch: refs/heads/branch-3.2 Commit: 5f7ed043c8e8f9d964a719d076a9d3d207d4cea7 Parents: a7785d2 Author: Wangda Tan <[email protected]> Authored: Thu Oct 18 10:57:11 2018 -0700 Committer: Wangda Tan <[email protected]> Committed: Thu Oct 18 12:12:14 2018 -0700 ---------------------------------------------------------------------- .../fpga/IntelFpgaOpenclPlugin.java | 4 +- .../resourceplugin/fpga/TestFpgaDiscoverer.java | 64 ++++++++++++++++++-- 2 files changed, 61 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java index 2d6cf6f..f8fb6d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/IntelFpgaOpenclPlugin.java @@ -124,7 +124,9 @@ public class IntelFpgaOpenclPlugin implements AbstractFpgaVendorPlugin { } else { binaryPath = new File(pluginDefaultPreferredPath + "/bin", pluginDefaultBinaryName); if (binaryPath.exists()) { - pathToExecutable = pluginDefaultPreferredPath; + pathToExecutable = binaryPath.getAbsolutePath(); + LOG.info("Succeed in finding FPGA discoverer executable: " + + pathToExecutable); } else { pathToExecutable = pluginDefaultBinaryName; LOG.warn("Failed to find FPGA discoverer executable in " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f7ed043/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java index 87fb4e9..d5bcdb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/TestFpgaDiscoverer.java @@ -32,9 +32,8 @@ import org.junit.Test; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Field; +import java.util.*; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyString; @@ -60,8 +59,41 @@ public class TestFpgaDiscoverer { f.mkdirs(); } + // A dirty hack to modify the env of the current JVM itself - Dirty, but + // should be okay for testing. + @SuppressWarnings({ "rawtypes", "unchecked" }) + private static void setNewEnvironmentHack(Map<String, String> newenv) + throws Exception { + try { + Class<?> cl = Class.forName("java.lang.ProcessEnvironment"); + Field field = cl.getDeclaredField("theEnvironment"); + field.setAccessible(true); + Map<String, String> env = (Map<String, String>) field.get(null); + env.clear(); + env.putAll(newenv); + Field ciField = cl.getDeclaredField("theCaseInsensitiveEnvironment"); + ciField.setAccessible(true); + Map<String, String> cienv = (Map<String, String>) ciField.get(null); + cienv.clear(); + cienv.putAll(newenv); + } catch (NoSuchFieldException e) { + Class[] classes = Collections.class.getDeclaredClasses(); + Map<String, String> env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map<String, String> map = (Map<String, String>) obj; + map.clear(); + map.putAll(newenv); + } + } + } + } + @Test - public void testLinuxFpgaResourceDiscoverPluginConfig() throws YarnException, IOException { + public void testLinuxFpgaResourceDiscoverPluginConfig() throws Exception { Configuration conf = new Configuration(false); FpgaDiscoverer discoverer = FpgaDiscoverer.getInstance(); @@ -73,8 +105,9 @@ public class TestFpgaDiscoverer { openclPlugin.setShell(mockPuginShell()); discoverer.initialize(conf); - // Case 1. No configuration set for binary - Assert.assertEquals("No configuration should return just a single binary name", + // Case 1. No configuration set for binary(no environment "ALTERAOCLSDKROOT" set) + Assert.assertEquals("No configuration(no environment ALTERAOCLSDKROOT set)" + + "should return just a single binary name", "aocl", openclPlugin.getPathToExecutable()); // Case 2. With correct configuration and file exists @@ -91,6 +124,25 @@ public class TestFpgaDiscoverer { Assert.assertEquals("Correct configuration but file doesn't exists should return just a single binary name", "aocl", openclPlugin.getPathToExecutable()); + // Case 4. Set a empty value + conf.set(YarnConfiguration.NM_FPGA_PATH_TO_EXEC, ""); + discoverer.initialize(conf); + Assert.assertEquals("configuration with empty string value, should use aocl", + "aocl", openclPlugin.getPathToExecutable()); + + // Case 5. No configuration set for binary, but set environment "ALTERAOCLSDKROOT" + // we load the default configuration to start with + conf = new Configuration(true); + fakeBinary = new File(getTestParentFolder() + "/bin/aocl"); + fakeBinary.getParentFile().mkdirs(); + touchFile(fakeBinary); + Map<String, String> newEnv = new HashMap<String, String>(); + newEnv.put("ALTERAOCLSDKROOT", getTestParentFolder()); + setNewEnvironmentHack(newEnv); + discoverer.initialize(conf); + Assert.assertEquals("No configuration but with environment ALTERAOCLSDKROOT set", + getTestParentFolder() + "/bin/aocl", openclPlugin.getPathToExecutable()); + } @Test --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
