This is an automated email from the ASF dual-hosted git repository. trohrmann pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit e1989bf3b598a565501f4d040ef9627aa34e3ded Author: Till Rohrmann <[email protected]> AuthorDate: Tue May 26 14:28:09 2020 +0200 [FLINK-17938] Move YarnTestBase.findFile and YarnTestBase.RootDirFilenameFilter to o.a.f.y.u.TestUtils Moving findFile and RootDirFilenameFilter to o.a.f.y.u.TestUtils allows to run the UtilsTest suite without requiring that yarn.classpath has been generated. The latter is required whenever one uses the YarnTestBase class because it reads this file when the class is being loaded. This closes #12341. --- .../test/java/org/apache/flink/yarn/UtilsTest.java | 3 +- .../flink/yarn/YARNFileReplicationITCase.java | 2 +- .../flink/yarn/YARNHighAvailabilityITCase.java | 2 +- .../java/org/apache/flink/yarn/YARNITCase.java | 2 +- .../yarn/YARNSessionCapacitySchedulerITCase.java | 3 +- .../java/org/apache/flink/yarn/YarnTestBase.java | 48 ++++------------------ .../java/org/apache/flink/yarn/util/TestUtils.java | 33 +++++++++++++++ 7 files changed, 48 insertions(+), 45 deletions(-) diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java index 53fd6ba..76aac0e 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/UtilsTest.java @@ -25,6 +25,7 @@ import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameter import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec; import org.apache.flink.runtime.clusterframework.TaskExecutorProcessUtils; import org.apache.flink.util.TestLogger; +import org.apache.flink.yarn.util.TestUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; @@ -66,7 +67,7 @@ public class UtilsTest extends TestLogger { @Test public void testUberjarLocator() { - File dir = YarnTestBase.findFile("..", new YarnTestBase.RootDirFilenameFilter()); + File dir = TestUtils.findFile("..", new TestUtils.RootDirFilenameFilter()); Assert.assertNotNull(dir); Assert.assertTrue(dir.getName().endsWith(".jar")); dir = dir.getParentFile().getParentFile(); // from uberjar to lib to root diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java index 3b0ea5b..b1f4560 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNFileReplicationITCase.java @@ -91,7 +91,7 @@ public class YARNFileReplicationITCase extends YarnTestBase { .setSlotsPerTaskManager(1) .createClusterSpecification(); - File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); + File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI())); try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java index 934df92..eba7140 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNHighAvailabilityITCase.java @@ -140,7 +140,7 @@ public class YARNHighAvailabilityITCase extends YarnTestBase { stopJobSignal = YarnTestJob.StopJobSignal.usingMarkerFile(FOLDER.newFile().toPath()); job = YarnTestJob.stoppableJob(stopJobSignal); final File testingJar = - YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); + TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); assertThat(testingJar, notNullValue()); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java index 47e523c..1e5a64e 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java @@ -116,7 +116,7 @@ public class YARNITCase extends YarnTestBase { .setSlotsPerTaskManager(1) .createClusterSpecification(); - File testingJar = YarnTestBase.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); + File testingJar = TestUtils.findFile("..", new TestUtils.TestJarFinder("flink-yarn-tests")); jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI())); try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java index 4145425..af0f532 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java @@ -40,6 +40,7 @@ import org.apache.flink.testutils.logging.TestLoggerResource; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.yarn.cli.FlinkYarnSessionCli; import org.apache.flink.yarn.configuration.YarnConfigOptions; +import org.apache.flink.yarn.util.TestUtils; import org.apache.flink.shaded.guava18.com.google.common.net.HostAndPort; @@ -561,7 +562,7 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase { Assert.assertTrue("Expected string 'der 29' or '(mind,1)' not found in string'" + content + "'", content.contains("der 29") || content.contains("(der,29)") || content.contains("(mind,1)")); // check if the heap size for the TaskManager was set correctly - File jobmanagerLog = YarnTestBase.findFile("..", new FilenameFilter() { + File jobmanagerLog = TestUtils.findFile("..", new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.contains("jobmanager.log") && dir.getAbsolutePath().contains(id.toString()); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 69031dd..70edeb8 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -28,6 +28,7 @@ import org.apache.flink.util.Preconditions; import org.apache.flink.util.TestLogger; import org.apache.flink.util.function.RunnableWithException; import org.apache.flink.yarn.cli.FlinkYarnSessionCli; +import org.apache.flink.yarn.util.TestUtils; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -207,7 +208,7 @@ public abstract class YarnTestBase extends TestLogger { private static String getYarnClasspath() { final String start = "../flink-yarn-tests"; try { - File classPathFile = findFile(start, (dir, name) -> name.equals("yarn.classpath")); + File classPathFile = TestUtils.findFile(start, (dir, name) -> name.equals("yarn.classpath")); return FileUtils.readFileToString(classPathFile); // potential NPE is supposed to be fatal } catch (Throwable t) { LOG.error("Error while getting YARN classpath in {}", new File(start).getAbsoluteFile(), t); @@ -305,29 +306,6 @@ public abstract class YarnTestBase extends TestLogger { return YARN_CONFIGURATION; } - /** - * Locate a file or directory. - */ - public static File findFile(String startAt, FilenameFilter fnf) { - File root = new File(startAt); - String[] files = root.list(); - if (files == null) { - return null; - } - for (String file : files) { - File f = new File(startAt + File.separator + file); - if (f.isDirectory()) { - File r = findFile(f.getAbsolutePath(), fnf); - if (r != null) { - return r; - } - } else if (fnf.accept(f.getParentFile(), f.getName())) { - return f; - } - } - return null; - } - @Nonnull YarnClusterDescriptor createYarnClusterDescriptor(org.apache.flink.configuration.Configuration flinkConfiguration) { final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptorWithoutLibDir(flinkConfiguration); @@ -347,16 +325,6 @@ public abstract class YarnTestBase extends TestLogger { } /** - * Filter to find root dir of the flink-yarn dist. - */ - public static class RootDirFilenameFilter implements FilenameFilter { - @Override - public boolean accept(File dir, String name) { - return name.startsWith("flink-dist") && name.endsWith(".jar") && dir.toString().contains("/lib"); - } - } - - /** * A simple {@link FilenameFilter} that only accepts files if their name contains every string in the array passed * to the constructor. */ @@ -428,7 +396,7 @@ public abstract class YarnTestBase extends TestLogger { Assert.assertTrue("Expecting directory " + cwd.getAbsolutePath() + " to be a directory", cwd.isDirectory()); List<String> prohibitedExcerpts = new ArrayList<>(); - File foundFile = findFile(cwd.getAbsolutePath(), new FilenameFilter() { + File foundFile = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() { @Override public boolean accept(File dir, String name) { // scan each file for prohibited strings. @@ -519,7 +487,7 @@ public abstract class YarnTestBase extends TestLogger { return false; } - File foundFile = findFile(cwd.getAbsolutePath(), new FilenameFilter() { + File foundFile = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (fileName != null && !name.equals(fileName)) { @@ -562,7 +530,7 @@ public abstract class YarnTestBase extends TestLogger { return false; } - File containerTokens = findFile(cwd.getAbsolutePath(), new FilenameFilter() { + File containerTokens = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.equals(containerId + ".tokens"); @@ -589,7 +557,7 @@ public abstract class YarnTestBase extends TestLogger { public static String getContainerIdByLogName(String logName) { File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); - File containerLog = findFile(cwd.getAbsolutePath(), new FilenameFilter() { + File containerLog = TestUtils.findFile(cwd.getAbsolutePath(), new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.equals(logName); @@ -653,7 +621,7 @@ public abstract class YarnTestBase extends TestLogger { System.setProperty("user.home", homeDir.getAbsolutePath()); String uberjarStartLoc = ".."; LOG.info("Trying to locate uberjar in {}", new File(uberjarStartLoc).getAbsolutePath()); - flinkUberjar = findFile(uberjarStartLoc, new RootDirFilenameFilter()); + flinkUberjar = TestUtils.findFile(uberjarStartLoc, new TestUtils.RootDirFilenameFilter()); Assert.assertNotNull("Flink uberjar not found", flinkUberjar); String flinkDistRootDir = flinkUberjar.getParentFile().getParent(); flinkLibFolder = flinkUberjar.getParentFile(); // the uberjar is located in lib/ @@ -681,7 +649,7 @@ public abstract class YarnTestBase extends TestLogger { Map<String, String> map = new HashMap<String, String>(System.getenv()); - File flinkConfDirPath = findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"})); + File flinkConfDirPath = TestUtils.findFile(flinkDistRootDir, new ContainsName(new String[]{"flink-conf.yaml"})); Assert.assertNotNull(flinkConfDirPath); final String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath(); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java index 5d852de..a8ae69a 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/TestUtils.java @@ -36,6 +36,29 @@ public class TestUtils { } /** + * Locate a file or directory. + */ + public static File findFile(String startAt, FilenameFilter fnf) { + File root = new File(startAt); + String[] files = root.list(); + if (files == null) { + return null; + } + for (String file : files) { + File f = new File(startAt + File.separator + file); + if (f.isDirectory()) { + File r = findFile(f.getAbsolutePath(), fnf); + if (r != null) { + return r; + } + } else if (fnf.accept(f.getParentFile(), f.getName())) { + return f; + } + } + return null; + } + + /** * Filename filter which finds the test jar for the given name. */ public static class TestJarFinder implements FilenameFilter { @@ -52,4 +75,14 @@ public class TestUtils { dir.getAbsolutePath().contains(File.separator + jarName + File.separator); } } + + /** + * Filter to find root dir of the flink-yarn dist. + */ + public static class RootDirFilenameFilter implements FilenameFilter { + @Override + public boolean accept(File dir, String name) { + return name.startsWith("flink-dist") && name.endsWith(".jar") && dir.toString().contains("/lib"); + } + } }
