[FLINK-9020][E2ETests] Use separate modules per testcase This closes #5717.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/129e2159 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/129e2159 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/129e2159 Branch: refs/heads/master Commit: 129e215961895ef3c759b4fbb919a3be2ce2f1a0 Parents: 91707e3 Author: Florian Schmidt <[email protected]> Authored: Mon Mar 19 13:26:14 2018 +0100 Committer: zentol <[email protected]> Committed: Wed Mar 21 21:01:52 2018 +0100 ---------------------------------------------------------------------- .../pom.xml | 108 +++++++++++++++++++ .../flink/runtime/taskmanager/TaskManager.java | 30 ++++++ .../streaming/tests/ClassLoaderTestProgram.java | 102 ++++++++++++++++++ .../src/main/resources/.version.properties | 1 + flink-end-to-end-tests/pom.xml | 78 ++------------ .../flink/runtime/taskmanager/TaskManager.java | 30 ------ .../streaming/tests/ClassLoaderTestProgram.java | 102 ------------------ .../src/main/resources/.version.properties | 1 - .../test-scripts/test_streaming_classloader.sh | 4 +- 9 files changed, 249 insertions(+), 207 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml new file mode 100644 index 0000000..ee43515 --- /dev/null +++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/pom.xml @@ -0,0 +1,108 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <parent> + <artifactId>flink-end-to-end-tests</artifactId> + <groupId>org.apache.flink</groupId> + <version>1.6-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <modelVersion>4.0.0</modelVersion> + + <artifactId>flink-parent-child-classloading-test_${scala.binary.version}</artifactId> + <name>flink-parent-child-classloading-test</name> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.4</version> + + <executions> + <!-- ClassLoaderTestProgram --> + <execution> + <id>ClassLoaderTestProgram</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <classifier>ClassLoaderTestProgram</classifier> + + <archive> + <manifestEntries> + <program-class>org.apache.flink.streaming.tests.ClassLoaderTestProgram</program-class> + </manifestEntries> + </archive> + + <includes> + <include>org/apache/flink/streaming/tests/ClassLoaderTestProgram.class</include> + <include>org/apache/flink/runtime/taskmanager/TaskManager.class</include> + <include>.version.properties</include> + </includes> + </configuration> + </execution> + </executions> + </plugin> + + <!--simplify the name of the testing JARs for referring to them in the end-to-end test scripts--> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.7</version> + <executions> + <execution> + <id>rename</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <copy file="${project.basedir}/target/flink-parent-child-classloading-test_${scala.binary.version}-${project.version}-ClassLoaderTestProgram.jar" tofile="${project.basedir}/target/ClassLoaderTestProgram.jar" /> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + +</project> + http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java new file mode 100644 index 0000000..3626885 --- /dev/null +++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.taskmanager; + +/** + * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this + * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do + * so. + */ +public class TaskManager { + public static String getMessage() { + return "Hello, World!"; + } +} http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java new file mode 100644 index 0000000..1d4ca4c --- /dev/null +++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.runtime.taskmanager.TaskManager; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import java.io.InputStream; +import java.net.URL; +import java.util.Enumeration; +import java.util.Properties; + +/** + * End-to-end test program for verifying that the {@code classloader.resolve-order} setting + * is being honored by Flink. We test this by creating a fake {@code TaskManager} with a single + * method that we call in the same package as the original Flink {@code TaskManager} and verify that + * we get a {@link NoSuchMethodError} if we're running with {@code parent-first} class loading + * and that we get the correct result from the method when we're running with {@code child-first} + * class loading. + */ +public class ClassLoaderTestProgram { + + public static void main(String[] args) throws Exception { + + final ParameterTool params = ParameterTool.fromArgs(args); + + final String resolveOrder = params.getRequired("resolve-order"); + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + env + .fromElements("Hello") + .map((MapFunction<String, String>) value -> { + + String gitUrl; + + try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream(".version.properties")) { + Properties properties = new Properties(); + properties.load(propFile); + gitUrl = properties.getProperty("git.remote.origin.url"); + } + + Enumeration<URL> resources = ClassLoaderTestProgram.class.getClassLoader().getResources( + ".version.properties"); + + StringBuilder sortedProperties = new StringBuilder(); + while (resources.hasMoreElements()) { + URL url = resources.nextElement(); + try (InputStream in = url.openStream()) { + Properties properties = new Properties(); + properties.load(in); + String orderedGitUrl = properties.getProperty("git.remote.origin.url"); + sortedProperties.append(orderedGitUrl); + } + } + + if (resolveOrder.equals("parent-first")) { + try { + @SuppressWarnings("unused") + String ignored = TaskManager.getMessage(); + + throw new RuntimeException( + "TaskManager.getMessage() should not be available with parent-first " + + "ClassLoader order."); + + } catch (NoSuchMethodError e) { + // expected + } + return "NoSuchMethodError:" + gitUrl + ":" + sortedProperties; + } else if (resolveOrder.equals("child-first")) { + String message = TaskManager.getMessage(); + if (!message.equals("Hello, World!")) { + throw new RuntimeException("Wrong message from fake TaskManager."); + } + return message + ":" + gitUrl + ":" + sortedProperties; + } else { + throw new RuntimeException("Unknown resolve order: " + resolveOrder); + } + }) + .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE); + + env.execute("ClassLoader Test Program"); + } +} http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties new file mode 100644 index 0000000..dc98aea --- /dev/null +++ b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties @@ -0,0 +1 @@ +git.remote.origin.url=hello-there-42 http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/pom.xml ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index a5bbc52..32a6e78 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -29,79 +29,13 @@ under the License. <relativePath>..</relativePath> </parent> - <artifactId>flink-end-to-end-tests_${scala.binary.version}</artifactId> - <name>flink-end-to-end-tests</name> - - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-core</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> - <version>${project.version}</version> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> + <packaging>pom</packaging> - <executions> - <!-- ClassLoaderTestProgram --> - <execution> - <id>ClassLoaderTestProgram</id> - <phase>package</phase> - <goals> - <goal>jar</goal> - </goals> - <configuration> - <classifier>ClassLoaderTestProgram</classifier> - - <archive> - <manifestEntries> - <program-class>org.apache.flink.streaming.tests.ClassLoaderTestProgram</program-class> - </manifestEntries> - </archive> - - <includes> - <include>org/apache/flink/streaming/tests/ClassLoaderTestProgram.class</include> - <include>org/apache/flink/runtime/taskmanager/TaskManager.class</include> - <include>.version.properties</include> - </includes> - </configuration> - </execution> - </executions> - </plugin> + <artifactId>flink-end-to-end-tests</artifactId> + <name>flink-end-to-end-tests</name> - <!--simplify the name of the testing JARs for referring to them in the end-to-end test scripts--> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.7</version> - <executions> - <execution> - <id>rename</id> - <phase>package</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - <copy file="${project.basedir}/target/flink-end-to-end-tests_${scala.binary.version}-${project.version}-ClassLoaderTestProgram.jar" tofile="${project.basedir}/target/ClassLoaderTestProgram.jar" /> - </target> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> + <modules> + <module>flink-parent-child-classloading-test</module> + </modules> </project> http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java b/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java deleted file mode 100644 index 3626885..0000000 --- a/flink-end-to-end-tests/src/main/java/org/apache/flink/runtime/taskmanager/TaskManager.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.taskmanager; - -/** - * A {@code Taskmanager} in the same package as the proper Flink {@link TaskManager}. We use this - * to check whether Flink correctly uses the child-first {@link ClassLoader} when configured to do - * so. - */ -public class TaskManager { - public static String getMessage() { - return "Hello, World!"; - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java b/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java deleted file mode 100644 index 1d4ca4c..0000000 --- a/flink-end-to-end-tests/src/main/java/org/apache/flink/streaming/tests/ClassLoaderTestProgram.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.tests; - -import org.apache.flink.api.common.functions.MapFunction; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.runtime.taskmanager.TaskManager; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import java.io.InputStream; -import java.net.URL; -import java.util.Enumeration; -import java.util.Properties; - -/** - * End-to-end test program for verifying that the {@code classloader.resolve-order} setting - * is being honored by Flink. We test this by creating a fake {@code TaskManager} with a single - * method that we call in the same package as the original Flink {@code TaskManager} and verify that - * we get a {@link NoSuchMethodError} if we're running with {@code parent-first} class loading - * and that we get the correct result from the method when we're running with {@code child-first} - * class loading. - */ -public class ClassLoaderTestProgram { - - public static void main(String[] args) throws Exception { - - final ParameterTool params = ParameterTool.fromArgs(args); - - final String resolveOrder = params.getRequired("resolve-order"); - - final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - env - .fromElements("Hello") - .map((MapFunction<String, String>) value -> { - - String gitUrl; - - try (InputStream propFile = ClassLoaderTestProgram.class.getClassLoader().getResourceAsStream(".version.properties")) { - Properties properties = new Properties(); - properties.load(propFile); - gitUrl = properties.getProperty("git.remote.origin.url"); - } - - Enumeration<URL> resources = ClassLoaderTestProgram.class.getClassLoader().getResources( - ".version.properties"); - - StringBuilder sortedProperties = new StringBuilder(); - while (resources.hasMoreElements()) { - URL url = resources.nextElement(); - try (InputStream in = url.openStream()) { - Properties properties = new Properties(); - properties.load(in); - String orderedGitUrl = properties.getProperty("git.remote.origin.url"); - sortedProperties.append(orderedGitUrl); - } - } - - if (resolveOrder.equals("parent-first")) { - try { - @SuppressWarnings("unused") - String ignored = TaskManager.getMessage(); - - throw new RuntimeException( - "TaskManager.getMessage() should not be available with parent-first " + - "ClassLoader order."); - - } catch (NoSuchMethodError e) { - // expected - } - return "NoSuchMethodError:" + gitUrl + ":" + sortedProperties; - } else if (resolveOrder.equals("child-first")) { - String message = TaskManager.getMessage(); - if (!message.equals("Hello, World!")) { - throw new RuntimeException("Wrong message from fake TaskManager."); - } - return message + ":" + gitUrl + ":" + sortedProperties; - } else { - throw new RuntimeException("Unknown resolve order: " + resolveOrder); - } - }) - .writeAsText(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE); - - env.execute("ClassLoader Test Program"); - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/src/main/resources/.version.properties ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/src/main/resources/.version.properties b/flink-end-to-end-tests/src/main/resources/.version.properties deleted file mode 100644 index dc98aea..0000000 --- a/flink-end-to-end-tests/src/main/resources/.version.properties +++ /dev/null @@ -1 +0,0 @@ -git.remote.origin.url=hello-there-42 http://git-wip-us.apache.org/repos/asf/flink/blob/129e2159/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh index 95c58f8..34c55f7 100755 --- a/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh +++ b/flink-end-to-end-tests/test-scripts/test_streaming_classloader.sh @@ -19,7 +19,7 @@ source "$(dirname "$0")"/common.sh -TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/target/ClassLoaderTestProgram.jar +TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-parent-child-classloading-test/target/ClassLoaderTestProgram.jar echo "Testing parent-first class loading" @@ -111,4 +111,4 @@ if [[ "$OUTPUT" != "$EXPECTED" ]]; then echo -e "EXPECTED: $EXPECTED" echo -e "ACTUAL: $OUTPUT" PASS="" -fi \ No newline at end of file +fi
