Repository: flink Updated Branches: refs/heads/master 14840809b -> d57215497
[FLINK-8455] [core] Make 'org.apache.hadoop.' a 'parent-first' classloading pattern. This change avoid duplication of Hadoop classes between the Flink runtime and the user code. Hadoop (and transitively its dependencies) should be part of the application class loader. The user code classloader is allowed to duplicate transitive dependencies, but not Hadoop's classes directly. This also adds tests to validate parent-first classloading patterns. Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d5721549 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d5721549 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d5721549 Branch: refs/heads/master Commit: d57215497f36f87939c1cdf3b090d00d8e59d90b Parents: f9aff59 Author: Stephan Ewen <[email protected]> Authored: Thu Jan 18 17:57:10 2018 +0100 Committer: Stephan Ewen <[email protected]> Committed: Fri Jan 19 13:40:00 2018 +0100 ---------------------------------------------------------------------- .../apache/flink/configuration/CoreOptions.java | 2 +- .../configuration/ParentFirstPatternsTest.java | 78 ++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/d5721549/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java ---------------------------------------------------------------------- diff --git a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java index 1182ed5..cd93b99 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/CoreOptions.java @@ -79,7 +79,7 @@ public class CoreOptions { */ public static final ConfigOption<String> ALWAYS_PARENT_FIRST_LOADER = ConfigOptions .key("classloader.parent-first-patterns") - .defaultValue("java.;scala.;org.apache.flink.;javax.annotation;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback"); + .defaultValue("java.;scala.;org.apache.flink.;org.apache.hadoop.;javax.annotation.;org.slf4j;org.apache.log4j;org.apache.logging.log4j;ch.qos.logback"); // ------------------------------------------------------------------------ // process parameters http://git-wip-us.apache.org/repos/asf/flink/blob/d5721549/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java ---------------------------------------------------------------------- diff --git a/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java new file mode 100644 index 0000000..784d099 --- /dev/null +++ b/flink-core/src/test/java/org/apache/flink/configuration/ParentFirstPatternsTest.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.configuration; + +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; + +import static org.junit.Assert.assertTrue; + +/** + * Test that checks that all packages that need to be loaded 'parent-first' are also + * in the parent-first patterns. + */ +public class ParentFirstPatternsTest extends TestLogger { + + private static final HashSet<String> PARENT_FIRST_PACKAGES = new HashSet<>( + Arrays.asList(CoreOptions.ALWAYS_PARENT_FIRST_LOADER.defaultValue().split(";"))); + + /** + * All java and Flink classes must be loaded parent first. + */ + @Test + public void testAllCorePatterns() { + assertTrue(PARENT_FIRST_PACKAGES.contains("java.")); + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.flink.")); + assertTrue(PARENT_FIRST_PACKAGES.contains("javax.annotation.")); + } + + /** + * To avoid multiple binding problems and warnings for logger frameworks, we load them + * parent-first. + */ + @Test + public void testLoggersParentFirst() { + assertTrue(PARENT_FIRST_PACKAGES.contains("org.slf4j")); + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.log4j")); + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.logging.log4j")); + assertTrue(PARENT_FIRST_PACKAGES.contains("ch.qos.logback")); + } + + /** + * As long as Scala is not a pure user library, but is also used in the Flink runtime, we need + * to load all Scala classes parent-first. + */ + @Test + public void testScalaParentFirst() { + assertTrue(PARENT_FIRST_PACKAGES.contains("scala.")); + } + + /** + * As long as we have Hadoop classes leaking through some of Flink's APIs (example bucketing sink), + * we need to make them parent first. + */ + @Test + public void testHadoopParentFirst() { + assertTrue(PARENT_FIRST_PACKAGES.contains("org.apache.hadoop.")); + } +}
