This is an automated email from the ASF dual-hosted git repository. sewen pushed a commit to branch release-1.8 in repository https://gitbox.apache.org/repos/asf/flink.git
commit 99d006426c29280e812cb34dd53300b7e695d013 Author: Stephan Ewen <[email protected]> AuthorDate: Tue Jul 30 11:04:23 2019 +0200 [hotfix][tests] Refactor MapR FS Tests --- .../runtime/fs/maprfs/FileSystemAccessTest.java | 42 ------ .../flink/runtime/fs/maprfs/MapRFreeTests.java | 74 ---------- .../flink/runtime/fs/maprfs/MapRFsFactoryTest.java | 84 +++--------- .../runtime/fs/maprfs/MapRNotInClassPathTest.java | 149 +++++++++++++++++++++ 4 files changed, 167 insertions(+), 182 deletions(-) diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java deleted file mode 100644 index aee50ba..0000000 --- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/FileSystemAccessTest.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.fs.maprfs; - -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * This test checks that the file system is properly accessible through the - * service loading abstraction. - */ -public class FileSystemAccessTest extends TestLogger { - - @Test - public void testGetMapRFs() throws Exception { - final Path path = new Path("maprfs:///my/path"); - - FileSystem fs = path.getFileSystem(); - assertEquals(path.toUri().getScheme(), fs.getUri().getScheme()); - } -} diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java deleted file mode 100644 index 110fce3..0000000 --- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFreeTests.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.fs.maprfs; - -import org.apache.flink.configuration.Configuration; - -import java.io.IOException; -import java.net.URI; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * A class with tests that require to be run in a MapR/Hadoop-free environment, - * to test proper error handling when no Hadoop classes are available. - * - * <p>This class must be dynamically loaded in a MapR/Hadoop-free class loader. - */ -// this class is only instantiated via reflection -@SuppressWarnings("unused") -public class MapRFreeTests { - - public static void test() throws Exception { - // make sure no MapR or Hadoop FS classes are in the classpath - try { - Class.forName("com.mapr.fs.MapRFileSystem"); - fail("Cannot run test when MapR classes are in the classpath"); - } - catch (ClassNotFoundException ignored) {} - - try { - Class.forName("org.apache.hadoop.fs.FileSystem"); - fail("Cannot run test when Hadoop classes are in the classpath"); - } - catch (ClassNotFoundException ignored) {} - - try { - Class.forName("org.apache.hadoop.conf.Configuration"); - fail("Cannot run test when Hadoop classes are in the classpath"); - } - catch (ClassNotFoundException ignored) {} - - // this method should complete without a linkage error - final MapRFsFactory factory = new MapRFsFactory(); - - // this method should also complete without a linkage error - factory.configure(new Configuration()); - - try { - factory.create(new URI("maprfs://somehost:9000/root/dir")); - fail("This statement should fail with an exception"); - } - catch (IOException e) { - assertTrue(e.getMessage().contains("MapR")); - assertTrue(e.getMessage().contains("classpath")); - } - } -} diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java index 984668f..2c8357a 100644 --- a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java +++ b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRFsFactoryTest.java @@ -18,99 +18,51 @@ package org.apache.flink.runtime.fs.maprfs; -import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.core.fs.FileSystemKind; +import org.apache.flink.core.fs.Path; import org.apache.flink.util.TestLogger; import org.junit.Test; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.net.URI; -import java.net.URLClassLoader; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; /** - * Tests for the MapRFsFactory. + * Tests for the {@link MapRFsFactory}. */ public class MapRFsFactoryTest extends TestLogger { - /** - * This test validates that the factory can be instantiated and configured even - * when MapR and Hadoop classes are missing from the classpath. - */ @Test - public void testInstantiationWithoutMapRClasses() throws Exception { - // we do reflection magic here to instantiate the test in another class - // loader, to make sure no MapR and Hadoop classes are in the classpath + public void testMapRFsScheme() throws Exception { + final Path path = new Path("maprfs:///my/path"); - final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRFreeTests"; + final FileSystem fs = path.getFileSystem(); - URLClassLoader parent = (URLClassLoader) getClass().getClassLoader(); - ClassLoader maprFreeClassLoader = new MapRFreeClassLoader(parent); - Class<?> testClass = Class.forName(testClassName, false, maprFreeClassLoader); - Method m = testClass.getDeclaredMethod("test"); - - try { - m.invoke(null); - } - catch (InvocationTargetException e) { - ExceptionUtils.rethrowException(e.getTargetException(), "exception in method"); - } + assertEquals(path.toUri().getScheme(), fs.getUri().getScheme()); } @Test - public void testCreateFsWithAuthority() throws Exception { - final URI uri = URI.create("maprfs://localhost:12345/"); + public void testMapRFsKind() throws Exception { + final Path path = new Path("maprfs:///my/path"); - MapRFsFactory factory = new MapRFsFactory(); + final FileSystem fs = path.getFileSystem(); - try { - factory.create(uri); - fail("should have failed with an exception"); - } - catch (IOException e) { - // expected, because we have no CLDB config available - } + assertEquals(FileSystemKind.FILE_SYSTEM, fs.getKind()); } @Test - public void testCreateFsWithMissingAuthority() throws Exception { - final URI uri = URI.create("maprfs:///my/path"); - - MapRFsFactory factory = new MapRFsFactory(); - factory.configure(new Configuration()); + public void testCreateWithAuthorityNoCldbFails() throws Exception { + final Path path = new Path("maprfs://localhost:12345/"); - FileSystem fs = factory.create(uri); - assertEquals("maprfs", fs.getUri().getScheme()); - } - - // ------------------------------------------------------------------------ - - private static final class MapRFreeClassLoader extends URLClassLoader { - - private final ClassLoader properParent; - - MapRFreeClassLoader(URLClassLoader parent) { - super(parent.getURLs(), null); - properParent = parent; + try { + path.getFileSystem(); + fail("should have failed with an exception"); } - - @Override - public Class<?> loadClass(String name) throws ClassNotFoundException { - if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) { - throw new ClassNotFoundException(name); - } - else if (name.startsWith("org.apache.log4j")) { - return properParent.loadClass(name); - } - else { - return super.loadClass(name); - } + catch (IOException e) { + // expected, because we have no CLDB config available } } } diff --git a/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java new file mode 100644 index 0000000..77ff62e --- /dev/null +++ b/flink-filesystems/flink-mapr-fs/src/test/java/org/apache/flink/runtime/fs/maprfs/MapRNotInClassPathTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.fs.maprfs; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.TestLogger; +import org.apache.flink.util.function.RunnableWithException; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A class with tests that require to be run in a MapR/Hadoop-free environment, + * to test proper error handling when no Hadoop classes are available. + * + * <p>This class must be dynamically loaded in a MapR/Hadoop-free class loader. + */ +public class MapRNotInClassPathTest extends TestLogger { + + @Test + public void testInstantiationWhenMapRClassesAreMissing() throws Exception { + final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRNotInClassPathTest$TestRunner"; + final ClassLoader cl = new MapRFreeClassLoader(getClass().getClassLoader()); + + final RunnableWithException testRunner = Class + .forName(testClassName, false, cl) + .asSubclass(RunnableWithException.class) + .newInstance(); + + testRunner.run(); + } + + // ------------------------------------------------------------------------ + + /** + * The tests that need to run in a special classloader. + */ + @SuppressWarnings("unused") + public static final class TestRunner implements RunnableWithException { + + @Override + public void run() throws Exception { + // make sure no MapR or Hadoop FS classes are in the classpath + try { + Class.forName("com.mapr.fs.MapRFileSystem"); + fail("Cannot run test when MapR classes are in the classpath"); + } + catch (ClassNotFoundException ignored) {} + + try { + Class.forName("org.apache.hadoop.fs.FileSystem"); + fail("Cannot run test when Hadoop classes are in the classpath"); + } + catch (ClassNotFoundException ignored) {} + + try { + Class.forName("org.apache.hadoop.conf.Configuration"); + fail("Cannot run test when Hadoop classes are in the classpath"); + } + catch (ClassNotFoundException ignored) {} + + // this method should complete without a linkage error + final MapRFsFactory factory = new MapRFsFactory(); + + // this method should also complete without a linkage error + factory.configure(new Configuration()); + + try { + factory.create(new URI("maprfs://somehost:9000/root/dir")); + fail("This statement should fail with an exception"); + } + catch (IOException e) { + assertTrue(e.getMessage().contains("MapR")); + assertTrue(e.getMessage().contains("classpath")); + } + } + } + + // ------------------------------------------------------------------------ + + /** + * A special classloader that filters "org.apache.hadoop.*" and "com.mapr.*" classes. + */ + private static final class MapRFreeClassLoader extends URLClassLoader { + + private final ClassLoader properParent; + + MapRFreeClassLoader(ClassLoader parent) { + super(getClasspathURLs(), null); + properParent = parent; + } + + @Override + public Class<?> loadClass(String name) throws ClassNotFoundException { + if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) { + throw new ClassNotFoundException(name); + } + else if (name.equals(RunnableWithException.class.getName()) || name.startsWith("org.apache.log4j")) { + return properParent.loadClass(name); + } + else { + return super.loadClass(name); + } + } + + private static URL[] getClasspathURLs() { + final String[] cp = System.getProperty("java.class.path").split(File.pathSeparator); + + return Arrays.stream(cp) + .filter(str -> !str.isEmpty()) + .map(MapRFreeClassLoader::parse) + .toArray(URL[]::new); + } + + private static URL parse(String fileName) { + try { + return new File(fileName).toURI().toURL(); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + } +}
