This is an automated email from the ASF dual-hosted git repository. jwills pushed a commit to branch jwills_great_version_upgrade in repository https://gitbox.apache.org/repos/asf/crunch.git
commit 7c677940061534f17ecaf3ef5efcd56ee07984e9 Author: Josh Wills <[email protected]> AuthorDate: Thu Nov 14 13:50:15 2019 -0800 Ever more fixes --- .../src/main/resources/archetype-resources/pom.xml | 4 ++-- .../io/parquet/AvroParquetFileSourceTargetIT.java | 2 +- .../types/avro/AvroChildClassloaderTestRunner.java | 22 ++++++++++++++++--- .../org/apache/crunch/types/avro/AvroModeTest.java | 25 ---------------------- pom.xml | 15 +++++++------ 5 files changed, 30 insertions(+), 38 deletions(-) diff --git a/crunch-archetype/src/main/resources/archetype-resources/pom.xml b/crunch-archetype/src/main/resources/archetype-resources/pom.xml index 6575417..34da675 100644 --- a/crunch-archetype/src/main/resources/archetype-resources/pom.xml +++ b/crunch-archetype/src/main/resources/archetype-resources/pom.xml @@ -89,8 +89,8 @@ <artifactId>maven-compiler-plugin</artifactId> <version>2.5.1</version> <configuration> - <source>1.6</source> - <target>1.6</target> + <source>1.8</source> + <target>1.8</target> </configuration> </plugin> <plugin> diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java index 1b2b151..3c80a3c 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java @@ -153,7 +153,7 @@ public class AvroParquetFileSourceTargetIT implements Serializable { Person person = Iterables.getOnlyElement(ageOnly.materialize()); assertNull(person.getName()); - assertEquals(person.getAge(), 42); + assertEquals(person.getAge().intValue(), 42); assertNull(person.getSiblingnames()); } diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java index a1f93b7..d0016fc 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java @@ -17,6 +17,8 @@ */ package org.apache.crunch.types.avro; +import java.io.File; +import java.lang.management.ManagementFactory; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; @@ -45,11 +47,25 @@ public class AvroChildClassloaderTestRunner extends BlockJUnit4ClassRunner { private static ClassLoader parentClassLoader; private static URL[] crunchURLs; static { - URLClassLoader systemClassLoader = (URLClassLoader) getSystemClassLoader(); + ClassLoader classLoader = getSystemClassLoader(); + URL[] urls = null; + if (classLoader instanceof URLClassLoader) { + urls = ((URLClassLoader) classLoader).getURLs(); + } else { + String[] pieces = ManagementFactory.getRuntimeMXBean().getClassPath().split(File.pathSeparator); + urls = new URL[pieces.length]; + for (int i = 0; i < pieces.length; i++) { + try { + urls[i] = new File(pieces[i]).toURI().toURL(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } Collection<URL> crunchURLs = new ArrayList<URL>(); Collection<URL> otherURLs = new ArrayList<URL>(); - for (URL url : systemClassLoader.getURLs()) { + for (URL url : urls) { if (url.getPath().matches("^.*/crunch-?.*/.*$")) { crunchURLs.add(url); } else { @@ -58,7 +74,7 @@ public class AvroChildClassloaderTestRunner extends BlockJUnit4ClassRunner { } TestClassLoader.crunchURLs = crunchURLs.toArray(new URL[crunchURLs.size()]); - parentClassLoader = new URLClassLoader(otherURLs.toArray(new URL[otherURLs.size()]), null); + parentClassLoader = new URLClassLoader(otherURLs.toArray(new URL[otherURLs.size()]), classLoader); } public TestClassLoader() { diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java index fe36520..8df49d3 100644 --- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java @@ -23,7 +23,6 @@ import static org.hamcrest.CoreMatchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -152,30 +151,6 @@ public class AvroModeTest { assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class))); } - @Test - public void testRegisterClassLoader() { - // First make sure things are in the default situation - AvroMode.setSpecificClassLoader(null); - - ClassLoader classLoaderA = mock(ClassLoader.class); - ClassLoader classLoaderB = mock(ClassLoader.class); - - // Basic sanity check to ensure that the class loader was really nulled out - assertNull(AvroMode.getSpecificClassLoader()); - - // Do an internal registration of a class loader. Because there is currently no internal class loader set, - // this should set the internal specific class loader - AvroMode.registerSpecificClassLoaderInternal(classLoaderA); - - assertEquals(classLoaderA, AvroMode.getSpecificClassLoader()); - - // Now we do an internal register of another class loader. Because there already is an internal specific class - // loader set, this should have no impact (as opposed to calling setSpecificClassLoader) - AvroMode.registerSpecificClassLoaderInternal(classLoaderB); - - assertEquals(classLoaderA, AvroMode.getSpecificClassLoader()); - } - private static class FakeReaderWriterFactory implements ReaderWriterFactory{ @Override diff --git a/pom.xml b/pom.xml index 05a21ad..9b8b526 100644 --- a/pom.xml +++ b/pom.xml @@ -85,9 +85,9 @@ under the License. <commons-lang.version>2.6</commons-lang.version> <commons-codec.version>1.4</commons-codec.version> <commons-cli.version>1.2</commons-cli.version> - <avro.version>1.9.1</avro.version> + <avro.version>1.8.2</avro.version> <hive.version>2.1.0</hive.version> - <parquet.version>1.8.1</parquet.version> + <parquet.version>1.10.1</parquet.version> <javassist.version>3.16.1-GA</javassist.version> <jackson.databind.version>2.6.1</jackson.databind.version> <protobuf-java.version>2.5.0</protobuf-java.version> @@ -106,7 +106,7 @@ under the License. <kafka.version>1.1.0</kafka.version> <scala.base.version>2.12</scala.base.version> - <scala.version>2.12.6</scala.version> + <scala.version>2.12.10</scala.version> <scalatest.version>3.0.1</scalatest.version> <spark.version>2.4.0</spark.version> <jline.version>2.12.1</jline.version> @@ -278,7 +278,7 @@ under the License. <exclusions> <exclusion> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> + <artifactId>hadoop-client</artifactId> </exclusion> </exclusions> </dependency> @@ -641,7 +641,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.3</version> + <version>3.8.1</version> <configuration> <source>${java.source.version}</source> <target>${java.target.version}</target> @@ -689,7 +689,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-assembly-plugin</artifactId> - <version>2.5.4</version> + <version>3.1.1</version> </plugin> <plugin> <groupId>net.alchim31.maven</groupId> @@ -701,6 +701,7 @@ under the License. <artifactId>maven-surefire-plugin</artifactId> <version>2.22.2</version> <configuration> + <useSystemClassLoader>false</useSystemClassLoader> <argLine>-Xmx2G -Dfile.encoding=UTF-8</argLine> </configuration> </plugin> @@ -712,7 +713,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> - <version>2.22.2</version> + <version>2.10</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId>
