This is an automated email from the ASF dual-hosted git repository. jwills pushed a commit to branch jwills_great_version_upgrade in repository https://gitbox.apache.org/repos/asf/crunch.git
commit d94800c556368829870a9bac64618dfe65d5699d Author: Josh Wills <[email protected]> AuthorDate: Wed Nov 13 15:14:51 2019 -0800 WIP for modernizing Crunch deps --- crunch-core/pom.xml | 9 +++-- .../io/parquet/AvroParquetFileSourceTargetIT.java | 2 +- .../crunch/io/parquet/AvroParquetFileSource.java | 2 +- .../crunch/util/CrunchRenameCopyListing.java | 4 +-- .../parquet/AvroParquetFileReaderFactoryTest.java | 2 +- crunch-kafka/pom.xml | 2 +- .../java/org/apache/crunch/kafka/KafkaUtils.java | 2 +- .../scrunch/spark/ByteBufferInputStream.scala | 2 +- crunch-test/pom.xml | 2 +- pom.xml | 41 ++++++++++++---------- 10 files changed, 38 insertions(+), 30 deletions(-) diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml index 1f487a2..26a0707 100644 --- a/crunch-core/pom.xml +++ b/crunch-core/pom.xml @@ -42,7 +42,6 @@ under the License. <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro-mapred</artifactId> - <classifier>${avro.classifier}</classifier> </dependency> <dependency> @@ -69,6 +68,12 @@ under the License. <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-distcp</artifactId> <scope>provided</scope> </dependency> @@ -142,7 +147,7 @@ under the License. <dependency> <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> + <artifactId>mockito-core</artifactId> <scope>test</scope> </dependency> diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java index a14a5e3..1b2b151 100644 --- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java @@ -153,7 +153,7 @@ public class AvroParquetFileSourceTargetIT implements Serializable { Person person = Iterables.getOnlyElement(ageOnly.materialize()); assertNull(person.getName()); - assertEquals(person.getAge(), new Integer(42)); + assertEquals(person.getAge(), 42); assertNull(person.getSiblingnames()); } diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java index 09703bd..9660b16 100644 --- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java +++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java @@ -186,7 +186,7 @@ public class AvroParquetFileSource<T extends IndexedRecord> extends FileSourceIm if (field == null) { throw new IllegalArgumentException("No field " + fieldName + " in schema: " + baseSchema.getName()); } - fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(), field.order())); + fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal(), field.order())); return this; } diff --git a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java index b930beb..7431d5d 100644 --- a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java +++ b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java @@ -219,7 +219,7 @@ public class CrunchRenameCopyListing extends SimpleCopyListing { DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath()), fileStatus.getPath()); } - if (!shouldCopy(fileStatus.getPath(), options)) { + if (!shouldCopy(fileStatus.getPath())) { return; } @@ -269,4 +269,4 @@ public class CrunchRenameCopyListing extends SimpleCopyListing { protected long getNumberOfPaths() { return totalPaths; } -} \ No newline at end of file +} diff --git a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java index d2ad5bc..95d4561 100644 --- a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java +++ b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java @@ -98,7 +98,7 @@ public class AvroParquetFileReaderFactoryTest { } public static Schema.Field cloneField(Schema.Field field) { - return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue()); + return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()); } } diff --git a/crunch-kafka/pom.xml b/crunch-kafka/pom.xml index 14817b6..1036808 100644 --- a/crunch-kafka/pom.xml +++ b/crunch-kafka/pom.xml @@ -40,7 +40,7 @@ under the License. </dependency> <dependency> <groupId>org.apache.kafka</groupId> - <artifactId>kafka_2.11</artifactId> + <artifactId>kafka_${scala.base.version}</artifactId> </dependency> <dependency> <groupId>org.scala-lang</groupId> diff --git a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java index 2681df0..0e9d750 100644 --- a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java +++ b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java @@ -34,7 +34,7 @@ import org.apache.crunch.CrunchRuntimeException; import org.apache.hadoop.conf.Configuration; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.network.ListenerName; -import org.apache.kafka.common.protocol.SecurityProtocol; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.Option; diff --git a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala index cc11b46..b43fa55 100644 --- a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala +++ b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala @@ -55,7 +55,7 @@ class ByteBufferInputStream(private var buffer: ByteBuffer) override def skip(bytes: Long): Long = { if (buffer != null) { val amountToSkip = math.min(bytes, buffer.remaining).toInt - buffer.position(buffer.position + amountToSkip) + buffer.position(buffer.position() + amountToSkip) if (buffer.remaining() == 0) { cleanUp() } diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml index b2c18dc..fb3110d 100644 --- a/crunch-test/pom.xml +++ b/crunch-test/pom.xml @@ -68,7 +68,7 @@ under the License. <dependency> <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> + <artifactId>mockito-core</artifactId> </dependency> <dependency> diff --git a/pom.xml b/pom.xml index 46d8838..05a21ad 100644 --- a/pom.xml +++ b/pom.xml @@ -72,8 +72,8 @@ under the License. <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <java.source.version>1.7</java.source.version> - <java.target.version>1.7</java.target.version> + <java.source.version>1.8</java.source.version> + <java.target.version>1.8</java.target.version> <!-- NOTE: Please don't change any dependency versions here without checking first that they match those on Hadoop's runtime @@ -85,7 +85,7 @@ under the License. <commons-lang.version>2.6</commons-lang.version> <commons-codec.version>1.4</commons-codec.version> <commons-cli.version>1.2</commons-cli.version> - <avro.version>1.7.7</avro.version> + <avro.version>1.9.1</avro.version> <hive.version>2.1.0</hive.version> <parquet.version>1.8.1</parquet.version> <javassist.version>3.16.1-GA</javassist.version> @@ -94,22 +94,21 @@ under the License. <libthrift.version>0.8.0</libthrift.version> <slf4j.version>1.6.1</slf4j.version> <log4j.version>1.2.15</log4j.version> - <algebird.version>0.9.0</algebird.version> + <algebird.version>0.13.2</algebird.version> <junit.version>4.10</junit.version> <hamcrest.version>1.1</hamcrest.version> - <mockito.version>1.9.0</mockito.version> + <mockito.version>2.23.4</mockito.version> <pkg>org.apache.crunch</pkg> - <hadoop.version>2.7.1</hadoop.version> + <hadoop.version>2.8.2</hadoop.version> <hbase.version>2.0.1</hbase.version> - <avro.classifier>hadoop2</avro.classifier> <hive.version>2.1.0</hive.version> - <kafka.version>0.10.2.1</kafka.version> - <scala.base.version>2.11</scala.base.version> - <scala.version>2.11.8</scala.version> - <scalatest.version>2.2.4</scalatest.version> - <spark.version>2.0.0</spark.version> + <kafka.version>1.1.0</kafka.version> + <scala.base.version>2.12</scala.base.version> + <scala.version>2.12.6</scala.version> + <scalatest.version>3.0.1</scalatest.version> + <spark.version>2.4.0</spark.version> <jline.version>2.12.1</jline.version> <jsr305.version>1.3.9</jsr305.version> </properties> @@ -202,6 +201,12 @@ under the License. <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs</artifactId> + <version>${hadoop.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> <version>${hadoop.version}</version> </dependency> @@ -258,7 +263,6 @@ under the License. <groupId>org.apache.avro</groupId> <artifactId>avro-mapred</artifactId> <version>${avro.version}</version> - <classifier>${avro.classifier}</classifier> <exclusions> <exclusion> <groupId>org.apache.avro</groupId> @@ -391,7 +395,7 @@ under the License. <dependency> <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> + <artifactId>mockito-core</artifactId> <version>${mockito.version}</version> </dependency> @@ -695,9 +699,9 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>2.18.1</version> + <version>2.22.2</version> <configuration> - <argLine>-Xmx2G -XX:PermSize=512m -XX:MaxPermSize=1G -Dfile.encoding=UTF-8</argLine> + <argLine>-Xmx2G -Dfile.encoding=UTF-8</argLine> </configuration> </plugin> <plugin> @@ -708,7 +712,7 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> - <version>2.10</version> + <version>2.22.2</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -837,9 +841,8 @@ under the License. <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId> - <version>2.18.1</version> + <version>2.22.2</version> <configuration> - <argLine>-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m -Dfile.encoding=UTF-8</argLine> <testSourceDirectory>${basedir}/src/it/java</testSourceDirectory> </configuration> <executions>
