Repository: hive Updated Branches: refs/heads/master b362de387 -> 41afa4e21
HIVE-18436: Upgrade to Spark 2.3.0 (Sahil Takiar, reviewed by Rui Li) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/41afa4e2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/41afa4e2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/41afa4e2 Branch: refs/heads/master Commit: 41afa4e2185b45b0bb6816baaea330d9ed40d127 Parents: b362de3 Author: Sahil Takiar <takiar.sa...@gmail.com> Authored: Fri Mar 9 18:54:42 2018 -0800 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Fri Mar 9 18:54:42 2018 -0800 ---------------------------------------------------------------------- itests/pom.xml | 4 ++-- pom.xml | 4 ++-- .../apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java | 2 +- .../org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java | 3 ++- .../test/java/org/apache/hive/spark/client/rpc/TestRpc.java | 6 ++++-- 5 files changed, 11 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/itests/pom.xml ---------------------------------------------------------------------- diff --git a/itests/pom.xml b/itests/pom.xml index a782cd2..40bb054 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -99,10 +99,10 @@ cd - fi tar -zxf $DOWNLOAD_DIR/$tarName -C $BASE_DIR - mv $BASE_DIR/spark-${spark.version}-bin-hadoop2-without-hive $BASE_DIR/$finalName + mv $BASE_DIR/spark-${spark.version}-bin-hadoop3-beta1-without-hive $BASE_DIR/$finalName } mkdir -p $DOWNLOAD_DIR - download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark" + download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop3-beta1-without-hive.tgz" "spark" cp -f $HIVE_ROOT/data/conf/spark/log4j2.properties $BASE_DIR/spark/conf/ </echo> </target> http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index a242fbf..5be30f6 100644 --- a/pom.xml +++ b/pom.xml @@ -185,7 +185,7 @@ <orc.version>1.4.3</orc.version> <mockito-all.version>1.10.19</mockito-all.version> <mina.version>2.0.0-M5</mina.version> - <netty.version>4.0.52.Final</netty.version> + <netty.version>4.1.17.Final</netty.version> <netty3.version>3.10.5.Final</netty3.version> <parquet.version>1.9.0</parquet.version> <pig.version>0.16.0</pig.version> @@ -198,7 +198,7 @@ <tez.version>0.9.1</tez.version> <slider.version>0.92.0-incubating</slider.version> <super-csv.version>2.2.0</super-csv.version> - <spark.version>2.2.0</spark.version> + <spark.version>2.3.0</spark.version> <scala.binary.version>2.11</scala.binary.version> <scala.version>2.11.8</scala.version> <tempus-fugit.version>1.1</tempus-fugit.version> http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java index f43b449..dff3b0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/LocalHiveSparkClient.java @@ -104,7 +104,7 @@ public class LocalHiveSparkClient implements HiveSparkClient { sc.addJar(regJar); } jobMetricsListener = new JobMetricsListener(); - sc.sc().listenerBus().addListener(jobMetricsListener); + sc.sc().addSparkListener(jobMetricsListener); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java ---------------------------------------------------------------------- diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java index 24858d7..fb73647 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestKryoMessageCodec.java @@ -72,7 +72,8 @@ public class TestKryoMessageCodec { c.writeAndFlush(MESSAGE); assertEquals(1, c.outboundMessages().size()); assertFalse(MESSAGE.getClass().equals(c.outboundMessages().peek().getClass())); - c.writeInbound(c.readOutbound()); + Object readOutboundResult = c.readOutbound(); + c.writeInbound(readOutboundResult); assertEquals(1, c.inboundMessages().size()); assertEquals(MESSAGE, c.readInbound()); c.close(); http://git-wip-us.apache.org/repos/asf/hive/blob/41afa4e2/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java ---------------------------------------------------------------------- diff --git a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java index 21e4595..fafdff7 100644 --- a/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java +++ b/spark-client/src/test/java/org/apache/hive/spark/client/rpc/TestRpc.java @@ -323,7 +323,8 @@ public class TestRpc { int count = 0; while (!client.outboundMessages().isEmpty()) { - server.writeInbound(client.readOutbound()); + Object readOutboundResult = client.readOutbound(); + server.writeInbound(readOutboundResult); count++; } server.flush(); @@ -331,7 +332,8 @@ public class TestRpc { count = 0; while (!server.outboundMessages().isEmpty()) { - client.writeInbound(server.readOutbound()); + Object readOutboundResult = server.readOutbound(); + client.writeInbound(readOutboundResult); count++; } client.flush();