BIGTOP-3082: Fix build failure with flume-1.8+kafka-0.10.2.2 When upgrading flume to 1.8 and kafka to 0.10.2.2, the build of flume is failed due to API changes and scala upgrade in kafka. This patch rebased FLUME-2662/FLUME-3026 patches to flume-1.8, and for the scala conflicts, used Anton's patch which can be found here: https://lists.apache.org/thread.html/%3CCAJZrK_tN49GM+vCCiB+rjLL==82U=7nswoapwWmyJbVAS5F=q...@mail.gmail.com%3E
This closes #397 Change-Id: Idf8d00c22ef60d82783056fea556e9a6f517054a Signed-off-by: Jun He <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/8488fb4c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/8488fb4c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/8488fb4c Branch: refs/heads/branch-1.3 Commit: 8488fb4c2f586cf54f3da22b2615e9e273b2503b Parents: 4d645af Author: Jun He <[email protected]> Authored: Thu Sep 20 01:48:25 2018 +0800 Committer: Jun He <[email protected]> Committed: Wed Sep 19 19:12:14 2018 +0000 ---------------------------------------------------------------------- .../src/common/flume/patch0-FLUME-2662.diff | 24 ++--- .../src/common/flume/patch1-FLUME-3026.diff | 92 ------------------ .../common/flume/patch1-FLUME-3026_rebased.diff | 99 ++++++++++++++++++++ bigtop.bom | 4 +- 4 files changed, 114 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/8488fb4c/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff b/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff index 9644c5e..61497b9 100644 --- a/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff +++ b/bigtop-packages/src/common/flume/patch0-FLUME-2662.diff @@ -1,11 +1,13 @@ ---- apache-flume-1.7.0-src/pom.xml. 2016-11-19 20:56:05.683682127 +0100 -+++ apache-flume-1.7.0-src/pom.xml 2016-11-19 20:56:22.707682094 +0100 -@@ -951,7 +951,7 @@ - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> -- <version>2.1</version> -+ <version>2.4</version> - </dependency> - - <dependency> +diff --git a/pom.xml b/pom.xml +index 3c82a47..bdd998d 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -58,7 +58,7 @@ limitations under the License. + <commons-collections.version>3.2.2</commons-collections.version> + <commons-compress.version>1.4.1</commons-compress.version> + <commons-dbcp.version>1.4</commons-dbcp.version> +- <commons-io.version>2.1</commons-io.version> ++ <commons-io.version>2.4</commons-io.version> + <commons-lang.version>2.5</commons-lang.version> + <curator.version>2.6.0</curator.version> + <derby.version>10.11.1.1</derby.version> http://git-wip-us.apache.org/repos/asf/bigtop/blob/8488fb4c/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff b/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff deleted file mode 100644 index 503de9d..0000000 --- a/bigtop-packages/src/common/flume/patch1-FLUME-3026.diff +++ /dev/null @@ -1,92 +0,0 @@ -diff --git a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java -index 5e5f2d0..917cee2 100644 ---- a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java -+++ b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java -@@ -20,6 +20,7 @@ package org.apache.flume.channel.kafka; - - import com.google.common.collect.Lists; - import kafka.admin.AdminUtils; -+import kafka.admin.RackAwareMode; - import kafka.utils.ZKGroupTopicDirs; - import kafka.utils.ZkUtils; - import org.apache.commons.lang.RandomStringUtils; -@@ -883,7 +884,8 @@ public class TestKafkaChannel { - ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false); - int replicationFactor = 1; - Properties topicConfig = new Properties(); -- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); -+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, -+ RackAwareMode.Disabled$.MODULE$); - } - - public static void deleteTopic(String topicName) { -diff --git a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java -index 7c66420..bc2a299 100644 ---- a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java -+++ b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java -@@ -21,6 +21,7 @@ package org.apache.flume.sink.kafka; - import com.google.common.base.Charsets; - - import kafka.admin.AdminUtils; -+import kafka.admin.RackAwareMode; - import kafka.message.MessageAndMetadata; - import kafka.utils.ZkUtils; - -@@ -551,7 +552,8 @@ public class TestKafkaSink { - ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false); - int replicationFactor = 1; - Properties topicConfig = new Properties(); -- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); -+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, -+ RackAwareMode.Disabled$.MODULE$); - } - - public static void deleteTopic(String topicName) { -diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java -index 53bd65c..ae5348c 100644 ---- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java -+++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java -@@ -17,6 +17,7 @@ - package org.apache.flume.source.kafka; - - import kafka.admin.AdminUtils; -+import kafka.admin.RackAwareMode; - import kafka.server.KafkaConfig; - import kafka.server.KafkaServerStartable; - import kafka.utils.ZkUtils; -@@ -131,7 +132,8 @@ public class KafkaSourceEmbeddedKafka { - ZkUtils zkUtils = ZkUtils.apply(zkClient, false); - int replicationFactor = 1; - Properties topicConfig = new Properties(); -- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); -+ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, -+ RackAwareMode.Disabled$.MODULE$); - } - - } -diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java -index d1daceb..cda91f9 100644 ---- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java -+++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java -@@ -20,7 +20,7 @@ package org.apache.flume.source.kafka; - import com.google.common.base.Charsets; - import com.google.common.collect.Lists; - import junit.framework.Assert; --import kafka.common.TopicExistsException; -+import org.apache.kafka.common.errors.TopicExistsException; - import kafka.utils.ZKGroupTopicDirs; - import kafka.utils.ZkUtils; - import org.apache.avro.io.BinaryEncoder; -diff --git a/pom.xml b/pom.xml -index f62c99a..fb2340f 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -52,7 +52,7 @@ limitations under the License. - <elasticsearch.version>0.90.1</elasticsearch.version> - <hadoop2.version>2.4.0</hadoop2.version> - <thrift.version>0.7.0</thrift.version> -- <kafka.version>0.9.0.1</kafka.version> -+ <kafka.version>0.10.1.0</kafka.version> - <kite.version>1.0.0</kite.version> - <hive.version>1.0.0</hive.version> - <xalan.verion>2.7.1</xalan.verion> http://git-wip-us.apache.org/repos/asf/bigtop/blob/8488fb4c/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff b/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff new file mode 100644 index 0000000..20a2ddd --- /dev/null +++ b/bigtop-packages/src/common/flume/patch1-FLUME-3026_rebased.diff @@ -0,0 +1,99 @@ +diff --git a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java +index 5e5f2d0..63607f7 100644 +--- a/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java ++++ b/flume-ng-channels/flume-kafka-channel/src/test/java/org/apache/flume/channel/kafka/TestKafkaChannel.java +@@ -20,6 +20,7 @@ package org.apache.flume.channel.kafka; + + import com.google.common.collect.Lists; + import kafka.admin.AdminUtils; ++import kafka.admin.RackAwareMode; + import kafka.utils.ZKGroupTopicDirs; + import kafka.utils.ZkUtils; + import org.apache.commons.lang.RandomStringUtils; +@@ -883,7 +884,8 @@ public class TestKafkaChannel { + ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false); + int replicationFactor = 1; + Properties topicConfig = new Properties(); +- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); ++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, ++ RackAwareMode.Disabled$.MODULE$); + } + + public static void deleteTopic(String topicName) { +diff --git a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java +index d92c71f..66c6fe3 100644 +--- a/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java ++++ b/flume-ng-sinks/flume-ng-kafka-sink/src/test/java/org/apache/flume/sink/kafka/TestKafkaSink.java +@@ -21,6 +21,7 @@ package org.apache.flume.sink.kafka; + import com.google.common.base.Charsets; + + import kafka.admin.AdminUtils; ++import kafka.admin.RackAwareMode; + import kafka.message.MessageAndMetadata; + import kafka.utils.ZkUtils; + +@@ -674,7 +675,8 @@ public class TestKafkaSink { + ZkUtils.apply(testUtil.getZkUrl(), sessionTimeoutMs, connectionTimeoutMs, false); + int replicationFactor = 1; + Properties topicConfig = new Properties(); +- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); ++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, ++ RackAwareMode.Disabled$.MODULE$); + } + + public static void deleteTopic(String topicName) { +@@ -698,4 +700,4 @@ public class TestKafkaSink { + return newTopic; + } + +-} +\ No newline at end of file ++} +diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java +index 53bd65c..ba75623 100644 +--- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java ++++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/KafkaSourceEmbeddedKafka.java +@@ -17,6 +17,7 @@ + package org.apache.flume.source.kafka; + + import kafka.admin.AdminUtils; ++import kafka.admin.RackAwareMode; + import kafka.server.KafkaConfig; + import kafka.server.KafkaServerStartable; + import kafka.utils.ZkUtils; +@@ -131,7 +132,8 @@ public class KafkaSourceEmbeddedKafka { + ZkUtils zkUtils = ZkUtils.apply(zkClient, false); + int replicationFactor = 1; + Properties topicConfig = new Properties(); +- AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig); ++ AdminUtils.createTopic(zkUtils, topicName, numPartitions, replicationFactor, topicConfig, ++ RackAwareMode.Disabled$.MODULE$); + } + + } +diff --git a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java +index 7804fa2..2d5bbf8 100644 +--- a/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java ++++ b/flume-ng-sources/flume-kafka-source/src/test/java/org/apache/flume/source/kafka/TestKafkaSource.java +@@ -20,7 +20,7 @@ package org.apache.flume.source.kafka; + import com.google.common.base.Charsets; + import com.google.common.collect.Lists; + import junit.framework.Assert; +-import kafka.common.TopicExistsException; ++import org.apache.kafka.common.errors.TopicExistsException; + import kafka.utils.ZKGroupTopicDirs; + import kafka.utils.ZkUtils; + import org.apache.avro.io.BinaryEncoder; +diff --git a/pom.xml b/pom.xml +index 3c82a47..2276355 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -77,7 +77,7 @@ limitations under the License. + <jetty.version>6.1.26</jetty.version> + <joda-time.version>2.9.9</joda-time.version> + <junit.version>4.10</junit.version> +- <kafka.version>0.9.0.1</kafka.version> ++ <kafka.version>0.10.2.2</kafka.version> + <kite.version>1.0.0</kite.version> + <hive.version>1.0.0</hive.version> + <lifecycle-mapping.version>1.0.0</lifecycle-mapping.version> http://git-wip-us.apache.org/repos/asf/bigtop/blob/8488fb4c/bigtop.bom ---------------------------------------------------------------------- diff --git a/bigtop.bom b/bigtop.bom index 3c90bd4..ca8caad 100644 --- a/bigtop.bom +++ b/bigtop.bom @@ -225,7 +225,7 @@ bigtop { 'flume' { name = 'flume' relNotes = 'Apache Flume' - version { base = '1.7.0'; pkg = base; release = 1 } + version { base = '1.8.0'; pkg = base; release = 1 } tarball { destination = "apache-$name-${version.base}-src.tar.gz" source = destination } url { download_path = "/$name/${version.base}/" @@ -373,7 +373,7 @@ bigtop { 'kafka' { name = 'kafka' relNotes = 'Apache Kafka' - version { base = '0.10.1.1'; pkg = base; release = 1 } + version { base = '0.10.2.2'; pkg = base; release = 1 } tarball { destination = "$name-${version.base}.tar.gz" source = "$name-${version.base}-src.tgz" } url { download_path = "/$name/${version.base}/"
