Repository: atlas Updated Branches: refs/heads/master ac2c1a779 -> 60789da3b
ATLAS-2495: updated Kafka version to 1.0.0 Project: http://git-wip-us.apache.org/repos/asf/atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/60789da3 Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/60789da3 Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/60789da3 Branch: refs/heads/master Commit: 60789da3b95366a256ae1020241bb9a9ec0817de Parents: ac2c1a7 Author: Madhan Neethiraj <mad...@apache.org> Authored: Sun Mar 11 17:34:08 2018 -0700 Committer: Madhan Neethiraj <mad...@apache.org> Committed: Tue Mar 13 17:40:40 2018 -0700 ---------------------------------------------------------------------- .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + .../src/test/resources/atlas-application.properties | 1 + intg/src/test/resources/atlas-application.properties | 1 + .../org/apache/atlas/kafka/EmbeddedKafkaServer.java | 15 +++++++++++++-- .../atlas/kafka/KafkaNotificationMockTest.java | 3 ++- pom.xml | 2 +- .../apache/atlas/web/integration/BaseResourceIT.java | 2 -- .../src/test/resources/atlas-application.properties | 1 + 14 files changed, 26 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/falcon-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/resources/atlas-application.properties b/addons/falcon-bridge/src/test/resources/atlas-application.properties index d1c3f71..d9fd0cc 100644 --- a/addons/falcon-bridge/src/test/resources/atlas-application.properties +++ b/addons/falcon-bridge/src/test/resources/atlas-application.properties @@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/hbase-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/hbase-bridge/src/test/resources/atlas-application.properties b/addons/hbase-bridge/src/test/resources/atlas-application.properties index d1c3f71..d9fd0cc 100644 --- a/addons/hbase-bridge/src/test/resources/atlas-application.properties +++ b/addons/hbase-bridge/src/test/resources/atlas-application.properties @@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/hive-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/resources/atlas-application.properties b/addons/hive-bridge/src/test/resources/atlas-application.properties index d1c3f71..d9fd0cc 100644 --- a/addons/hive-bridge/src/test/resources/atlas-application.properties +++ b/addons/hive-bridge/src/test/resources/atlas-application.properties @@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/sqoop-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/resources/atlas-application.properties b/addons/sqoop-bridge/src/test/resources/atlas-application.properties index d1c3f71..d9fd0cc 100644 --- a/addons/sqoop-bridge/src/test/resources/atlas-application.properties +++ b/addons/sqoop-bridge/src/test/resources/atlas-application.properties @@ -79,6 +79,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/addons/storm-bridge/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/test/resources/atlas-application.properties b/addons/storm-bridge/src/test/resources/atlas-application.properties index 6e9042a..b92c791 100644 --- a/addons/storm-bridge/src/test/resources/atlas-application.properties +++ b/addons/storm-bridge/src/test/resources/atlas-application.properties @@ -81,6 +81,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/authorization/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/authorization/src/test/resources/atlas-application.properties b/authorization/src/test/resources/atlas-application.properties index e1357b1..3e41581 100644 --- a/authorization/src/test/resources/atlas-application.properties +++ b/authorization/src/test/resources/atlas-application.properties @@ -97,6 +97,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/graphdb/janus/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/graphdb/janus/src/test/resources/atlas-application.properties b/graphdb/janus/src/test/resources/atlas-application.properties index 5c9bd7a..a355234 100644 --- a/graphdb/janus/src/test/resources/atlas-application.properties +++ b/graphdb/janus/src/test/resources/atlas-application.properties @@ -74,6 +74,7 @@ atlas.kafka.consumer.timeout.ms=100 atlas.kafka.auto.commit.interval.ms=100 atlas.kafka.hook.group.id=atlas atlas.kafka.entities.group.id=atlas_entities +atlas.kafka.offsets.topic.replication.factor=1 ######### Entity Audit Configs ######### atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/graphdb/titan0/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/graphdb/titan0/src/test/resources/atlas-application.properties b/graphdb/titan0/src/test/resources/atlas-application.properties index c6ea4e0..6188873 100644 --- a/graphdb/titan0/src/test/resources/atlas-application.properties +++ b/graphdb/titan0/src/test/resources/atlas-application.properties @@ -75,6 +75,7 @@ atlas.kafka.consumer.timeout.ms=100 atlas.kafka.auto.commit.interval.ms=100 atlas.kafka.hook.group.id=atlas atlas.kafka.entities.group.id=atlas_entities +atlas.kafka.offsets.topic.replication.factor=1 ######### Entity Audit Configs ######### atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/intg/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/intg/src/test/resources/atlas-application.properties b/intg/src/test/resources/atlas-application.properties index 9015f89..5470163 100644 --- a/intg/src/test/resources/atlas-application.properties +++ b/intg/src/test/resources/atlas-application.properties @@ -97,6 +97,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1 http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java ---------------------------------------------------------------------- diff --git a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java index 33c8296..32b597f 100644 --- a/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java +++ b/notification/src/main/java/org/apache/atlas/kafka/EmbeddedKafkaServer.java @@ -17,15 +17,16 @@ */ package org.apache.atlas.kafka; +import kafka.metrics.KafkaMetricsReporter; import kafka.server.KafkaConfig; import kafka.server.KafkaServer; -import kafka.utils.Time; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.service.Service; import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.ConfigurationConverter; import org.apache.kafka.clients.producer.*; +import org.apache.kafka.common.utils.Time; import org.apache.zookeeper.server.NIOServerCnxnFactory; import org.apache.zookeeper.server.ServerCnxnFactory; import org.apache.zookeeper.server.ZooKeeperServer; @@ -34,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import scala.Option; +import scala.collection.mutable.Buffer; import javax.inject.Inject; import java.io.File; @@ -43,6 +45,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.*; +import java.util.concurrent.TimeUnit; @Component @@ -135,7 +138,10 @@ public class EmbeddedKafkaServer implements Service { brokerConfig.setProperty("log.dirs", constructDir("kafka").getAbsolutePath()); brokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1)); - kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(), Option.apply(this.getClass().getName())); + List<KafkaMetricsReporter> metrics = new ArrayList<>(); + Buffer<KafkaMetricsReporter> metricsReporters = scala.collection.JavaConversions.asScalaBuffer(metrics); + + kafkaServer = new KafkaServer(KafkaConfig.fromProps(brokerConfig), new SystemTime(), Option.apply(this.getClass().getName()), metricsReporters); kafkaServer.startup(); @@ -174,6 +180,11 @@ public class EmbeddedKafkaServer implements Service { } @Override + public long hiResClockMs() { + return TimeUnit.NANOSECONDS.toMillis(nanoseconds()); + } + + @Override public void sleep(long arg0) { try { Thread.sleep(arg0); http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java ---------------------------------------------------------------------- diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java index 78d2a90..263903b 100644 --- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java +++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationMockTest.java @@ -77,7 +77,8 @@ public class KafkaNotificationMockTest { String topicName = kafkaNotification.getTopicName(NotificationInterface.NotificationType.HOOK); String message = "This is a test message"; Future returnValue = mock(Future.class); - when(returnValue.get()).thenReturn(new RecordMetadata(new TopicPartition(topicName, 0), 0, 0)); + TopicPartition topicPartition = new TopicPartition(topicName, 0); + when(returnValue.get()).thenReturn(new RecordMetadata(topicPartition, 0, 0, 0, Long.valueOf(0), 0, 0)); ProducerRecord expectedRecord = new ProducerRecord(topicName, message); when(producer.send(expectedRecord)).thenReturn(returnValue); http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 34e1ad0..9ddc8e1 100644 --- a/pom.xml +++ b/pom.xml @@ -546,7 +546,7 @@ <hadoop.version>2.7.1</hadoop.version> <hbase.version>1.1.2</hbase.version> <solr.version>5.5.1</solr.version> - <kafka.version>0.10.0.0</kafka.version> + <kafka.version>1.0.0</kafka.version> <kafka.scala.binary.version>2.11</kafka.scala.binary.version> <curator.version>2.11.0</curator.version> <zookeeper.version>3.4.6</zookeeper.version> http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java index 6a7aece..6f6a7ee 100755 --- a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java +++ b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java @@ -36,7 +36,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; import org.apache.atlas.notification.NotificationInterface; -import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.v1.model.instance.Id; import org.apache.atlas.v1.model.instance.Referenceable; import org.apache.atlas.v1.model.instance.Struct; @@ -57,7 +56,6 @@ import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.BeforeClass; -import javax.inject.Inject; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; http://git-wip-us.apache.org/repos/asf/atlas/blob/60789da3/webapp/src/test/resources/atlas-application.properties ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/atlas-application.properties b/webapp/src/test/resources/atlas-application.properties index 402d362..1e251bc 100644 --- a/webapp/src/test/resources/atlas-application.properties +++ b/webapp/src/test/resources/atlas-application.properties @@ -80,6 +80,7 @@ atlas.kafka.entities.group.id=atlas_entities atlas.kafka.enable.auto.commit=false atlas.kafka.auto.offset.reset=earliest atlas.kafka.session.timeout.ms=30000 +atlas.kafka.offsets.topic.replication.factor=1