This is an automated email from the ASF dual-hosted git repository.
mikexue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/eventmesh.git
The following commit(s) were added to refs/heads/master by this push:
new 6c94d9deb [ISSUE #4011] add kafkaConnector module (#4180)
6c94d9deb is described below
commit 6c94d9deb1b24af946fa3ae3e92beaead72f5ffc
Author: Jett <[email protected]>
AuthorDate: Fri Jul 21 10:02:52 2023 +0800
[ISSUE #4011] add kafkaConnector module (#4180)
* [ISSUE apache#4011] add kafkaConnector module
* [ISSUE apache#4011] add kafkaConnector module
* [ISSUE apache#4011] fix codestyle
* Fixes #4011.
add default config for kafka sink and source
---
.../eventmesh-connector-kafka/build.gradle | 24 +++++
.../eventmesh-connector-kafka/gradle.properties | 18 ++++
.../connector/kafka/config/KafkaServerConfig.java | 31 ++++++
.../connector/kafka/server/KafkaConnectServer.java | 46 ++++++++
.../kafka/sink/config/KafkaSinkConfig.java | 29 +++++
.../kafka/sink/config/SinkConnectorConfig.java | 39 +++++++
.../kafka/sink/connector/KafkaSinkConnector.java | 116 ++++++++++++++++++++
.../kafka/source/config/KafkaSourceConfig.java | 28 +++++
.../kafka/source/config/SourceConnectorConfig.java | 37 +++++++
.../source/connector/KafkaSourceConnector.java | 119 +++++++++++++++++++++
.../src/main/resources/server-config.yml | 19 ++++
.../src/main/resources/sink-config.yml | 33 ++++++
.../src/main/resources/source-config.yml | 32 ++++++
settings.gradle | 1 +
14 files changed, 572 insertions(+)
diff --git a/eventmesh-connectors/eventmesh-connector-kafka/build.gradle
b/eventmesh-connectors/eventmesh-connector-kafka/build.gradle
new file mode 100644
index 000000000..98176995b
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-kafka/build.gradle
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+dependencies {
+ implementation project(":eventmesh-openconnect:eventmesh-openconnect-java")
+ implementation 'io.cloudevents:cloudevents-kafka:2.4.2'
+ implementation 'org.apache.kafka:kafka-clients:3.0.0'
+ compileOnly 'org.projectlombok:lombok'
+ annotationProcessor 'org.projectlombok:lombok'
+}
\ No newline at end of file
diff --git a/eventmesh-connectors/eventmesh-connector-kafka/gradle.properties
b/eventmesh-connectors/eventmesh-connector-kafka/gradle.properties
new file mode 100644
index 000000000..4486939e8
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-kafka/gradle.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+pluginType=connector
+pluginName=kafka
\ No newline at end of file
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java
new file mode 100644
index 000000000..2c9ef71fb
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.config;
+
+import org.apache.eventmesh.openconnect.api.config.Config;
+
+import lombok.Data;
+
+@Data
+public class KafkaServerConfig extends Config {
+
+ private boolean sourceEnable;
+
+ private boolean sinkEnable;
+
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java
new file mode 100644
index 000000000..3115b5bf4
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/server/KafkaConnectServer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.server;
+
+import org.apache.eventmesh.connector.kafka.config.KafkaServerConfig;
+import org.apache.eventmesh.connector.kafka.sink.connector.KafkaSinkConnector;
+import
org.apache.eventmesh.connector.kafka.source.connector.KafkaSourceConnector;
+import org.apache.eventmesh.openconnect.Application;
+import org.apache.eventmesh.openconnect.util.ConfigUtil;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class KafkaConnectServer {
+
+
+ public static void main(String[] args) throws Exception {
+
+ KafkaServerConfig serverConfig =
ConfigUtil.parse(KafkaServerConfig.class, "server-config.yml");
+
+ if (serverConfig.isSourceEnable()) {
+ Application kafkaSourceApp = new Application();
+ kafkaSourceApp.run(KafkaSourceConnector.class);
+ }
+
+ if (serverConfig.isSinkEnable()) {
+ Application kafkaSinkApp = new Application();
+ kafkaSinkApp.run(KafkaSinkConnector.class);
+ }
+ }
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java
new file mode 100644
index 000000000..9bb79551d
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.sink.config;
+
+import org.apache.eventmesh.openconnect.api.config.SinkConfig;
+
+import lombok.Data;
+
+@Data
+public class KafkaSinkConfig extends SinkConfig {
+
+ public SinkConnectorConfig connectorConfig;
+
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java
new file mode 100644
index 000000000..a240bf4f4
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.sink.config;
+
+import lombok.Data;
+
+@Data
+public class SinkConnectorConfig {
+
+ private String connectorName = "kafkaSink";
+ private String topic = "TopicTest";
+ private String ack = "all";
+ private String bootstrapServers = "127.0.0.1:9092";
+ private String keyConverter =
"org.apache.kafka.common.serialization.StringSerializer";
+ private String valueConverter =
"org.apache.kafka.common.serialization.StringSerializer";
+ private String maxRequestSize = "1048576";
+ private String bufferMemory = "33554432";
+ private String batchSize = "16384";
+ private String lingerMs = "0";
+ private String requestTimeoutMs = "30000";
+ private String maxInFightRequestsPerConnection = "5";
+ private String retries = "0";
+ private String compressionType = "none";
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java
new file mode 100644
index 000000000..01a36208e
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.sink.connector;
+
+import org.apache.eventmesh.connector.kafka.sink.config.KafkaSinkConfig;
+import org.apache.eventmesh.openconnect.api.config.Config;
+import org.apache.eventmesh.openconnect.api.data.ConnectRecord;
+import org.apache.eventmesh.openconnect.api.sink.Sink;
+
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.Producer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.header.Header;
+import org.apache.kafka.common.header.internals.RecordHeader;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class KafkaSinkConnector implements Sink {
+
+ private KafkaSinkConfig sinkConfig;
+
+ private Properties props = new Properties();
+ Producer<String, String> producer;
+
+ @Override
+ public Class<? extends Config> configClass() {
+ return KafkaSinkConfig.class;
+ }
+
+ @Override
+ public void init(Config config) {
+ this.sinkConfig = (KafkaSinkConfig) config;
+ props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG,
sinkConfig.getConnectorConfig().getBootstrapServers());
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
sinkConfig.getConnectorConfig().getKeyConverter());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
sinkConfig.getConnectorConfig().getValueConverter());
+ props.put(ProducerConfig.ACKS_CONFIG,
sinkConfig.getConnectorConfig().getAck());
+ props.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG,
sinkConfig.getConnectorConfig().getMaxRequestSize());
+ props.put(ProducerConfig.BUFFER_MEMORY_CONFIG,
sinkConfig.getConnectorConfig().getBufferMemory());
+ props.put(ProducerConfig.BATCH_SIZE_CONFIG,
sinkConfig.getConnectorConfig().getBatchSize());
+ props.put(ProducerConfig.LINGER_MS_CONFIG,
sinkConfig.getConnectorConfig().getLingerMs());
+ props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG,
sinkConfig.getConnectorConfig().getRequestTimeoutMs());
+ props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION,
sinkConfig.getConnectorConfig().getMaxInFightRequestsPerConnection());
+ props.put(ProducerConfig.RETRIES_CONFIG,
sinkConfig.getConnectorConfig().getRetries());
+ props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG,
sinkConfig.getConnectorConfig().getCompressionType());
+ producer = new KafkaProducer<>(props);
+ }
+
+ @Override
+ public void start() throws Exception {
+ }
+
+ @Override
+ public void commit(ConnectRecord record) {
+
+ }
+
+ @Override
+ public String name() {
+ return this.sinkConfig.getConnectorConfig().getConnectorName();
+ }
+
+ @Override
+ public void stop() {
+ producer.close();
+ }
+
+ @Override
+ public void put(List<ConnectRecord> sinkRecords) {
+ try {
+ for (ConnectRecord connectRecord : sinkRecords) {
+ ProducerRecord message = convertRecordToMessage(connectRecord);
+ producer.send(message, (metadata, exception) -> {
+ if (exception == null) {
+ log.debug("Produced message to
topic:{},partition:{},offset:{}", metadata.topic(), metadata.partition(),
metadata.offset());
+ } else {
+ log.error("Failed to produce message:{}",
exception.getMessage());
+ }
+ });
+ }
+ } catch (Exception e) {
+ log.error("Failed to produce message:{}", e.getMessage());
+ }
+ }
+
+ public ProducerRecord convertRecordToMessage(ConnectRecord connectRecord) {
+ List<Header> headers = new ArrayList<>();
+ for (String key : connectRecord.getExtensions().keySet()) {
+ headers.add(new RecordHeader(key,
connectRecord.getExtension(key).getBytes(StandardCharsets.UTF_8)));
+ }
+ ProducerRecord message = new
ProducerRecord(this.sinkConfig.getConnectorConfig().getTopic(), null, "",
+ new String((byte[]) connectRecord.getData(),
StandardCharsets.UTF_8), headers);
+ return message;
+ }
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java
new file mode 100644
index 000000000..4319ec96d
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.source.config;
+
+import org.apache.eventmesh.openconnect.api.config.SourceConfig;
+
+import lombok.Data;
+
+@Data
+public class KafkaSourceConfig extends SourceConfig {
+
+ public SourceConnectorConfig connectorConfig;
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java
new file mode 100644
index 000000000..acfb9ffe7
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.eventmesh.connector.kafka.source.config;
+
+import lombok.Data;
+
+@Data
+public class SourceConnectorConfig {
+
+ private String connectorName = "kafkaSource";
+ private String topic = "TopicTest";
+ private String bootstrapServers = "127.0.0.1:9092";
+ private String groupID = "kafkaSource";
+ private String keyConverter =
"org.apache.kafka.common.serialization.StringSerializer";
+ private String valueConverter =
"org.apache.kafka.common.serialization.StringSerializer";
+ private String autoCommitIntervalMS = "1000";
+ private String enableAutoCommit = "false";
+ private String sessionTimeoutMS = "3000";
+ private String maxPollRecords = "1000";
+ private int pollTimeOut = 100;
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java
new file mode 100644
index 000000000..64487c63d
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.kafka.source.connector;
+
+import org.apache.eventmesh.connector.kafka.source.config.KafkaSourceConfig;
+import org.apache.eventmesh.openconnect.api.config.Config;
+import org.apache.eventmesh.openconnect.api.data.ConnectRecord;
+import org.apache.eventmesh.openconnect.api.data.RecordOffset;
+import org.apache.eventmesh.openconnect.api.data.RecordPartition;
+import org.apache.eventmesh.openconnect.api.source.Source;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class KafkaSourceConnector implements Source {
+
+ private KafkaSourceConfig sourceConfig;
+
+ private KafkaConsumer<String, String> kafkaConsumer;
+
+ private int pollTimeOut = 100;
+
+ @Override
+ public Class<? extends Config> configClass() {
+ return KafkaSourceConfig.class;
+ }
+
+ @Override
+ public void init(Config config) throws Exception {
+ this.sourceConfig = (KafkaSourceConfig) config;
+ Properties props = new Properties();
+ props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
sourceConfig.getConnectorConfig().getBootstrapServers());
+ props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
sourceConfig.getConnectorConfig().getKeyConverter());
+ props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
sourceConfig.getConnectorConfig().getValueConverter());
+ props.put(ConsumerConfig.GROUP_ID_CONFIG,
sourceConfig.getConnectorConfig().getGroupID());
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,
sourceConfig.getConnectorConfig().getEnableAutoCommit());
+ props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG,
sourceConfig.getConnectorConfig().getMaxPollRecords());
+ props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,
sourceConfig.getConnectorConfig().getAutoCommitIntervalMS());
+ props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG,
sourceConfig.getConnectorConfig().getSessionTimeoutMS());
+ this.pollTimeOut = sourceConfig.getConnectorConfig().getPollTimeOut();
+ this.kafkaConsumer = new KafkaConsumer<String, String>(props);
+ }
+
+ @Override
+ public void start() throws Exception {
+
kafkaConsumer.subscribe(Collections.singleton(sourceConfig.getConnectorConfig().getTopic()));
+ }
+
+ @Override
+ public void commit(ConnectRecord record) {
+
+ }
+
+ @Override
+ public String name() {
+ return this.sourceConfig.getConnectorConfig().getConnectorName();
+ }
+
+ @Override
+ public void stop() {
+ kafkaConsumer.unsubscribe();
+ }
+
+ @Override
+ public List<ConnectRecord> poll() {
+ ConsumerRecords<String, String> records =
kafkaConsumer.poll(Duration.ofMillis(pollTimeOut));
+ List<ConnectRecord> connectRecords = new ArrayList<>(records.count());
+ for (ConsumerRecord<String, String> record : records) {
+ Long timestamp = System.currentTimeMillis();
+ String key = record.key();
+ String value = record.value();
+ RecordPartition recordPartition =
convertToRecordPartition(record.topic(), record.partition());
+ RecordOffset recordOffset = convertToRecordOffset(record.offset());
+ ConnectRecord connectRecord = new ConnectRecord(recordPartition,
recordOffset, timestamp, value);
+ connectRecord.addExtension("key", key);
+ connectRecords.add(connectRecord);
+ }
+ kafkaConsumer.commitAsync();
+ return connectRecords;
+ }
+
+ public static RecordOffset convertToRecordOffset(Long offset) {
+ Map<String, String> offsetMap = new HashMap<>();
+ offsetMap.put("queueOffset", offset + "");
+ return new RecordOffset(offsetMap);
+ }
+
+ public static RecordPartition convertToRecordPartition(String topic, int
partition) {
+ Map<String, String> map = new HashMap<>();
+ map.put("topic", topic);
+ map.put("partition", String.valueOf(partition));
+ return new RecordPartition(map);
+ }
+}
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml
new file mode 100644
index 000000000..0cd7b5b5a
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/server-config.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+sourceEnable: true
+sinkEnable: false
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml
new file mode 100644
index 000000000..241687825
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/sink-config.yml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pubSubConfig:
+ meshAddress: 127.0.0.1:10000
+ subject: TopicTest
+ idc: FT
+ env: PRD
+ group: kafkaSink
+ appId: 5031
+ userName: kafkaSinkUser
+ passWord: kafkaPassWord
+connectorConfig:
+ connectorName: kafkaSink
+ bootstrapServers:
+ topic: TopicTest
+ keyConverter: org.apache.kafka.common.serialization.StringSerializer
+ valueConverter: org.apache.kafka.common.serialization.StringSerializer
+
diff --git
a/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml
new file mode 100644
index 000000000..4376df801
--- /dev/null
+++
b/eventmesh-connectors/eventmesh-connector-kafka/src/main/resources/source-config.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pubSubConfig:
+ meshAddress: 127.0.0.1:10000
+ subject: TopicTest
+ idc: FT
+ env: PRD
+ group: kafkaSource
+ appId: 5032
+ userName: kafkaSourceUser
+ passWord: kafkaPassWord
+connectorConfig:
+ connectorName: kafkaSource
+ bootstrapServers: 127.0.0.1:9090
+ topic: TopicTest
+ groupID: kafkaSource
+ maxPollRecords: 1000
diff --git a/settings.gradle b/settings.gradle
index 405d64d26..84ee32037 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -26,6 +26,7 @@ include 'eventmesh-spi'
include 'eventmesh-openconnect:eventmesh-openconnect-java'
include 'eventmesh-connectors:eventmesh-connector-rocketmq'
include 'eventmesh-connectors:eventmesh-connector-openfunction'
+include 'eventmesh-connectors:eventmesh-connector-kafka'
include 'eventmesh-storage-plugin:eventmesh-storage-api'
include 'eventmesh-storage-plugin:eventmesh-storage-standalone'
include 'eventmesh-storage-plugin:eventmesh-storage-kafka'
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
