sijie commented on a change in pull request #2998: support mysql binlog sync to
pulsar by canal
URL: https://github.com/apache/pulsar/pull/2998#discussion_r234357262
##########
File path:
pulsar-io/canal/src/main/java/org/apache/pulsar/io/canal/CanalSource.java
##########
@@ -0,0 +1,152 @@
+package org.apache.pulsar.io.canal;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalConnectors;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.pulsar.functions.api.Record;
+import org.apache.pulsar.io.core.PushSource;
+import org.apache.pulsar.io.core.SourceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.util.Assert;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+@Slf4j
+public class CanalSource extends PushSource<byte[]> {
+
+ protected Thread thread = null;
+
+ protected volatile boolean running = false;
+
+ private CanalConnector connector;
+
+ private CanalSourceConfig canalSourceConfig;
+
+ protected Thread.UncaughtExceptionHandler handler = new
Thread.UncaughtExceptionHandler() {
+
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ log.error("parse events has an error", e);
+ }
+ };
+
+ @Override
+ public void open(Map<String, Object> config, SourceContext sourceContext)
throws Exception {
+ canalSourceConfig = CanalSourceConfig.load(config);
+ if (canalSourceConfig.getCluster()) {
+ connector =
CanalConnectors.newClusterConnector(canalSourceConfig.getZkServers(),
+ canalSourceConfig.getDestination(),
canalSourceConfig.getUsername(), canalSourceConfig.getPassword());
+ } else {
+ connector = CanalConnectors.newSingleConnector(
+ new
InetSocketAddress(canalSourceConfig.getSingleHostname(),
canalSourceConfig.getSinglePort()),
+ canalSourceConfig.getDestination(),
canalSourceConfig.getUsername(), canalSourceConfig.getPassword());
+ }
+ log.info("start canal connect");
Review comment:
can you improve the logging here?
- in cluster mode, can you print out the zookeeper user and destination?
- in single mode, can you print out the single host name, port and
destination?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
