[GitHub] jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface
jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface URL: https://github.com/apache/incubator-pulsar/pull/1708#discussion_r185603311 ## File path: pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/RuntimeSink.java ## @@ -29,7 +32,9 @@ * There is a default implementation provided for wrapping up the user provided {@link Sink}. Pulsar sink * should be implemented using this interface to ensure supporting effective-once. */ -public interface RuntimeSink extends Sink { +//public interface RuntimeSink extends Sink { Review comment: yup will remove This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface
jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface URL: https://github.com/apache/incubator-pulsar/pull/1708#discussion_r185603124 ## File path: pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java ## @@ -18,5 +18,249 @@ */ package org.apache.pulsar.functions.sink; -public class PulsarSink { +import com.google.common.annotations.VisibleForTesting; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import net.jodah.typetools.TypeResolver; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerEventListener; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.connect.core.RecordContext; +import org.apache.pulsar.functions.api.SerDe; +import org.apache.pulsar.functions.api.utils.DefaultSerDe; +import org.apache.pulsar.functions.instance.InstanceUtils; +import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.MultiConsumersOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.Producers; +import org.apache.pulsar.functions.source.PulsarRecord; +import org.apache.pulsar.functions.utils.FunctionConfig; + +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +@Slf4j +public class PulsarSink implements RuntimeSink { + +private PulsarClient client; +private PulsarSinkConfig pulsarSinkConfig; +private SerDe outputSerDe; + +private PulsarSinkProcessor pulsarSinkProcessor; + +private interface PulsarSinkProcessor { +void initializeOutputProducer(String outputTopic) throws Exception; + +void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception; + +void close() throws Exception; +} + +private class PulsarSinkAtMostOnceProcessor implements PulsarSinkProcessor { +private Producerproducer; + +@Override +public void initializeOutputProducer(String outputTopic) throws Exception { +this.producer = AbstractOneOuputTopicProducers.createProducer( +client, pulsarSinkConfig.getTopic()); +} + +@Override +public void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception { +Message outputMsg = outputMsgBuilder.build(); +this.producer.sendAsync(outputMsg); +} + +@Override +public void close() throws Exception { +if (null != producer) { +try { +producer.close(); +} catch (PulsarClientException e) { +log.warn("Fail to close producer for processor {}", pulsarSinkConfig.getTopic(), e); +} +} +} +} + +private class PulsarSinkAtLeastOnceProcessor implements PulsarSinkProcessor { +private Producer producer; + +@Override +public void initializeOutputProducer(String outputTopic) throws Exception { +this.producer = AbstractOneOuputTopicProducers.createProducer( +client, pulsarSinkConfig.getTopic()); +} + +@Override +public void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception { +Message outputMsg = outputMsgBuilder.build(); +this.producer.sendAsync(outputMsg).thenAccept(messageId -> pulsarRecord.ack()); +} + +@Override +public void close() throws Exception { +if (null != producer) { +try { +producer.close(); +} catch (PulsarClientException e) { +log.warn("Fail to close producer for processor {}", pulsarSinkConfig.getTopic(), e); +} +} +} +} + +private class PulsarSinkEffectivelyOnceProcessor implements PulsarSinkProcessor, ConsumerEventListener { + +@Getter(AccessLevel.PACKAGE) +protected Producers outputProducer; + +@Override +public void initializeOutputProducer(String outputTopic) throws Exception { +outputProducer = new MultiConsumersOneOuputTopicProducers(client, outputTopic); +outputProducer.initialize(); +} + +@Override +public void sendOutputMessage(MessageBuilder outputMsgBuilder, PulsarRecord pulsarRecord) +throws Exception { + +//
[GitHub] jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface
jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface URL: https://github.com/apache/incubator-pulsar/pull/1708#discussion_r185602904 ## File path: pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java ## @@ -18,5 +18,249 @@ */ package org.apache.pulsar.functions.sink; -public class PulsarSink { +import com.google.common.annotations.VisibleForTesting; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import net.jodah.typetools.TypeResolver; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerEventListener; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.connect.core.RecordContext; +import org.apache.pulsar.functions.api.SerDe; +import org.apache.pulsar.functions.api.utils.DefaultSerDe; +import org.apache.pulsar.functions.instance.InstanceUtils; +import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.MultiConsumersOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.Producers; +import org.apache.pulsar.functions.source.PulsarRecord; +import org.apache.pulsar.functions.utils.FunctionConfig; + +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +@Slf4j +public class PulsarSink implements RuntimeSink { + +private PulsarClient client; +private PulsarSinkConfig pulsarSinkConfig; +private SerDe outputSerDe; + +private PulsarSinkProcessor pulsarSinkProcessor; + +private interface PulsarSinkProcessor { +void initializeOutputProducer(String outputTopic) throws Exception; + +void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception; + +void close() throws Exception; +} + +private class PulsarSinkAtMostOnceProcessor implements PulsarSinkProcessor { +private Producerproducer; + +@Override +public void initializeOutputProducer(String outputTopic) throws Exception { +this.producer = AbstractOneOuputTopicProducers.createProducer( +client, pulsarSinkConfig.getTopic()); +} + +@Override +public void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception { +Message outputMsg = outputMsgBuilder.build(); +this.producer.sendAsync(outputMsg); +} + +@Override +public void close() throws Exception { +if (null != producer) { +try { +producer.close(); +} catch (PulsarClientException e) { +log.warn("Fail to close producer for processor {}", pulsarSinkConfig.getTopic(), e); +} +} +} +} + +private class PulsarSinkAtLeastOnceProcessor implements PulsarSinkProcessor { Review comment: this cannot be static. I am using the PulsarClient from the parent class This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface
jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface URL: https://github.com/apache/incubator-pulsar/pull/1708#discussion_r185602874 ## File path: pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java ## @@ -18,5 +18,249 @@ */ package org.apache.pulsar.functions.sink; -public class PulsarSink { +import com.google.common.annotations.VisibleForTesting; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import net.jodah.typetools.TypeResolver; +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.ConsumerEventListener; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageBuilder; +import org.apache.pulsar.client.api.Producer; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.connect.core.RecordContext; +import org.apache.pulsar.functions.api.SerDe; +import org.apache.pulsar.functions.api.utils.DefaultSerDe; +import org.apache.pulsar.functions.instance.InstanceUtils; +import org.apache.pulsar.functions.instance.producers.AbstractOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.MultiConsumersOneOuputTopicProducers; +import org.apache.pulsar.functions.instance.producers.Producers; +import org.apache.pulsar.functions.source.PulsarRecord; +import org.apache.pulsar.functions.utils.FunctionConfig; + +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +@Slf4j +public class PulsarSink implements RuntimeSink { + +private PulsarClient client; +private PulsarSinkConfig pulsarSinkConfig; +private SerDe outputSerDe; + +private PulsarSinkProcessor pulsarSinkProcessor; + +private interface PulsarSinkProcessor { +void initializeOutputProducer(String outputTopic) throws Exception; + +void sendOutputMessage(MessageBuilder outputMsgBuilder, + PulsarRecord pulsarRecord) throws Exception; + +void close() throws Exception; +} + +private class PulsarSinkAtMostOnceProcessor implements PulsarSinkProcessor { Review comment: this cannot be static. I am using the PulsarClient from the parent class This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface
jerrypeng commented on a change in pull request #1708: Refactor functions to use Sink interface URL: https://github.com/apache/incubator-pulsar/pull/1708#discussion_r185602541 ## File path: pulsar-connect/core/src/main/java/org/apache/pulsar/connect/core/Sink.java ## @@ -45,5 +45,5 @@ * @param value output value * @return Completable future fo async publish request */ -CompletableFuture write(T value); -} \ No newline at end of file +CompletableFuture write(T value) throws Exception; Review comment: I agree. I will change This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services