[incubator-pulsar] branch master updated: Correcting SQL getting started title (#2594)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new e357253 Correcting SQL getting started title (#2594) e357253 is described below commit e3572530f904d3cb0b3e80c177dbf33b1f4b2903 Author: Boyang Jerry Peng AuthorDate: Wed Sep 19 10:06:47 2018 -0700 Correcting SQL getting started title (#2594) --- .../versioned_docs/version-2.1.1-incubating/sql-getting-started.md| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md b/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md index aec0934..3fb42a3 100644 --- a/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md +++ b/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md @@ -1,7 +1,7 @@ --- id: version-2.1.1-incubating-sql-getting-started -title: Pulsar SQL Overview -sidebar_label: Overview +title: Pulsar SQL Getting Started +sidebar_label: Getting Started original_id: sql-getting-started ---
[incubator-pulsar] branch jerrypeng-patch-1 deleted (was 6d8b8ba)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a change to branch jerrypeng-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git. was 6d8b8ba Merge remote-tracking branch 'apache/jerrypeng-patch-1' into jerrypeng-patch-1 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-pulsar] branch jerrypeng-patch-1 updated (e256a79 -> 6d8b8ba)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a change to branch jerrypeng-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git. from e256a79 Correcting SQL getting started title add e4eb459 Correcting SQL getting started title add 9e1691e Merge remote-tracking branch 'apache/master' into jerrypeng-patch-1 add e677679 Merge remote-tracking branch 'apache/master' into jerrypeng-patch-1 add 49fc5e5 [Function] avoid creating assignment snapshot and publish individual assigment msg (#2549) add 701f3ea fix behavior of JSONSchema for derived classes (#2577) add 3097283 docs for 2.1.1-incubating release (#2601) add 8e14175 add auto ack and timeout configurable (#2503) add 4fdc21a [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604) add 5bdfa8a Merge remote-tracking branch 'apache/master' into jerrypeng-patch-1 add 6d8b8ba Merge remote-tracking branch 'apache/jerrypeng-patch-1' into jerrypeng-patch-1 No new revisions were added by this update. Summary of changes: conf/functions_worker.yml | 2 + .../pulsar/compaction/TwoPhaseCompactor.java | 2 +- .../apache/pulsar/compaction/CompactionTest.java | 2 +- .../worker/PulsarWorkerAssignmentTest.java | 370 + pulsar-client-schema/pom.xml | 6 + .../pulsar/client/impl/schema/JSONSchema.java | 41 ++- .../pulsar/client/schema/JSONSchemaTest.java | 67 +++- .../pulsar/client/schema/SchemaTestUtils.java | 42 ++- .../java/org/apache/pulsar/admin/cli/CmdSinks.java | 19 +- .../org/apache/pulsar/admin/cli/TestCmdSinks.java | 1 + .../proto/src/main/proto/Request.proto | 5 - .../apache/pulsar/functions/utils/SinkConfig.java | 3 + .../functions/worker/FunctionAssignmentTailer.java | 47 ++- .../functions/worker/FunctionRuntimeManager.java | 283 +++- .../pulsar/functions/worker/SchedulerManager.java | 126 --- .../pulsar/functions/worker/WorkerConfig.java | 5 +- .../pulsar/functions/worker/WorkerService.java | 15 +- .../worker/scheduler/RoundRobinScheduler.java | 10 +- .../worker/FunctionRuntimeManagerTest.java | 40 +-- .../functions/worker/MembershipManagerTest.java| 3 + .../functions/worker/SchedulerManagerTest.java | 298 + site2/docs/reference-pulsar-admin.md | 4 + site2/website/release-notes.md | 17 + site2/website/releases.json| 1 + site2/website/sidebars.json| 5 - .../version-2.1.1-incubating}/adaptors-kafka.md| 3 +- .../administration-proxy.md| 3 +- .../client-libraries-cpp.md| 3 +- .../client-libraries-python.md | 3 +- .../cookbooks-tiered-storage.md| 3 +- .../version-2.1.1-incubating}/deploy-bare-metal.md | 3 +- .../version-2.1.1-incubating}/functions-api.md | 3 +- .../functions-deploying.md | 3 +- .../functions-guarantees.md| 3 +- .../functions-overview.md | 3 +- .../functions-quickstart.md| 3 +- .../getting-started-clients.md | 3 +- .../getting-started-standalone.md | 3 +- .../version-2.1.1-incubating}/io-managing.md | 3 +- .../version-2.1.1-incubating}/io-quickstart.md | 7 +- .../reference-configuration.md | 3 +- .../reference-pulsar-admin.md | 3 +- .../sql-deployment-configurations.md | 3 +- .../sql-getting-started.md | 3 +- .../version-2.1.1-incubating}/sql-overview.md | 3 +- site2/website/versions.json| 1 + 46 files changed, 1016 insertions(+), 463 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/adaptors-kafka.md (99%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/administration-proxy.md (98%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/client-libraries-cpp.md (98%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/client-libraries-python.md (97%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/cookbooks-tiered-storage.md (99%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/deploy-bare-metal.md (99%) copy site2/{docs => website/versioned_docs/version-2.1.1-incubating}/functions-api.md (99%) copy site2/{docs => website/versioned_docs/version-2.1.1-incuba
[incubator-pulsar] branch master updated: Add ServiceUrlProvider and add method forceCloseConnection in PulsarC… (#2543)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new ee5afa5 Add ServiceUrlProvider and add method forceCloseConnection in PulsarC… (#2543) ee5afa5 is described below commit ee5afa5007e260891140956d244a865d73368321 Author: penghui AuthorDate: Wed Sep 19 15:44:15 2018 +0800 Add ServiceUrlProvider and add method forceCloseConnection in PulsarC… (#2543) Support build Pulsar client with serviceUrlProvider method. ### Motivation With serviceUrlProvider we can store the pulsar service url in zookeeper or any other config service. And we can watch the service url change event then control the pulsar client, such as change pulsar client serviceUrl, force close client connection or re-connect with new service url. ### Modifications Add ServiceUrlProvider interface. Add forceCloseConnection method in PulsarClient. --- .../pulsar/client/api/ServiceUrlProviderTest.java | 156 + .../apache/pulsar/client/api/ClientBuilder.java| 7 + .../org/apache/pulsar/client/api/PulsarClient.java | 22 +++ .../pulsar/client/api/ServiceUrlProvider.java | 42 ++ .../pulsar/client/impl/ClientBuilderImpl.java | 20 ++- .../apache/pulsar/client/impl/HandlerState.java| 4 + .../apache/pulsar/client/impl/LookupService.java | 2 +- .../client/impl/PartitionedProducerImpl.java | 4 + .../pulsar/client/impl/PulsarClientImpl.java | 29 +++- .../client/impl/conf/ClientConfigurationData.java | 2 + 10 files changed, 283 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java new file mode 100644 index 000..ab86f12 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api; + +import org.apache.bookkeeper.test.PortManager; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.client.impl.ConsumerImpl; +import org.apache.pulsar.client.impl.ProducerImpl; +import org.apache.pulsar.client.impl.PulsarClientImpl; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.concurrent.TimeUnit; + +public class ServiceUrlProviderTest extends ProducerConsumerBase { + +@BeforeClass +@Override +protected void setup() throws Exception { +super.internalSetup(); +super.producerBaseSetup(); + +} + +@AfterClass +@Override +protected void cleanup() throws Exception { +super.internalCleanup(); +} + +@Test +public void testCreateClientWithServiceUrlProvider() throws Exception { + +PulsarClient client = PulsarClient.builder() +.serviceUrlProvider(new TestServiceUrlProvider(pulsar.getBrokerServiceUrl())) +.statsInterval(1, TimeUnit.SECONDS) +.build(); +Assert.assertTrue(((PulsarClientImpl) client).getConfiguration().getServiceUrlProvider() instanceof TestServiceUrlProvider); +Producer producer = client.newProducer(Schema.STRING) +.topic("persistent://my-property/my-ns/my-topic") +.create(); +Consumer consumer = client.newConsumer(Schema.STRING) +.topic("persistent://my-property/my-ns/my-topic") +.subscriptionName("my-subscribe") +.subscribe(); +for (int i = 0; i < 100; i++) { +producer.send("Hello Pulsar[" + i + "]"); +} +client.forceCloseConnection(); +for (int i = 100; i < 200; i++) { +producer.send("Hello Pulsar[" + i + "]"); +
[incubator-pulsar] branch master updated: fixing/improving logging for function instance (#2586)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 3f8bd00 fixing/improving logging for function instance (#2586) 3f8bd00 is described below commit 3f8bd00d961f1f56233c629746c11cbd711bf06e Author: Boyang Jerry Peng AuthorDate: Tue Sep 18 23:30:39 2018 -0700 fixing/improving logging for function instance (#2586) ### Motivation logging for java function instance (process mode and localrun) is not working well. Log files are created at inappropriate locations for logs not captured by the routing appender, functions bookkeeper logs are missing because we are not using the shaded classpath. The best for logging is just to put all the logs from the java instance log in a single log and not worry about the routing which is only useful when running in threaded mode --- .../pulsar/functions/runtime/JavaInstanceMain.java | 9 --- .../pulsar/functions/runtime/ProcessRuntime.java | 22 +++--- .../pulsar/functions/runtime/ThreadRuntime.java| 4 +- .../src/main/resources/java_instance_log4j2.yml| 78 +++--- .../functions/runtime/ProcessRuntimeTest.java | 4 +- 5 files changed, 31 insertions(+), 86 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java index 11da7c3..083686b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java @@ -22,27 +22,18 @@ package org.apache.pulsar.functions.runtime; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.converters.StringConverter; -import com.google.gson.Gson; import com.google.protobuf.Empty; import com.google.protobuf.util.JsonFormat; import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; - -import static org.apache.commons.lang3.StringUtils.isNotBlank; - import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; -import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceControlGrpc; -import java.util.Map; import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java index a978a09..c5159b0 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java @@ -22,32 +22,32 @@ package org.apache.pulsar.functions.runtime; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.Gson; import com.google.protobuf.Empty; import com.google.protobuf.util.JsonFormat; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; - -import static org.apache.commons.lang3.StringUtils.isNotBlank; - -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; import org.apache.pulsar.functions.proto.InstanceControlGrpc; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry; import java.io.InputStream; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.apache.commons.lang3
[incubator-pulsar] branch master updated: [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 4fdc21a [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604) 4fdc21a is described below commit 4fdc21afba77184109757f62679eec4d972dbbc8 Author: Sijie Guo AuthorDate: Tue Sep 18 21:27:41 2018 -0700 [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604) *Motivation* There is no ways to gurantee how fast a consumer can consume. so it makes no sense to compare the different permits between two consumers. *Changes* Remove PersistentQueueE2ETest#testConsumersWithDifferentPermits
[incubator-pulsar] branch master updated: [tests] improve connector related integration tests (#2587)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 7530d64 [tests] improve connector related integration tests (#2587) 7530d64 is described below commit 7530d64a679a0783122b18058c1148c89c0fee0a Author: Sijie Guo AuthorDate: Tue Sep 18 14:50:51 2018 -0700 [tests] improve connector related integration tests (#2587) *Motivation* with more and more connector are added, it becomes expensive to start all external services at the begin. *Changes* - refactor the connector testing framework to start external service before methods - fix kafka, cassandra and mysql connectors --- distribution/io/src/assemble/io.xml| 6 + .../broker/service/PersistentQueueE2ETest.java | 60 -- .../functions/instance/JavaInstanceRunnable.java | 3 +- .../pulsar/functions/source/TopicSchema.java | 3 + ...rchAbstractSink.java => ElasticSearchSink.java} | 9 +- .../io/elasticsearch/ElasticSearchStringSink.java | 35 -- .../resources/META-INF/services/pulsar-io.yaml | 4 +- .../io/elasticsearch/ElasticSearchSinkTests.java | 4 +- .../apache/pulsar/io/kafka/KafkaAbstractSink.java | 6 +- .../{KafkaStringSink.java => KafkaBytesSink.java} | 25 - .../resources/META-INF/services/pulsar-io.yaml | 2 +- site2/docs/io-quickstart.md| 2 +- .../version-2.1.0-incubating/io-quickstart.md | 2 +- .../integration/functions/PulsarFunctionsTest.java | 39 +-- .../functions/utils/CommandGenerator.java | 2 +- .../integration/io/CassandraSinkArchiveTester.java | 121 - .../tests/integration/io/CassandraSinkTester.java | 45 +--- .../integration/io/ElasticSearchSinkTester.java| 34 +++--- .../tests/integration/io/HdfsSinkTester.java | 22 ++-- .../tests/integration/io/JdbcSinkTester.java | 38 --- .../tests/integration/io/KafkaSinkTester.java | 28 +++-- .../tests/integration/io/KafkaSourceTester.java| 12 +- .../pulsar/tests/integration/io/SinkTester.java| 25 - .../pulsar/tests/integration/io/SourceTester.java | 4 +- .../tests/integration/suites/PulsarTestSuite.java | 50 - .../integration/topologies/PulsarCluster.java | 17 +++ 26 files changed, 212 insertions(+), 386 deletions(-) diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 08ff859..a509e19 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -92,5 +92,11 @@ connectors 644 + + + ${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar + connectors + 644 + diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java index 2937ca0..fb7a76c 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java @@ -233,66 +233,6 @@ public class PersistentQueueE2ETest extends BrokerTestBase { deleteTopic(topicName); } -@Test -public void testConsumersWithDifferentPermits() throws Exception { -final String topicName = "persistent://prop/use/ns-abc/shared-topic4"; -final String subName = "sub4"; -final int numMsgs = 1; - -final AtomicInteger msgCountConsumer1 = new AtomicInteger(0); -final AtomicInteger msgCountConsumer2 = new AtomicInteger(0); -final CountDownLatch latch = new CountDownLatch(numMsgs); - -int recvQ1 = 10; -Consumer consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) - .subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1) -.messageListener((consumer, msg) -> { -msgCountConsumer1.incrementAndGet(); -try { -consumer.acknowledge(msg); -latch.countDown(); -} catch (PulsarClientException e) { -fail("Should not fail"); -} -}).subscribe(); - -int recvQ2 = 1; -Consumer consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName) - .subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2) -.messageListener((consumer, msg) -> { -msgCountConsumer2.incrementAndGet(); -try { -
[incubator-pulsar] branch master updated: [build] Fix docker organization parameter (#2597)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new fb396bf [build] Fix docker organization parameter (#2597) fb396bf is described below commit fb396bf8313f728f5659286e4d8924bcf37d4832 Author: Sijie Guo AuthorDate: Tue Sep 18 14:50:12 2018 -0700 [build] Fix docker organization parameter (#2597) *Motivation* docker orgnization is missing for building test image. so the build will be failing with `-Pdocker`. *Changes* Move the docker organization parameter to root pom file. --- docker/pom.xml | 3 --- pom.xml| 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index bdc99f7..302bda8 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -31,9 +31,6 @@ org.apache.pulsar docker-images Apache Pulsar :: Docker Images - -apachepulsar - pulsar grafana diff --git a/pom.xml b/pom.xml index 6d45a00..91ba14c 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,7 @@ flexible messaging model and an intuitive client API. true false 1 +apachepulsar 2.1.0-incubating
[incubator-pulsar] branch master updated: Fix: Compaction with last deleted keys not completing compaction (#2591)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8d50617 Fix: Compaction with last deleted keys not completing compaction (#2591) 8d50617 is described below commit 8d5061779e7a869e6e5981d2010e671ed0afa322 Author: Rajan Dhabalia AuthorDate: Tue Sep 18 03:09:50 2018 -0700 Fix: Compaction with last deleted keys not completing compaction (#2591) ### Motivation Right now, topic-compaction ignores the message-id with empty payload but if the last message in the ledger has empty payload then compactor doesn't complete the compaction because compactor ignores last message and doesn't complete the result-future so, caller never sees complete result. ### Modifications - Compactor calculates` from` and `to` position for compacted ledger according to last non-deleted active key. - Compactor handles tail deleted keys from the ledger and completes compaction process gracefully. ### Result compactor can successfully compact ledger whose last message is also deleted. --- .../pulsar/compaction/TwoPhaseCompactor.java | 90 +- .../apache/pulsar/compaction/CompactionTest.java | 56 ++ 2 files changed, 111 insertions(+), 35 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java index cc3f710..425e049 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java @@ -76,7 +76,7 @@ public class TwoPhaseCompactor extends Compactor { @Override protected CompletableFuture doCompaction(RawReader reader, BookKeeper bk) { return phaseOne(reader).thenCompose( -(r) -> phaseTwo(reader, r.from, r.to, r.latestForKey, bk)); +(r) -> phaseTwo(reader, r.from, r.to, r.lastReadId, r.latestForKey, bk)); } private CompletableFuture phaseOne(RawReader reader) { @@ -90,7 +90,8 @@ public class TwoPhaseCompactor extends Compactor { } else { log.info("Commencing phase one of compaction for {}, reading to {}", reader.getTopic(), lastMessageId); -phaseOneLoop(reader, Optional.empty(), lastMessageId, latestForKey, loopPromise); +phaseOneLoop(reader, Optional.empty(), Optional.empty(), lastMessageId, latestForKey, +loopPromise); } }); return loopPromise; @@ -98,6 +99,7 @@ public class TwoPhaseCompactor extends Compactor { private void phaseOneLoop(RawReader reader, Optional firstMessageId, + Optional toMessageId, MessageId lastMessageId, Map latestForKey, CompletableFuture loopPromise) { @@ -114,6 +116,7 @@ public class TwoPhaseCompactor extends Compactor { return; } MessageId id = m.getMessageId(); +boolean deletedMessage = false; if (RawBatchConverter.isReadableBatch(m)) { try { RawBatchConverter.extractIdsAndKeys(m) @@ -125,16 +128,23 @@ public class TwoPhaseCompactor extends Compactor { } else { Pair keyAndSize = extractKeyAndSize(m); if (keyAndSize != null) { -latestForKey.put(keyAndSize.getLeft(), id); +if(keyAndSize.getRight() > 0) { +latestForKey.put(keyAndSize.getLeft(), id); +} else { +deletedMessage = true; +latestForKey.remove(keyAndSize.getLeft()); +} } } +MessageId first = firstMessageId.orElse(deletedMessage ? null : id); +MessageId to = deletedMessage ? toMessageId.orElse(null) : id; if (id.compareTo(lastMessageId) == 0) { -loopPromise.complete(new PhaseOneResult(firstMessageId.orElse(id), -id, latestForKey)); +loopPromise.complete(new PhaseOneRes
[incubator-pulsar] branch master updated: [java-client] Issue #2384: ConnectionHandler: Log stack trace instead of printing (#2599)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new c866279 [java-client] Issue #2384: ConnectionHandler: Log stack trace instead of printing (#2599) c866279 is described below commit c86627948d4abf4b50517338c739f0dea8029849 Author: Sijie Guo AuthorDate: Tue Sep 18 03:05:40 2018 -0700 [java-client] Issue #2384: ConnectionHandler: Log stack trace instead of printing (#2599) *Motivation* Fixes #2384. The pulsar java client is currently very noisy in case of connection loss and it cannot be muted, as the stack trace is printed with .printStackTrace(). Moving this to the logging system will allow developers to configure this behavior. *Changes* Remove `printStackTrace`. --- .../src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java index d9bdc74..d04e8a8 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java @@ -71,7 +71,6 @@ class ConnectionHandler { } private Void handleConnectionError(Throwable exception) { -exception.printStackTrace(); log.warn("[{}] [{}] Error connecting to broker: {}", state.topic, state.getHandlerName(), exception.getMessage()); connection.connectionFailed(new PulsarClientException(exception));
[incubator-pulsar] branch master updated: [tests] Flaky Test ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 6c07c81 [tests] Flaky Test ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598) 6c07c81 is described below commit 6c07c81bf85eaa6d2a23c026049c57305284876e Author: Sijie Guo AuthorDate: Tue Sep 18 02:18:17 2018 -0700 [tests] Flaky Test ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598) * [tests] Flaky Test ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache *Motivation* The problem is zookeeper watcher notification is non-deterministic. That says if you create N paths in zookeeper, you might receive x notification. x is between 1 and N. so the test is written in a very non-deterministic way. ``` java.lang.AssertionError: expected [1] but found [2] at org.testng.Assert.fail(Assert.java:96) at org.testng.Assert.failNotEquals(Assert.java:776) at org.testng.Assert.assertEqualsImpl(Assert.java:137) at org.testng.Assert.assertEquals(Assert.java:118) at org.testng.Assert.assertEquals(Assert.java:652) at org.testng.Assert.assertEquals(Assert.java:662) at org.apache.pulsar.zookeeper.ZookeeperCacheTest.testChildrenCacheZnodeCreatedAfterCache(ZookeeperCacheTest.java:214) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124) at org.testng.internal.InvokeMethodRunnable.runOne(InvokeMethodRunnable.java:54) at org.testng.internal.InvokeMethodRunnable.run(InvokeMethodRunnable.java:44) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) ``` *Changes* rewrite the test to make it more robust. --- .../test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java index 09c3ea1..39f23ac 100644 --- a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java +++ b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java @@ -209,12 +209,14 @@ public class ZookeeperCacheTest { Thread.sleep(1); } +final int recvNotifications = notificationCount.get(); + assertEquals(cache.get(), new TreeSet(Lists.newArrayList("z1"))); assertEquals(cache.get("/test"), new TreeSet(Lists.newArrayList("z1"))); -assertEquals(notificationCount.get(), 1); +assertTrue(recvNotifications == 1 || recvNotifications == 2); zkClient.delete("/test/z1", -1); -while (notificationCount.get() < 2) { +while (notificationCount.get() < (recvNotifications + 1)) { Thread.sleep(1); } @@ -230,7 +232,7 @@ public class ZookeeperCacheTest { // Ok } -assertEquals(notificationCount.get(), 2); +assertEquals(notificationCount.get(), (recvNotifications + 1)); } @Test(timeOut = 1)
[incubator-pulsar] branch master updated: [tests] make PersistentQueueE2ETest tolerant failures during deleting topic at cleanup (#2600)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new aebfbba [tests] make PersistentQueueE2ETest tolerant failures during deleting topic at cleanup (#2600) aebfbba is described below commit aebfbba2a9f2882944bd1854672f4538a9d27177 Author: Sijie Guo AuthorDate: Tue Sep 18 00:43:50 2018 -0700 [tests] make PersistentQueueE2ETest tolerant failures during deleting topic at cleanup (#2600) *Motivation* "Topic has active producers/subscriptions" can be thrown during deleting topic at cleanup. This kind of error can be ignored during cleaning up to make the tests more robust. *Changes* Make PersistentQueueE2ETest toelrant failures during cleaning up topics. --- .../broker/service/PersistentQueueE2ETest.java | 23 +- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java index c41a07a..2937ca0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java @@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.collections.CollectionUtils; import org.apache.pulsar.broker.service.persistent.PersistentSubscription; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.ConsumerBuilder; import org.apache.pulsar.client.api.Message; @@ -76,6 +77,14 @@ public class PersistentQueueE2ETest extends BrokerTestBase { private static final Logger log = LoggerFactory.getLogger(PersistentQueueE2ETest.class); +private void deleteTopic(String topicName) { +try { +admin.topics().delete(topicName); +} catch (PulsarAdminException pae) { +// it is okay to get exception if it is cleaning up. +} +} + @Test public void testSimpleConsumerEvents() throws Exception { final String topicName = "persistent://prop/use/ns-abc/shared-topic1"; @@ -172,7 +181,7 @@ public class PersistentQueueE2ETest extends BrokerTestBase { producer.close(); consumer2.close(); -admin.topics().delete(topicName); +deleteTopic(topicName); } @Test @@ -220,7 +229,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase { assertTrue(CollectionUtils.subtract(messagesProduced, messagesConsumed).isEmpty()); consumer1.close(); -admin.topics().delete(topicName); + +deleteTopic(topicName); } @Test @@ -279,7 +289,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase { consumer1.close(); consumer2.close(); -admin.topics().delete(topicName); + +deleteTopic(topicName); } // this test is good to have to see the distribution, but every now and then it gets slightly different than the @@ -352,7 +363,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase { consumer1.close(); consumer2.close(); consumer3.close(); -admin.topics().delete(topicName); + +deleteTopic(topicName); } @Test(timeOut = 30) @@ -550,6 +562,7 @@ public class PersistentQueueE2ETest extends BrokerTestBase { producer.close(); consumer1.close(); consumer2.close(); -admin.topics().delete(topicName); + +deleteTopic(topicName); } }
[incubator-pulsar] branch master updated: Preserve filename of the original function submission (#2588)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new a123380 Preserve filename of the original function submission (#2588) a123380 is described below commit a123380327fbe95eef1066c4c456020c19136e07 Author: Sanjeev Kulkarni AuthorDate: Mon Sep 17 01:03:06 2018 -0700 Preserve filename of the original function submission (#2588) ### Motivation When we submit a function, currently we don't note down the original filename that was uploaded from the client. For regular java functions this works fine. For python workers we manage by doing tricks wrt tricks like filename using classnames. However when we add things like wheel support, the filename encodes information that cannot be discarded away. This pr preserves that filename --- pulsar-functions/proto/src/main/proto/Function.proto | 1 + .../org/apache/pulsar/functions/utils/FunctionDetailsUtils.java | 8 +++- .../java/org/apache/pulsar/functions/worker/FunctionActioner.java | 2 +- .../apache/pulsar/functions/worker/rest/api/FunctionsImpl.java| 6 ++ 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/proto/src/main/proto/Function.proto b/pulsar-functions/proto/src/main/proto/Function.proto index 6e969ae..a76cf8d 100644 --- a/pulsar-functions/proto/src/main/proto/Function.proto +++ b/pulsar-functions/proto/src/main/proto/Function.proto @@ -116,6 +116,7 @@ message SinkSpec { message PackageLocationMetaData { string packagePath = 1; +string originalFileName = 2; } message FunctionMetaData { diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java index 5ecdd4f..a9787ba 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java @@ -19,6 +19,8 @@ package org.apache.pulsar.functions.utils; +import org.apache.commons.lang.StringUtils; +import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; public class FunctionDetailsUtils { @@ -43,7 +45,11 @@ public class FunctionDetailsUtils { return fullyQualifiedName.split("/")[2]; } -public static String getDownloadFileName(FunctionDetails FunctionDetails) { +public static String getDownloadFileName(FunctionDetails FunctionDetails, + Function.PackageLocationMetaData packageLocation) { +if (!StringUtils.isEmpty(packageLocation.getOriginalFileName())) { +return packageLocation.getOriginalFileName(); +} String[] hierarchy = FunctionDetails.getClassName().split("\\."); String fileName; if (hierarchy.length <= 0) { diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java index f5a7e96..724d80c 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java @@ -156,7 +156,7 @@ public class FunctionActioner implements AutoCloseable { pkgFile = new File( pkgDir, -new File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails())).getName()); +new File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails(), functionMetaData.getPackageLocation())).getName()); downloadFile(pkgFile, isPkgUrlProvided, functionMetaData, instanceId); } diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index df82c0d..8bb9194 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java @@ -170,6 +170,9 @@ public class FunctionsImpl { } else { packageLocationMetaDataBuilder.setPackagePath(isPkgUrlProvided ? functionPkgUrl : createPackagePath(tenant, namespace, functionName, fileDetail.getFileName())); +if (!isPkgUrlProvided) { + packageLocationMetaDataBuilder.setOriginalFil
[incubator-pulsar] branch master updated: [schema] add schemas for primtive types (#2500)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 5a2d75c [schema] add schemas for primtive types (#2500) 5a2d75c is described below commit 5a2d75c11acd8c6014ea1f9551405a6bcb41e4f8 Author: Sijie Guo AuthorDate: Mon Sep 17 00:51:59 2018 -0700 [schema] add schemas for primtive types (#2500) * [schema] add schemas for primtive types * Fix issues with Schema.BYTES --- .../client/api/SchemaSerializationException.java | 5 + .../client/impl/schema/ByteBufferSchema.java | 75 +++ .../schema/{StringSchema.java => ByteSchema.java} | 50 +- .../pulsar/client/impl/schema/BytesSchema.java | 17 +++- .../pulsar/client/impl/schema/DoubleSchema.java| 80 .../pulsar/client/impl/schema/FloatSchema.java | 76 +++ .../pulsar/client/impl/schema/IntSchema.java | 75 +++ .../pulsar/client/impl/schema/LongSchema.java | 79 .../pulsar/client/impl/schema/ShortSchema.java | 73 +++ .../pulsar/client/impl/schema/StringSchema.java| 29 -- .../pulsar/client/schema/PrimitiveSchemaTest.java | 102 + .../apache/pulsar/client/impl/ConsumerImpl.java| 11 ++- .../apache/pulsar/client/impl/ProducerImpl.java| 3 + .../apache/pulsar/common/schema/SchemaInfo.java| 2 + .../apache/pulsar/common/schema/SchemaType.java| 40 15 files changed, 686 insertions(+), 31 deletions(-) diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java index 39248d2..46a38c1 100644 --- a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java @@ -19,6 +19,11 @@ package org.apache.pulsar.client.api; public class SchemaSerializationException extends RuntimeException { + +public SchemaSerializationException(String message) { +super(message); +} + public SchemaSerializationException(Throwable cause) { super(cause); } diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java new file mode 100644 index 000..ee8ba66 --- /dev/null +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.impl.schema; + +import java.nio.ByteBuffer; +import org.apache.pulsar.client.api.Schema; +import org.apache.pulsar.common.schema.SchemaInfo; +import org.apache.pulsar.common.schema.SchemaType; + +/** + * A bytebuffer schema. + */ +public class ByteBufferSchema implements Schema { + +public static ByteBufferSchema of() { +return INSTANCE; +} + +private static final ByteBufferSchema INSTANCE = new ByteBufferSchema(); +private static final SchemaInfo SCHEMA_INFO = new SchemaInfo() +.setName("ByteBuffer") +.setType(SchemaType.BYTEBUFFER) +.setSchema(new byte[0]); + +@Override +public byte[] encode(ByteBuffer data) { +if (data == null) { +return null; +} + +data.rewind(); + +if (data.hasArray()) { +byte[] arr = data.array(); +if (data.arrayOffset() == 0 && arr.length == data.remaining()) { +return arr; +} +} + +byte[] ret = new byte[data.remaining()]; +data.get(ret, 0, ret.length); +data.rewind(); +return ret; +} + +@Override +public ByteBuffer decode(byte[] data) { +if (null == data) { +return null; +} else {
[incubator-pulsar] branch master updated: Change repository tag for test image (#2571)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 844f46f Change repository tag for test image (#2571) 844f46f is described below commit 844f46f156687fad1a08d7e88716fa5ee6937943 Author: Ali Ahmed AuthorDate: Mon Sep 17 00:26:27 2018 -0700 Change repository tag for test image (#2571) This uses the docker.organization property to tag the test image to make it consistent with other pom files --- tests/docker-images/latest-version-image/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml index 78a12e2..b58530c 100644 --- a/tests/docker-images/latest-version-image/pom.xml +++ b/tests/docker-images/latest-version-image/pom.xml @@ -68,13 +68,13 @@ tag - apachepulsar/pulsar-test-latest-version + ${docker.organization}/pulsar-test-latest-version latest - apachepulsar/pulsar-test-latest-version + ${docker.organization}/pulsar-test-latest-version ${project.version} false true
[incubator-pulsar.wiki] branch master updated: Updated Home (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 83e4c5c Updated Home (markdown) 83e4c5c is described below commit 83e4c5cfacdb4ba3fe3b75d3239d65c18b7071c7 Author: Sijie Guo AuthorDate: Sun Sep 16 16:30:08 2018 -0700 Updated Home (markdown) --- Home.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Home.md b/Home.md index 74324a2..bf95c54 100644 --- a/Home.md +++ b/Home.md @@ -9,12 +9,12 @@ We encourage to document any big change or feature or any addition to public use ### Proposed -* [[PIP 23: Message Tracing By Interceptors]] -* [[PIP 22: Pulsar Dead Letter Topic]] * [[PIP 21: Pulsar Edge Component]] ### Accepted +* [[PIP 23: Message Tracing By Interceptors]] +* [[PIP 22: Pulsar Dead Letter Topic]] * [PIP 20: Mechanism to revoke TLS authentication](https://github.com/apache/incubator-pulsar/wiki/PIP-20:-Mechanism-to-revoke-TLS-authentication) * [PIP 19: Pulsar SQL](https://github.com/apache/incubator-pulsar/wiki/PIP-19:-Pulsar-SQL) * [PIP 17: Tiered storage for Pulsar topics](https://github.com/apache/incubator-pulsar/wiki/PIP-17:-Tiered-storage-for-Pulsar-topics)
[incubator-pulsar.wiki] branch master updated: Updated PIP 22: Pulsar Dead Letter Topic (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 04cecc4 Updated PIP 22: Pulsar Dead Letter Topic (markdown) 04cecc4 is described below commit 04cecc4bf2f95096ec9ec54b2f85ceb62135eece Author: Sijie Guo AuthorDate: Sun Sep 16 16:29:44 2018 -0700 Updated PIP 22: Pulsar Dead Letter Topic (markdown) --- PIP-22:-Pulsar-Dead-Letter-Topic.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PIP-22:-Pulsar-Dead-Letter-Topic.md b/PIP-22:-Pulsar-Dead-Letter-Topic.md index c259aa2..cd49a3f 100644 --- a/PIP-22:-Pulsar-Dead-Letter-Topic.md +++ b/PIP-22:-Pulsar-Dead-Letter-Topic.md @@ -1,7 +1,8 @@ -- **Status**: Proposed +- **Status**: Done - **Author**: [Penghui Li](https://github.com/codelipenghui) -- **Pull Request**: - +- **Pull Request**: [#2508](https://github.com/apache/incubator-pulsar/pull/2508) - **Mailing List discussion**: +- **Release**: 2.2.0 ## Motivation
[incubator-pulsar.wiki] branch master updated: Updated PIP 23: Message Tracing By Interceptors (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 5044279 Updated PIP 23: Message Tracing By Interceptors (markdown) 5044279 is described below commit 5044279a618f45aca535171dbbb8e3eaa3aabded Author: Sijie Guo AuthorDate: Sun Sep 16 16:28:35 2018 -0700 Updated PIP 23: Message Tracing By Interceptors (markdown) --- PIP-23:-Message-Tracing-By-Interceptors.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PIP-23:-Message-Tracing-By-Interceptors.md b/PIP-23:-Message-Tracing-By-Interceptors.md index 58d5336..c29439c 100644 --- a/PIP-23:-Message-Tracing-By-Interceptors.md +++ b/PIP-23:-Message-Tracing-By-Interceptors.md @@ -1,7 +1,8 @@ -- **Status**: Proposed +- **Status**: Done - **Author**: [Penghui Li](https://github.com/codelipenghui), [Jia Zhai](https://github.com/zhaijack) -- **Pull Request**: - +- **Pull Request**: [#2471](https://github.com/apache/incubator-pulsar/pull/2471) - **Mailing List discussion**: +- **Release**: 2.2.0 ## Requirement & Overview
[incubator-pulsar] branch master updated: Misc fixes (#2579)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 858648e Misc fixes (#2579) 858648e is described below commit 858648eb24aeba9d95a7e83513830afd40cb21fe Author: Sanjeev Kulkarni AuthorDate: Fri Sep 14 22:33:03 2018 -0700 Misc fixes (#2579) This pr does a couple of minor nits 1. Add a log statement in Sink to make it consistent with the Source. 2. Added AllArgs/NoArgs constructor for some test related objects --- .../src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java| 2 ++ .../org/apache/pulsar/functions/api/examples/CustomBaseObject.java| 2 ++ .../org/apache/pulsar/functions/api/examples/CustomDerivedObject.java | 4 3 files changed, 8 insertions(+) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java index 835e288..4449045 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java @@ -205,6 +205,8 @@ public class PulsarSink implements Sink { @Override public void open(Map config, SinkContext sinkContext) throws Exception { +log.info("Opening pulsar sink with config: {}", pulsarSinkConfig); + Schema schema = initializeSchema(); FunctionConfig.ProcessingGuarantees processingGuarantees = this.pulsarSinkConfig.getProcessingGuarantees(); diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java index 173393b..6102887 100644 --- a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java @@ -20,11 +20,13 @@ package org.apache.pulsar.functions.api.examples; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; @Getter @Setter @AllArgsConstructor +@NoArgsConstructor public class CustomBaseObject { private long baseValue; } diff --git a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java index 464e977..d700231 100644 --- a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java +++ b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java @@ -18,11 +18,15 @@ */ package org.apache.pulsar.functions.api.examples; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; @Getter @Setter +@AllArgsConstructor +@NoArgsConstructor public class CustomDerivedObject extends CustomBaseObject { private int derivedValue; public CustomDerivedObject(long baseValue, int derivedValue) {
[incubator-pulsar] branch master updated: Issue #2574: Timeout message not get redeliver in TopicsConsumer when use message listener
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new ec74355 Issue #2574: Timeout message not get redeliver in TopicsConsumer when use message listener ec74355 is described below commit ec7435565520ee6ac3ed2ce4a0cd13e8de7dbca3 Author: Jia Zhai AuthorDate: Sat Sep 15 00:15:13 2018 +0800 Issue #2574: Timeout message not get redeliver in TopicsConsumer when use message listener ### Motivation fix issue #2574 . Timeout message not get redeliver in TopicsConsumer when use message listener. This is caused by message listener wrongly set in individual sub-ConsumerImpl. ### Modifications set message listener to null for individual sub-ConsumerImpl. Add a UT ### Result UT passed. --- .../pulsar/client/impl/TopicsConsumerImplTest.java | 58 ++ .../client/impl/MultiTopicsConsumerImpl.java | 1 + 2 files changed, 59 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java index d28d1d5..55be9ec 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.client.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -615,4 +616,61 @@ public class TopicsConsumerImplTest extends ProducerConsumerBase { } } +/** + * Test Listener for github issue #2547 + */ +@Test(timeOut = 3) +public void testMultiTopicsMessageListener() throws Exception { +String key = "MultiTopicsMessageListenerTest"; +final String subscriptionName = "my-ex-subscription-" + key; +final String messagePredicate = "my-message-" + key + "-"; +final int totalMessages = 6; + +// set latch larger than totalMessages, so timeout message get resend +CountDownLatch latch = new CountDownLatch(totalMessages * 3); + +final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key; +List topicNames = Lists.newArrayList(topicName1); + +admin.tenants().createTenant("prop", new TenantInfo()); +admin.topics().createPartitionedTopic(topicName1, 2); + +// 1. producer connect +Producer producer1 = pulsarClient.newProducer().topic(topicName1) +.enableBatching(false) +.messageRoutingMode(MessageRoutingMode.SinglePartition) +.create(); + +// 2. Create consumer, set not ack in message listener, so time-out message will resend +Consumer consumer = pulsarClient.newConsumer() +.topics(topicNames) +.subscriptionName(subscriptionName) +.subscriptionType(SubscriptionType.Shared) +.ackTimeout(1000, TimeUnit.MILLISECONDS) +.receiverQueueSize(100) +.messageListener((c1, msg) -> { +assertNotNull(msg, "Message cannot be null"); +String receivedMessage = new String(msg.getData()); +latch.countDown(); + +log.info("Received message [{}] in the listener, latch: {}", +receivedMessage, latch.getCount()); +// since not acked, it should retry another time +//c1.acknowledgeAsync(msg); +}) +.subscribe(); +assertTrue(consumer instanceof MultiTopicsConsumerImpl); + +MultiTopicsConsumerImpl topicsConsumer = (MultiTopicsConsumerImpl) consumer; + +// 3. producer publish messages +for (int i = 0; i < totalMessages; i++) { +producer1.send((messagePredicate + "producer1-" + i).getBytes()); +} + +// verify should not time out, because of message redelivered several times. +latch.await(); + +consumer.close(); +} } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 75fdac6..8b8556a 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -485,6 +485,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { ConsumerConfigurationData interna
[incubator-pulsar] branch master updated: [proxy][functions] Issue #2154: proxy should be able to forward rest requests to function workers cluster (#2560)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 07d4226 [proxy][functions] Issue #2154: proxy should be able to forward rest requests to function workers cluster (#2560) 07d4226 is described below commit 07d42261eb2d685de6f9a5ab214f4d08944ffb6e Author: Sijie Guo AuthorDate: Fri Sep 14 01:10:35 2018 -0700 [proxy][functions] Issue #2154: proxy should be able to forward rest requests to function workers cluster (#2560) *Motivation* Function workers can be deployed as a separate cluster. If so, proxy is not able to forward the functions related rest calls to the correct server. *Changes* Add two settings in proxy configuration to allow proxy configuring forwarding functions related rest calls to function worker cluster. *Tests* Verified with changes in integration tests (manually). It is hard to add the integration tests based on current integration tests. will add them in a separate PR. --- conf/proxy.conf | 5 + .../apache/pulsar/proxy/server/AdminProxyHandler.java| 16 ++-- .../apache/pulsar/proxy/server/ProxyConfiguration.java | 12 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/conf/proxy.conf b/conf/proxy.conf index 9b307cc..b95c4d3 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -23,6 +23,11 @@ zookeeperServers= # Configuration store connection string (as a comma-separated list) configurationStoreServers= +# If function workers are setup in a separate cluster, configure the following 2 settings +# to point to the function workers cluster +functionWorkerWebServiceURL= +functionWorkerWebServiceURLTLS= + # ZooKeeper session timeout (in milliseconds) zookeeperSessionTimeoutMs=3 diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java index 49c789c..d6c32df 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java @@ -49,12 +49,15 @@ class AdminProxyHandler extends AsyncProxyServlet { private final ProxyConfiguration config; private final BrokerDiscoveryProvider discoveryProvider; private final String brokerWebServiceUrl; +private final String functionWorkerWebServiceUrl; AdminProxyHandler(ProxyConfiguration config, BrokerDiscoveryProvider discoveryProvider) { this.config = config; this.discoveryProvider = discoveryProvider; this.brokerWebServiceUrl = config.isTlsEnabledWithBroker() ? config.getBrokerWebServiceURLTLS() : config.getBrokerWebServiceURL(); +this.functionWorkerWebServiceUrl = config.isTlsEnabledWithBroker() ? config.getFunctionWorkerWebServiceURLTLS() +: config.getFunctionWorkerWebServiceURL(); } @Override @@ -122,7 +125,16 @@ class AdminProxyHandler extends AsyncProxyServlet { protected String rewriteTarget(HttpServletRequest request) { StringBuilder url = new StringBuilder(); -if (isBlank(brokerWebServiceUrl)) { +boolean isFunctionsRestRequest = false; +String requestUri = request.getRequestURI(); +if (requestUri.startsWith("/admin/v2/functions") +|| requestUri.startsWith("/admin/functions")) { +isFunctionsRestRequest = true; +} + +if (isFunctionsRestRequest && !isBlank(functionWorkerWebServiceUrl)) { +url.append(functionWorkerWebServiceUrl); +} else if (isBlank(brokerWebServiceUrl)) { try { ServiceLookupData availableBroker = discoveryProvider.nextBroker(); @@ -148,7 +160,7 @@ class AdminProxyHandler extends AsyncProxyServlet { if (url.lastIndexOf("/") == url.length() - 1) { url.deleteCharAt(url.lastIndexOf("/")); } -url.append(request.getRequestURI()); +url.append(requestUri); String query = request.getQueryString(); if (query != null) { diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index b4e8afb..0155baa 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java @@ -48,6 +48,10 @@ public class ProxyConfiguration implements PulsarConfiguration { private String brokerWebServiceURL; private String brokerWebServiceURLTLS; +
[incubator-pulsar] branch master updated: PIP-22: Dead Letter Topic (#2508)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 95fe84c PIP-22: Dead Letter Topic (#2508) 95fe84c is described below commit 95fe84c7b43060102bb08f0dfe85dadb152f4542 Author: penghui AuthorDate: Fri Sep 14 15:37:58 2018 +0800 PIP-22: Dead Letter Topic (#2508) ### Motivation Fixes #189 When consumer got messages from pulsar, It's difficult to ensure every message can be consume success. Pulsar support message redelivery feature by set acknowledge timeout when create a new consumer. This is a good feature guarantee consumer will not lost messages. But however, some message will redelivery so many times possible, even to the extent that it can be never stop. So, It's necessary to support a feature to control it by pulsar. Users can use this feature and customize this feature to control the message redelivery behavior. The feature named Dead Letter Topic. ### Modifications Consumer can set maximum number of redeliveries by java client. Consumer can set the name of Dead Letter Topic by java client, It’s not necessary. Message exceeding the maximum number of redeliveries should send to Dead Letter Topic and acknowledged automatic. ### Result If consumer enable future of dead letter topic. When Message exceeding the maximum number of redeliveries, message will send to the Dead Letter Topic and acknowledged automatic. --- .../org/apache/pulsar/broker/service/Consumer.java | 3 +- .../apache/pulsar/broker/service/Dispatcher.java | 2 + .../broker/service/InMemoryRedeliveryTracker.java | 58 + .../pulsar/broker/service/RedeliveryTracker.java | 36 +++ .../broker/service/RedeliveryTrackerDisabled.java | 55 + .../NonPersistentDispatcherMultipleConsumers.java | 9 + ...onPersistentDispatcherSingleActiveConsumer.java | 9 + .../PersistentDispatcherMultipleConsumers.java | 14 +- .../PersistentDispatcherSingleActiveConsumer.java | 11 + .../service/persistent/PersistentSubscription.java | 1 + .../apache/pulsar/client/impl/RawReaderImpl.java | 2 +- .../pulsar/client/api/DeadLetterTopicTest.java | 249 + .../client/impl/CompactedOutBatchMessageTest.java | 2 +- .../apache/pulsar/client/api/ConsumerBuilder.java | 24 ++ .../apache/pulsar/client/api/DeadLetterPolicy.java | 32 +++ .../org/apache/pulsar/client/impl/ClientCnx.java | 2 +- .../pulsar/client/impl/ConsumerBuilderImpl.java| 7 + .../apache/pulsar/client/impl/ConsumerImpl.java| 111 - .../impl/conf/ConsumerConfigurationData.java | 3 + .../org/apache/pulsar/common/api/Commands.java | 5 +- .../apache/pulsar/common/api/proto/PulsarApi.java | 57 + pulsar-common/src/main/proto/PulsarApi.proto | 1 + 22 files changed, 680 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index 883bf55..5752034 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -265,7 +265,8 @@ public class Consumer { if (i == (entries.size() - 1)) { promise = writePromise; } -ctx.write(Commands.newMessage(consumerId, messageId, metadataAndPayload), promise); +int redeliveryCount = subscription.getDispatcher().getRedeliveryTracker().getRedeliveryCount(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId())); +ctx.write(Commands.newMessage(consumerId, messageId, redeliveryCount, metadataAndPayload), promise); messageId.recycle(); messageIdBuilder.recycle(); entry.release(); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java index 0d7b7d6..43f65cd 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java @@ -70,4 +70,6 @@ public interface Dispatcher { void addUnAckedMessages(int unAckMessages); +RedeliveryTracker getRedeliveryTracker(); + } diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java new file mode 100644 index 000..99b38cc --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTr
[incubator-pulsar] branch master updated: optimizing throughput in Pulsar Presto connector (#2564)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 6ef7aca optimizing throughput in Pulsar Presto connector (#2564) 6ef7aca is described below commit 6ef7acaf37d57769c4d9dbf7558ef627ce061339 Author: Boyang Jerry Peng AuthorDate: Thu Sep 13 14:50:01 2018 -0700 optimizing throughput in Pulsar Presto connector (#2564) ### Motivation 1. Currently, the presto pulsar connector will read synchronously from bookkeeper when it has run out of entries go process. Basically, we process a batch of entries and then we read more. Ideally should be doing reading and processing in parallel to increase throughput. 2. Each split initializes their own ManagedLedgerFactory/Bookkeeper client. We really just need one bookkeeper client to be shared among threads. ### Modifications 1. Rewrote the logic in the Presto Pulsar connector to read async and process in parallel 2. Cache ManagedLedgerFactory to be used across splits ### Result I see about 2X throughput improvement on single node as well as cluster (2 brokers, 3 bookies, 4 presto workers including coordinator) on AWS --- conf/presto/catalog/pulsar.properties | 6 +- .../apache/pulsar/sql/presto/PulsarConnector.java | 5 + .../pulsar/sql/presto/PulsarConnectorCache.java| 64 +++ .../pulsar/sql/presto/PulsarConnectorConfig.java | 26 ++- .../pulsar/sql/presto/PulsarRecordCursor.java | 201 +++-- .../pulsar/sql/presto/TestPulsarConnector.java | 146 --- .../pulsar/sql/presto/TestPulsarRecordCursor.java | 1 + 7 files changed, 326 insertions(+), 123 deletions(-) diff --git a/conf/presto/catalog/pulsar.properties b/conf/presto/catalog/pulsar.properties index 23b945e..5f922e5 100644 --- a/conf/presto/catalog/pulsar.properties +++ b/conf/presto/catalog/pulsar.properties @@ -26,4 +26,8 @@ pulsar.zookeeper-uri=localhost:2181 # minimum number of entries to read at a single time pulsar.entry-read-batch-size=100 # default number of splits to use per query -pulsar.target-num-splits=4 +pulsar.target-num-splits=2 +# max message queue size +pulsar.max-split-message-queue-size=1 +# max entry queue size +pulsar.max-split-entry-queue-size = 1000 diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java index 1d89b51..498583d 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnector.java @@ -87,6 +87,11 @@ public class PulsarConnector implements Connector { log.error(e, "Failed to close pulsar connector"); } try { +PulsarConnectorCache.shutdown(); +} catch (Exception e) { +log.error("Failed to shutdown pulsar connector cache"); +} +try { lifeCycleManager.stop(); } catch (Exception e) { log.error(e, "Error shutting down connector"); diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java new file mode 100644 index 000..d13ddcd --- /dev/null +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorCache.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.sql.presto; + +import org.apache.bookkeeper.mledger.ManagedLedgerException; +import org.apache.bookkeeper.mledger.ManagedLedgerFactory; +import org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl; +import org.apache.pulsar.shade.org.apache.bookkeeper.conf.ClientConfiguration; + +public class PulsarConnectorCache { + +private static P
[incubator-pulsar] branch master updated: Stop all functions gracefully on closing worker-service (#2548)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 95ed9b9 Stop all functions gracefully on closing worker-service (#2548) 95ed9b9 is described below commit 95ed9b91af20bcdd396ee34d3ea6cc94a52fea31 Author: Rajan Dhabalia AuthorDate: Thu Sep 13 14:50:35 2018 -0700 Stop all functions gracefully on closing worker-service (#2548) ### Motivation Right now, if stopping [WorkerService](https://github.com/apache/incubator-pulsar/blob/master/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java#L189) doesn't stop functions and all the threads stayed alive event `WorkerService` is stopped. ### Modifications Stop all function resource gracefully while stopping worker service. ### Result Function threads will not stay alive while stopping worker-service. --- .../functions/worker/FunctionRuntimeManager.java | 20 1 file changed, 20 insertions(+) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java index ee6eeec..43cd27b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java @@ -389,6 +389,25 @@ public class FunctionRuntimeManager implements AutoCloseable{ return Response.status(Status.OK).build(); } +/** + * It stops all functions instances owned by current worker + * @throws Exception + */ +public void stopAllOwnedFunctions() throws Exception { +final String workerId = this.workerConfig.getWorkerId(); +Map assignments = workerIdToAssignments.get(workerId); +if (assignments != null) { +assignments.values().forEach(assignment -> { +String fullyQualifiedInstanceId = Utils.getFullyQualifiedInstanceId(assignment.getInstance()); +try { +stopFunction(fullyQualifiedInstanceId, false); +} catch (Exception e) { +log.warn("Failed to stop function {} - {}", fullyQualifiedInstanceId, e.getMessage()); +} +}); +} +} + private void stopFunction(String fullyQualifiedInstanceId, boolean restart) throws Exception { log.info("[{}] {}..", restart ? "restarting" : "stopping", fullyQualifiedInstanceId); FunctionRuntimeInfo functionRuntimeInfo = this.getFunctionRuntimeInfo(fullyQualifiedInstanceId); @@ -647,6 +666,7 @@ public class FunctionRuntimeManager implements AutoCloseable{ @Override public void close() throws Exception { +stopAllOwnedFunctions(); this.functionActioner.close(); this.functionAssignmentTailer.close(); if (runtimeFactory != null) {
[incubator-pulsar] branch master updated: Another fix for website build (#2567)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 0964dad Another fix for website build (#2567) 0964dad is described below commit 0964dadcd3f425a2d4c424d4ab1de1fb7e35ce84 Author: Matteo Merli AuthorDate: Thu Sep 13 17:49:36 2018 -0400 Another fix for website build (#2567) ### Motivation Comma is missing in the javascript list. --- site2/website/scripts/replace.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site2/website/scripts/replace.js b/site2/website/scripts/replace.js index 13c9569..2970688 100644 --- a/site2/website/scripts/replace.js +++ b/site2/website/scripts/replace.js @@ -118,7 +118,7 @@ for (v of versions) { `${v}`, binaryReleaseUrl(`${v}`), connectorReleaseUrl(`${v}`), - prestoPulsarReleaseUrl(`${latestVersion}`) + prestoPulsarReleaseUrl(`${latestVersion}`), downloadPageUrl(), rpmReleaseUrl(`${v}`, ""), rpmReleaseUrl(`${v}`, "-debuginfo"),
[incubator-pulsar] branch master updated: Fix: get function status with auth enable (#2516)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new e5a7b53 Fix: get function status with auth enable (#2516) e5a7b53 is described below commit e5a7b532ffbf1d8813fc31403162fe3c17e937f5 Author: Rajan Dhabalia AuthorDate: Thu Sep 13 11:29:37 2018 -0700 Fix: get function status with auth enable (#2516) --- .../pulsar/broker/admin/impl/FunctionsBase.java| 2 +- .../client/admin/internal/FunctionsImpl.java | 5 ++- .../functions/worker/FunctionRuntimeManager.java | 41 -- .../functions/worker/rest/api/FunctionsImpl.java | 6 ++-- .../worker/rest/api/v2/FunctionApiV2Resource.java | 2 +- 5 files changed, 23 insertions(+), 33 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java index def5452..62d12ec 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java @@ -160,7 +160,7 @@ public class FunctionsBase extends AdminResource implements Supplierhttp://%s:%d/admin/functions/%s/%s/%s/%d/status";, -workerInfo.getWorkerHostname(), workerInfo.getPort(), tenant, namespace, functionName, instanceId)) -.request(MediaType.TEXT_PLAIN) -.get(String.class); - -InstanceCommunication.FunctionStatus.Builder functionStatusBuilder = InstanceCommunication.FunctionStatus.newBuilder(); -try { - org.apache.pulsar.functions.utils.Utils.mergeJson(jsonResponse, functionStatusBuilder); -} catch (IOException e) { -log.warn("Got invalid function status response from {}", workerInfo, e); -throw new RuntimeException(e); -} -functionStatusBuilder.setWorkerId(assignedWorkerId); -functionStatus = functionStatusBuilder.build(); +URI redirect = UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build(); +throw new WebApplicationException(Response.temporaryRedirect(redirect).build()); } return functionStatus; @@ -426,9 +412,10 @@ public class FunctionRuntimeManager implements AutoCloseable{ * @param namespace the namespace the function belongs to * @param functionName the function name * @return a list of function statuses + * @throws PulsarAdminException */ public InstanceCommunication.FunctionStatusList getAllFunctionStatus( -String tenant, String namespace, String functionName) { +String tenant, String namespace, String functionName) throws PulsarAdminException { Collection assignments = this.findFunctionAssignments(tenant, namespace, functionName); @@ -438,13 +425,15 @@ public class FunctionRuntimeManager implements AutoCloseable{ } for (Assignment assignment : assignments) { - -InstanceCommunication.FunctionStatus functionStatus = this.getFunctionInstanceStatus( - assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), - assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), - assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), -assignment.getInstance().getInstanceId()); - +boolean isOwner = this.workerConfig.getWorkerId().equals(assignment.getWorkerId()); +InstanceCommunication.FunctionStatus functionStatus = isOwner +? (getFunctionInstanceStatus(tenant, namespace, functionName, +assignment.getInstance().getInstanceId(), null)) +: this.functionAdmin.functions().getFunctionStatus( + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(), + assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(), +assignment.getInstance().getInstanceId()); functionStatusListBuilder.addFunctionStatusList(functionStatus); } return functionStatusListBuilder.build(); diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java index 136bab0..df82c0d 100644 --- a/pulsar-functions/
[incubator-pulsar] branch master updated: Remove property term from dashboard. (#2553)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 5506b54 Remove property term from dashboard. (#2553) 5506b54 is described below commit 5506b54d99a5f85ee8d6b5fcb1e652b83bad3ef1 Author: cckellogg AuthorDate: Wed Sep 12 02:45:44 2018 -0700 Remove property term from dashboard. (#2553) This updates the html to display the new v2 naming. ### Motivation This brings the dashboard up to date with the new v2 naming. Property/properties are now tenant/tenants. --- dashboard/django/stats/templates/stats/base.html | 2 +- dashboard/django/stats/templates/stats/home.html | 4 ++-- dashboard/django/stats/views.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboard/django/stats/templates/stats/base.html b/dashboard/django/stats/templates/stats/base.html index 1e5843f..9ffdb77 100644 --- a/dashboard/django/stats/templates/stats/base.html +++ b/dashboard/django/stats/templates/stats/base.html @@ -50,7 +50,7 @@ {% block nav-global %}   -Properties  |  +Tenants  |  Brokers  |  Topics  |  Clusters diff --git a/dashboard/django/stats/templates/stats/home.html b/dashboard/django/stats/templates/stats/home.html index 828bfae..7e9cb80 100644 --- a/dashboard/django/stats/templates/stats/home.html +++ b/dashboard/django/stats/templates/stats/home.html @@ -30,7 +30,7 @@ -{% column_header properties 'name' 'Property' %} +{% column_header properties 'name' 'Tenant' %} {% column_header properties 'numNamespaces' 'Namespaces' %} {% column_header properties 'numTopics' 'Topics' %} {% column_header properties 'numProducers' 'Producers' %} @@ -64,7 +64,7 @@ {{property.storage | filesizeformat}} {% empty %} -No properties +No tenants {% endfor %} diff --git a/dashboard/django/stats/views.py b/dashboard/django/stats/views.py index 9ab6859..6aba518 100644 --- a/dashboard/django/stats/views.py +++ b/dashboard/django/stats/views.py @@ -62,7 +62,7 @@ def home(request): return render(request, 'stats/home.html', { 'properties': properties, -'title' : 'Properties', +'title' : 'Tenants', }) def property(request, property_name):
[incubator-pulsar] branch master updated: [documentation][example] Flink Source & Sink Connector (#2561)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 71003e2 [documentation][example] Flink Source & Sink Connector (#2561) 71003e2 is described below commit 71003e20e6606527c2f538ba2059850d9539d878 Author: Sijie Guo AuthorDate: Wed Sep 12 01:38:56 2018 -0700 [documentation][example] Flink Source & Sink Connector (#2561) ### Motivation We added flink source connector (#2441) and sink connector (#2434). It would be great to an example to show how to use flink source & sink connector. ### Modifications - introduce an `examples` module - introduce an `examples/flink-consumer-source` module - add a word count example to use flink source and sink connector ### Result be able to know how to use flink source & sink connector --- examples/flink-consumer-source/README.md | 76 + examples/flink-consumer-source/pom.xml | 97 .../flink/PulsarConsumerSourceWordCount.java | 126 + examples/pom.xml | 39 +++ pom.xml| 3 + .../connectors/pulsar/FlinkPulsarProducer.java | 10 +- .../connectors/pulsar/PulsarConsumerSource.java| 1 - .../pulsar/partitioner/PulsarKeyExtractor.java | 4 +- 8 files changed, 350 insertions(+), 6 deletions(-) diff --git a/examples/flink-consumer-source/README.md b/examples/flink-consumer-source/README.md new file mode 100644 index 000..38b75b6 --- /dev/null +++ b/examples/flink-consumer-source/README.md @@ -0,0 +1,76 @@ +## Apache Flink Connectors for Pulsar + +This page describes how to use the connectors to read and write Pulsar topics with [Apache Flink](https://flink.apache.org/) stream processing applications. + +Build end-to-end stream processing pipelines that use Pulsar as the stream storage and message bus, and Apache Flink for computation over the streams. +See the [Pulsar Concepts](https://pulsar.incubator.apache.org/docs/en/concepts-overview/) page for more information. + +## Example + +### PulsarConsumerSourceWordCount + +This Flink streaming job is consuming from a Pulsar topic and couting the wordcount in a streaming fashion. The job can write the word count results +to stdout or another Pulsar topic. + +The steps to run the example: + +1. Start Pulsar Standalone. + +You can follow the [instructions](https://pulsar.incubator.apache.org/docs/en/standalone/) to start a Pulsar standalone locally. + +```shell +$ bin/pulsar standalone +``` + +2. Start Flink locally. + +You can follow the [instructions](https://ci.apache.org/projects/flink/flink-docs-release-1.6/quickstart/setup_quickstart.html) to download and start Flink. + +```shell +$ ./bin/start-cluster.sh +``` + +3. Build the examples. + +```shell +$ cd ${PULSAR_HOME} +$ mvn clean install -DskipTests +``` + +4. Run the word count example to print results to stdout. + +```shell +$ ./bin/flink run ${PULSAR_HOME}/examples/flink-consumer-source/target/pulsar-flink-streaming-wordcount.jar --service-url pulsar://localhost:6650 --input-topic test_src --subscription test_sub +``` + +5. Produce messages to topic `test_src`. + +```shell +$ bin/pulsar-client produce -m "hello world test again" -n 100 test_src +``` + +6. You can check the flink taskexecutor `.out` file. The `.out` file will print the counts at the end of each time window as long as words are floating in, e.g.: + +```shell +PulsarConsumerSourceWordCount.WordWithCount(word=hello, count=200) +PulsarConsumerSourceWordCount.WordWithCount(word=again, count=200) +PulsarConsumerSourceWordCount.WordWithCount(word=test, count=200) +PulsarConsumerSourceWordCount.WordWithCount(word=world, count=200) +PulsarConsumerSourceWordCount.WordWithCount(word=hello, count=100) +PulsarConsumerSourceWordCount.WordWithCount(word=again, count=100) +PulsarConsumerSourceWordCount.WordWithCount(word=test, count=100) +``` + +Alternatively, when you run the flink word count example at step 4, you can choose dump the result to another pulsar topic. + +```shell +$ ./bin/flink run ${PULSAR_HOME}/examples/flink-consumer-source/target/pulsar-flink-streaming-wordcount.jar --service-url pulsar://localhost:6650 --input-topic test_src --subscription test_sub --output-topic test_dest +``` + +Once the flink word count example is running, you can use `bin/pulsar-client` to tail the results produced into topic `test_dest`. + +```shell +$ bin/pulsar-client consume -n 0 -s test test_dest +``` + +You will see similar results as what you see at step 6 when running the word count example to print results to stdout. diff --git a/exam
[incubator-pulsar] branch master updated: Removed sha1 from release signing script (#2563)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 0ce9e1f Removed sha1 from release signing script (#2563) 0ce9e1f is described below commit 0ce9e1f4ef5f372d8199eedbaa04811b10115dab Author: Matteo Merli AuthorDate: Wed Sep 12 03:11:26 2018 -0400 Removed sha1 from release signing script (#2563) ### Motivation We have removed sha1 links from main download page but we should also remove from the release signing script. --- site2/docs/client-libraries-cpp.md | 10 +- .../version-2.1.0-incubating/client-libraries-cpp.md | 6 +++--- src/sign-release.sh| 3 --- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/site2/docs/client-libraries-cpp.md b/site2/docs/client-libraries-cpp.md index ee22377..864efdd 100644 --- a/site2/docs/client-libraries-cpp.md +++ b/site2/docs/client-libraries-cpp.md @@ -19,9 +19,9 @@ The Pulsar C++ client has been successfully tested on **MacOS** and **Linux**. | Link | Crypto files | |--|--| -| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc), [sha1]({{pulsar:rpm:client}}.sha1), [sha512]({{pulsar:rpm:client}}.sha512) | -| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) | [asc]({{pulsar:rpm:client-debuginfo}}.asc), [sha1]({{pulsar:rpm:client-debuginfo}}.sha1), [sha512]({{pulsar:rpm:client-debuginfo}}.sha512) | -| [client-devel]({{pulsar:rpm:client-devel}}) | [asc]({{pulsar:rpm:client-devel}}.asc), [sha1]({{pulsar:rpm:client-devel}}.sha1), [sha512]({{pulsar:rpm:client-devel}}.sha512) | +| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc), [sha512]({{pulsar:rpm:client}}.sha512) | +| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) | [asc]({{pulsar:rpm:client-debuginfo}}.asc), [sha512]({{pulsar:rpm:client-debuginfo}}.sha512) | +| [client-devel]({{pulsar:rpm:client-devel}}) | [asc]({{pulsar:rpm:client-devel}}.asc), [sha512]({{pulsar:rpm:client-devel}}.sha512) | To install a RPM package, download the RPM packages and install them using the following command: @@ -33,8 +33,8 @@ $ rpm -ivh apache-pulsar-client*.rpm | Link | Crypto files | |--|--| -| [client]({{pulsar:deb:client}}) | [asc]({{pulsar:deb:client}}.asc), [sha1]({{pulsar:deb:client}}.sha1), [sha512]({{pulsar:deb:client}}.sha512) | -| [client-devel]({{pulsar:deb:client-devel}}) | [asc]({{pulsar:deb:client-devel}}.asc), [sha1]({{pulsar:deb:client-devel}}.sha1), [sha512]({{pulsar:deb:client-devel}}.sha512) | +| [client]({{pulsar:deb:client}}) | [asc]({{pulsar:deb:client}}.asc), [sha512]({{pulsar:deb:client}}.sha512) | +| [client-devel]({{pulsar:deb:client-devel}}) | [asc]({{pulsar:deb:client-devel}}.asc), [sha512]({{pulsar:deb:client-devel}}.sha512) | To install a DEB package, download the DEB packages and install them using the following command: diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md b/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md index 2b6d137..ae67ebc 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md @@ -20,9 +20,9 @@ The Pulsar C++ client has been successfully tested on **MacOS** and **Linux**. | Link | Crypto files | |--|--| -| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc), [sha1]({{pulsar:rpm:client}}.sha1), [sha512]({{pulsar:rpm:client}}.sha512) | -| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) | [asc]({{pulsar:rpm:client-debuginfo}}.asc), [sha1]({{pulsar:rpm:client-debuginfo}}.sha1), [sha512]({{pulsar:rpm:client-debuginfo}}.sha512) | -| [client-devel]({{pulsar:rpm:client-devel}}) | [asc]({{pulsar:rpm:client-devel}}.asc), [sha1]({{pulsar:rpm:client-devel}}.sha1), [sha512]({{pulsar:rpm:client-devel}}.sha512) | +| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc), [sha512]({{pulsar:rpm:client}}.sha512) | +| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) | [asc]({{pulsar:rpm:client-debuginfo}}.asc), [sha512]({{pulsar:rpm:client-debuginfo}}.sha512) | +| [client-devel]({{pulsar:rpm:client-devel}}) | [asc]({{pulsar:rpm:client-devel}}.asc), [sha512]({{pulsar:rpm:client-devel}}.sha512) | To install a RPM package, down the RPM packages and install them using following command: diff --git a/src/sign-release.sh b/src/sign-release.sh index b26e643..70977db 100755 --- a/src/sign-release.sh +++ b/src/sign-release.sh @@ -27,9 +27,6 @@ do echo "Signing $FILE" gpg --armor --output $FILE.asc --detach-sig $FILE - # SHA-1 signature - shasum -a 1 $FILE > $FILE.sha1 -
[incubator-pulsar] branch master updated: [ecosystem] Flink pulsar source connector (#2555)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new fc61056 [ecosystem] Flink pulsar source connector (#2555) fc61056 is described below commit fc61056ccab29bb068b65d71120364d14503f69f Author: Sijie Guo AuthorDate: Tue Sep 11 12:11:41 2018 -0700 [ecosystem] Flink pulsar source connector (#2555) *Motivation* This ports apache/flink#6200 to apache pulsar repo. It adds a pulsar source connector which will enable flink jobs to process messages from pulsar topics. *Changes* Add a PulsarConsumerSource connector --- .../connectors/pulsar/PulsarConsumerSource.java| 204 .../connectors/pulsar/PulsarSourceBase.java| 31 ++ .../connectors/pulsar/PulsarSourceBuilder.java | 118 + .../pulsar/PulsarConsumerSourceTests.java | 524 + 4 files changed, 877 insertions(+) diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java new file mode 100644 index 000..f1b2595 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.flink.streaming.connectors.pulsar; + +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.streaming.api.functions.source.MessageAcknowledgingSourceBase; +import org.apache.flink.streaming.api.operators.StreamingRuntimeContext; +import org.apache.flink.util.IOUtils; + +import org.apache.pulsar.client.api.Consumer; +import org.apache.pulsar.client.api.Message; +import org.apache.pulsar.client.api.MessageId; +import org.apache.pulsar.client.api.PulsarClient; +import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.SubscriptionType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +/** + * Pulsar source (consumer) which receives messages from a topic and acknowledges messages. + * When checkpointing is enabled, it guarantees at least once processing semantics. + * + * When checkpointing is disabled, it auto acknowledges messages based on the number of messages it has + * received. In this mode messages may be dropped. + */ +class PulsarConsumerSource extends MessageAcknowledgingSourceBase implements PulsarSourceBase { + +private static final Logger LOG = LoggerFactory.getLogger(PulsarConsumerSource.class); + +private final int messageReceiveTimeoutMs = 100; +private final String serviceUrl; +private final String topic; +private final String subscriptionName; +private final DeserializationSchema deserializer; + +private PulsarClient client; +private Consumer consumer; + +private boolean isCheckpointingEnabled; + +private final long acknowledgementBatchSize; +private long batchCount; +private long totalMessageCount; + +private transient volatile boolean isRunning; + +PulsarConsumerSource(PulsarSourceBuilder builder) { +super(MessageId.class); +this.serviceUrl = builder.serviceUrl; +this.topic = builder.topic; +this.deserializer = builder.deserializationSchema; +this.subscriptionName = builder.subscriptionName; +this.acknowledgementBatchSize = builder.acknowledgementBatchSize; +} + +@Override +public void open(Configuration parameters) throws Exception { +super.open(parameters); + +final RuntimeContext context
[incubator-pulsar] branch master updated: Readd MockBookKeeper to Pulsar (#2544)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 69571f8 Readd MockBookKeeper to Pulsar (#2544) 69571f8 is described below commit 69571f8ef67b99d66c7d405e17870100d0aa5080 Author: Ivan Kelly AuthorDate: Tue Sep 11 01:31:14 2018 +0200 Readd MockBookKeeper to Pulsar (#2544) These mocks were moved out of pulsar to the bookkeeper project a few months ago. While it is good to have mocks generally available for bookkeeper, if the mock is not in the pulsar code base and we want to change the behaviour of the mock for a specific case, we need to wait for a bookkeeper release cycle to do so. It's better to have the mock in Pulsar, so we can bend it to our needs. --- .../bookkeeper/client/PulsarMockBookKeeper.java| 284 + .../bookkeeper/client/PulsarMockLedgerHandle.java | 243 ++ .../bookkeeper/client/PulsarMockReadHandle.java| 125 + .../mledger/impl/ManagedLedgerErrorsTest.java | 4 +- .../bookkeeper/test/MockedBookKeeperTestCase.java | 6 +- .../broker/MockedBookKeeperClientFactory.java | 18 +- .../broker/auth/MockedPulsarServiceBaseTest.java | 31 ++- .../PersistentDispatcherFailoverConsumerTest.java | 3 +- .../pulsar/broker/service/PersistentTopicTest.java | 3 +- .../pulsar/broker/service/ServerCnxTest.java | 3 +- .../impl/BlobStoreManagedLedgerOffloaderTest.java | 6 +- 11 files changed, 704 insertions(+), 22 deletions(-) diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java b/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java new file mode 100644 index 000..f3689e9 --- /dev/null +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java @@ -0,0 +1,284 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.client; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.bookkeeper.client.AsyncCallback.CreateCallback; +import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback; +import org.apache.bookkeeper.client.AsyncCallback.OpenCallback; +import org.apache.bookkeeper.client.api.OpenBuilder; +import org.apache.bookkeeper.client.api.ReadHandle; +import org.apache.bookkeeper.client.impl.OpenBuilderBase; +import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.zookeeper.ZooKeeper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Mocked version of BookKeeper client that keeps all ledgers data in memory. + * + * This mocked client is meant to be used in unit tests for applications using the BookKeeper API. + */ +public class PulsarMockBookKeeper extends BookKeeper { + +final ExecutorService executor; +final ZooKeeper zkc; + +@Override +public ZooKeeper getZkHandle() { +return zkc; +} + +@Override +public ClientConfiguration getConf() { +return super.getConf(); +} + +Map ledgers = new ConcurrentHashMap<>(); +AtomicLong sequence = new AtomicLong(3); + +CompletableFuture defaultResponse = CompletableFuture.completedFuture(null); +List> failures = new ArrayList<>(); + +public PulsarMockBookKeeper(ZooKeeper zkc, ExecutorService executor) throws Exception { +this.zkc = zkc; +this.executor = executor; +} + +@Override +public LedgerHandle createLedger(DigestType digestType, byte passwd[]) +throws BKException, InterruptedException { +return createLedger(3, 2, dige
[incubator-pulsar] branch master updated: Fix dashboard to work with python3. (#2552)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new fc2d1d3 Fix dashboard to work with python3. (#2552) fc2d1d3 is described below commit fc2d1d3da7ffb8bb47e30ec49c07cf8d9f735246 Author: cckellogg AuthorDate: Mon Sep 10 13:33:29 2018 -0700 Fix dashboard to work with python3. (#2552) ### Motivation Make dashboard work with python 3. --- dashboard/django/collector.py | 10 +- dashboard/django/stats/views.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dashboard/django/collector.py b/dashboard/django/collector.py index 76c0646..c8c3b13 100755 --- a/dashboard/django/collector.py +++ b/dashboard/django/collector.py @@ -63,7 +63,7 @@ def fetch_broker_stats(cluster, broker_url, timestamp): def _fetch_broker_stats(cluster, broker_host_port, timestamp): broker_url = 'http://%s/' % broker_host_port -print 'Getting stats for %s' % broker_host_port +print('Getting stats for %s' % broker_host_port) broker, _ = Broker.objects.get_or_create( url = broker_host_port, @@ -262,7 +262,7 @@ def fetch_stats(): if cluster_name == 'global': continue cluster_url = get(args.serviceUrl, '/admin/clusters/' + cluster_name)['serviceUrl'] -print 'Cluster:', cluster_name, '->', cluster_url +print('Cluster:', cluster_name, '->', cluster_url) cluster, created = Cluster.objects.get_or_create(name=cluster_name) if cluster_url != cluster.serviceUrl: cluster.serviceUrl = cluster_url @@ -275,7 +275,7 @@ def fetch_stats(): f = pool.apply_async(fetch_broker_stats, (cluster, broker_host_port, timestamp)) futures.append(f) except Exception as e: -print 'ERROR: ', e +print('ERROR: ', e) pool.close() @@ -300,10 +300,10 @@ def purge_db(): Consumer.objects.filter(timestamp__lt = threshold).delete() def collect_and_purge(): -print '-- Starting stats collection' +print('-- Starting stats collection') fetch_stats() purge_db() -print '-- Finished collecting stats' +print('-- Finished collecting stats') if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings") diff --git a/dashboard/django/stats/views.py b/dashboard/django/stats/views.py index 426e9b1..9ab6859 100644 --- a/dashboard/django/stats/views.py +++ b/dashboard/django/stats/views.py @@ -56,7 +56,7 @@ def home(request): throughputOut = Sum('namespace__topic__msgThroughputOut'), ) -print properties.query +print(properties.query) properties = Table(request, properties, default_sort='name')
[incubator-pulsar] branch master updated: Elastic connector (#2546)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 928c3e1 Elastic connector (#2546) 928c3e1 is described below commit 928c3e1a9e1e1d27b3bf5bf816890d1316c4b5d2 Author: David Kjerrumgaard <35466513+david-stream...@users.noreply.github.com> AuthorDate: Mon Sep 10 13:31:49 2018 -0700 Elastic connector (#2546) ### Motivation Added a sink connector that writes JSON documents into ElasticSearch ### Modifications Added new pulsar-io module and associated integration tests ### Result An ElasticSearch sink connector will be available for use. --- pulsar-io/{hdfs => elastic-search}/pom.xml | 38 -- .../elasticsearch/ElasticSearchAbstractSink.java | 150 + .../io/elasticsearch/ElasticSearchConfig.java | 88 .../io/elasticsearch/ElasticSearchStringSink.java | 35 + .../pulsar/io/elasticsearch/package-info.java | 19 +++ .../resources/META-INF/services/pulsar-io.yaml | 22 +++ .../io/elasticsearch/ElasticSearchConfigTests.java | 125 + .../io/elasticsearch/ElasticSearchSinkTests.java | 144 .../pulsar/io/elasticsearch/data/Profile.java | 33 + .../pulsar/io/elasticsearch/data/UserProfile.java | 35 + .../src/test/resources/sinkConfig.yaml | 25 pulsar-io/hdfs/pom.xml | 1 + pulsar-io/pom.xml | 1 + tests/integration/pom.xml | 7 + .../containers/ElasticSearchContainer.java | 47 +++ .../integration/functions/PulsarFunctionsTest.java | 7 +- .../integration/io/ElasticSearchSinkTester.java| 75 +++ .../pulsar/tests/integration/io/SinkTester.java| 3 +- .../tests/integration/suites/PulsarTestSuite.java | 6 + 19 files changed, 851 insertions(+), 10 deletions(-) diff --git a/pulsar-io/hdfs/pom.xml b/pulsar-io/elastic-search/pom.xml similarity index 70% copy from pulsar-io/hdfs/pom.xml copy to pulsar-io/elastic-search/pom.xml index 0d55207..1339b2f 100644 --- a/pulsar-io/hdfs/pom.xml +++ b/pulsar-io/elastic-search/pom.xml @@ -25,15 +25,24 @@ pulsar-io 2.2.0-incubating-SNAPSHOT - pulsar-io-hdfs + pulsar-io-elastic-search + Pulsar IO :: ElasticSearch + + +jcenter +https://jcenter.bintray.com/ + + + - + + ${project.groupId} pulsar-io-core ${project.version} - + com.fasterxml.jackson.core jackson-databind @@ -44,15 +53,28 @@ jackson-dataformat-yaml + + org.apache.commons + commons-lang3 + 3.4 + + - org.apache.hadoop - hadoop-client - 3.1.1 + org.elasticsearch.client + elasticsearch-rest-high-level-client + 6.3.2 + + +net.andreinc.mockneat +mockneat +0.2.2 +test + - org.testng - testng + com.google.code.gson + gson test diff --git a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java new file mode 100644 index 000..3760d40 --- /dev/null +++ b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java @@ -0,0 +1,150 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.elasticsearch; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Usernam
[incubator-pulsar] branch master updated: [docker] introduce a pulsar standalone image (#2545)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 2812fef [docker] introduce a pulsar standalone image (#2545) 2812fef is described below commit 2812fefb72aa8f7277a8a97d8842b0ad2dc6719b Author: Sijie Guo AuthorDate: Mon Sep 10 13:31:19 2018 -0700 [docker] introduce a pulsar standalone image (#2545) ## Motivation `pulsar` and `pulsar-all` are designed for running pulsar components on production. although it can be used for running standalone, people still need to run a separate docker container for pulsar dashboard. ## Changes introduce a `pulsar-standalone` image to package everything into one image. so people can run launch a pulsar standalone in one line command in docker, including dashboard. --- docker/pom.xml | 1 + docker/pulsar-standalone/Dockerfile| 55 docker/pulsar-standalone/conf/nginx-app.conf | 37 +++ docker/pulsar-standalone/conf/postgresql.conf | 38 +++ docker/pulsar-standalone/conf/supervisor-app.conf | 34 ++ docker/pulsar-standalone/conf/uwsgi.ini| 45 +++ docker/pulsar-standalone/conf/uwsgi_params | 16 + docker/pulsar-standalone/django/collector.py | 345 + docker/pulsar-standalone/django/collector.sh | 23 ++ .../pulsar-standalone/django/dashboard/__init__.py | 19 ++ .../pulsar-standalone/django/dashboard/settings.py | 181 +++ docker/pulsar-standalone/django/dashboard/urls.py | 45 +++ docker/pulsar-standalone/django/dashboard/wsgi.py | 35 +++ docker/pulsar-standalone/django/manage.py | 41 +++ docker/pulsar-standalone/django/stats/__init__.py | 19 ++ docker/pulsar-standalone/django/stats/admin.py | 34 ++ docker/pulsar-standalone/django/stats/apps.py | 26 ++ .../django/stats/migrations/0001_initial.py| 221 + .../django/stats/migrations/__init__.py| 20 ++ docker/pulsar-standalone/django/stats/models.py| 200 .../django/stats/templates/stats/base.html | 98 ++ .../django/stats/templates/stats/broker.html | 71 + .../django/stats/templates/stats/brokers.html | 90 ++ .../django/stats/templates/stats/clusters.html | 106 +++ .../django/stats/templates/stats/home.html | 74 + .../django/stats/templates/stats/namespace.html| 96 ++ .../django/stats/templates/stats/property.html | 72 + .../django/stats/templates/stats/topic.html| 197 .../django/stats/templates/stats/topics.html | 91 ++ .../django/stats/templatetags/__init__.py | 19 ++ .../django/stats/templatetags/stats_extras.py | 64 .../django/stats/templatetags/table.py | 91 ++ docker/pulsar-standalone/django/stats/tests.py | 22 ++ docker/pulsar-standalone/django/stats/urls.py | 37 +++ docker/pulsar-standalone/django/stats/views.py | 283 + docker/pulsar-standalone/pom.xml | 81 + site2/docs/getting-started-standalone.md | 27 +- .../getting-started-standalone.md | 14 +- 38 files changed, 2957 insertions(+), 11 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index 675656a..bdc99f7 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -38,5 +38,6 @@ pulsar grafana pulsar-all +pulsar-standalone diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile new file mode 100644 index 000..869b4c0 --- /dev/null +++ b/docker/pulsar-standalone/Dockerfile @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM apachepulsar/pulsar-all:latest + +RUN apt-get update +RUN apt-get -y install postgresql sudo nginx supervisor + +# Python dependencies +RUN pip install uwsgi 'Django<2.0' psycopg2 pytz requests + +# Postgres configuration +COPY conf/postgresql.conf /etc/postgresql/9
[incubator-pulsar] branch master updated: [documentation] Add section about licensing at contribution guide (#2511)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 3e9a445 [documentation] Add section about licensing at contribution guide (#2511) 3e9a445 is described below commit 3e9a445e9ced577c1e494875e9a490c3dee9a5ab Author: Sijie Guo AuthorDate: Sun Sep 9 16:51:03 2018 -0700 [documentation] Add section about licensing at contribution guide (#2511) --- site2/website/contributing.md | 9 + 1 file changed, 9 insertions(+) diff --git a/site2/website/contributing.md b/site2/website/contributing.md index 9552244..310045a 100644 --- a/site2/website/contributing.md +++ b/site2/website/contributing.md @@ -181,6 +181,15 @@ Then you can push your local, committed changes to your (forked) repository on G All code should have appropriate unit testing coverage. New code should have new tests in the same contribution. Bug fixes should include a regression test to prevent the issue from reoccurring. +### Licensing + +All code contributed to Pulsar will be licensed under [Apache License V2](https://www.apache.org/licenses/LICENSE-2.0). You need to ensure every new files you are adding have the right +license header. You can add license header to your files by running following command: + +```shell +$ mvn license:format +``` + ### Develop C++/Python Client Currently C++ and Python client are developed under `pulsar-client-cpp` directory. You can use following commands to build and test your c++ or python code changes.
[incubator-pulsar] branch master updated: Added cmake install for lib and headers (#2529)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 1c72ac3 Added cmake install for lib and headers (#2529) 1c72ac3 is described below commit 1c72ac3fb1d20b4f12300c4d339cb1a383d718ae Author: Matteo Merli AuthorDate: Sun Sep 9 16:50:49 2018 -0700 Added cmake install for lib and headers (#2529) ### Motivation Added `install` target to do local install for C++ client lib --- pulsar-client-cpp/lib/CMakeLists.txt | 4 1 file changed, 4 insertions(+) diff --git a/pulsar-client-cpp/lib/CMakeLists.txt b/pulsar-client-cpp/lib/CMakeLists.txt index 5b2dbf9..2116ed6 100644 --- a/pulsar-client-cpp/lib/CMakeLists.txt +++ b/pulsar-client-cpp/lib/CMakeLists.txt @@ -45,3 +45,7 @@ set_target_properties(pulsarShared PROPERTIES OUTPUT_NAME ${LIB_NAME} VERSION ${ target_link_libraries(pulsarStatic ${COMMON_LIBS}) target_link_libraries(pulsarShared ${COMMON_LIBS}) +install(TARGETS pulsarStatic DESTINATION lib) +install(TARGETS pulsarShared DESTINATION lib) + +install(DIRECTORY "../include/pulsar" DESTINATION include) \ No newline at end of file
[incubator-pulsar] branch master updated: fixed zookeeper name references in helm charts. (#2525)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 879c730 fixed zookeeper name references in helm charts. (#2525) 879c730 is described below commit 879c730daf781ef27ad1344803ca97ad60cd4b62 Author: Victor AuthorDate: Fri Sep 7 12:25:23 2018 -0700 fixed zookeeper name references in helm charts. (#2525) ### Motivation zookeeper failed to start because of wrong ZOOKEEPER_SERVERS was set. ### Modifications Changed the reference of zookeeper names by how they were created. ### Result zookeeper started successfully and broker worked as expected. --- .../kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml | 2 +- deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml | 2 +- deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml | 4 ++-- deployment/kubernetes/helm/pulsar/templates/proxy-configmap.yaml | 4 ++-- .../kubernetes/helm/pulsar/templates/zookeeper-statefulset.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml b/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml index 2e25112..4200e6c 100644 --- a/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml +++ b/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml @@ -33,6 +33,6 @@ metadata: data: zkServers: {{- $global := . }} -{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component }}{{ end }} +{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }} {{ toYaml .Values.autoRecovery.configData | indent 2 }} {{- end }} diff --git a/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml b/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml index 31c66df..45a8546 100644 --- a/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml +++ b/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml @@ -32,7 +32,7 @@ metadata: data: zkServers: {{- $global := . }} -{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component }}{{ end }} +{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }} # disable auto recovery on bookies since we will start AutoRecovery in separated pods autoRecoveryDaemonEnabled: "false" {{ toYaml .Values.bookkeeper.configData | indent 2 }} diff --git a/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml b/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml index 7d7df75..d577826 100644 --- a/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml +++ b/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml @@ -32,10 +32,10 @@ metadata: data: zookeeperServers: {{- $global := . }} -{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component }}{{ end }} +{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template "pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }} configurationStoreServers: {{- $global := . }} -{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne $i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name $global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name $global.Chart.Name $global.Values.z
[incubator-pulsar] branch master updated: Typo in CSR generation docs (#2530) (#2531)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 6860a90 Typo in CSR generation docs (#2530) (#2531) 6860a90 is described below commit 6860a90bc685003afef17ab8bfa33e2701c99bfe Author: Ivan Kelly AuthorDate: Thu Sep 6 19:56:52 2018 +0200 Typo in CSR generation docs (#2530) (#2531) The output file for CSR generation was .cert.pem rather than .csr.pem, which causes confusion. It should be .csr.pem --- site2/docs/security-tls-authentication.md | 2 +- site2/docs/security-tls-transport.md| 2 +- .../version-2.1.0-incubating/security-tls-authentication.md | 2 +- .../versioned_docs/version-2.1.0-incubating/security-tls-transport.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/site2/docs/security-tls-authentication.md b/site2/docs/security-tls-authentication.md index d5eb7d7..10578d2 100644 --- a/site2/docs/security-tls-authentication.md +++ b/site2/docs/security-tls-authentication.md @@ -30,7 +30,7 @@ Generate the certificate request. When asked for a **common name**, enter the ** ```bash $ openssl req -config openssl.cnf \ - -key admin.key.pem -new -sha256 -out admin.cert.pem + -key admin.key.pem -new -sha256 -out admin.csr.pem ``` Sign with request with the certificate authority. Note that that client certs uses the **usr_cert** extension, which allows the cert to be used for client authentication. diff --git a/site2/docs/security-tls-transport.md b/site2/docs/security-tls-transport.md index 70db61f..460d2d0 100644 --- a/site2/docs/security-tls-transport.md +++ b/site2/docs/security-tls-transport.md @@ -96,7 +96,7 @@ Generate the certificate request... ```bash $ openssl req -config openssl.cnf \ - -key broker.key.pem -new -sha256 -out broker.cert.pem + -key broker.key.pem -new -sha256 -out broker.csr.pem ``` ... and sign it with the certificate authority. diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md index a9af2b7..121ab17 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md @@ -31,7 +31,7 @@ Generate the certificate request. When asked for a **common name**, enter the ** ```bash $ openssl req -config openssl.cnf \ - -key admin.key.pem -new -sha256 -out admin.cert.pem + -key admin.key.pem -new -sha256 -out admin.csr.pem ``` Sign with request with the certificate authority. Note that that client certs uses the **usr_cert** extension, which allows the cert to be used for client authentication. diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md index 9924ed5..6981b8f 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md @@ -97,7 +97,7 @@ Generate the certificate request... ```bash $ openssl req -config openssl.cnf \ - -key broker.key.pem -new -sha256 -out broker.cert.pem + -key broker.key.pem -new -sha256 -out broker.csr.pem ``` ... and sign it with the certificate authority.
[incubator-pulsar] branch master updated: TopicPatterns are now supported by python functions (#2506)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 669196c TopicPatterns are now supported by python functions (#2506) 669196c is described below commit 669196c27d3c2028a112d939774744dae8ab1b88 Author: Sanjeev Kulkarni AuthorDate: Thu Sep 6 01:01:03 2018 -0700 TopicPatterns are now supported by python functions (#2506) --- .../instance/src/main/python/python_instance.py| 26 ++ .../src/main/python/python_instance_main.py| 3 --- .../functions/utils/validation/ValidatorImpls.java | 9 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py index b43b882..ddd546e 100644 --- a/pulsar-functions/instance/src/main/python/python_instance.py +++ b/pulsar-functions/instance/src/main/python/python_instance.py @@ -172,7 +172,7 @@ class PythonInstance(object): self.consumers[topic] = self.pulsar_client.subscribe( str(topic), subscription_name, consumer_type=mode, -message_listener=partial(self.message_listener, topic, self.input_serdes[topic]), +message_listener=partial(self.message_listener, self.input_serdes[topic]), unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None ) @@ -183,12 +183,20 @@ class PythonInstance(object): serde_kclass = util.import_class(os.path.dirname(self.user_code), consumer_conf.serdeClassName) self.input_serdes[topic] = serde_kclass() Log.info("Setting up consumer for topic %s with subname %s" % (topic, subscription_name)) - self.consumers[topic] = self.pulsar_client.subscribe( -str(topic), subscription_name, -consumer_type=mode, -message_listener=partial(self.message_listener, topic, self.input_serdes[topic]), -unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None - ) + if consumer_conf.isRegexPattern: +self.consumers[topic] = self.pulsar_client.subscribe_pattern( + str(topic), subscription_name, + consumer_type=mode, + message_listener=partial(self.message_listener, self.input_serdes[topic]), + unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None +) + else: +self.consumers[topic] = self.pulsar_client.subscribe( + str(topic), subscription_name, + consumer_type=mode, + message_listener=partial(self.message_listener, self.input_serdes[topic]), + unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms else None +) function_kclass = util.import_class(os.path.dirname(self.user_code), self.instance_config.function_details.className) if function_kclass is None: @@ -301,8 +309,8 @@ class PythonInstance(object): batching_max_publish_delay_ms=1, max_pending_messages=10) - def message_listener(self, topic, serde, consumer, message): -item = InternalMessage(message, topic, serde, consumer) + def message_listener(self, serde, consumer, message): +item = InternalMessage(message, consumer.topic(), serde, consumer) self.queue.put(item, True) if self.atmost_once and self.auto_ack: consumer.acknowledge(message) diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py b/pulsar-functions/instance/src/main/python/python_instance_main.py index 514d7fe..d9f1132 100644 --- a/pulsar-functions/instance/src/main/python/python_instance_main.py +++ b/pulsar-functions/instance/src/main/python/python_instance_main.py @@ -82,9 +82,6 @@ def main(): Log.info("Starting Python instance with %s" % str(args)) - if function_details.source.topicsPattern: -raise ValueError('topicsPattern is not supported by python client') - authentication = None use_tls = False tls_allow_insecure_connection = False diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java index cbeb970..f264d2a 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java @@ -506,15 +506,6 @@ public class ValidatorImpls { if (functionConfig.getWindowConfig() != null) { throw new IllegalArgumentException("There is currently no support windowing in python"); } - -if (functionConfig.getTopicsPattern() != null &&
[incubator-pulsar] branch master updated: better way to estimate object size for sql (#2528)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 90a7448 better way to estimate object size for sql (#2528) 90a7448 is described below commit 90a7448041aca5d5189b6a0b641453a50369faad Author: Boyang Jerry Peng AuthorDate: Thu Sep 6 01:00:12 2018 -0700 better way to estimate object size for sql (#2528) Since we mocked ClassLayout.parseClass() because we need to exclude org.openjdk.jol for license issues. Lets come up with another way to estimate class size --- pulsar-sql/presto-distribution/LICENSE | 3 +++ pulsar-sql/presto-distribution/pom.xml | 14 .../java/org/openjdk/jol/info/ClassLayout.java | 26 ++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 51d4923..d7f2fc8 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -359,6 +359,7 @@ The Apache Software License, Version 2.0 - jsr305-3.0.2.jar * Objenesis - objenesis-2.1.jar +- objenesis-2.6.jar * Okio - okio-1.13.0.jar * Presto @@ -379,6 +380,8 @@ The Apache Software License, Version 2.0 - snappy-java-1.1.1.3.jar * Bean Validation API - validation-api-1.1.0.Final.jar + * Objectsize +- objectsize-0.0.12.jar Protocol Buffers License * Protocol Buffers diff --git a/pulsar-sql/presto-distribution/pom.xml b/pulsar-sql/presto-distribution/pom.xml index ab55b3d..a355777 100644 --- a/pulsar-sql/presto-distribution/pom.xml +++ b/pulsar-sql/presto-distribution/pom.xml @@ -30,6 +30,8 @@ 0.206 0.170 +2.6 +0.0.12 com.facebook.presto.server.PrestoServer ${project.artifactId} @@ -80,6 +82,18 @@ provided + +org.objenesis +objenesis +${objenesis.version} + + + +com.twitter.common +objectsize +${objectsize.version} + + diff --git a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java b/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java index 16feb29..2a8bfa3 100644 --- a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java +++ b/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java @@ -18,18 +18,36 @@ */ package org.openjdk.jol.info; +import com.twitter.common.objectsize.ObjectSizeCalculator; +import io.airlift.log.Logger; +import org.objenesis.ObjenesisStd; + /** * Mock class avoid a dependency on OpenJDK JOL, * which is incompatible with the Apache License. */ public class ClassLayout { -public static ClassLayout parseClass(Class ignored) { -return new ClassLayout(); +private static final Logger log = Logger.get(ClassLayout.class); + +private int size; +private static final int DEFAULT_SIZE = 64; + +private ClassLayout(int size) { +this.size = size; +} + +public static ClassLayout parseClass(Class clazz) { +long size = DEFAULT_SIZE; +try { +size = ObjectSizeCalculator.getObjectSize(new ObjenesisStd().newInstance(clazz)); +} catch (Throwable th) { +log.info("Error estimating size of class %s",clazz, th); +} +return new ClassLayout(Math.toIntExact(size)); } -// TODO find a better estimate of class size public int instanceSize() { -return 64; // random, means nothing +return size; } } \ No newline at end of file
[incubator-pulsar] branch master updated: PIP-23: Pulsar Java Client Interceptors. (#2471)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 7a92111 PIP-23: Pulsar Java Client Interceptors. (#2471) 7a92111 is described below commit 7a92111fe7824d422d1739abd06d85955b652212 Author: penghui AuthorDate: Thu Sep 6 05:25:30 2018 +0800 PIP-23: Pulsar Java Client Interceptors. (#2471) ### Motivation Support user to add interceptors to producer and consumer. ### Modifications Add Consumer interceptors. ```java Message beforeConsume(Message message); void onAcknowledge(MessageId messageId, Throwable cause); void onAcknowledgeCumulative(MessageId messageId, Throwable cause); ``` Add Producer interceptors. ```java Message beforeSend(Message message); void onSendAcknowledgement(Message message, MessageId msgId, Throwable cause); ``` ### Result Users can using interceptors in multiple scenarios, such as for applications to add custom logging or processing. Master Issue: #2476 --- .../apache/pulsar/client/impl/RawReaderImpl.java | 2 +- .../pulsar/broker/service/PersistentTopicTest.java | 7 +- .../pulsar/broker/service/ReplicatorTest.java | 3 +- .../broker/service/v1/V1_ReplicatorTest.java | 3 +- .../apache/pulsar/client/api/InterceptorsTest.java | 359 + .../org/apache/pulsar/client/api/Consumer.java | 6 + .../apache/pulsar/client/api/ConsumerBuilder.java | 8 + .../pulsar/client/api/ConsumerInterceptor.java | 99 ++ .../apache/pulsar/client/api/ProducerBuilder.java | 8 + .../pulsar/client/api/ProducerInterceptor.java | 92 ++ .../apache/pulsar/client/impl/ConsumerBase.java| 25 +- .../pulsar/client/impl/ConsumerBuilderImpl.java| 20 +- .../apache/pulsar/client/impl/ConsumerImpl.java| 40 ++- .../pulsar/client/impl/ConsumerInterceptors.java | 132 .../org/apache/pulsar/client/impl/MessageImpl.java | 4 +- .../client/impl/MultiTopicsConsumerImpl.java | 12 +- .../client/impl/PartitionedProducerImpl.java | 6 +- .../impl/PatternMultiTopicsConsumerImpl.java | 4 +- .../apache/pulsar/client/impl/ProducerBase.java| 18 +- .../pulsar/client/impl/ProducerBuilderImpl.java| 18 +- .../apache/pulsar/client/impl/ProducerImpl.java| 72 +++-- .../pulsar/client/impl/ProducerInterceptors.java | 110 +++ .../pulsar/client/impl/PulsarClientImpl.java | 45 +-- .../org/apache/pulsar/client/impl/ReaderImpl.java | 2 +- .../pulsar/client/impl/TopicMessageImpl.java | 4 + .../pulsar/functions/instance/ContextImplTest.java | 2 +- .../MultiConsumersOneOutputTopicProducersTest.java | 27 +- 27 files changed, 1033 insertions(+), 95 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index ae1a4db..99aec93 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -111,7 +111,7 @@ public class RawReaderImpl implements RawReader { consumerFuture, SubscriptionMode.Durable, MessageId.earliest, -Schema.BYTES); +Schema.BYTES, null); incomingRawMessages = new GrowableArrayBlockingQueue<>(); pendingRawReceives = new ConcurrentLinkedQueue<>(); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java index f8ac40a..8c175d7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java @@ -24,6 +24,7 @@ import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyObject; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Matchers.matches; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.doAnswer; @@ -36,7 +37,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; -import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; @@ -80,7 +80,6 @@ import org.apache.pulsar.broker.admin.AdminResou
[incubator-pulsar] branch master updated: [schema] implement generic schema/record for Schema.JSON (#2497)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8453d73 [schema] implement generic schema/record for Schema.JSON (#2497) 8453d73 is described below commit 8453d73e94f0a22de53a4a003c2084e1573ac855 Author: Sijie Guo AuthorDate: Tue Sep 4 15:56:07 2018 -0700 [schema] implement generic schema/record for Schema.JSON (#2497) ### Motivation The `GenericSchema` and `GenericRecord` are used for schema auto detection. Currently it only supports Avro. This PR is to support json. ### Changes Introduce generic schema and generic record for json schema --- .../pulsar/client/api/schema/GenericRecord.java| 4 +- .../schema/{ => generic}/GenericAvroRecord.java| 7 +- .../schema/{ => generic}/GenericAvroSchema.java| 33 ++-- .../impl/schema/generic/GenericJsonRecord.java | 74 .../impl/schema/generic/GenericJsonSchema.java | 63 ++ .../client/impl/schema/generic/GenericSchema.java | 77 + .../impl/schema/generic/GenericSchemaTest.java | 99 ++ .../pulsar/client/schema/AvroSchemaTest.java | 93 +--- .../pulsar/client/schema/JSONSchemaTest.java | 38 ++--- .../pulsar/client/schema/SchemaTestUtils.java | 61 + .../pulsar/client/impl/PulsarClientImpl.java | 10 +-- 11 files changed, 398 insertions(+), 161 deletions(-) diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java index 0a4fce4..46a49a1 100644 --- a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java @@ -38,7 +38,9 @@ public interface GenericRecord { * @param field the field to retrieve the value * @return the value object */ -Object getField(Field field); +default Object getField(Field field) { +return getField(field.getName()); +} /** * Retrieve the value of the provided fieldName. diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java similarity index 94% rename from pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java rename to pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java index fb65c7a..c9dbeb7 100644 --- a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.client.impl.schema; +package org.apache.pulsar.client.impl.schema.generic; import java.util.List; import java.util.stream.Collectors; @@ -49,11 +49,6 @@ class GenericAvroRecord implements GenericRecord { } @Override -public Object getField(Field field) { -return getField(field.getName()); -} - -@Override public Object getField(String fieldName) { Object value = record.get(fieldName); if (value instanceof Utf8) { diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java similarity index 74% rename from pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java rename to pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java index 4ccfe55..5fe4459 100644 --- a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java @@ -16,58 +16,40 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.pulsar.client.impl.schema; +package org.apache.pulsar.client.impl.schema.generic; import static com.google.common.base.Preconditions.checkArgument; -import static java.nio.charset.StandardCharsets.UTF_8; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericDatumWriter;
[incubator-pulsar] branch branch-2.1 updated: Handle /managed-ledgers znode existance on cluster init (#2379) (#2510)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new bf4a50b Handle /managed-ledgers znode existance on cluster init (#2379) (#2510) bf4a50b is described below commit bf4a50b45dcaec7bd48b38a739bc4e7d43d36af1 Author: Ivan Kelly AuthorDate: Tue Sep 4 23:41:47 2018 +0200 Handle /managed-ledgers znode existance on cluster init (#2379) (#2510) ManagedLedger clients may create the /managed-ledger znode on boot, so if a broker starts before the metadata is initialized, it could potentially block initialization. This patch changes this by making the existance of /managed-ledger a non-error condition. --- .../main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java| 7 ++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java index e35daef..9dd065a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java @@ -147,7 +147,12 @@ public class PulsarClusterMetadataSetup { } } -localZk.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); +try { +localZk.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); +} catch (NodeExistsException e) { +// Ignore +} + localZk.create("/namespace", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); try {
[incubator-pulsar] branch master updated: Issue 2313: create a JDBC sink connector (#2440)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 79ac46a Issue 2313: create a JDBC sink connector (#2440) 79ac46a is described below commit 79ac46a6ae50e48bed4ccb680d1f7945611f0565 Author: Jia Zhai AuthorDate: Wed Sep 5 04:20:42 2018 +0800 Issue 2313: create a JDBC sink connector (#2440) ### Motivation This change is trying to add a basic JDBC sink connector. ### Modifications Add the jdbc module to the pulsar-io sub-module. Add unit test and integration test for it. ### Result ut and integration test pass. Master Issue: #2442 --- distribution/io/src/assemble/io.xml| 5 + pom.xml| 9 +- pulsar-io/jdbc/lombok.config | 23 +++ pulsar-io/jdbc/pom.xml | 96 ++ .../apache/pulsar/io/jdbc/JdbcAbstractSink.java| 197 + .../apache/pulsar/io/jdbc/JdbcAvroSchemaSink.java | 92 ++ .../org/apache/pulsar/io/jdbc/JdbcSinkConfig.java | 63 +++ .../java/org/apache/pulsar/io/jdbc/JdbcUtils.java | 178 +++ .../resources/META-INF/services/pulsar-io.yaml | 22 +++ .../org/apache/pulsar/io/jdbc/JdbcSinkTest.java| 135 ++ .../org/apache/pulsar/io/jdbc/JdbcUtilsTest.java | 95 ++ .../org/apache/pulsar/io/jdbc/SqliteUtils.java | 111 pulsar-io/pom.xml | 1 + tests/integration/pom.xml | 21 +++ .../integration/functions/PulsarFunctionsTest.java | 54 +- .../functions/PulsarFunctionsTestBase.java | 2 +- .../tests/integration/io/JdbcSinkTester.java | 137 ++ .../tests/integration/suites/PulsarTestSuite.java | 11 ++ 18 files changed, 1245 insertions(+), 7 deletions(-) diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 8cf7fce..bb75e84 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -75,6 +75,11 @@ 644 + ${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar + connectors + 644 + + ${basedir}/../../pulsar-io/data-genenator/target/pulsar-io-data-generator-${project.version}.nar connectors 644 diff --git a/pom.xml b/pom.xml index 1ed2802..681f18e 100644 --- a/pom.xml +++ b/pom.xml @@ -167,6 +167,8 @@ flexible messaging model and an intuitive client API. 1.11.297 1.8.2 2.1.1 +3.8.11.2 +8.0.11 0.206 1.6.0 2.11 @@ -822,6 +824,11 @@ flexible messaging model and an intuitive client API. ${testcontainers.version} +org.testcontainers +mysql +${testcontainers.version} + + org.arquillian.cube arquillian-cube-docker ${arquillian-cube.version} @@ -1086,7 +1093,7 @@ flexible messaging model and an intuitive client API. src/main/java/org/apache/pulsar/common/api/proto/PulsarApi.java src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java bin/proto/MLDataFormats_pb2.py - + src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java diff --git a/pulsar-io/jdbc/lombok.config b/pulsar-io/jdbc/lombok.config new file mode 100644 index 000..9a9adee --- /dev/null +++ b/pulsar-io/jdbc/lombok.config @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +## This file is to fix the conflict with jackson error like this: +##com.fasterxml.jackson.databind.exc.MismatchedInputException: Cannot construct instance of ... +lombok.anyConstructor.addConstructorProperties=true +config.stopBubbling = true diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml new file mode 100644 in
[incubator-pulsar] branch branch-2.1 updated: Don't initialize /managed-ledgers on client creation (#2379) (#2509)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 8a4f5cc Don't initialize /managed-ledgers on client creation (#2379) (#2509) 8a4f5cc is described below commit 8a4f5ccad08522a5774dbc0500b3f48ea73a6fb6 Author: Ivan Kelly AuthorDate: Tue Sep 4 22:18:46 2018 +0200 Don't initialize /managed-ledgers on client creation (#2379) (#2509) Normally the /managed-ledgers znode is created by the initialize-cluster-metadata command when a cluster is being turned up. However, the ManagedLedger client also creates it on boot. This has caused issues in the past, where if a broker is started before initialize-cluster-metadata is run, then initialize-cluster-metadata fails because it sees the /managed-ledger znode. This patch removes the automatic creation of this znode from the client boot process. --- .../mledger/impl/MetaStoreImplZookeeper.java | 64 +++--- .../mledger/impl/MetaStoreImplZookeeperTest.java | 52 ++ .../bookkeeper/test/BookKeeperClusterTestCase.java | 4 ++ .../bookkeeper/test/MockedBookKeeperTestCase.java | 4 ++ 4 files changed, 117 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java index 354c04f..4d90b91 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java @@ -25,9 +25,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat.ParseException; +import java.io.File; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -35,7 +37,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; -import org.apache.bookkeeper.util.ZkUtils; import org.apache.zookeeper.AsyncCallback.StringCallback; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -95,10 +96,6 @@ public class MetaStoreImplZookeeper implements MetaStore { throws Exception { this.zk = zk; this.executor = executor; - -if (zk.exists(prefixName, false) == null) { -zk.create(prefixName, new byte[0], Acl, CreateMode.PERSISTENT); -} } // @@ -157,8 +154,8 @@ public class MetaStoreImplZookeeper implements MetaStore { } }; -ZkUtils.asyncCreateFullPathOptimistic(zk, prefix + ledgerName, new byte[0], Acl, -CreateMode.PERSISTENT, createcb, null); +asyncCreateFullPathOptimistic(zk, prefixName, ledgerName, new byte[0], Acl, + CreateMode.PERSISTENT, createcb); } else { // Tried to open a managed ledger but it doesn't exist and we shouldn't creating it at this // point @@ -365,5 +362,58 @@ public class MetaStoreImplZookeeper implements MetaStore { } +public static void asyncCreateFullPathOptimistic( +final ZooKeeper zk, final String basePath, final String nodePath, final byte[] data, +final List acl, final CreateMode createMode, final StringCallback callback) { +String fullPath = basePath + "/" + nodePath; + +zk.create(fullPath, data, acl, createMode, + (rc, path, ignoreCtx1, name) -> { + Runnable retry = () -> { + asyncCreateFullPathOptimistic(zk, basePath, nodePath, data, +acl, createMode, callback); + }; + + Consumer complete = (finalrc) -> { + callback.processResult(finalrc, path, null, name); + }; + + if (rc != Code.NONODE.intValue()) { + complete.accept(rc); + return; + } + +
[incubator-pulsar] branch master updated: Don't initialize /managed-ledgers on client creation (#2379) (#2509)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8763c74 Don't initialize /managed-ledgers on client creation (#2379) (#2509) 8763c74 is described below commit 8763c746b5b38901d7fbb77e5766f55c563ba25e Author: Ivan Kelly AuthorDate: Tue Sep 4 22:18:46 2018 +0200 Don't initialize /managed-ledgers on client creation (#2379) (#2509) Normally the /managed-ledgers znode is created by the initialize-cluster-metadata command when a cluster is being turned up. However, the ManagedLedger client also creates it on boot. This has caused issues in the past, where if a broker is started before initialize-cluster-metadata is run, then initialize-cluster-metadata fails because it sees the /managed-ledger znode. This patch removes the automatic creation of this znode from the client boot process. --- .../mledger/impl/MetaStoreImplZookeeper.java | 64 +++--- .../mledger/impl/MetaStoreImplZookeeperTest.java | 52 ++ .../bookkeeper/test/BookKeeperClusterTestCase.java | 4 ++ .../bookkeeper/test/MockedBookKeeperTestCase.java | 4 ++ 4 files changed, 117 insertions(+), 7 deletions(-) diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java index 354c04f..4d90b91 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java @@ -25,9 +25,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat.ParseException; +import java.io.File; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; +import java.util.function.Consumer; import org.apache.bookkeeper.common.util.OrderedExecutor; import org.apache.bookkeeper.mledger.ManagedLedgerException; @@ -35,7 +37,6 @@ import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException; import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo; import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo; -import org.apache.bookkeeper.util.ZkUtils; import org.apache.zookeeper.AsyncCallback.StringCallback; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -95,10 +96,6 @@ public class MetaStoreImplZookeeper implements MetaStore { throws Exception { this.zk = zk; this.executor = executor; - -if (zk.exists(prefixName, false) == null) { -zk.create(prefixName, new byte[0], Acl, CreateMode.PERSISTENT); -} } // @@ -157,8 +154,8 @@ public class MetaStoreImplZookeeper implements MetaStore { } }; -ZkUtils.asyncCreateFullPathOptimistic(zk, prefix + ledgerName, new byte[0], Acl, -CreateMode.PERSISTENT, createcb, null); +asyncCreateFullPathOptimistic(zk, prefixName, ledgerName, new byte[0], Acl, + CreateMode.PERSISTENT, createcb); } else { // Tried to open a managed ledger but it doesn't exist and we shouldn't creating it at this // point @@ -365,5 +362,58 @@ public class MetaStoreImplZookeeper implements MetaStore { } +public static void asyncCreateFullPathOptimistic( +final ZooKeeper zk, final String basePath, final String nodePath, final byte[] data, +final List acl, final CreateMode createMode, final StringCallback callback) { +String fullPath = basePath + "/" + nodePath; + +zk.create(fullPath, data, acl, createMode, + (rc, path, ignoreCtx1, name) -> { + Runnable retry = () -> { + asyncCreateFullPathOptimistic(zk, basePath, nodePath, data, +acl, createMode, callback); + }; + + Consumer complete = (finalrc) -> { + callback.processResult(finalrc, path, null, name); + }; + + if (rc != Code.NONODE.intValue()) { + complete.accept(rc); + return; + } + + //
[incubator-pulsar] branch branch-2.1 updated: Ensure standalone service comes back quickly after ungraceful restarts (#2487)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 113418d Ensure standalone service comes back quickly after ungraceful restarts (#2487) 113418d is described below commit 113418d3e7b4d02597bed7a939a46e5f66829621 Author: Matteo Merli AuthorDate: Tue Sep 4 12:34:48 2018 -0700 Ensure standalone service comes back quickly after ungraceful restarts (#2487) * Ensure standalone service comes back quickly after ungraceful restarts * Handle NoNode errors when deleting * Added NoopLoadManager for standalone mode --- conf/standalone.conf | 2 + .../pulsar/broker/loadbalance/NoopLoadManager.java | 158 + .../loadbalance/impl/ModularLoadManagerImpl.java | 3 +- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 15 +- 4 files changed, 176 insertions(+), 2 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index 74a5702..755b76e 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -311,6 +311,8 @@ autoSkipNonRecoverableData=false ### --- Load balancer --- ### +loadManagerClassName=org.apache.pulsar.broker.loadbalance.NoopLoadManager + # Enable load balancer loadBalancerEnabled=false diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java new file mode 100644 index 000..5773c61 --- /dev/null +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.loadbalance; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import org.apache.bookkeeper.util.ZkUtils; +import org.apache.pulsar.broker.PulsarServerException; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription; +import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit; +import org.apache.pulsar.common.naming.ServiceUnitId; +import org.apache.pulsar.common.stats.Metrics; +import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport; +import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; +import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData; +import org.apache.pulsar.zookeeper.ZooKeeperCache.Deserializer; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException.NoNodeException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.ZooKeeper; + +public class NoopLoadManager implements LoadManager { + +private String lookupServiceAddress; +private ResourceUnit localResourceUnit; +private ZooKeeper zkClient; + +LocalBrokerData localData; + +private static final Deserializer loadReportDeserializer = (key, content) -> ObjectMapperFactory +.getThreadLocal() +.readValue(content, LocalBrokerData.class); + +@Override +public void initialize(PulsarService pulsar) { +lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getWebServicePort(); +localResourceUnit = new SimpleResourceUnit(String.format("http://%s";, lookupServiceAddress), +new PulsarResourceDescription()); +zkClient = pulsar.getZkClient(); + +localData = new LocalBrokerData(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), +pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); +} + +@Override +public void start() throws PulsarServerException { +String brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; + +try { +// When running in standalone, this er
[incubator-pulsar] branch master updated: Add pulsar flink sink connector (#2434)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new ad03a07 Add pulsar flink sink connector (#2434) ad03a07 is described below commit ad03a07d5f5d93654f1d5afb8694989db5688095 Author: Ali Ahmed AuthorDate: Tue Sep 4 11:59:03 2018 -0700 Add pulsar flink sink connector (#2434) @XiaoZYang introduces a module for a pulsar sink connector for flink. This PR is moving the work from flink repo to pulsar repo, so the flink connector can be released faster along with Pulsar releases. Original Flink Github Issue: apache/flink#5845 Jira Issue: https://issues.apache.org/jira/browse/FLINK-9168 Original Author: @XiaoZYang (Zong Yang Xiao) --- pom.xml| 3 + pulsar-flink/pom.xml | 137 + .../connectors/pulsar/FlinkPulsarProducer.java | 314 + .../connectors/pulsar/PulsarJsonTableSink.java | 61 .../connectors/pulsar/PulsarProduceMode.java | 37 +++ .../connectors/pulsar/PulsarTableSink.java | 163 +++ .../pulsar/partitioner/PulsarKeyExtractor.java | 36 +++ .../pulsar/serde/JsonRowDeserializationSchema.java | 130 + .../pulsar/serde/JsonRowSerializationSchema.java | 92 ++ 9 files changed, 973 insertions(+) diff --git a/pom.xml b/pom.xml index 4bb3cf6..1ed2802 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ flexible messaging model and an intuitive client API. pulsar-proxy pulsar-discovery-service pulsar-storm +pulsar-flink pulsar-spark pulsar-zookeeper-utils pulsar-testclient @@ -167,6 +168,8 @@ flexible messaging model and an intuitive client API. 1.8.2 2.1.1 0.206 +1.6.0 +2.11 1.15.1 diff --git a/pulsar-flink/pom.xml b/pulsar-flink/pom.xml new file mode 100644 index 000..97ab4fb --- /dev/null +++ b/pulsar-flink/pom.xml @@ -0,0 +1,137 @@ + +http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"; + xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + 4.0.0 + + +org.apache.pulsar +pulsar +2.2.0-incubating-SNAPSHOT +.. + + + pulsar-flink + Pulsar Flink Connectors + + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + provided + + + + org.apache.flink + flink-table_${scala.binary.version} + ${flink.version} + provided + + true + + + + ${project.groupId} + pulsar-client + ${project.version} + + + + org.apache.flink + flink-runtime_${scala.binary.version} + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-tests_${scala.binary.version} + ${flink.version} + test + test-jar + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${flink.version} + test + + + + org.javassist + javassist + 3.20.0-GA + test + + + + + + + +src/main/resources +true + + + + +org.apache.maven.plugins +maven-shade-plugin + + +package + + shade + + + true + true + + + com.google.guava:guava + + + + + com.google + pulsar-flink-shade.com.google + + + + + + + + + diff --git a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java new file mode 100644 index 000..bddfee4 --- /dev/null +++ b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java @@ -0,0 +1,314 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless r
[incubator-pulsar] branch branch-2.1 updated: Allow to configure bookie settings when running in standalone mode (#2480)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new f5a01f7 Allow to configure bookie settings when running in standalone mode (#2480) f5a01f7 is described below commit f5a01f7c82569d3a4dcfc4bed2cc94c790909117 Author: Matteo Merli AuthorDate: Tue Sep 4 11:16:11 2018 -0700 Allow to configure bookie settings when running in standalone mode (#2480) ### Motivation Most bookie configs are set in stone when running Pulsar in standalone mode. We need to be able to tune these settings to have the Pulsar standalone to adapt for running with small memory settings. --- conf/standalone.conf | 79 ++ .../java/org/apache/pulsar/PulsarStandalone.java | 9 ++- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 20 ++ 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index a9fb288..74a5702 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -411,3 +411,82 @@ exposePublisherStats=true # Deprecated. Use configurationStoreServers globalZookeeperServers= + +### --- BookKeeper Configuration --- # + +ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage + +# Size of Write Cache. Memory is allocated from JVM direct memory. +# Write cache is used to buffer entries before flushing into the entry log +# For good performance, it should be big enough to hold a sub +dbStorage_writeCacheMaxSizeMb=256 + +# Size of Read cache. Memory is allocated from JVM direct memory. +# This read cache is pre-filled doing read-ahead whenever a cache miss happens +dbStorage_readAheadCacheMaxSizeMb=64 + +# How many entries to pre-fill in cache after a read cache miss +dbStorage_readAheadCacheBatchSize=1000 + +flushInterval=6 + +## RocksDB specific configurations +## DbLedgerStorage uses RocksDB to store the indexes from +## (ledgerId, entryId) -> (entryLog, offset) + +# Size of RocksDB block-cache. For best performance, this cache +# should be big enough to hold a significant portion of the index +# database which can reach ~2GB in some cases +# Default is 16 MBytes +dbStorage_rocksDB_blockCacheSize=16777216 + +# Other RocksDB specific tunables +dbStorage_rocksDB_writeBufferSizeMB=4 +dbStorage_rocksDB_sstSizeInMB=4 +dbStorage_rocksDB_blockSize=4096 +dbStorage_rocksDB_bloomFilterBitsPerKey=10 +dbStorage_rocksDB_numLevels=-1 +dbStorage_rocksDB_numFilesInLevel0=4 +dbStorage_rocksDB_maxSizeInLevel1MB=256 + +# Maximum latency to impose on a journal write to achieve grouping +journalMaxGroupWaitMSec=1 + +# Should the data be fsynced on journal before acknowledgment. +journalSyncData=false + + +# For each ledger dir, maximum disk space which can be used. +# Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will +# be written to that partition. If all ledger dir partions are full, then bookie +# will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will +# shutdown. +# Valid values should be in between 0 and 1 (exclusive). +diskUsageThreshold=0.99 + +# The disk free space low water mark threshold. +# Disk is considered full when usage threshold is exceeded. +# Disk returns back to non-full state when usage is below low water mark threshold. +# This prevents it from going back and forth between these states frequently +# when concurrent writes and compaction are happening. This also prevent bookie from +# switching frequently between read-only and read-writes states in the same cases. +diskUsageWarnThreshold=0.99 + +# Whether the bookie allowed to use a loopback interface as its primary +# interface(i.e. the interface it uses to establish its identity)? +# By default, loopback interfaces are not allowed as the primary +# interface. +# Using a loopback interface as the primary interface usually indicates +# a configuration error. For example, its fairly common in some VPS setups +# to not configure a hostname, or to have the hostname resolve to +# 127.0.0.1. If this is the case, then all bookies in the cluster will +# establish their identities as 127.0.0.1:3181, and only one will be able +# to join the cluster. For VPSs configured like this, you should explicitly +# set the listening interface. +allowLoopback=true + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +gcWaitTime=30 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index b6105c9..bb30bfa 100644 --- a/pulsar-br
[incubator-pulsar] branch master updated: Allow to configure bookie settings when running in standalone mode (#2480)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8f4f371 Allow to configure bookie settings when running in standalone mode (#2480) 8f4f371 is described below commit 8f4f371def2983f64e5dfa8f31e9fe04a2e21462 Author: Matteo Merli AuthorDate: Tue Sep 4 11:16:11 2018 -0700 Allow to configure bookie settings when running in standalone mode (#2480) ### Motivation Most bookie configs are set in stone when running Pulsar in standalone mode. We need to be able to tune these settings to have the Pulsar standalone to adapt for running with small memory settings. --- conf/standalone.conf | 79 ++ .../java/org/apache/pulsar/PulsarStandalone.java | 9 ++- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 20 ++ 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index 09d369c..b09fcd6 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -420,3 +420,82 @@ exposePublisherStats=true # Deprecated. Use configurationStoreServers globalZookeeperServers= + +### --- BookKeeper Configuration --- # + +ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage + +# Size of Write Cache. Memory is allocated from JVM direct memory. +# Write cache is used to buffer entries before flushing into the entry log +# For good performance, it should be big enough to hold a sub +dbStorage_writeCacheMaxSizeMb=256 + +# Size of Read cache. Memory is allocated from JVM direct memory. +# This read cache is pre-filled doing read-ahead whenever a cache miss happens +dbStorage_readAheadCacheMaxSizeMb=64 + +# How many entries to pre-fill in cache after a read cache miss +dbStorage_readAheadCacheBatchSize=1000 + +flushInterval=6 + +## RocksDB specific configurations +## DbLedgerStorage uses RocksDB to store the indexes from +## (ledgerId, entryId) -> (entryLog, offset) + +# Size of RocksDB block-cache. For best performance, this cache +# should be big enough to hold a significant portion of the index +# database which can reach ~2GB in some cases +# Default is 16 MBytes +dbStorage_rocksDB_blockCacheSize=16777216 + +# Other RocksDB specific tunables +dbStorage_rocksDB_writeBufferSizeMB=4 +dbStorage_rocksDB_sstSizeInMB=4 +dbStorage_rocksDB_blockSize=4096 +dbStorage_rocksDB_bloomFilterBitsPerKey=10 +dbStorage_rocksDB_numLevels=-1 +dbStorage_rocksDB_numFilesInLevel0=4 +dbStorage_rocksDB_maxSizeInLevel1MB=256 + +# Maximum latency to impose on a journal write to achieve grouping +journalMaxGroupWaitMSec=1 + +# Should the data be fsynced on journal before acknowledgment. +journalSyncData=false + + +# For each ledger dir, maximum disk space which can be used. +# Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will +# be written to that partition. If all ledger dir partions are full, then bookie +# will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will +# shutdown. +# Valid values should be in between 0 and 1 (exclusive). +diskUsageThreshold=0.99 + +# The disk free space low water mark threshold. +# Disk is considered full when usage threshold is exceeded. +# Disk returns back to non-full state when usage is below low water mark threshold. +# This prevents it from going back and forth between these states frequently +# when concurrent writes and compaction are happening. This also prevent bookie from +# switching frequently between read-only and read-writes states in the same cases. +diskUsageWarnThreshold=0.99 + +# Whether the bookie allowed to use a loopback interface as its primary +# interface(i.e. the interface it uses to establish its identity)? +# By default, loopback interfaces are not allowed as the primary +# interface. +# Using a loopback interface as the primary interface usually indicates +# a configuration error. For example, its fairly common in some VPS setups +# to not configure a hostname, or to have the hostname resolve to +# 127.0.0.1. If this is the case, then all bookies in the cluster will +# establish their identities as 127.0.0.1:3181, and only one will be able +# to join the cluster. For VPSs configured like this, you should explicitly +# set the listening interface. +allowLoopback=true + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +gcWaitTime=30 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index b6105c9..bb30bfa 100644 --- a/pulsar-broker/src/
[incubator-pulsar] branch branch-2.1 updated: Fix log4j config to not redirect BK logs into separate file (#2482)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new a4b20bb Fix log4j config to not redirect BK logs into separate file (#2482) a4b20bb is described below commit a4b20bb28fb77a5f84be61481e126a698788ae79 Author: Matteo Merli AuthorDate: Tue Sep 4 11:15:24 2018 -0700 Fix log4j config to not redirect BK logs into separate file (#2482) ### Motivation Currently all `org.apache.bookkeeper` messages printed by broker are being routed to a different log file `broker.log.bk`. Removing for now the special config for BK logs. --- conf/log4j2.yaml | 50 -- 1 file changed, 50 deletions(-) diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index ab2b9ab..45ed5d9 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -35,10 +35,6 @@ Configuration: value: "info" - name: "pulsar.routing.appender.default" value: "Console" - - name: "bk.log.level" -value: "info" - - name: "bk.log.appender" -value: "BkRollingFile" # Example: logger-filter script Scripts: @@ -85,34 +81,6 @@ Configuration: IfLastModified: age: 30d -# Rolling file appender configuration for bk -RollingRandomAccessFile: - name: BkRollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-}-%i.log.gz" - immediateFlush: true - PatternLayout: -Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: -TimeBasedTriggeringPolicy: - interval: 1 - modulate: true -SizeBasedTriggeringPolicy: - size: 1 GB -# Trigger every day at midnight that also scan -# roll-over strategy that deletes older file -CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: -basePath: ${sys:pulsar.log.dir} -maxDepth: 2 -IfFileName: - glob: "*/${sys:pulsar.log.file}.bk*log.gz" -IfLastModified: - age: 30d - # Routing Routing: name: RoutingAppender @@ -173,24 +141,6 @@ Configuration: AppenderRef: - ref: Console - - name: org.apache.bookkeeper -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - - name: org.apache.distributedlog -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - - name: org.apache.zookeeper -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - name: verbose level: info additivity: false
[incubator-pulsar] branch master updated: Fix log4j config to not redirect BK logs into separate file (#2482)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 37f266b Fix log4j config to not redirect BK logs into separate file (#2482) 37f266b is described below commit 37f266bb0357fa825f1b3c2e99582526c43bba84 Author: Matteo Merli AuthorDate: Tue Sep 4 11:15:24 2018 -0700 Fix log4j config to not redirect BK logs into separate file (#2482) ### Motivation Currently all `org.apache.bookkeeper` messages printed by broker are being routed to a different log file `broker.log.bk`. Removing for now the special config for BK logs. --- conf/log4j2.yaml | 50 -- 1 file changed, 50 deletions(-) diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml index ab2b9ab..45ed5d9 100644 --- a/conf/log4j2.yaml +++ b/conf/log4j2.yaml @@ -35,10 +35,6 @@ Configuration: value: "info" - name: "pulsar.routing.appender.default" value: "Console" - - name: "bk.log.level" -value: "info" - - name: "bk.log.appender" -value: "BkRollingFile" # Example: logger-filter script Scripts: @@ -85,34 +81,6 @@ Configuration: IfLastModified: age: 30d -# Rolling file appender configuration for bk -RollingRandomAccessFile: - name: BkRollingFile - fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk" - filePattern: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-}-%i.log.gz" - immediateFlush: true - PatternLayout: -Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" - Policies: -TimeBasedTriggeringPolicy: - interval: 1 - modulate: true -SizeBasedTriggeringPolicy: - size: 1 GB -# Trigger every day at midnight that also scan -# roll-over strategy that deletes older file -CronTriggeringPolicy: - schedule: "0 0 0 * * ?" - # Delete file older than 30days - DefaultRolloverStrategy: - Delete: -basePath: ${sys:pulsar.log.dir} -maxDepth: 2 -IfFileName: - glob: "*/${sys:pulsar.log.file}.bk*log.gz" -IfLastModified: - age: 30d - # Routing Routing: name: RoutingAppender @@ -173,24 +141,6 @@ Configuration: AppenderRef: - ref: Console - - name: org.apache.bookkeeper -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - - name: org.apache.distributedlog -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - - name: org.apache.zookeeper -level: "${sys:bk.log.level}" -additivity: false -AppenderRef: - - ref: "${sys:bk.log.appender}" - - name: verbose level: info additivity: false
[incubator-pulsar] branch branch-2.1 updated: [functions] Don't set the state storage url if state is not enabled. (#2488)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 04c5c48 [functions] Don't set the state storage url if state is not enabled. (#2488) 04c5c48 is described below commit 04c5c48f2fcba7724892f66acf7784518877ba3c Author: Sijie Guo AuthorDate: Tue Sep 4 00:17:29 2018 -0700 [functions] Don't set the state storage url if state is not enabled. (#2488) --- pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index cf07d8c..b6105c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -273,7 +273,10 @@ public class PulsarStandalone implements AutoCloseable { // worker talks to local broker workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort()); workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:"; + config.getWebServicePort()); -workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" + this.getStreamStoragePort()); +if (!this.isNoStreamStorage()) { +// only set the state storage service url when state is enabled. +workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" + this.getStreamStoragePort()); +} String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress( config.getAdvertisedAddress()); workerConfig.setWorkerHostname(hostname);
[incubator-pulsar] branch master updated: [functions] Don't set the state storage url if state is not enabled. (#2488)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 4124de1 [functions] Don't set the state storage url if state is not enabled. (#2488) 4124de1 is described below commit 4124de11c407e7a5efa8aa2f4b669e22543ee427 Author: Sijie Guo AuthorDate: Tue Sep 4 00:17:29 2018 -0700 [functions] Don't set the state storage url if state is not enabled. (#2488) --- pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index cf07d8c..b6105c9 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -273,7 +273,10 @@ public class PulsarStandalone implements AutoCloseable { // worker talks to local broker workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" + config.getBrokerServicePort()); workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:"; + config.getWebServicePort()); -workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" + this.getStreamStoragePort()); +if (!this.isNoStreamStorage()) { +// only set the state storage service url when state is enabled. +workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" + this.getStreamStoragePort()); +} String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress( config.getAdvertisedAddress()); workerConfig.setWorkerHostname(hostname);
[incubator-pulsar] branch branch-2.1 updated: Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 588f6f2 Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484) 588f6f2 is described below commit 588f6f2c5d32288e7404eb5049ac03dc0697 Author: Victor AuthorDate: Sat Sep 1 00:09:30 2018 -0700 Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484) ### Motivation Command `./bin/pulsar-admin clusters list` failed with `Error: Could not find or load main class "` in my k8s deployment by using helm charts. ### Modifications With helm deployment the env var PULSAR_MEM is quoted with `"`, taking it for `pulsar-admin` will make the exec command to be: ```exec /docker-java-home/bin/java -cp '/pulsar/conf:::/pulsar/lib/*:' -Dlog4j.configurationFile=log4j2.yaml -Djava.net.preferIPv4Stack=true '"' -Xms1g -Xmx1g '-XX:MaxDirectMemorySize=1g"' -client -Dio.netty.leakDetectionLevel=disabled ... org.apache.pulsar.admin.cli.PulsarAdminTool /pulsar/conf/client.conf clusters list``` The fix is to take out the use of env var PULSAR_MEM in `conf/pulsar_tools_env.sh`. ### Result `pulsar-admin` works as expected. Fixes #2481 --- conf/pulsar_tools_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh index 7ad022f..1a701fe 100755 --- a/conf/pulsar_tools_env.sh +++ b/conf/pulsar_tools_env.sh @@ -42,7 +42,7 @@ # PULSAR_GLOBAL_ZK_CONF= # Extra options to be passed to the jvm -PULSAR_MEM=${PULSAR_MEM:-"-Xmx256m -XX:MaxDirectMemorySize=256m"} +PULSAR_MEM="-Xmx128m -XX:MaxDirectMemorySize=128m" # Garbage collection options PULSAR_GC=" -client "
[incubator-pulsar] branch master updated: Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new cfdcdb5 Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484) cfdcdb5 is described below commit cfdcdb5c096918187ab52725ba091c401f28b62c Author: Victor AuthorDate: Sat Sep 1 00:09:30 2018 -0700 Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484) ### Motivation Command `./bin/pulsar-admin clusters list` failed with `Error: Could not find or load main class "` in my k8s deployment by using helm charts. ### Modifications With helm deployment the env var PULSAR_MEM is quoted with `"`, taking it for `pulsar-admin` will make the exec command to be: ```exec /docker-java-home/bin/java -cp '/pulsar/conf:::/pulsar/lib/*:' -Dlog4j.configurationFile=log4j2.yaml -Djava.net.preferIPv4Stack=true '"' -Xms1g -Xmx1g '-XX:MaxDirectMemorySize=1g"' -client -Dio.netty.leakDetectionLevel=disabled ... org.apache.pulsar.admin.cli.PulsarAdminTool /pulsar/conf/client.conf clusters list``` The fix is to take out the use of env var PULSAR_MEM in `conf/pulsar_tools_env.sh`. ### Result `pulsar-admin` works as expected. Fixes #2481 --- conf/pulsar_tools_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh index 38e2859..1a701fe 100755 --- a/conf/pulsar_tools_env.sh +++ b/conf/pulsar_tools_env.sh @@ -42,7 +42,7 @@ # PULSAR_GLOBAL_ZK_CONF= # Extra options to be passed to the jvm -PULSAR_MEM=${PULSAR_TOOLS_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"} +PULSAR_MEM="-Xmx128m -XX:MaxDirectMemorySize=128m" # Garbage collection options PULSAR_GC=" -client "
[incubator-pulsar] branch master updated: [Schema] Introduce Schema.AUTO to detect schema automatically for consumers and readers (#2494)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 42b598a [Schema] Introduce Schema.AUTO to detect schema automatically for consumers and readers (#2494) 42b598a is described below commit 42b598a17b64d76428d48438cfb169cf3a1fcd5d Author: Sijie Guo AuthorDate: Sat Sep 1 00:07:12 2018 -0700 [Schema] Introduce Schema.AUTO to detect schema automatically for consumers and readers (#2494) ### Motivation Sometimes consumers and readers don't have the pojo for the message schemas. so we can use `Schema.Auto` to detect the schema and reflect the messages into generic records. ### Changes - Introduce AutoSchema and `SchemaType.AUTO` - fetch schema information first when `AutoSchema` is used --- .../api/SimpleTypedProducerConsumerTest.java | 95 ++ .../java/org/apache/pulsar/client/api/Schema.java | 6 ++ .../pulsar/client/impl/schema/AutoSchema.java | 63 ++ .../pulsar/client/schema/AvroSchemaTest.java | 21 - .../pulsar/client/impl/PulsarClientImpl.java | 52 .../apache/pulsar/common/schema/SchemaType.java| 7 +- 6 files changed, 242 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java index b880953..24573d5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java @@ -26,6 +26,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.pulsar.broker.service.schema.SchemaRegistry; +import org.apache.pulsar.client.api.schema.GenericRecord; import org.apache.pulsar.client.impl.schema.AvroSchema; import org.apache.pulsar.client.impl.schema.JSONSchema; import org.apache.pulsar.client.impl.schema.ProtobufSchema; @@ -440,4 +441,98 @@ public class SimpleTypedProducerConsumerTest extends ProducerConsumerBase { } } +@Test +public void testAvroProducerAndAutoSchemaConsumer() throws Exception { + log.info("-- Starting {} test --", methodName); + + AvroSchema avroSchema = + AvroSchema.of(AvroEncodedPojo.class); + + Producer producer = pulsarClient + .newProducer(avroSchema) + .topic("persistent://my-property/use/my-ns/my-topic1") + .create(); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(new AvroEncodedPojo(message)); + } + + Consumer consumer = pulsarClient + .newConsumer(Schema.AUTO()) + .topic("persistent://my-property/use/my-ns/my-topic1") + .subscriptionName("my-subscriber-name") + .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) + .subscribe(); + + Message msg = null; + Set messageSet = Sets.newHashSet(); + for (int i = 0; i < 10; i++) { + msg = consumer.receive(5, TimeUnit.SECONDS); + GenericRecord receivedMessage = msg.getValue(); + log.debug("Received message: [{}]", receivedMessage); + String expectedMessage = "my-message-" + i; + String actualMessage = (String) receivedMessage.getField("message"); + testMessageOrderAndDuplicates(messageSet, actualMessage, expectedMessage); + } + // Acknowledge the consumption of all messages at once + consumer.acknowledgeCumulative(msg); + consumer.close(); + + SchemaRegistry.SchemaAndMetadata storedSchema = pulsar.getSchemaRegistryService() + .getSchema("my-property/my-ns/my-topic1") + .get(); + + Assert.assertEquals(storedSchema.schema.getData(), avroSchema.getSchemaInfo().getSchema()); + + log.info("-- Exiting {} test --", methodName); + + } + + @Test +public void testAvroProducerAndAutoSchemaReader() throws Exception { + log.info("-- Starting {} test --", methodName); + + AvroSchema avroSchema = + AvroSchema.of(AvroEncodedPojo.class); + + Producer producer = pulsarClient + .newProducer(avroSchema) + .topic("persistent://my-property/use/my-ns/my-topic1") + .create(); + + for (int i = 0; i < 10; i++) { + String message = "my-message-" + i; + producer.send(new AvroEncodedPojo(message)); + } + + Reader reader = pulsarClient +
[incubator-pulsar] branch master updated: [tiered storage] Use NAR plugin to package offloaders (#2393)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 9c1503b [tiered storage] Use NAR plugin to package offloaders (#2393) 9c1503b is described below commit 9c1503b67eb122d2801fd9a9b5a94e5f7780efc0 Author: Sijie Guo AuthorDate: Wed Aug 29 00:52:29 2018 -0700 [tiered storage] Use NAR plugin to package offloaders (#2393) ### Motivation Offloader typically involves a new storage system, which usually involves dependencies that might be conflicting with the dependencies of Pulsar. We want to package offloader implementations as what we did for connectors, so people can decide which offloader to use and only include it. People would also become easier to write its offloader if needed. ### Changes - Update `tiered-storage/jcloud` to package it using nifi-nar plugin. - Add bunch of utils in managed-ledger to locate offloader nar packages and load `LedgerOffloaderFactory` from corresponding NAR package. - Add a new distribution module to package offloaders into one distribution, as what we did for connectors. - Update pulsar-all image to include offloaders ### NOTES This change doesn't get rid of jcloud-shaded. because jcloud is using `ServiceLoad` and Guice injection. It makes things very tricky on class loading. Not attempt to address the problem any time soon. We can pin `jcloud-shaded` version as what we did for `protobuf-shaded` in next release. --- conf/broker.conf | 3 + distribution/{ => offloaders}/pom.xml | 42 +- distribution/offloaders/src/assemble/README| 10 ++ .../offloaders/src/assemble/offloaders.xml | 54 distribution/pom.xml | 1 + distribution/server/src/assemble/LICENSE.bin.txt | 14 -- docker/pulsar-all/Dockerfile | 4 + docker/pulsar-all/pom.xml | 21 +++ .../mledger/offload/OffloaderDefinition.java | 46 +++ .../bookkeeper/mledger/offload/OffloaderUtils.java | 149 + .../bookkeeper/mledger/offload/Offloaders.java | 57 .../apache/pulsar/broker/ServiceConfiguration.java | 11 ++ pulsar-broker/pom.xml | 6 - .../org/apache/pulsar/broker/PulsarService.java| 20 ++- tiered-storage/jcloud/pom.xml | 10 +- .../impl/BlobStoreManagedLedgerOffloader.java | 6 + .../META-INF/services/pulsar-offloader.yaml| 9 +- 17 files changed, 422 insertions(+), 41 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 11670d3..8927a85 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -513,6 +513,9 @@ schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.Bookkeepe ### --- Ledger Offloading --- ### +# The directory for all the offloader implementations +offloadersDirectory=./offloaders + # Driver to use to offload old data to long term storage (Possible values: S3, aws-s3, google-cloud-storage) # When using google-cloud-storage, Make sure both Google Cloud Storage and Google Cloud Storage JSON API are enabled for # the project (check from Developers Console -> Api&auth -> APIs). diff --git a/distribution/pom.xml b/distribution/offloaders/pom.xml similarity index 53% copy from distribution/pom.xml copy to distribution/offloaders/pom.xml index 36b4917..84349da 100644 --- a/distribution/pom.xml +++ b/distribution/offloaders/pom.xml @@ -24,17 +24,45 @@ org.apache.pulsar -pulsar +distribution 2.2.0-incubating-SNAPSHOT .. - distribution + pulsar-offloader-distribution pom - Pulsar :: Distribution + Pulsar :: Distribution :: Offloader - -server -io - + + + org.apache.pulsar + managed-ledger-original + ${project.version} + + + + + + +maven-assembly-plugin + + +distro-assembly +package + + single + + + true + posix + apache-pulsar-offloaders-${project.version} + +src/assemble/offloaders.xml + + + + + + + diff --git a/distribution/offloaders/src/assemble/README b/distribution/offloaders/src/assemble/README new file mode 100644 index 000..6777ec2 --- /dev/null +++ b/distribution/offloaders/src/assemble/README @@ -0,0 +1,10 @@ + +Please refer to http://pulsar.incubator.apache.org/ for access to documentation. + +This package contains Pulsar offloader archives. Each archive +contains: + + * the offloader code plus all the dependencies + + * ME
[incubator-pulsar] branch branch-2.1 updated: Increase default maxConcurrentLookupRequests to 50k (#2380)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 333b896 Increase default maxConcurrentLookupRequests to 50k (#2380) 333b896 is described below commit 333b896a37ccd27729efd778aa97710b7f8286db Author: Matteo Merli AuthorDate: Tue Aug 28 10:41:14 2018 -0700 Increase default maxConcurrentLookupRequests to 50k (#2380) ### Motivation Increasing the default value of `maxConcurrentLookupRequests` from 10K to 50K to increase limit of sustainable lookups before rejecting requests. --- conf/broker.conf | 3 +-- conf/proxy.conf| 2 +- conf/standalone.conf | 2 +- deployment/terraform-ansible/templates/broker.conf | 2 +- .../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +- .../main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index cd4a505..5eeb936 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 @@ -511,4 +511,3 @@ s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576 # Deprecated. Use configurationStoreServers globalZookeeperServers= - diff --git a/conf/proxy.conf b/conf/proxy.conf index 3bde301..ed5420f 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false maxConcurrentInboundConnections=1 # Max concurrent outbound connections. The proxy will error out requests beyond that. -maxConcurrentLookupRequests=1 +maxConcurrentLookupRequests=5 # --- TLS --- # diff --git a/conf/standalone.conf b/conf/standalone.conf index cc03b95..a9fb288 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -141,7 +141,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index a7456a4..8eaa0df 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index ddc7728..7b2e85b 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -170,7 +170,7 @@ public class ServiceConfiguration implements PulsarConfiguration { private boolean dispatchThrottlingOnNonBacklogConsumerEnabled = false; // Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic @FieldContext(dynamic = true) -private int maxConcurrentLookupRequest = 1; +private int maxConcurrentLookupRequest = 5; // Max number of concurrent topic loading request broker allows to control number of zk-operations @FieldContext(dynamic = true) private int maxConcurrentTopicLoadRequest = 5000; diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java index 4d02786..b4e8afb 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
[incubator-pulsar] branch master updated: Increase default maxConcurrentLookupRequests to 50k (#2380)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new fa11a3c Increase default maxConcurrentLookupRequests to 50k (#2380) fa11a3c is described below commit fa11a3ca5d36ae6c1cc169e32cd0164fd8e47487 Author: Matteo Merli AuthorDate: Tue Aug 28 10:41:14 2018 -0700 Increase default maxConcurrentLookupRequests to 50k (#2380) ### Motivation Increasing the default value of `maxConcurrentLookupRequests` from 10K to 50K to increase limit of sustainable lookups before rejecting requests. --- conf/broker.conf | 3 +-- conf/proxy.conf | 2 +- conf/standalone.conf | 2 +- deployment/terraform-ansible/templates/broker.conf| 2 +- deployment/terraform-ansible/templates/proxy.conf | 2 +- .../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +- .../main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java | 2 +- site2/docs/administration-proxy.md| 2 +- site2/docs/reference-configuration.md | 4 ++-- 9 files changed, 10 insertions(+), 11 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index a8dd44b..11670d3 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -161,7 +161,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 @@ -557,4 +557,3 @@ gcsManagedLedgerOffloadServiceAccountKeyFile= # Deprecated. Use configurationStoreServers globalZookeeperServers= - diff --git a/conf/proxy.conf b/conf/proxy.conf index 7b7e690..9b307cc 100644 --- a/conf/proxy.conf +++ b/conf/proxy.conf @@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false maxConcurrentInboundConnections=1 # Max concurrent outbound connections. The proxy will error out requests beyond that. -maxConcurrentLookupRequests=1 +maxConcurrentLookupRequests=5 # --- TLS --- # diff --git a/conf/standalone.conf b/conf/standalone.conf index 13bef8a..09d369c 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -144,7 +144,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index 8e8c6c2..a4fad1c 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0 dispatchThrottlingOnNonBacklogConsumerEnabled=true # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic -maxConcurrentLookupRequest=1 +maxConcurrentLookupRequest=5 # Max number of concurrent topic loading request broker allows to control number of zk-operations maxConcurrentTopicLoadRequest=5000 diff --git a/deployment/terraform-ansible/templates/proxy.conf b/deployment/terraform-ansible/templates/proxy.conf index 21fa327..b631aca 100644 --- a/deployment/terraform-ansible/templates/proxy.conf +++ b/deployment/terraform-ansible/templates/proxy.conf @@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false maxConcurrentInboundConnections=1 # Max concurrent outbound connections. The proxy will error out requests beyond that. -maxConcurrentLookupRequests=1 +maxConcurrentLookupRequests=5 # --- TLS --- # diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 8057540..e94f6b9 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -179,7 +179,7 @@ public class ServiceConfiguration implements PulsarConfiguration { private boolean dispatchThrottlingOnNonBacklogConsumerEnabled = false; // Max number of concurrent lookup request broker allows to throttle heavy
[incubator-pulsar] branch master updated: Fix flake in DiscoveryServiceTest (#1081) (#2406)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 774cafa Fix flake in DiscoveryServiceTest (#1081) (#2406) 774cafa is described below commit 774cafa25d6340c778c5c281045446086aff96a0 Author: Ivan Kelly AuthorDate: Tue Aug 28 07:30:42 2018 +0200 Fix flake in DiscoveryServiceTest (#1081) (#2406) This test was flaking because it was only waiting for 1 second for connection and message exchange to complete, which is not enough time when there's heavy load on the machine (simulated with stress-ng). The fix is to increase the timeout to 10 seconds. I've also cleaned up the test to use a CompletableFuture rather than a CountDownLatch so tha the test thread can be notified of failures in the handlers. --- .../discovery/service/DiscoveryServiceTest.java| 77 -- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java b/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java index 7d4a5ee..24bcd6a 100644 --- a/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java +++ b/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java @@ -21,7 +21,6 @@ package org.apache.pulsar.discovery.service; import static org.apache.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import java.lang.reflect.Field; @@ -31,16 +30,17 @@ import java.net.URISyntaxException; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.apache.bookkeeper.util.ZkUtils; import org.apache.pulsar.common.api.Commands; +import org.apache.pulsar.common.api.proto.PulsarApi.BaseCommand; import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.SecurityUtility; +import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream; import org.apache.pulsar.discovery.service.web.ZookeeperCacheLoader; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; import org.apache.pulsar.zookeeper.ZooKeeperChildrenCache; @@ -66,8 +66,13 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class DiscoveryServiceTest extends BaseDiscoveryTestSetup { +private static final Logger log = LoggerFactory.getLogger(DiscoveryServiceTest.class); + private final static String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt"; private final static String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key"; @@ -120,48 +125,40 @@ public class DiscoveryServiceTest extends BaseDiscoveryTestSetup { /** * It verifies: client connects to Discovery-service and receives discovery response successfully. - * + * * @throws Exception */ @Test public void testClientServerConnection() throws Exception { addBrokerToZk(2); -// 1. client connects to DiscoveryService, 2. Client receive service-lookup response -final int messageTransfer = 2; -final CountDownLatch latch = new CountDownLatch(messageTransfer); -NioEventLoopGroup workerGroup = connectToService(service.getServiceUrl(), latch, false); -try { -assertTrue(latch.await(1, TimeUnit.SECONDS)); -} catch (InterruptedException e) { -fail("should have received lookup response message from server", e); -} + +final CompletableFuture promise = new CompletableFuture<>(); +NioEventLoopGroup workerGroup = connectToService(service.getServiceUrl(), promise, false); +assertEquals(promise.get(10, TimeUnit.SECONDS).getType(), BaseCommand.Type.CONNECTED); workerGroup.shutdownGracefully(); } @Test public void testClientServerConnectionTls() throws Exception { addBrokerToZk(2); -// 1. client connects to DiscoveryService, 2. Client receive service-lookup response -
[incubator-pulsar] branch master updated: Fixed {{pulsar.version}} into {{pulsar:version}} in few pages (#2455)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8da0669 Fixed {{pulsar.version}} into {{pulsar:version}} in few pages (#2455) 8da0669 is described below commit 8da066947b339d1f71025f8d3857bb7e5b9e70a2 Author: Matteo Merli AuthorDate: Mon Aug 27 22:26:10 2018 -0700 Fixed {{pulsar.version}} into {{pulsar:version}} in few pages (#2455) ### Motivation In few places in website we're displaying the literal `{{pulsar.version}}` instead of the actual Pulsar version string. The correct variable name is expected to use `:` as separator. --- site2/docs/client-libraries-go.md| 2 +- site2/docs/deploy-bare-metal.md | 12 ++-- site2/docs/getting-started-standalone.md | 12 ++-- site2/docs/io-quickstart.md | 12 ++-- .../version-2.1.0-incubating/client-libraries-go.md | 2 +- .../version-2.1.0-incubating/deploy-bare-metal.md| 12 ++-- .../version-2.1.0-incubating/getting-started-standalone.md | 12 ++-- .../versioned_docs/version-2.1.0-incubating/io-quickstart.md | 12 ++-- 8 files changed, 38 insertions(+), 38 deletions(-) diff --git a/site2/docs/client-libraries-go.md b/site2/docs/client-libraries-go.md index 2433eb9..421d457 100644 --- a/site2/docs/client-libraries-go.md +++ b/site2/docs/client-libraries-go.md @@ -34,7 +34,7 @@ $ go get -u github.com/apache/incubator-pulsar/pulsar-client-go/pulsar Or you can use [dep](https://github.com/golang/dep) for managing the dependencies. ```bash -$ dep ensure -add github.com/apache/incubator-pulsar/pulsar-client-go/pulsar@v{{pulsar.version}} +$ dep ensure -add github.com/apache/incubator-pulsar/pulsar-client-go/pulsar@v{{pulsar:version}} ``` Once installed locally, you can import it into your project: diff --git a/site2/docs/deploy-bare-metal.md b/site2/docs/deploy-bare-metal.md index 78ea557..b5793c6 100644 --- a/site2/docs/deploy-bare-metal.md +++ b/site2/docs/deploy-bare-metal.md @@ -125,12 +125,12 @@ $ tar xvfz apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz $ mv apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors $ ls connectors -pulsar-io-aerospike-{{pulsar.version}}.nar -pulsar-io-cassandra-{{pulsar.version}}.nar -pulsar-io-kafka-{{pulsar.version}}.nar -pulsar-io-kinesis-{{pulsar.version}}.nar -pulsar-io-rabbitmq-{{pulsar.version}}.nar -pulsar-io-twitter-{{pulsar.version}}.nar +pulsar-io-aerospike-{{pulsar:version}}.nar +pulsar-io-cassandra-{{pulsar:version}}.nar +pulsar-io-kafka-{{pulsar:version}}.nar +pulsar-io-kinesis-{{pulsar:version}}.nar +pulsar-io-rabbitmq-{{pulsar:version}}.nar +pulsar-io-twitter-{{pulsar:version}}.nar ... ``` diff --git a/site2/docs/getting-started-standalone.md b/site2/docs/getting-started-standalone.md index 4090451..0dcf024 100644 --- a/site2/docs/getting-started-standalone.md +++ b/site2/docs/getting-started-standalone.md @@ -87,12 +87,12 @@ $ tar xvfz /path/to/apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz $ cd apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors $ ls connectors -pulsar-io-aerospike-{{pulsar.version}}.nar -pulsar-io-cassandra-{{pulsar.version}}.nar -pulsar-io-kafka-{{pulsar.version}}.nar -pulsar-io-kinesis-{{pulsar.version}}.nar -pulsar-io-rabbitmq-{{pulsar.version}}.nar -pulsar-io-twitter-{{pulsar.version}}.nar +pulsar-io-aerospike-{{pulsar:version}}.nar +pulsar-io-cassandra-{{pulsar:version}}.nar +pulsar-io-kafka-{{pulsar:version}}.nar +pulsar-io-kinesis-{{pulsar:version}}.nar +pulsar-io-rabbitmq-{{pulsar:version}}.nar +pulsar-io-twitter-{{pulsar:version}}.nar ... ``` diff --git a/site2/docs/io-quickstart.md b/site2/docs/io-quickstart.md index 042ce8e..914429b 100644 --- a/site2/docs/io-quickstart.md +++ b/site2/docs/io-quickstart.md @@ -69,12 +69,12 @@ $ tar xvfz /path/to/apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz $ cp -r apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors $ ls connectors -pulsar-io-aerospike-{{pulsar.version}}.nar -pulsar-io-cassandra-{{pulsar.version}}.nar -pulsar-io-kafka-{{pulsar.version}}.nar -pulsar-io-kinesis-{{pulsar.version}}.nar -pulsar-io-rabbitmq-{{pulsar.version}}.nar -pulsar-io-twitter-{{pulsar.version}}.nar +pulsar-io-aerospike-{{pulsar:version}}.nar +pulsar-io-cassandra-{{pulsar:version}}.nar +pulsar-io-kafka-{{pulsar:version}}.nar +pulsar-io-kinesis-{{pulsar:version}}.nar +pulsar-io-rabbitmq-{{pulsar:version}}.nar +pulsar-io-twitter-{{pulsar:version}}.nar ... ``` diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-go.md b/site2/website/versioned_docs/version-2.1.0-incub
[incubator-pulsar] branch master updated: [Schema] Provide a generic record interface for representing a typed message (#2452)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 757222d [Schema] Provide a generic record interface for representing a typed message (#2452) 757222d is described below commit 757222d9d232dbe03293df890609aa85ff63556c Author: Sijie Guo AuthorDate: Mon Aug 27 22:21:27 2018 -0700 [Schema] Provide a generic record interface for representing a typed message (#2452) * [Schema] Provide a generic record interface for representing a typed message ### Motivation In some use cases, the publishers and consumers don't know the type or schema of the messages ahead of time. For example, in pulsar io connector, when connecting a topic to a jdbc table, the connector doesn't know the tyep of the messages ahead of time; the connector can only fetch schema info from schema registry and that is the only information connector knows. It is impossible for mapping the messages to a relational database table. So we need a way to present a generic `Struct` record with fields. ### Changes Introduce `Field` and `GenericRecord` to represent `Struct` records deserialized with a schema. ### NotCovered This change only introduces the interfaces. It doesn't integrate with the producer and consumer workflow. That would be done in subsequent changes if we agree on the interfaces. --- .../org/apache/pulsar/client/api/schema/Field.java | 43 + .../pulsar/client/api/schema/GenericRecord.java| 51 +++ .../pulsar/client/impl/schema/AvroSchema.java | 17 ++-- .../client/impl/schema/GenericAvroRecord.java | 79 .../client/impl/schema/GenericAvroSchema.java | 102 + .../pulsar/client/schema/AvroSchemaTest.java | 47 +- 6 files changed, 329 insertions(+), 10 deletions(-) diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java new file mode 100644 index 000..653b5d6 --- /dev/null +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.client.api.schema; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +/** + * A field in a record, consisting of a field name, index, and + * {@link org.apache.pulsar.client.api.Schema} for the field value. + */ +@Data +@EqualsAndHashCode +@ToString +public class Field { + +/** + * The field name. + */ +private final String name; +/** + * The index of the field within the record. + */ +private final int index; + +} diff --git a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java new file mode 100644 index 000..0a4fce4 --- /dev/null +++ b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the Lic
[incubator-pulsar] branch branch-2.1 updated: Revert "Kinesis-sink consider topic-name as partition-key if record key empty (#2372)"
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 9ede4c8 Revert "Kinesis-sink consider topic-name as partition-key if record key empty (#2372)" 9ede4c8 is described below commit 9ede4c80802626065f70fb5e20b81bafb6f33969 Author: Sijie Guo AuthorDate: Mon Aug 27 11:47:56 2018 -0700 Revert "Kinesis-sink consider topic-name as partition-key if record key empty (#2372)" This reverts commit 703305b5426856bab7bab30a41e4f242e7782dc7. --- .../kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java index cdfadde..67de21a 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java @@ -114,7 +114,7 @@ public class KinesisSink implements Sink { record.getRecordSequence()); throw new IllegalStateException("kinesis queue has publish failure"); } -String partitionedKey = record.getKey().orElse(record.getTopicName().orElse(defaultPartitionedKey)); +String partitionedKey = record.getKey().orElse(defaultPartitionedKey); partitionedKey = partitionedKey.length() > maxPartitionedKeyLength ? partitionedKey.substring(0, maxPartitionedKeyLength - 1) : partitionedKey; // partitionedKey Length must be at least one, and at most 256
[incubator-pulsar] branch branch-2.1 updated: Added Reader.HasNext in Go client (#2450)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new b6dc7c1 Added Reader.HasNext in Go client (#2450) b6dc7c1 is described below commit b6dc7c1eae314e65cf782a7b0201408279dc9e8d Author: Matteo Merli AuthorDate: Mon Aug 27 11:41:24 2018 -0700 Added Reader.HasNext in Go client (#2450) ### Motivation Added `Reader.HasNext()` in Go client library --- pulsar-client-go/pulsar/c_reader.go| 13 + pulsar-client-go/pulsar/reader.go | 3 +++ pulsar-client-go/pulsar/reader_test.go | 12 ++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pulsar-client-go/pulsar/c_reader.go b/pulsar-client-go/pulsar/c_reader.go index 730f9b8..12c1103 100644 --- a/pulsar-client-go/pulsar/c_reader.go +++ b/pulsar-client-go/pulsar/c_reader.go @@ -146,6 +146,19 @@ func (r *reader) Next(ctx context.Context) (Message, error) { } } +func (r *reader) HasNext() (bool, error) { + value := C.int(0) + res := C.pulsar_reader_has_message_available(r.ptr, &value) + + if res != C.pulsar_result_Ok { + return false, newError(res, "Failed to check if next message is available") + } else if value == C.int(1) { + return true, nil + } else { + return false, nil + } +} + func (r *reader) Close() error { channel := make(chan error) r.CloseAsync(func(err error) { channel <- err; close(channel) }) diff --git a/pulsar-client-go/pulsar/reader.go b/pulsar-client-go/pulsar/reader.go index f61ebd7..7015c9c 100644 --- a/pulsar-client-go/pulsar/reader.go +++ b/pulsar-client-go/pulsar/reader.go @@ -67,6 +67,9 @@ type Reader interface { // Read the next message in the topic, blocking until a message is available Next(context.Context) (Message, error) + // Check if there is any message available to read from the current position + HasNext() (bool, error) + // Close the reader and stop the broker to push more messages Close() error } diff --git a/pulsar-client-go/pulsar/reader_test.go b/pulsar-client-go/pulsar/reader_test.go index 11d1b36..3b075e1 100644 --- a/pulsar-client-go/pulsar/reader_test.go +++ b/pulsar-client-go/pulsar/reader_test.go @@ -20,9 +20,9 @@ package pulsar import ( - "testing" - "fmt" "context" + "fmt" + "testing" ) func TestReaderConnectError(t *testing.T) { @@ -80,12 +80,20 @@ func TestReader(t *testing.T) { t.Fatal(err) } + hasNext, err := reader.HasNext() + assertNil(t, err) + assertEqual(t, hasNext, true) + msg, err := reader.Next(ctx) assertNil(t, err) assertNotNil(t, msg) assertEqual(t, string(msg.Payload()), fmt.Sprintf("hello-%d", i)) } + + hasNext, err := reader.HasNext() + assertNil(t, err) + assertEqual(t, hasNext, false) } func TestReaderWithInvalidConf(t *testing.T) {
[incubator-pulsar] branch master updated: Added Reader.HasNext in Go client (#2450)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 73bc19d Added Reader.HasNext in Go client (#2450) 73bc19d is described below commit 73bc19dc41b6a06bd7051216682cc5d2fc5d5592 Author: Matteo Merli AuthorDate: Mon Aug 27 11:41:24 2018 -0700 Added Reader.HasNext in Go client (#2450) ### Motivation Added `Reader.HasNext()` in Go client library --- pulsar-client-go/pulsar/c_reader.go| 13 + pulsar-client-go/pulsar/reader.go | 3 +++ pulsar-client-go/pulsar/reader_test.go | 12 ++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pulsar-client-go/pulsar/c_reader.go b/pulsar-client-go/pulsar/c_reader.go index 730f9b8..12c1103 100644 --- a/pulsar-client-go/pulsar/c_reader.go +++ b/pulsar-client-go/pulsar/c_reader.go @@ -146,6 +146,19 @@ func (r *reader) Next(ctx context.Context) (Message, error) { } } +func (r *reader) HasNext() (bool, error) { + value := C.int(0) + res := C.pulsar_reader_has_message_available(r.ptr, &value) + + if res != C.pulsar_result_Ok { + return false, newError(res, "Failed to check if next message is available") + } else if value == C.int(1) { + return true, nil + } else { + return false, nil + } +} + func (r *reader) Close() error { channel := make(chan error) r.CloseAsync(func(err error) { channel <- err; close(channel) }) diff --git a/pulsar-client-go/pulsar/reader.go b/pulsar-client-go/pulsar/reader.go index f61ebd7..7015c9c 100644 --- a/pulsar-client-go/pulsar/reader.go +++ b/pulsar-client-go/pulsar/reader.go @@ -67,6 +67,9 @@ type Reader interface { // Read the next message in the topic, blocking until a message is available Next(context.Context) (Message, error) + // Check if there is any message available to read from the current position + HasNext() (bool, error) + // Close the reader and stop the broker to push more messages Close() error } diff --git a/pulsar-client-go/pulsar/reader_test.go b/pulsar-client-go/pulsar/reader_test.go index 11d1b36..3b075e1 100644 --- a/pulsar-client-go/pulsar/reader_test.go +++ b/pulsar-client-go/pulsar/reader_test.go @@ -20,9 +20,9 @@ package pulsar import ( - "testing" - "fmt" "context" + "fmt" + "testing" ) func TestReaderConnectError(t *testing.T) { @@ -80,12 +80,20 @@ func TestReader(t *testing.T) { t.Fatal(err) } + hasNext, err := reader.HasNext() + assertNil(t, err) + assertEqual(t, hasNext, true) + msg, err := reader.Next(ctx) assertNil(t, err) assertNotNil(t, msg) assertEqual(t, string(msg.Payload()), fmt.Sprintf("hello-%d", i)) } + + hasNext, err := reader.HasNext() + assertNil(t, err) + assertEqual(t, hasNext, false) } func TestReaderWithInvalidConf(t *testing.T) {
[incubator-pulsar] branch branch-2.1 updated: Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 2595941 Issue #2330: change getTopicName in MultiTopicsConsumer (#2346) 2595941 is described below commit 259594165f5ee3c69a29629534af81011cab4ef1 Author: Jia Zhai AuthorDate: Tue Aug 28 02:38:41 2018 +0800 Issue #2330: change getTopicName in MultiTopicsConsumer (#2346) * change getTopicName in MultiTopicsConsumer * change following sijie's comments * keep both topicName and topicPartitonName in consumer to avoid new string --- .../apache/pulsar/client/impl/ConsumerImpl.java| 8 +++ .../client/impl/MultiTopicsConsumerImpl.java | 9 .../pulsar/client/impl/TopicMessageIdImpl.java | 27 ++ .../pulsar/client/impl/TopicMessageImpl.java | 20 .../client/impl/UnAckedTopicMessageTracker.java| 4 ++-- .../pulsar/client/impl/MessageIdCompareToTest.java | 6 - .../tests/integration/semantics/SemanticsTest.java | 2 +- 7 files changed, 60 insertions(+), 16 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1a7b67b..fe0e2c1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -131,6 +131,8 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final SubscriptionInitialPosition subscriptionInitialPosition; private final ConnectionHandler connectionHandler; +private final String topicNameWithoutPartition; + enum SubscriptionMode { // Make the subscription to be backed by a durable cursor that will retain messages and persist the current // position @@ -203,6 +205,8 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle NonPersistentAcknowledgmentGroupingTracker.of(); } +topicNameWithoutPartition = topicName.getPartitionedTopicName(); + grabCnx(); } @@ -1458,6 +1462,10 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle this.connectionHandler.grabCnx(); } +public String getTopicNameWithoutPartition() { +return topicNameWithoutPartition; +} + private static final Logger log = LoggerFactory.getLogger(ConsumerImpl.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index bbfb3f3..6d5d56d 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -230,7 +230,8 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { checkArgument(message instanceof MessageImpl); lock.writeLock().lock(); try { -TopicMessageImpl topicMessage = new TopicMessageImpl<>(consumer.getTopic(), message); +TopicMessageImpl topicMessage = new TopicMessageImpl<>( +consumer.getTopic(), consumer.getTopicNameWithoutPartition(), message); unAckedMessageTracker.add(topicMessage.getMessageId()); if (log.isDebugEnabled()) { @@ -369,7 +370,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { } if (ackType == AckType.Cumulative) { -Consumer individualConsumer = consumers.get(topicMessageId.getTopicName()); +Consumer individualConsumer = consumers.get(topicMessageId.getTopicPartitionName()); if (individualConsumer != null) { MessageId innerId = topicMessageId.getInnerMessageId(); return individualConsumer.acknowledgeCumulativeAsync(innerId); @@ -377,7 +378,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException()); } } else { -ConsumerImpl consumer = consumers.get(topicMessageId.getTopicName()); +ConsumerImpl consumer = consumers.get(topicMessageId.getTopicPartitionName()); MessageId innerId = topicMessageId.getInnerMessageId(); return consumer.doAcknowledge(innerId, ackType, properties) @@ -510,7 +511,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { } removeExpiredMessagesFromQueue(messageIds); messageIds.stream().map(messageId -> (TopicMessageIdImpl)messageId) -
[incubator-pulsar] branch master updated: Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8688b67 Issue #2330: change getTopicName in MultiTopicsConsumer (#2346) 8688b67 is described below commit 8688b6733177ee4a48d805df18a1d82865cf7320 Author: Jia Zhai AuthorDate: Tue Aug 28 02:38:41 2018 +0800 Issue #2330: change getTopicName in MultiTopicsConsumer (#2346) * change getTopicName in MultiTopicsConsumer * change following sijie's comments * keep both topicName and topicPartitonName in consumer to avoid new string --- .../apache/pulsar/client/impl/ConsumerImpl.java| 8 +++ .../client/impl/MultiTopicsConsumerImpl.java | 9 .../pulsar/client/impl/TopicMessageIdImpl.java | 27 ++ .../pulsar/client/impl/TopicMessageImpl.java | 20 .../client/impl/UnAckedTopicMessageTracker.java| 4 ++-- .../pulsar/client/impl/MessageIdCompareToTest.java | 6 - .../tests/integration/semantics/SemanticsTest.java | 2 +- 7 files changed, 60 insertions(+), 16 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index da04534..fe37b69 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -131,6 +131,8 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle private final SubscriptionInitialPosition subscriptionInitialPosition; private final ConnectionHandler connectionHandler; +private final String topicNameWithoutPartition; + enum SubscriptionMode { // Make the subscription to be backed by a durable cursor that will retain messages and persist the current // position @@ -203,6 +205,8 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle NonPersistentAcknowledgmentGroupingTracker.of(); } +topicNameWithoutPartition = topicName.getPartitionedTopicName(); + grabCnx(); } @@ -1458,6 +1462,10 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle this.connectionHandler.grabCnx(); } +public String getTopicNameWithoutPartition() { +return topicNameWithoutPartition; +} + private static final Logger log = LoggerFactory.getLogger(ConsumerImpl.class); } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java index 799bec8..f1cb9cf 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java @@ -231,7 +231,8 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { checkArgument(message instanceof MessageImpl); lock.writeLock().lock(); try { -TopicMessageImpl topicMessage = new TopicMessageImpl<>(consumer.getTopic(), message); +TopicMessageImpl topicMessage = new TopicMessageImpl<>( +consumer.getTopic(), consumer.getTopicNameWithoutPartition(), message); unAckedMessageTracker.add(topicMessage.getMessageId()); if (log.isDebugEnabled()) { @@ -370,7 +371,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { } if (ackType == AckType.Cumulative) { -Consumer individualConsumer = consumers.get(topicMessageId.getTopicName()); +Consumer individualConsumer = consumers.get(topicMessageId.getTopicPartitionName()); if (individualConsumer != null) { MessageId innerId = topicMessageId.getInnerMessageId(); return individualConsumer.acknowledgeCumulativeAsync(innerId); @@ -378,7 +379,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException()); } } else { -ConsumerImpl consumer = consumers.get(topicMessageId.getTopicName()); +ConsumerImpl consumer = consumers.get(topicMessageId.getTopicPartitionName()); MessageId innerId = topicMessageId.getInnerMessageId(); return consumer.doAcknowledge(innerId, ackType, properties) @@ -511,7 +512,7 @@ public class MultiTopicsConsumerImpl extends ConsumerBase { } removeExpiredMessagesFromQueue(messageIds); messageIds.stream().map(messageId -> (TopicMessageIdImpl)messageId) -.collect(C
[incubator-pulsar] branch branch-2.1 updated: fix issue with protocol version not being correct (#2444)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 96bf00f fix issue with protocol version not being correct (#2444) 96bf00f is described below commit 96bf00fa12bf74db57d36f40989e233752288351 Author: Boyang Jerry Peng AuthorDate: Fri Aug 24 22:40:58 2018 -0700 fix issue with protocol version not being correct (#2444) --- .../src/main/java/org/apache/pulsar/client/impl/ClientCnx.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index 6a8ab74..b2d9478 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -232,8 +232,9 @@ public class ClientCnx extends PulsarHandler { if (log.isDebugEnabled()) { log.debug("{} Connection is ready", ctx.channel()); } -connectionFuture.complete(null); +// set remote protocol version to the correct version before we complete the connection future remoteEndpointProtocolVersion = connected.getProtocolVersion(); +connectionFuture.complete(null); state = State.Ready; }
[incubator-pulsar] branch branch-2.1 updated: Fixed initialization order of acknowledgmentsGroupingTracker in ConsumerImpl (#2399)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 93b6d20 Fixed initialization order of acknowledgmentsGroupingTracker in ConsumerImpl (#2399) 93b6d20 is described below commit 93b6d209e8c1bfdc3c84eb066dc46da0bed165c5 Author: Matteo Merli AuthorDate: Mon Aug 20 10:39:54 2018 -0700 Fixed initialization order of acknowledgmentsGroupingTracker in ConsumerImpl (#2399) ### Motivation With delayed acks enabled (the default), there is a potential race condition that lead to a NPE: ``` java.lang.NullPointerException at org.apache.pulsar.client.impl.ConsumerImpl.getClientCnx(ConsumerImpl.java:1446) at org.apache.pulsar.client.impl.PersistentAcknowledgmentsGroupingTracker.flush(PersistentAcknowledgmentsGroupingTracker.java:154) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ... ``` The reason is that the delayed ack commit task gets scheduled (eg: in 100ms) and might be executed before the the main thread has finished initializing the `ConsumerImpl` instance. ### Modifications Reordered the initialization in `ConsumerImpl` constructor to make sure `connectionHandler` is already set when we create the `PersistentAcknowledgmentsGroupingTracker` instance. --- .../org/apache/pulsar/client/impl/ConsumerImpl.java| 18 +- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 1c69c75..1a7b67b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -160,15 +160,6 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle this.readCompacted = conf.isReadCompacted(); this.subscriptionInitialPosition = conf.getSubscriptionInitialPosition(); -TopicName topicName = TopicName.get(topic); -if (topicName.isPersistent()) { -this.acknowledgmentsGroupingTracker = -new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); -} else { -this.acknowledgmentsGroupingTracker = -NonPersistentAcknowledgmentGroupingTracker.of(); -} - if (client.getConfiguration().getStatsIntervalSeconds() > 0) { stats = new ConsumerStatsRecorderImpl(client, conf, this); } else { @@ -203,6 +194,15 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle new Backoff(100, TimeUnit.MILLISECONDS, 60, TimeUnit.SECONDS, 0, TimeUnit.MILLISECONDS), this); +TopicName topicName = TopicName.get(topic); +if (topicName.isPersistent()) { +this.acknowledgmentsGroupingTracker = +new PersistentAcknowledgmentsGroupingTracker(this, conf, client.eventLoopGroup()); +} else { +this.acknowledgmentsGroupingTracker = +NonPersistentAcknowledgmentGroupingTracker.of(); +} + grabCnx(); }
[incubator-pulsar] branch branch-2.1 updated: leak assign schema in MessageImpl constructor (#2439)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new acd77a0 leak assign schema in MessageImpl constructor (#2439) acd77a0 is described below commit acd77a0fe6f09de1919040b65a90fb145f7f1bfa Author: Jia Zhai AuthorDate: Fri Aug 24 11:42:55 2018 +0800 leak assign schema in MessageImpl constructor (#2439) --- .../src/main/java/org/apache/pulsar/client/impl/MessageImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java index 5ea8bfe..96b68c1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java @@ -155,6 +155,7 @@ public class MessageImpl implements Message { this.cnx = null; this.payload = payload; this.properties = Collections.unmodifiableMap(properties); +this.schema = schema; } public static MessageImpl deserialize(ByteBuf headersAndPayload) throws IOException {
[incubator-pulsar] branch branch-2.1 updated: Increased default brokerShutdownTimeout to 60 seconds (#2377)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new caf8d67 Increased default brokerShutdownTimeout to 60 seconds (#2377) caf8d67 is described below commit caf8d676f0a49896269b57032bd07ae3b4287564 Author: Matteo Merli AuthorDate: Wed Aug 15 11:50:18 2018 -0700 Increased default brokerShutdownTimeout to 60 seconds (#2377) ### Motivation The default timeout for broker graceful shutdown is set to 3 seconds. This can give little room to do graceful shutdown when the broker is serving a lot of topics. There is no big downside in increasing the timeout to a much bigger value. --- conf/broker.conf| 2 +- conf/standalone.conf| 2 +- deployment/terraform-ansible/templates/broker.conf | 2 +- .../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java| 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 2e40d82..cd4a505 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -53,7 +53,7 @@ failureDomainsEnabled=false zooKeeperSessionTimeoutMillis=3 # Time to wait for broker graceful shutdown. After this time elapses, the process will be killed -brokerShutdownTimeoutMs=3000 +brokerShutdownTimeoutMs=6 # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true diff --git a/conf/standalone.conf b/conf/standalone.conf index 0d7a395..cc03b95 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -46,7 +46,7 @@ failureDomainsEnabled=false zooKeeperSessionTimeoutMillis=3 # Time to wait for broker graceful shutdown. After this time elapses, the process will be killed -brokerShutdownTimeoutMs=3000 +brokerShutdownTimeoutMs=6 # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true diff --git a/deployment/terraform-ansible/templates/broker.conf b/deployment/terraform-ansible/templates/broker.conf index 73d4e2f..a7456a4 100644 --- a/deployment/terraform-ansible/templates/broker.conf +++ b/deployment/terraform-ansible/templates/broker.conf @@ -53,7 +53,7 @@ failureDomainsEnabled=false zooKeeperSessionTimeoutMillis=3 # Time to wait for broker graceful shutdown. After this time elapses, the process will be killed -brokerShutdownTimeoutMs=3000 +brokerShutdownTimeoutMs=6 # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true diff --git a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java index 1bb6fff..ddc7728 100644 --- a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java @@ -78,7 +78,7 @@ public class ServiceConfiguration implements PulsarConfiguration { // Time to wait for broker graceful shutdown. After this time elapses, the // process will be killed @FieldContext(dynamic = true) -private long brokerShutdownTimeoutMs = 3000; +private long brokerShutdownTimeoutMs = 6; // Enable backlog quota check. Enforces action on topic when the quota is // reached private boolean backlogQuotaCheckEnabled = true;
[incubator-pulsar] branch branch-2.1 updated: [client] add properties to producer for cpp & python client (#2420)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new d7ec1c5 [client] add properties to producer for cpp & python client (#2420) d7ec1c5 is described below commit d7ec1c5051ee3f7066cd1ba1572606760a3c371e Author: Sijie Guo AuthorDate: Wed Aug 22 09:32:46 2018 -0700 [client] add properties to producer for cpp & python client (#2420) * [client] add properties to producer for cpp & python client ### Motivation This is a caught-up change to enable properties for producer as java clients. ### Changes Enable properties on producer for both cpp & python client ### Results Properties are added as metadata for CommandProducer. However there is no way to verify the producer properties. so I didn't add any specific tests, just adding properties for both cpp and python clients in the tests, that should excerise the corresponding code path. * Add `properties` to pydoc --- .../include/pulsar/ProducerConfiguration.h | 34 ++ pulsar-client-cpp/lib/Commands.cc | 11 ++- pulsar-client-cpp/lib/Commands.h | 3 +- pulsar-client-cpp/lib/ProducerConfiguration.cc | 34 +- pulsar-client-cpp/lib/ProducerConfigurationImpl.h | 1 + pulsar-client-cpp/lib/ProducerImpl.cc | 3 +- pulsar-client-cpp/python/pulsar/__init__.py| 11 ++- pulsar-client-cpp/python/src/config.cc | 1 + pulsar-client-cpp/python/test_producer.py | 6 +++- pulsar-client-cpp/tests/BasicEndToEndTest.cc | 2 ++ 10 files changed, 100 insertions(+), 6 deletions(-) diff --git a/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h b/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h index 9f7bf1f..45154c5 100644 --- a/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h +++ b/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h @@ -127,6 +127,40 @@ class ProducerConfiguration { bool isEncryptionEnabled() const; ProducerConfiguration& addEncryptionKey(std::string key); +/** + * Check whether the message has a specific property attached. + * + * @param name the name of the property to check + * @return true if the message has the specified property + * @return false if the property is not defined + */ +bool hasProperty(const std::string& name) const; + +/** + * Get the value of a specific property + * + * @param name the name of the property + * @return the value of the property or null if the property was not defined + */ +const std::string& getProperty(const std::string& name) const; + +/** + * Get all the properties attached to this producer. + */ +std::map& getProperties() const; + +/** + * Sets a new property on a message. + * @param name the name of the property + * @param value the associated value + */ +ProducerConfiguration& setProperty(const std::string& name, const std::string& value); + +/** + * Add all the properties in the provided map + */ +ProducerConfiguration& setProperties(const std::map& properties); + friend class PulsarWrapper; private: diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc index 8bd0128..4d9f1be 100644 --- a/pulsar-client-cpp/lib/Commands.cc +++ b/pulsar-client-cpp/lib/Commands.cc @@ -22,6 +22,7 @@ #include "pulsar/MessageBuilder.h" #include "PulsarApi.pb.h" #include "LogUtils.h" +#include "PulsarApi.pb.h" #include "Utils.h" #include "Url.h" #include "checksum/ChecksumProvider.h" @@ -231,13 +232,21 @@ SharedBuffer Commands::newUnsubscribe(uint64_t consumerId, uint64_t requestId) { } SharedBuffer Commands::newProducer(const std::string& topic, uint64_t producerId, - const std::string& producerName, uint64_t requestId) { + const std::string& producerName, uint64_t requestId, + const std::map& metadata) { BaseCommand cmd; cmd.set_type(BaseCommand::PRODUCER); CommandProducer* producer = cmd.mutable_producer(); producer->set_topic(topic); producer->set_producer_id(producerId); producer->set_request_id(requestId); +for (std::map::const_iterator it = metadata.begin(); it != metadata.end(); + it++) { +proto::KeyValue* keyValue = proto::KeyValue().New(); +keyValue->set_key(it->first); +keyValue->set_value(it->second); +producer->mutable_metad
[incubator-pulsar] branch branch-2.1 updated: [client] add properties to consumer for cpp & python client (#2423)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new ffde1bc [client] add properties to consumer for cpp & python client (#2423) ffde1bc is described below commit ffde1bcd45c73e94d5f41a05fc6fb2c6f31d3764 Author: Sijie Guo AuthorDate: Wed Aug 22 02:41:00 2018 -0700 [client] add properties to consumer for cpp & python client (#2423) * [client] add properties to consumer for cpp & python client ### Motivation This is a caught-up change to enable properties for consumer as java clients. ### Changes Enable properties on consumer for both cpp & python client ### Results Properties are added as metadata for CommandSubscribe. However there is no way to verify the consumer properties. so I didn't add any specific tests, just adding properties for both cpp and python clients in the tests, that should excerise the corresponding code path. * remove "make format" --- .../include/pulsar/ConsumerConfiguration.h | 34 ++ pulsar-client-cpp/lib/Commands.cc | 12 +++- pulsar-client-cpp/lib/Commands.h | 2 +- pulsar-client-cpp/lib/ConsumerConfiguration.cc | 32 pulsar-client-cpp/lib/ConsumerConfigurationImpl.h | 4 ++- pulsar-client-cpp/lib/ConsumerImpl.cc | 2 +- pulsar-client-cpp/python/pulsar/__init__.py| 11 ++- pulsar-client-cpp/python/src/config.cc | 1 + pulsar-client-cpp/python/test_consumer.py | 6 +++- pulsar-client-cpp/tests/BasicEndToEndTest.cc | 6 +++- 10 files changed, 103 insertions(+), 7 deletions(-) diff --git a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h index 36e5808..0687166 100644 --- a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h +++ b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h @@ -162,6 +162,40 @@ class ConsumerConfiguration { void setPatternAutoDiscoveryPeriod(int periodInSeconds); int getPatternAutoDiscoveryPeriod() const; +/** + * Check whether the message has a specific property attached. + * + * @param name the name of the property to check + * @return true if the message has the specified property + * @return false if the property is not defined + */ +bool hasProperty(const std::string& name) const; + +/** + * Get the value of a specific property + * + * @param name the name of the property + * @return the value of the property or null if the property was not defined + */ +const std::string& getProperty(const std::string& name) const; + +/** + * Get all the properties attached to this producer. + */ +std::map& getProperties() const; + +/** + * Sets a new property on a message. + * @param name the name of the property + * @param value the associated value + */ +ConsumerConfiguration& setProperty(const std::string& name, const std::string& value); + +/** + * Add all the properties in the provided map + */ +ConsumerConfiguration& setProperties(const std::map& properties); + friend class PulsarWrapper; private: diff --git a/pulsar-client-cpp/lib/Commands.cc b/pulsar-client-cpp/lib/Commands.cc index 8a1933b..8bd0128 100644 --- a/pulsar-client-cpp/lib/Commands.cc +++ b/pulsar-client-cpp/lib/Commands.cc @@ -20,6 +20,7 @@ #include "MessageImpl.h" #include "Version.h" #include "pulsar/MessageBuilder.h" +#include "PulsarApi.pb.h" #include "LogUtils.h" #include "Utils.h" #include "Url.h" @@ -27,6 +28,7 @@ #include #include +using namespace pulsar; namespace pulsar { using namespace pulsar::proto; @@ -185,7 +187,8 @@ SharedBuffer Commands::newConnect(const AuthenticationPtr& authentication, const SharedBuffer Commands::newSubscribe(const std::string& topic, const std::string& subscription, uint64_t consumerId, uint64_t requestId, CommandSubscribe_SubType subType, const std::string& consumerName, SubscriptionMode subscriptionMode, -Optional startMessageId, bool readCompacted) { +Optional startMessageId, bool readCompacted, +const std::map& metadata) { BaseCommand cmd; cmd.set_type(BaseCommand::SUBSCRIBE); CommandSubscribe* subscribe = cmd.mutable_subscribe(); @@ -206,6 +209,13 @@ SharedBuffer Commands::newSubscribe(const std::string& t
[incubator-pulsar] branch branch-2.1 updated: Fixed race condition during expansion of concurrent open hash maps (#2387)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 6146ca9 Fixed race condition during expansion of concurrent open hash maps (#2387) 6146ca9 is described below commit 6146ca9cc4c307bf4b19915ca6f13280efd7d428 Author: Matteo Merli AuthorDate: Fri Aug 17 08:47:30 2018 -0700 Fixed race condition during expansion of concurrent open hash maps (#2387) ### Motivation Porting same fix as https://github.com/apache/bookkeeper/pull/1607 to correct issue reported on https://github.com/apache/bookkeeper/issues/1606. There is a race condition in the concurrent open hash maps implementation. The race happens when the maps gets re-hashed after the expansion and the new arrays are substituting the old ones. The race itself is that a thread doing a `get()` on the map is first checking the current capacity of the map, uses that to get the bucket and then tries to do optimistic read of the value in that bucket. This assumes `capacity` update is visible only after the `values` array is already swapped, but that is not always the case in current code. ### Changes * Use `volatile` qualifier for `capacity` and `values` arrays to ensure ordering of memory read is respected by compiler * In rehashing, update `capacity` after `values` where it was not already the case --- .../util/collections/ConcurrentLongHashMap.java| 8 +-- .../util/collections/ConcurrentLongPairSet.java| 15 +++--- .../util/collections/ConcurrentOpenHashMap.java| 6 +-- .../util/collections/ConcurrentOpenHashSet.java| 6 +-- .../collections/ConcurrentLongHashMapTest.java | 57 -- 5 files changed, 72 insertions(+), 20 deletions(-) diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java index fcb0c10..60c24c0 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java @@ -30,7 +30,7 @@ import com.google.common.collect.Lists; /** * Map from long to an Object. - * + * * Provides similar methods as a ConcurrentMap with 2 differences: * * No boxing/unboxing from long -> Long @@ -187,10 +187,10 @@ public class ConcurrentLongHashMap { // A section is a portion of the hash map that is covered by a single @SuppressWarnings("serial") private static final class Section extends StampedLock { -private long[] keys; -private V[] values; +private volatile long[] keys; +private volatile V[] values; -private int capacity; +private volatile int capacity; private volatile int size; private int usedBuckets; private int resizeThreshold; diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java index 74d4314..4634b40 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java @@ -26,7 +26,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.StampedLock; -import java.util.function.Predicate; /** * Concurrent hash set where values are composed of pairs of longs. @@ -163,11 +162,11 @@ public class ConcurrentLongPairSet { /** * Removes all of the elements of this collection that satisfy the given predicate. - * + * * @param filter *a predicate which returns {@code true} for elements to be removed * @return {@code true} if any elements were removed - * + * * @return number of removed values */ public int removeIf(LongPairPredicate filter) { @@ -209,9 +208,9 @@ public class ConcurrentLongPairSet { @SuppressWarnings("serial") private static final class Section extends StampedLock { // Keys and values are stored interleaved in the table array -private long[] table; +private volatile long[] table; -private int capacity; +private volatile int capacity; private volatile int size; private int usedBuckets; private int resizeThreshold; @@ -449,9 +448,11 @@ public class ConcurrentLongPairSet { } } -capacity = newCapacity;
[incubator-pulsar] branch branch-2.1 updated: Kinesis-sink consider topic-name as partition-key if record key empty (#2372)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 703305b Kinesis-sink consider topic-name as partition-key if record key empty (#2372) 703305b is described below commit 703305b5426856bab7bab30a41e4f242e7782dc7 Author: Rajan Dhabalia AuthorDate: Tue Aug 14 15:41:45 2018 -0700 Kinesis-sink consider topic-name as partition-key if record key empty (#2372) --- .../kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java index 67de21a..cdfadde 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java @@ -114,7 +114,7 @@ public class KinesisSink implements Sink { record.getRecordSequence()); throw new IllegalStateException("kinesis queue has publish failure"); } -String partitionedKey = record.getKey().orElse(defaultPartitionedKey); +String partitionedKey = record.getKey().orElse(record.getTopicName().orElse(defaultPartitionedKey)); partitionedKey = partitionedKey.length() > maxPartitionedKeyLength ? partitionedKey.substring(0, maxPartitionedKeyLength - 1) : partitionedKey; // partitionedKey Length must be at least one, and at most 256
[incubator-pulsar] branch branch-2.1 updated: [compaction] make topic compaction works with partitioned topic (#2367)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new f97a8f7 [compaction] make topic compaction works with partitioned topic (#2367) f97a8f7 is described below commit f97a8f7d136ddc609d01ce54e20f45e9949b04df Author: Sijie Guo AuthorDate: Tue Aug 14 11:02:24 2018 -0700 [compaction] make topic compaction works with partitioned topic (#2367) * [compaction] make topic compaction works with partitioned topic ### Motivation Topic compaction doesn't work with partitioned topic. ### Changes - make `RawReaderImpl` and `ReaderImpl` return message with partition idx - make broker service `Consumer` deliver MessageIdData with partition idx - add an integration test to ensure compaction work with partitioned topic --- .../java/org/apache/pulsar/broker/service/Consumer.java | 9 +++-- .../org/apache/pulsar/client/impl/RawReaderImpl.java | 16 .../org/apache/pulsar/compaction/TwoPhaseCompactor.java | 8 .../PersistentDispatcherFailoverConsumerTest.java| 2 +- .../java/org/apache/pulsar/client/impl/ReaderImpl.java | 4 +++- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java index c3befa5..4c54018 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java @@ -73,6 +73,7 @@ public class Consumer { private final String appId; private AuthenticationDataSource authenticationData; private final String topicName; +private final int partitionIdx; private final InitialPosition subscriptionInitialPosition; private final long consumerId; @@ -119,6 +120,7 @@ public class Consumer { this.subscription = subscription; this.subType = subType; this.topicName = topicName; +this.partitionIdx = TopicName.getPartitionIndex(topicName); this.consumerId = consumerId; this.priorityLevel = priorityLevel; this.readCompacted = readCompacted; @@ -239,8 +241,11 @@ public class Consumer { Entry entry = entries.get(i); PositionImpl pos = (PositionImpl) entry.getPosition(); MessageIdData.Builder messageIdBuilder = MessageIdData.newBuilder(); -MessageIdData messageId = messageIdBuilder.setLedgerId(pos.getLedgerId()).setEntryId(pos.getEntryId()) -.build(); +MessageIdData messageId = messageIdBuilder +.setLedgerId(pos.getLedgerId()) +.setEntryId(pos.getEntryId()) +.setPartition(partitionIdx) +.build(); ByteBuf metadataAndPayload = entry.getDataBuffer(); // increment ref-count of data and release at the end of process: so, we can get chance to call entry.release diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java index e768c3e..ae1a4db 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java @@ -35,6 +35,7 @@ import org.apache.pulsar.client.api.SubscriptionType; import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData; import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType; import org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData; +import org.apache.pulsar.common.naming.TopicName; import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -102,8 +103,15 @@ public class RawReaderImpl implements RawReader { RawConsumerImpl(PulsarClientImpl client, ConsumerConfigurationData conf, CompletableFuture> consumerFuture) { -super(client, conf.getSingleTopic(), conf, client.externalExecutorProvider().getExecutor(), -1, -consumerFuture, SubscriptionMode.Durable, MessageId.earliest, Schema.BYTES); +super(client, +conf.getSingleTopic(), +conf, +client.externalExecutorProvider().getExecutor(), +TopicName.getPartitionIndex(conf.getSingleTopic()), +consumerFuture, +SubscriptionMode.Durable, +MessageId.earliest, +Schema.BYTES); incomingRawMessages = new GrowableArrayBlock
[incubator-pulsar] 02/02: [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git commit bcb3d3f6d15995b74c51d0507fece569361928da Author: Jia Zhai AuthorDate: Tue Aug 14 06:05:59 2018 +0800 [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311) * catch up receiver queue size support in multi topics consumer * add python config --- pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc | 5 - pulsar-client-cpp/python/pulsar/__init__.py| 13 pulsar-client-cpp/python/src/config.cc | 4 .../client/impl/MultiTopicsConsumerImpl.java | 23 +- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 7be197c..6750273 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -160,9 +160,12 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, config.setBrokerConsumerStatsCacheTimeInMs(conf_.getBrokerConsumerStatsCacheTimeInMs()); config.setMessageListener( boost::bind(&MultiTopicsConsumerImpl::messageReceived, shared_from_this(), _1, _2)); -config.setReceiverQueueSize(conf_.getReceiverQueueSize()); int numPartitions = partitionMetadata->getPartitions() >= 1 ? partitionMetadata->getPartitions() : 1; +// Apply total limit of receiver queue size across partitions +config.setReceiverQueueSize( +std::min(conf_.getReceiverQueueSize(), + (int)(conf_.getMaxTotalReceiverQueueSizeAcrossPartitions() / numPartitions))); Lock lock(mutex_); topicsPartitions_.insert(std::make_pair(topicName->toString(), numPartitions)); diff --git a/pulsar-client-cpp/python/pulsar/__init__.py b/pulsar-client-cpp/python/pulsar/__init__.py index 434fb07..f3b560b 100644 --- a/pulsar-client-cpp/python/pulsar/__init__.py +++ b/pulsar-client-cpp/python/pulsar/__init__.py @@ -315,6 +315,7 @@ class Client: send_timeout_millis=3, compression_type=CompressionType.NONE, max_pending_messages=1000, +max_pending_messages_across_partitions=5, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, @@ -352,6 +353,9 @@ class Client: * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. +* `max_pending_messages_across_partitions`: + Set the max size of the queue holding the messages pending to receive + an acknowledgment across partitions from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: @@ -364,6 +368,7 @@ class Client: _check_type(int, send_timeout_millis, 'send_timeout_millis') _check_type(CompressionType, compression_type, 'compression_type') _check_type(int, max_pending_messages, 'max_pending_messages') +_check_type(int, max_pending_messages_across_partitions, 'max_pending_messages_across_partitions') _check_type(bool, block_if_queue_full, 'block_if_queue_full') _check_type(bool, batching_enabled, 'batching_enabled') _check_type(int, batching_max_messages, 'batching_max_messages') @@ -374,6 +379,7 @@ class Client: conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) + conf.max_pending_messages_across_partitions(max_pending_messages_across_partitions) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) @@ -392,6 +398,7 @@ class Client: consumer_type=ConsumerType.Exclusive, message_listener=None, receiver_queue_size=1000, + max_total_receiver_queue_size_across_partitions=5, consumer_name=None, unacked_messages_timeout_ms=None, broker_consumer_stats_cache_time_ms=3, @@ -434,6 +441,9 @@ class Client: should not be interrupted when the consumer queue size is zero. The default value is 1000 messages and should work well for most use cases. +* `max_total_receiver_queue_size_across_partitions` + Set the max total receiver queue size across
[incubator-pulsar] branch branch-2.1 updated (b070cfd -> bcb3d3f)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a change to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git. from b070cfd [docker] Publish pulsar-all image to docker hub (#2361) new 068db59 Fix: function with multi-topic not acking on effectively-once (#2347) new bcb3d3f [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../api/PartitionedProducerConsumerTest.java | 9 ++--- .../org/apache/pulsar/io/PulsarSinkE2ETest.java| 10 +++--- pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc | 5 ++- pulsar-client-cpp/python/pulsar/__init__.py| 13 +++ pulsar-client-cpp/python/src/config.cc | 4 +++ .../client/impl/MultiTopicsConsumerImpl.java | 40 ++ .../pulsar/functions/source/PulsarSource.java | 24 +++-- 7 files changed, 67 insertions(+), 38 deletions(-)
[incubator-pulsar] 01/02: Fix: function with multi-topic not acking on effectively-once (#2347)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git commit 068db598adb529d1ae441084c280f0e776d2cbbc Author: Rajan Dhabalia AuthorDate: Mon Aug 13 14:19:46 2018 -0700 Fix: function with multi-topic not acking on effectively-once (#2347) ### Motivation `MultiTopicsConsumerImpl` doesn't support `acknowledgeCumulativeAsync` and therefore, function with multi-topic and `EFFECTIVELY_ONCE` processing is not acking message and failing `EFFECTIVELY_ONCE` behavior. ### Modifications Function should ack message for a specific topic consumer if `inputTopicConsumer` is multi-topic consumer. ### Result Function should able to ack messages for multi-topic consumer when processing-guarantee is `EFFECTIVELY_ONCE` --- .../api/PartitionedProducerConsumerTest.java | 9 ++-- .../org/apache/pulsar/io/PulsarSinkE2ETest.java| 10 + .../client/impl/MultiTopicsConsumerImpl.java | 17 +-- .../pulsar/functions/source/PulsarSource.java | 24 -- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java index a599532..ae7757d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java @@ -290,19 +290,14 @@ public class PartitionedProducerConsumerTest extends ProducerConsumerBase { } try { -producer = pulsarClient.newProducer().topic(topicName.toString()) -.enableBatching(false) -.messageRoutingMode(MessageRoutingMode.SinglePartition) -.create(); +producer = pulsarClient.newProducer().topic(topicName.toString()).enableBatching(false) + .messageRoutingMode(MessageRoutingMode.SinglePartition).create(); consumer = pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName("my-sub").subscribe(); producer.send("message1".getBytes()); producer.send("message2".getBytes()); /* Message msg1 = */ consumer.receive(); Message msg2 = consumer.receive(); consumer.acknowledgeCumulative(msg2); -Assert.fail("should fail since ack cumulative is not supported for partitioned topic"); -} catch (PulsarClientException e) { -Assert.assertTrue(e instanceof PulsarClientException.NotSupportedException); } finally { producer.close(); consumer.unsubscribe(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java index 5398bc9..2001981 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java @@ -62,6 +62,7 @@ import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.instance.JavaInstanceRunnable; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.Function.FunctionDetails; +import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; import org.apache.pulsar.functions.proto.Function.SinkSpec; import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; @@ -356,11 +357,11 @@ public class PulsarSinkE2ETest { retryStrategically((test) -> { try { SubscriptionStats subStats = admin.topics().getStats(sourceTopic).subscriptions.get(subscriptionName); -return subStats.unackedMessages == 0; +return subStats.unackedMessages == 0 && subStats.msgThroughputOut == totalMsgs; } catch (PulsarAdminException e) { return false; } -}, 5, 500); +}, 5, 200); FunctionRuntimeManager functionRuntimeManager = functionsWorkerService.getFunctionRuntimeManager(); functionRuntimeManager.updateRates(); @@ -400,11 +401,12 @@ public class PulsarSinkE2ETest { functionDetailsBuilder.setRuntime(FunctionDetails.Runtime.JAVA); functionDetailsBuilder.setParallelism(1); functionDetailsBuilder.setClassName(IdentityFunction.class.getName()); + functionDetailsBuilder.setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE); // set source spec // source spec classname should
[incubator-pulsar] branch branch-2.1 updated: [docker] Publish pulsar-all image to docker hub (#2361)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new b070cfd [docker] Publish pulsar-all image to docker hub (#2361) b070cfd is described below commit b070cfd76c3fb03fcff35f647b711695b3bc1993 Author: Sijie Guo AuthorDate: Mon Aug 13 09:07:44 2018 -0700 [docker] Publish pulsar-all image to docker hub (#2361) ### Motivation We have `pulsar-all` image since 2.1.0 release. However this image is not pushed to docker hub. ### Changes Publish `pulsar-all` image to docker hub. --- docker/publish.sh | 4 1 file changed, 4 insertions(+) diff --git a/docker/publish.sh b/docker/publish.sh index 411fd0d..f73bdac 100755 --- a/docker/publish.sh +++ b/docker/publish.sh @@ -63,18 +63,22 @@ set -x set -e docker tag pulsar:latest ${docker_registry_org}/pulsar:latest +docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:latest docker tag pulsar-grafana:latest ${docker_registry_org}/pulsar-grafana:latest docker tag pulsar-dashboard:latest ${docker_registry_org}/pulsar-dashboard:latest docker tag pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION +docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION docker tag pulsar-grafana:latest ${docker_registry_org}/pulsar-grafana:$MVN_VERSION docker tag pulsar-dashboard:latest ${docker_registry_org}/pulsar-dashboard:$MVN_VERSION # Push all images and tags docker push ${docker_registry_org}/pulsar:latest +docker push ${docker_registry_org}/pulsar-all:latest docker push ${docker_registry_org}/pulsar-grafana:latest docker push ${docker_registry_org}/pulsar-dashboard:latest docker push ${docker_registry_org}/pulsar:$MVN_VERSION +docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION docker push ${docker_registry_org}/pulsar-grafana:$MVN_VERSION docker push ${docker_registry_org}/pulsar-dashboard:$MVN_VERSION
[incubator-pulsar] branch branch-2.1 updated: add encoding for WebTargets parts (#2359)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 1ef9d0d add encoding for WebTargets parts (#2359) 1ef9d0d is described below commit 1ef9d0daa1492d948ba31625a09c0f1792709121 Author: Jia Zhai AuthorDate: Mon Aug 13 04:36:41 2018 +0800 add encoding for WebTargets parts (#2359) ### Motivation Fixes #2304 Issue #2304 reported that Pulsar admin commands will fail with some url style parts, ``` root@pulsar-admin:/pulsar# pulsar-admin namespaces grant-permission --actions produce,consume --role 'spiffe://developer/passport' core-platform/testing HTTP 404 Not Found Reason: HTTP 404 Not Found ``` This PR try to fix it. ### Modifications - use URLEncoder to encode webTarget parts. - add tests to verify ### Result all UT passed --- .../java/org/apache/pulsar/broker/admin/AdminApiTest.java | 6 ++ .../apache/pulsar/client/admin/internal/WebTargets.java| 14 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java index c9d973d..ae72737 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java @@ -620,11 +620,15 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest { // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list. } +// test with url style role. +admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1", +"spiffe://developer/passport-role", EnumSet.allOf(AuthAction.class)); admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1", "my-role", EnumSet.allOf(AuthAction.class)); Policies policies = new Policies(); policies.replication_clusters = Sets.newHashSet("test"); policies.bundles = Policies.defaultBundle(); + policies.auth_policies.namespace_auth.put("spiffe://developer/passport-role", EnumSet.allOf(AuthAction.class)); policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class)); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies); @@ -632,7 +636,9 @@ public class AdminApiTest extends MockedPulsarServiceBaseTest { assertEquals(admin.namespaces().getTopics("prop-xyz/ns1"), Lists.newArrayList()); +admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1", "spiffe://developer/passport-role"); admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1", "my-role"); + policies.auth_policies.namespace_auth.remove("spiffe://developer/passport-role"); policies.auth_policies.namespace_auth.remove("my-role"); assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies); diff --git a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java index 35b04f1..3de248c 100644 --- a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java +++ b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java @@ -18,14 +18,26 @@ */ package org.apache.pulsar.client.admin.internal; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import javax.ws.rs.client.WebTarget; +import lombok.extern.slf4j.Slf4j; +@Slf4j class WebTargets { static WebTarget addParts(WebTarget target, String[] parts) { if (parts != null && parts.length > 0) { for (String part : parts) { -target = target.path(part); +String encode; +try { +encode = URLEncoder.encode(part, StandardCharsets.UTF_8.toString()); +} catch (UnsupportedEncodingException e) { +log.error(String.format("%s is Unknown", StandardCharsets.UTF_8.toString()) + "exception - [{}]", e); +encode = part; +} +target = target.path(encode); } } return target;
[incubator-pulsar] branch branch-2.1 updated: Issue 1067: Problems with Partitioned Topics which name contains -partition-N (#2342)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 05488a0 Issue 1067: Problems with Partitioned Topics which name contains -partition-N (#2342) 05488a0 is described below commit 05488a0be6e27292344a093f694d2a5f48886601 Author: Sijie Guo AuthorDate: Fri Aug 10 19:40:44 2018 -0700 Issue 1067: Problems with Partitioned Topics which name contains -partition-N (#2342) * Issue 1067: Problems with Partitioned Topics which name contains -partition-N ### Motivation Fixes #1067. Someone accidentally created a partitioned topic in one of our cluster with a name which contains -partition-2. This raised all sorts of issues, it would seem that only one partition was created for this topic, but metadata exists saying that it has 10 partitions. ### Changes Disallow creating a partitioned topic contains '-partition-' in the name. * only validate partitioned topic name when create/delete/update partitioned topics * Changed to use a CONSTANT --- .../apache/pulsar/broker/admin/AdminResource.java | 9 +++ .../pulsar/broker/admin/v2/PersistentTopics.java | 42 + .../pulsar/broker/admin/AdminResourceTest.java | 69 ++ .../org/apache/pulsar/common/naming/TopicName.java | 2 +- 4 files changed, 108 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java index be1b7dd..de7dd0b 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java @@ -254,6 +254,15 @@ public abstract class AdminResource extends PulsarWebResource { this.topicName = TopicName.get(domain(), namespaceName, topic); } +protected void validatePartitionedTopicName(String tenant, String namespace, String encodedTopic) { +// first, it has to be a validate topic name +validateTopicName(tenant, namespace, encodedTopic); +// second, "-partition-" is not allowed +if (encodedTopic.contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) { +throw new RestException(Status.PRECONDITION_FAILED, "Partitioned Topic Name should not contain '-partition-'"); +} +} + @Deprecated protected void validateTopicName(String property, String cluster, String namespace, String encodedTopic) { String topic = Codec.decode(encodedTopic); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java index 6a246b8..771a057 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java @@ -124,12 +124,15 @@ public class PersistentTopics extends PersistentTopicsBase { @PUT @Path("/{tenant}/{namespace}/{topic}/partitions") @ApiOperation(value = "Create a partitioned topic.", notes = "It needs to be called before creating a producer on a partitioned topic.") -@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), -@ApiResponse(code = 409, message = "Partitioned topic already exist") }) +@ApiResponses(value = { +@ApiResponse(code = 403, message = "Don't have admin permission"), +@ApiResponse(code = 409, message = "Partitioned topic already exist"), +@ApiResponse(code = 412, message = "Partitioned topic name is invalid") +}) public void createPartitionedTopic(@PathParam("tenant") String tenant, @PathParam("namespace") String namespace, @PathParam("topic") @Encoded String encodedTopic, int numPartitions, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { -validateTopicName(tenant, namespace, encodedTopic); +validatePartitionedTopicName(tenant, namespace, encodedTopic); internalCreatePartitionedTopic(numPartitions, authoritative); } @@ -151,18 +154,25 @@ public class PersistentTopics extends PersistentTopicsBase { @POST @Path("/{tenant}/{namespace}/{topic}/partitions") @ApiOperation(value = "Increment partitons of an existing partitioned topic.", notes = "It only increments partitions of existing non-global partitioned-topic") -@ApiResponses(value
[incubator-pulsar] branch branch-2.1 updated: Remove snapshot-topic ref (#2355)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new f2e6ea7 Remove snapshot-topic ref (#2355) f2e6ea7 is described below commit f2e6ea73aab47a5c977d44c9deb3f99a07e53a94 Author: Rajan Dhabalia AuthorDate: Thu Aug 9 23:43:02 2018 -0700 Remove snapshot-topic ref (#2355) ### Motivation remove unused config `functionMetadataSnapshotsTopicPath` --- conf/functions_worker.yml| 1 - .../src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java | 1 - 2 files changed, 2 deletions(-) diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml index 4194337..8c0d903 100644 --- a/conf/functions_worker.yml +++ b/conf/functions_worker.yml @@ -24,7 +24,6 @@ workerPort: 6750 connectorsDirectory: ./connectors functionMetadataTopicName: metadata -functionMetadataSnapshotsTopicPath: snapshots clusterCoordinationTopicName: coordinate pulsarFunctionsNamespace: public/functions pulsarFunctionsCluster: standalone diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java index eda9b15..348d245 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java @@ -58,7 +58,6 @@ public class WorkerConfig implements Serializable, PulsarConfiguration { private String pulsarServiceUrl; private String pulsarWebServiceUrl; private String clusterCoordinationTopicName; -private String functionMetadataSnapshotsTopicPath; private String pulsarFunctionsNamespace; private String pulsarFunctionsCluster; private int numFunctionPackageReplicas;
[incubator-pulsar] branch branch-2.1 updated: Standalone bookie should always advertise 'localhost' when running in docker container (#2425)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new cb88ae0 Standalone bookie should always advertise 'localhost' when running in docker container (#2425) cb88ae0 is described below commit cb88ae029d748017d88cdc7583b5dca1eb3a5b1c Author: Matteo Merli AuthorDate: Fri Aug 24 12:55:12 2018 -0700 Standalone bookie should always advertise 'localhost' when running in docker container (#2425) * Standalone bookie should always advertise 'localhost' when running in docker container * Fixed advertised address coming from config file --- pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index 3e44f76..cf07d8c 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java @@ -253,7 +253,8 @@ public class PulsarStandalone implements AutoCloseable { if (!this.isOnlyBroker()) { // Start LocalBookKeeper bkEnsemble = new LocalBookkeeperEnsemble( -this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(), this.getBkDir(), this.isWipeData(), config.getAdvertisedAddress()); +this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(), +this.getBkDir(), this.isWipeData(), "127.0.0.1"); bkEnsemble.startStandalone(!this.isNoStreamStorage()); } @@ -352,7 +353,7 @@ public class PulsarStandalone implements AutoCloseable { log.info(e.getMessage()); } } - + /** this methods gets a buidler to use to build and an embedded pulsar instance ` * i.e. *
[incubator-pulsar] branch branch-2.1 updated: Set correct exception if function runnable fails with Error (#2353)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 7a0ca5a Set correct exception if function runnable fails with Error (#2353) 7a0ca5a is described below commit 7a0ca5ae1be8d03828ef224350576755fa5f8af9 Author: Rajan Dhabalia AuthorDate: Mon Aug 13 21:20:02 2018 -0700 Set correct exception if function runnable fails with Error (#2353) * Set correct exception if function runnable fails with Error * fix throwable set --- .../org/apache/pulsar/functions/instance/JavaInstanceRunnable.java | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java index b64224f..68df84a 100644 --- a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java +++ b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java @@ -181,8 +181,10 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { */ @Override public void run() { +String functionName = null; try { ContextImpl contextImpl = setupContext(); +functionName = String.format("%s-%s", contextImpl.getTenant(), contextImpl.getFunctionName()); javaInstance = setupJavaInstance(contextImpl); if (null != stateTable) { StateContextImpl stateContext = new StateContextImpl(stateTable); @@ -229,8 +231,8 @@ public class JavaInstanceRunnable implements AutoCloseable, Runnable { } } } catch (Throwable t) { -log.error("Uncaught exception in Java Instance", t); -deathException = (Exception) t; +log.error("[{}] Uncaught exception in Java Instance", functionName, t); +deathException = t; return; } finally { log.info("Closing instance");
[incubator-pulsar] branch branch-2.1 updated: Fix NPE when splitting and unloading bundle (#2348)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 4db7f38 Fix NPE when splitting and unloading bundle (#2348) 4db7f38 is described below commit 4db7f38cfaf96c5a2f524a780b854b8df1878b57 Author: massakam AuthorDate: Fri Aug 10 13:38:52 2018 +0900 Fix NPE when splitting and unloading bundle (#2348) --- .../pulsar/broker/namespace/NamespaceService.java | 6 ++-- .../broker/namespace/NamespaceServiceTest.java | 42 ++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java index bf44519..d9a1ccf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java @@ -593,7 +593,7 @@ public class NamespaceService { boolean unload, AtomicInteger counter, CompletableFuture unloadFuture) { -CompletableFuture updateFuture = new CompletableFuture<>(); +CompletableFuture> updateFuture = new CompletableFuture<>(); final Pair> splittedBundles = bundleFactory.splitBundles(bundle, 2 /* by default split into 2 */); @@ -622,7 +622,7 @@ public class NamespaceService { // namespace bundle bundleFactory.invalidateBundleCache(bundle.getNamespaceObject()); -updateFuture.complete(splittedBundles.getLeft()); +updateFuture.complete(splittedBundles.getRight()); } else if (rc == Code.BADVERSION.intValue()) { KeeperException keeperException = KeeperException.create(KeeperException.Code.get(rc)); String msg = format("failed to update namespace policies [%s], NamespaceBundle: %s " + @@ -680,7 +680,7 @@ public class NamespaceService { if (unload) { // unload new split bundles -r.getBundles().forEach(splitBundle -> { +r.forEach(splitBundle -> { try { unloadNamespaceBundle(splitBundle); } catch (Exception e) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java index a2ef397..2c51e16 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java @@ -429,6 +429,48 @@ public class NamespaceServiceTest extends BrokerTestBase { } +@Test +public void testRemoveOwnershipAndSplitBundle() throws Exception { +OwnershipCache ownershipCache = spy(pulsar.getNamespaceService().getOwnershipCache()); + doNothing().when(ownershipCache).disableOwnership(any(NamespaceBundle.class)); + +Field ownership = NamespaceService.class.getDeclaredField("ownershipCache"); +ownership.setAccessible(true); +ownership.set(pulsar.getNamespaceService(), ownershipCache); + +NamespaceService namespaceService = pulsar.getNamespaceService(); +NamespaceName nsname = NamespaceName.get("pulsar/global/ns1"); +TopicName topicName = TopicName.get("persistent://pulsar/global/ns1/topic-1"); +NamespaceBundles bundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname); +NamespaceBundle originalBundle = bundles.findBundle(topicName); + +CompletableFuture result1 = namespaceService.splitAndOwnBundle(originalBundle, false); +try { +result1.get(); +} catch (Exception e) { +fail("split bundle faild", e); +} + +NamespaceBundles updatedNsBundles = namespaceService.getNamespaceBundleFactory().getBundles(nsname); +assertNotNull(updatedNsBundles); +NamespaceBundle splittedBundle = updatedNsBundles.findBundle(topicName); + +updatedNsBundles.getBundles().stream().filter(bundle -> !bundle.equals(splittedBundle)).forEach(bundle -> { +try { +ownershipCache.removeOwnership(bundle).get(); +} catch (Exception e) { +fail("faild to remove ownership", e); +} +}); + +Completab
[incubator-pulsar] branch branch-2.1 updated: Issue 2272: Fix git-commit-id-plugin (#2327)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 046e500 Issue 2272: Fix git-commit-id-plugin (#2327) 046e500 is described below commit 046e5007fec67ab89f2093d2ee85db0b09391d58 Author: Sijie Guo AuthorDate: Tue Aug 7 08:06:05 2018 -0700 Issue 2272: Fix git-commit-id-plugin (#2327) ### Motivation Fixes #2272. We are using git-commit-id plugin for generating a properties file containing all the build information. However you have to skip git-commit-id-plugin when the source code is not in a git repo. ### Changes add `false` to git-commit-plugin --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0a2fae5..e04cce4 100644 --- a/pom.xml +++ b/pom.xml @@ -1137,6 +1137,7 @@ flexible messaging model and an intuitive client API. false true ${project.build.outputDirectory}/${project.artifactId}-git.properties + false false properties
[incubator-pulsar] branch branch-2.1 updated: Issue #1452: remove reachedEndOfTopic in addConsumer (#2301)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 0cc5e81 Issue #1452: remove reachedEndOfTopic in addConsumer (#2301) 0cc5e81 is described below commit 0cc5e81fcc5ac3859ee463556e6c1ed7e05d29a3 Author: Jia Zhai AuthorDate: Mon Aug 6 07:49:42 2018 +0800 Issue #1452: remove reachedEndOfTopic in addConsumer (#2301) ### Motivation Fixes #1452 In issue #1452 , reachedEndOfTopic was called twice if a topic has been terminated before subscription. It may be better to call `reachedEndOfTopic`, when real read/ack happened to the subscription, so delete the calling in `addConsumer` to avoid dup calling. ### Modifications remove dup calling in `addConsumer` . add related ut. ### Result Expected all ut passed. --- .../service/persistent/PersistentSubscription.java | 5 --- .../client/api/SimpleProducerConsumerTest.java | 42 -- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java index 8a77bfa..0842742 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java @@ -103,11 +103,6 @@ public class PersistentSubscription implements Subscription { throw new SubscriptionFencedException("Subscription is fenced"); } -if (topic.getManagedLedger().isTerminated() && cursor.getNumberOfEntriesInBacklog() == 0) { -// Immediately notify the consumer that there are no more available messages -consumer.reachedEndOfTopic(); -} - if (dispatcher == null || !dispatcher.isConsumerConnected()) { switch (consumer.subType()) { case Exclusive: diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java index 79166a1..61bdad0 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java @@ -2434,7 +2434,7 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase { Producer producer = pulsarClient.newProducer().topic("persistent://my-property/my-ns/myrsa-topic1") .addEncryptionKey(encryptionKeyName).compressionType(CompressionType.LZ4) .cryptoKeyReader(new EncKeyReader()).create(); - + Consumer consumer = pulsarClient.newConsumer().topicsPattern("persistent://my-property/my-ns/myrsa-topic1") .subscriptionName("my-subscriber-name").cryptoFailureAction(ConsumerCryptoFailureAction.CONSUME) .subscribe(); @@ -2450,7 +2450,7 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase { consumer.close(); log.info("-- Exiting {} test --", methodName); } - + private String decryptMessage(TopicMessageImpl msg, String encryptionKeyName, CryptoKeyReader reader) throws Exception { Optional ctx = msg.getEncryptionCtx(); @@ -2624,4 +2624,42 @@ public class SimpleProducerConsumerTest extends ProducerConsumerBase { log.info("-- Exiting {} test --", methodName); } +// Issue 1452: https://github.com/apache/incubator-pulsar/issues/1452 +// reachedEndOfTopic should be called only once if a topic has been terminated before subscription +@Test +public void testReachedEndOfTopic() throws Exception +{ +String topicName = "persistent://my-property/my-ns/testReachedEndOfTopic"; +Producer producer = pulsarClient.newProducer() +.topic(topicName) +.enableBatching(false).create(); +producer.close(); + +admin.topics().terminateTopicAsync(topicName).get(); + +CountDownLatch latch = new CountDownLatch(2); +Consumer consumer = pulsarClient.newConsumer() +.topic(topicName) +.subscriptionName("my-subscriber-name") +.messageListener(new MessageListener() +{ +@Override +public void reachedEndOfTopic(Consumer consumer) +{ +log.info("called reachedEndOfTopic {}", methodName); +latch.countDown(); +
[incubator-pulsar] branch branch-2.1 updated: Ignore unknown Properties from config file when parsing (#2336)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 5ac1322 Ignore unknown Properties from config file when parsing (#2336) 5ac1322 is described below commit 5ac1322a7e919217b3979ff1a13c9a7e6ea34541 Author: Sanjeev Kulkarni AuthorDate: Tue Aug 7 20:41:13 2018 -0700 Ignore unknown Properties from config file when parsing (#2336) ### Motivation As Functions evolve, we will be adding new properties/fields to the FunctionConfig. There will be fields present in the config files that won't make sense for previous admin tool installation. This especially happens when submitting pulsar functions using newer client to run over kubernetes with an older installation of pulsar. This pr will mark the unknown fields to be ignored to get over that problem --- .../src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java index 0cd922e..e38249c 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.utils; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -48,6 +49,7 @@ import java.util.Map; @EqualsAndHashCode @ToString @isValidFunctionConfig +@JsonIgnoreProperties(ignoreUnknown = true) public class FunctionConfig { public enum ProcessingGuarantees {
[incubator-pulsar] branch branch-2.1 updated: Cpp client: add PatternMultiTopicsConsumerImpl to support regex subscribe (#2219)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new b23134c Cpp client: add PatternMultiTopicsConsumerImpl to support regex subscribe (#2219) b23134c is described below commit b23134c422456f45ed14c3c49de53eb8889e45fc Author: Jia Zhai AuthorDate: Sat Aug 4 14:56:56 2018 +0800 Cpp client: add PatternMultiTopicsConsumerImpl to support regex subscribe (#2219) In PR #1279 and #1298 we added regex based subscription. This is a catch up work to add `PatternMultiTopicsConsumerImpl` in cpp client. --- pulsar-client-cpp/include/pulsar/Client.h | 16 ++ .../include/pulsar/ConsumerConfiguration.h | 10 + pulsar-client-cpp/lib/BinaryProtoLookupService.cc | 42 pulsar-client-cpp/lib/BinaryProtoLookupService.h | 9 + pulsar-client-cpp/lib/Client.cc| 24 ++ pulsar-client-cpp/lib/ClientConnection.cc | 86 ++- pulsar-client-cpp/lib/ClientConnection.h | 13 +- pulsar-client-cpp/lib/ClientImpl.cc| 58 + pulsar-client-cpp/lib/ClientImpl.h | 8 + pulsar-client-cpp/lib/Commands.cc | 18 ++ pulsar-client-cpp/lib/Commands.h | 1 + pulsar-client-cpp/lib/ConsumerConfiguration.cc | 6 + pulsar-client-cpp/lib/ConsumerConfigurationImpl.h | 4 +- pulsar-client-cpp/lib/HTTPLookupService.cc | 112 +++-- pulsar-client-cpp/lib/HTTPLookupService.h | 9 +- pulsar-client-cpp/lib/LookupService.h | 11 + pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h| 2 +- .../lib/PatternMultiTopicsConsumerImpl.cc | 235 +++ .../lib/PatternMultiTopicsConsumerImpl.h | 76 ++ pulsar-client-cpp/tests/BasicEndToEndTest.cc | 257 + pulsar-client-cpp/tests/BinaryLookupServiceTest.cc | 67 ++ 21 files changed, 1032 insertions(+), 32 deletions(-) diff --git a/pulsar-client-cpp/include/pulsar/Client.h b/pulsar-client-cpp/include/pulsar/Client.h index 6a9e487..1913851 100644 --- a/pulsar-client-cpp/include/pulsar/Client.h +++ b/pulsar-client-cpp/include/pulsar/Client.h @@ -99,6 +99,9 @@ class Client { void subscribeAsync(const std::string& topic, const std::string& consumerName, const ConsumerConfiguration& conf, SubscribeCallback callback); +/** + * subscribe for multiple topics under the same namespace. + */ Result subscribe(const std::vector& topics, const std::string& subscriptionName, Consumer& consumer); Result subscribe(const std::vector& topics, const std::string& subscriptionName, @@ -109,6 +112,19 @@ class Client { const ConsumerConfiguration& conf, SubscribeCallback callback); /** + * subscribe for multiple topics, which match given regexPattern, under the same namespace. + */ +Result subscribeWithRegex(const std::string& regexPattern, const std::string& consumerName, + Consumer& consumer); +Result subscribeWithRegex(const std::string& regexPattern, const std::string& consumerName, + const ConsumerConfiguration& conf, Consumer& consumer); + +void subscribeWithRegexAsync(const std::string& regexPattern, const std::string& consumerName, + SubscribeCallback callback); +void subscribeWithRegexAsync(const std::string& regexPattern, const std::string& consumerName, + const ConsumerConfiguration& conf, SubscribeCallback callback); + +/** * Create a topic reader with given {@code ReaderConfiguration} for reading messages from the specified * topic. * diff --git a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h index c9584e3..36e5808 100644 --- a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h +++ b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h @@ -152,6 +152,16 @@ class ConsumerConfiguration { bool isReadCompacted() const; void setReadCompacted(bool compacted); +/** + * Set the time duration in minutes, for which the PatternMultiTopicsConsumer will do a pattern auto + * discovery. + * The default value is 60 seconds. less than 0 will disable auto discovery. + * + * @param periodInSeconds period in seconds to do an auto discovery + */ +void setPatternAutoDiscoveryPeriod(int periodInSeconds); +int getPatternAutoDiscoveryPeriod() const; + friend class PulsarWrapper; private: diff --git a/pulsar-client-cpp/lib/BinaryProtoLookupService.cc b/p
[incubator-pulsar] branch branch-2.1 updated: handle subscription-already-exist exception on partitioned-topic for create-sub admin-api (#2269)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new d15cd7b handle subscription-already-exist exception on partitioned-topic for create-sub admin-api (#2269) d15cd7b is described below commit d15cd7bc850cb85f8e3038dbb62c98429e542787 Author: Rajan Dhabalia AuthorDate: Tue Jul 31 11:33:02 2018 -0700 handle subscription-already-exist exception on partitioned-topic for create-sub admin-api (#2269) --- .../broker/admin/impl/PersistentTopicsBase.java| 35 +- .../broker/admin/CreateSubscriptionTest.java | 26 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index cf8355f..f85c617 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -38,9 +38,11 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import javax.ws.rs.WebApplicationException; @@ -829,15 +831,34 @@ public class PersistentTopicsBase extends AdminResource { try { if (partitionMetadata.partitions > 0) { // Create the subscription on each partition -List> futures = Lists.newArrayList(); PulsarAdmin admin = pulsar().getAdminClient(); +CountDownLatch latch = new CountDownLatch(partitionMetadata.partitions); +AtomicReference exception = new AtomicReference<>(); +AtomicInteger failureCount = new AtomicInteger(0); + for (int i = 0; i < partitionMetadata.partitions; i++) { - futures.add(admin.topics().createSubscriptionAsync(topicName.getPartition(i).toString(), -subscriptionName, messageId)); +admin.persistentTopics() + .createSubscriptionAsync(topicName.getPartition(i).toString(), subscriptionName, messageId) +.handle((result, ex) -> { +if (ex != null) { +int c = failureCount.incrementAndGet(); +// fail the operation on unknown exception or if all the partitioned failed due to +// subscription-already-exist +if (c == partitionMetadata.partitions +|| !(ex instanceof PulsarAdminException.ConflictException)) { +exception.set(ex); +} +} +latch.countDown(); +return null; +}); } -FutureUtil.waitForAll(futures).join(); +latch.await(); +if (exception.get() != null) { +throw exception.get(); +} } else { validateAdminOperationOnTopic(authoritative); @@ -850,10 +871,10 @@ public class PersistentTopicsBase extends AdminResource { PersistentSubscription subscription = (PersistentSubscription) topic .createSubscription(subscriptionName, InitialPosition.Latest).get(); subscription.resetCursor(PositionImpl.get(messageId.getLedgerId(), messageId.getEntryId())).get(); -log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(), -topicName, subscriptionName, messageId); +log.info("[{}][{}] Successfully created subscription {} at message id {}", clientAppId(), topicName, +subscriptionName, messageId); } -} catch (Exception e) { +} catch (Throwable e) { Throwable t = e.getCause(); log.warn("[{}] [{}] Failed to create subscription {} at message id {}", clientAppId(), topicName, subscriptionName, messageId, e); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-bro
[incubator-pulsar] branch branch-2.1 updated: kinesis-sink: manage msg ordering for publish callback failure (#2285)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new f449006 kinesis-sink: manage msg ordering for publish callback failure (#2285) f449006 is described below commit f4490067739ad82023df4bc7721c16dbb7245538 Author: Rajan Dhabalia AuthorDate: Thu Aug 2 12:35:48 2018 -0700 kinesis-sink: manage msg ordering for publish callback failure (#2285) --- .../org/apache/pulsar/io/kinesis/KinesisSink.java | 49 ++ .../pulsar/io/kinesis/KinesisSinkConfig.java | 1 + 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java index dc70b98..67de21a 100644 --- a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java +++ b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java @@ -44,6 +44,7 @@ import java.lang.reflect.Constructor; import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -89,6 +90,12 @@ public class KinesisSink implements Sink { private static final String defaultPartitionedKey = "default"; private static final int maxPartitionedKeyLength = 256; private SinkContext sinkContext; +// +private static final int FALSE = 0; +private static final int TRUE = 1; +private volatile int previousPublishFailed = FALSE; +private static final AtomicIntegerFieldUpdater IS_PUBLISH_FAILED = +AtomicIntegerFieldUpdater.newUpdater(KinesisSink.class, "previousPublishFailed"); public static final String ACCESS_KEY_NAME = "accessKey"; public static final String SECRET_KEY_NAME = "secretKey"; @@ -101,6 +108,12 @@ public class KinesisSink implements Sink { @Override public void write(Record record) throws Exception { +// kpl-thread captures publish-failure. fail the publish on main pulsar-io-thread to maintain the ordering +if (kinesisSinkConfig.isRetainOrdering() && previousPublishFailed == TRUE) { +LOG.warn("Skip acking message to retain ordering with previous failed message {}-{}", this.streamName, +record.getRecordSequence()); +throw new IllegalStateException("kinesis queue has publish failure"); +} String partitionedKey = record.getKey().orElse(defaultPartitionedKey); partitionedKey = partitionedKey.length() > maxPartitionedKeyLength ? partitionedKey.substring(0, maxPartitionedKeyLength - 1) @@ -109,7 +122,7 @@ public class KinesisSink implements Sink { ListenableFuture addRecordResult = kinesisProducer.addUserRecord(this.streamName, partitionedKey, data); addCallback(addRecordResult, -ProducerSendCallback.create(this.streamName, record, System.nanoTime(), sinkContext), directExecutor()); +ProducerSendCallback.create(this, record, System.nanoTime()), directExecutor()); if (sinkContext != null) { sinkContext.recordMetric(METRICS_TOTAL_INCOMING, 1); sinkContext.recordMetric(METRICS_TOTAL_INCOMING_BYTES, data.array().length); @@ -151,6 +164,7 @@ public class KinesisSink implements Sink { this.streamName = kinesisSinkConfig.getAwsKinesisStreamName(); this.kinesisProducer = new KinesisProducer(kinesisConfig); +IS_PUBLISH_FAILED.set(this, FALSE); LOG.info("Kinesis sink started. {}", (ReflectionToStringBuilder.toString(kinesisConfig, ToStringStyle.SHORT_PREFIX_STYLE))); } @@ -167,30 +181,26 @@ public class KinesisSink implements Sink { private static final class ProducerSendCallback implements FutureCallback { private Record resultContext; -private String streamName; private long startTime = 0; private final Handle recyclerHandle; -private SinkContext sinkContext; +private KinesisSink kinesisSink; private ProducerSendCallback(Handle recyclerHandle) { this.recyclerHandle = recyclerHandle; } -static ProducerSendCallback create(String streamName, Record resultContext, long startTime, -SinkContext sinkContext) { +static ProducerSendCallback create(KinesisSink kinesisSink, Record resultContext, long startTime) { ProducerSendCallback sendCallback = RECYCLER.get(); sendCallback
[incubator-pulsar] branch branch-2.1 updated: Derive source/sink arg-class name from function-class for file-url (#2258)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 752f1b4 Derive source/sink arg-class name from function-class for file-url (#2258) 752f1b4 is described below commit 752f1b44359bc83f0f11ddd2f0cb18f4f70b712c Author: Rajan Dhabalia AuthorDate: Tue Jul 31 11:32:49 2018 -0700 Derive source/sink arg-class name from function-class for file-url (#2258) * Derive source/sink arg-class name from functio-class for file-url archive * fix set type-arg if src/sink arg-class is not set * add unit-test --- .../org/apache/pulsar/io/PulsarSinkE2ETest.java| 99 -- .../org/apache/pulsar/functions/utils/Utils.java | 15 ++-- .../functions/worker/rest/api/FunctionsImpl.java | 42 ++--- 3 files changed, 113 insertions(+), 43 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java index 16f1a76..5398bc9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java @@ -21,6 +21,10 @@ package org.apache.pulsar.io; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically; import static org.mockito.Mockito.spy; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertTrue; import java.io.File; import java.lang.reflect.Method; @@ -62,7 +66,6 @@ import org.apache.pulsar.functions.proto.Function.SinkSpec; import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatusList; -import org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData; import org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData.DataDigest; import org.apache.pulsar.functions.sink.PulsarSink; import org.apache.pulsar.functions.utils.Reflections; @@ -70,11 +73,9 @@ import org.apache.pulsar.functions.utils.Utils; import org.apache.pulsar.functions.worker.FunctionRuntimeManager; import org.apache.pulsar.functions.worker.WorkerConfig; import org.apache.pulsar.functions.worker.WorkerService; -import org.apache.pulsar.functions.worker.rest.WorkerServer; import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.DataProvider; @@ -124,7 +125,7 @@ public class PulsarSinkE2ETest { public Object[][] validRoleName() { return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } }; } - + @BeforeMethod void setup(Method method) throws Exception { @@ -147,7 +148,6 @@ public class PulsarSinkE2ETest { config.setBrokerServicePortTls(brokerServiceTlsPort); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); - Set providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); config.setAuthenticationEnabled(true); @@ -156,7 +156,6 @@ public class PulsarSinkE2ETest { config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); config.setTlsAllowInsecureConnection(true); - functionsWorkerService = createPulsarFunctionWorker(config); urlTls = new URL(brokerServiceUrl); @@ -190,12 +189,12 @@ public class PulsarSinkE2ETest { workerConfig.getClientAuthenticationParameters()); } pulsarClient = clientBuilder.build(); - + TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); - + Thread.sleep(100); } @@ -237,7 +236,7 @@ public class PulsarSinkE2ETest { workerConfig.setUseTls(true); workerConfig.setTlsAllowInsecureConnection(true); workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH); - + workerConfig.setAuthenticationEnabled(true); workerConfig.setAuthorizationEnabled(true); @@ -285,7 +284,7 @@ public class PulsarSinkE2ETest { } }, 5, 150); // validate pulsar sink consume
[incubator-pulsar] branch branch-2.1 updated: Fix: function-cli avoid arg-type check and class-loading for file url (#2257)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 6a05513 Fix: function-cli avoid arg-type check and class-loading for file url (#2257) 6a05513 is described below commit 6a05513f8cbcb5f9414af034a8f1e14754b94045 Author: Rajan Dhabalia AuthorDate: Mon Jul 30 11:31:31 2018 -0700 Fix: function-cli avoid arg-type check and class-loading for file url (#2257) --- .../src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java | 9 - .../java/org/apache/pulsar/functions/utils/FunctionConfig.java | 1 - .../apache/pulsar/functions/utils/validation/ValidatorImpls.java | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java index 2a38c81..15aa086 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java @@ -545,7 +545,14 @@ public class CmdFunctions extends CmdBase { Class[] typeArgs = null; if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { -typeArgs = Utils.getFunctionTypes(functionConfig); +if (functionConfig.getJar().startsWith(Utils.FILE)) { +// server derives the arg-type by loading a class +if (isBlank(functionConfig.getClassName())) { +throw new ParameterException("Class-name must be present for jar with file-url"); +} +} else { +typeArgs = Utils.getFunctionTypes(functionConfig); +} } FunctionDetails.Builder functionDetailsBuilder = FunctionDetails.newBuilder(); diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java index 3140f95..0cd922e 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java @@ -69,7 +69,6 @@ public class FunctionConfig { @NotNull private String name; @NotNull -@isImplementationOfClasses(implementsClasses = {Function.class, java.util.function.Function.class}) private String className; @isListEntryCustom(entryValidatorClasses = {ValidatorImpls.TopicNameValidator.class}) private Collection inputs; diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java index 297a2f2..925f7bc 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java @@ -502,7 +502,9 @@ public class ValidatorImpls { FunctionConfig functionConfig = (FunctionConfig) o; doCommonChecks(functionConfig); if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) { -doJavaChecks(functionConfig, name); +if (!functionConfig.getJar().startsWith(Utils.FILE)) { +doJavaChecks(functionConfig, name); +} } else { doPythonChecks(functionConfig, name); }
[incubator-pulsar] branch branch-2.1 updated: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 (#2182)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 406f8fa Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 (#2182) 406f8fa is described below commit 406f8fa4de6d02839add4b50d31ef4d630b4c857 Author: Sijie Guo AuthorDate: Wed Jul 25 23:02:25 2018 -0700 Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 (#2182) * Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 ### Motivation Fixes #2110. infinite retention is not correctly propagated. ### Changes Fix the command tool to set infinite retention correctly. Update command line description and website to reflect the fact retention size less than 1MB is treated as no retention. --- .../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 -- site/_data/cli/pulsar-admin.yaml | 2 +- .../pulsar/tests/integration/cli/CLITest.java | 38 ++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java index 0eaf81b..4f898a5 100644 --- a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java +++ b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java @@ -342,7 +342,7 @@ public class CmdNamespaces extends CmdBase { private String retentionTimeStr; @Parameter(names = { "--size", "-s" }, description = "Retention size limit (eg: 10M, 16G, 3T). " -+ "0 means no retention and -1 means infinite size retention", required = true) ++ "0 or less than 1MB means no retention and -1 means infinite size retention", required = true) private String limitStr; @Override @@ -351,8 +351,12 @@ public class CmdNamespaces extends CmdBase { long sizeLimit = validateSizeString(limitStr); int retentionTimeInMin = validateTimeString(retentionTimeStr); -sizeLimit = sizeLimit / (1024 * 1024); -int retentionSizeInMB = (int) sizeLimit; +int retentionSizeInMB; +if (sizeLimit != -1) { +retentionSizeInMB = (int) (sizeLimit / (1024 * 1024)); +} else { +retentionSizeInMB = -1; +} admin.namespaces().setRetention(namespace, new RetentionPolicies(retentionTimeInMin, retentionSizeInMB)); } } diff --git a/site/_data/cli/pulsar-admin.yaml b/site/_data/cli/pulsar-admin.yaml index c0d483f..94ec02c 100644 --- a/site/_data/cli/pulsar-admin.yaml +++ b/site/_data/cli/pulsar-admin.yaml @@ -431,7 +431,7 @@ commands: argument: tenant/cluster/namespace options: - flags: -s, --size - description: The retention size limits (for example `10M`, `16G` or `3T`). 0 means no retention and -1 means infinite size retention + description: The retention size limits (for example `10M`, `16G` or `3T`). 0 or less than 1MB means no retention and -1 means infinite size retention - flags: -t, --time description: "The retention time in minutes, hours, days, or weeks. Examples: `100m`, `13h`, `2d`, `5w`. 0 means no retention and -1 means infinite time retention" - name: unload diff --git a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java index 8d9ed20..0f2163a 100644 --- a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java +++ b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java @@ -164,4 +164,42 @@ public class CLITest extends PulsarClusterTestBase { ); assertTrue(result.getStderr().contains("Reason: HTTP 404 Not Found")); } + +@Test +public void testSetInfiniteRetention() throws Exception { +ContainerExecResult result; + +String namespace = "get-and-set-retention" + randomName(8); +pulsarCluster.createNamespace(namespace); + +String[] setCommand = { +"namespaces", "set-retention", "public/" + namespace, +"--size", "-1", +"--time", "-1" +}; + +result = pulsarCluster.runAdminCommandOnAnyBroker(setCommand); +assertEquals(0, result.getExitCode()); +assertTrue( +result.getStdout().isEmpty(), +result.getStdout()
[incubator-pulsar] branch branch-2.1 updated: add function-name reference with function failure log (#2246)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new cab977e add function-name reference with function failure log (#2246) cab977e is described below commit cab977e785d9db6e8a96f304585b8858f70c97b0 Author: Rajan Dhabalia AuthorDate: Thu Jul 26 23:36:06 2018 -0700 add function-name reference with function failure log (#2246) --- .../main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java index 8329ed7..6abb5fb 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java @@ -82,7 +82,9 @@ public class RuntimeSpawner implements AutoCloseable { @Override public void run() { if (!runtime.isAlive()) { -log.error("Function Container is dead with exception", runtime.getDeathException()); +log.error("[{}-{}] Function Container is dead with exception", +instanceConfig.getFunctionDetails().getName(), instanceConfig.getInstanceId(), +runtime.getDeathException()); log.error("Restarting..."); // Just for the sake of sanity, just destroy the runtime runtime.stop();
[incubator-pulsar] branch branch-2.1 updated: Set empty response to complete grpc txn (#2225)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new b335e94 Set empty response to complete grpc txn (#2225) b335e94 is described below commit b335e94b8541c4b1e4536aa12d700ea06082dc5f Author: Rajan Dhabalia AuthorDate: Wed Jul 25 13:07:52 2018 -0700 Set empty response to complete grpc txn (#2225) --- .../java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java index 0c10fdb..966740d 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java @@ -370,9 +370,10 @@ public class JavaInstanceMain implements AutoCloseable { if (runtime != null) { try { runtime.resetMetrics().get(); + responseObserver.onNext(com.google.protobuf.Empty.getDefaultInstance()); responseObserver.onCompleted(); } catch (InterruptedException | ExecutionException e) { -log.error("Exception in JavaInstance doing getAndResetMetrics", e); +log.error("Exception in JavaInstance doing resetMetrics", e); throw new RuntimeException(e); } }
[incubator-pulsar] branch branch-2.1 updated: Fix implementation of equals() for MessageId (#2226)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 3eaa79b Fix implementation of equals() for MessageId (#2226) 3eaa79b is described below commit 3eaa79beb20a759d9dba49a3dd75a5257c29d2e1 Author: massakam AuthorDate: Wed Jul 25 19:53:08 2018 +0900 Fix implementation of equals() for MessageId (#2226) --- .../pulsar/client/impl/BatchMessageIdImpl.java | 4 +++- .../apache/pulsar/client/impl/MessageIdImpl.java | 8 .../pulsar/client/impl/BatchMessageIdImplTest.java | 24 ++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java index 78109c4..d66e242 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java @@ -92,7 +92,9 @@ public class BatchMessageIdImpl extends MessageIdImpl { return ledgerId == other.ledgerId && entryId == other.entryId && partitionIndex == other.partitionIndex && batchIndex == other.batchIndex; } else if (obj instanceof MessageIdImpl) { -return batchIndex == NO_BATCH && obj.equals(this); +MessageIdImpl other = (MessageIdImpl) obj; +return ledgerId == other.ledgerId && entryId == other.entryId && partitionIndex == other.partitionIndex +&& batchIndex == NO_BATCH; } return false; } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java index 41cc9c9..3686b12 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java @@ -70,12 +70,12 @@ public class MessageIdImpl implements MessageId { @Override public boolean equals(Object obj) { -if (obj instanceof MessageIdImpl) { -MessageIdImpl other = (MessageIdImpl) obj; -return ledgerId == other.ledgerId && entryId == other.entryId && partitionIndex == other.partitionIndex; -} else if (obj instanceof BatchMessageIdImpl){ +if (obj instanceof BatchMessageIdImpl) { BatchMessageIdImpl other = (BatchMessageIdImpl) obj; return other.equals(this); +} else if (obj instanceof MessageIdImpl) { +MessageIdImpl other = (MessageIdImpl) obj; +return ledgerId == other.ledgerId && entryId == other.entryId && partitionIndex == other.partitionIndex; } return false; } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java index a3b5f72..af21fcf 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java @@ -44,4 +44,28 @@ public class BatchMessageIdImplTest { Assert.assertTrue(batchMsgId1.hashCode() != batchMsgId2.hashCode()); } + +@Test +public void equalsTest() { +BatchMessageIdImpl batchMsgId1 = new BatchMessageIdImpl(0, 0, 0, 0); +BatchMessageIdImpl batchMsgId2 = new BatchMessageIdImpl(1, 1, 1, 1); +BatchMessageIdImpl batchMsgId3 = new BatchMessageIdImpl(0, 0, 0, 1); +BatchMessageIdImpl batchMsgId4 = new BatchMessageIdImpl(0, 0, 0, -1); +MessageIdImpl msgId = new MessageIdImpl(0, 0, 0); + +Assert.assertTrue(batchMsgId1.equals(batchMsgId1)); +Assert.assertFalse(batchMsgId1.equals(batchMsgId2)); +Assert.assertFalse(batchMsgId1.equals(batchMsgId3)); +Assert.assertFalse(batchMsgId1.equals(batchMsgId4)); +Assert.assertFalse(batchMsgId1.equals(msgId)); + +Assert.assertTrue(msgId.equals(msgId)); +Assert.assertFalse(msgId.equals(batchMsgId1)); +Assert.assertFalse(msgId.equals(batchMsgId2)); +Assert.assertFalse(msgId.equals(batchMsgId3)); +Assert.assertTrue(msgId.equals(batchMsgId4)); + +Assert.assertTrue(batchMsgId4.equals(msgId)); +} + }
[incubator-pulsar] branch branch-2.1 updated: Fixed ack message tracker cleanup for single message batches (#2224)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new 638eb55 Fixed ack message tracker cleanup for single message batches (#2224) 638eb55 is described below commit 638eb55970f2d30eb09282731f0402903b5ee469 Author: Matteo Merli AuthorDate: Wed Jul 25 02:00:16 2018 -0700 Fixed ack message tracker cleanup for single message batches (#2224) ### Motivation Fixes #2221 The issue is that we were removing a batch message id impl from the tracking set which contains the batch-wide ids. --- .../impl/UnAcknowledgedMessagesTimeoutTest.java| 32 ++ .../apache/pulsar/client/impl/ConsumerImpl.java| 28 +-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java index 0b5126b..da53760 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java @@ -19,6 +19,8 @@ package org.apache.pulsar.client.impl; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; import java.util.HashSet; import java.util.concurrent.TimeUnit; @@ -30,6 +32,7 @@ import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.MessageRoutingMode; import org.apache.pulsar.client.api.Producer; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.client.api.Schema; import org.apache.pulsar.client.api.SubscriptionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -386,4 +389,33 @@ public class UnAcknowledgedMessagesTimeoutTest extends BrokerTestBase { assertEquals(consumer.getUnAckedMessageTracker().size(), 0); } + +@Test +public void testSingleMessageBatch() throws Exception { +String topicName = "prop/ns-abc/topic-estSingleMessageBatch"; + +Producer producer = pulsarClient.newProducer(Schema.STRING) +.topic(topicName) +.enableBatching(true) +.batchingMaxPublishDelay(10, TimeUnit.SECONDS) +.create(); + +Consumer consumer = pulsarClient.newConsumer(Schema.STRING) +.topic(topicName) +.subscriptionName("subscription") +.ackTimeout(1, TimeUnit.HOURS) +.subscribe(); + +// Force the creation of a batch with a single message +producer.sendAsync("hello"); +producer.flush(); + +Message message = consumer.receive(); + +assertFalse(((ConsumerImpl) consumer).getUnAckedMessageTracker().isEmpty()); + +consumer.acknowledge(message); + +assertTrue(((ConsumerImpl) consumer).getUnAckedMessageTracker().isEmpty()); +} } diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java index 6f78119..1c69c75 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java @@ -385,11 +385,6 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle log.debug("[{}] [{}] can ack message to broker {}, acktype {}, cardinality {}, length {}", subscription, consumerName, batchMessageId, ackType, outstandingAcks, batchSize); } -// increment Acknowledge-msg counter with number of messages in batch only if AckType is Individual. -// CumulativeAckType is handled while sending ack to broker -if (ackType == AckType.Individual) { -stats.incrementNumAcksSent(batchSize); -} return true; } else { if (AckType.Cumulative == ackType @@ -433,11 +428,16 @@ public class ConsumerImpl extends ConsumerBase implements ConnectionHandle Map properties) { MessageIdImpl msgId = (MessageIdImpl) messageId; - if (ackType == AckType.Individual) { -unAckedMessageTracker.remove(msgId); -// increment counter by 1 for non-batch msg -if (!(messageId instanceof BatchMessageIdImpl)) { +if (messageId instanceof BatchMessageIdImpl) { +BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl) messageId; + +