[incubator-pulsar] branch master updated: Correcting SQL getting started title (#2594)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new e357253 Correcting SQL getting started title (#2594) e357253 is described below commit e3572530f904d3cb0b3e80c177dbf33b1f4b2903 Author: Boyang Jerry Peng AuthorDate: Wed Sep 19 10:06:47 2018 -0700 Correcting SQL getting started title (#2594) --- .../versioned_docs/version-2.1.1-incubating/sql-getting-started.md| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md b/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md index aec0934..3fb42a3 100644 --- a/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md +++ b/site2/website/versioned_docs/version-2.1.1-incubating/sql-getting-started.md @@ -1,7 +1,7 @@ --- id: version-2.1.1-incubating-sql-getting-started -title: Pulsar SQL Overview -sidebar_label: Overview +title: Pulsar SQL Getting Started +sidebar_label: Getting Started original_id: sql-getting-started ---
[incubator-pulsar] branch jerrypeng-patch-1 deleted (was 6d8b8ba)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a change to branch jerrypeng-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git. was 6d8b8ba Merge remote-tracking branch 'apache/jerrypeng-patch-1' into jerrypeng-patch-1 The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-pulsar] branch jerrypeng-patch-1 updated (e256a79 -> 6d8b8ba)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a change to branch jerrypeng-patch-1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git.
from e256a79 Correcting SQL getting started title
add e4eb459 Correcting SQL getting started title
add 9e1691e Merge remote-tracking branch 'apache/master' into
jerrypeng-patch-1
add e677679 Merge remote-tracking branch 'apache/master' into
jerrypeng-patch-1
add 49fc5e5 [Function] avoid creating assignment snapshot and publish
individual assigment msg (#2549)
add 701f3ea fix behavior of JSONSchema for derived classes (#2577)
add 3097283 docs for 2.1.1-incubating release (#2601)
add 8e14175 add auto ack and timeout configurable (#2503)
add 4fdc21a [tests] remove
PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604)
add 5bdfa8a Merge remote-tracking branch 'apache/master' into
jerrypeng-patch-1
add 6d8b8ba Merge remote-tracking branch 'apache/jerrypeng-patch-1' into
jerrypeng-patch-1
No new revisions were added by this update.
Summary of changes:
conf/functions_worker.yml | 2 +
.../pulsar/compaction/TwoPhaseCompactor.java | 2 +-
.../apache/pulsar/compaction/CompactionTest.java | 2 +-
.../worker/PulsarWorkerAssignmentTest.java | 370 +
pulsar-client-schema/pom.xml | 6 +
.../pulsar/client/impl/schema/JSONSchema.java | 41 ++-
.../pulsar/client/schema/JSONSchemaTest.java | 67 +++-
.../pulsar/client/schema/SchemaTestUtils.java | 42 ++-
.../java/org/apache/pulsar/admin/cli/CmdSinks.java | 19 +-
.../org/apache/pulsar/admin/cli/TestCmdSinks.java | 1 +
.../proto/src/main/proto/Request.proto | 5 -
.../apache/pulsar/functions/utils/SinkConfig.java | 3 +
.../functions/worker/FunctionAssignmentTailer.java | 47 ++-
.../functions/worker/FunctionRuntimeManager.java | 283 +++-
.../pulsar/functions/worker/SchedulerManager.java | 126 ---
.../pulsar/functions/worker/WorkerConfig.java | 5 +-
.../pulsar/functions/worker/WorkerService.java | 15 +-
.../worker/scheduler/RoundRobinScheduler.java | 10 +-
.../worker/FunctionRuntimeManagerTest.java | 40 +--
.../functions/worker/MembershipManagerTest.java| 3 +
.../functions/worker/SchedulerManagerTest.java | 298 +
site2/docs/reference-pulsar-admin.md | 4 +
site2/website/release-notes.md | 17 +
site2/website/releases.json| 1 +
site2/website/sidebars.json| 5 -
.../version-2.1.1-incubating}/adaptors-kafka.md| 3 +-
.../administration-proxy.md| 3 +-
.../client-libraries-cpp.md| 3 +-
.../client-libraries-python.md | 3 +-
.../cookbooks-tiered-storage.md| 3 +-
.../version-2.1.1-incubating}/deploy-bare-metal.md | 3 +-
.../version-2.1.1-incubating}/functions-api.md | 3 +-
.../functions-deploying.md | 3 +-
.../functions-guarantees.md| 3 +-
.../functions-overview.md | 3 +-
.../functions-quickstart.md| 3 +-
.../getting-started-clients.md | 3 +-
.../getting-started-standalone.md | 3 +-
.../version-2.1.1-incubating}/io-managing.md | 3 +-
.../version-2.1.1-incubating}/io-quickstart.md | 7 +-
.../reference-configuration.md | 3 +-
.../reference-pulsar-admin.md | 3 +-
.../sql-deployment-configurations.md | 3 +-
.../sql-getting-started.md | 3 +-
.../version-2.1.1-incubating}/sql-overview.md | 3 +-
site2/website/versions.json| 1 +
46 files changed, 1016 insertions(+), 463 deletions(-)
create mode 100644
pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarWorkerAssignmentTest.java
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/adaptors-kafka.md (99%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/administration-proxy.md (98%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/client-libraries-cpp.md (98%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/client-libraries-python.md
(97%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/cookbooks-tiered-storage.md
(99%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/deploy-bare-metal.md (99%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/functions-api.md (99%)
copy site2/{docs =>
website/versioned_docs/version-2.1.1-incubating}/functions-deploying.md (99%)
copy
[incubator-pulsar] branch master updated: Add ServiceUrlProvider and add method forceCloseConnection in PulsarC… (#2543)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ee5afa5 Add ServiceUrlProvider and add method forceCloseConnection in
PulsarC… (#2543)
ee5afa5 is described below
commit ee5afa5007e260891140956d244a865d73368321
Author: penghui
AuthorDate: Wed Sep 19 15:44:15 2018 +0800
Add ServiceUrlProvider and add method forceCloseConnection in PulsarC…
(#2543)
Support build Pulsar client with serviceUrlProvider method.
### Motivation
With serviceUrlProvider we can store the pulsar service url in zookeeper or
any other config service.
And we can watch the service url change event then control the pulsar
client, such as change pulsar client serviceUrl, force close client connection
or re-connect with new service url.
### Modifications
Add ServiceUrlProvider interface.
Add forceCloseConnection method in PulsarClient.
---
.../pulsar/client/api/ServiceUrlProviderTest.java | 156 +
.../apache/pulsar/client/api/ClientBuilder.java| 7 +
.../org/apache/pulsar/client/api/PulsarClient.java | 22 +++
.../pulsar/client/api/ServiceUrlProvider.java | 42 ++
.../pulsar/client/impl/ClientBuilderImpl.java | 20 ++-
.../apache/pulsar/client/impl/HandlerState.java| 4 +
.../apache/pulsar/client/impl/LookupService.java | 2 +-
.../client/impl/PartitionedProducerImpl.java | 4 +
.../pulsar/client/impl/PulsarClientImpl.java | 29 +++-
.../client/impl/conf/ClientConfigurationData.java | 2 +
10 files changed, 283 insertions(+), 5 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java
new file mode 100644
index 000..ab86f12
--- /dev/null
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/ServiceUrlProviderTest.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api;
+
+import org.apache.bookkeeper.test.PortManager;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.client.impl.ConsumerImpl;
+import org.apache.pulsar.client.impl.ProducerImpl;
+import org.apache.pulsar.client.impl.PulsarClientImpl;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.TimeUnit;
+
+public class ServiceUrlProviderTest extends ProducerConsumerBase {
+
+@BeforeClass
+@Override
+protected void setup() throws Exception {
+super.internalSetup();
+super.producerBaseSetup();
+
+}
+
+@AfterClass
+@Override
+protected void cleanup() throws Exception {
+super.internalCleanup();
+}
+
+@Test
+public void testCreateClientWithServiceUrlProvider() throws Exception {
+
+PulsarClient client = PulsarClient.builder()
+.serviceUrlProvider(new
TestServiceUrlProvider(pulsar.getBrokerServiceUrl()))
+.statsInterval(1, TimeUnit.SECONDS)
+.build();
+Assert.assertTrue(((PulsarClientImpl)
client).getConfiguration().getServiceUrlProvider() instanceof
TestServiceUrlProvider);
+Producer producer = client.newProducer(Schema.STRING)
+.topic("persistent://my-property/my-ns/my-topic")
+.create();
+Consumer consumer = client.newConsumer(Schema.STRING)
+.topic("persistent://my-property/my-ns/my-topic")
+.subscriptionName("my-subscribe")
+.subscribe();
+for (int i = 0; i < 100; i++) {
+producer.send("Hello Pulsar[" + i + "]");
+}
+client.forceCloseConnection();
+for (int i = 100; i < 200; i++) {
+producer.send("Hello Pulsar[" + i + "]");
+
[incubator-pulsar] branch master updated: fixing/improving logging for function instance (#2586)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 3f8bd00 fixing/improving logging for function instance (#2586) 3f8bd00 is described below commit 3f8bd00d961f1f56233c629746c11cbd711bf06e Author: Boyang Jerry Peng AuthorDate: Tue Sep 18 23:30:39 2018 -0700 fixing/improving logging for function instance (#2586) ### Motivation logging for java function instance (process mode and localrun) is not working well. Log files are created at inappropriate locations for logs not captured by the routing appender, functions bookkeeper logs are missing because we are not using the shaded classpath. The best for logging is just to put all the logs from the java instance log in a single log and not worry about the routing which is only useful when running in threaded mode --- .../pulsar/functions/runtime/JavaInstanceMain.java | 9 --- .../pulsar/functions/runtime/ProcessRuntime.java | 22 +++--- .../pulsar/functions/runtime/ThreadRuntime.java| 4 +- .../src/main/resources/java_instance_log4j2.yml| 78 +++--- .../functions/runtime/ProcessRuntimeTest.java | 4 +- 5 files changed, 31 insertions(+), 86 deletions(-) diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java index 11da7c3..083686b 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java @@ -22,27 +22,18 @@ package org.apache.pulsar.functions.runtime; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.converters.StringConverter; -import com.google.gson.Gson; import com.google.protobuf.Empty; import com.google.protobuf.util.JsonFormat; import io.grpc.Server; import io.grpc.ServerBuilder; import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; - -import static org.apache.commons.lang3.StringUtils.isNotBlank; - import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; -import org.apache.pulsar.functions.proto.Function; -import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees; -import org.apache.pulsar.functions.proto.Function.SinkSpec; -import org.apache.pulsar.functions.proto.Function.SourceSpec; import org.apache.pulsar.functions.proto.Function.FunctionDetails; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceControlGrpc; -import java.util.Map; import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; diff --git a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java index a978a09..c5159b0 100644 --- a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java +++ b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java @@ -22,32 +22,32 @@ package org.apache.pulsar.functions.runtime; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.gson.Gson; import com.google.protobuf.Empty; import com.google.protobuf.util.JsonFormat; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import lombok.Getter; import lombok.extern.slf4j.Slf4j; - -import static org.apache.commons.lang3.StringUtils.isNotBlank; - -import org.apache.commons.lang3.StringUtils; import org.apache.pulsar.functions.instance.AuthenticationConfig; import org.apache.pulsar.functions.instance.InstanceConfig; import org.apache.pulsar.functions.proto.Function; import org.apache.pulsar.functions.proto.InstanceCommunication; import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus; import org.apache.pulsar.functions.proto.InstanceControlGrpc; +import org.apache.pulsar.functions.utils.FunctionDetailsUtils; import org.apache.pulsar.functions.utils.functioncache.FunctionCacheEntry; import java.io.InputStream; -import java.util.*; +import java.util.LinkedList; +import java.util.List; +import java.util.TimerTask; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.apache.commons.lang3
[incubator-pulsar] branch master updated: [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 4fdc21a [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604) 4fdc21a is described below commit 4fdc21afba77184109757f62679eec4d972dbbc8 Author: Sijie Guo AuthorDate: Tue Sep 18 21:27:41 2018 -0700 [tests] remove PersistentQueueE2ETest#testConsumersWithDifferentPermits (#2604) *Motivation* There is no ways to gurantee how fast a consumer can consume. so it makes no sense to compare the different permits between two consumers. *Changes* Remove PersistentQueueE2ETest#testConsumersWithDifferentPermits
[incubator-pulsar] branch master updated: [tests] improve connector related integration tests (#2587)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 7530d64 [tests] improve connector related integration tests (#2587)
7530d64 is described below
commit 7530d64a679a0783122b18058c1148c89c0fee0a
Author: Sijie Guo
AuthorDate: Tue Sep 18 14:50:51 2018 -0700
[tests] improve connector related integration tests (#2587)
*Motivation*
with more and more connector are added, it becomes expensive to start all
external services at the begin.
*Changes*
- refactor the connector testing framework to start external service before
methods
- fix kafka, cassandra and mysql connectors
---
distribution/io/src/assemble/io.xml| 6 +
.../broker/service/PersistentQueueE2ETest.java | 60 --
.../functions/instance/JavaInstanceRunnable.java | 3 +-
.../pulsar/functions/source/TopicSchema.java | 3 +
...rchAbstractSink.java => ElasticSearchSink.java} | 9 +-
.../io/elasticsearch/ElasticSearchStringSink.java | 35 --
.../resources/META-INF/services/pulsar-io.yaml | 4 +-
.../io/elasticsearch/ElasticSearchSinkTests.java | 4 +-
.../apache/pulsar/io/kafka/KafkaAbstractSink.java | 6 +-
.../{KafkaStringSink.java => KafkaBytesSink.java} | 25 -
.../resources/META-INF/services/pulsar-io.yaml | 2 +-
site2/docs/io-quickstart.md| 2 +-
.../version-2.1.0-incubating/io-quickstart.md | 2 +-
.../integration/functions/PulsarFunctionsTest.java | 39 +--
.../functions/utils/CommandGenerator.java | 2 +-
.../integration/io/CassandraSinkArchiveTester.java | 121 -
.../tests/integration/io/CassandraSinkTester.java | 45 +---
.../integration/io/ElasticSearchSinkTester.java| 34 +++---
.../tests/integration/io/HdfsSinkTester.java | 22 ++--
.../tests/integration/io/JdbcSinkTester.java | 38 ---
.../tests/integration/io/KafkaSinkTester.java | 28 +++--
.../tests/integration/io/KafkaSourceTester.java| 12 +-
.../pulsar/tests/integration/io/SinkTester.java| 25 -
.../pulsar/tests/integration/io/SourceTester.java | 4 +-
.../tests/integration/suites/PulsarTestSuite.java | 50 -
.../integration/topologies/PulsarCluster.java | 17 +++
26 files changed, 212 insertions(+), 386 deletions(-)
diff --git a/distribution/io/src/assemble/io.xml
b/distribution/io/src/assemble/io.xml
index 08ff859..a509e19 100644
--- a/distribution/io/src/assemble/io.xml
+++ b/distribution/io/src/assemble/io.xml
@@ -92,5 +92,11 @@
connectors
644
+
+
+
${basedir}/../../pulsar-io/elastic-search/target/pulsar-io-elastic-search-${project.version}.nar
+ connectors
+ 644
+
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
index 2937ca0..fb7a76c 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
@@ -233,66 +233,6 @@ public class PersistentQueueE2ETest extends BrokerTestBase
{
deleteTopic(topicName);
}
-@Test
-public void testConsumersWithDifferentPermits() throws Exception {
-final String topicName = "persistent://prop/use/ns-abc/shared-topic4";
-final String subName = "sub4";
-final int numMsgs = 1;
-
-final AtomicInteger msgCountConsumer1 = new AtomicInteger(0);
-final AtomicInteger msgCountConsumer2 = new AtomicInteger(0);
-final CountDownLatch latch = new CountDownLatch(numMsgs);
-
-int recvQ1 = 10;
-Consumer consumer1 =
pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
-
.subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1)
-.messageListener((consumer, msg) -> {
-msgCountConsumer1.incrementAndGet();
-try {
-consumer.acknowledge(msg);
-latch.countDown();
-} catch (PulsarClientException e) {
-fail("Should not fail");
-}
-}).subscribe();
-
-int recvQ2 = 1;
-Consumer consumer2 =
pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
-
.subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2)
-.messageListener((consumer, msg) -> {
-msgCountConsumer2.incrementAndGet();
-try {
-
[incubator-pulsar] branch master updated: [build] Fix docker organization parameter (#2597)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new fb396bf [build] Fix docker organization parameter (#2597) fb396bf is described below commit fb396bf8313f728f5659286e4d8924bcf37d4832 Author: Sijie Guo AuthorDate: Tue Sep 18 14:50:12 2018 -0700 [build] Fix docker organization parameter (#2597) *Motivation* docker orgnization is missing for building test image. so the build will be failing with `-Pdocker`. *Changes* Move the docker organization parameter to root pom file. --- docker/pom.xml | 3 --- pom.xml| 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index bdc99f7..302bda8 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -31,9 +31,6 @@ org.apache.pulsar docker-images Apache Pulsar :: Docker Images - -apachepulsar - pulsar grafana diff --git a/pom.xml b/pom.xml index 6d45a00..91ba14c 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,7 @@ flexible messaging model and an intuitive client API. true false 1 +apachepulsar 2.1.0-incubating
[incubator-pulsar] branch master updated: Fix: Compaction with last deleted keys not completing compaction (#2591)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8d50617 Fix: Compaction with last deleted keys not completing
compaction (#2591)
8d50617 is described below
commit 8d5061779e7a869e6e5981d2010e671ed0afa322
Author: Rajan Dhabalia
AuthorDate: Tue Sep 18 03:09:50 2018 -0700
Fix: Compaction with last deleted keys not completing compaction (#2591)
### Motivation
Right now, topic-compaction ignores the message-id with empty payload but
if the last message in the ledger has empty payload then compactor doesn't
complete the compaction because compactor ignores last message and doesn't
complete the result-future so, caller never sees complete result.
### Modifications
- Compactor calculates` from` and `to` position for compacted ledger
according to last non-deleted active key.
- Compactor handles tail deleted keys from the ledger and completes
compaction process gracefully.
### Result
compactor can successfully compact ledger whose last message is also
deleted.
---
.../pulsar/compaction/TwoPhaseCompactor.java | 90 +-
.../apache/pulsar/compaction/CompactionTest.java | 56 ++
2 files changed, 111 insertions(+), 35 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
index cc3f710..425e049 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/compaction/TwoPhaseCompactor.java
@@ -76,7 +76,7 @@ public class TwoPhaseCompactor extends Compactor {
@Override
protected CompletableFuture doCompaction(RawReader reader,
BookKeeper bk) {
return phaseOne(reader).thenCompose(
-(r) -> phaseTwo(reader, r.from, r.to, r.latestForKey, bk));
+(r) -> phaseTwo(reader, r.from, r.to, r.lastReadId,
r.latestForKey, bk));
}
private CompletableFuture phaseOne(RawReader reader) {
@@ -90,7 +90,8 @@ public class TwoPhaseCompactor extends Compactor {
} else {
log.info("Commencing phase one of compaction for {},
reading to {}",
reader.getTopic(), lastMessageId);
-phaseOneLoop(reader, Optional.empty(), lastMessageId,
latestForKey, loopPromise);
+phaseOneLoop(reader, Optional.empty(),
Optional.empty(), lastMessageId, latestForKey,
+loopPromise);
}
});
return loopPromise;
@@ -98,6 +99,7 @@ public class TwoPhaseCompactor extends Compactor {
private void phaseOneLoop(RawReader reader,
Optional firstMessageId,
+ Optional toMessageId,
MessageId lastMessageId,
Map latestForKey,
CompletableFuture loopPromise) {
@@ -114,6 +116,7 @@ public class TwoPhaseCompactor extends Compactor {
return;
}
MessageId id = m.getMessageId();
+boolean deletedMessage = false;
if (RawBatchConverter.isReadableBatch(m)) {
try {
RawBatchConverter.extractIdsAndKeys(m)
@@ -125,16 +128,23 @@ public class TwoPhaseCompactor extends Compactor {
} else {
Pair keyAndSize =
extractKeyAndSize(m);
if (keyAndSize != null) {
-latestForKey.put(keyAndSize.getLeft(), id);
+if(keyAndSize.getRight() > 0) {
+latestForKey.put(keyAndSize.getLeft(),
id);
+} else {
+deletedMessage = true;
+latestForKey.remove(keyAndSize.getLeft());
+}
}
}
+MessageId first = firstMessageId.orElse(deletedMessage
? null : id);
+MessageId to = deletedMessage ?
toMessageId.orElse(null) : id;
if (id.compareTo(lastMessageId) == 0) {
-loopPromise.complete(new
PhaseOneResult(firstMessageId.orElse(id),
-id,
latestForKey));
+loopPromise.complete(new PhaseOneResult(first,
[incubator-pulsar] branch master updated: [java-client] Issue #2384: ConnectionHandler: Log stack trace instead of printing (#2599)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new c866279 [java-client] Issue #2384: ConnectionHandler: Log stack trace
instead of printing (#2599)
c866279 is described below
commit c86627948d4abf4b50517338c739f0dea8029849
Author: Sijie Guo
AuthorDate: Tue Sep 18 03:05:40 2018 -0700
[java-client] Issue #2384: ConnectionHandler: Log stack trace instead of
printing (#2599)
*Motivation*
Fixes #2384.
The pulsar java client is currently very noisy in case of connection loss
and it cannot be muted, as the stack trace is printed with .printStackTrace().
Moving this to the logging system will allow developers to configure this
behavior.
*Changes*
Remove `printStackTrace`.
---
.../src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java | 1 -
1 file changed, 1 deletion(-)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java
index d9bdc74..d04e8a8 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConnectionHandler.java
@@ -71,7 +71,6 @@ class ConnectionHandler {
}
private Void handleConnectionError(Throwable exception) {
-exception.printStackTrace();
log.warn("[{}] [{}] Error connecting to broker: {}", state.topic,
state.getHandlerName(), exception.getMessage());
connection.connectionFailed(new PulsarClientException(exception));
[incubator-pulsar] branch master updated: [tests] Flaky Test ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 6c07c81 [tests] Flaky Test
ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598)
6c07c81 is described below
commit 6c07c81bf85eaa6d2a23c026049c57305284876e
Author: Sijie Guo
AuthorDate: Tue Sep 18 02:18:17 2018 -0700
[tests] Flaky Test
ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache (#2598)
* [tests] Flaky Test
ZooKeeperCacheTest#testChildrenCacheZnodeCreatedAfterCache
*Motivation*
The problem is zookeeper watcher notification is non-deterministic. That
says if you create N paths in zookeeper, you might receive
x notification. x is between 1 and N. so the test is written in a very
non-deterministic way.
```
java.lang.AssertionError: expected [1] but found [2]
at org.testng.Assert.fail(Assert.java:96)
at org.testng.Assert.failNotEquals(Assert.java:776)
at org.testng.Assert.assertEqualsImpl(Assert.java:137)
at org.testng.Assert.assertEquals(Assert.java:118)
at org.testng.Assert.assertEquals(Assert.java:652)
at org.testng.Assert.assertEquals(Assert.java:662)
at
org.apache.pulsar.zookeeper.ZookeeperCacheTest.testChildrenCacheZnodeCreatedAfterCache(ZookeeperCacheTest.java:214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:124)
at
org.testng.internal.InvokeMethodRunnable.runOne(InvokeMethodRunnable.java:54)
at
org.testng.internal.InvokeMethodRunnable.run(InvokeMethodRunnable.java:44)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
```
*Changes*
rewrite the test to make it more robust.
---
.../test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java
b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java
index 09c3ea1..39f23ac 100644
---
a/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java
+++
b/pulsar-zookeeper-utils/src/test/java/org/apache/pulsar/zookeeper/ZookeeperCacheTest.java
@@ -209,12 +209,14 @@ public class ZookeeperCacheTest {
Thread.sleep(1);
}
+final int recvNotifications = notificationCount.get();
+
assertEquals(cache.get(), new
TreeSet(Lists.newArrayList("z1")));
assertEquals(cache.get("/test"), new
TreeSet(Lists.newArrayList("z1")));
-assertEquals(notificationCount.get(), 1);
+assertTrue(recvNotifications == 1 || recvNotifications == 2);
zkClient.delete("/test/z1", -1);
-while (notificationCount.get() < 2) {
+while (notificationCount.get() < (recvNotifications + 1)) {
Thread.sleep(1);
}
@@ -230,7 +232,7 @@ public class ZookeeperCacheTest {
// Ok
}
-assertEquals(notificationCount.get(), 2);
+assertEquals(notificationCount.get(), (recvNotifications + 1));
}
@Test(timeOut = 1)
[incubator-pulsar] branch master updated: [tests] make PersistentQueueE2ETest tolerant failures during deleting topic at cleanup (#2600)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new aebfbba [tests] make PersistentQueueE2ETest tolerant failures during
deleting topic at cleanup (#2600)
aebfbba is described below
commit aebfbba2a9f2882944bd1854672f4538a9d27177
Author: Sijie Guo
AuthorDate: Tue Sep 18 00:43:50 2018 -0700
[tests] make PersistentQueueE2ETest tolerant failures during deleting topic
at cleanup (#2600)
*Motivation*
"Topic has active producers/subscriptions" can be thrown during deleting
topic at cleanup.
This kind of error can be ignored during cleaning up to make the tests more
robust.
*Changes*
Make PersistentQueueE2ETest toelrant failures during cleaning up topics.
---
.../broker/service/PersistentQueueE2ETest.java | 23 +-
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
index c41a07a..2937ca0 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentQueueE2ETest.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.collections.CollectionUtils;
import org.apache.pulsar.broker.service.persistent.PersistentSubscription;
import org.apache.pulsar.broker.service.persistent.PersistentTopic;
+import org.apache.pulsar.client.admin.PulsarAdminException;
import org.apache.pulsar.client.api.Consumer;
import org.apache.pulsar.client.api.ConsumerBuilder;
import org.apache.pulsar.client.api.Message;
@@ -76,6 +77,14 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
private static final Logger log =
LoggerFactory.getLogger(PersistentQueueE2ETest.class);
+private void deleteTopic(String topicName) {
+try {
+admin.topics().delete(topicName);
+} catch (PulsarAdminException pae) {
+// it is okay to get exception if it is cleaning up.
+}
+}
+
@Test
public void testSimpleConsumerEvents() throws Exception {
final String topicName = "persistent://prop/use/ns-abc/shared-topic1";
@@ -172,7 +181,7 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
producer.close();
consumer2.close();
-admin.topics().delete(topicName);
+deleteTopic(topicName);
}
@Test
@@ -220,7 +229,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
assertTrue(CollectionUtils.subtract(messagesProduced,
messagesConsumed).isEmpty());
consumer1.close();
-admin.topics().delete(topicName);
+
+deleteTopic(topicName);
}
@Test
@@ -279,7 +289,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
consumer1.close();
consumer2.close();
-admin.topics().delete(topicName);
+
+deleteTopic(topicName);
}
// this test is good to have to see the distribution, but every now and
then it gets slightly different than the
@@ -352,7 +363,8 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
consumer1.close();
consumer2.close();
consumer3.close();
-admin.topics().delete(topicName);
+
+deleteTopic(topicName);
}
@Test(timeOut = 30)
@@ -550,6 +562,7 @@ public class PersistentQueueE2ETest extends BrokerTestBase {
producer.close();
consumer1.close();
consumer2.close();
-admin.topics().delete(topicName);
+
+deleteTopic(topicName);
}
}
[incubator-pulsar] branch master updated: Preserve filename of the original function submission (#2588)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new a123380 Preserve filename of the original function submission (#2588)
a123380 is described below
commit a123380327fbe95eef1066c4c456020c19136e07
Author: Sanjeev Kulkarni
AuthorDate: Mon Sep 17 01:03:06 2018 -0700
Preserve filename of the original function submission (#2588)
### Motivation
When we submit a function, currently we don't note down the original
filename that was uploaded from the client. For regular java functions this
works fine. For python workers we manage by doing tricks wrt tricks like
filename using classnames. However when we add things like wheel support, the
filename encodes information that cannot be discarded away.
This pr preserves that filename
---
pulsar-functions/proto/src/main/proto/Function.proto | 1 +
.../org/apache/pulsar/functions/utils/FunctionDetailsUtils.java | 8 +++-
.../java/org/apache/pulsar/functions/worker/FunctionActioner.java | 2 +-
.../apache/pulsar/functions/worker/rest/api/FunctionsImpl.java| 6 ++
4 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/pulsar-functions/proto/src/main/proto/Function.proto
b/pulsar-functions/proto/src/main/proto/Function.proto
index 6e969ae..a76cf8d 100644
--- a/pulsar-functions/proto/src/main/proto/Function.proto
+++ b/pulsar-functions/proto/src/main/proto/Function.proto
@@ -116,6 +116,7 @@ message SinkSpec {
message PackageLocationMetaData {
string packagePath = 1;
+string originalFileName = 2;
}
message FunctionMetaData {
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
index 5ecdd4f..a9787ba 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionDetailsUtils.java
@@ -19,6 +19,8 @@
package org.apache.pulsar.functions.utils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
public class FunctionDetailsUtils {
@@ -43,7 +45,11 @@ public class FunctionDetailsUtils {
return fullyQualifiedName.split("/")[2];
}
-public static String getDownloadFileName(FunctionDetails FunctionDetails) {
+public static String getDownloadFileName(FunctionDetails FunctionDetails,
+ Function.PackageLocationMetaData
packageLocation) {
+if (!StringUtils.isEmpty(packageLocation.getOriginalFileName())) {
+return packageLocation.getOriginalFileName();
+}
String[] hierarchy = FunctionDetails.getClassName().split("\\.");
String fileName;
if (hierarchy.length <= 0) {
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
index f5a7e96..724d80c 100644
---
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
+++
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionActioner.java
@@ -156,7 +156,7 @@ public class FunctionActioner implements AutoCloseable {
pkgFile = new File(
pkgDir,
-new
File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails())).getName());
+new
File(FunctionDetailsUtils.getDownloadFileName(functionMetaData.getFunctionDetails(),
functionMetaData.getPackageLocation())).getName());
downloadFile(pkgFile, isPkgUrlProvided, functionMetaData,
instanceId);
}
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index df82c0d..8bb9194 100644
---
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
+++
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
@@ -170,6 +170,9 @@ public class FunctionsImpl {
} else {
packageLocationMetaDataBuilder.setPackagePath(isPkgUrlProvided ?
functionPkgUrl
: createPackagePath(tenant, namespace, functionName,
fileDetail.getFileName()));
+if (!isPkgUrlProvided) {
+
packageLocationMetaDataBuilder.setOriginalFileName
[incubator-pulsar] branch master updated: [schema] add schemas for primtive types (#2500)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 5a2d75c [schema] add schemas for primtive types (#2500)
5a2d75c is described below
commit 5a2d75c11acd8c6014ea1f9551405a6bcb41e4f8
Author: Sijie Guo
AuthorDate: Mon Sep 17 00:51:59 2018 -0700
[schema] add schemas for primtive types (#2500)
* [schema] add schemas for primtive types
* Fix issues with Schema.BYTES
---
.../client/api/SchemaSerializationException.java | 5 +
.../client/impl/schema/ByteBufferSchema.java | 75 +++
.../schema/{StringSchema.java => ByteSchema.java} | 50 +-
.../pulsar/client/impl/schema/BytesSchema.java | 17 +++-
.../pulsar/client/impl/schema/DoubleSchema.java| 80
.../pulsar/client/impl/schema/FloatSchema.java | 76 +++
.../pulsar/client/impl/schema/IntSchema.java | 75 +++
.../pulsar/client/impl/schema/LongSchema.java | 79
.../pulsar/client/impl/schema/ShortSchema.java | 73 +++
.../pulsar/client/impl/schema/StringSchema.java| 29 --
.../pulsar/client/schema/PrimitiveSchemaTest.java | 102 +
.../apache/pulsar/client/impl/ConsumerImpl.java| 11 ++-
.../apache/pulsar/client/impl/ProducerImpl.java| 3 +
.../apache/pulsar/common/schema/SchemaInfo.java| 2 +
.../apache/pulsar/common/schema/SchemaType.java| 40
15 files changed, 686 insertions(+), 31 deletions(-)
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java
index 39248d2..46a38c1 100644
---
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/SchemaSerializationException.java
@@ -19,6 +19,11 @@
package org.apache.pulsar.client.api;
public class SchemaSerializationException extends RuntimeException {
+
+public SchemaSerializationException(String message) {
+super(message);
+}
+
public SchemaSerializationException(Throwable cause) {
super(cause);
}
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java
new file mode 100644
index 000..ee8ba66
--- /dev/null
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/ByteBufferSchema.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.impl.schema;
+
+import java.nio.ByteBuffer;
+import org.apache.pulsar.client.api.Schema;
+import org.apache.pulsar.common.schema.SchemaInfo;
+import org.apache.pulsar.common.schema.SchemaType;
+
+/**
+ * A bytebuffer schema.
+ */
+public class ByteBufferSchema implements Schema {
+
+public static ByteBufferSchema of() {
+return INSTANCE;
+}
+
+private static final ByteBufferSchema INSTANCE = new ByteBufferSchema();
+private static final SchemaInfo SCHEMA_INFO = new SchemaInfo()
+.setName("ByteBuffer")
+.setType(SchemaType.BYTEBUFFER)
+.setSchema(new byte[0]);
+
+@Override
+public byte[] encode(ByteBuffer data) {
+if (data == null) {
+return null;
+}
+
+data.rewind();
+
+if (data.hasArray()) {
+byte[] arr = data.array();
+if (data.arrayOffset() == 0 && arr.length == data.remaining()) {
+return arr;
+}
+}
+
+byte[] ret = new byte[data.remaining()];
+data.get(ret, 0, ret.length);
+data.rewind();
+return ret;
+}
+
+@Override
+public ByteBuffer decode(byte[] data) {
+if (null == data) {
+return null;
+} else {
[incubator-pulsar] branch master updated: Change repository tag for test image (#2571)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 844f46f Change repository tag for test image (#2571)
844f46f is described below
commit 844f46f156687fad1a08d7e88716fa5ee6937943
Author: Ali Ahmed
AuthorDate: Mon Sep 17 00:26:27 2018 -0700
Change repository tag for test image (#2571)
This uses the docker.organization property to tag the test image to make it
consistent with other pom files
---
tests/docker-images/latest-version-image/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/docker-images/latest-version-image/pom.xml
b/tests/docker-images/latest-version-image/pom.xml
index 78a12e2..b58530c 100644
--- a/tests/docker-images/latest-version-image/pom.xml
+++ b/tests/docker-images/latest-version-image/pom.xml
@@ -68,13 +68,13 @@
tag
-
apachepulsar/pulsar-test-latest-version
+
${docker.organization}/pulsar-test-latest-version
latest
- apachepulsar/pulsar-test-latest-version
+
${docker.organization}/pulsar-test-latest-version
${project.version}
false
true
[incubator-pulsar.wiki] branch master updated: Updated Home (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 83e4c5c Updated Home (markdown) 83e4c5c is described below commit 83e4c5cfacdb4ba3fe3b75d3239d65c18b7071c7 Author: Sijie Guo AuthorDate: Sun Sep 16 16:30:08 2018 -0700 Updated Home (markdown) --- Home.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Home.md b/Home.md index 74324a2..bf95c54 100644 --- a/Home.md +++ b/Home.md @@ -9,12 +9,12 @@ We encourage to document any big change or feature or any addition to public use ### Proposed -* [[PIP 23: Message Tracing By Interceptors]] -* [[PIP 22: Pulsar Dead Letter Topic]] * [[PIP 21: Pulsar Edge Component]] ### Accepted +* [[PIP 23: Message Tracing By Interceptors]] +* [[PIP 22: Pulsar Dead Letter Topic]] * [PIP 20: Mechanism to revoke TLS authentication](https://github.com/apache/incubator-pulsar/wiki/PIP-20:-Mechanism-to-revoke-TLS-authentication) * [PIP 19: Pulsar SQL](https://github.com/apache/incubator-pulsar/wiki/PIP-19:-Pulsar-SQL) * [PIP 17: Tiered storage for Pulsar topics](https://github.com/apache/incubator-pulsar/wiki/PIP-17:-Tiered-storage-for-Pulsar-topics)
[incubator-pulsar.wiki] branch master updated: Updated PIP 22: Pulsar Dead Letter Topic (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 04cecc4 Updated PIP 22: Pulsar Dead Letter Topic (markdown) 04cecc4 is described below commit 04cecc4bf2f95096ec9ec54b2f85ceb62135eece Author: Sijie Guo AuthorDate: Sun Sep 16 16:29:44 2018 -0700 Updated PIP 22: Pulsar Dead Letter Topic (markdown) --- PIP-22:-Pulsar-Dead-Letter-Topic.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PIP-22:-Pulsar-Dead-Letter-Topic.md b/PIP-22:-Pulsar-Dead-Letter-Topic.md index c259aa2..cd49a3f 100644 --- a/PIP-22:-Pulsar-Dead-Letter-Topic.md +++ b/PIP-22:-Pulsar-Dead-Letter-Topic.md @@ -1,7 +1,8 @@ -- **Status**: Proposed +- **Status**: Done - **Author**: [Penghui Li](https://github.com/codelipenghui) -- **Pull Request**: - +- **Pull Request**: [#2508](https://github.com/apache/incubator-pulsar/pull/2508) - **Mailing List discussion**: +- **Release**: 2.2.0 ## Motivation
[incubator-pulsar.wiki] branch master updated: Updated PIP 23: Message Tracing By Interceptors (markdown)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.wiki.git The following commit(s) were added to refs/heads/master by this push: new 5044279 Updated PIP 23: Message Tracing By Interceptors (markdown) 5044279 is described below commit 5044279a618f45aca535171dbbb8e3eaa3aabded Author: Sijie Guo AuthorDate: Sun Sep 16 16:28:35 2018 -0700 Updated PIP 23: Message Tracing By Interceptors (markdown) --- PIP-23:-Message-Tracing-By-Interceptors.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PIP-23:-Message-Tracing-By-Interceptors.md b/PIP-23:-Message-Tracing-By-Interceptors.md index 58d5336..c29439c 100644 --- a/PIP-23:-Message-Tracing-By-Interceptors.md +++ b/PIP-23:-Message-Tracing-By-Interceptors.md @@ -1,7 +1,8 @@ -- **Status**: Proposed +- **Status**: Done - **Author**: [Penghui Li](https://github.com/codelipenghui), [Jia Zhai](https://github.com/zhaijack) -- **Pull Request**: - +- **Pull Request**: [#2471](https://github.com/apache/incubator-pulsar/pull/2471) - **Mailing List discussion**: +- **Release**: 2.2.0 ## Requirement & Overview
[incubator-pulsar] branch master updated: Misc fixes (#2579)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 858648e Misc fixes (#2579)
858648e is described below
commit 858648eb24aeba9d95a7e83513830afd40cb21fe
Author: Sanjeev Kulkarni
AuthorDate: Fri Sep 14 22:33:03 2018 -0700
Misc fixes (#2579)
This pr does a couple of minor nits
1. Add a log statement in Sink to make it consistent with the Source.
2. Added AllArgs/NoArgs constructor for some test related objects
---
.../src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java| 2 ++
.../org/apache/pulsar/functions/api/examples/CustomBaseObject.java| 2 ++
.../org/apache/pulsar/functions/api/examples/CustomDerivedObject.java | 4
3 files changed, 8 insertions(+)
diff --git
a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
index 835e288..4449045 100644
---
a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
+++
b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/sink/PulsarSink.java
@@ -205,6 +205,8 @@ public class PulsarSink implements Sink {
@Override
public void open(Map config, SinkContext sinkContext)
throws Exception {
+log.info("Opening pulsar sink with config: {}", pulsarSinkConfig);
+
Schema schema = initializeSchema();
FunctionConfig.ProcessingGuarantees processingGuarantees =
this.pulsarSinkConfig.getProcessingGuarantees();
diff --git
a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java
b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java
index 173393b..6102887 100644
---
a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java
+++
b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomBaseObject.java
@@ -20,11 +20,13 @@ package org.apache.pulsar.functions.api.examples;
import lombok.AllArgsConstructor;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
@AllArgsConstructor
+@NoArgsConstructor
public class CustomBaseObject {
private long baseValue;
}
diff --git
a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java
b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java
index 464e977..d700231 100644
---
a/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java
+++
b/pulsar-functions/java-examples/src/main/java/org/apache/pulsar/functions/api/examples/CustomDerivedObject.java
@@ -18,11 +18,15 @@
*/
package org.apache.pulsar.functions.api.examples;
+import lombok.AllArgsConstructor;
import lombok.Getter;
+import lombok.NoArgsConstructor;
import lombok.Setter;
@Getter
@Setter
+@AllArgsConstructor
+@NoArgsConstructor
public class CustomDerivedObject extends CustomBaseObject {
private int derivedValue;
public CustomDerivedObject(long baseValue, int derivedValue) {
[incubator-pulsar] branch master updated: Issue #2574: Timeout message not get redeliver in TopicsConsumer when use message listener
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ec74355 Issue #2574: Timeout message not get redeliver in
TopicsConsumer when use message listener
ec74355 is described below
commit ec7435565520ee6ac3ed2ce4a0cd13e8de7dbca3
Author: Jia Zhai
AuthorDate: Sat Sep 15 00:15:13 2018 +0800
Issue #2574: Timeout message not get redeliver in TopicsConsumer when use
message listener
### Motivation
fix issue #2574 .
Timeout message not get redeliver in TopicsConsumer when use message
listener.
This is caused by message listener wrongly set in individual
sub-ConsumerImpl.
### Modifications
set message listener to null for individual sub-ConsumerImpl.
Add a UT
### Result
UT passed.
---
.../pulsar/client/impl/TopicsConsumerImplTest.java | 58 ++
.../client/impl/MultiTopicsConsumerImpl.java | 1 +
2 files changed, 59 insertions(+)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
index d28d1d5..55be9ec 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/TopicsConsumerImplTest.java
@@ -19,6 +19,7 @@
package org.apache.pulsar.client.impl;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@@ -615,4 +616,61 @@ public class TopicsConsumerImplTest extends
ProducerConsumerBase {
}
}
+/**
+ * Test Listener for github issue #2547
+ */
+@Test(timeOut = 3)
+public void testMultiTopicsMessageListener() throws Exception {
+String key = "MultiTopicsMessageListenerTest";
+final String subscriptionName = "my-ex-subscription-" + key;
+final String messagePredicate = "my-message-" + key + "-";
+final int totalMessages = 6;
+
+// set latch larger than totalMessages, so timeout message get resend
+CountDownLatch latch = new CountDownLatch(totalMessages * 3);
+
+final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" +
key;
+List topicNames = Lists.newArrayList(topicName1);
+
+admin.tenants().createTenant("prop", new TenantInfo());
+admin.topics().createPartitionedTopic(topicName1, 2);
+
+// 1. producer connect
+Producer producer1 =
pulsarClient.newProducer().topic(topicName1)
+.enableBatching(false)
+.messageRoutingMode(MessageRoutingMode.SinglePartition)
+.create();
+
+// 2. Create consumer, set not ack in message listener, so time-out
message will resend
+Consumer consumer = pulsarClient.newConsumer()
+.topics(topicNames)
+.subscriptionName(subscriptionName)
+.subscriptionType(SubscriptionType.Shared)
+.ackTimeout(1000, TimeUnit.MILLISECONDS)
+.receiverQueueSize(100)
+.messageListener((c1, msg) -> {
+assertNotNull(msg, "Message cannot be null");
+String receivedMessage = new String(msg.getData());
+latch.countDown();
+
+log.info("Received message [{}] in the listener, latch: {}",
+receivedMessage, latch.getCount());
+// since not acked, it should retry another time
+//c1.acknowledgeAsync(msg);
+})
+.subscribe();
+assertTrue(consumer instanceof MultiTopicsConsumerImpl);
+
+MultiTopicsConsumerImpl topicsConsumer = (MultiTopicsConsumerImpl)
consumer;
+
+// 3. producer publish messages
+for (int i = 0; i < totalMessages; i++) {
+producer1.send((messagePredicate + "producer1-" + i).getBytes());
+}
+
+// verify should not time out, because of message redelivered several
times.
+latch.await();
+
+consumer.close();
+}
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 75fdac6..8b8556a 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -485,6 +485,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
ConsumerConfigurationData interna
[incubator-pulsar] branch master updated: [proxy][functions] Issue #2154: proxy should be able to forward rest requests to function workers cluster (#2560)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 07d4226 [proxy][functions] Issue #2154: proxy should be able to
forward rest requests to function workers cluster (#2560)
07d4226 is described below
commit 07d42261eb2d685de6f9a5ab214f4d08944ffb6e
Author: Sijie Guo
AuthorDate: Fri Sep 14 01:10:35 2018 -0700
[proxy][functions] Issue #2154: proxy should be able to forward rest
requests to function workers cluster (#2560)
*Motivation*
Function workers can be deployed as a separate cluster. If so, proxy is not
able to forward the functions
related rest calls to the correct server.
*Changes*
Add two settings in proxy configuration to allow proxy configuring
forwarding functions related rest calls
to function worker cluster.
*Tests*
Verified with changes in integration tests (manually). It is hard to add
the integration tests based on
current integration tests. will add them in a separate PR.
---
conf/proxy.conf | 5 +
.../apache/pulsar/proxy/server/AdminProxyHandler.java| 16 ++--
.../apache/pulsar/proxy/server/ProxyConfiguration.java | 12
3 files changed, 31 insertions(+), 2 deletions(-)
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 9b307cc..b95c4d3 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -23,6 +23,11 @@ zookeeperServers=
# Configuration store connection string (as a comma-separated list)
configurationStoreServers=
+# If function workers are setup in a separate cluster, configure the following
2 settings
+# to point to the function workers cluster
+functionWorkerWebServiceURL=
+functionWorkerWebServiceURLTLS=
+
# ZooKeeper session timeout (in milliseconds)
zookeeperSessionTimeoutMs=3
diff --git
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
index 49c789c..d6c32df 100644
---
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
+++
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/AdminProxyHandler.java
@@ -49,12 +49,15 @@ class AdminProxyHandler extends AsyncProxyServlet {
private final ProxyConfiguration config;
private final BrokerDiscoveryProvider discoveryProvider;
private final String brokerWebServiceUrl;
+private final String functionWorkerWebServiceUrl;
AdminProxyHandler(ProxyConfiguration config, BrokerDiscoveryProvider
discoveryProvider) {
this.config = config;
this.discoveryProvider = discoveryProvider;
this.brokerWebServiceUrl = config.isTlsEnabledWithBroker() ?
config.getBrokerWebServiceURLTLS()
: config.getBrokerWebServiceURL();
+this.functionWorkerWebServiceUrl = config.isTlsEnabledWithBroker() ?
config.getFunctionWorkerWebServiceURLTLS()
+: config.getFunctionWorkerWebServiceURL();
}
@Override
@@ -122,7 +125,16 @@ class AdminProxyHandler extends AsyncProxyServlet {
protected String rewriteTarget(HttpServletRequest request) {
StringBuilder url = new StringBuilder();
-if (isBlank(brokerWebServiceUrl)) {
+boolean isFunctionsRestRequest = false;
+String requestUri = request.getRequestURI();
+if (requestUri.startsWith("/admin/v2/functions")
+|| requestUri.startsWith("/admin/functions")) {
+isFunctionsRestRequest = true;
+}
+
+if (isFunctionsRestRequest && !isBlank(functionWorkerWebServiceUrl)) {
+url.append(functionWorkerWebServiceUrl);
+} else if (isBlank(brokerWebServiceUrl)) {
try {
ServiceLookupData availableBroker =
discoveryProvider.nextBroker();
@@ -148,7 +160,7 @@ class AdminProxyHandler extends AsyncProxyServlet {
if (url.lastIndexOf("/") == url.length() - 1) {
url.deleteCharAt(url.lastIndexOf("/"));
}
-url.append(request.getRequestURI());
+url.append(requestUri);
String query = request.getQueryString();
if (query != null) {
diff --git
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
index b4e8afb..0155baa 100644
---
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
+++
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
@@ -48,6 +48,10 @@ public class ProxyConfiguration implements
PulsarConfiguration {
private String brokerWebServiceURL;
private String brokerWebServiceURLTLS;
+
[incubator-pulsar] branch master updated: PIP-22: Dead Letter Topic (#2508)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 95fe84c PIP-22: Dead Letter Topic (#2508)
95fe84c is described below
commit 95fe84c7b43060102bb08f0dfe85dadb152f4542
Author: penghui
AuthorDate: Fri Sep 14 15:37:58 2018 +0800
PIP-22: Dead Letter Topic (#2508)
### Motivation
Fixes #189
When consumer got messages from pulsar, It's difficult to ensure every
message can be consume success. Pulsar support message redelivery feature by
set acknowledge timeout when create a new consumer. This is a good feature
guarantee consumer will not lost messages.
But however, some message will redelivery so many times possible, even to
the extent that it can be never stop.
So, It's necessary to support a feature to control it by pulsar. Users can
use this feature and customize this feature to control the message redelivery
behavior. The feature named Dead Letter Topic.
### Modifications
Consumer can set maximum number of redeliveries by java client.
Consumer can set the name of Dead Letter Topic by java client, It’s not
necessary.
Message exceeding the maximum number of redeliveries should send to Dead
Letter Topic and acknowledged automatic.
### Result
If consumer enable future of dead letter topic. When Message exceeding the
maximum number of redeliveries, message will send to the Dead Letter Topic and
acknowledged automatic.
---
.../org/apache/pulsar/broker/service/Consumer.java | 3 +-
.../apache/pulsar/broker/service/Dispatcher.java | 2 +
.../broker/service/InMemoryRedeliveryTracker.java | 58 +
.../pulsar/broker/service/RedeliveryTracker.java | 36 +++
.../broker/service/RedeliveryTrackerDisabled.java | 55 +
.../NonPersistentDispatcherMultipleConsumers.java | 9 +
...onPersistentDispatcherSingleActiveConsumer.java | 9 +
.../PersistentDispatcherMultipleConsumers.java | 14 +-
.../PersistentDispatcherSingleActiveConsumer.java | 11 +
.../service/persistent/PersistentSubscription.java | 1 +
.../apache/pulsar/client/impl/RawReaderImpl.java | 2 +-
.../pulsar/client/api/DeadLetterTopicTest.java | 249 +
.../client/impl/CompactedOutBatchMessageTest.java | 2 +-
.../apache/pulsar/client/api/ConsumerBuilder.java | 24 ++
.../apache/pulsar/client/api/DeadLetterPolicy.java | 32 +++
.../org/apache/pulsar/client/impl/ClientCnx.java | 2 +-
.../pulsar/client/impl/ConsumerBuilderImpl.java| 7 +
.../apache/pulsar/client/impl/ConsumerImpl.java| 111 -
.../impl/conf/ConsumerConfigurationData.java | 3 +
.../org/apache/pulsar/common/api/Commands.java | 5 +-
.../apache/pulsar/common/api/proto/PulsarApi.java | 57 +
pulsar-common/src/main/proto/PulsarApi.proto | 1 +
22 files changed, 680 insertions(+), 13 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index 883bf55..5752034 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -265,7 +265,8 @@ public class Consumer {
if (i == (entries.size() - 1)) {
promise = writePromise;
}
-ctx.write(Commands.newMessage(consumerId, messageId,
metadataAndPayload), promise);
+int redeliveryCount =
subscription.getDispatcher().getRedeliveryTracker().getRedeliveryCount(PositionImpl.get(messageId.getLedgerId(),
messageId.getEntryId()));
+ctx.write(Commands.newMessage(consumerId, messageId,
redeliveryCount, metadataAndPayload), promise);
messageId.recycle();
messageIdBuilder.recycle();
entry.release();
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
index 0d7b7d6..43f65cd 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Dispatcher.java
@@ -70,4 +70,6 @@ public interface Dispatcher {
void addUnAckedMessages(int unAckMessages);
+RedeliveryTracker getRedeliveryTracker();
+
}
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java
new file mode 100644
index 000..99b38cc
--- /dev/null
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/InMemoryRedeliveryTracker.java
[incubator-pulsar] branch master updated: Stop all functions gracefully on closing worker-service (#2548)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 95ed9b9 Stop all functions gracefully on closing worker-service
(#2548)
95ed9b9 is described below
commit 95ed9b91af20bcdd396ee34d3ea6cc94a52fea31
Author: Rajan Dhabalia
AuthorDate: Thu Sep 13 14:50:35 2018 -0700
Stop all functions gracefully on closing worker-service (#2548)
### Motivation
Right now, if stopping
[WorkerService](https://github.com/apache/incubator-pulsar/blob/master/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerService.java#L189)
doesn't stop functions and all the threads stayed alive event `WorkerService`
is stopped.
### Modifications
Stop all function resource gracefully while stopping worker service.
### Result
Function threads will not stay alive while stopping worker-service.
---
.../functions/worker/FunctionRuntimeManager.java | 20
1 file changed, 20 insertions(+)
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
index ee6eeec..43cd27b 100644
---
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
+++
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
@@ -389,6 +389,25 @@ public class FunctionRuntimeManager implements
AutoCloseable{
return Response.status(Status.OK).build();
}
+/**
+ * It stops all functions instances owned by current worker
+ * @throws Exception
+ */
+public void stopAllOwnedFunctions() throws Exception {
+final String workerId = this.workerConfig.getWorkerId();
+Map assignments =
workerIdToAssignments.get(workerId);
+if (assignments != null) {
+assignments.values().forEach(assignment -> {
+String fullyQualifiedInstanceId =
Utils.getFullyQualifiedInstanceId(assignment.getInstance());
+try {
+stopFunction(fullyQualifiedInstanceId, false);
+} catch (Exception e) {
+log.warn("Failed to stop function {} - {}",
fullyQualifiedInstanceId, e.getMessage());
+}
+});
+}
+}
+
private void stopFunction(String fullyQualifiedInstanceId, boolean
restart) throws Exception {
log.info("[{}] {}..", restart ? "restarting" : "stopping",
fullyQualifiedInstanceId);
FunctionRuntimeInfo functionRuntimeInfo =
this.getFunctionRuntimeInfo(fullyQualifiedInstanceId);
@@ -647,6 +666,7 @@ public class FunctionRuntimeManager implements
AutoCloseable{
@Override
public void close() throws Exception {
+stopAllOwnedFunctions();
this.functionActioner.close();
this.functionAssignmentTailer.close();
if (runtimeFactory != null) {
[incubator-pulsar] branch master updated: Another fix for website build (#2567)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 0964dad Another fix for website build (#2567)
0964dad is described below
commit 0964dadcd3f425a2d4c424d4ab1de1fb7e35ce84
Author: Matteo Merli
AuthorDate: Thu Sep 13 17:49:36 2018 -0400
Another fix for website build (#2567)
### Motivation
Comma is missing in the javascript list.
---
site2/website/scripts/replace.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/site2/website/scripts/replace.js b/site2/website/scripts/replace.js
index 13c9569..2970688 100644
--- a/site2/website/scripts/replace.js
+++ b/site2/website/scripts/replace.js
@@ -118,7 +118,7 @@ for (v of versions) {
`${v}`,
binaryReleaseUrl(`${v}`),
connectorReleaseUrl(`${v}`),
- prestoPulsarReleaseUrl(`${latestVersion}`)
+ prestoPulsarReleaseUrl(`${latestVersion}`),
downloadPageUrl(),
rpmReleaseUrl(`${v}`, ""),
rpmReleaseUrl(`${v}`, "-debuginfo"),
[incubator-pulsar] branch master updated: Fix: get function status with auth enable (#2516)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new e5a7b53 Fix: get function status with auth enable (#2516)
e5a7b53 is described below
commit e5a7b532ffbf1d8813fc31403162fe3c17e937f5
Author: Rajan Dhabalia
AuthorDate: Thu Sep 13 11:29:37 2018 -0700
Fix: get function status with auth enable (#2516)
---
.../pulsar/broker/admin/impl/FunctionsBase.java| 2 +-
.../client/admin/internal/FunctionsImpl.java | 5 ++-
.../functions/worker/FunctionRuntimeManager.java | 41 --
.../functions/worker/rest/api/FunctionsImpl.java | 6 ++--
.../worker/rest/api/v2/FunctionApiV2Resource.java | 2 +-
5 files changed, 23 insertions(+), 33 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
index def5452..62d12ec 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/FunctionsBase.java
@@ -160,7 +160,7 @@ public class FunctionsBase extends AdminResource implements
Supplierhttp://%s:%d/admin/functions/%s/%s/%s/%d/status;,
-workerInfo.getWorkerHostname(), workerInfo.getPort(),
tenant, namespace, functionName, instanceId))
-.request(MediaType.TEXT_PLAIN)
-.get(String.class);
-
-InstanceCommunication.FunctionStatus.Builder functionStatusBuilder
= InstanceCommunication.FunctionStatus.newBuilder();
-try {
-
org.apache.pulsar.functions.utils.Utils.mergeJson(jsonResponse,
functionStatusBuilder);
-} catch (IOException e) {
-log.warn("Got invalid function status response from {}",
workerInfo, e);
-throw new RuntimeException(e);
-}
-functionStatusBuilder.setWorkerId(assignedWorkerId);
-functionStatus = functionStatusBuilder.build();
+URI redirect =
UriBuilder.fromUri(uri).host(workerInfo.getWorkerHostname()).port(workerInfo.getPort()).build();
+throw new
WebApplicationException(Response.temporaryRedirect(redirect).build());
}
return functionStatus;
@@ -426,9 +412,10 @@ public class FunctionRuntimeManager implements
AutoCloseable{
* @param namespace the namespace the function belongs to
* @param functionName the function name
* @return a list of function statuses
+ * @throws PulsarAdminException
*/
public InstanceCommunication.FunctionStatusList getAllFunctionStatus(
-String tenant, String namespace, String functionName) {
+String tenant, String namespace, String functionName) throws
PulsarAdminException {
Collection assignments =
this.findFunctionAssignments(tenant, namespace, functionName);
@@ -438,13 +425,15 @@ public class FunctionRuntimeManager implements
AutoCloseable{
}
for (Assignment assignment : assignments) {
-
-InstanceCommunication.FunctionStatus functionStatus =
this.getFunctionInstanceStatus(
-
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(),
-
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(),
-
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(),
-assignment.getInstance().getInstanceId());
-
+boolean isOwner =
this.workerConfig.getWorkerId().equals(assignment.getWorkerId());
+InstanceCommunication.FunctionStatus functionStatus = isOwner
+? (getFunctionInstanceStatus(tenant, namespace,
functionName,
+assignment.getInstance().getInstanceId(), null))
+: this.functionAdmin.functions().getFunctionStatus(
+
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getTenant(),
+
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getNamespace(),
+
assignment.getInstance().getFunctionMetaData().getFunctionDetails().getName(),
+assignment.getInstance().getInstanceId());
functionStatusListBuilder.addFunctionStatusList(functionStatus);
}
return functionStatusListBuilder.build();
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/FunctionsImpl.java
index 136bab0..df82c0d 100644
---
a/pulsar-functions/worker/src
[incubator-pulsar] branch master updated: Remove property term from dashboard. (#2553)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 5506b54 Remove property term from dashboard. (#2553)
5506b54 is described below
commit 5506b54d99a5f85ee8d6b5fcb1e652b83bad3ef1
Author: cckellogg
AuthorDate: Wed Sep 12 02:45:44 2018 -0700
Remove property term from dashboard. (#2553)
This updates the html to display the new v2 naming.
### Motivation
This brings the dashboard up to date with the new v2 naming.
Property/properties are now tenant/tenants.
---
dashboard/django/stats/templates/stats/base.html | 2 +-
dashboard/django/stats/templates/stats/home.html | 4 ++--
dashboard/django/stats/views.py | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/dashboard/django/stats/templates/stats/base.html
b/dashboard/django/stats/templates/stats/base.html
index 1e5843f..9ffdb77 100644
--- a/dashboard/django/stats/templates/stats/base.html
+++ b/dashboard/django/stats/templates/stats/base.html
@@ -50,7 +50,7 @@
{% block nav-global %}
-Properties |
+Tenants |
Brokers |
Topics |
Clusters
diff --git a/dashboard/django/stats/templates/stats/home.html
b/dashboard/django/stats/templates/stats/home.html
index 828bfae..7e9cb80 100644
--- a/dashboard/django/stats/templates/stats/home.html
+++ b/dashboard/django/stats/templates/stats/home.html
@@ -30,7 +30,7 @@
-{% column_header properties 'name' 'Property' %}
+{% column_header properties 'name' 'Tenant' %}
{% column_header properties 'numNamespaces' 'Namespaces' %}
{% column_header properties 'numTopics' 'Topics' %}
{% column_header properties 'numProducers' 'Producers' %}
@@ -64,7 +64,7 @@
{{property.storage | filesizeformat}}
{% empty %}
-No properties
+No tenants
{% endfor %}
diff --git a/dashboard/django/stats/views.py b/dashboard/django/stats/views.py
index 9ab6859..6aba518 100644
--- a/dashboard/django/stats/views.py
+++ b/dashboard/django/stats/views.py
@@ -62,7 +62,7 @@ def home(request):
return render(request, 'stats/home.html', {
'properties': properties,
-'title' : 'Properties',
+'title' : 'Tenants',
})
def property(request, property_name):
[incubator-pulsar] branch master updated: [documentation][example] Flink Source & Sink Connector (#2561)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 71003e2 [documentation][example] Flink Source & Sink Connector
(#2561)
71003e2 is described below
commit 71003e20e6606527c2f538ba2059850d9539d878
Author: Sijie Guo
AuthorDate: Wed Sep 12 01:38:56 2018 -0700
[documentation][example] Flink Source & Sink Connector (#2561)
### Motivation
We added flink source connector (#2441) and sink connector (#2434). It
would be great to an example to show how to use flink source & sink connector.
### Modifications
- introduce an `examples` module
- introduce an `examples/flink-consumer-source` module
- add a word count example to use flink source and sink connector
### Result
be able to know how to use flink source & sink connector
---
examples/flink-consumer-source/README.md | 76 +
examples/flink-consumer-source/pom.xml | 97
.../flink/PulsarConsumerSourceWordCount.java | 126 +
examples/pom.xml | 39 +++
pom.xml| 3 +
.../connectors/pulsar/FlinkPulsarProducer.java | 10 +-
.../connectors/pulsar/PulsarConsumerSource.java| 1 -
.../pulsar/partitioner/PulsarKeyExtractor.java | 4 +-
8 files changed, 350 insertions(+), 6 deletions(-)
diff --git a/examples/flink-consumer-source/README.md
b/examples/flink-consumer-source/README.md
new file mode 100644
index 000..38b75b6
--- /dev/null
+++ b/examples/flink-consumer-source/README.md
@@ -0,0 +1,76 @@
+## Apache Flink Connectors for Pulsar
+
+This page describes how to use the connectors to read and write Pulsar topics
with [Apache Flink](https://flink.apache.org/) stream processing applications.
+
+Build end-to-end stream processing pipelines that use Pulsar as the stream
storage and message bus, and Apache Flink for computation over the streams.
+See the [Pulsar
Concepts](https://pulsar.incubator.apache.org/docs/en/concepts-overview/) page
for more information.
+
+## Example
+
+### PulsarConsumerSourceWordCount
+
+This Flink streaming job is consuming from a Pulsar topic and couting the
wordcount in a streaming fashion. The job can write the word count results
+to stdout or another Pulsar topic.
+
+The steps to run the example:
+
+1. Start Pulsar Standalone.
+
+You can follow the
[instructions](https://pulsar.incubator.apache.org/docs/en/standalone/) to
start a Pulsar standalone locally.
+
+```shell
+$ bin/pulsar standalone
+```
+
+2. Start Flink locally.
+
+You can follow the
[instructions](https://ci.apache.org/projects/flink/flink-docs-release-1.6/quickstart/setup_quickstart.html)
to download and start Flink.
+
+```shell
+$ ./bin/start-cluster.sh
+```
+
+3. Build the examples.
+
+```shell
+$ cd ${PULSAR_HOME}
+$ mvn clean install -DskipTests
+```
+
+4. Run the word count example to print results to stdout.
+
+```shell
+$ ./bin/flink run
${PULSAR_HOME}/examples/flink-consumer-source/target/pulsar-flink-streaming-wordcount.jar
--service-url pulsar://localhost:6650 --input-topic test_src --subscription
test_sub
+```
+
+5. Produce messages to topic `test_src`.
+
+```shell
+$ bin/pulsar-client produce -m "hello world test again" -n 100 test_src
+```
+
+6. You can check the flink taskexecutor `.out` file. The `.out` file will
print the counts at the end of each time window as long as words are floating
in, e.g.:
+
+```shell
+PulsarConsumerSourceWordCount.WordWithCount(word=hello, count=200)
+PulsarConsumerSourceWordCount.WordWithCount(word=again, count=200)
+PulsarConsumerSourceWordCount.WordWithCount(word=test, count=200)
+PulsarConsumerSourceWordCount.WordWithCount(word=world, count=200)
+PulsarConsumerSourceWordCount.WordWithCount(word=hello, count=100)
+PulsarConsumerSourceWordCount.WordWithCount(word=again, count=100)
+PulsarConsumerSourceWordCount.WordWithCount(word=test, count=100)
+```
+
+Alternatively, when you run the flink word count example at step 4, you can
choose dump the result to another pulsar topic.
+
+```shell
+$ ./bin/flink run
${PULSAR_HOME}/examples/flink-consumer-source/target/pulsar-flink-streaming-wordcount.jar
--service-url pulsar://localhost:6650 --input-topic test_src --subscription
test_sub --output-topic test_dest
+```
+
+Once the flink word count example is running, you can use `bin/pulsar-client`
to tail the results produced into topic `test_dest`.
+
+```shell
+$ bin/pulsar-client consume -n 0 -s test test_dest
+```
+
+You will see similar results as what you see at step 6 when running the word
count example to print results to stdout.
diff --git a/exam
[incubator-pulsar] branch master updated: Removed sha1 from release signing script (#2563)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 0ce9e1f Removed sha1 from release signing script (#2563)
0ce9e1f is described below
commit 0ce9e1f4ef5f372d8199eedbaa04811b10115dab
Author: Matteo Merli
AuthorDate: Wed Sep 12 03:11:26 2018 -0400
Removed sha1 from release signing script (#2563)
### Motivation
We have removed sha1 links from main download page but we should also
remove from the release signing script.
---
site2/docs/client-libraries-cpp.md | 10 +-
.../version-2.1.0-incubating/client-libraries-cpp.md | 6 +++---
src/sign-release.sh| 3 ---
3 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/site2/docs/client-libraries-cpp.md
b/site2/docs/client-libraries-cpp.md
index ee22377..864efdd 100644
--- a/site2/docs/client-libraries-cpp.md
+++ b/site2/docs/client-libraries-cpp.md
@@ -19,9 +19,9 @@ The Pulsar C++ client has been successfully tested on
**MacOS** and **Linux**.
| Link | Crypto files |
|--|--|
-| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc),
[sha1]({{pulsar:rpm:client}}.sha1), [sha512]({{pulsar:rpm:client}}.sha512) |
-| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) |
[asc]({{pulsar:rpm:client-debuginfo}}.asc),
[sha1]({{pulsar:rpm:client-debuginfo}}.sha1),
[sha512]({{pulsar:rpm:client-debuginfo}}.sha512) |
-| [client-devel]({{pulsar:rpm:client-devel}}) |
[asc]({{pulsar:rpm:client-devel}}.asc),
[sha1]({{pulsar:rpm:client-devel}}.sha1),
[sha512]({{pulsar:rpm:client-devel}}.sha512) |
+| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc),
[sha512]({{pulsar:rpm:client}}.sha512) |
+| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) |
[asc]({{pulsar:rpm:client-debuginfo}}.asc),
[sha512]({{pulsar:rpm:client-debuginfo}}.sha512) |
+| [client-devel]({{pulsar:rpm:client-devel}}) |
[asc]({{pulsar:rpm:client-devel}}.asc),
[sha512]({{pulsar:rpm:client-devel}}.sha512) |
To install a RPM package, download the RPM packages and install them using the
following command:
@@ -33,8 +33,8 @@ $ rpm -ivh apache-pulsar-client*.rpm
| Link | Crypto files |
|--|--|
-| [client]({{pulsar:deb:client}}) | [asc]({{pulsar:deb:client}}.asc),
[sha1]({{pulsar:deb:client}}.sha1), [sha512]({{pulsar:deb:client}}.sha512) |
-| [client-devel]({{pulsar:deb:client-devel}}) |
[asc]({{pulsar:deb:client-devel}}.asc),
[sha1]({{pulsar:deb:client-devel}}.sha1),
[sha512]({{pulsar:deb:client-devel}}.sha512) |
+| [client]({{pulsar:deb:client}}) | [asc]({{pulsar:deb:client}}.asc),
[sha512]({{pulsar:deb:client}}.sha512) |
+| [client-devel]({{pulsar:deb:client-devel}}) |
[asc]({{pulsar:deb:client-devel}}.asc),
[sha512]({{pulsar:deb:client-devel}}.sha512) |
To install a DEB package, download the DEB packages and install them using the
following command:
diff --git
a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md
b/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md
index 2b6d137..ae67ebc 100644
---
a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md
+++
b/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-cpp.md
@@ -20,9 +20,9 @@ The Pulsar C++ client has been successfully tested on
**MacOS** and **Linux**.
| Link | Crypto files |
|--|--|
-| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc),
[sha1]({{pulsar:rpm:client}}.sha1), [sha512]({{pulsar:rpm:client}}.sha512) |
-| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) |
[asc]({{pulsar:rpm:client-debuginfo}}.asc),
[sha1]({{pulsar:rpm:client-debuginfo}}.sha1),
[sha512]({{pulsar:rpm:client-debuginfo}}.sha512) |
-| [client-devel]({{pulsar:rpm:client-devel}}) |
[asc]({{pulsar:rpm:client-devel}}.asc),
[sha1]({{pulsar:rpm:client-devel}}.sha1),
[sha512]({{pulsar:rpm:client-devel}}.sha512) |
+| [client]({{pulsar:rpm:client}}) | [asc]({{pulsar:rpm:client}}.asc),
[sha512]({{pulsar:rpm:client}}.sha512) |
+| [client-debuginfo]({{pulsar:rpm:client-debuginfo}}) |
[asc]({{pulsar:rpm:client-debuginfo}}.asc),
[sha512]({{pulsar:rpm:client-debuginfo}}.sha512) |
+| [client-devel]({{pulsar:rpm:client-devel}}) |
[asc]({{pulsar:rpm:client-devel}}.asc),
[sha512]({{pulsar:rpm:client-devel}}.sha512) |
To install a RPM package, down the RPM packages and install them using
following command:
diff --git a/src/sign-release.sh b/src/sign-release.sh
index b26e643..70977db 100755
--- a/src/sign-release.sh
+++ b/src/sign-release.sh
@@ -27,9 +27,6 @@ do
echo "Signing $FILE"
gpg --armor --output $FILE.asc --detach-sig $FILE
- # SHA-1 signature
- shasum -a 1 $FILE > $FILE.sha1
-
[incubator-pulsar] branch master updated: [ecosystem] Flink pulsar source connector (#2555)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new fc61056 [ecosystem] Flink pulsar source connector (#2555)
fc61056 is described below
commit fc61056ccab29bb068b65d71120364d14503f69f
Author: Sijie Guo
AuthorDate: Tue Sep 11 12:11:41 2018 -0700
[ecosystem] Flink pulsar source connector (#2555)
*Motivation*
This ports apache/flink#6200 to apache pulsar repo. It adds a pulsar source
connector
which will enable flink jobs to process messages from pulsar topics.
*Changes*
Add a PulsarConsumerSource connector
---
.../connectors/pulsar/PulsarConsumerSource.java| 204
.../connectors/pulsar/PulsarSourceBase.java| 31 ++
.../connectors/pulsar/PulsarSourceBuilder.java | 118 +
.../pulsar/PulsarConsumerSourceTests.java | 524 +
4 files changed, 877 insertions(+)
diff --git
a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java
b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java
new file mode 100644
index 000..f1b2595
--- /dev/null
+++
b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/PulsarConsumerSource.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flink.streaming.connectors.pulsar;
+
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.Configuration;
+import
org.apache.flink.streaming.api.functions.source.MessageAcknowledgingSourceBase;
+import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
+import org.apache.flink.util.IOUtils;
+
+import org.apache.pulsar.client.api.Consumer;
+import org.apache.pulsar.client.api.Message;
+import org.apache.pulsar.client.api.MessageId;
+import org.apache.pulsar.client.api.PulsarClient;
+import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.SubscriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Pulsar source (consumer) which receives messages from a topic and
acknowledges messages.
+ * When checkpointing is enabled, it guarantees at least once processing
semantics.
+ *
+ * When checkpointing is disabled, it auto acknowledges messages based on
the number of messages it has
+ * received. In this mode messages may be dropped.
+ */
+class PulsarConsumerSource extends MessageAcknowledgingSourceBase implements PulsarSourceBase {
+
+private static final Logger LOG =
LoggerFactory.getLogger(PulsarConsumerSource.class);
+
+private final int messageReceiveTimeoutMs = 100;
+private final String serviceUrl;
+private final String topic;
+private final String subscriptionName;
+private final DeserializationSchema deserializer;
+
+private PulsarClient client;
+private Consumer consumer;
+
+private boolean isCheckpointingEnabled;
+
+private final long acknowledgementBatchSize;
+private long batchCount;
+private long totalMessageCount;
+
+private transient volatile boolean isRunning;
+
+PulsarConsumerSource(PulsarSourceBuilder builder) {
+super(MessageId.class);
+this.serviceUrl = builder.serviceUrl;
+this.topic = builder.topic;
+this.deserializer = builder.deserializationSchema;
+this.subscriptionName = builder.subscriptionName;
+this.acknowledgementBatchSize = builder.acknowledgementBatchSize;
+}
+
+@Override
+public void open(Configuration parameters) throws Exception {
+super.open(parameters);
+
+final RuntimeContext context
[incubator-pulsar] branch master updated: Readd MockBookKeeper to Pulsar (#2544)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 69571f8 Readd MockBookKeeper to Pulsar (#2544)
69571f8 is described below
commit 69571f8ef67b99d66c7d405e17870100d0aa5080
Author: Ivan Kelly
AuthorDate: Tue Sep 11 01:31:14 2018 +0200
Readd MockBookKeeper to Pulsar (#2544)
These mocks were moved out of pulsar to the bookkeeper project a few
months ago. While it is good to have mocks generally available for
bookkeeper, if the mock is not in the pulsar code base and we want to
change the behaviour of the mock for a specific case, we need to wait
for a bookkeeper release cycle to do so. It's better to have the mock
in Pulsar, so we can bend it to our needs.
---
.../bookkeeper/client/PulsarMockBookKeeper.java| 284 +
.../bookkeeper/client/PulsarMockLedgerHandle.java | 243 ++
.../bookkeeper/client/PulsarMockReadHandle.java| 125 +
.../mledger/impl/ManagedLedgerErrorsTest.java | 4 +-
.../bookkeeper/test/MockedBookKeeperTestCase.java | 6 +-
.../broker/MockedBookKeeperClientFactory.java | 18 +-
.../broker/auth/MockedPulsarServiceBaseTest.java | 31 ++-
.../PersistentDispatcherFailoverConsumerTest.java | 3 +-
.../pulsar/broker/service/PersistentTopicTest.java | 3 +-
.../pulsar/broker/service/ServerCnxTest.java | 3 +-
.../impl/BlobStoreManagedLedgerOffloaderTest.java | 6 +-
11 files changed, 704 insertions(+), 22 deletions(-)
diff --git
a/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java
b/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java
new file mode 100644
index 000..f3689e9
--- /dev/null
+++
b/managed-ledger/src/test/java/org/apache/bookkeeper/client/PulsarMockBookKeeper.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.bookkeeper.client;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
+import org.apache.bookkeeper.client.AsyncCallback.DeleteCallback;
+import org.apache.bookkeeper.client.AsyncCallback.OpenCallback;
+import org.apache.bookkeeper.client.api.OpenBuilder;
+import org.apache.bookkeeper.client.api.ReadHandle;
+import org.apache.bookkeeper.client.impl.OpenBuilderBase;
+import org.apache.bookkeeper.common.concurrent.FutureUtils;
+import org.apache.bookkeeper.conf.ClientConfiguration;
+import org.apache.zookeeper.ZooKeeper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Mocked version of BookKeeper client that keeps all ledgers data in memory.
+ *
+ * This mocked client is meant to be used in unit tests for applications
using the BookKeeper API.
+ */
+public class PulsarMockBookKeeper extends BookKeeper {
+
+final ExecutorService executor;
+final ZooKeeper zkc;
+
+@Override
+public ZooKeeper getZkHandle() {
+return zkc;
+}
+
+@Override
+public ClientConfiguration getConf() {
+return super.getConf();
+}
+
+Map ledgers = new ConcurrentHashMap<>();
+AtomicLong sequence = new AtomicLong(3);
+
+CompletableFuture defaultResponse =
CompletableFuture.completedFuture(null);
+List> failures = new ArrayList<>();
+
+public PulsarMockBookKeeper(ZooKeeper zkc, ExecutorService executor)
throws Exception {
+this.zkc = zkc;
+this.executor = executor;
+}
+
+@Override
+public LedgerHandle createLedger(DigestType digestType, byte passwd[])
+throws BKException, InterruptedException {
+return createLedger(3, 2, digestTyp
[incubator-pulsar] branch master updated: Fix dashboard to work with python3. (#2552)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new fc2d1d3 Fix dashboard to work with python3. (#2552)
fc2d1d3 is described below
commit fc2d1d3da7ffb8bb47e30ec49c07cf8d9f735246
Author: cckellogg
AuthorDate: Mon Sep 10 13:33:29 2018 -0700
Fix dashboard to work with python3. (#2552)
### Motivation
Make dashboard work with python 3.
---
dashboard/django/collector.py | 10 +-
dashboard/django/stats/views.py | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/dashboard/django/collector.py b/dashboard/django/collector.py
index 76c0646..c8c3b13 100755
--- a/dashboard/django/collector.py
+++ b/dashboard/django/collector.py
@@ -63,7 +63,7 @@ def fetch_broker_stats(cluster, broker_url, timestamp):
def _fetch_broker_stats(cluster, broker_host_port, timestamp):
broker_url = 'http://%s/' % broker_host_port
-print 'Getting stats for %s' % broker_host_port
+print('Getting stats for %s' % broker_host_port)
broker, _ = Broker.objects.get_or_create(
url = broker_host_port,
@@ -262,7 +262,7 @@ def fetch_stats():
if cluster_name == 'global': continue
cluster_url = get(args.serviceUrl, '/admin/clusters/' +
cluster_name)['serviceUrl']
-print 'Cluster:', cluster_name, '->', cluster_url
+print('Cluster:', cluster_name, '->', cluster_url)
cluster, created = Cluster.objects.get_or_create(name=cluster_name)
if cluster_url != cluster.serviceUrl:
cluster.serviceUrl = cluster_url
@@ -275,7 +275,7 @@ def fetch_stats():
f = pool.apply_async(fetch_broker_stats, (cluster,
broker_host_port, timestamp))
futures.append(f)
except Exception as e:
-print 'ERROR: ', e
+print('ERROR: ', e)
pool.close()
@@ -300,10 +300,10 @@ def purge_db():
Consumer.objects.filter(timestamp__lt = threshold).delete()
def collect_and_purge():
-print '-- Starting stats collection'
+print('-- Starting stats collection')
fetch_stats()
purge_db()
-print '-- Finished collecting stats'
+print('-- Finished collecting stats')
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
diff --git a/dashboard/django/stats/views.py b/dashboard/django/stats/views.py
index 426e9b1..9ab6859 100644
--- a/dashboard/django/stats/views.py
+++ b/dashboard/django/stats/views.py
@@ -56,7 +56,7 @@ def home(request):
throughputOut = Sum('namespace__topic__msgThroughputOut'),
)
-print properties.query
+print(properties.query)
properties = Table(request, properties, default_sort='name')
[incubator-pulsar] branch master updated: Elastic connector (#2546)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 928c3e1 Elastic connector (#2546)
928c3e1 is described below
commit 928c3e1a9e1e1d27b3bf5bf816890d1316c4b5d2
Author: David Kjerrumgaard <35466513+david-stream...@users.noreply.github.com>
AuthorDate: Mon Sep 10 13:31:49 2018 -0700
Elastic connector (#2546)
### Motivation
Added a sink connector that writes JSON documents into ElasticSearch
### Modifications
Added new pulsar-io module and associated integration tests
### Result
An ElasticSearch sink connector will be available for use.
---
pulsar-io/{hdfs => elastic-search}/pom.xml | 38 --
.../elasticsearch/ElasticSearchAbstractSink.java | 150 +
.../io/elasticsearch/ElasticSearchConfig.java | 88
.../io/elasticsearch/ElasticSearchStringSink.java | 35 +
.../pulsar/io/elasticsearch/package-info.java | 19 +++
.../resources/META-INF/services/pulsar-io.yaml | 22 +++
.../io/elasticsearch/ElasticSearchConfigTests.java | 125 +
.../io/elasticsearch/ElasticSearchSinkTests.java | 144
.../pulsar/io/elasticsearch/data/Profile.java | 33 +
.../pulsar/io/elasticsearch/data/UserProfile.java | 35 +
.../src/test/resources/sinkConfig.yaml | 25
pulsar-io/hdfs/pom.xml | 1 +
pulsar-io/pom.xml | 1 +
tests/integration/pom.xml | 7 +
.../containers/ElasticSearchContainer.java | 47 +++
.../integration/functions/PulsarFunctionsTest.java | 7 +-
.../integration/io/ElasticSearchSinkTester.java| 75 +++
.../pulsar/tests/integration/io/SinkTester.java| 3 +-
.../tests/integration/suites/PulsarTestSuite.java | 6 +
19 files changed, 851 insertions(+), 10 deletions(-)
diff --git a/pulsar-io/hdfs/pom.xml b/pulsar-io/elastic-search/pom.xml
similarity index 70%
copy from pulsar-io/hdfs/pom.xml
copy to pulsar-io/elastic-search/pom.xml
index 0d55207..1339b2f 100644
--- a/pulsar-io/hdfs/pom.xml
+++ b/pulsar-io/elastic-search/pom.xml
@@ -25,15 +25,24 @@
pulsar-io
2.2.0-incubating-SNAPSHOT
- pulsar-io-hdfs
+ pulsar-io-elastic-search
+ Pulsar IO :: ElasticSearch
+
+
+jcenter
+https://jcenter.bintray.com/
+
+
+
-
+
+
${project.groupId}
pulsar-io-core
${project.version}
-
+
com.fasterxml.jackson.core
jackson-databind
@@ -44,15 +53,28 @@
jackson-dataformat-yaml
+
+ org.apache.commons
+ commons-lang3
+ 3.4
+
+
- org.apache.hadoop
- hadoop-client
- 3.1.1
+ org.elasticsearch.client
+ elasticsearch-rest-high-level-client
+ 6.3.2
+
+
+net.andreinc.mockneat
+mockneat
+0.2.2
+test
+
- org.testng
- testng
+ com.google.code.gson
+ gson
test
diff --git
a/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java
b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java
new file mode 100644
index 000..3760d40
--- /dev/null
+++
b/pulsar-io/elastic-search/src/main/java/org/apache/pulsar/io/elasticsearch/ElasticSearchAbstractSink.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.io.elasticsearch;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Map;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Usernam
[incubator-pulsar] branch master updated: [docker] introduce a pulsar standalone image (#2545)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 2812fef [docker] introduce a pulsar standalone image (#2545) 2812fef is described below commit 2812fefb72aa8f7277a8a97d8842b0ad2dc6719b Author: Sijie Guo AuthorDate: Mon Sep 10 13:31:19 2018 -0700 [docker] introduce a pulsar standalone image (#2545) ## Motivation `pulsar` and `pulsar-all` are designed for running pulsar components on production. although it can be used for running standalone, people still need to run a separate docker container for pulsar dashboard. ## Changes introduce a `pulsar-standalone` image to package everything into one image. so people can run launch a pulsar standalone in one line command in docker, including dashboard. --- docker/pom.xml | 1 + docker/pulsar-standalone/Dockerfile| 55 docker/pulsar-standalone/conf/nginx-app.conf | 37 +++ docker/pulsar-standalone/conf/postgresql.conf | 38 +++ docker/pulsar-standalone/conf/supervisor-app.conf | 34 ++ docker/pulsar-standalone/conf/uwsgi.ini| 45 +++ docker/pulsar-standalone/conf/uwsgi_params | 16 + docker/pulsar-standalone/django/collector.py | 345 + docker/pulsar-standalone/django/collector.sh | 23 ++ .../pulsar-standalone/django/dashboard/__init__.py | 19 ++ .../pulsar-standalone/django/dashboard/settings.py | 181 +++ docker/pulsar-standalone/django/dashboard/urls.py | 45 +++ docker/pulsar-standalone/django/dashboard/wsgi.py | 35 +++ docker/pulsar-standalone/django/manage.py | 41 +++ docker/pulsar-standalone/django/stats/__init__.py | 19 ++ docker/pulsar-standalone/django/stats/admin.py | 34 ++ docker/pulsar-standalone/django/stats/apps.py | 26 ++ .../django/stats/migrations/0001_initial.py| 221 + .../django/stats/migrations/__init__.py| 20 ++ docker/pulsar-standalone/django/stats/models.py| 200 .../django/stats/templates/stats/base.html | 98 ++ .../django/stats/templates/stats/broker.html | 71 + .../django/stats/templates/stats/brokers.html | 90 ++ .../django/stats/templates/stats/clusters.html | 106 +++ .../django/stats/templates/stats/home.html | 74 + .../django/stats/templates/stats/namespace.html| 96 ++ .../django/stats/templates/stats/property.html | 72 + .../django/stats/templates/stats/topic.html| 197 .../django/stats/templates/stats/topics.html | 91 ++ .../django/stats/templatetags/__init__.py | 19 ++ .../django/stats/templatetags/stats_extras.py | 64 .../django/stats/templatetags/table.py | 91 ++ docker/pulsar-standalone/django/stats/tests.py | 22 ++ docker/pulsar-standalone/django/stats/urls.py | 37 +++ docker/pulsar-standalone/django/stats/views.py | 283 + docker/pulsar-standalone/pom.xml | 81 + site2/docs/getting-started-standalone.md | 27 +- .../getting-started-standalone.md | 14 +- 38 files changed, 2957 insertions(+), 11 deletions(-) diff --git a/docker/pom.xml b/docker/pom.xml index 675656a..bdc99f7 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -38,5 +38,6 @@ pulsar grafana pulsar-all +pulsar-standalone diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile new file mode 100644 index 000..869b4c0 --- /dev/null +++ b/docker/pulsar-standalone/Dockerfile @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM apachepulsar/pulsar-all:latest + +RUN apt-get update +RUN apt-get -y install postgresql sudo nginx supervisor + +# Python dependencies +RUN pip install uwsgi 'Django<2.0' psycopg2 pytz requests + +# Postgres configuration +COPY conf/postgresql.conf /etc/postgresql/9.6/mai
[incubator-pulsar] branch master updated: [documentation] Add section about licensing at contribution guide (#2511)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 3e9a445 [documentation] Add section about licensing at contribution guide (#2511) 3e9a445 is described below commit 3e9a445e9ced577c1e494875e9a490c3dee9a5ab Author: Sijie Guo AuthorDate: Sun Sep 9 16:51:03 2018 -0700 [documentation] Add section about licensing at contribution guide (#2511) --- site2/website/contributing.md | 9 + 1 file changed, 9 insertions(+) diff --git a/site2/website/contributing.md b/site2/website/contributing.md index 9552244..310045a 100644 --- a/site2/website/contributing.md +++ b/site2/website/contributing.md @@ -181,6 +181,15 @@ Then you can push your local, committed changes to your (forked) repository on G All code should have appropriate unit testing coverage. New code should have new tests in the same contribution. Bug fixes should include a regression test to prevent the issue from reoccurring. +### Licensing + +All code contributed to Pulsar will be licensed under [Apache License V2](https://www.apache.org/licenses/LICENSE-2.0). You need to ensure every new files you are adding have the right +license header. You can add license header to your files by running following command: + +```shell +$ mvn license:format +``` + ### Develop C++/Python Client Currently C++ and Python client are developed under `pulsar-client-cpp` directory. You can use following commands to build and test your c++ or python code changes.
[incubator-pulsar] branch master updated: Added cmake install for lib and headers (#2529)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 1c72ac3 Added cmake install for lib and headers (#2529)
1c72ac3 is described below
commit 1c72ac3fb1d20b4f12300c4d339cb1a383d718ae
Author: Matteo Merli
AuthorDate: Sun Sep 9 16:50:49 2018 -0700
Added cmake install for lib and headers (#2529)
### Motivation
Added `install` target to do local install for C++ client lib
---
pulsar-client-cpp/lib/CMakeLists.txt | 4
1 file changed, 4 insertions(+)
diff --git a/pulsar-client-cpp/lib/CMakeLists.txt
b/pulsar-client-cpp/lib/CMakeLists.txt
index 5b2dbf9..2116ed6 100644
--- a/pulsar-client-cpp/lib/CMakeLists.txt
+++ b/pulsar-client-cpp/lib/CMakeLists.txt
@@ -45,3 +45,7 @@ set_target_properties(pulsarShared PROPERTIES OUTPUT_NAME
${LIB_NAME} VERSION ${
target_link_libraries(pulsarStatic ${COMMON_LIBS})
target_link_libraries(pulsarShared ${COMMON_LIBS})
+install(TARGETS pulsarStatic DESTINATION lib)
+install(TARGETS pulsarShared DESTINATION lib)
+
+install(DIRECTORY "../include/pulsar" DESTINATION include)
\ No newline at end of file
[incubator-pulsar] branch master updated: fixed zookeeper name references in helm charts. (#2525)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 879c730 fixed zookeeper name references in helm charts. (#2525)
879c730 is described below
commit 879c730daf781ef27ad1344803ca97ad60cd4b62
Author: Victor
AuthorDate: Fri Sep 7 12:25:23 2018 -0700
fixed zookeeper name references in helm charts. (#2525)
### Motivation
zookeeper failed to start because of wrong ZOOKEEPER_SERVERS was set.
### Modifications
Changed the reference of zookeeper names by how they were created.
### Result
zookeeper started successfully and broker worked as expected.
---
.../kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml | 2 +-
deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml | 2 +-
deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml | 4 ++--
deployment/kubernetes/helm/pulsar/templates/proxy-configmap.yaml | 4 ++--
.../kubernetes/helm/pulsar/templates/zookeeper-statefulset.yaml | 2 +-
5 files changed, 7 insertions(+), 7 deletions(-)
diff --git
a/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml
b/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml
index 2e25112..4200e6c 100644
--- a/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml
+++ b/deployment/kubernetes/helm/pulsar/templates/autorecovery-configmap.yaml
@@ -33,6 +33,6 @@ metadata:
data:
zkServers:
{{- $global := . }}
-{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component }}{{ end }}
+{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{
$global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template
"pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }}
{{ toYaml .Values.autoRecovery.configData | indent 2 }}
{{- end }}
diff --git
a/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml
b/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml
index 31c66df..45a8546 100644
--- a/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml
+++ b/deployment/kubernetes/helm/pulsar/templates/bookkeeper-configmap.yaml
@@ -32,7 +32,7 @@ metadata:
data:
zkServers:
{{- $global := . }}
-{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component }}{{ end }}
+{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{
$global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template
"pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }}
# disable auto recovery on bookies since we will start AutoRecovery in
separated pods
autoRecoveryDaemonEnabled: "false"
{{ toYaml .Values.bookkeeper.configData | indent 2 }}
diff --git a/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml
b/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml
index 7d7df75..d577826 100644
--- a/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml
+++ b/deployment/kubernetes/helm/pulsar/templates/broker-configmap.yaml
@@ -32,10 +32,10 @@ metadata:
data:
zookeeperServers:
{{- $global := . }}
-{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component }}{{ end }}
+{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ template "pulsar.fullname" $global }}-{{
$global.Values.zookeeper.component }}-{{ printf "%d" $i }}.{{ template
"pulsar.fullname" $global }}-{{ $global.Values.zookeeper.component }}{{ end }}
configurationStoreServers:
{{- $global := . }}
-{{ range $i, $e := until (.Values.zookeeper.replicaCount | int) }}{{ if ne
$i 0 }},{{ end }}{{ printf "%s-%s-%s-%d.%s-%s-%s" $global.Release.Name
$global.Chart.Name $global.Values.zookeeper.component $i $global.Release.Name
$global.Chart.Name $global.Values.z
[incubator-pulsar] branch master updated: Typo in CSR generation docs (#2530) (#2531)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 6860a90 Typo in CSR generation docs (#2530) (#2531) 6860a90 is described below commit 6860a90bc685003afef17ab8bfa33e2701c99bfe Author: Ivan Kelly AuthorDate: Thu Sep 6 19:56:52 2018 +0200 Typo in CSR generation docs (#2530) (#2531) The output file for CSR generation was .cert.pem rather than .csr.pem, which causes confusion. It should be .csr.pem --- site2/docs/security-tls-authentication.md | 2 +- site2/docs/security-tls-transport.md| 2 +- .../version-2.1.0-incubating/security-tls-authentication.md | 2 +- .../versioned_docs/version-2.1.0-incubating/security-tls-transport.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/site2/docs/security-tls-authentication.md b/site2/docs/security-tls-authentication.md index d5eb7d7..10578d2 100644 --- a/site2/docs/security-tls-authentication.md +++ b/site2/docs/security-tls-authentication.md @@ -30,7 +30,7 @@ Generate the certificate request. When asked for a **common name**, enter the ** ```bash $ openssl req -config openssl.cnf \ - -key admin.key.pem -new -sha256 -out admin.cert.pem + -key admin.key.pem -new -sha256 -out admin.csr.pem ``` Sign with request with the certificate authority. Note that that client certs uses the **usr_cert** extension, which allows the cert to be used for client authentication. diff --git a/site2/docs/security-tls-transport.md b/site2/docs/security-tls-transport.md index 70db61f..460d2d0 100644 --- a/site2/docs/security-tls-transport.md +++ b/site2/docs/security-tls-transport.md @@ -96,7 +96,7 @@ Generate the certificate request... ```bash $ openssl req -config openssl.cnf \ - -key broker.key.pem -new -sha256 -out broker.cert.pem + -key broker.key.pem -new -sha256 -out broker.csr.pem ``` ... and sign it with the certificate authority. diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md index a9af2b7..121ab17 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-authentication.md @@ -31,7 +31,7 @@ Generate the certificate request. When asked for a **common name**, enter the ** ```bash $ openssl req -config openssl.cnf \ - -key admin.key.pem -new -sha256 -out admin.cert.pem + -key admin.key.pem -new -sha256 -out admin.csr.pem ``` Sign with request with the certificate authority. Note that that client certs uses the **usr_cert** extension, which allows the cert to be used for client authentication. diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md index 9924ed5..6981b8f 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/security-tls-transport.md @@ -97,7 +97,7 @@ Generate the certificate request... ```bash $ openssl req -config openssl.cnf \ - -key broker.key.pem -new -sha256 -out broker.cert.pem + -key broker.key.pem -new -sha256 -out broker.csr.pem ``` ... and sign it with the certificate authority.
[incubator-pulsar] branch master updated: TopicPatterns are now supported by python functions (#2506)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 669196c TopicPatterns are now supported by python functions (#2506)
669196c is described below
commit 669196c27d3c2028a112d939774744dae8ab1b88
Author: Sanjeev Kulkarni
AuthorDate: Thu Sep 6 01:01:03 2018 -0700
TopicPatterns are now supported by python functions (#2506)
---
.../instance/src/main/python/python_instance.py| 26 ++
.../src/main/python/python_instance_main.py| 3 ---
.../functions/utils/validation/ValidatorImpls.java | 9
3 files changed, 17 insertions(+), 21 deletions(-)
diff --git a/pulsar-functions/instance/src/main/python/python_instance.py
b/pulsar-functions/instance/src/main/python/python_instance.py
index b43b882..ddd546e 100644
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -172,7 +172,7 @@ class PythonInstance(object):
self.consumers[topic] = self.pulsar_client.subscribe(
str(topic), subscription_name,
consumer_type=mode,
-message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]),
+message_listener=partial(self.message_listener,
self.input_serdes[topic]),
unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms
else None
)
@@ -183,12 +183,20 @@ class PythonInstance(object):
serde_kclass = util.import_class(os.path.dirname(self.user_code),
consumer_conf.serdeClassName)
self.input_serdes[topic] = serde_kclass()
Log.info("Setting up consumer for topic %s with subname %s" % (topic,
subscription_name))
- self.consumers[topic] = self.pulsar_client.subscribe(
-str(topic), subscription_name,
-consumer_type=mode,
-message_listener=partial(self.message_listener, topic,
self.input_serdes[topic]),
-unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms
else None
- )
+ if consumer_conf.isRegexPattern:
+self.consumers[topic] = self.pulsar_client.subscribe_pattern(
+ str(topic), subscription_name,
+ consumer_type=mode,
+ message_listener=partial(self.message_listener,
self.input_serdes[topic]),
+ unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms
else None
+)
+ else:
+self.consumers[topic] = self.pulsar_client.subscribe(
+ str(topic), subscription_name,
+ consumer_type=mode,
+ message_listener=partial(self.message_listener,
self.input_serdes[topic]),
+ unacked_messages_timeout_ms=int(self.timeout_ms) if self.timeout_ms
else None
+)
function_kclass = util.import_class(os.path.dirname(self.user_code),
self.instance_config.function_details.className)
if function_kclass is None:
@@ -301,8 +309,8 @@ class PythonInstance(object):
batching_max_publish_delay_ms=1,
max_pending_messages=10)
- def message_listener(self, topic, serde, consumer, message):
-item = InternalMessage(message, topic, serde, consumer)
+ def message_listener(self, serde, consumer, message):
+item = InternalMessage(message, consumer.topic(), serde, consumer)
self.queue.put(item, True)
if self.atmost_once and self.auto_ack:
consumer.acknowledge(message)
diff --git a/pulsar-functions/instance/src/main/python/python_instance_main.py
b/pulsar-functions/instance/src/main/python/python_instance_main.py
index 514d7fe..d9f1132 100644
--- a/pulsar-functions/instance/src/main/python/python_instance_main.py
+++ b/pulsar-functions/instance/src/main/python/python_instance_main.py
@@ -82,9 +82,6 @@ def main():
Log.info("Starting Python instance with %s" % str(args))
- if function_details.source.topicsPattern:
-raise ValueError('topicsPattern is not supported by python client')
-
authentication = None
use_tls = False
tls_allow_insecure_connection = False
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
index cbeb970..f264d2a 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
@@ -506,15 +506,6 @@ public class ValidatorImpls {
if (functionConfig.getWindowConfig() != null) {
throw new IllegalArgumentException("There is currently no
support windowing in python");
}
-
-if (functionConfig.getTopicsPattern() != null &&
!fun
[incubator-pulsar] branch master updated: better way to estimate object size for sql (#2528)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 90a7448 better way to estimate object size for sql (#2528)
90a7448 is described below
commit 90a7448041aca5d5189b6a0b641453a50369faad
Author: Boyang Jerry Peng
AuthorDate: Thu Sep 6 01:00:12 2018 -0700
better way to estimate object size for sql (#2528)
Since we mocked ClassLayout.parseClass() because we need to exclude
org.openjdk.jol for license issues. Lets come up with another way to estimate
class size
---
pulsar-sql/presto-distribution/LICENSE | 3 +++
pulsar-sql/presto-distribution/pom.xml | 14
.../java/org/openjdk/jol/info/ClassLayout.java | 26 ++
3 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/pulsar-sql/presto-distribution/LICENSE
b/pulsar-sql/presto-distribution/LICENSE
index 51d4923..d7f2fc8 100644
--- a/pulsar-sql/presto-distribution/LICENSE
+++ b/pulsar-sql/presto-distribution/LICENSE
@@ -359,6 +359,7 @@ The Apache Software License, Version 2.0
- jsr305-3.0.2.jar
* Objenesis
- objenesis-2.1.jar
+- objenesis-2.6.jar
* Okio
- okio-1.13.0.jar
* Presto
@@ -379,6 +380,8 @@ The Apache Software License, Version 2.0
- snappy-java-1.1.1.3.jar
* Bean Validation API
- validation-api-1.1.0.Final.jar
+ * Objectsize
+- objectsize-0.0.12.jar
Protocol Buffers License
* Protocol Buffers
diff --git a/pulsar-sql/presto-distribution/pom.xml
b/pulsar-sql/presto-distribution/pom.xml
index ab55b3d..a355777 100644
--- a/pulsar-sql/presto-distribution/pom.xml
+++ b/pulsar-sql/presto-distribution/pom.xml
@@ -30,6 +30,8 @@
0.206
0.170
+2.6
+0.0.12
com.facebook.presto.server.PrestoServer
${project.artifactId}
@@ -80,6 +82,18 @@
provided
+
+org.objenesis
+objenesis
+${objenesis.version}
+
+
+
+com.twitter.common
+objectsize
+${objectsize.version}
+
+
diff --git
a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java
b/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java
index 16feb29..2a8bfa3 100644
---
a/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java
+++
b/pulsar-sql/presto-distribution/src/main/java/org/openjdk/jol/info/ClassLayout.java
@@ -18,18 +18,36 @@
*/
package org.openjdk.jol.info;
+import com.twitter.common.objectsize.ObjectSizeCalculator;
+import io.airlift.log.Logger;
+import org.objenesis.ObjenesisStd;
+
/**
* Mock class avoid a dependency on OpenJDK JOL,
* which is incompatible with the Apache License.
*/
public class ClassLayout {
-public static ClassLayout parseClass(Class ignored) {
-return new ClassLayout();
+private static final Logger log = Logger.get(ClassLayout.class);
+
+private int size;
+private static final int DEFAULT_SIZE = 64;
+
+private ClassLayout(int size) {
+this.size = size;
+}
+
+public static ClassLayout parseClass(Class clazz) {
+long size = DEFAULT_SIZE;
+try {
+size = ObjectSizeCalculator.getObjectSize(new
ObjenesisStd().newInstance(clazz));
+} catch (Throwable th) {
+log.info("Error estimating size of class %s",clazz, th);
+}
+return new ClassLayout(Math.toIntExact(size));
}
-// TODO find a better estimate of class size
public int instanceSize() {
-return 64; // random, means nothing
+return size;
}
}
\ No newline at end of file
[incubator-pulsar] branch master updated: PIP-23: Pulsar Java Client Interceptors. (#2471)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 7a92111 PIP-23: Pulsar Java Client Interceptors. (#2471)
7a92111 is described below
commit 7a92111fe7824d422d1739abd06d85955b652212
Author: penghui
AuthorDate: Thu Sep 6 05:25:30 2018 +0800
PIP-23: Pulsar Java Client Interceptors. (#2471)
### Motivation
Support user to add interceptors to producer and consumer.
### Modifications
Add Consumer interceptors.
```java
Message beforeConsume(Message message);
void onAcknowledge(MessageId messageId, Throwable cause);
void onAcknowledgeCumulative(MessageId messageId, Throwable cause);
```
Add Producer interceptors.
```java
Message beforeSend(Message message);
void onSendAcknowledgement(Message message, MessageId msgId, Throwable
cause);
```
### Result
Users can using interceptors in multiple scenarios, such as for
applications to add
custom logging or processing.
Master Issue: #2476
---
.../apache/pulsar/client/impl/RawReaderImpl.java | 2 +-
.../pulsar/broker/service/PersistentTopicTest.java | 7 +-
.../pulsar/broker/service/ReplicatorTest.java | 3 +-
.../broker/service/v1/V1_ReplicatorTest.java | 3 +-
.../apache/pulsar/client/api/InterceptorsTest.java | 359 +
.../org/apache/pulsar/client/api/Consumer.java | 6 +
.../apache/pulsar/client/api/ConsumerBuilder.java | 8 +
.../pulsar/client/api/ConsumerInterceptor.java | 99 ++
.../apache/pulsar/client/api/ProducerBuilder.java | 8 +
.../pulsar/client/api/ProducerInterceptor.java | 92 ++
.../apache/pulsar/client/impl/ConsumerBase.java| 25 +-
.../pulsar/client/impl/ConsumerBuilderImpl.java| 20 +-
.../apache/pulsar/client/impl/ConsumerImpl.java| 40 ++-
.../pulsar/client/impl/ConsumerInterceptors.java | 132
.../org/apache/pulsar/client/impl/MessageImpl.java | 4 +-
.../client/impl/MultiTopicsConsumerImpl.java | 12 +-
.../client/impl/PartitionedProducerImpl.java | 6 +-
.../impl/PatternMultiTopicsConsumerImpl.java | 4 +-
.../apache/pulsar/client/impl/ProducerBase.java| 18 +-
.../pulsar/client/impl/ProducerBuilderImpl.java| 18 +-
.../apache/pulsar/client/impl/ProducerImpl.java| 72 +++--
.../pulsar/client/impl/ProducerInterceptors.java | 110 +++
.../pulsar/client/impl/PulsarClientImpl.java | 45 +--
.../org/apache/pulsar/client/impl/ReaderImpl.java | 2 +-
.../pulsar/client/impl/TopicMessageImpl.java | 4 +
.../pulsar/functions/instance/ContextImplTest.java | 2 +-
.../MultiConsumersOneOutputTopicProducersTest.java | 27 +-
27 files changed, 1033 insertions(+), 95 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
index ae1a4db..99aec93 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
@@ -111,7 +111,7 @@ public class RawReaderImpl implements RawReader {
consumerFuture,
SubscriptionMode.Durable,
MessageId.earliest,
-Schema.BYTES);
+Schema.BYTES, null);
incomingRawMessages = new GrowableArrayBlockingQueue<>();
pendingRawReceives = new ConcurrentLinkedQueue<>();
}
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
index f8ac40a..8c175d7 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/PersistentTopicTest.java
@@ -24,6 +24,7 @@ import static
org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.matches;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.doAnswer;
@@ -36,7 +37,6 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
@@ -80,7 +80,6 @@ import org.apache.pulsar.broker.admin.AdminResou
[incubator-pulsar] branch master updated: [schema] implement generic schema/record for Schema.JSON (#2497)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8453d73 [schema] implement generic schema/record for Schema.JSON
(#2497)
8453d73 is described below
commit 8453d73e94f0a22de53a4a003c2084e1573ac855
Author: Sijie Guo
AuthorDate: Tue Sep 4 15:56:07 2018 -0700
[schema] implement generic schema/record for Schema.JSON (#2497)
### Motivation
The `GenericSchema` and `GenericRecord` are used for schema auto detection.
Currently it only supports Avro.
This PR is to support json.
### Changes
Introduce generic schema and generic record for json schema
---
.../pulsar/client/api/schema/GenericRecord.java| 4 +-
.../schema/{ => generic}/GenericAvroRecord.java| 7 +-
.../schema/{ => generic}/GenericAvroSchema.java| 33 ++--
.../impl/schema/generic/GenericJsonRecord.java | 74
.../impl/schema/generic/GenericJsonSchema.java | 63 ++
.../client/impl/schema/generic/GenericSchema.java | 77 +
.../impl/schema/generic/GenericSchemaTest.java | 99 ++
.../pulsar/client/schema/AvroSchemaTest.java | 93 +---
.../pulsar/client/schema/JSONSchemaTest.java | 38 ++---
.../pulsar/client/schema/SchemaTestUtils.java | 61 +
.../pulsar/client/impl/PulsarClientImpl.java | 10 +--
11 files changed, 398 insertions(+), 161 deletions(-)
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
index 0a4fce4..46a49a1 100644
---
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
@@ -38,7 +38,9 @@ public interface GenericRecord {
* @param field the field to retrieve the value
* @return the value object
*/
-Object getField(Field field);
+default Object getField(Field field) {
+return getField(field.getName());
+}
/**
* Retrieve the value of the provided fieldName.
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java
similarity index 94%
rename from
pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java
rename to
pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java
index fb65c7a..c9dbeb7 100644
---
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroRecord.java
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroRecord.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.client.impl.schema;
+package org.apache.pulsar.client.impl.schema.generic;
import java.util.List;
import java.util.stream.Collectors;
@@ -49,11 +49,6 @@ class GenericAvroRecord implements GenericRecord {
}
@Override
-public Object getField(Field field) {
-return getField(field.getName());
-}
-
-@Override
public Object getField(String fieldName) {
Object value = record.get(fieldName);
if (value instanceof Utf8) {
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java
similarity index 74%
rename from
pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java
rename to
pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java
index 4ccfe55..5fe4459 100644
---
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/GenericAvroSchema.java
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/impl/schema/generic/GenericAvroSchema.java
@@ -16,58 +16,40 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.pulsar.client.impl.schema;
+package org.apache.pulsar.client.impl.schema.generic;
import static com.google.common.base.Preconditions.checkArgument;
-import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
[incubator-pulsar] branch branch-2.1 updated: Handle /managed-ledgers znode existance on cluster init (#2379) (#2510)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new bf4a50b Handle /managed-ledgers znode existance on cluster init
(#2379) (#2510)
bf4a50b is described below
commit bf4a50b45dcaec7bd48b38a739bc4e7d43d36af1
Author: Ivan Kelly
AuthorDate: Tue Sep 4 23:41:47 2018 +0200
Handle /managed-ledgers znode existance on cluster init (#2379) (#2510)
ManagedLedger clients may create the /managed-ledger znode on boot, so
if a broker starts before the metadata is initialized, it could
potentially block initialization.
This patch changes this by making the existance of /managed-ledger a
non-error condition.
---
.../main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java| 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
index e35daef..9dd065a 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarClusterMetadataSetup.java
@@ -147,7 +147,12 @@ public class PulsarClusterMetadataSetup {
}
}
-localZk.create("/managed-ledgers", new byte[0],
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+try {
+localZk.create("/managed-ledgers", new byte[0],
ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+} catch (NodeExistsException e) {
+// Ignore
+}
+
localZk.create("/namespace", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
try {
[incubator-pulsar] branch master updated: Issue 2313: create a JDBC sink connector (#2440)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 79ac46a Issue 2313: create a JDBC sink connector (#2440)
79ac46a is described below
commit 79ac46a6ae50e48bed4ccb680d1f7945611f0565
Author: Jia Zhai
AuthorDate: Wed Sep 5 04:20:42 2018 +0800
Issue 2313: create a JDBC sink connector (#2440)
### Motivation
This change is trying to add a basic JDBC sink connector.
### Modifications
Add the jdbc module to the pulsar-io sub-module.
Add unit test and integration test for it.
### Result
ut and integration test pass.
Master Issue: #2442
---
distribution/io/src/assemble/io.xml| 5 +
pom.xml| 9 +-
pulsar-io/jdbc/lombok.config | 23 +++
pulsar-io/jdbc/pom.xml | 96 ++
.../apache/pulsar/io/jdbc/JdbcAbstractSink.java| 197 +
.../apache/pulsar/io/jdbc/JdbcAvroSchemaSink.java | 92 ++
.../org/apache/pulsar/io/jdbc/JdbcSinkConfig.java | 63 +++
.../java/org/apache/pulsar/io/jdbc/JdbcUtils.java | 178 +++
.../resources/META-INF/services/pulsar-io.yaml | 22 +++
.../org/apache/pulsar/io/jdbc/JdbcSinkTest.java| 135 ++
.../org/apache/pulsar/io/jdbc/JdbcUtilsTest.java | 95 ++
.../org/apache/pulsar/io/jdbc/SqliteUtils.java | 111
pulsar-io/pom.xml | 1 +
tests/integration/pom.xml | 21 +++
.../integration/functions/PulsarFunctionsTest.java | 54 +-
.../functions/PulsarFunctionsTestBase.java | 2 +-
.../tests/integration/io/JdbcSinkTester.java | 137 ++
.../tests/integration/suites/PulsarTestSuite.java | 11 ++
18 files changed, 1245 insertions(+), 7 deletions(-)
diff --git a/distribution/io/src/assemble/io.xml
b/distribution/io/src/assemble/io.xml
index 8cf7fce..bb75e84 100644
--- a/distribution/io/src/assemble/io.xml
+++ b/distribution/io/src/assemble/io.xml
@@ -75,6 +75,11 @@
644
+
${basedir}/../../pulsar-io/jdbc/target/pulsar-io-jdbc-${project.version}.nar
+ connectors
+ 644
+
+
${basedir}/../../pulsar-io/data-genenator/target/pulsar-io-data-generator-${project.version}.nar
connectors
644
diff --git a/pom.xml b/pom.xml
index 1ed2802..681f18e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -167,6 +167,8 @@ flexible messaging model and an intuitive client
API.
1.11.297
1.8.2
2.1.1
+3.8.11.2
+8.0.11
0.206
1.6.0
2.11
@@ -822,6 +824,11 @@ flexible messaging model and an intuitive client
API.
${testcontainers.version}
+org.testcontainers
+mysql
+${testcontainers.version}
+
+
org.arquillian.cube
arquillian-cube-docker
${arquillian-cube.version}
@@ -1086,7 +1093,7 @@ flexible messaging model and an intuitive client
API.
src/main/java/org/apache/pulsar/common/api/proto/PulsarApi.java
src/main/java/org/apache/pulsar/broker/service/schema/proto/SchemaRegistryFormat.java
bin/proto/MLDataFormats_pb2.py
-
+
src/main/java/org/apache/pulsar/io/kinesis/fbs/CompressionType.java
src/main/java/org/apache/pulsar/io/kinesis/fbs/EncryptionCtx.java
diff --git a/pulsar-io/jdbc/lombok.config b/pulsar-io/jdbc/lombok.config
new file mode 100644
index 000..9a9adee
--- /dev/null
+++ b/pulsar-io/jdbc/lombok.config
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+## This file is to fix the conflict with jackson error like this:
+##com.fasterxml.jackson.databind.exc.MismatchedInputException: Cannot
construct instance of ...
+lombok.anyConstructor.addConstructorProperties=true
+config.stopBubbling = true
diff --git a/pulsar-io/jdbc/pom.xml b/pulsar-io/jdbc/pom.xml
new file mode 100644
in
[incubator-pulsar] branch master updated: Don't initialize /managed-ledgers on client creation (#2379) (#2509)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8763c74 Don't initialize /managed-ledgers on client creation (#2379)
(#2509)
8763c74 is described below
commit 8763c746b5b38901d7fbb77e5766f55c563ba25e
Author: Ivan Kelly
AuthorDate: Tue Sep 4 22:18:46 2018 +0200
Don't initialize /managed-ledgers on client creation (#2379) (#2509)
Normally the /managed-ledgers znode is created by the
initialize-cluster-metadata command when a cluster is being turned
up.
However, the ManagedLedger client also creates it on boot. This has
caused issues in the past, where if a broker is started before
initialize-cluster-metadata is run, then initialize-cluster-metadata
fails because it sees the /managed-ledger znode.
This patch removes the automatic creation of this znode from the
client boot process.
---
.../mledger/impl/MetaStoreImplZookeeper.java | 64 +++---
.../mledger/impl/MetaStoreImplZookeeperTest.java | 52 ++
.../bookkeeper/test/BookKeeperClusterTestCase.java | 4 ++
.../bookkeeper/test/MockedBookKeeperTestCase.java | 4 ++
4 files changed, 117 insertions(+), 7 deletions(-)
diff --git
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java
index 354c04f..4d90b91 100644
---
a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java
+++
b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeper.java
@@ -25,9 +25,11 @@ import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.TextFormat;
import com.google.protobuf.TextFormat.ParseException;
+import java.io.File;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
+import java.util.function.Consumer;
import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.bookkeeper.mledger.ManagedLedgerException;
@@ -35,7 +37,6 @@ import
org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException;
import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.bookkeeper.util.ZkUtils;
import org.apache.zookeeper.AsyncCallback.StringCallback;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -95,10 +96,6 @@ public class MetaStoreImplZookeeper implements MetaStore {
throws Exception {
this.zk = zk;
this.executor = executor;
-
-if (zk.exists(prefixName, false) == null) {
-zk.create(prefixName, new byte[0], Acl, CreateMode.PERSISTENT);
-}
}
//
@@ -157,8 +154,8 @@ public class MetaStoreImplZookeeper implements MetaStore {
}
};
-ZkUtils.asyncCreateFullPathOptimistic(zk, prefix +
ledgerName, new byte[0], Acl,
-CreateMode.PERSISTENT, createcb, null);
+asyncCreateFullPathOptimistic(zk, prefixName,
ledgerName, new byte[0], Acl,
+
CreateMode.PERSISTENT, createcb);
} else {
// Tried to open a managed ledger but it doesn't
exist and we shouldn't creating it at this
// point
@@ -365,5 +362,58 @@ public class MetaStoreImplZookeeper implements MetaStore {
}
+public static void asyncCreateFullPathOptimistic(
+final ZooKeeper zk, final String basePath, final String nodePath,
final byte[] data,
+final List acl, final CreateMode createMode, final
StringCallback callback) {
+String fullPath = basePath + "/" + nodePath;
+
+zk.create(fullPath, data, acl, createMode,
+ (rc, path, ignoreCtx1, name) -> {
+ Runnable retry = () -> {
+ asyncCreateFullPathOptimistic(zk, basePath,
nodePath, data,
+acl, createMode,
callback);
+ };
+
+ Consumer complete = (finalrc) -> {
+ callback.processResult(finalrc, path, null, name);
+ };
+
+ if (rc != Code.NONODE.intValue()) {
+ complete.accept(rc);
+ return;
+ }
+
+ // Since I got a nonod
[incubator-pulsar] branch branch-2.1 updated: Ensure standalone service comes back quickly after ungraceful restarts (#2487)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 113418d Ensure standalone service comes back quickly after ungraceful
restarts (#2487)
113418d is described below
commit 113418d3e7b4d02597bed7a939a46e5f66829621
Author: Matteo Merli
AuthorDate: Tue Sep 4 12:34:48 2018 -0700
Ensure standalone service comes back quickly after ungraceful restarts
(#2487)
* Ensure standalone service comes back quickly after ungraceful restarts
* Handle NoNode errors when deleting
* Added NoopLoadManager for standalone mode
---
conf/standalone.conf | 2 +
.../pulsar/broker/loadbalance/NoopLoadManager.java | 158 +
.../loadbalance/impl/ModularLoadManagerImpl.java | 3 +-
.../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 15 +-
4 files changed, 176 insertions(+), 2 deletions(-)
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 74a5702..755b76e 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -311,6 +311,8 @@ autoSkipNonRecoverableData=false
### --- Load balancer --- ###
+loadManagerClassName=org.apache.pulsar.broker.loadbalance.NoopLoadManager
+
# Enable load balancer
loadBalancerEnabled=false
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
new file mode 100644
index 000..5773c61
--- /dev/null
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/loadbalance/NoopLoadManager.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.loadbalance;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.bookkeeper.util.ZkUtils;
+import org.apache.pulsar.broker.PulsarServerException;
+import org.apache.pulsar.broker.PulsarService;
+import org.apache.pulsar.broker.loadbalance.impl.PulsarResourceDescription;
+import org.apache.pulsar.broker.loadbalance.impl.SimpleResourceUnit;
+import org.apache.pulsar.common.naming.ServiceUnitId;
+import org.apache.pulsar.common.stats.Metrics;
+import org.apache.pulsar.common.util.ObjectMapperFactory;
+import org.apache.pulsar.policies.data.loadbalancer.LoadManagerReport;
+import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData;
+import org.apache.pulsar.policies.data.loadbalancer.ServiceLookupData;
+import org.apache.pulsar.zookeeper.ZooKeeperCache.Deserializer;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+
+public class NoopLoadManager implements LoadManager {
+
+private String lookupServiceAddress;
+private ResourceUnit localResourceUnit;
+private ZooKeeper zkClient;
+
+LocalBrokerData localData;
+
+private static final Deserializer loadReportDeserializer
= (key, content) -> ObjectMapperFactory
+.getThreadLocal()
+.readValue(content, LocalBrokerData.class);
+
+@Override
+public void initialize(PulsarService pulsar) {
+lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" +
pulsar.getConfiguration().getWebServicePort();
+localResourceUnit = new SimpleResourceUnit(String.format("http://%s;,
lookupServiceAddress),
+new PulsarResourceDescription());
+zkClient = pulsar.getZkClient();
+
+localData = new LocalBrokerData(pulsar.getWebServiceAddress(),
pulsar.getWebServiceAddressTls(),
+pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls());
+}
+
+@Override
+public void start() throws PulsarServerException {
+String brokerZnodePath = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" +
lookupServiceAddress;
+
+try {
+// When running in standalone, this error c
[incubator-pulsar] branch master updated: Add pulsar flink sink connector (#2434)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new ad03a07 Add pulsar flink sink connector (#2434)
ad03a07 is described below
commit ad03a07d5f5d93654f1d5afb8694989db5688095
Author: Ali Ahmed
AuthorDate: Tue Sep 4 11:59:03 2018 -0700
Add pulsar flink sink connector (#2434)
@XiaoZYang introduces a module for a pulsar sink connector for flink. This
PR is moving the work from flink repo to pulsar repo, so the flink connector
can be released faster along with Pulsar releases.
Original Flink Github Issue: apache/flink#5845
Jira Issue: https://issues.apache.org/jira/browse/FLINK-9168
Original Author: @XiaoZYang (Zong Yang Xiao)
---
pom.xml| 3 +
pulsar-flink/pom.xml | 137 +
.../connectors/pulsar/FlinkPulsarProducer.java | 314 +
.../connectors/pulsar/PulsarJsonTableSink.java | 61
.../connectors/pulsar/PulsarProduceMode.java | 37 +++
.../connectors/pulsar/PulsarTableSink.java | 163 +++
.../pulsar/partitioner/PulsarKeyExtractor.java | 36 +++
.../pulsar/serde/JsonRowDeserializationSchema.java | 130 +
.../pulsar/serde/JsonRowSerializationSchema.java | 92 ++
9 files changed, 973 insertions(+)
diff --git a/pom.xml b/pom.xml
index 4bb3cf6..1ed2802 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,6 +92,7 @@ flexible messaging model and an intuitive client
API.
pulsar-proxy
pulsar-discovery-service
pulsar-storm
+pulsar-flink
pulsar-spark
pulsar-zookeeper-utils
pulsar-testclient
@@ -167,6 +168,8 @@ flexible messaging model and an intuitive client
API.
1.8.2
2.1.1
0.206
+1.6.0
+2.11
1.15.1
diff --git a/pulsar-flink/pom.xml b/pulsar-flink/pom.xml
new file mode 100644
index 000..97ab4fb
--- /dev/null
+++ b/pulsar-flink/pom.xml
@@ -0,0 +1,137 @@
+
+http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd;
+ xmlns="http://maven.apache.org/POM/4.0.0;
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;>
+ 4.0.0
+
+
+org.apache.pulsar
+pulsar
+2.2.0-incubating-SNAPSHOT
+..
+
+
+ pulsar-flink
+ Pulsar Flink Connectors
+
+
+
+
+ org.apache.flink
+ flink-streaming-java_${scala.binary.version}
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-table_${scala.binary.version}
+ ${flink.version}
+ provided
+
+ true
+
+
+
+ ${project.groupId}
+ pulsar-client
+ ${project.version}
+
+
+
+ org.apache.flink
+ flink-runtime_${scala.binary.version}
+ ${flink.version}
+ test-jar
+ test
+
+
+
+ org.apache.flink
+ flink-tests_${scala.binary.version}
+ ${flink.version}
+ test
+ test-jar
+
+
+
+ org.apache.flink
+ flink-streaming-java_${scala.binary.version}
+ ${flink.version}
+ test-jar
+ test
+
+
+
+ org.apache.flink
+ flink-test-utils_${scala.binary.version}
+ ${flink.version}
+ test
+
+
+
+ org.javassist
+ javassist
+ 3.20.0-GA
+ test
+
+
+
+
+
+
+
+src/main/resources
+true
+
+
+
+
+org.apache.maven.plugins
+maven-shade-plugin
+
+
+package
+
+ shade
+
+
+ true
+
true
+
+
+ com.google.guava:guava
+
+
+
+
+ com.google
+ pulsar-flink-shade.com.google
+
+
+
+
+
+
+
+
+
diff --git
a/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java
b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java
new file mode 100644
index 000..bddfee4
--- /dev/null
+++
b/pulsar-flink/src/main/java/org/apache/flink/streaming/connectors/pulsar/FlinkPulsarProducer.java
@@ -0,0 +1,314 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by a
[incubator-pulsar] branch branch-2.1 updated: Allow to configure bookie settings when running in standalone mode (#2480)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/branch-2.1 by this push: new f5a01f7 Allow to configure bookie settings when running in standalone mode (#2480) f5a01f7 is described below commit f5a01f7c82569d3a4dcfc4bed2cc94c790909117 Author: Matteo Merli AuthorDate: Tue Sep 4 11:16:11 2018 -0700 Allow to configure bookie settings when running in standalone mode (#2480) ### Motivation Most bookie configs are set in stone when running Pulsar in standalone mode. We need to be able to tune these settings to have the Pulsar standalone to adapt for running with small memory settings. --- conf/standalone.conf | 79 ++ .../java/org/apache/pulsar/PulsarStandalone.java | 9 ++- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 20 ++ 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index a9fb288..74a5702 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -411,3 +411,82 @@ exposePublisherStats=true # Deprecated. Use configurationStoreServers globalZookeeperServers= + +### --- BookKeeper Configuration --- # + +ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage + +# Size of Write Cache. Memory is allocated from JVM direct memory. +# Write cache is used to buffer entries before flushing into the entry log +# For good performance, it should be big enough to hold a sub +dbStorage_writeCacheMaxSizeMb=256 + +# Size of Read cache. Memory is allocated from JVM direct memory. +# This read cache is pre-filled doing read-ahead whenever a cache miss happens +dbStorage_readAheadCacheMaxSizeMb=64 + +# How many entries to pre-fill in cache after a read cache miss +dbStorage_readAheadCacheBatchSize=1000 + +flushInterval=6 + +## RocksDB specific configurations +## DbLedgerStorage uses RocksDB to store the indexes from +## (ledgerId, entryId) -> (entryLog, offset) + +# Size of RocksDB block-cache. For best performance, this cache +# should be big enough to hold a significant portion of the index +# database which can reach ~2GB in some cases +# Default is 16 MBytes +dbStorage_rocksDB_blockCacheSize=16777216 + +# Other RocksDB specific tunables +dbStorage_rocksDB_writeBufferSizeMB=4 +dbStorage_rocksDB_sstSizeInMB=4 +dbStorage_rocksDB_blockSize=4096 +dbStorage_rocksDB_bloomFilterBitsPerKey=10 +dbStorage_rocksDB_numLevels=-1 +dbStorage_rocksDB_numFilesInLevel0=4 +dbStorage_rocksDB_maxSizeInLevel1MB=256 + +# Maximum latency to impose on a journal write to achieve grouping +journalMaxGroupWaitMSec=1 + +# Should the data be fsynced on journal before acknowledgment. +journalSyncData=false + + +# For each ledger dir, maximum disk space which can be used. +# Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will +# be written to that partition. If all ledger dir partions are full, then bookie +# will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will +# shutdown. +# Valid values should be in between 0 and 1 (exclusive). +diskUsageThreshold=0.99 + +# The disk free space low water mark threshold. +# Disk is considered full when usage threshold is exceeded. +# Disk returns back to non-full state when usage is below low water mark threshold. +# This prevents it from going back and forth between these states frequently +# when concurrent writes and compaction are happening. This also prevent bookie from +# switching frequently between read-only and read-writes states in the same cases. +diskUsageWarnThreshold=0.99 + +# Whether the bookie allowed to use a loopback interface as its primary +# interface(i.e. the interface it uses to establish its identity)? +# By default, loopback interfaces are not allowed as the primary +# interface. +# Using a loopback interface as the primary interface usually indicates +# a configuration error. For example, its fairly common in some VPS setups +# to not configure a hostname, or to have the hostname resolve to +# 127.0.0.1. If this is the case, then all bookies in the cluster will +# establish their identities as 127.0.0.1:3181, and only one will be able +# to join the cluster. For VPSs configured like this, you should explicitly +# set the listening interface. +allowLoopback=true + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +gcWaitTime=30 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index b6105c9..bb30bfa 100644 --- a/pulsar-broker/src/m
[incubator-pulsar] branch master updated: Allow to configure bookie settings when running in standalone mode (#2480)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 8f4f371 Allow to configure bookie settings when running in standalone mode (#2480) 8f4f371 is described below commit 8f4f371def2983f64e5dfa8f31e9fe04a2e21462 Author: Matteo Merli AuthorDate: Tue Sep 4 11:16:11 2018 -0700 Allow to configure bookie settings when running in standalone mode (#2480) ### Motivation Most bookie configs are set in stone when running Pulsar in standalone mode. We need to be able to tune these settings to have the Pulsar standalone to adapt for running with small memory settings. --- conf/standalone.conf | 79 ++ .../java/org/apache/pulsar/PulsarStandalone.java | 9 ++- .../pulsar/zookeeper/LocalBookkeeperEnsemble.java | 20 ++ 3 files changed, 92 insertions(+), 16 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index 09d369c..b09fcd6 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -420,3 +420,82 @@ exposePublisherStats=true # Deprecated. Use configurationStoreServers globalZookeeperServers= + +### --- BookKeeper Configuration --- # + +ledgerStorageClass=org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage + +# Size of Write Cache. Memory is allocated from JVM direct memory. +# Write cache is used to buffer entries before flushing into the entry log +# For good performance, it should be big enough to hold a sub +dbStorage_writeCacheMaxSizeMb=256 + +# Size of Read cache. Memory is allocated from JVM direct memory. +# This read cache is pre-filled doing read-ahead whenever a cache miss happens +dbStorage_readAheadCacheMaxSizeMb=64 + +# How many entries to pre-fill in cache after a read cache miss +dbStorage_readAheadCacheBatchSize=1000 + +flushInterval=6 + +## RocksDB specific configurations +## DbLedgerStorage uses RocksDB to store the indexes from +## (ledgerId, entryId) -> (entryLog, offset) + +# Size of RocksDB block-cache. For best performance, this cache +# should be big enough to hold a significant portion of the index +# database which can reach ~2GB in some cases +# Default is 16 MBytes +dbStorage_rocksDB_blockCacheSize=16777216 + +# Other RocksDB specific tunables +dbStorage_rocksDB_writeBufferSizeMB=4 +dbStorage_rocksDB_sstSizeInMB=4 +dbStorage_rocksDB_blockSize=4096 +dbStorage_rocksDB_bloomFilterBitsPerKey=10 +dbStorage_rocksDB_numLevels=-1 +dbStorage_rocksDB_numFilesInLevel0=4 +dbStorage_rocksDB_maxSizeInLevel1MB=256 + +# Maximum latency to impose on a journal write to achieve grouping +journalMaxGroupWaitMSec=1 + +# Should the data be fsynced on journal before acknowledgment. +journalSyncData=false + + +# For each ledger dir, maximum disk space which can be used. +# Default is 0.95f. i.e. 95% of disk can be used at most after which nothing will +# be written to that partition. If all ledger dir partions are full, then bookie +# will turn to readonly mode if 'readOnlyModeEnabled=true' is set, else it will +# shutdown. +# Valid values should be in between 0 and 1 (exclusive). +diskUsageThreshold=0.99 + +# The disk free space low water mark threshold. +# Disk is considered full when usage threshold is exceeded. +# Disk returns back to non-full state when usage is below low water mark threshold. +# This prevents it from going back and forth between these states frequently +# when concurrent writes and compaction are happening. This also prevent bookie from +# switching frequently between read-only and read-writes states in the same cases. +diskUsageWarnThreshold=0.99 + +# Whether the bookie allowed to use a loopback interface as its primary +# interface(i.e. the interface it uses to establish its identity)? +# By default, loopback interfaces are not allowed as the primary +# interface. +# Using a loopback interface as the primary interface usually indicates +# a configuration error. For example, its fairly common in some VPS setups +# to not configure a hostname, or to have the hostname resolve to +# 127.0.0.1. If this is the case, then all bookies in the cluster will +# establish their identities as 127.0.0.1:3181, and only one will be able +# to join the cluster. For VPSs configured like this, you should explicitly +# set the listening interface. +allowLoopback=true + +# How long the interval to trigger next garbage collection, in milliseconds +# Since garbage collection is running in background, too frequent gc +# will heart performance. It is better to give a higher number of gc +# interval if there is enough disk capacity. +gcWaitTime=30 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java index b6105c9..bb30bfa 100644 --- a/pulsar-broker/src/main/java/
[incubator-pulsar] branch branch-2.1 updated: Fix log4j config to not redirect BK logs into separate file (#2482)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a4b20bb Fix log4j config to not redirect BK logs into separate file
(#2482)
a4b20bb is described below
commit a4b20bb28fb77a5f84be61481e126a698788ae79
Author: Matteo Merli
AuthorDate: Tue Sep 4 11:15:24 2018 -0700
Fix log4j config to not redirect BK logs into separate file (#2482)
### Motivation
Currently all `org.apache.bookkeeper` messages printed by broker are being
routed to a different log file `broker.log.bk`. Removing for now the special
config for BK logs.
---
conf/log4j2.yaml | 50 --
1 file changed, 50 deletions(-)
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index ab2b9ab..45ed5d9 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -35,10 +35,6 @@ Configuration:
value: "info"
- name: "pulsar.routing.appender.default"
value: "Console"
- - name: "bk.log.level"
-value: "info"
- - name: "bk.log.appender"
-value: "BkRollingFile"
# Example: logger-filter script
Scripts:
@@ -85,34 +81,6 @@ Configuration:
IfLastModified:
age: 30d
-# Rolling file appender configuration for bk
-RollingRandomAccessFile:
- name: BkRollingFile
- fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk"
- filePattern:
"${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-}-%i.log.gz"
- immediateFlush: true
- PatternLayout:
-Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
- Policies:
-TimeBasedTriggeringPolicy:
- interval: 1
- modulate: true
-SizeBasedTriggeringPolicy:
- size: 1 GB
-# Trigger every day at midnight that also scan
-# roll-over strategy that deletes older file
-CronTriggeringPolicy:
- schedule: "0 0 0 * * ?"
- # Delete file older than 30days
- DefaultRolloverStrategy:
- Delete:
-basePath: ${sys:pulsar.log.dir}
-maxDepth: 2
-IfFileName:
- glob: "*/${sys:pulsar.log.file}.bk*log.gz"
-IfLastModified:
- age: 30d
-
# Routing
Routing:
name: RoutingAppender
@@ -173,24 +141,6 @@ Configuration:
AppenderRef:
- ref: Console
- - name: org.apache.bookkeeper
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- - name: org.apache.distributedlog
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- - name: org.apache.zookeeper
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- name: verbose
level: info
additivity: false
[incubator-pulsar] branch master updated: Fix log4j config to not redirect BK logs into separate file (#2482)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 37f266b Fix log4j config to not redirect BK logs into separate file
(#2482)
37f266b is described below
commit 37f266bb0357fa825f1b3c2e99582526c43bba84
Author: Matteo Merli
AuthorDate: Tue Sep 4 11:15:24 2018 -0700
Fix log4j config to not redirect BK logs into separate file (#2482)
### Motivation
Currently all `org.apache.bookkeeper` messages printed by broker are being
routed to a different log file `broker.log.bk`. Removing for now the special
config for BK logs.
---
conf/log4j2.yaml | 50 --
1 file changed, 50 deletions(-)
diff --git a/conf/log4j2.yaml b/conf/log4j2.yaml
index ab2b9ab..45ed5d9 100644
--- a/conf/log4j2.yaml
+++ b/conf/log4j2.yaml
@@ -35,10 +35,6 @@ Configuration:
value: "info"
- name: "pulsar.routing.appender.default"
value: "Console"
- - name: "bk.log.level"
-value: "info"
- - name: "bk.log.appender"
-value: "BkRollingFile"
# Example: logger-filter script
Scripts:
@@ -85,34 +81,6 @@ Configuration:
IfLastModified:
age: 30d
-# Rolling file appender configuration for bk
-RollingRandomAccessFile:
- name: BkRollingFile
- fileName: "${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk"
- filePattern:
"${sys:pulsar.log.dir}/${sys:pulsar.log.file}.bk-%d{MM-dd-}-%i.log.gz"
- immediateFlush: true
- PatternLayout:
-Pattern: "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"
- Policies:
-TimeBasedTriggeringPolicy:
- interval: 1
- modulate: true
-SizeBasedTriggeringPolicy:
- size: 1 GB
-# Trigger every day at midnight that also scan
-# roll-over strategy that deletes older file
-CronTriggeringPolicy:
- schedule: "0 0 0 * * ?"
- # Delete file older than 30days
- DefaultRolloverStrategy:
- Delete:
-basePath: ${sys:pulsar.log.dir}
-maxDepth: 2
-IfFileName:
- glob: "*/${sys:pulsar.log.file}.bk*log.gz"
-IfLastModified:
- age: 30d
-
# Routing
Routing:
name: RoutingAppender
@@ -173,24 +141,6 @@ Configuration:
AppenderRef:
- ref: Console
- - name: org.apache.bookkeeper
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- - name: org.apache.distributedlog
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- - name: org.apache.zookeeper
-level: "${sys:bk.log.level}"
-additivity: false
-AppenderRef:
- - ref: "${sys:bk.log.appender}"
-
- name: verbose
level: info
additivity: false
[incubator-pulsar] branch branch-2.1 updated: [functions] Don't set the state storage url if state is not enabled. (#2488)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 04c5c48 [functions] Don't set the state storage url if state is not
enabled. (#2488)
04c5c48 is described below
commit 04c5c48f2fcba7724892f66acf7784518877ba3c
Author: Sijie Guo
AuthorDate: Tue Sep 4 00:17:29 2018 -0700
[functions] Don't set the state storage url if state is not enabled. (#2488)
---
pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
index cf07d8c..b6105c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
@@ -273,7 +273,10 @@ public class PulsarStandalone implements AutoCloseable {
// worker talks to local broker
workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" +
config.getBrokerServicePort());
workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:; +
config.getWebServicePort());
-workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" +
this.getStreamStoragePort());
+if (!this.isNoStreamStorage()) {
+// only set the state storage service url when state is
enabled.
+workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" +
this.getStreamStoragePort());
+}
String hostname =
ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
config.getAdvertisedAddress());
workerConfig.setWorkerHostname(hostname);
[incubator-pulsar] branch master updated: [functions] Don't set the state storage url if state is not enabled. (#2488)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 4124de1 [functions] Don't set the state storage url if state is not
enabled. (#2488)
4124de1 is described below
commit 4124de11c407e7a5efa8aa2f4b669e22543ee427
Author: Sijie Guo
AuthorDate: Tue Sep 4 00:17:29 2018 -0700
[functions] Don't set the state storage url if state is not enabled. (#2488)
---
pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
index cf07d8c..b6105c9 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
@@ -273,7 +273,10 @@ public class PulsarStandalone implements AutoCloseable {
// worker talks to local broker
workerConfig.setPulsarServiceUrl("pulsar://127.0.0.1:" +
config.getBrokerServicePort());
workerConfig.setPulsarWebServiceUrl("http://127.0.0.1:; +
config.getWebServicePort());
-workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" +
this.getStreamStoragePort());
+if (!this.isNoStreamStorage()) {
+// only set the state storage service url when state is
enabled.
+workerConfig.setStateStorageServiceUrl("bk://127.0.0.1:" +
this.getStreamStoragePort());
+}
String hostname =
ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
config.getAdvertisedAddress());
workerConfig.setWorkerHostname(hostname);
[incubator-pulsar] branch master updated: Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new cfdcdb5 Fixed the issue 2481 by not to take PULSAR_MEM from env.
(#2484)
cfdcdb5 is described below
commit cfdcdb5c096918187ab52725ba091c401f28b62c
Author: Victor
AuthorDate: Sat Sep 1 00:09:30 2018 -0700
Fixed the issue 2481 by not to take PULSAR_MEM from env. (#2484)
### Motivation
Command `./bin/pulsar-admin clusters list` failed with `Error: Could not
find or load main class "` in my k8s deployment by using helm charts.
### Modifications
With helm deployment the env var PULSAR_MEM is quoted with `"`, taking it
for `pulsar-admin` will make the exec command to be:
```exec /docker-java-home/bin/java -cp '/pulsar/conf:::/pulsar/lib/*:'
-Dlog4j.configurationFile=log4j2.yaml -Djava.net.preferIPv4Stack=true '"'
-Xms1g -Xmx1g '-XX:MaxDirectMemorySize=1g"' -client
-Dio.netty.leakDetectionLevel=disabled ...
org.apache.pulsar.admin.cli.PulsarAdminTool /pulsar/conf/client.conf clusters
list```
The fix is to take out the use of env var PULSAR_MEM in
`conf/pulsar_tools_env.sh`.
### Result
`pulsar-admin` works as expected.
Fixes #2481
---
conf/pulsar_tools_env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/pulsar_tools_env.sh b/conf/pulsar_tools_env.sh
index 38e2859..1a701fe 100755
--- a/conf/pulsar_tools_env.sh
+++ b/conf/pulsar_tools_env.sh
@@ -42,7 +42,7 @@
# PULSAR_GLOBAL_ZK_CONF=
# Extra options to be passed to the jvm
-PULSAR_MEM=${PULSAR_TOOLS_MEM:-"-Xmx128m -XX:MaxDirectMemorySize=128m"}
+PULSAR_MEM="-Xmx128m -XX:MaxDirectMemorySize=128m"
# Garbage collection options
PULSAR_GC=" -client "
[incubator-pulsar] branch master updated: [Schema] Introduce Schema.AUTO to detect schema automatically for consumers and readers (#2494)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 42b598a [Schema] Introduce Schema.AUTO to detect schema automatically
for consumers and readers (#2494)
42b598a is described below
commit 42b598a17b64d76428d48438cfb169cf3a1fcd5d
Author: Sijie Guo
AuthorDate: Sat Sep 1 00:07:12 2018 -0700
[Schema] Introduce Schema.AUTO to detect schema automatically for consumers
and readers (#2494)
### Motivation
Sometimes consumers and readers don't have the pojo for the message
schemas. so we can use `Schema.Auto` to detect
the schema and reflect the messages into generic records.
### Changes
- Introduce AutoSchema and `SchemaType.AUTO`
- fetch schema information first when `AutoSchema` is used
---
.../api/SimpleTypedProducerConsumerTest.java | 95 ++
.../java/org/apache/pulsar/client/api/Schema.java | 6 ++
.../pulsar/client/impl/schema/AutoSchema.java | 63 ++
.../pulsar/client/schema/AvroSchemaTest.java | 21 -
.../pulsar/client/impl/PulsarClientImpl.java | 52
.../apache/pulsar/common/schema/SchemaType.java| 7 +-
6 files changed, 242 insertions(+), 2 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java
index b880953..24573d5 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleTypedProducerConsumerTest.java
@@ -26,6 +26,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.pulsar.broker.service.schema.SchemaRegistry;
+import org.apache.pulsar.client.api.schema.GenericRecord;
import org.apache.pulsar.client.impl.schema.AvroSchema;
import org.apache.pulsar.client.impl.schema.JSONSchema;
import org.apache.pulsar.client.impl.schema.ProtobufSchema;
@@ -440,4 +441,98 @@ public class SimpleTypedProducerConsumerTest extends
ProducerConsumerBase {
}
}
+@Test
+public void testAvroProducerAndAutoSchemaConsumer() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ AvroSchema avroSchema =
+ AvroSchema.of(AvroEncodedPojo.class);
+
+ Producer producer = pulsarClient
+ .newProducer(avroSchema)
+ .topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
+
+ for (int i = 0; i < 10; i++) {
+ String message = "my-message-" + i;
+ producer.send(new AvroEncodedPojo(message));
+ }
+
+ Consumer consumer = pulsarClient
+ .newConsumer(Schema.AUTO())
+ .topic("persistent://my-property/use/my-ns/my-topic1")
+ .subscriptionName("my-subscriber-name")
+ .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
+ .subscribe();
+
+ Message msg = null;
+ Set messageSet = Sets.newHashSet();
+ for (int i = 0; i < 10; i++) {
+ msg = consumer.receive(5, TimeUnit.SECONDS);
+ GenericRecord receivedMessage = msg.getValue();
+ log.debug("Received message: [{}]", receivedMessage);
+ String expectedMessage = "my-message-" + i;
+ String actualMessage = (String) receivedMessage.getField("message");
+ testMessageOrderAndDuplicates(messageSet, actualMessage,
expectedMessage);
+ }
+ // Acknowledge the consumption of all messages at once
+ consumer.acknowledgeCumulative(msg);
+ consumer.close();
+
+ SchemaRegistry.SchemaAndMetadata storedSchema =
pulsar.getSchemaRegistryService()
+ .getSchema("my-property/my-ns/my-topic1")
+ .get();
+
+ Assert.assertEquals(storedSchema.schema.getData(),
avroSchema.getSchemaInfo().getSchema());
+
+ log.info("-- Exiting {} test --", methodName);
+
+ }
+
+ @Test
+public void testAvroProducerAndAutoSchemaReader() throws Exception {
+ log.info("-- Starting {} test --", methodName);
+
+ AvroSchema avroSchema =
+ AvroSchema.of(AvroEncodedPojo.class);
+
+ Producer producer = pulsarClient
+ .newProducer(avroSchema)
+ .topic("persistent://my-property/use/my-ns/my-topic1")
+ .create();
+
+ for (int i = 0; i < 10; i++) {
+ String message = "my-message-" + i;
+ producer.send(new AvroEncodedPojo(message));
+ }
+
+ Reader reader = pulsarClient
+
[incubator-pulsar] branch master updated: [tiered storage] Use NAR plugin to package offloaders (#2393)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 9c1503b [tiered storage] Use NAR plugin to package offloaders
(#2393)
9c1503b is described below
commit 9c1503b67eb122d2801fd9a9b5a94e5f7780efc0
Author: Sijie Guo
AuthorDate: Wed Aug 29 00:52:29 2018 -0700
[tiered storage] Use NAR plugin to package offloaders (#2393)
### Motivation
Offloader typically involves a new storage system, which usually involves
dependencies that
might be conflicting with the dependencies of Pulsar. We want to package
offloader implementations
as what we did for connectors, so people can decide which offloader to use
and only include it.
People would also become easier to write its offloader if needed.
### Changes
- Update `tiered-storage/jcloud` to package it using nifi-nar plugin.
- Add bunch of utils in managed-ledger to locate offloader nar packages and
load `LedgerOffloaderFactory` from corresponding NAR package.
- Add a new distribution module to package offloaders into one
distribution, as what we did for connectors.
- Update pulsar-all image to include offloaders
### NOTES
This change doesn't get rid of jcloud-shaded. because jcloud is using
`ServiceLoad` and Guice injection. It makes
things very tricky on class loading. Not attempt to address the problem any
time soon. We can pin `jcloud-shaded` version
as what we did for `protobuf-shaded` in next release.
---
conf/broker.conf | 3 +
distribution/{ => offloaders}/pom.xml | 42 +-
distribution/offloaders/src/assemble/README| 10 ++
.../offloaders/src/assemble/offloaders.xml | 54
distribution/pom.xml | 1 +
distribution/server/src/assemble/LICENSE.bin.txt | 14 --
docker/pulsar-all/Dockerfile | 4 +
docker/pulsar-all/pom.xml | 21 +++
.../mledger/offload/OffloaderDefinition.java | 46 +++
.../bookkeeper/mledger/offload/OffloaderUtils.java | 149 +
.../bookkeeper/mledger/offload/Offloaders.java | 57
.../apache/pulsar/broker/ServiceConfiguration.java | 11 ++
pulsar-broker/pom.xml | 6 -
.../org/apache/pulsar/broker/PulsarService.java| 20 ++-
tiered-storage/jcloud/pom.xml | 10 +-
.../impl/BlobStoreManagedLedgerOffloader.java | 6 +
.../META-INF/services/pulsar-offloader.yaml| 9 +-
17 files changed, 422 insertions(+), 41 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index 11670d3..8927a85 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -513,6 +513,9 @@
schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.Bookkeepe
### --- Ledger Offloading --- ###
+# The directory for all the offloader implementations
+offloadersDirectory=./offloaders
+
# Driver to use to offload old data to long term storage (Possible values: S3,
aws-s3, google-cloud-storage)
# When using google-cloud-storage, Make sure both Google Cloud Storage and
Google Cloud Storage JSON API are enabled for
# the project (check from Developers Console -> Api -> APIs).
diff --git a/distribution/pom.xml b/distribution/offloaders/pom.xml
similarity index 53%
copy from distribution/pom.xml
copy to distribution/offloaders/pom.xml
index 36b4917..84349da 100644
--- a/distribution/pom.xml
+++ b/distribution/offloaders/pom.xml
@@ -24,17 +24,45 @@
org.apache.pulsar
-pulsar
+distribution
2.2.0-incubating-SNAPSHOT
..
- distribution
+ pulsar-offloader-distribution
pom
- Pulsar :: Distribution
+ Pulsar :: Distribution :: Offloader
-
-server
-io
-
+
+
+ org.apache.pulsar
+ managed-ledger-original
+ ${project.version}
+
+
+
+
+
+
+maven-assembly-plugin
+
+
+distro-assembly
+package
+
+ single
+
+
+ true
+ posix
+
apache-pulsar-offloaders-${project.version}
+
+src/assemble/offloaders.xml
+
+
+
+
+
+
+
diff --git a/distribution/offloaders/src/assemble/README
b/distribution/offloaders/src/assemble/README
new file mode 100644
index 000..6777ec2
--- /dev/null
+++ b/distribution/offloaders/src/assemble/README
@@ -0,0 +1,10 @@
+
+Please refer to http://pulsar.incubator.apache.org/ for access to
documentation.
+
+This package contains Pulsar offloader archives. Each archive
+contains:
+
+ * the offloader code plus all the dependencies
+
+ * META-INF/DEPEDEN
[incubator-pulsar] branch branch-2.1 updated: Increase default maxConcurrentLookupRequests to 50k (#2380)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 333b896 Increase default maxConcurrentLookupRequests to 50k (#2380)
333b896 is described below
commit 333b896a37ccd27729efd778aa97710b7f8286db
Author: Matteo Merli
AuthorDate: Tue Aug 28 10:41:14 2018 -0700
Increase default maxConcurrentLookupRequests to 50k (#2380)
### Motivation
Increasing the default value of `maxConcurrentLookupRequests` from 10K to
50K to increase limit of sustainable lookups before rejecting requests.
---
conf/broker.conf | 3 +--
conf/proxy.conf| 2 +-
conf/standalone.conf | 2 +-
deployment/terraform-ansible/templates/broker.conf | 2 +-
.../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +-
.../main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java | 2 +-
6 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index cd4a505..5eeb936 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
@@ -511,4 +511,3 @@ s3ManagedLedgerOffloadReadBufferSizeInBytes=1048576
# Deprecated. Use configurationStoreServers
globalZookeeperServers=
-
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 3bde301..ed5420f 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false
maxConcurrentInboundConnections=1
# Max concurrent outbound connections. The proxy will error out requests
beyond that.
-maxConcurrentLookupRequests=1
+maxConcurrentLookupRequests=5
# --- TLS --- #
diff --git a/conf/standalone.conf b/conf/standalone.conf
index cc03b95..a9fb288 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -141,7 +141,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
diff --git a/deployment/terraform-ansible/templates/broker.conf
b/deployment/terraform-ansible/templates/broker.conf
index a7456a4..8eaa0df 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index ddc7728..7b2e85b 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -170,7 +170,7 @@ public class ServiceConfiguration implements
PulsarConfiguration {
private boolean dispatchThrottlingOnNonBacklogConsumerEnabled = false;
// Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
@FieldContext(dynamic = true)
-private int maxConcurrentLookupRequest = 1;
+private int maxConcurrentLookupRequest = 5;
// Max number of concurrent topic loading request broker allows to control
number of zk-operations
@FieldContext(dynamic = true)
private int maxConcurrentTopicLoadRequest = 5000;
diff --git
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
index 4d02786..b4e8afb 100644
---
a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
+++
b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java
[incubator-pulsar] branch master updated: Increase default maxConcurrentLookupRequests to 50k (#2380)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new fa11a3c Increase default maxConcurrentLookupRequests to 50k (#2380)
fa11a3c is described below
commit fa11a3ca5d36ae6c1cc169e32cd0164fd8e47487
Author: Matteo Merli
AuthorDate: Tue Aug 28 10:41:14 2018 -0700
Increase default maxConcurrentLookupRequests to 50k (#2380)
### Motivation
Increasing the default value of `maxConcurrentLookupRequests` from 10K to
50K to increase limit of sustainable lookups before rejecting requests.
---
conf/broker.conf | 3 +--
conf/proxy.conf | 2 +-
conf/standalone.conf | 2 +-
deployment/terraform-ansible/templates/broker.conf| 2 +-
deployment/terraform-ansible/templates/proxy.conf | 2 +-
.../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java | 2 +-
.../main/java/org/apache/pulsar/proxy/server/ProxyConfiguration.java | 2 +-
site2/docs/administration-proxy.md| 2 +-
site2/docs/reference-configuration.md | 4 ++--
9 files changed, 10 insertions(+), 11 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index a8dd44b..11670d3 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -161,7 +161,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
@@ -557,4 +557,3 @@ gcsManagedLedgerOffloadServiceAccountKeyFile=
# Deprecated. Use configurationStoreServers
globalZookeeperServers=
-
diff --git a/conf/proxy.conf b/conf/proxy.conf
index 7b7e690..9b307cc 100644
--- a/conf/proxy.conf
+++ b/conf/proxy.conf
@@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false
maxConcurrentInboundConnections=1
# Max concurrent outbound connections. The proxy will error out requests
beyond that.
-maxConcurrentLookupRequests=1
+maxConcurrentLookupRequests=5
# --- TLS --- #
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 13bef8a..09d369c 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -144,7 +144,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
diff --git a/deployment/terraform-ansible/templates/broker.conf
b/deployment/terraform-ansible/templates/broker.conf
index 8e8c6c2..a4fad1c 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -152,7 +152,7 @@ dispatchThrottlingRatePerTopicInByte=0
dispatchThrottlingOnNonBacklogConsumerEnabled=true
# Max number of concurrent lookup request broker allows to throttle heavy
incoming lookup traffic
-maxConcurrentLookupRequest=1
+maxConcurrentLookupRequest=5
# Max number of concurrent topic loading request broker allows to control
number of zk-operations
maxConcurrentTopicLoadRequest=5000
diff --git a/deployment/terraform-ansible/templates/proxy.conf
b/deployment/terraform-ansible/templates/proxy.conf
index 21fa327..b631aca 100644
--- a/deployment/terraform-ansible/templates/proxy.conf
+++ b/deployment/terraform-ansible/templates/proxy.conf
@@ -81,7 +81,7 @@ forwardAuthorizationCredentials=false
maxConcurrentInboundConnections=1
# Max concurrent outbound connections. The proxy will error out requests
beyond that.
-maxConcurrentLookupRequests=1
+maxConcurrentLookupRequests=5
# --- TLS --- #
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 8057540..e94f6b9 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -179,7 +179,7 @@ public class ServiceConfiguration implements
PulsarConfiguration {
private boolean dispatchThrottlingOnNonBacklogConsumerEnabled = false;
// Max number of concurrent lookup request broker allows to throttle heavy
[incubator-pulsar] branch master updated: Fix flake in DiscoveryServiceTest (#1081) (#2406)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 774cafa Fix flake in DiscoveryServiceTest (#1081) (#2406)
774cafa is described below
commit 774cafa25d6340c778c5c281045446086aff96a0
Author: Ivan Kelly
AuthorDate: Tue Aug 28 07:30:42 2018 +0200
Fix flake in DiscoveryServiceTest (#1081) (#2406)
This test was flaking because it was only waiting for 1 second for
connection and message exchange to complete, which is not enough time
when there's heavy load on the machine (simulated with stress-ng).
The fix is to increase the timeout to 10 seconds. I've also cleaned up
the test to use a CompletableFuture rather than a CountDownLatch so
tha the test thread can be notified of failures in the handlers.
---
.../discovery/service/DiscoveryServiceTest.java| 77 --
1 file changed, 43 insertions(+), 34 deletions(-)
diff --git
a/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java
b/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java
index 7d4a5ee..24bcd6a 100644
---
a/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java
+++
b/pulsar-discovery-service/src/test/java/org/apache/pulsar/discovery/service/DiscoveryServiceTest.java
@@ -21,7 +21,6 @@ package org.apache.pulsar.discovery.service;
import static
org.apache.pulsar.discovery.service.web.ZookeeperCacheLoader.LOADBALANCE_BROKERS_ROOT;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
import java.lang.reflect.Field;
@@ -31,16 +30,17 @@ import java.net.URISyntaxException;
import java.security.PrivateKey;
import java.security.cert.X509Certificate;
import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.bookkeeper.util.ZkUtils;
import org.apache.pulsar.common.api.Commands;
+import org.apache.pulsar.common.api.proto.PulsarApi.BaseCommand;
import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
import org.apache.pulsar.common.util.ObjectMapperFactory;
import org.apache.pulsar.common.util.SecurityUtility;
+import org.apache.pulsar.common.util.protobuf.ByteBufCodedInputStream;
import org.apache.pulsar.discovery.service.web.ZookeeperCacheLoader;
import org.apache.pulsar.policies.data.loadbalancer.LoadReport;
import org.apache.pulsar.zookeeper.ZooKeeperChildrenCache;
@@ -66,8 +66,13 @@ import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class DiscoveryServiceTest extends BaseDiscoveryTestSetup {
+private static final Logger log =
LoggerFactory.getLogger(DiscoveryServiceTest.class);
+
private final static String TLS_CLIENT_CERT_FILE_PATH =
"./src/test/resources/certificate/client.crt";
private final static String TLS_CLIENT_KEY_FILE_PATH =
"./src/test/resources/certificate/client.key";
@@ -120,48 +125,40 @@ public class DiscoveryServiceTest extends
BaseDiscoveryTestSetup {
/**
* It verifies: client connects to Discovery-service and receives
discovery response successfully.
- *
+ *
* @throws Exception
*/
@Test
public void testClientServerConnection() throws Exception {
addBrokerToZk(2);
-// 1. client connects to DiscoveryService, 2. Client receive
service-lookup response
-final int messageTransfer = 2;
-final CountDownLatch latch = new CountDownLatch(messageTransfer);
-NioEventLoopGroup workerGroup =
connectToService(service.getServiceUrl(), latch, false);
-try {
-assertTrue(latch.await(1, TimeUnit.SECONDS));
-} catch (InterruptedException e) {
-fail("should have received lookup response message from server",
e);
-}
+
+final CompletableFuture promise = new
CompletableFuture<>();
+NioEventLoopGroup workerGroup =
connectToService(service.getServiceUrl(), promise, false);
+assertEquals(promise.get(10, TimeUnit.SECONDS).getType(),
BaseCommand.Type.CONNECTED);
workerGroup.shutdownGracefully();
}
@Test
public void testClientServerConnectionTls() throws Exception {
addBrokerToZk(2);
-// 1. client connects to DiscoveryService, 2. Client receive
service-lookup response
-final int
[incubator-pulsar] branch master updated: Fixed {{pulsar.version}} into {{pulsar:version}} in few pages (#2455)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8da0669 Fixed {{pulsar.version}} into {{pulsar:version}} in few pages
(#2455)
8da0669 is described below
commit 8da066947b339d1f71025f8d3857bb7e5b9e70a2
Author: Matteo Merli
AuthorDate: Mon Aug 27 22:26:10 2018 -0700
Fixed {{pulsar.version}} into {{pulsar:version}} in few pages (#2455)
### Motivation
In few places in website we're displaying the literal `{{pulsar.version}}`
instead of the actual Pulsar version string. The correct variable name is
expected to use `:` as separator.
---
site2/docs/client-libraries-go.md| 2 +-
site2/docs/deploy-bare-metal.md | 12 ++--
site2/docs/getting-started-standalone.md | 12 ++--
site2/docs/io-quickstart.md | 12 ++--
.../version-2.1.0-incubating/client-libraries-go.md | 2 +-
.../version-2.1.0-incubating/deploy-bare-metal.md| 12 ++--
.../version-2.1.0-incubating/getting-started-standalone.md | 12 ++--
.../versioned_docs/version-2.1.0-incubating/io-quickstart.md | 12 ++--
8 files changed, 38 insertions(+), 38 deletions(-)
diff --git a/site2/docs/client-libraries-go.md
b/site2/docs/client-libraries-go.md
index 2433eb9..421d457 100644
--- a/site2/docs/client-libraries-go.md
+++ b/site2/docs/client-libraries-go.md
@@ -34,7 +34,7 @@ $ go get -u
github.com/apache/incubator-pulsar/pulsar-client-go/pulsar
Or you can use [dep](https://github.com/golang/dep) for managing the
dependencies.
```bash
-$ dep ensure -add
github.com/apache/incubator-pulsar/pulsar-client-go/pulsar@v{{pulsar.version}}
+$ dep ensure -add
github.com/apache/incubator-pulsar/pulsar-client-go/pulsar@v{{pulsar:version}}
```
Once installed locally, you can import it into your project:
diff --git a/site2/docs/deploy-bare-metal.md b/site2/docs/deploy-bare-metal.md
index 78ea557..b5793c6 100644
--- a/site2/docs/deploy-bare-metal.md
+++ b/site2/docs/deploy-bare-metal.md
@@ -125,12 +125,12 @@ $ tar xvfz
apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz
$ mv apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors
$ ls connectors
-pulsar-io-aerospike-{{pulsar.version}}.nar
-pulsar-io-cassandra-{{pulsar.version}}.nar
-pulsar-io-kafka-{{pulsar.version}}.nar
-pulsar-io-kinesis-{{pulsar.version}}.nar
-pulsar-io-rabbitmq-{{pulsar.version}}.nar
-pulsar-io-twitter-{{pulsar.version}}.nar
+pulsar-io-aerospike-{{pulsar:version}}.nar
+pulsar-io-cassandra-{{pulsar:version}}.nar
+pulsar-io-kafka-{{pulsar:version}}.nar
+pulsar-io-kinesis-{{pulsar:version}}.nar
+pulsar-io-rabbitmq-{{pulsar:version}}.nar
+pulsar-io-twitter-{{pulsar:version}}.nar
...
```
diff --git a/site2/docs/getting-started-standalone.md
b/site2/docs/getting-started-standalone.md
index 4090451..0dcf024 100644
--- a/site2/docs/getting-started-standalone.md
+++ b/site2/docs/getting-started-standalone.md
@@ -87,12 +87,12 @@ $ tar xvfz
/path/to/apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz
$ cd apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors
$ ls connectors
-pulsar-io-aerospike-{{pulsar.version}}.nar
-pulsar-io-cassandra-{{pulsar.version}}.nar
-pulsar-io-kafka-{{pulsar.version}}.nar
-pulsar-io-kinesis-{{pulsar.version}}.nar
-pulsar-io-rabbitmq-{{pulsar.version}}.nar
-pulsar-io-twitter-{{pulsar.version}}.nar
+pulsar-io-aerospike-{{pulsar:version}}.nar
+pulsar-io-cassandra-{{pulsar:version}}.nar
+pulsar-io-kafka-{{pulsar:version}}.nar
+pulsar-io-kinesis-{{pulsar:version}}.nar
+pulsar-io-rabbitmq-{{pulsar:version}}.nar
+pulsar-io-twitter-{{pulsar:version}}.nar
...
```
diff --git a/site2/docs/io-quickstart.md b/site2/docs/io-quickstart.md
index 042ce8e..914429b 100644
--- a/site2/docs/io-quickstart.md
+++ b/site2/docs/io-quickstart.md
@@ -69,12 +69,12 @@ $ tar xvfz
/path/to/apache-pulsar-io-connectors-{{pulsar:version}}-bin.tar.gz
$ cp -r apache-pulsar-io-connectors-{{pulsar:version}}/connectors connectors
$ ls connectors
-pulsar-io-aerospike-{{pulsar.version}}.nar
-pulsar-io-cassandra-{{pulsar.version}}.nar
-pulsar-io-kafka-{{pulsar.version}}.nar
-pulsar-io-kinesis-{{pulsar.version}}.nar
-pulsar-io-rabbitmq-{{pulsar.version}}.nar
-pulsar-io-twitter-{{pulsar.version}}.nar
+pulsar-io-aerospike-{{pulsar:version}}.nar
+pulsar-io-cassandra-{{pulsar:version}}.nar
+pulsar-io-kafka-{{pulsar:version}}.nar
+pulsar-io-kinesis-{{pulsar:version}}.nar
+pulsar-io-rabbitmq-{{pulsar:version}}.nar
+pulsar-io-twitter-{{pulsar:version}}.nar
...
```
diff --git
a/site2/website/versioned_docs/version-2.1.0-incubating/client-libraries-go.md
b/site2/website/versioned_docs/version-2.1.0-incubating
[incubator-pulsar] branch master updated: [Schema] Provide a generic record interface for representing a typed message (#2452)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 757222d [Schema] Provide a generic record interface for representing
a typed message (#2452)
757222d is described below
commit 757222d9d232dbe03293df890609aa85ff63556c
Author: Sijie Guo
AuthorDate: Mon Aug 27 22:21:27 2018 -0700
[Schema] Provide a generic record interface for representing a typed
message (#2452)
* [Schema] Provide a generic record interface for representing a typed
message
### Motivation
In some use cases, the publishers and consumers don't know the type or
schema of the messages ahead of time.
For example, in pulsar io connector, when connecting a topic to a jdbc
table, the connector doesn't know
the tyep of the messages ahead of time; the connector can only fetch schema
info from schema registry and
that is the only information connector knows. It is impossible for mapping
the messages to a relational database
table.
So we need a way to present a generic `Struct` record with fields.
### Changes
Introduce `Field` and `GenericRecord` to represent `Struct` records
deserialized with a schema.
### NotCovered
This change only introduces the interfaces. It doesn't integrate with the
producer and consumer workflow.
That would be done in subsequent changes if we agree on the interfaces.
---
.../org/apache/pulsar/client/api/schema/Field.java | 43 +
.../pulsar/client/api/schema/GenericRecord.java| 51 +++
.../pulsar/client/impl/schema/AvroSchema.java | 17 ++--
.../client/impl/schema/GenericAvroRecord.java | 79
.../client/impl/schema/GenericAvroSchema.java | 102 +
.../pulsar/client/schema/AvroSchemaTest.java | 47 +-
6 files changed, 329 insertions(+), 10 deletions(-)
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java
new file mode 100644
index 000..653b5d6
--- /dev/null
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/Field.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.client.api.schema;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+/**
+ * A field in a record, consisting of a field name, index, and
+ * {@link org.apache.pulsar.client.api.Schema} for the field value.
+ */
+@Data
+@EqualsAndHashCode
+@ToString
+public class Field {
+
+/**
+ * The field name.
+ */
+private final String name;
+/**
+ * The index of the field within the record.
+ */
+private final int index;
+
+}
diff --git
a/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
new file mode 100644
index 000..0a4fce4
--- /dev/null
+++
b/pulsar-client-schema/src/main/java/org/apache/pulsar/client/api/schema/GenericRecord.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+pac
[incubator-pulsar] branch branch-2.1 updated: Revert "Kinesis-sink consider topic-name as partition-key if record key empty (#2372)"
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9ede4c8 Revert "Kinesis-sink consider topic-name as partition-key if
record key empty (#2372)"
9ede4c8 is described below
commit 9ede4c80802626065f70fb5e20b81bafb6f33969
Author: Sijie Guo
AuthorDate: Mon Aug 27 11:47:56 2018 -0700
Revert "Kinesis-sink consider topic-name as partition-key if record key
empty (#2372)"
This reverts commit 703305b5426856bab7bab30a41e4f242e7782dc7.
---
.../kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
index cdfadde..67de21a 100644
---
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
+++
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
@@ -114,7 +114,7 @@ public class KinesisSink implements Sink {
record.getRecordSequence());
throw new IllegalStateException("kinesis queue has publish
failure");
}
-String partitionedKey =
record.getKey().orElse(record.getTopicName().orElse(defaultPartitionedKey));
+String partitionedKey = record.getKey().orElse(defaultPartitionedKey);
partitionedKey = partitionedKey.length() > maxPartitionedKeyLength
? partitionedKey.substring(0, maxPartitionedKeyLength - 1)
: partitionedKey; // partitionedKey Length must be at least
one, and at most 256
[incubator-pulsar] branch branch-2.1 updated: Added Reader.HasNext in Go client (#2450)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b6dc7c1 Added Reader.HasNext in Go client (#2450)
b6dc7c1 is described below
commit b6dc7c1eae314e65cf782a7b0201408279dc9e8d
Author: Matteo Merli
AuthorDate: Mon Aug 27 11:41:24 2018 -0700
Added Reader.HasNext in Go client (#2450)
### Motivation
Added `Reader.HasNext()` in Go client library
---
pulsar-client-go/pulsar/c_reader.go| 13 +
pulsar-client-go/pulsar/reader.go | 3 +++
pulsar-client-go/pulsar/reader_test.go | 12 ++--
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/pulsar-client-go/pulsar/c_reader.go
b/pulsar-client-go/pulsar/c_reader.go
index 730f9b8..12c1103 100644
--- a/pulsar-client-go/pulsar/c_reader.go
+++ b/pulsar-client-go/pulsar/c_reader.go
@@ -146,6 +146,19 @@ func (r *reader) Next(ctx context.Context) (Message,
error) {
}
}
+func (r *reader) HasNext() (bool, error) {
+ value := C.int(0)
+ res := C.pulsar_reader_has_message_available(r.ptr, )
+
+ if res != C.pulsar_result_Ok {
+ return false, newError(res, "Failed to check if next message is
available")
+ } else if value == C.int(1) {
+ return true, nil
+ } else {
+ return false, nil
+ }
+}
+
func (r *reader) Close() error {
channel := make(chan error)
r.CloseAsync(func(err error) { channel <- err; close(channel) })
diff --git a/pulsar-client-go/pulsar/reader.go
b/pulsar-client-go/pulsar/reader.go
index f61ebd7..7015c9c 100644
--- a/pulsar-client-go/pulsar/reader.go
+++ b/pulsar-client-go/pulsar/reader.go
@@ -67,6 +67,9 @@ type Reader interface {
// Read the next message in the topic, blocking until a message is
available
Next(context.Context) (Message, error)
+ // Check if there is any message available to read from the current
position
+ HasNext() (bool, error)
+
// Close the reader and stop the broker to push more messages
Close() error
}
diff --git a/pulsar-client-go/pulsar/reader_test.go
b/pulsar-client-go/pulsar/reader_test.go
index 11d1b36..3b075e1 100644
--- a/pulsar-client-go/pulsar/reader_test.go
+++ b/pulsar-client-go/pulsar/reader_test.go
@@ -20,9 +20,9 @@
package pulsar
import (
- "testing"
- "fmt"
"context"
+ "fmt"
+ "testing"
)
func TestReaderConnectError(t *testing.T) {
@@ -80,12 +80,20 @@ func TestReader(t *testing.T) {
t.Fatal(err)
}
+ hasNext, err := reader.HasNext()
+ assertNil(t, err)
+ assertEqual(t, hasNext, true)
+
msg, err := reader.Next(ctx)
assertNil(t, err)
assertNotNil(t, msg)
assertEqual(t, string(msg.Payload()), fmt.Sprintf("hello-%d",
i))
}
+
+ hasNext, err := reader.HasNext()
+ assertNil(t, err)
+ assertEqual(t, hasNext, false)
}
func TestReaderWithInvalidConf(t *testing.T) {
[incubator-pulsar] branch master updated: Added Reader.HasNext in Go client (#2450)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 73bc19d Added Reader.HasNext in Go client (#2450)
73bc19d is described below
commit 73bc19dc41b6a06bd7051216682cc5d2fc5d5592
Author: Matteo Merli
AuthorDate: Mon Aug 27 11:41:24 2018 -0700
Added Reader.HasNext in Go client (#2450)
### Motivation
Added `Reader.HasNext()` in Go client library
---
pulsar-client-go/pulsar/c_reader.go| 13 +
pulsar-client-go/pulsar/reader.go | 3 +++
pulsar-client-go/pulsar/reader_test.go | 12 ++--
3 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/pulsar-client-go/pulsar/c_reader.go
b/pulsar-client-go/pulsar/c_reader.go
index 730f9b8..12c1103 100644
--- a/pulsar-client-go/pulsar/c_reader.go
+++ b/pulsar-client-go/pulsar/c_reader.go
@@ -146,6 +146,19 @@ func (r *reader) Next(ctx context.Context) (Message,
error) {
}
}
+func (r *reader) HasNext() (bool, error) {
+ value := C.int(0)
+ res := C.pulsar_reader_has_message_available(r.ptr, )
+
+ if res != C.pulsar_result_Ok {
+ return false, newError(res, "Failed to check if next message is
available")
+ } else if value == C.int(1) {
+ return true, nil
+ } else {
+ return false, nil
+ }
+}
+
func (r *reader) Close() error {
channel := make(chan error)
r.CloseAsync(func(err error) { channel <- err; close(channel) })
diff --git a/pulsar-client-go/pulsar/reader.go
b/pulsar-client-go/pulsar/reader.go
index f61ebd7..7015c9c 100644
--- a/pulsar-client-go/pulsar/reader.go
+++ b/pulsar-client-go/pulsar/reader.go
@@ -67,6 +67,9 @@ type Reader interface {
// Read the next message in the topic, blocking until a message is
available
Next(context.Context) (Message, error)
+ // Check if there is any message available to read from the current
position
+ HasNext() (bool, error)
+
// Close the reader and stop the broker to push more messages
Close() error
}
diff --git a/pulsar-client-go/pulsar/reader_test.go
b/pulsar-client-go/pulsar/reader_test.go
index 11d1b36..3b075e1 100644
--- a/pulsar-client-go/pulsar/reader_test.go
+++ b/pulsar-client-go/pulsar/reader_test.go
@@ -20,9 +20,9 @@
package pulsar
import (
- "testing"
- "fmt"
"context"
+ "fmt"
+ "testing"
)
func TestReaderConnectError(t *testing.T) {
@@ -80,12 +80,20 @@ func TestReader(t *testing.T) {
t.Fatal(err)
}
+ hasNext, err := reader.HasNext()
+ assertNil(t, err)
+ assertEqual(t, hasNext, true)
+
msg, err := reader.Next(ctx)
assertNil(t, err)
assertNotNil(t, msg)
assertEqual(t, string(msg.Payload()), fmt.Sprintf("hello-%d",
i))
}
+
+ hasNext, err := reader.HasNext()
+ assertNil(t, err)
+ assertEqual(t, hasNext, false)
}
func TestReaderWithInvalidConf(t *testing.T) {
[incubator-pulsar] branch branch-2.1 updated: Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 2595941 Issue #2330: change getTopicName in MultiTopicsConsumer
(#2346)
2595941 is described below
commit 259594165f5ee3c69a29629534af81011cab4ef1
Author: Jia Zhai
AuthorDate: Tue Aug 28 02:38:41 2018 +0800
Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
* change getTopicName in MultiTopicsConsumer
* change following sijie's comments
* keep both topicName and topicPartitonName in consumer to avoid new string
---
.../apache/pulsar/client/impl/ConsumerImpl.java| 8 +++
.../client/impl/MultiTopicsConsumerImpl.java | 9
.../pulsar/client/impl/TopicMessageIdImpl.java | 27 ++
.../pulsar/client/impl/TopicMessageImpl.java | 20
.../client/impl/UnAckedTopicMessageTracker.java| 4 ++--
.../pulsar/client/impl/MessageIdCompareToTest.java | 6 -
.../tests/integration/semantics/SemanticsTest.java | 2 +-
7 files changed, 60 insertions(+), 16 deletions(-)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 1a7b67b..fe0e2c1 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -131,6 +131,8 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
private final SubscriptionInitialPosition subscriptionInitialPosition;
private final ConnectionHandler connectionHandler;
+private final String topicNameWithoutPartition;
+
enum SubscriptionMode {
// Make the subscription to be backed by a durable cursor that will
retain messages and persist the current
// position
@@ -203,6 +205,8 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
NonPersistentAcknowledgmentGroupingTracker.of();
}
+topicNameWithoutPartition = topicName.getPartitionedTopicName();
+
grabCnx();
}
@@ -1458,6 +1462,10 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
this.connectionHandler.grabCnx();
}
+public String getTopicNameWithoutPartition() {
+return topicNameWithoutPartition;
+}
+
private static final Logger log =
LoggerFactory.getLogger(ConsumerImpl.class);
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index bbfb3f3..6d5d56d 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -230,7 +230,8 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
checkArgument(message instanceof MessageImpl);
lock.writeLock().lock();
try {
-TopicMessageImpl topicMessage = new
TopicMessageImpl<>(consumer.getTopic(), message);
+TopicMessageImpl topicMessage = new TopicMessageImpl<>(
+consumer.getTopic(), consumer.getTopicNameWithoutPartition(),
message);
unAckedMessageTracker.add(topicMessage.getMessageId());
if (log.isDebugEnabled()) {
@@ -369,7 +370,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
}
if (ackType == AckType.Cumulative) {
-Consumer individualConsumer =
consumers.get(topicMessageId.getTopicName());
+Consumer individualConsumer =
consumers.get(topicMessageId.getTopicPartitionName());
if (individualConsumer != null) {
MessageId innerId = topicMessageId.getInnerMessageId();
return individualConsumer.acknowledgeCumulativeAsync(innerId);
@@ -377,7 +378,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
return FutureUtil.failedFuture(new
PulsarClientException.NotConnectedException());
}
} else {
-ConsumerImpl consumer =
consumers.get(topicMessageId.getTopicName());
+ConsumerImpl consumer =
consumers.get(topicMessageId.getTopicPartitionName());
MessageId innerId = topicMessageId.getInnerMessageId();
return consumer.doAcknowledge(innerId, ackType, properties)
@@ -510,7 +511,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
}
removeExpiredMessagesFromQueue(messageIds);
messageIds.stream().map(messageId -> (TopicMessageIdImpl)messageId)
-
[incubator-pulsar] branch master updated: Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8688b67 Issue #2330: change getTopicName in MultiTopicsConsumer
(#2346)
8688b67 is described below
commit 8688b6733177ee4a48d805df18a1d82865cf7320
Author: Jia Zhai
AuthorDate: Tue Aug 28 02:38:41 2018 +0800
Issue #2330: change getTopicName in MultiTopicsConsumer (#2346)
* change getTopicName in MultiTopicsConsumer
* change following sijie's comments
* keep both topicName and topicPartitonName in consumer to avoid new string
---
.../apache/pulsar/client/impl/ConsumerImpl.java| 8 +++
.../client/impl/MultiTopicsConsumerImpl.java | 9
.../pulsar/client/impl/TopicMessageIdImpl.java | 27 ++
.../pulsar/client/impl/TopicMessageImpl.java | 20
.../client/impl/UnAckedTopicMessageTracker.java| 4 ++--
.../pulsar/client/impl/MessageIdCompareToTest.java | 6 -
.../tests/integration/semantics/SemanticsTest.java | 2 +-
7 files changed, 60 insertions(+), 16 deletions(-)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index da04534..fe37b69 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -131,6 +131,8 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
private final SubscriptionInitialPosition subscriptionInitialPosition;
private final ConnectionHandler connectionHandler;
+private final String topicNameWithoutPartition;
+
enum SubscriptionMode {
// Make the subscription to be backed by a durable cursor that will
retain messages and persist the current
// position
@@ -203,6 +205,8 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
NonPersistentAcknowledgmentGroupingTracker.of();
}
+topicNameWithoutPartition = topicName.getPartitionedTopicName();
+
grabCnx();
}
@@ -1458,6 +1462,10 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
this.connectionHandler.grabCnx();
}
+public String getTopicNameWithoutPartition() {
+return topicNameWithoutPartition;
+}
+
private static final Logger log =
LoggerFactory.getLogger(ConsumerImpl.class);
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
index 799bec8..f1cb9cf 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MultiTopicsConsumerImpl.java
@@ -231,7 +231,8 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
checkArgument(message instanceof MessageImpl);
lock.writeLock().lock();
try {
-TopicMessageImpl topicMessage = new
TopicMessageImpl<>(consumer.getTopic(), message);
+TopicMessageImpl topicMessage = new TopicMessageImpl<>(
+consumer.getTopic(), consumer.getTopicNameWithoutPartition(),
message);
unAckedMessageTracker.add(topicMessage.getMessageId());
if (log.isDebugEnabled()) {
@@ -370,7 +371,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
}
if (ackType == AckType.Cumulative) {
-Consumer individualConsumer =
consumers.get(topicMessageId.getTopicName());
+Consumer individualConsumer =
consumers.get(topicMessageId.getTopicPartitionName());
if (individualConsumer != null) {
MessageId innerId = topicMessageId.getInnerMessageId();
return individualConsumer.acknowledgeCumulativeAsync(innerId);
@@ -378,7 +379,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
return FutureUtil.failedFuture(new
PulsarClientException.NotConnectedException());
}
} else {
-ConsumerImpl consumer =
consumers.get(topicMessageId.getTopicName());
+ConsumerImpl consumer =
consumers.get(topicMessageId.getTopicPartitionName());
MessageId innerId = topicMessageId.getInnerMessageId();
return consumer.doAcknowledge(innerId, ackType, properties)
@@ -511,7 +512,7 @@ public class MultiTopicsConsumerImpl extends
ConsumerBase {
}
removeExpiredMessagesFromQueue(messageIds);
messageIds.stream().map(messageId -> (TopicMessageIdImpl)messageId)
-.collect(Collec
[incubator-pulsar] branch branch-2.1 updated: fix issue with protocol version not being correct (#2444)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 96bf00f fix issue with protocol version not being correct (#2444)
96bf00f is described below
commit 96bf00fa12bf74db57d36f40989e233752288351
Author: Boyang Jerry Peng
AuthorDate: Fri Aug 24 22:40:58 2018 -0700
fix issue with protocol version not being correct (#2444)
---
.../src/main/java/org/apache/pulsar/client/impl/ClientCnx.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
index 6a8ab74..b2d9478 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java
@@ -232,8 +232,9 @@ public class ClientCnx extends PulsarHandler {
if (log.isDebugEnabled()) {
log.debug("{} Connection is ready", ctx.channel());
}
-connectionFuture.complete(null);
+// set remote protocol version to the correct version before we
complete the connection future
remoteEndpointProtocolVersion = connected.getProtocolVersion();
+connectionFuture.complete(null);
state = State.Ready;
}
[incubator-pulsar] branch branch-2.1 updated: leak assign schema in MessageImpl constructor (#2439)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new acd77a0 leak assign schema in MessageImpl constructor (#2439)
acd77a0 is described below
commit acd77a0fe6f09de1919040b65a90fb145f7f1bfa
Author: Jia Zhai
AuthorDate: Fri Aug 24 11:42:55 2018 +0800
leak assign schema in MessageImpl constructor (#2439)
---
.../src/main/java/org/apache/pulsar/client/impl/MessageImpl.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
index 5ea8bfe..96b68c1 100644
--- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
+++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageImpl.java
@@ -155,6 +155,7 @@ public class MessageImpl implements Message {
this.cnx = null;
this.payload = payload;
this.properties = Collections.unmodifiableMap(properties);
+this.schema = schema;
}
public static MessageImpl deserialize(ByteBuf headersAndPayload)
throws IOException {
[incubator-pulsar] branch branch-2.1 updated: Increased default brokerShutdownTimeout to 60 seconds (#2377)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new caf8d67 Increased default brokerShutdownTimeout to 60 seconds (#2377)
caf8d67 is described below
commit caf8d676f0a49896269b57032bd07ae3b4287564
Author: Matteo Merli
AuthorDate: Wed Aug 15 11:50:18 2018 -0700
Increased default brokerShutdownTimeout to 60 seconds (#2377)
### Motivation
The default timeout for broker graceful shutdown is set to 3 seconds. This
can give little room to do graceful shutdown when the broker is serving a lot
of topics.
There is no big downside in increasing the timeout to a much bigger value.
---
conf/broker.conf| 2 +-
conf/standalone.conf| 2 +-
deployment/terraform-ansible/templates/broker.conf | 2 +-
.../src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java| 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/conf/broker.conf b/conf/broker.conf
index 2e40d82..cd4a505 100644
--- a/conf/broker.conf
+++ b/conf/broker.conf
@@ -53,7 +53,7 @@ failureDomainsEnabled=false
zooKeeperSessionTimeoutMillis=3
# Time to wait for broker graceful shutdown. After this time elapses, the
process will be killed
-brokerShutdownTimeoutMs=3000
+brokerShutdownTimeoutMs=6
# Enable backlog quota check. Enforces action on topic when the quota is
reached
backlogQuotaCheckEnabled=true
diff --git a/conf/standalone.conf b/conf/standalone.conf
index 0d7a395..cc03b95 100644
--- a/conf/standalone.conf
+++ b/conf/standalone.conf
@@ -46,7 +46,7 @@ failureDomainsEnabled=false
zooKeeperSessionTimeoutMillis=3
# Time to wait for broker graceful shutdown. After this time elapses, the
process will be killed
-brokerShutdownTimeoutMs=3000
+brokerShutdownTimeoutMs=6
# Enable backlog quota check. Enforces action on topic when the quota is
reached
backlogQuotaCheckEnabled=true
diff --git a/deployment/terraform-ansible/templates/broker.conf
b/deployment/terraform-ansible/templates/broker.conf
index 73d4e2f..a7456a4 100644
--- a/deployment/terraform-ansible/templates/broker.conf
+++ b/deployment/terraform-ansible/templates/broker.conf
@@ -53,7 +53,7 @@ failureDomainsEnabled=false
zooKeeperSessionTimeoutMillis=3
# Time to wait for broker graceful shutdown. After this time elapses, the
process will be killed
-brokerShutdownTimeoutMs=3000
+brokerShutdownTimeoutMs=6
# Enable backlog quota check. Enforces action on topic when the quota is
reached
backlogQuotaCheckEnabled=true
diff --git
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
index 1bb6fff..ddc7728 100644
---
a/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
+++
b/pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java
@@ -78,7 +78,7 @@ public class ServiceConfiguration implements
PulsarConfiguration {
// Time to wait for broker graceful shutdown. After this time elapses, the
// process will be killed
@FieldContext(dynamic = true)
-private long brokerShutdownTimeoutMs = 3000;
+private long brokerShutdownTimeoutMs = 6;
// Enable backlog quota check. Enforces action on topic when the quota is
// reached
private boolean backlogQuotaCheckEnabled = true;
[incubator-pulsar] branch branch-2.1 updated: [client] add properties to producer for cpp & python client (#2420)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d7ec1c5 [client] add properties to producer for cpp & python client
(#2420)
d7ec1c5 is described below
commit d7ec1c5051ee3f7066cd1ba1572606760a3c371e
Author: Sijie Guo
AuthorDate: Wed Aug 22 09:32:46 2018 -0700
[client] add properties to producer for cpp & python client (#2420)
* [client] add properties to producer for cpp & python client
### Motivation
This is a caught-up change to enable properties for producer as java
clients.
### Changes
Enable properties on producer for both cpp & python client
### Results
Properties are added as metadata for CommandProducer. However there is no
way
to verify the producer properties. so I didn't add any specific tests, just
adding properties for both cpp and python clients in the tests, that should
excerise the corresponding code path.
* Add `properties` to pydoc
---
.../include/pulsar/ProducerConfiguration.h | 34 ++
pulsar-client-cpp/lib/Commands.cc | 11 ++-
pulsar-client-cpp/lib/Commands.h | 3 +-
pulsar-client-cpp/lib/ProducerConfiguration.cc | 34 +-
pulsar-client-cpp/lib/ProducerConfigurationImpl.h | 1 +
pulsar-client-cpp/lib/ProducerImpl.cc | 3 +-
pulsar-client-cpp/python/pulsar/__init__.py| 11 ++-
pulsar-client-cpp/python/src/config.cc | 1 +
pulsar-client-cpp/python/test_producer.py | 6 +++-
pulsar-client-cpp/tests/BasicEndToEndTest.cc | 2 ++
10 files changed, 100 insertions(+), 6 deletions(-)
diff --git a/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h
b/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h
index 9f7bf1f..45154c5 100644
--- a/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h
+++ b/pulsar-client-cpp/include/pulsar/ProducerConfiguration.h
@@ -127,6 +127,40 @@ class ProducerConfiguration {
bool isEncryptionEnabled() const;
ProducerConfiguration& addEncryptionKey(std::string key);
+/**
+ * Check whether the message has a specific property attached.
+ *
+ * @param name the name of the property to check
+ * @return true if the message has the specified property
+ * @return false if the property is not defined
+ */
+bool hasProperty(const std::string& name) const;
+
+/**
+ * Get the value of a specific property
+ *
+ * @param name the name of the property
+ * @return the value of the property or null if the property was not
defined
+ */
+const std::string& getProperty(const std::string& name) const;
+
+/**
+ * Get all the properties attached to this producer.
+ */
+std::map& getProperties() const;
+
+/**
+ * Sets a new property on a message.
+ * @param name the name of the property
+ * @param value the associated value
+ */
+ProducerConfiguration& setProperty(const std::string& name, const
std::string& value);
+
+/**
+ * Add all the properties in the provided map
+ */
+ProducerConfiguration& setProperties(const std::map& properties);
+
friend class PulsarWrapper;
private:
diff --git a/pulsar-client-cpp/lib/Commands.cc
b/pulsar-client-cpp/lib/Commands.cc
index 8bd0128..4d9f1be 100644
--- a/pulsar-client-cpp/lib/Commands.cc
+++ b/pulsar-client-cpp/lib/Commands.cc
@@ -22,6 +22,7 @@
#include "pulsar/MessageBuilder.h"
#include "PulsarApi.pb.h"
#include "LogUtils.h"
+#include "PulsarApi.pb.h"
#include "Utils.h"
#include "Url.h"
#include "checksum/ChecksumProvider.h"
@@ -231,13 +232,21 @@ SharedBuffer Commands::newUnsubscribe(uint64_t
consumerId, uint64_t requestId) {
}
SharedBuffer Commands::newProducer(const std::string& topic, uint64_t
producerId,
- const std::string& producerName, uint64_t
requestId) {
+ const std::string& producerName, uint64_t
requestId,
+ const std::map&
metadata) {
BaseCommand cmd;
cmd.set_type(BaseCommand::PRODUCER);
CommandProducer* producer = cmd.mutable_producer();
producer->set_topic(topic);
producer->set_producer_id(producerId);
producer->set_request_id(requestId);
+for (std::map::const_iterator it =
metadata.begin(); it != metadata.end();
+ it++) {
+proto::KeyValue* keyValue = proto::KeyValue().New();
+keyValue->set_key(it->first);
+keyValue->set_value(it->second);
+producer->mutable_metadata()
[incubator-pulsar] branch branch-2.1 updated: [client] add properties to consumer for cpp & python client (#2423)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ffde1bc [client] add properties to consumer for cpp & python client
(#2423)
ffde1bc is described below
commit ffde1bcd45c73e94d5f41a05fc6fb2c6f31d3764
Author: Sijie Guo
AuthorDate: Wed Aug 22 02:41:00 2018 -0700
[client] add properties to consumer for cpp & python client (#2423)
* [client] add properties to consumer for cpp & python client
### Motivation
This is a caught-up change to enable properties for consumer as java
clients.
### Changes
Enable properties on consumer for both cpp & python client
### Results
Properties are added as metadata for CommandSubscribe. However there is no
way
to verify the consumer properties. so I didn't add any specific tests, just
adding properties for both cpp and python clients in the tests, that should
excerise the corresponding code path.
* remove "make format"
---
.../include/pulsar/ConsumerConfiguration.h | 34 ++
pulsar-client-cpp/lib/Commands.cc | 12 +++-
pulsar-client-cpp/lib/Commands.h | 2 +-
pulsar-client-cpp/lib/ConsumerConfiguration.cc | 32
pulsar-client-cpp/lib/ConsumerConfigurationImpl.h | 4 ++-
pulsar-client-cpp/lib/ConsumerImpl.cc | 2 +-
pulsar-client-cpp/python/pulsar/__init__.py| 11 ++-
pulsar-client-cpp/python/src/config.cc | 1 +
pulsar-client-cpp/python/test_consumer.py | 6 +++-
pulsar-client-cpp/tests/BasicEndToEndTest.cc | 6 +++-
10 files changed, 103 insertions(+), 7 deletions(-)
diff --git a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
index 36e5808..0687166 100644
--- a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
+++ b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
@@ -162,6 +162,40 @@ class ConsumerConfiguration {
void setPatternAutoDiscoveryPeriod(int periodInSeconds);
int getPatternAutoDiscoveryPeriod() const;
+/**
+ * Check whether the message has a specific property attached.
+ *
+ * @param name the name of the property to check
+ * @return true if the message has the specified property
+ * @return false if the property is not defined
+ */
+bool hasProperty(const std::string& name) const;
+
+/**
+ * Get the value of a specific property
+ *
+ * @param name the name of the property
+ * @return the value of the property or null if the property was not
defined
+ */
+const std::string& getProperty(const std::string& name) const;
+
+/**
+ * Get all the properties attached to this producer.
+ */
+std::map& getProperties() const;
+
+/**
+ * Sets a new property on a message.
+ * @param name the name of the property
+ * @param value the associated value
+ */
+ConsumerConfiguration& setProperty(const std::string& name, const
std::string& value);
+
+/**
+ * Add all the properties in the provided map
+ */
+ConsumerConfiguration& setProperties(const std::map& properties);
+
friend class PulsarWrapper;
private:
diff --git a/pulsar-client-cpp/lib/Commands.cc
b/pulsar-client-cpp/lib/Commands.cc
index 8a1933b..8bd0128 100644
--- a/pulsar-client-cpp/lib/Commands.cc
+++ b/pulsar-client-cpp/lib/Commands.cc
@@ -20,6 +20,7 @@
#include "MessageImpl.h"
#include "Version.h"
#include "pulsar/MessageBuilder.h"
+#include "PulsarApi.pb.h"
#include "LogUtils.h"
#include "Utils.h"
#include "Url.h"
@@ -27,6 +28,7 @@
#include
#include
+using namespace pulsar;
namespace pulsar {
using namespace pulsar::proto;
@@ -185,7 +187,8 @@ SharedBuffer Commands::newConnect(const AuthenticationPtr&
authentication, const
SharedBuffer Commands::newSubscribe(const std::string& topic, const
std::string& subscription,
uint64_t consumerId, uint64_t requestId,
CommandSubscribe_SubType subType,
const std::string& consumerName,
SubscriptionMode subscriptionMode,
-Optional startMessageId, bool
readCompacted) {
+Optional startMessageId, bool
readCompacted,
+const std::map&
metadata) {
BaseCommand cmd;
cmd.set_type(BaseCommand::SUBSCRIBE);
CommandSubscribe* subscribe = cmd.mutable_subscribe();
@@ -206,6 +209,13 @@ SharedBuffer Commands::newSubscribe(const std::string&
top
[incubator-pulsar] branch branch-2.1 updated: Fixed race condition during expansion of concurrent open hash maps (#2387)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 6146ca9 Fixed race condition during expansion of concurrent open hash
maps (#2387)
6146ca9 is described below
commit 6146ca9cc4c307bf4b19915ca6f13280efd7d428
Author: Matteo Merli
AuthorDate: Fri Aug 17 08:47:30 2018 -0700
Fixed race condition during expansion of concurrent open hash maps (#2387)
### Motivation
Porting same fix as https://github.com/apache/bookkeeper/pull/1607 to
correct issue reported on https://github.com/apache/bookkeeper/issues/1606.
There is a race condition in the concurrent open hash maps implementation.
The race happens when the maps gets re-hashed after the expansion and the new
arrays are substituting the old ones.
The race itself is that a thread doing a `get()` on the map is first
checking the current capacity of the map, uses that to get the bucket and then
tries to do optimistic read of the value in that bucket.
This assumes `capacity` update is visible only after the `values` array is
already swapped, but that is not always the case in current code.
### Changes
* Use `volatile` qualifier for `capacity` and `values` arrays to ensure
ordering of memory read is respected by compiler
* In rehashing, update `capacity` after `values` where it was not already
the case
---
.../util/collections/ConcurrentLongHashMap.java| 8 +--
.../util/collections/ConcurrentLongPairSet.java| 15 +++---
.../util/collections/ConcurrentOpenHashMap.java| 6 +--
.../util/collections/ConcurrentOpenHashSet.java| 6 +--
.../collections/ConcurrentLongHashMapTest.java | 57 --
5 files changed, 72 insertions(+), 20 deletions(-)
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java
index fcb0c10..60c24c0 100644
---
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java
+++
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongHashMap.java
@@ -30,7 +30,7 @@ import com.google.common.collect.Lists;
/**
* Map from long to an Object.
- *
+ *
* Provides similar methods as a ConcurrentMap with 2 differences:
*
* No boxing/unboxing from long -> Long
@@ -187,10 +187,10 @@ public class ConcurrentLongHashMap {
// A section is a portion of the hash map that is covered by a single
@SuppressWarnings("serial")
private static final class Section extends StampedLock {
-private long[] keys;
-private V[] values;
+private volatile long[] keys;
+private volatile V[] values;
-private int capacity;
+private volatile int capacity;
private volatile int size;
private int usedBuckets;
private int resizeThreshold;
diff --git
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java
index 74d4314..4634b40 100644
---
a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java
+++
b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/ConcurrentLongPairSet.java
@@ -26,7 +26,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.StampedLock;
-import java.util.function.Predicate;
/**
* Concurrent hash set where values are composed of pairs of longs.
@@ -163,11 +162,11 @@ public class ConcurrentLongPairSet {
/**
* Removes all of the elements of this collection that satisfy the given
predicate.
- *
+ *
* @param filter
*a predicate which returns {@code true} for elements to be
removed
* @return {@code true} if any elements were removed
- *
+ *
* @return number of removed values
*/
public int removeIf(LongPairPredicate filter) {
@@ -209,9 +208,9 @@ public class ConcurrentLongPairSet {
@SuppressWarnings("serial")
private static final class Section extends StampedLock {
// Keys and values are stored interleaved in the table array
-private long[] table;
+private volatile long[] table;
-private int capacity;
+private volatile int capacity;
private volatile int size;
private int usedBuckets;
private int resizeThreshold;
@@ -449,9 +448,11 @@ public class ConcurrentLongPairSet {
}
}
-capacity = newCapacity;
[incubator-pulsar] branch branch-2.1 updated: Kinesis-sink consider topic-name as partition-key if record key empty (#2372)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 703305b Kinesis-sink consider topic-name as partition-key if record
key empty (#2372)
703305b is described below
commit 703305b5426856bab7bab30a41e4f242e7782dc7
Author: Rajan Dhabalia
AuthorDate: Tue Aug 14 15:41:45 2018 -0700
Kinesis-sink consider topic-name as partition-key if record key empty
(#2372)
---
.../kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
index 67de21a..cdfadde 100644
---
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
+++
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
@@ -114,7 +114,7 @@ public class KinesisSink implements Sink {
record.getRecordSequence());
throw new IllegalStateException("kinesis queue has publish
failure");
}
-String partitionedKey = record.getKey().orElse(defaultPartitionedKey);
+String partitionedKey =
record.getKey().orElse(record.getTopicName().orElse(defaultPartitionedKey));
partitionedKey = partitionedKey.length() > maxPartitionedKeyLength
? partitionedKey.substring(0, maxPartitionedKeyLength - 1)
: partitionedKey; // partitionedKey Length must be at least
one, and at most 256
[incubator-pulsar] branch branch-2.1 updated: [compaction] make topic compaction works with partitioned topic (#2367)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new f97a8f7 [compaction] make topic compaction works with partitioned
topic (#2367)
f97a8f7 is described below
commit f97a8f7d136ddc609d01ce54e20f45e9949b04df
Author: Sijie Guo
AuthorDate: Tue Aug 14 11:02:24 2018 -0700
[compaction] make topic compaction works with partitioned topic (#2367)
* [compaction] make topic compaction works with partitioned topic
### Motivation
Topic compaction doesn't work with partitioned topic.
### Changes
- make `RawReaderImpl` and `ReaderImpl` return message with partition idx
- make broker service `Consumer` deliver MessageIdData with partition idx
- add an integration test to ensure compaction work with partitioned topic
---
.../java/org/apache/pulsar/broker/service/Consumer.java | 9 +++--
.../org/apache/pulsar/client/impl/RawReaderImpl.java | 16
.../org/apache/pulsar/compaction/TwoPhaseCompactor.java | 8
.../PersistentDispatcherFailoverConsumerTest.java| 2 +-
.../java/org/apache/pulsar/client/impl/ReaderImpl.java | 4 +++-
5 files changed, 27 insertions(+), 12 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
index c3befa5..4c54018 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/Consumer.java
@@ -73,6 +73,7 @@ public class Consumer {
private final String appId;
private AuthenticationDataSource authenticationData;
private final String topicName;
+private final int partitionIdx;
private final InitialPosition subscriptionInitialPosition;
private final long consumerId;
@@ -119,6 +120,7 @@ public class Consumer {
this.subscription = subscription;
this.subType = subType;
this.topicName = topicName;
+this.partitionIdx = TopicName.getPartitionIndex(topicName);
this.consumerId = consumerId;
this.priorityLevel = priorityLevel;
this.readCompacted = readCompacted;
@@ -239,8 +241,11 @@ public class Consumer {
Entry entry = entries.get(i);
PositionImpl pos = (PositionImpl) entry.getPosition();
MessageIdData.Builder messageIdBuilder =
MessageIdData.newBuilder();
-MessageIdData messageId =
messageIdBuilder.setLedgerId(pos.getLedgerId()).setEntryId(pos.getEntryId())
-.build();
+MessageIdData messageId = messageIdBuilder
+.setLedgerId(pos.getLedgerId())
+.setEntryId(pos.getEntryId())
+.setPartition(partitionIdx)
+.build();
ByteBuf metadataAndPayload = entry.getDataBuffer();
// increment ref-count of data and release at the end of
process: so, we can get chance to call entry.release
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
index e768c3e..ae1a4db 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/client/impl/RawReaderImpl.java
@@ -35,6 +35,7 @@ import org.apache.pulsar.client.api.SubscriptionType;
import org.apache.pulsar.client.impl.conf.ConsumerConfigurationData;
import org.apache.pulsar.common.api.proto.PulsarApi.CommandAck.AckType;
import org.apache.pulsar.common.api.proto.PulsarApi.MessageIdData;
+import org.apache.pulsar.common.naming.TopicName;
import org.apache.pulsar.common.util.collections.GrowableArrayBlockingQueue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,8 +103,15 @@ public class RawReaderImpl implements RawReader {
RawConsumerImpl(PulsarClientImpl client,
ConsumerConfigurationData conf,
CompletableFuture> consumerFuture) {
-super(client, conf.getSingleTopic(), conf,
client.externalExecutorProvider().getExecutor(), -1,
-consumerFuture, SubscriptionMode.Durable,
MessageId.earliest, Schema.BYTES);
+super(client,
+conf.getSingleTopic(),
+conf,
+client.externalExecutorProvider().getExecutor(),
+TopicName.getPartitionIndex(conf.getSingleTopic()),
+consumerFuture,
+SubscriptionMode.Durable,
+MessageId.earliest,
+Schema.BYTES);
incomingRawMessages = new GrowableArrayBlockingQu
[incubator-pulsar] 02/02: [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git commit bcb3d3f6d15995b74c51d0507fece569361928da Author: Jia Zhai AuthorDate: Tue Aug 14 06:05:59 2018 +0800 [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311) * catch up receiver queue size support in multi topics consumer * add python config --- pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc | 5 - pulsar-client-cpp/python/pulsar/__init__.py| 13 pulsar-client-cpp/python/src/config.cc | 4 .../client/impl/MultiTopicsConsumerImpl.java | 23 +- 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc index 7be197c..6750273 100644 --- a/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc +++ b/pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc @@ -160,9 +160,12 @@ void MultiTopicsConsumerImpl::subscribeTopicPartitions(const Result result, config.setBrokerConsumerStatsCacheTimeInMs(conf_.getBrokerConsumerStatsCacheTimeInMs()); config.setMessageListener( boost::bind(::messageReceived, shared_from_this(), _1, _2)); -config.setReceiverQueueSize(conf_.getReceiverQueueSize()); int numPartitions = partitionMetadata->getPartitions() >= 1 ? partitionMetadata->getPartitions() : 1; +// Apply total limit of receiver queue size across partitions +config.setReceiverQueueSize( +std::min(conf_.getReceiverQueueSize(), + (int)(conf_.getMaxTotalReceiverQueueSizeAcrossPartitions() / numPartitions))); Lock lock(mutex_); topicsPartitions_.insert(std::make_pair(topicName->toString(), numPartitions)); diff --git a/pulsar-client-cpp/python/pulsar/__init__.py b/pulsar-client-cpp/python/pulsar/__init__.py index 434fb07..f3b560b 100644 --- a/pulsar-client-cpp/python/pulsar/__init__.py +++ b/pulsar-client-cpp/python/pulsar/__init__.py @@ -315,6 +315,7 @@ class Client: send_timeout_millis=3, compression_type=CompressionType.NONE, max_pending_messages=1000, +max_pending_messages_across_partitions=5, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, @@ -352,6 +353,9 @@ class Client: * `max_pending_messages`: Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. +* `max_pending_messages_across_partitions`: + Set the max size of the queue holding the messages pending to receive + an acknowledgment across partitions from the broker. * `block_if_queue_full`: Set whether `send_async` operations should block when the outgoing message queue is full. * `message_routing_mode`: @@ -364,6 +368,7 @@ class Client: _check_type(int, send_timeout_millis, 'send_timeout_millis') _check_type(CompressionType, compression_type, 'compression_type') _check_type(int, max_pending_messages, 'max_pending_messages') +_check_type(int, max_pending_messages_across_partitions, 'max_pending_messages_across_partitions') _check_type(bool, block_if_queue_full, 'block_if_queue_full') _check_type(bool, batching_enabled, 'batching_enabled') _check_type(int, batching_max_messages, 'batching_max_messages') @@ -374,6 +379,7 @@ class Client: conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) + conf.max_pending_messages_across_partitions(max_pending_messages_across_partitions) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) @@ -392,6 +398,7 @@ class Client: consumer_type=ConsumerType.Exclusive, message_listener=None, receiver_queue_size=1000, + max_total_receiver_queue_size_across_partitions=5, consumer_name=None, unacked_messages_timeout_ms=None, broker_consumer_stats_cache_time_ms=3, @@ -434,6 +441,9 @@ class Client: should not be interrupted when the consumer queue size is zero. The default value is 1000 messages and should work well for most use cases. +* `max_total_receiver_queue_size_across_partitions` + Set the max total receiver queue size across partitions. + This setting will be used to reduce the receiver queue size for individua
[incubator-pulsar] branch branch-2.1 updated (b070cfd -> bcb3d3f)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a change to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git. from b070cfd [docker] Publish pulsar-all image to docker hub (#2361) new 068db59 Fix: function with multi-topic not acking on effectively-once (#2347) new bcb3d3f [cpp] receiver queue size config acorss partitions in multi-topics-consumer (#2311) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../api/PartitionedProducerConsumerTest.java | 9 ++--- .../org/apache/pulsar/io/PulsarSinkE2ETest.java| 10 +++--- pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc | 5 ++- pulsar-client-cpp/python/pulsar/__init__.py| 13 +++ pulsar-client-cpp/python/src/config.cc | 4 +++ .../client/impl/MultiTopicsConsumerImpl.java | 40 ++ .../pulsar/functions/source/PulsarSource.java | 24 +++-- 7 files changed, 67 insertions(+), 38 deletions(-)
[incubator-pulsar] 01/02: Fix: function with multi-topic not acking on effectively-once (#2347)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
commit 068db598adb529d1ae441084c280f0e776d2cbbc
Author: Rajan Dhabalia
AuthorDate: Mon Aug 13 14:19:46 2018 -0700
Fix: function with multi-topic not acking on effectively-once (#2347)
### Motivation
`MultiTopicsConsumerImpl` doesn't support `acknowledgeCumulativeAsync` and
therefore, function with multi-topic and `EFFECTIVELY_ONCE` processing is not
acking message and failing `EFFECTIVELY_ONCE` behavior.
### Modifications
Function should ack message for a specific topic consumer if
`inputTopicConsumer` is multi-topic consumer.
### Result
Function should able to ack messages for multi-topic consumer when
processing-guarantee is `EFFECTIVELY_ONCE`
---
.../api/PartitionedProducerConsumerTest.java | 9 ++--
.../org/apache/pulsar/io/PulsarSinkE2ETest.java| 10 +
.../client/impl/MultiTopicsConsumerImpl.java | 17 +--
.../pulsar/functions/source/PulsarSource.java | 24 --
4 files changed, 32 insertions(+), 28 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
index a599532..ae7757d 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/PartitionedProducerConsumerTest.java
@@ -290,19 +290,14 @@ public class PartitionedProducerConsumerTest extends
ProducerConsumerBase {
}
try {
-producer = pulsarClient.newProducer().topic(topicName.toString())
-.enableBatching(false)
-.messageRoutingMode(MessageRoutingMode.SinglePartition)
-.create();
+producer =
pulsarClient.newProducer().topic(topicName.toString()).enableBatching(false)
+
.messageRoutingMode(MessageRoutingMode.SinglePartition).create();
consumer =
pulsarClient.newConsumer().topic(topicName.toString()).subscriptionName("my-sub").subscribe();
producer.send("message1".getBytes());
producer.send("message2".getBytes());
/* Message msg1 = */ consumer.receive();
Message msg2 = consumer.receive();
consumer.acknowledgeCumulative(msg2);
-Assert.fail("should fail since ack cumulative is not supported for
partitioned topic");
-} catch (PulsarClientException e) {
-Assert.assertTrue(e instanceof
PulsarClientException.NotSupportedException);
} finally {
producer.close();
consumer.unsubscribe();
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
index 5398bc9..2001981 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
@@ -62,6 +62,7 @@ import org.apache.pulsar.functions.api.utils.IdentityFunction;
import org.apache.pulsar.functions.instance.JavaInstanceRunnable;
import org.apache.pulsar.functions.proto.Function;
import org.apache.pulsar.functions.proto.Function.FunctionDetails;
+import org.apache.pulsar.functions.proto.Function.ProcessingGuarantees;
import org.apache.pulsar.functions.proto.Function.SinkSpec;
import org.apache.pulsar.functions.proto.Function.SourceSpec;
import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus;
@@ -356,11 +357,11 @@ public class PulsarSinkE2ETest {
retryStrategically((test) -> {
try {
SubscriptionStats subStats =
admin.topics().getStats(sourceTopic).subscriptions.get(subscriptionName);
-return subStats.unackedMessages == 0;
+return subStats.unackedMessages == 0 &&
subStats.msgThroughputOut == totalMsgs;
} catch (PulsarAdminException e) {
return false;
}
-}, 5, 500);
+}, 5, 200);
FunctionRuntimeManager functionRuntimeManager =
functionsWorkerService.getFunctionRuntimeManager();
functionRuntimeManager.updateRates();
@@ -400,11 +401,12 @@ public class PulsarSinkE2ETest {
functionDetailsBuilder.setRuntime(FunctionDetails.Runtime.JAVA);
functionDetailsBuilder.setParallelism(1);
functionDetailsBuilder.setClassName(IdentityFunction.class.getName());
+
functionDetailsBuilder.setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE);
// set source spec
// source spec classname should be em
[incubator-pulsar] branch branch-2.1 updated: [docker] Publish pulsar-all image to docker hub (#2361)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b070cfd [docker] Publish pulsar-all image to docker hub (#2361)
b070cfd is described below
commit b070cfd76c3fb03fcff35f647b711695b3bc1993
Author: Sijie Guo
AuthorDate: Mon Aug 13 09:07:44 2018 -0700
[docker] Publish pulsar-all image to docker hub (#2361)
### Motivation
We have `pulsar-all` image since 2.1.0 release. However this image is not
pushed to docker hub.
### Changes
Publish `pulsar-all` image to docker hub.
---
docker/publish.sh | 4
1 file changed, 4 insertions(+)
diff --git a/docker/publish.sh b/docker/publish.sh
index 411fd0d..f73bdac 100755
--- a/docker/publish.sh
+++ b/docker/publish.sh
@@ -63,18 +63,22 @@ set -x
set -e
docker tag pulsar:latest ${docker_registry_org}/pulsar:latest
+docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:latest
docker tag pulsar-grafana:latest ${docker_registry_org}/pulsar-grafana:latest
docker tag pulsar-dashboard:latest
${docker_registry_org}/pulsar-dashboard:latest
docker tag pulsar:latest ${docker_registry_org}/pulsar:$MVN_VERSION
+docker tag pulsar-all:latest ${docker_registry_org}/pulsar-all:$MVN_VERSION
docker tag pulsar-grafana:latest
${docker_registry_org}/pulsar-grafana:$MVN_VERSION
docker tag pulsar-dashboard:latest
${docker_registry_org}/pulsar-dashboard:$MVN_VERSION
# Push all images and tags
docker push ${docker_registry_org}/pulsar:latest
+docker push ${docker_registry_org}/pulsar-all:latest
docker push ${docker_registry_org}/pulsar-grafana:latest
docker push ${docker_registry_org}/pulsar-dashboard:latest
docker push ${docker_registry_org}/pulsar:$MVN_VERSION
+docker push ${docker_registry_org}/pulsar-all:$MVN_VERSION
docker push ${docker_registry_org}/pulsar-grafana:$MVN_VERSION
docker push ${docker_registry_org}/pulsar-dashboard:$MVN_VERSION
[incubator-pulsar] branch branch-2.1 updated: add encoding for WebTargets parts (#2359)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 1ef9d0d add encoding for WebTargets parts (#2359)
1ef9d0d is described below
commit 1ef9d0daa1492d948ba31625a09c0f1792709121
Author: Jia Zhai
AuthorDate: Mon Aug 13 04:36:41 2018 +0800
add encoding for WebTargets parts (#2359)
### Motivation
Fixes #2304
Issue #2304 reported that Pulsar admin commands will fail with some url
style parts,
```
root@pulsar-admin:/pulsar# pulsar-admin namespaces grant-permission
--actions produce,consume --role 'spiffe://developer/passport'
core-platform/testing
HTTP 404 Not Found
Reason: HTTP 404 Not Found
```
This PR try to fix it.
### Modifications
- use URLEncoder to encode webTarget parts.
- add tests to verify
### Result
all UT passed
---
.../java/org/apache/pulsar/broker/admin/AdminApiTest.java | 6 ++
.../apache/pulsar/client/admin/internal/WebTargets.java| 14 +-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
index c9d973d..ae72737 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
@@ -620,11 +620,15 @@ public class AdminApiTest extends
MockedPulsarServiceBaseTest {
// Ok, got the non authorized exception since usc cluster is not
in the allowed clusters list.
}
+// test with url style role.
+admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1",
+"spiffe://developer/passport-role",
EnumSet.allOf(AuthAction.class));
admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1",
"my-role", EnumSet.allOf(AuthAction.class));
Policies policies = new Policies();
policies.replication_clusters = Sets.newHashSet("test");
policies.bundles = Policies.defaultBundle();
+
policies.auth_policies.namespace_auth.put("spiffe://developer/passport-role",
EnumSet.allOf(AuthAction.class));
policies.auth_policies.namespace_auth.put("my-role",
EnumSet.allOf(AuthAction.class));
assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies);
@@ -632,7 +636,9 @@ public class AdminApiTest extends
MockedPulsarServiceBaseTest {
assertEquals(admin.namespaces().getTopics("prop-xyz/ns1"),
Lists.newArrayList());
+admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1",
"spiffe://developer/passport-role");
admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1",
"my-role");
+
policies.auth_policies.namespace_auth.remove("spiffe://developer/passport-role");
policies.auth_policies.namespace_auth.remove("my-role");
assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies);
diff --git
a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
index 35b04f1..3de248c 100644
---
a/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
+++
b/pulsar-client-admin/src/main/java/org/apache/pulsar/client/admin/internal/WebTargets.java
@@ -18,14 +18,26 @@
*/
package org.apache.pulsar.client.admin.internal;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
import javax.ws.rs.client.WebTarget;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
class WebTargets {
static WebTarget addParts(WebTarget target, String[] parts) {
if (parts != null && parts.length > 0) {
for (String part : parts) {
-target = target.path(part);
+String encode;
+try {
+encode = URLEncoder.encode(part,
StandardCharsets.UTF_8.toString());
+} catch (UnsupportedEncodingException e) {
+log.error(String.format("%s is Unknown",
StandardCharsets.UTF_8.toString()) + "exception - [{}]", e);
+encode = part;
+}
+target = target.path(encode);
}
}
return target;
[incubator-pulsar] branch branch-2.1 updated: Issue 1067: Problems with Partitioned Topics which name contains -partition-N (#2342)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 05488a0 Issue 1067: Problems with Partitioned Topics which name
contains -partition-N (#2342)
05488a0 is described below
commit 05488a0be6e27292344a093f694d2a5f48886601
Author: Sijie Guo
AuthorDate: Fri Aug 10 19:40:44 2018 -0700
Issue 1067: Problems with Partitioned Topics which name contains
-partition-N (#2342)
* Issue 1067: Problems with Partitioned Topics which name contains
-partition-N
### Motivation
Fixes #1067.
Someone accidentally created a partitioned topic in one of our cluster with
a name which contains -partition-2.
This raised all sorts of issues, it would seem that only one partition was
created for this topic, but metadata exists saying that it has 10 partitions.
### Changes
Disallow creating a partitioned topic contains '-partition-' in the name.
* only validate partitioned topic name when create/delete/update
partitioned topics
* Changed to use a CONSTANT
---
.../apache/pulsar/broker/admin/AdminResource.java | 9 +++
.../pulsar/broker/admin/v2/PersistentTopics.java | 42 +
.../pulsar/broker/admin/AdminResourceTest.java | 69 ++
.../org/apache/pulsar/common/naming/TopicName.java | 2 +-
4 files changed, 108 insertions(+), 14 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
index be1b7dd..de7dd0b 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/AdminResource.java
@@ -254,6 +254,15 @@ public abstract class AdminResource extends
PulsarWebResource {
this.topicName = TopicName.get(domain(), namespaceName, topic);
}
+protected void validatePartitionedTopicName(String tenant, String
namespace, String encodedTopic) {
+// first, it has to be a validate topic name
+validateTopicName(tenant, namespace, encodedTopic);
+// second, "-partition-" is not allowed
+if (encodedTopic.contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) {
+throw new RestException(Status.PRECONDITION_FAILED, "Partitioned
Topic Name should not contain '-partition-'");
+}
+}
+
@Deprecated
protected void validateTopicName(String property, String cluster, String
namespace, String encodedTopic) {
String topic = Codec.decode(encodedTopic);
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
index 6a246b8..771a057 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/PersistentTopics.java
@@ -124,12 +124,15 @@ public class PersistentTopics extends
PersistentTopicsBase {
@PUT
@Path("/{tenant}/{namespace}/{topic}/partitions")
@ApiOperation(value = "Create a partitioned topic.", notes = "It needs to
be called before creating a producer on a partitioned topic.")
-@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have
admin permission"),
-@ApiResponse(code = 409, message = "Partitioned topic already
exist") })
+@ApiResponses(value = {
+@ApiResponse(code = 403, message = "Don't have admin permission"),
+@ApiResponse(code = 409, message = "Partitioned topic already exist"),
+@ApiResponse(code = 412, message = "Partitioned topic name is invalid")
+})
public void createPartitionedTopic(@PathParam("tenant") String tenant,
@PathParam("namespace") String namespace,
@PathParam("topic") @Encoded String encodedTopic, int
numPartitions,
@QueryParam("authoritative") @DefaultValue("false") boolean
authoritative) {
-validateTopicName(tenant, namespace, encodedTopic);
+validatePartitionedTopicName(tenant, namespace, encodedTopic);
internalCreatePartitionedTopic(numPartitions, authoritative);
}
@@ -151,18 +154,25 @@ public class PersistentTopics extends
PersistentTopicsBase {
@POST
@Path("/{tenant}/{namespace}/{topic}/partitions")
@ApiOperation(value = "Increment partitons of an existing partitioned
topic.", notes = "It only increments partitions of existing non-global
partitioned-topic")
-@ApiResponses(value = { @ApiResponse(code = 403, m
[incubator-pulsar] branch branch-2.1 updated: Remove snapshot-topic ref (#2355)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new f2e6ea7 Remove snapshot-topic ref (#2355)
f2e6ea7 is described below
commit f2e6ea73aab47a5c977d44c9deb3f99a07e53a94
Author: Rajan Dhabalia
AuthorDate: Thu Aug 9 23:43:02 2018 -0700
Remove snapshot-topic ref (#2355)
### Motivation
remove unused config `functionMetadataSnapshotsTopicPath`
---
conf/functions_worker.yml| 1 -
.../src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java | 1 -
2 files changed, 2 deletions(-)
diff --git a/conf/functions_worker.yml b/conf/functions_worker.yml
index 4194337..8c0d903 100644
--- a/conf/functions_worker.yml
+++ b/conf/functions_worker.yml
@@ -24,7 +24,6 @@ workerPort: 6750
connectorsDirectory: ./connectors
functionMetadataTopicName: metadata
-functionMetadataSnapshotsTopicPath: snapshots
clusterCoordinationTopicName: coordinate
pulsarFunctionsNamespace: public/functions
pulsarFunctionsCluster: standalone
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
index eda9b15..348d245 100644
---
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
+++
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/WorkerConfig.java
@@ -58,7 +58,6 @@ public class WorkerConfig implements Serializable,
PulsarConfiguration {
private String pulsarServiceUrl;
private String pulsarWebServiceUrl;
private String clusterCoordinationTopicName;
-private String functionMetadataSnapshotsTopicPath;
private String pulsarFunctionsNamespace;
private String pulsarFunctionsCluster;
private int numFunctionPackageReplicas;
[incubator-pulsar] branch branch-2.1 updated: Standalone bookie should always advertise 'localhost' when running in docker container (#2425)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new cb88ae0 Standalone bookie should always advertise 'localhost' when
running in docker container (#2425)
cb88ae0 is described below
commit cb88ae029d748017d88cdc7583b5dca1eb3a5b1c
Author: Matteo Merli
AuthorDate: Fri Aug 24 12:55:12 2018 -0700
Standalone bookie should always advertise 'localhost' when running in
docker container (#2425)
* Standalone bookie should always advertise 'localhost' when running in
docker container
* Fixed advertised address coming from config file
---
pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
index 3e44f76..cf07d8c 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
@@ -253,7 +253,8 @@ public class PulsarStandalone implements AutoCloseable {
if (!this.isOnlyBroker()) {
// Start LocalBookKeeper
bkEnsemble = new LocalBookkeeperEnsemble(
-this.getNumOfBk(), this.getZkPort(), this.getBkPort(),
this.getStreamStoragePort(), this.getZkDir(), this.getBkDir(),
this.isWipeData(), config.getAdvertisedAddress());
+this.getNumOfBk(), this.getZkPort(), this.getBkPort(),
this.getStreamStoragePort(), this.getZkDir(),
+this.getBkDir(), this.isWipeData(), "127.0.0.1");
bkEnsemble.startStandalone(!this.isNoStreamStorage());
}
@@ -352,7 +353,7 @@ public class PulsarStandalone implements AutoCloseable {
log.info(e.getMessage());
}
}
-
+
/** this methods gets a buidler to use to build and an embedded pulsar
instance `
* i.e.
*
[incubator-pulsar] branch branch-2.1 updated: Set correct exception if function runnable fails with Error (#2353)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 7a0ca5a Set correct exception if function runnable fails with Error
(#2353)
7a0ca5a is described below
commit 7a0ca5ae1be8d03828ef224350576755fa5f8af9
Author: Rajan Dhabalia
AuthorDate: Mon Aug 13 21:20:02 2018 -0700
Set correct exception if function runnable fails with Error (#2353)
* Set correct exception if function runnable fails with Error
* fix throwable set
---
.../org/apache/pulsar/functions/instance/JavaInstanceRunnable.java | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git
a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
index b64224f..68df84a 100644
---
a/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
+++
b/pulsar-functions/instance/src/main/java/org/apache/pulsar/functions/instance/JavaInstanceRunnable.java
@@ -181,8 +181,10 @@ public class JavaInstanceRunnable implements
AutoCloseable, Runnable {
*/
@Override
public void run() {
+String functionName = null;
try {
ContextImpl contextImpl = setupContext();
+functionName = String.format("%s-%s", contextImpl.getTenant(),
contextImpl.getFunctionName());
javaInstance = setupJavaInstance(contextImpl);
if (null != stateTable) {
StateContextImpl stateContext = new
StateContextImpl(stateTable);
@@ -229,8 +231,8 @@ public class JavaInstanceRunnable implements AutoCloseable,
Runnable {
}
}
} catch (Throwable t) {
-log.error("Uncaught exception in Java Instance", t);
-deathException = (Exception) t;
+log.error("[{}] Uncaught exception in Java Instance",
functionName, t);
+deathException = t;
return;
} finally {
log.info("Closing instance");
[incubator-pulsar] branch branch-2.1 updated: Fix NPE when splitting and unloading bundle (#2348)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4db7f38 Fix NPE when splitting and unloading bundle (#2348)
4db7f38 is described below
commit 4db7f38cfaf96c5a2f524a780b854b8df1878b57
Author: massakam
AuthorDate: Fri Aug 10 13:38:52 2018 +0900
Fix NPE when splitting and unloading bundle (#2348)
---
.../pulsar/broker/namespace/NamespaceService.java | 6 ++--
.../broker/namespace/NamespaceServiceTest.java | 42 ++
2 files changed, 45 insertions(+), 3 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
index bf44519..d9a1ccf 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/namespace/NamespaceService.java
@@ -593,7 +593,7 @@ public class NamespaceService {
boolean unload,
AtomicInteger counter,
CompletableFuture unloadFuture) {
-CompletableFuture updateFuture = new
CompletableFuture<>();
+CompletableFuture> updateFuture = new
CompletableFuture<>();
final Pair> splittedBundles =
bundleFactory.splitBundles(bundle,
2 /* by default split into 2 */);
@@ -622,7 +622,7 @@ public class NamespaceService {
// namespace bundle
bundleFactory.invalidateBundleCache(bundle.getNamespaceObject());
-updateFuture.complete(splittedBundles.getLeft());
+updateFuture.complete(splittedBundles.getRight());
} else if (rc == Code.BADVERSION.intValue()) {
KeeperException keeperException =
KeeperException.create(KeeperException.Code.get(rc));
String msg = format("failed to update namespace
policies [%s], NamespaceBundle: %s " +
@@ -680,7 +680,7 @@ public class NamespaceService {
if (unload) {
// unload new split bundles
-r.getBundles().forEach(splitBundle -> {
+r.forEach(splitBundle -> {
try {
unloadNamespaceBundle(splitBundle);
} catch (Exception e) {
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
index a2ef397..2c51e16 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceServiceTest.java
@@ -429,6 +429,48 @@ public class NamespaceServiceTest extends BrokerTestBase {
}
+@Test
+public void testRemoveOwnershipAndSplitBundle() throws Exception {
+OwnershipCache ownershipCache =
spy(pulsar.getNamespaceService().getOwnershipCache());
+
doNothing().when(ownershipCache).disableOwnership(any(NamespaceBundle.class));
+
+Field ownership =
NamespaceService.class.getDeclaredField("ownershipCache");
+ownership.setAccessible(true);
+ownership.set(pulsar.getNamespaceService(), ownershipCache);
+
+NamespaceService namespaceService = pulsar.getNamespaceService();
+NamespaceName nsname = NamespaceName.get("pulsar/global/ns1");
+TopicName topicName =
TopicName.get("persistent://pulsar/global/ns1/topic-1");
+NamespaceBundles bundles =
namespaceService.getNamespaceBundleFactory().getBundles(nsname);
+NamespaceBundle originalBundle = bundles.findBundle(topicName);
+
+CompletableFuture result1 =
namespaceService.splitAndOwnBundle(originalBundle, false);
+try {
+result1.get();
+} catch (Exception e) {
+fail("split bundle faild", e);
+}
+
+NamespaceBundles updatedNsBundles =
namespaceService.getNamespaceBundleFactory().getBundles(nsname);
+assertNotNull(updatedNsBundles);
+NamespaceBundle splittedBundle =
updatedNsBundles.findBundle(topicName);
+
+updatedNsBundles.getBundles().stream().filter(bundle ->
!bundle.equals(splittedBundle)).forEach(bundle -> {
+try {
+ownershipCache.removeOwnership(bundle).get();
+} catch (Exception e) {
+fail("faild to remove ownership", e);
+}
+});
+
+Completab
[incubator-pulsar] branch branch-2.1 updated: Issue 2272: Fix git-commit-id-plugin (#2327)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 046e500 Issue 2272: Fix git-commit-id-plugin (#2327)
046e500 is described below
commit 046e5007fec67ab89f2093d2ee85db0b09391d58
Author: Sijie Guo
AuthorDate: Tue Aug 7 08:06:05 2018 -0700
Issue 2272: Fix git-commit-id-plugin (#2327)
### Motivation
Fixes #2272.
We are using git-commit-id plugin for generating a properties file
containing all the build information.
However you have to skip git-commit-id-plugin when the source code is not
in a git repo.
### Changes
add `false` to
git-commit-plugin
---
pom.xml | 1 +
1 file changed, 1 insertion(+)
diff --git a/pom.xml b/pom.xml
index 0a2fae5..e04cce4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1137,6 +1137,7 @@ flexible messaging model and an intuitive client
API.
false
true
${project.build.outputDirectory}/${project.artifactId}-git.properties
+ false
false
properties
[incubator-pulsar] branch branch-2.1 updated: Issue #1452: remove reachedEndOfTopic in addConsumer (#2301)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 0cc5e81 Issue #1452: remove reachedEndOfTopic in addConsumer (#2301)
0cc5e81 is described below
commit 0cc5e81fcc5ac3859ee463556e6c1ed7e05d29a3
Author: Jia Zhai
AuthorDate: Mon Aug 6 07:49:42 2018 +0800
Issue #1452: remove reachedEndOfTopic in addConsumer (#2301)
### Motivation
Fixes #1452
In issue #1452 , reachedEndOfTopic was called twice if a topic has been
terminated before subscription.
It may be better to call `reachedEndOfTopic`, when real read/ack happened
to the subscription, so delete the calling in `addConsumer` to avoid dup
calling.
### Modifications
remove dup calling in `addConsumer` .
add related ut.
### Result
Expected all ut passed.
---
.../service/persistent/PersistentSubscription.java | 5 ---
.../client/api/SimpleProducerConsumerTest.java | 42 --
2 files changed, 40 insertions(+), 7 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
index 8a77bfa..0842742 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentSubscription.java
@@ -103,11 +103,6 @@ public class PersistentSubscription implements
Subscription {
throw new SubscriptionFencedException("Subscription is fenced");
}
-if (topic.getManagedLedger().isTerminated() &&
cursor.getNumberOfEntriesInBacklog() == 0) {
-// Immediately notify the consumer that there are no more
available messages
-consumer.reachedEndOfTopic();
-}
-
if (dispatcher == null || !dispatcher.isConsumerConnected()) {
switch (consumer.subType()) {
case Exclusive:
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
index 79166a1..61bdad0 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleProducerConsumerTest.java
@@ -2434,7 +2434,7 @@ public class SimpleProducerConsumerTest extends
ProducerConsumerBase {
Producer producer =
pulsarClient.newProducer().topic("persistent://my-property/my-ns/myrsa-topic1")
.addEncryptionKey(encryptionKeyName).compressionType(CompressionType.LZ4)
.cryptoKeyReader(new EncKeyReader()).create();
-
+
Consumer consumer =
pulsarClient.newConsumer().topicsPattern("persistent://my-property/my-ns/myrsa-topic1")
.subscriptionName("my-subscriber-name").cryptoFailureAction(ConsumerCryptoFailureAction.CONSUME)
.subscribe();
@@ -2450,7 +2450,7 @@ public class SimpleProducerConsumerTest extends
ProducerConsumerBase {
consumer.close();
log.info("-- Exiting {} test --", methodName);
}
-
+
private String decryptMessage(TopicMessageImpl msg, String
encryptionKeyName, CryptoKeyReader reader)
throws Exception {
Optional ctx = msg.getEncryptionCtx();
@@ -2624,4 +2624,42 @@ public class SimpleProducerConsumerTest extends
ProducerConsumerBase {
log.info("-- Exiting {} test --", methodName);
}
+// Issue 1452: https://github.com/apache/incubator-pulsar/issues/1452
+// reachedEndOfTopic should be called only once if a topic has been
terminated before subscription
+@Test
+public void testReachedEndOfTopic() throws Exception
+{
+String topicName =
"persistent://my-property/my-ns/testReachedEndOfTopic";
+Producer producer = pulsarClient.newProducer()
+.topic(topicName)
+.enableBatching(false).create();
+producer.close();
+
+admin.topics().terminateTopicAsync(topicName).get();
+
+CountDownLatch latch = new CountDownLatch(2);
+Consumer consumer = pulsarClient.newConsumer()
+.topic(topicName)
+.subscriptionName("my-subscriber-name")
+.messageListener(new MessageListener()
+{
+@Override
+public void reachedEndOfTopic(Consumer consumer)
+{
+log.info("called reachedEndOfTopic {}", methodName);
+latch.countDown();
+
[incubator-pulsar] branch branch-2.1 updated: Ignore unknown Properties from config file when parsing (#2336)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5ac1322 Ignore unknown Properties from config file when parsing
(#2336)
5ac1322 is described below
commit 5ac1322a7e919217b3979ff1a13c9a7e6ea34541
Author: Sanjeev Kulkarni
AuthorDate: Tue Aug 7 20:41:13 2018 -0700
Ignore unknown Properties from config file when parsing (#2336)
### Motivation
As Functions evolve, we will be adding new properties/fields to the
FunctionConfig. There will be fields present in the config files that won't
make sense for previous admin tool installation. This especially happens when
submitting pulsar functions using newer client to run over kubernetes with an
older installation of pulsar. This pr will mark the unknown fields to be
ignored to get over that problem
---
.../src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java | 2 ++
1 file changed, 2 insertions(+)
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
index 0cd922e..e38249c 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
@@ -18,6 +18,7 @@
*/
package org.apache.pulsar.functions.utils;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.Getter;
@@ -48,6 +49,7 @@ import java.util.Map;
@EqualsAndHashCode
@ToString
@isValidFunctionConfig
+@JsonIgnoreProperties(ignoreUnknown = true)
public class FunctionConfig {
public enum ProcessingGuarantees {
[incubator-pulsar] branch branch-2.1 updated: Cpp client: add PatternMultiTopicsConsumerImpl to support regex subscribe (#2219)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b23134c Cpp client: add PatternMultiTopicsConsumerImpl to support
regex subscribe (#2219)
b23134c is described below
commit b23134c422456f45ed14c3c49de53eb8889e45fc
Author: Jia Zhai
AuthorDate: Sat Aug 4 14:56:56 2018 +0800
Cpp client: add PatternMultiTopicsConsumerImpl to support regex subscribe
(#2219)
In PR #1279 and #1298 we added regex based subscription. This is a catch up
work to add `PatternMultiTopicsConsumerImpl` in cpp client.
---
pulsar-client-cpp/include/pulsar/Client.h | 16 ++
.../include/pulsar/ConsumerConfiguration.h | 10 +
pulsar-client-cpp/lib/BinaryProtoLookupService.cc | 42
pulsar-client-cpp/lib/BinaryProtoLookupService.h | 9 +
pulsar-client-cpp/lib/Client.cc| 24 ++
pulsar-client-cpp/lib/ClientConnection.cc | 86 ++-
pulsar-client-cpp/lib/ClientConnection.h | 13 +-
pulsar-client-cpp/lib/ClientImpl.cc| 58 +
pulsar-client-cpp/lib/ClientImpl.h | 8 +
pulsar-client-cpp/lib/Commands.cc | 18 ++
pulsar-client-cpp/lib/Commands.h | 1 +
pulsar-client-cpp/lib/ConsumerConfiguration.cc | 6 +
pulsar-client-cpp/lib/ConsumerConfigurationImpl.h | 4 +-
pulsar-client-cpp/lib/HTTPLookupService.cc | 112 +++--
pulsar-client-cpp/lib/HTTPLookupService.h | 9 +-
pulsar-client-cpp/lib/LookupService.h | 11 +
pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h| 2 +-
.../lib/PatternMultiTopicsConsumerImpl.cc | 235 +++
.../lib/PatternMultiTopicsConsumerImpl.h | 76 ++
pulsar-client-cpp/tests/BasicEndToEndTest.cc | 257 +
pulsar-client-cpp/tests/BinaryLookupServiceTest.cc | 67 ++
21 files changed, 1032 insertions(+), 32 deletions(-)
diff --git a/pulsar-client-cpp/include/pulsar/Client.h
b/pulsar-client-cpp/include/pulsar/Client.h
index 6a9e487..1913851 100644
--- a/pulsar-client-cpp/include/pulsar/Client.h
+++ b/pulsar-client-cpp/include/pulsar/Client.h
@@ -99,6 +99,9 @@ class Client {
void subscribeAsync(const std::string& topic, const std::string&
consumerName,
const ConsumerConfiguration& conf, SubscribeCallback
callback);
+/**
+ * subscribe for multiple topics under the same namespace.
+ */
Result subscribe(const std::vector& topics, const
std::string& subscriptionName,
Consumer& consumer);
Result subscribe(const std::vector& topics, const
std::string& subscriptionName,
@@ -109,6 +112,19 @@ class Client {
const ConsumerConfiguration& conf, SubscribeCallback
callback);
/**
+ * subscribe for multiple topics, which match given regexPattern, under
the same namespace.
+ */
+Result subscribeWithRegex(const std::string& regexPattern, const
std::string& consumerName,
+ Consumer& consumer);
+Result subscribeWithRegex(const std::string& regexPattern, const
std::string& consumerName,
+ const ConsumerConfiguration& conf, Consumer&
consumer);
+
+void subscribeWithRegexAsync(const std::string& regexPattern, const
std::string& consumerName,
+ SubscribeCallback callback);
+void subscribeWithRegexAsync(const std::string& regexPattern, const
std::string& consumerName,
+ const ConsumerConfiguration& conf,
SubscribeCallback callback);
+
+/**
* Create a topic reader with given {@code ReaderConfiguration} for
reading messages from the specified
* topic.
*
diff --git a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
index c9584e3..36e5808 100644
--- a/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
+++ b/pulsar-client-cpp/include/pulsar/ConsumerConfiguration.h
@@ -152,6 +152,16 @@ class ConsumerConfiguration {
bool isReadCompacted() const;
void setReadCompacted(bool compacted);
+/**
+ * Set the time duration in minutes, for which the
PatternMultiTopicsConsumer will do a pattern auto
+ * discovery.
+ * The default value is 60 seconds. less than 0 will disable auto
discovery.
+ *
+ * @param periodInSeconds period in seconds to do an auto discovery
+ */
+void setPatternAutoDiscoveryPeriod(int periodInSeconds);
+int getPatternAutoDiscoveryPeriod() const;
+
friend class PulsarWrapper;
private:
diff --git a/pulsar-client-cpp/lib/BinaryProtoLookupService.c
[incubator-pulsar] branch branch-2.1 updated: handle subscription-already-exist exception on partitioned-topic for create-sub admin-api (#2269)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d15cd7b handle subscription-already-exist exception on
partitioned-topic for create-sub admin-api (#2269)
d15cd7b is described below
commit d15cd7bc850cb85f8e3038dbb62c98429e542787
Author: Rajan Dhabalia
AuthorDate: Tue Jul 31 11:33:02 2018 -0700
handle subscription-already-exist exception on partitioned-topic for
create-sub admin-api (#2269)
---
.../broker/admin/impl/PersistentTopicsBase.java| 35 +-
.../broker/admin/CreateSubscriptionTest.java | 26
2 files changed, 54 insertions(+), 7 deletions(-)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
index cf8355f..f85c617 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java
@@ -38,9 +38,11 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import javax.ws.rs.WebApplicationException;
@@ -829,15 +831,34 @@ public class PersistentTopicsBase extends AdminResource {
try {
if (partitionMetadata.partitions > 0) {
// Create the subscription on each partition
-List> futures = Lists.newArrayList();
PulsarAdmin admin = pulsar().getAdminClient();
+CountDownLatch latch = new
CountDownLatch(partitionMetadata.partitions);
+AtomicReference exception = new AtomicReference<>();
+AtomicInteger failureCount = new AtomicInteger(0);
+
for (int i = 0; i < partitionMetadata.partitions; i++) {
-
futures.add(admin.topics().createSubscriptionAsync(topicName.getPartition(i).toString(),
-subscriptionName, messageId));
+admin.persistentTopics()
+
.createSubscriptionAsync(topicName.getPartition(i).toString(),
subscriptionName, messageId)
+.handle((result, ex) -> {
+if (ex != null) {
+int c = failureCount.incrementAndGet();
+// fail the operation on unknown exception
or if all the partitioned failed due to
+// subscription-already-exist
+if (c == partitionMetadata.partitions
+|| !(ex instanceof
PulsarAdminException.ConflictException)) {
+exception.set(ex);
+}
+}
+latch.countDown();
+return null;
+});
}
-FutureUtil.waitForAll(futures).join();
+latch.await();
+if (exception.get() != null) {
+throw exception.get();
+}
} else {
validateAdminOperationOnTopic(authoritative);
@@ -850,10 +871,10 @@ public class PersistentTopicsBase extends AdminResource {
PersistentSubscription subscription = (PersistentSubscription)
topic
.createSubscription(subscriptionName,
InitialPosition.Latest).get();
subscription.resetCursor(PositionImpl.get(messageId.getLedgerId(),
messageId.getEntryId())).get();
-log.info("[{}][{}] Successfully created subscription {} at
message id {}", clientAppId(),
-topicName, subscriptionName, messageId);
+log.info("[{}][{}] Successfully created subscription {} at
message id {}", clientAppId(), topicName,
+subscriptionName, messageId);
}
-} catch (Exception e) {
+} catch (Throwable e) {
Throwable t = e.getCause();
log.warn("[{}] [{}] Failed to create subscription {} at message id
{}", clientAppId(),
topicName, subscriptionName, messageId, e);
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
b/pulsar-bro
[incubator-pulsar] branch branch-2.1 updated: kinesis-sink: manage msg ordering for publish callback failure (#2285)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new f449006 kinesis-sink: manage msg ordering for publish callback
failure (#2285)
f449006 is described below
commit f4490067739ad82023df4bc7721c16dbb7245538
Author: Rajan Dhabalia
AuthorDate: Thu Aug 2 12:35:48 2018 -0700
kinesis-sink: manage msg ordering for publish callback failure (#2285)
---
.../org/apache/pulsar/io/kinesis/KinesisSink.java | 49 ++
.../pulsar/io/kinesis/KinesisSinkConfig.java | 1 +
2 files changed, 33 insertions(+), 17 deletions(-)
diff --git
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
index dc70b98..67de21a 100644
---
a/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
+++
b/pulsar-io/kinesis/src/main/java/org/apache/pulsar/io/kinesis/KinesisSink.java
@@ -44,6 +44,7 @@ import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import org.apache.commons.lang3.builder.ReflectionToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
@@ -89,6 +90,12 @@ public class KinesisSink implements Sink {
private static final String defaultPartitionedKey = "default";
private static final int maxPartitionedKeyLength = 256;
private SinkContext sinkContext;
+//
+private static final int FALSE = 0;
+private static final int TRUE = 1;
+private volatile int previousPublishFailed = FALSE;
+private static final AtomicIntegerFieldUpdater
IS_PUBLISH_FAILED =
+AtomicIntegerFieldUpdater.newUpdater(KinesisSink.class,
"previousPublishFailed");
public static final String ACCESS_KEY_NAME = "accessKey";
public static final String SECRET_KEY_NAME = "secretKey";
@@ -101,6 +108,12 @@ public class KinesisSink implements Sink {
@Override
public void write(Record record) throws Exception {
+// kpl-thread captures publish-failure. fail the publish on main
pulsar-io-thread to maintain the ordering
+if (kinesisSinkConfig.isRetainOrdering() && previousPublishFailed ==
TRUE) {
+LOG.warn("Skip acking message to retain ordering with previous
failed message {}-{}", this.streamName,
+record.getRecordSequence());
+throw new IllegalStateException("kinesis queue has publish
failure");
+}
String partitionedKey = record.getKey().orElse(defaultPartitionedKey);
partitionedKey = partitionedKey.length() > maxPartitionedKeyLength
? partitionedKey.substring(0, maxPartitionedKeyLength - 1)
@@ -109,7 +122,7 @@ public class KinesisSink implements Sink {
ListenableFuture addRecordResult =
kinesisProducer.addUserRecord(this.streamName,
partitionedKey, data);
addCallback(addRecordResult,
-ProducerSendCallback.create(this.streamName, record,
System.nanoTime(), sinkContext), directExecutor());
+ProducerSendCallback.create(this, record, System.nanoTime()),
directExecutor());
if (sinkContext != null) {
sinkContext.recordMetric(METRICS_TOTAL_INCOMING, 1);
sinkContext.recordMetric(METRICS_TOTAL_INCOMING_BYTES,
data.array().length);
@@ -151,6 +164,7 @@ public class KinesisSink implements Sink {
this.streamName = kinesisSinkConfig.getAwsKinesisStreamName();
this.kinesisProducer = new KinesisProducer(kinesisConfig);
+IS_PUBLISH_FAILED.set(this, FALSE);
LOG.info("Kinesis sink started. {}",
(ReflectionToStringBuilder.toString(kinesisConfig,
ToStringStyle.SHORT_PREFIX_STYLE)));
}
@@ -167,30 +181,26 @@ public class KinesisSink implements Sink {
private static final class ProducerSendCallback implements
FutureCallback {
private Record resultContext;
-private String streamName;
private long startTime = 0;
private final Handle recyclerHandle;
-private SinkContext sinkContext;
+private KinesisSink kinesisSink;
private ProducerSendCallback(Handle
recyclerHandle) {
this.recyclerHandle = recyclerHandle;
}
-static ProducerSendCallback create(String streamName, Record
resultContext, long startTime,
-SinkContext sinkContext) {
+static ProducerSendCallback create(KinesisSink kinesisSink,
Record resultContext, long startTime) {
ProducerSendCallback sendCallback = RECYCLER.get();
sendCallback
[incubator-pulsar] branch branch-2.1 updated: Derive source/sink arg-class name from function-class for file-url (#2258)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 752f1b4 Derive source/sink arg-class name from function-class for
file-url (#2258)
752f1b4 is described below
commit 752f1b44359bc83f0f11ddd2f0cb18f4f70b712c
Author: Rajan Dhabalia
AuthorDate: Tue Jul 31 11:32:49 2018 -0700
Derive source/sink arg-class name from function-class for file-url (#2258)
* Derive source/sink arg-class name from functio-class for file-url archive
* fix set type-arg if src/sink arg-class is not set
* add unit-test
---
.../org/apache/pulsar/io/PulsarSinkE2ETest.java| 99 --
.../org/apache/pulsar/functions/utils/Utils.java | 15 ++--
.../functions/worker/rest/api/FunctionsImpl.java | 42 ++---
3 files changed, 113 insertions(+), 43 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
index 16f1a76..5398bc9 100644
--- a/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
+++ b/pulsar-broker/src/test/java/org/apache/pulsar/io/PulsarSinkE2ETest.java
@@ -21,6 +21,10 @@ package org.apache.pulsar.io;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static
org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.retryStrategically;
import static org.mockito.Mockito.spy;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.Assert.assertTrue;
import java.io.File;
import java.lang.reflect.Method;
@@ -62,7 +66,6 @@ import org.apache.pulsar.functions.proto.Function.SinkSpec;
import org.apache.pulsar.functions.proto.Function.SourceSpec;
import org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatus;
import
org.apache.pulsar.functions.proto.InstanceCommunication.FunctionStatusList;
-import org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData;
import
org.apache.pulsar.functions.proto.InstanceCommunication.MetricsData.DataDigest;
import org.apache.pulsar.functions.sink.PulsarSink;
import org.apache.pulsar.functions.utils.Reflections;
@@ -70,11 +73,9 @@ import org.apache.pulsar.functions.utils.Utils;
import org.apache.pulsar.functions.worker.FunctionRuntimeManager;
import org.apache.pulsar.functions.worker.WorkerConfig;
import org.apache.pulsar.functions.worker.WorkerService;
-import org.apache.pulsar.functions.worker.rest.WorkerServer;
import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
@@ -124,7 +125,7 @@ public class PulsarSinkE2ETest {
public Object[][] validRoleName() {
return new Object[][] { { Boolean.TRUE }, { Boolean.FALSE } };
}
-
+
@BeforeMethod
void setup(Method method) throws Exception {
@@ -147,7 +148,6 @@ public class PulsarSinkE2ETest {
config.setBrokerServicePortTls(brokerServiceTlsPort);
config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
-
Set providers = new HashSet<>();
providers.add(AuthenticationProviderTls.class.getName());
config.setAuthenticationEnabled(true);
@@ -156,7 +156,6 @@ public class PulsarSinkE2ETest {
config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
config.setTlsAllowInsecureConnection(true);
-
functionsWorkerService = createPulsarFunctionWorker(config);
urlTls = new URL(brokerServiceUrl);
@@ -190,12 +189,12 @@ public class PulsarSinkE2ETest {
workerConfig.getClientAuthenticationParameters());
}
pulsarClient = clientBuilder.build();
-
+
TenantInfo propAdmin = new TenantInfo();
propAdmin.getAdminRoles().add("superUser");
propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use")));
admin.tenants().updateTenant(tenant, propAdmin);
-
+
Thread.sleep(100);
}
@@ -237,7 +236,7 @@ public class PulsarSinkE2ETest {
workerConfig.setUseTls(true);
workerConfig.setTlsAllowInsecureConnection(true);
workerConfig.setTlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH);
-
+
workerConfig.setAuthenticationEnabled(true);
workerConfig.setAuthorizationEnabled(true);
@@ -285,7 +284,7 @@ public class PulsarSinkE2ETest {
}
}, 5, 150);
// validate pulsar sink consume
[incubator-pulsar] branch branch-2.1 updated: Fix: function-cli avoid arg-type check and class-loading for file url (#2257)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 6a05513 Fix: function-cli avoid arg-type check and class-loading for
file url (#2257)
6a05513 is described below
commit 6a05513f8cbcb5f9414af034a8f1e14754b94045
Author: Rajan Dhabalia
AuthorDate: Mon Jul 30 11:31:31 2018 -0700
Fix: function-cli avoid arg-type check and class-loading for file url
(#2257)
---
.../src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java | 9 -
.../java/org/apache/pulsar/functions/utils/FunctionConfig.java | 1 -
.../apache/pulsar/functions/utils/validation/ValidatorImpls.java | 4 +++-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
index 2a38c81..15aa086 100644
---
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
+++
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
@@ -545,7 +545,14 @@ public class CmdFunctions extends CmdBase {
Class[] typeArgs = null;
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
-typeArgs = Utils.getFunctionTypes(functionConfig);
+if (functionConfig.getJar().startsWith(Utils.FILE)) {
+// server derives the arg-type by loading a class
+if (isBlank(functionConfig.getClassName())) {
+throw new ParameterException("Class-name must be
present for jar with file-url");
+}
+} else {
+typeArgs = Utils.getFunctionTypes(functionConfig);
+}
}
FunctionDetails.Builder functionDetailsBuilder =
FunctionDetails.newBuilder();
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
index 3140f95..0cd922e 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/FunctionConfig.java
@@ -69,7 +69,6 @@ public class FunctionConfig {
@NotNull
private String name;
@NotNull
-@isImplementationOfClasses(implementsClasses = {Function.class,
java.util.function.Function.class})
private String className;
@isListEntryCustom(entryValidatorClasses =
{ValidatorImpls.TopicNameValidator.class})
private Collection inputs;
diff --git
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
index 297a2f2..925f7bc 100644
---
a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
+++
b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/validation/ValidatorImpls.java
@@ -502,7 +502,9 @@ public class ValidatorImpls {
FunctionConfig functionConfig = (FunctionConfig) o;
doCommonChecks(functionConfig);
if (functionConfig.getRuntime() == FunctionConfig.Runtime.JAVA) {
-doJavaChecks(functionConfig, name);
+if (!functionConfig.getJar().startsWith(Utils.FILE)) {
+doJavaChecks(functionConfig, name);
+}
} else {
doPythonChecks(functionConfig, name);
}
[incubator-pulsar] branch branch-2.1 updated: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 (#2182)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 406f8fa Issue 2110: get-retention returns retentionSizeInMB as 0 when
set to -1 (#2182)
406f8fa is described below
commit 406f8fa4de6d02839add4b50d31ef4d630b4c857
Author: Sijie Guo
AuthorDate: Wed Jul 25 23:02:25 2018 -0700
Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
(#2182)
* Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
### Motivation
Fixes #2110. infinite retention is not correctly propagated.
### Changes
Fix the command tool to set infinite retention correctly. Update command
line description and website to reflect the fact retention size less than 1MB
is treated as no retention.
---
.../org/apache/pulsar/admin/cli/CmdNamespaces.java | 10 --
site/_data/cli/pulsar-admin.yaml | 2 +-
.../pulsar/tests/integration/cli/CLITest.java | 38 ++
3 files changed, 46 insertions(+), 4 deletions(-)
diff --git
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
index 0eaf81b..4f898a5 100644
---
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
+++
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
@@ -342,7 +342,7 @@ public class CmdNamespaces extends CmdBase {
private String retentionTimeStr;
@Parameter(names = { "--size", "-s" }, description = "Retention size
limit (eg: 10M, 16G, 3T). "
-+ "0 means no retention and -1 means infinite size retention",
required = true)
++ "0 or less than 1MB means no retention and -1 means infinite
size retention", required = true)
private String limitStr;
@Override
@@ -351,8 +351,12 @@ public class CmdNamespaces extends CmdBase {
long sizeLimit = validateSizeString(limitStr);
int retentionTimeInMin = validateTimeString(retentionTimeStr);
-sizeLimit = sizeLimit / (1024 * 1024);
-int retentionSizeInMB = (int) sizeLimit;
+int retentionSizeInMB;
+if (sizeLimit != -1) {
+retentionSizeInMB = (int) (sizeLimit / (1024 * 1024));
+} else {
+retentionSizeInMB = -1;
+}
admin.namespaces().setRetention(namespace, new
RetentionPolicies(retentionTimeInMin, retentionSizeInMB));
}
}
diff --git a/site/_data/cli/pulsar-admin.yaml b/site/_data/cli/pulsar-admin.yaml
index c0d483f..94ec02c 100644
--- a/site/_data/cli/pulsar-admin.yaml
+++ b/site/_data/cli/pulsar-admin.yaml
@@ -431,7 +431,7 @@ commands:
argument: tenant/cluster/namespace
options:
- flags: -s, --size
- description: The retention size limits (for example `10M`, `16G` or
`3T`). 0 means no retention and -1 means infinite size retention
+ description: The retention size limits (for example `10M`, `16G` or
`3T`). 0 or less than 1MB means no retention and -1 means infinite size
retention
- flags: -t, --time
description: "The retention time in minutes, hours, days, or weeks.
Examples: `100m`, `13h`, `2d`, `5w`. 0 means no retention and -1 means infinite
time retention"
- name: unload
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java
index 8d9ed20..0f2163a 100644
---
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/cli/CLITest.java
@@ -164,4 +164,42 @@ public class CLITest extends PulsarClusterTestBase {
);
assertTrue(result.getStderr().contains("Reason: HTTP 404 Not Found"));
}
+
+@Test
+public void testSetInfiniteRetention() throws Exception {
+ContainerExecResult result;
+
+String namespace = "get-and-set-retention" + randomName(8);
+pulsarCluster.createNamespace(namespace);
+
+String[] setCommand = {
+"namespaces", "set-retention", "public/" + namespace,
+"--size", "-1",
+"--time", "-1"
+};
+
+result = pulsarCluster.runAdminCommandOnAnyBroker(setCommand);
+assertEquals(0, result.getExitCode());
+assertTrue(
+result.getStdout().isEmpty(),
+result.getStdout()
[incubator-pulsar] branch branch-2.1 updated: add function-name reference with function failure log (#2246)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new cab977e add function-name reference with function failure log (#2246)
cab977e is described below
commit cab977e785d9db6e8a96f304585b8858f70c97b0
Author: Rajan Dhabalia
AuthorDate: Thu Jul 26 23:36:06 2018 -0700
add function-name reference with function failure log (#2246)
---
.../main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
index 8329ed7..6abb5fb 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/RuntimeSpawner.java
@@ -82,7 +82,9 @@ public class RuntimeSpawner implements AutoCloseable {
@Override
public void run() {
if (!runtime.isAlive()) {
-log.error("Function Container is dead with exception",
runtime.getDeathException());
+log.error("[{}-{}] Function Container is dead with
exception",
+instanceConfig.getFunctionDetails().getName(),
instanceConfig.getInstanceId(),
+runtime.getDeathException());
log.error("Restarting...");
// Just for the sake of sanity, just destroy the
runtime
runtime.stop();
[incubator-pulsar] branch branch-2.1 updated: Set empty response to complete grpc txn (#2225)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b335e94 Set empty response to complete grpc txn (#2225)
b335e94 is described below
commit b335e94b8541c4b1e4536aa12d700ea06082dc5f
Author: Rajan Dhabalia
AuthorDate: Wed Jul 25 13:07:52 2018 -0700
Set empty response to complete grpc txn (#2225)
---
.../java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
index 0c10fdb..966740d 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/JavaInstanceMain.java
@@ -370,9 +370,10 @@ public class JavaInstanceMain implements AutoCloseable {
if (runtime != null) {
try {
runtime.resetMetrics().get();
+
responseObserver.onNext(com.google.protobuf.Empty.getDefaultInstance());
responseObserver.onCompleted();
} catch (InterruptedException | ExecutionException e) {
-log.error("Exception in JavaInstance doing
getAndResetMetrics", e);
+log.error("Exception in JavaInstance doing resetMetrics",
e);
throw new RuntimeException(e);
}
}
[incubator-pulsar] branch branch-2.1 updated: Fix implementation of equals() for MessageId (#2226)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 3eaa79b Fix implementation of equals() for MessageId (#2226)
3eaa79b is described below
commit 3eaa79beb20a759d9dba49a3dd75a5257c29d2e1
Author: massakam
AuthorDate: Wed Jul 25 19:53:08 2018 +0900
Fix implementation of equals() for MessageId (#2226)
---
.../pulsar/client/impl/BatchMessageIdImpl.java | 4 +++-
.../apache/pulsar/client/impl/MessageIdImpl.java | 8
.../pulsar/client/impl/BatchMessageIdImplTest.java | 24 ++
3 files changed, 31 insertions(+), 5 deletions(-)
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
index 78109c4..d66e242 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/BatchMessageIdImpl.java
@@ -92,7 +92,9 @@ public class BatchMessageIdImpl extends MessageIdImpl {
return ledgerId == other.ledgerId && entryId == other.entryId &&
partitionIndex == other.partitionIndex
&& batchIndex == other.batchIndex;
} else if (obj instanceof MessageIdImpl) {
-return batchIndex == NO_BATCH && obj.equals(this);
+MessageIdImpl other = (MessageIdImpl) obj;
+return ledgerId == other.ledgerId && entryId == other.entryId &&
partitionIndex == other.partitionIndex
+&& batchIndex == NO_BATCH;
}
return false;
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
index 41cc9c9..3686b12 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/MessageIdImpl.java
@@ -70,12 +70,12 @@ public class MessageIdImpl implements MessageId {
@Override
public boolean equals(Object obj) {
-if (obj instanceof MessageIdImpl) {
-MessageIdImpl other = (MessageIdImpl) obj;
-return ledgerId == other.ledgerId && entryId == other.entryId &&
partitionIndex == other.partitionIndex;
-} else if (obj instanceof BatchMessageIdImpl){
+if (obj instanceof BatchMessageIdImpl) {
BatchMessageIdImpl other = (BatchMessageIdImpl) obj;
return other.equals(this);
+} else if (obj instanceof MessageIdImpl) {
+MessageIdImpl other = (MessageIdImpl) obj;
+return ledgerId == other.ledgerId && entryId == other.entryId &&
partitionIndex == other.partitionIndex;
}
return false;
}
diff --git
a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
index a3b5f72..af21fcf 100644
---
a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
+++
b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/BatchMessageIdImplTest.java
@@ -44,4 +44,28 @@ public class BatchMessageIdImplTest {
Assert.assertTrue(batchMsgId1.hashCode() != batchMsgId2.hashCode());
}
+
+@Test
+public void equalsTest() {
+BatchMessageIdImpl batchMsgId1 = new BatchMessageIdImpl(0, 0, 0, 0);
+BatchMessageIdImpl batchMsgId2 = new BatchMessageIdImpl(1, 1, 1, 1);
+BatchMessageIdImpl batchMsgId3 = new BatchMessageIdImpl(0, 0, 0, 1);
+BatchMessageIdImpl batchMsgId4 = new BatchMessageIdImpl(0, 0, 0, -1);
+MessageIdImpl msgId = new MessageIdImpl(0, 0, 0);
+
+Assert.assertTrue(batchMsgId1.equals(batchMsgId1));
+Assert.assertFalse(batchMsgId1.equals(batchMsgId2));
+Assert.assertFalse(batchMsgId1.equals(batchMsgId3));
+Assert.assertFalse(batchMsgId1.equals(batchMsgId4));
+Assert.assertFalse(batchMsgId1.equals(msgId));
+
+Assert.assertTrue(msgId.equals(msgId));
+Assert.assertFalse(msgId.equals(batchMsgId1));
+Assert.assertFalse(msgId.equals(batchMsgId2));
+Assert.assertFalse(msgId.equals(batchMsgId3));
+Assert.assertTrue(msgId.equals(batchMsgId4));
+
+Assert.assertTrue(batchMsgId4.equals(msgId));
+}
+
}
[incubator-pulsar] branch branch-2.1 updated: Fixed ack message tracker cleanup for single message batches (#2224)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 638eb55 Fixed ack message tracker cleanup for single message batches
(#2224)
638eb55 is described below
commit 638eb55970f2d30eb09282731f0402903b5ee469
Author: Matteo Merli
AuthorDate: Wed Jul 25 02:00:16 2018 -0700
Fixed ack message tracker cleanup for single message batches (#2224)
### Motivation
Fixes #2221
The issue is that we were removing a batch message id impl from the
tracking set which contains the batch-wide ids.
---
.../impl/UnAcknowledgedMessagesTimeoutTest.java| 32 ++
.../apache/pulsar/client/impl/ConsumerImpl.java| 28 +--
2 files changed, 46 insertions(+), 14 deletions(-)
diff --git
a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
index 0b5126b..da53760 100644
---
a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
+++
b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/UnAcknowledgedMessagesTimeoutTest.java
@@ -19,6 +19,8 @@
package org.apache.pulsar.client.impl;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
import java.util.HashSet;
import java.util.concurrent.TimeUnit;
@@ -30,6 +32,7 @@ import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.MessageRoutingMode;
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClientException;
+import org.apache.pulsar.client.api.Schema;
import org.apache.pulsar.client.api.SubscriptionType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -386,4 +389,33 @@ public class UnAcknowledgedMessagesTimeoutTest extends
BrokerTestBase {
assertEquals(consumer.getUnAckedMessageTracker().size(), 0);
}
+
+@Test
+public void testSingleMessageBatch() throws Exception {
+String topicName = "prop/ns-abc/topic-estSingleMessageBatch";
+
+Producer producer = pulsarClient.newProducer(Schema.STRING)
+.topic(topicName)
+.enableBatching(true)
+.batchingMaxPublishDelay(10, TimeUnit.SECONDS)
+.create();
+
+Consumer consumer = pulsarClient.newConsumer(Schema.STRING)
+.topic(topicName)
+.subscriptionName("subscription")
+.ackTimeout(1, TimeUnit.HOURS)
+.subscribe();
+
+// Force the creation of a batch with a single message
+producer.sendAsync("hello");
+producer.flush();
+
+Message message = consumer.receive();
+
+assertFalse(((ConsumerImpl)
consumer).getUnAckedMessageTracker().isEmpty());
+
+consumer.acknowledge(message);
+
+assertTrue(((ConsumerImpl)
consumer).getUnAckedMessageTracker().isEmpty());
+}
}
diff --git
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
index 6f78119..1c69c75 100644
---
a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
+++
b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ConsumerImpl.java
@@ -385,11 +385,6 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
log.debug("[{}] [{}] can ack message to broker {}, acktype {},
cardinality {}, length {}", subscription,
consumerName, batchMessageId, ackType,
outstandingAcks, batchSize);
}
-// increment Acknowledge-msg counter with number of messages in
batch only if AckType is Individual.
-// CumulativeAckType is handled while sending ack to broker
-if (ackType == AckType.Individual) {
-stats.incrementNumAcksSent(batchSize);
-}
return true;
} else {
if (AckType.Cumulative == ackType
@@ -433,11 +428,16 @@ public class ConsumerImpl extends ConsumerBase
implements ConnectionHandle
Map
properties) {
MessageIdImpl msgId = (MessageIdImpl) messageId;
-
if (ackType == AckType.Individual) {
-unAckedMessageTracker.remove(msgId);
-// increment counter by 1 for non-batch msg
-if (!(messageId instanceof BatchMessageIdImpl)) {
+if (messageId instanceof BatchMessageIdImpl) {
+BatchMessageIdImpl batchMessageId = (BatchMessageIdImpl)
messageId;
+
+
[incubator-pulsar] branch branch-2.1 updated: C++ should default not allow insecure connections with TLS (#2229)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 227e1df C++ should default not allow insecure connections with TLS
(#2229)
227e1df is described below
commit 227e1dfd9dc3b62cdb13780064ff1751f9c34542
Author: Ivan Kelly
AuthorDate: Wed Jul 25 20:40:34 2018 +0100
C++ should default not allow insecure connections with TLS (#2229)
If tlsAllowInsecureConnection is true, then the client doesn't
validate the server's TLS cert, allowing for MITM attacks. This
shouldn't be the default option.
---
pulsar-client-cpp/lib/ClientConfigurationImpl.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pulsar-client-cpp/lib/ClientConfigurationImpl.h
b/pulsar-client-cpp/lib/ClientConfigurationImpl.h
index 7160a65..cb23a14 100644
--- a/pulsar-client-cpp/lib/ClientConfigurationImpl.h
+++ b/pulsar-client-cpp/lib/ClientConfigurationImpl.h
@@ -44,7 +44,7 @@ struct ClientConfigurationImpl {
concurrentLookupRequest(5),
logConfFilePath(),
useTls(false),
- tlsAllowInsecureConnection(true),
+ tlsAllowInsecureConnection(false),
statsIntervalInSeconds(600), // 10 minutes
loggerFactory() {}
};
[incubator-pulsar] branch branch-2.1 updated: fix bug in FunctionRuntimeManager involving not cleaning up old invalid assignments (#2223)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new c300e3e fix bug in FunctionRuntimeManager involving not cleaning up
old invalid assignments (#2223)
c300e3e is described below
commit c300e3e5f2f6118619e7d06d5db9629bbf29ad3f
Author: Boyang Jerry Peng
AuthorDate: Wed Jul 25 11:37:48 2018 -0700
fix bug in FunctionRuntimeManager involving not cleaning up old invalid
assignments (#2223)
### Motivation
Old invalid assignments of functions are not cleaned up and left over after
a function is rescheduled.
Getting the status of functions rely on assignments to route the get status
to the correct worker running the function. If the assignments are not
correct, get status requests will be routed incorrectly.
Also fixed a test that was asserting the incorrect value. The test would
have caught this problem if it was asserting the correct value.
---
.../pulsar/functions/worker/FunctionRuntimeManager.java | 12 ++--
.../pulsar/functions/worker/FunctionRuntimeManagerTest.java | 4 +---
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
index 05a79d8..5e1995e 100644
---
a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
+++
b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/FunctionRuntimeManager.java
@@ -82,7 +82,6 @@ public class FunctionRuntimeManager implements AutoCloseable{
private MembershipManager membershipManager;
private final ConnectorsManager connectorsManager;
-
public FunctionRuntimeManager(WorkerConfig workerConfig,
PulsarClient pulsarClient,
Namespace dlogNamespace,
@@ -354,6 +353,7 @@ public class FunctionRuntimeManager implements
AutoCloseable{
* @param assignmentsUpdate the assignment update
*/
public synchronized void processAssignmentUpdate(MessageId messageId,
AssignmentsUpdate assignmentsUpdate) {
+
if (assignmentsUpdate.getVersion() > this.currentAssignmentVersion) {
Map assignmentMap = new HashMap<>();
@@ -430,8 +430,16 @@ public class FunctionRuntimeManager implements
AutoCloseable{
newFunctionRuntimeInfo.setFunctionInstance(assignment.getInstance());
this.insertStartAction(newFunctionRuntimeInfo);
this.setFunctionRuntimeInfo(fullyQualifiedInstanceId,
newFunctionRuntimeInfo);
-this.setAssignment(assignment);
}
+
+// find existing assignment
+Assignment existing_assignment =
this.findAssignment(assignment);
+if (existing_assignment != null) {
+// delete old assignment that could have old data
+this.deleteAssignment(existing_assignment);
+}
+// set to newest assignment
+this.setAssignment(assignment);
}
}
diff --git
a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
index 8ab7473..4f618c4 100644
---
a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
+++
b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/FunctionRuntimeManagerTest.java
@@ -378,12 +378,10 @@ public class FunctionRuntimeManagerTest {
.build();
Assert.assertEquals(functionRuntimeManager.functionRuntimeInfoMap.size(), 2);
-
-
Assert.assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 2);
+
Assert.assertEquals(functionRuntimeManager.workerIdToAssignments.size(), 1);
Assert.assertEquals(functionRuntimeManager.workerIdToAssignments
.get("worker-1").get("test-tenant/test-namespace/func-1:0"),
assignment1);
Assert.assertEquals(functionRuntimeManager.workerIdToAssignments
.get("worker-1").get("test-tenant/test-namespace/func-2:0"),
assignment3);
}
-
}
[incubator-pulsar] branch branch-2.1 updated: set worker port to be broker service port when running worker with broker (#2218)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b88d087 set worker port to be broker service port when running worker
with broker (#2218)
b88d087 is described below
commit b88d087a87b4d26f901408b31a41e77597ead41e
Author: Boyang Jerry Peng
AuthorDate: Tue Jul 24 01:54:07 2018 -0700
set worker port to be broker service port when running worker with broker
(#2218)
The port for the function worker should be same as the broker service port
when the worker service is run as part of the broker
---
pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java | 1 +
pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java| 1 +
2 files changed, 2 insertions(+)
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
index 24e3fda..42e5f6e 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarBrokerStarter.java
@@ -156,6 +156,7 @@ public class PulsarBrokerStarter {
String hostname =
ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
brokerConfig.getAdvertisedAddress());
workerConfig.setWorkerHostname(hostname);
+workerConfig.setWorkerPort(brokerConfig.getWebServicePort());
workerConfig.setWorkerId(
"c-" + brokerConfig.getClusterName()
+ "-fw-" + hostname
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
index 28e839b..3e44f76 100644
--- a/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
+++ b/pulsar-broker/src/main/java/org/apache/pulsar/PulsarStandalone.java
@@ -276,6 +276,7 @@ public class PulsarStandalone implements AutoCloseable {
String hostname =
ServiceConfigurationUtils.getDefaultOrConfiguredAddress(
config.getAdvertisedAddress());
workerConfig.setWorkerHostname(hostname);
+workerConfig.setWorkerPort(config.getWebServicePort());
workerConfig.setWorkerId(
"c-" + config.getClusterName()
+ "-fw-" + hostname
[incubator-pulsar] branch branch-2.1 updated: Cpp client: add multiTopicsConsumer (#1996)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 500e393 Cpp client: add multiTopicsConsumer (#1996)
500e393 is described below
commit 500e3936d520b1854a096e4ef71be7cb7edfc502
Author: Jia Zhai
AuthorDate: Tue Jul 24 05:25:45 2018 +0800
Cpp client: add multiTopicsConsumer (#1996)
In PR #1103, we add Multi-Topics-Consumer in java client. This is a catch
up work to add it in cpp client.
---
pulsar-client-cpp/include/pulsar/Client.h | 9 +
pulsar-client-cpp/include/pulsar/Consumer.h| 1 +
pulsar-client-cpp/include/pulsar/Message.h | 1 +
pulsar-client-cpp/include/pulsar/MessageId.h | 12 +
pulsar-client-cpp/lib/Client.cc| 24 +
pulsar-client-cpp/lib/ClientImpl.cc| 36 ++
pulsar-client-cpp/lib/ClientImpl.h | 3 +
pulsar-client-cpp/lib/MessageId.cc | 4 +
pulsar-client-cpp/lib/MessageIdImpl.h | 17 +-
pulsar-client-cpp/lib/MessageImpl.cc | 10 +-
pulsar-client-cpp/lib/MessageImpl.h| 11 +
.../lib/MultiTopicsBrokerConsumerStatsImpl.cc | 158 +
.../lib/MultiTopicsBrokerConsumerStatsImpl.h | 92 +++
pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc | 644 +
pulsar-client-cpp/lib/MultiTopicsConsumerImpl.h| 133 +
pulsar-client-cpp/lib/NamespaceName.cc | 5 +
pulsar-client-cpp/lib/NamespaceName.h | 5 +
pulsar-client-cpp/lib/TopicName.cc | 3 +
pulsar-client-cpp/lib/TopicName.h | 1 +
.../lib/UnAckedMessageTrackerDisabled.h| 1 +
.../lib/UnAckedMessageTrackerEnabled.cc| 20 +
.../lib/UnAckedMessageTrackerEnabled.h | 1 +
.../lib/UnAckedMessageTrackerInterface.h | 3 +
pulsar-client-cpp/tests/BasicEndToEndTest.cc | 173 +-
pulsar-client-cpp/tests/NamespaceNameTest.cc | 1 +
25 files changed, 1364 insertions(+), 4 deletions(-)
diff --git a/pulsar-client-cpp/include/pulsar/Client.h
b/pulsar-client-cpp/include/pulsar/Client.h
index 07b4355..6a9e487 100644
--- a/pulsar-client-cpp/include/pulsar/Client.h
+++ b/pulsar-client-cpp/include/pulsar/Client.h
@@ -99,6 +99,15 @@ class Client {
void subscribeAsync(const std::string& topic, const std::string&
consumerName,
const ConsumerConfiguration& conf, SubscribeCallback
callback);
+Result subscribe(const std::vector& topics, const
std::string& subscriptionName,
+ Consumer& consumer);
+Result subscribe(const std::vector& topics, const
std::string& subscriptionName,
+ const ConsumerConfiguration& conf, Consumer& consumer);
+void subscribeAsync(const std::vector& topics, const
std::string& subscriptionName,
+SubscribeCallback callback);
+void subscribeAsync(const std::vector& topics, const
std::string& subscriptionName,
+const ConsumerConfiguration& conf, SubscribeCallback
callback);
+
/**
* Create a topic reader with given {@code ReaderConfiguration} for
reading messages from the specified
* topic.
diff --git a/pulsar-client-cpp/include/pulsar/Consumer.h
b/pulsar-client-cpp/include/pulsar/Consumer.h
index 4272166..4486515 100644
--- a/pulsar-client-cpp/include/pulsar/Consumer.h
+++ b/pulsar-client-cpp/include/pulsar/Consumer.h
@@ -243,6 +243,7 @@ class Consumer {
friend class PulsarFriend;
friend class PulsarWrapper;
friend class PartitionedConsumerImpl;
+friend class MultiTopicsConsumerImpl;
friend class ConsumerImpl;
friend class ClientImpl;
friend class ConsumerTest;
diff --git a/pulsar-client-cpp/include/pulsar/Message.h
b/pulsar-client-cpp/include/pulsar/Message.h
index aff0d94..a3b9af0 100644
--- a/pulsar-client-cpp/include/pulsar/Message.h
+++ b/pulsar-client-cpp/include/pulsar/Message.h
@@ -134,6 +134,7 @@ class Message {
proto::SingleMessageMetadata& singleMetadata);
friend class PartitionedProducerImpl;
friend class PartitionedConsumerImpl;
+friend class MultiTopicsConsumerImpl;
friend class MessageBuilder;
friend class ConsumerImpl;
friend class ProducerImpl;
diff --git a/pulsar-client-cpp/include/pulsar/MessageId.h
b/pulsar-client-cpp/include/pulsar/MessageId.h
index dfe3a51..e9ff133 100644
--- a/pulsar-client-cpp/include/pulsar/MessageId.h
+++ b/pulsar-client-cpp/include/pulsar/MessageId.h
@@ -22,6 +22,7 @@
#include
#include
#include
+//#include
#pragma GCC visibility push(default)
@@ -51,6 +52,16 @@ class MessageId {
void serialize(std::string& result) const;
/**
+
