[GitHub] jerrypeng commented on issue #2181: augmenting protoschema with info for parsing
jerrypeng commented on issue #2181: augmenting protoschema with info for parsing URL: https://github.com/apache/incubator-pulsar/pull/2181#issuecomment-405816645 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat opened a new pull request #2191: Removed shading relocations for Circe-checksum and lz4 libraries
merlimat opened a new pull request #2191: Removed shading relocations for Circe-checksum and lz4 libraries URL: https://github.com/apache/incubator-pulsar/pull/2191 ### Motivation Circe-checksum and lz4 both use JNI code embedded in the Jar as the best option to do compression and checksums. If the classes are shaded, the JNI package names will be different and therefore we would be falling back on the slower Java based implementations. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on a change in pull request #1996: Cpp client: add multiTopicsConsumer
zhaijack commented on a change in pull request #1996: Cpp client: add
multiTopicsConsumer
URL: https://github.com/apache/incubator-pulsar/pull/1996#discussion_r203255677
##
File path: pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc
##
@@ -0,0 +1,621 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include "MultiTopicsConsumerImpl.h"
+
+DECLARE_LOG_OBJECT()
+
+using namespace pulsar;
+
+MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const
std::vector& topics,
+ const std::string&
subscriptionName, TopicNamePtr topicName,
+ const ConsumerConfiguration&
conf,
+ const LookupServicePtr
lookupServicePtr)
+: client_(client),
+ subscriptionName_(subscriptionName),
+ topic_(topicName ? topicName->toString() : "EmptyTopics"),
+ conf_(conf),
+ state_(Pending),
+ messages_(1000),
+ listenerExecutor_(client->getListenerExecutorProvider()->get()),
+ messageListener_(conf.getMessageListener()),
+ namespaceName_(topicName ? topicName->getNamespaceName() :
boost::shared_ptr()),
+ lookupServicePtr_(lookupServicePtr),
+ allTopicPartitionsNumber_(boost::make_shared>(0)),
+ topics_(topics) {
+std::stringstream consumerStrStream;
+consumerStrStream << "[Muti Topics Consumer: "
+ << "TopicName - " << topic_ << " - Subscription - " <<
subscriptionName << "]";
+consumerStr_ = consumerStrStream.str();
+
+if (conf.getUnAckedMessagesTimeoutMs() != 0) {
+unAckedMessageTrackerPtr_.reset(
+new
UnAckedMessageTrackerEnabled(conf.getUnAckedMessagesTimeoutMs(), client,
*this));
+} else {
+unAckedMessageTrackerPtr_.reset(new UnAckedMessageTrackerDisabled());
+}
+}
+
+void MultiTopicsConsumerImpl::start() {
+if (topics_.empty()) {
+setState(Ready);
+LOG_DEBUG("No topics passed in when create MultiTopicsConsumer.");
+multiTopicsConsumerCreatedPromise_.setValue(shared_from_this());
+return;
+}
+
+// start call subscribeOneTopicAsync for each single topic
+int topicsNumber = topics_.size();
+boost::shared_ptr> topicsNeedCreate =
boost::make_shared>(topicsNumber);
+// subscribe for each passed in topic
+for (std::vector::const_iterator itr = topics_.begin(); itr
!= topics_.end(); itr++) {
+subscribeOneTopicAsync(*itr).addListener(
+boost::bind(::handleOneTopicSubscribed,
shared_from_this(), _1, _2, *itr,
+topicsNeedCreate));
+}
+}
+
+void MultiTopicsConsumerImpl::handleOneTopicSubscribed(Result result, Consumer
consumer,
+ const std::string&
topic,
+
boost::shared_ptr> topicsNeedCreate) {
+int previous = topicsNeedCreate->fetch_sub(1);
+assert(previous > 0);
+
+if (result != ResultOk) {
+state_ = Failed;
+multiTopicsConsumerCreatedPromise_.setFailed(result);
+// unsubscribed all of the successfully subscribed partitioned
consumers
+ResultCallback nullCallbackForCleanup = NULL;
+closeAsync(nullCallbackForCleanup);
+LOG_ERROR("Unable to create Consumer - "
+ << " Error - " << result);
+return;
+}
+
+LOG_DEBUG("Successfully Subscribed to topic " << topic << " in
TopicsConsumer ");
+
+if (topicsNeedCreate->load() == 0) {
+LOG_INFO("Successfully Subscribed to Topics");
+setState(Ready);
+if (!namespaceName_) {
+namespaceName_ = TopicName::get(topic)->getNamespaceName();
+}
+multiTopicsConsumerCreatedPromise_.setValue(shared_from_this());
+return;
+}
+}
+
+// subscribe for passed in topic
+Future MultiTopicsConsumerImpl::subscribeOneTopicAsync(const
std::string& topic) {
+TopicNamePtr topicName;
+ConsumerSubResultPromisePtr topicPromise =
boost::make_shared>();
+if (!(topicName = TopicName::get(topic))) {
+LOG_ERROR("TopicName invalid: " << topic);
+
[GitHub] grantwwu commented on issue #2174: V2 doc changes
grantwwu commented on issue #2174: V2 doc changes URL: https://github.com/apache/incubator-pulsar/pull/2174#issuecomment-405812906 @merlimat uh, did you see my question in the comments? Asking whether I should remove the whole command or just the deprecated REST endpoint. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
merlimat commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203244621
##
File path: pulsar-io/kinesis/pom.xml
##
@@ -38,6 +38,13 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-functions-instance
+ ${project.version}
+ provided
Review comment:
Look at
https://github.com/apache/incubator-pulsar/pull/2187/files#diff-5df4e50a01720cb0f526ef575cc4d66fR71
for how it got changed for the kinesis sink to access the encryption context.
More than lightweight, I think it's preferable to separate the API
dependencies. Also, that interface really only applies to PulsarRecord and not
any other implementations. For the dependency, yes it could be introduced a
functions-common, but depending on pulsar-function-instance doesn't add any
"real" dependency, since that module is by definition always in the class path,
in every execution mode.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] merlimat closed pull request #2186: Issue 1288: Provide documentation for running BookKeeper auto-recovery
merlimat closed pull request #2186: Issue 1288: Provide documentation for
running BookKeeper auto-recovery
URL: https://github.com/apache/incubator-pulsar/pull/2186
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/deployment/terraform-ansible/templates/bookkeeper.conf
b/deployment/terraform-ansible/templates/bookkeeper.conf
index 942dcffbbe..2f5dd4ae59 100644
--- a/deployment/terraform-ansible/templates/bookkeeper.conf
+++ b/deployment/terraform-ansible/templates/bookkeeper.conf
@@ -232,7 +232,7 @@ rereplicationEntryBatchSize=5000
# openLedgerRereplicationGracePeriod=30
# Whether the bookie itself can start auto-recovery service also or not
-# autoRecoveryDaemonEnabled=false
+autoRecoveryDaemonEnabled=true
# How long to wait, in seconds, before starting auto recovery of a lost bookie
# lostBookieRecoveryDelay=0
diff --git a/site/docs/latest/deployment/cluster.md
b/site/docs/latest/deployment/cluster.md
index d23423ce71..0d6ddeeba7 100644
--- a/site/docs/latest/deployment/cluster.md
+++ b/site/docs/latest/deployment/cluster.md
@@ -195,6 +195,10 @@ $ bin/bookkeeper shell bookiesanity
This will create an ephemeral BookKeeper {% popover ledger %} on the local
bookie, write a few entries, read them back, and finally delete the ledger.
+Starting from Pulsar 2.1 release, Bookies start
[AutoRecovery](http://bookkeeper.apache.org/docs/latest/admin/autorecovery/)
daemons by default. Those AutoRecovery daemons will monitor the healthy of the
bookkeeper cluster and automatically re-replicate entries for those
under-replicated ledgers.
+Since `AutoRecovery` deamons are stateless processes, if you would like to run
`AutoRecovery` as a separate service, you can also disable `AutoRecovery` on
bookies by setting `autoRecoveryDaemonEnabled` to `false` in your
`bookkeeper.conf` file. And you can follow the instructions in
+[bookkeeper
documentation](http://bookkeeper.apache.org/docs/latest/admin/autorecovery/) to
run `AutoRecovery` as separate processes.
+
## Deploying Pulsar brokers
Pulsar {% popover brokers %} are the last thing you need to deploy in your
Pulsar cluster. Brokers handle Pulsar messages and provide Pulsar's
administrative interface. We recommend running **3 brokers**, one for each
machine that's already running a BookKeeper bookie.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[incubator-pulsar] branch master updated: Issue 1288: Provide documentation for running BookKeeper auto-recovery (#2186)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new fd7b32c Issue 1288: Provide documentation for running BookKeeper
auto-recovery (#2186)
fd7b32c is described below
commit fd7b32c6ed4d8fd814c90718c143fcb7818c11e8
Author: Sijie Guo
AuthorDate: Tue Jul 17 20:47:27 2018 -0700
Issue 1288: Provide documentation for running BookKeeper auto-recovery
(#2186)
* Issue 1288: Provide documentation for running BookKeeper auto-recovery
### Motivation
Fixes #1288. We mentioned auto-recovery in DCOS deployment. but we don't
have any instructions to run auto-recovery in bare-mental deployment.
### Changes
Update the documentation in bare-mental deployment.
* Enable autorecovery for ansible
---
deployment/terraform-ansible/templates/bookkeeper.conf | 2 +-
site/docs/latest/deployment/cluster.md | 4
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/deployment/terraform-ansible/templates/bookkeeper.conf
b/deployment/terraform-ansible/templates/bookkeeper.conf
index 942dcff..2f5dd4a 100644
--- a/deployment/terraform-ansible/templates/bookkeeper.conf
+++ b/deployment/terraform-ansible/templates/bookkeeper.conf
@@ -232,7 +232,7 @@ rereplicationEntryBatchSize=5000
# openLedgerRereplicationGracePeriod=30
# Whether the bookie itself can start auto-recovery service also or not
-# autoRecoveryDaemonEnabled=false
+autoRecoveryDaemonEnabled=true
# How long to wait, in seconds, before starting auto recovery of a lost bookie
# lostBookieRecoveryDelay=0
diff --git a/site/docs/latest/deployment/cluster.md
b/site/docs/latest/deployment/cluster.md
index d23423c..0d6ddee 100644
--- a/site/docs/latest/deployment/cluster.md
+++ b/site/docs/latest/deployment/cluster.md
@@ -195,6 +195,10 @@ $ bin/bookkeeper shell bookiesanity
This will create an ephemeral BookKeeper {% popover ledger %} on the local
bookie, write a few entries, read them back, and finally delete the ledger.
+Starting from Pulsar 2.1 release, Bookies start
[AutoRecovery](http://bookkeeper.apache.org/docs/latest/admin/autorecovery/)
daemons by default. Those AutoRecovery daemons will monitor the healthy of the
bookkeeper cluster and automatically re-replicate entries for those
under-replicated ledgers.
+Since `AutoRecovery` deamons are stateless processes, if you would like to run
`AutoRecovery` as a separate service, you can also disable `AutoRecovery` on
bookies by setting `autoRecoveryDaemonEnabled` to `false` in your
`bookkeeper.conf` file. And you can follow the instructions in
+[bookkeeper
documentation](http://bookkeeper.apache.org/docs/latest/admin/autorecovery/) to
run `AutoRecovery` as separate processes.
+
## Deploying Pulsar brokers
Pulsar {% popover brokers %} are the last thing you need to deploy in your
Pulsar cluster. Brokers handle Pulsar messages and provide Pulsar's
administrative interface. We recommend running **3 brokers**, one for each
machine that's already running a BookKeeper bookie.
[GitHub] merlimat closed issue #1288: Provide documentation for running BookKeeper auto-recovery
merlimat closed issue #1288: Provide documentation for running BookKeeper auto-recovery URL: https://github.com/apache/incubator-pulsar/issues/1288 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on a change in pull request #1996: Cpp client: add multiTopicsConsumer
zhaijack commented on a change in pull request #1996: Cpp client: add multiTopicsConsumer URL: https://github.com/apache/incubator-pulsar/pull/1996#discussion_r203242596 ## File path: pulsar-client-cpp/lib/MultiTopicsConsumerImpl.cc ## @@ -0,0 +1,621 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include "MultiTopicsConsumerImpl.h" + +DECLARE_LOG_OBJECT() + +using namespace pulsar; + +MultiTopicsConsumerImpl::MultiTopicsConsumerImpl(ClientImplPtr client, const std::vector& topics, + const std::string& subscriptionName, TopicNamePtr topicName, + const ConsumerConfiguration& conf, + const LookupServicePtr lookupServicePtr) +: client_(client), + subscriptionName_(subscriptionName), + topic_(topicName ? topicName->toString() : "EmptyTopics"), + conf_(conf), + state_(Pending), + messages_(1000), + listenerExecutor_(client->getListenerExecutorProvider()->get()), + messageListener_(conf.getMessageListener()), + namespaceName_(topicName ? topicName->getNamespaceName() : boost::shared_ptr()), Review comment: Yes, only from single namespace. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on a change in pull request #1996: Cpp client: add multiTopicsConsumer
zhaijack commented on a change in pull request #1996: Cpp client: add
multiTopicsConsumer
URL: https://github.com/apache/incubator-pulsar/pull/1996#discussion_r203238921
##
File path: pulsar-client-cpp/lib/MessageImpl.h
##
@@ -43,13 +43,24 @@ class MessageImpl {
SharedBuffer payload;
MessageId messageId;
ClientConnection* cnx_;
+std::string topicName_;
const std::string& getPartitionKey() const;
bool hasPartitionKey() const;
uint64_t getPublishTimestamp() const;
uint64_t getEventTimestamp() const;
+/**
+ * Only for MultiTopicsConsumer to get a valid topicName
Review comment:
Thanks, right, I found that request. cpp work tracked in issue #2190
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhaijack opened a new issue #2190: cpp client work to get the topic information from consumed message
zhaijack opened a new issue #2190: cpp client work to get the topic information from consumed message URL: https://github.com/apache/incubator-pulsar/issues/2190 The issue details is in issue #2007 and PR #2135, This is mainly track Cpp client side. When a user receives a message, allow them to get the topic from the message itself. This is mostly useful in the case where a single consumer is subscribed to multiple topics. The change also contains a minor cleanup of the constructors in MessageImpl. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on a change in pull request #1996: Cpp client: add multiTopicsConsumer
zhaijack commented on a change in pull request #1996: Cpp client: add
multiTopicsConsumer
URL: https://github.com/apache/incubator-pulsar/pull/1996#discussion_r203238921
##
File path: pulsar-client-cpp/lib/MessageImpl.h
##
@@ -43,13 +43,24 @@ class MessageImpl {
SharedBuffer payload;
MessageId messageId;
ClientConnection* cnx_;
+std::string topicName_;
const std::string& getPartitionKey() const;
bool hasPartitionKey() const;
uint64_t getPublishTimestamp() const;
uint64_t getEventTimestamp() const;
+/**
+ * Only for MultiTopicsConsumer to get a valid topicName
Review comment:
Thanks, right, I found that request, will do that support later
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on issue #2181: augmenting protoschema with info for parsing
sijie commented on issue #2181: augmenting protoschema with info for parsing URL: https://github.com/apache/incubator-pulsar/pull/2181#issuecomment-405775628 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie closed pull request #2183: Issue 2121: Improve logging around "Namespace not served by this instance"
sijie closed pull request #2183: Issue 2121: Improve logging around "Namespace
not served by this instance"
URL: https://github.com/apache/incubator-pulsar/pull/2183
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
index 5bdf8129a3..741afbfefb 100644
---
a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
+++
b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java
@@ -934,8 +934,8 @@ void checkTopicNsOwnership(final String topic) throws
RuntimeException {
}
if (!ownedByThisInstance) {
-String msg = String.format("Namespace not served by this instance.
Please redo the lookup. "
-+ "Request is denied: namespace=%s",
topicName.getNamespace());
+String msg = String.format("Namespace bundle for topic (%s) not
served by this instance. Please redo the lookup. "
++ "Request is denied: namespace=%s", topic,
topicName.getNamespace());
log.warn(msg);
throw new RuntimeException(new ServiceUnitNotReadyException(msg));
}
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[incubator-pulsar] branch master updated: Issue 2121: Improve logging around "Namespace not served by this instance" (#2183)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new f0024d2 Issue 2121: Improve logging around "Namespace not served by this instance" (#2183) f0024d2 is described below commit f0024d2d95b947f1e2ac09165133ed1fde146cb9 Author: Sijie Guo AuthorDate: Tue Jul 17 18:02:28 2018 -0700 Issue 2121: Improve logging around "Namespace not served by this instance" (#2183) ### Motivation Fixes #2121. Improve the logging around "namespace not served by this instance" to make things clearer. ### Changes Improve the logging to say "namespace bundle for topic not served by", rahter than "namespace not served by" --- .../src/main/java/org/apache/pulsar/broker/service/BrokerService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 5bdf812..741afbf 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -934,8 +934,8 @@ public class BrokerService implements Closeable, ZooKeeperCacheListener
[GitHub] sijie closed issue #2121: Improve logging around "Namespace not served by this instance"
sijie closed issue #2121: Improve logging around "Namespace not served by this instance" URL: https://github.com/apache/incubator-pulsar/issues/2121 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on a change in pull request #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
sijie commented on a change in pull request #2182: Issue 2110: get-retention
returns retentionSizeInMB as 0 when set to -1
URL: https://github.com/apache/incubator-pulsar/pull/2182#discussion_r203225243
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
##
@@ -342,7 +342,7 @@ void run() throws PulsarAdminException {
private String retentionTimeStr;
@Parameter(names = { "--size", "-s" }, description = "Retention size
limit (eg: 10M, 16G, 3T). "
-+ "0 means no retention and -1 means infinite size retention",
required = true)
++ "0 or less than 1MB means no retention and -1 means infinite
size retention", required = true)
Review comment:
because that is the current logic. the broker only takes MB as inputs. so if
you specify less than 1MB, it is 0. This change here is just to update the
documentation. we can discuss if we want to change this behavior, which will be
a bunch of BC considerations.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie opened a new pull request #2189: dumpContainerDirToTargetCompressed should copy files to a containerName named directory
sijie opened a new pull request #2189: dumpContainerDirToTargetCompressed should copy files to a containerName named directory URL: https://github.com/apache/incubator-pulsar/pull/2189 *Motivation* dumpContainerLogToTarget copies the files to a containerName named directory. However dumpContainerDirToTargetCompressed doesn't. *Changes* Change `dumpContainerDirToTargetCompressed` to copy the tarball to a containerName named directory. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] srkukarni commented on a change in pull request #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
srkukarni commented on a change in pull request #2182: Issue 2110:
get-retention returns retentionSizeInMB as 0 when set to -1
URL: https://github.com/apache/incubator-pulsar/pull/2182#discussion_r203217003
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdNamespaces.java
##
@@ -342,7 +342,7 @@ void run() throws PulsarAdminException {
private String retentionTimeStr;
@Parameter(names = { "--size", "-s" }, description = "Retention size
limit (eg: 10M, 16G, 3T). "
-+ "0 means no retention and -1 means infinite size retention",
required = true)
++ "0 or less than 1MB means no retention and -1 means infinite
size retention", required = true)
Review comment:
This is actually very confusing. Why can't i specify 1kb?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie opened a new pull request #2188: client configuration on authentication page is out of date
sijie opened a new pull request #2188: client configuration on authentication page is out of date URL: https://github.com/apache/incubator-pulsar/pull/2188 *Motivation* Client configuration for auth is out of date. *Changes* - configure client.conf using webServiceUrl instead of serviceUrl - change PulsarAdmin to use builder This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] rdhabalia commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
rdhabalia commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203210773
##
File path: pulsar-io/kinesis/pom.xml
##
@@ -38,6 +38,13 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-functions-instance
+ ${project.version}
+ provided
Review comment:
> I have moved EncryptionContext out of Record (main reason is that it's
really tied to Pulsar client API and not relevant to other sources and sinks).
But if Pulsar record is encrypted then this context anyway will be required
by any sink. So, I think `EncryptionContext` should be part of `Record`.??
> I have added a new interface RecordWithEncryptionContext in
pulsar-functions-instance,
yes, it seems we should have light weight pulsar-function-common which keeps
common types for other modules. so, artifact doesn't have to depend on
actual-impl-artifacts such as pulsar-function-instance.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] merlimat commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
merlimat commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203208896
##
File path: pulsar-io/core/pom.xml
##
@@ -38,16 +38,10 @@
${project.groupId}
- pulsar-common
+ pulsar-functions-api
${project.version}
-
Review comment:
Actually, the diff is misleading, but the change is correct:
* Removed dep on `pulsar-common`
* Added dep on `pulsar-functions-api` since record is moved there
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] merlimat commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
merlimat commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203207753
##
File path: pulsar-io/kinesis/pom.xml
##
@@ -38,6 +38,13 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-functions-instance
+ ${project.version}
+ provided
Review comment:
I have moved `EncryptionContext` out of `Record` (main reason is that it's
really tied to Pulsar client API and not relevant to other sources and sinks).
I have added a new interface `RecordWithEncryptionContext` in
`pulsar-functions-instance`, alongside `PulsarSource` implemenentation.
Sink implementations, can check `s instanceof RecordWithEncryptionContext`
and downcast from there to access the `EncryptionContext`.
The dependency here is "provided" because the jar will always be added by
function runtime. In this case we need it to access the interface
`RecordWithEncryptionContext`, but there's no need to package it again with the
sink.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] merlimat commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
merlimat commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203207180
##
File path: pulsar-io/core/pom.xml
##
@@ -38,16 +38,10 @@
${project.groupId}
- pulsar-common
+ pulsar-functions-api
${project.version}
-
Review comment:
Oh, true, I was intending to remove the dependency alltogether, not just the
exclusions.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] aahmed-se commented on issue #2180: Cleanup Arquillian consolidate projects
aahmed-se commented on issue #2180: Cleanup Arquillian consolidate projects URL: https://github.com/apache/incubator-pulsar/pull/2180#issuecomment-405752528 consolidated the projects and updated the docs. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
sijie commented on a change in pull request #2179: Issue 2081: Improve
documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
URL: https://github.com/apache/incubator-pulsar/pull/2179#discussion_r203204228
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
##
@@ -222,15 +222,27 @@ void processArguments() throws Exception {
description = "Path to the main Python file for the function
(if the function is written in Python)",
listConverter = StringConverter.class)
protected String pyFile;
-@Parameter(names = "--inputs", description = "The function's input
topic or topics (multiple topics can be specified as a comma-separated list)")
+@Parameter(
+names = "--inputs",
+description =
+"The function's input topic or topics (multiple topics can be
specified as a comma-separated list)."
++ " A default SerDe is used for serializing/deserializing
messages. If you want to use your own SerDe,"
++ " specify the topic and the SerDe class in
[--customSerDeInputs]. The total list of input topics is"
++ " a union of [--inputs] and
([--topicsPattern]/[--customSerDeInputs])."
++ " A topic should only be specified in either [--inputs] or
[--customSerDeInputs].")
Review comment:
if you are specifying a topic, it can only be in one place, either --inputs
or --customSerDeInputs.
If you are specifying a topic pattern, it has to be in --topicsPattern; if
you are using a topic pattern and want to use a customized serde, the topic
pattern needs to be specified in --customerSerDeInputs.
any suggestions on improving this?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on issue #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
sijie commented on issue #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 URL: https://github.com/apache/incubator-pulsar/pull/2182#issuecomment-405750820 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] jerrypeng commented on issue #2181: augmenting protoschema with info for parsing
jerrypeng commented on issue #2181: augmenting protoschema with info for parsing URL: https://github.com/apache/incubator-pulsar/pull/2181#issuecomment-405750834 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] grantwwu commented on a change in pull request #2174: V2 doc changes
grantwwu commented on a change in pull request #2174: V2 doc changes
URL: https://github.com/apache/incubator-pulsar/pull/2174#discussion_r203196517
##
File path: site/docs/latest/cookbooks/RetentionExpiry.md
##
@@ -97,7 +97,7 @@ $ pulsar-admin namespaces set-retention
my-prop/my-cluster/my-ns \
REST API
-{% endpoint POST /admin/namespaces/:property/:cluster/:namespace/retention %}
+{% endpoint POST /admin/v2/namespaces/:property/:cluster/:namespace/retention
%}
Review comment:
Done
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] grantwwu commented on a change in pull request #2174: V2 doc changes
grantwwu commented on a change in pull request #2174: V2 doc changes
URL: https://github.com/apache/incubator-pulsar/pull/2174#discussion_r203195426
##
File path: site/docs/latest/admin-api/namespaces.md
##
@@ -92,7 +92,7 @@ $ pulsar-admin namespaces policies test-tenant/test-namespace
REST API
-{% endpoint GET /admin/namespaces/:tenant/:cluster/:namespace %}
+{% endpoint GET /admin/v2/namespaces/:tenant/:cluster/:namespace %}
Review comment:
Done
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] grantwwu commented on a change in pull request #2174: V2 doc changes
grantwwu commented on a change in pull request #2174: V2 doc changes
URL: https://github.com/apache/incubator-pulsar/pull/2174#discussion_r203195319
##
File path: site/docs/latest/admin-api/namespaces.md
##
@@ -47,7 +47,7 @@ $ pulsar-admin namespaces create test-tenant/test-namespace
REST API
-{% endpoint PUT /admin/namespaces/:tenant/:cluster/:namespace %}
+{% endpoint PUT /admin/v2/namespaces/:tenant/:cluster/:namespace %}
Review comment:
Done
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] rdhabalia commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
rdhabalia commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203198101
##
File path: pulsar-io/core/pom.xml
##
@@ -38,16 +38,10 @@
${project.groupId}
- pulsar-common
+ pulsar-functions-api
${project.version}
-
Review comment:
pulsar-common brings many different dependencies which doesn't require for
pulsar-io. so, I think we should exclude all transitive dependencies unless it
actually requires.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on issue #2177: Allow construction of c++ builtin auth plugins via factory
sijie commented on issue #2177: Allow construction of c++ builtin auth plugins via factory URL: https://github.com/apache/incubator-pulsar/pull/2177#issuecomment-405744743 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2183: Issue 2121: Improve logging around "Namespace not served by this instance"
sijie commented on issue #2183: Issue 2121: Improve logging around "Namespace not served by this instance" URL: https://github.com/apache/incubator-pulsar/pull/2183#issuecomment-405744899 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] rdhabalia commented on a change in pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
rdhabalia commented on a change in pull request #2187: Removed the
`Context.ack(byte[] messageId)` and expose `Record` instead
URL: https://github.com/apache/incubator-pulsar/pull/2187#discussion_r203197794
##
File path: pulsar-io/kinesis/pom.xml
##
@@ -38,6 +38,13 @@
${project.version}
+
+ ${project.groupId}
+ pulsar-functions-instance
+ ${project.version}
+ provided
Review comment:
why provided?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie closed pull request #2175: Don't dirty the tree when building in CI
sijie closed pull request #2175: Don't dirty the tree when building in CI URL: https://github.com/apache/incubator-pulsar/pull/2175 This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/.gitignore b/.gitignore index cd4f8addb6..8808c8f834 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,6 @@ pulsar-client-cpp/python/pkg/osx/**/*.whl pulsar-client-cpp/python/pkg/osx/**/*.template2 pulsar-client-cpp/python/wheelhouse - +# CI generated files +.repository +docker.debug-info This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie closed pull request #2171: Add integration test for kafka source
sijie closed pull request #2171: Add integration test for kafka source
URL: https://github.com/apache/incubator-pulsar/pull/2171
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pom.xml b/pom.xml
index a6d645379b..263d3624d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -799,6 +799,11 @@ flexible messaging model and an intuitive client
API.
testcontainers
${testcontainers.version}
+
+org.testcontainers
+kafka
+${testcontainers.version}
+
org.arquillian.cube
arquillian-cube-docker
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
deleted file mode 100644
index 83b5e42ba0..00
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.tests.containers;
-
-import lombok.extern.slf4j.Slf4j;
-import org.testcontainers.containers.BindMode;
-import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
-
-/**
- * Cassandra Container.
- */
-@Slf4j
-public class KafkaContainer> extends
ChaosContainer {
-
-public static final String NAME = "kafka";
-public static final int INTERNAL_PORT = 9092;
-public static final int PORT = 9093;
-
-public KafkaContainer(String clusterName) {
-super(clusterName, "confluentinc/cp-kafka:4.1.1");
-}
-
-@Override
-protected void configure() {
-super.configure();
-this.withNetworkAliases(NAME)
-.withExposedPorts(INTERNAL_PORT, PORT)
-.withClasspathResourceMapping(
-"kafka-zookeeper.properties", "/zookeeper.properties",
-BindMode.READ_ONLY)
-.withCommand("sh", "-c", "zookeeper-server-start
/zookeeper.properties & /etc/confluent/docker/run")
-.withEnv("KAFKA_LISTENERS",
-"INTERNAL://kafka:" + INTERNAL_PORT + ",PLAINTEXT://" +
"0.0.0.0" + ":" + PORT)
-.withEnv("KAFKA_ZOOKEEPER_CONNECT", "localhost:2181")
-.withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP",
"INTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT")
-.withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "INTERNAL")
-.withEnv("KAFKA_BROKER_ID", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1")
-.withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + "")
-.withCreateContainerCmdModifier(createContainerCmd -> {
-createContainerCmd.withHostName(NAME);
-createContainerCmd.withName(clusterName + "-" + NAME);
-})
-.waitingFor(new HostPortWaitStrategy());
-}
-}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
deleted file mode 100644
index 052db7e8b0..00
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed
[GitHub] sijie closed pull request #2170: Add integration test for kafka sink
sijie closed pull request #2170: Add integration test for kafka sink
URL: https://github.com/apache/incubator-pulsar/pull/2170
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pom.xml b/pom.xml
index a6d645379b..263d3624d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -799,6 +799,11 @@ flexible messaging model and an intuitive client
API.
testcontainers
${testcontainers.version}
+
+org.testcontainers
+kafka
+${testcontainers.version}
+
org.arquillian.cube
arquillian-cube-docker
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
deleted file mode 100644
index 83b5e42ba0..00
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.tests.containers;
-
-import lombok.extern.slf4j.Slf4j;
-import org.testcontainers.containers.BindMode;
-import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
-
-/**
- * Cassandra Container.
- */
-@Slf4j
-public class KafkaContainer> extends
ChaosContainer {
-
-public static final String NAME = "kafka";
-public static final int INTERNAL_PORT = 9092;
-public static final int PORT = 9093;
-
-public KafkaContainer(String clusterName) {
-super(clusterName, "confluentinc/cp-kafka:4.1.1");
-}
-
-@Override
-protected void configure() {
-super.configure();
-this.withNetworkAliases(NAME)
-.withExposedPorts(INTERNAL_PORT, PORT)
-.withClasspathResourceMapping(
-"kafka-zookeeper.properties", "/zookeeper.properties",
-BindMode.READ_ONLY)
-.withCommand("sh", "-c", "zookeeper-server-start
/zookeeper.properties & /etc/confluent/docker/run")
-.withEnv("KAFKA_LISTENERS",
-"INTERNAL://kafka:" + INTERNAL_PORT + ",PLAINTEXT://" +
"0.0.0.0" + ":" + PORT)
-.withEnv("KAFKA_ZOOKEEPER_CONNECT", "localhost:2181")
-.withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP",
"INTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT")
-.withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "INTERNAL")
-.withEnv("KAFKA_BROKER_ID", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1")
-.withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + "")
-.withCreateContainerCmdModifier(createContainerCmd -> {
-createContainerCmd.withHostName(NAME);
-createContainerCmd.withName(clusterName + "-" + NAME);
-})
-.waitingFor(new HostPortWaitStrategy());
-}
-}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
deleted file mode 100644
index 052db7e8b0..00
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaProxyContainer.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under
[incubator-pulsar] branch master updated: Don't dirty the tree when building in CI (#2175)
This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/master by this push: new 3dfc217 Don't dirty the tree when building in CI (#2175) 3dfc217 is described below commit 3dfc217e9c2855ab4b42c81d453a3023c5dade32 Author: Ivan Kelly AuthorDate: Tue Jul 17 23:08:39 2018 +0100 Don't dirty the tree when building in CI (#2175) When we build a package, mvn inserts the sha of the commit we are building from into the tarball. If there have been any changes to the work tree, it will add the suffix "(dirty)". CI makes a couple of modifications to the worktree. It creates its own .repository for maven artifacts. And it create a docker debugging logfile. This patch adds these to .gitignore so that they won't be picked up by git status and won't cause the tree to be 'dirty'. --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cd4f8ad..8808c8f 100644 --- a/.gitignore +++ b/.gitignore @@ -64,4 +64,6 @@ pulsar-client-cpp/python/pkg/osx/**/*.whl pulsar-client-cpp/python/pkg/osx/**/*.template2 pulsar-client-cpp/python/wheelhouse - +# CI generated files +.repository +docker.debug-info
[incubator-pulsar] branch master updated: Add integration test for kafka source (#2171)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 8de127d Add integration test for kafka source (#2171)
8de127d is described below
commit 8de127d65f0a3eb8eb9bad0194d305f24d3ed791
Author: Sijie Guo
AuthorDate: Tue Jul 17 15:08:13 2018 -0700
Add integration test for kafka source (#2171)
*Motivation*
We added integration tests for kafka & cassandra sinks. We need test
coverage on kafka sources.
*Changes*
- Add `PulsarIOSourceTest` and `SourceTester` for testing sources
- Implement `KafkaSourceTester` for testing kafka source
---
.../tests/integration/io/KafkaSourceTester.java| 140 +++
.../tests/integration/io/PulsarIOSourceTest.java | 255 +
.../pulsar/tests/integration/io/SourceTester.java | 52 +
3 files changed, 447 insertions(+)
diff --git
a/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java
new file mode 100644
index 000..e690d6b
--- /dev/null
+++
b/tests/integration/semantics/src/test/java/org/apache/pulsar/tests/integration/io/KafkaSourceTester.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.integration.io;
+
+import static org.testng.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.kafka.common.serialization.StringSerializer;
+import org.apache.pulsar.tests.integration.utils.TestUtils;
+import org.testcontainers.containers.Container.ExecResult;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.KafkaContainer;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+import org.testng.collections.Maps;
+
+/**
+ * A tester for testing kafka source.
+ */
+@Slf4j
+public class KafkaSourceTester extends SourceTester {
+
+private static final String NAME = "kafka";
+
+private final String kafkaTopicName;
+
+private KafkaContainer kafkaContainer;
+
+private KafkaConsumer kafkaConsumer;
+
+public KafkaSourceTester() {
+super(NAME);
+String suffix = TestUtils.randomName(8) + "_" +
System.currentTimeMillis();
+this.kafkaTopicName = "kafka_source_topic_" + suffix;
+
+sourceConfig.put("bootstrapServers", NAME + ":9092");
+sourceConfig.put("groupId", "test-source-group");
+sourceConfig.put("fetchMinBytes", 1L);
+sourceConfig.put("autoCommitIntervalMs", 10L);
+sourceConfig.put("sessionTimeoutMs", 1L);
+sourceConfig.put("topic", kafkaTopicName);
+sourceConfig.put("valueDeserializationClass",
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
+}
+
+@Override
+protected Map> newSourceService(String
clusterName) {
+this.kafkaContainer = new KafkaContainer()
+.withEmbeddedZookeeper()
+.withNetworkAliases(NAME)
+.withCreateContainerCmdModifier(createContainerCmd ->
createContainerCmd
+.withName(NAME)
+.withHostName(clusterName + "-" + NAME));
+
+Map> containers = Maps.newHashMap();
+containers.put("kafka", kafkaContainer);
+return containers;
+}
+
+@Override
+protected void prepareSource() throws Exception {
+ExecResult execResult = kafkaContainer.execInContainer(
+"/usr/bin/kafka-topics",
+"--create",
+
[incubator-pulsar] branch master updated: Add integration test for kafka sink (#2170)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new d6fb900 Add integration test for kafka sink (#2170)
d6fb900 is described below
commit d6fb900f3741603290f1b93fdb86fe8c1e8d70e5
Author: Sijie Guo
AuthorDate: Tue Jul 17 15:07:44 2018 -0700
Add integration test for kafka sink (#2170)
*Motivation*
#2167 introduces a sink test for cassandra. this PR adds a test for kafka
sink.
*Changes*
- Add a KafkaSinkTester so `PulsarIOSinkTest` will test kafka sink
- Remove customized `KafkaContainer` and `KafkaProxyContainer` and use
testcontainers's KafkaContainer
---
pom.xml| 5 +
.../pulsar/tests/containers/KafkaContainer.java| 63 --
.../tests/containers/KafkaProxyContainer.java | 51
.../pulsar/tests/topologies/PulsarCluster.java | 7 +-
tests/integration/semantics/pom.xml| 11 ++
.../tests/integration/io/CassandraSinkTester.java | 11 +-
.../tests/integration/io/KafkaSinkTester.java | 130 +
.../tests/integration/io/PulsarIOSinkTest.java | 13 ++-
.../pulsar/tests/integration/io/SinkTester.java| 6 +-
9 files changed, 169 insertions(+), 128 deletions(-)
diff --git a/pom.xml b/pom.xml
index a6d6453..263d362 100644
--- a/pom.xml
+++ b/pom.xml
@@ -800,6 +800,11 @@ flexible messaging model and an intuitive client
API.
${testcontainers.version}
+org.testcontainers
+kafka
+${testcontainers.version}
+
+
org.arquillian.cube
arquillian-cube-docker
${arquillian-cube.version}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
deleted file mode 100644
index 83b5e42..000
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/KafkaContainer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.tests.containers;
-
-import lombok.extern.slf4j.Slf4j;
-import org.testcontainers.containers.BindMode;
-import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy;
-
-/**
- * Cassandra Container.
- */
-@Slf4j
-public class KafkaContainer> extends
ChaosContainer {
-
-public static final String NAME = "kafka";
-public static final int INTERNAL_PORT = 9092;
-public static final int PORT = 9093;
-
-public KafkaContainer(String clusterName) {
-super(clusterName, "confluentinc/cp-kafka:4.1.1");
-}
-
-@Override
-protected void configure() {
-super.configure();
-this.withNetworkAliases(NAME)
-.withExposedPorts(INTERNAL_PORT, PORT)
-.withClasspathResourceMapping(
-"kafka-zookeeper.properties", "/zookeeper.properties",
-BindMode.READ_ONLY)
-.withCommand("sh", "-c", "zookeeper-server-start
/zookeeper.properties & /etc/confluent/docker/run")
-.withEnv("KAFKA_LISTENERS",
-"INTERNAL://kafka:" + INTERNAL_PORT + ",PLAINTEXT://" +
"0.0.0.0" + ":" + PORT)
-.withEnv("KAFKA_ZOOKEEPER_CONNECT", "localhost:2181")
-.withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP",
"INTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT")
-.withEnv("KAFKA_INTER_BROKER_LISTENER_NAME", "INTERNAL")
-.withEnv("KAFKA_BROKER_ID", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1")
-.withEnv("KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS", "1")
-.withEnv("KAFKA_LOG_FLUSH_INTERVAL_MESSAGES", Long.MAX_VALUE + "")
-.withCreateContainerCmdModifier(createContainerCmd -> {
-createContainerCmd.withHostName(NAME);
-createContainerCmd.withName(clusterName + "-" + NAME);
-})
-.waitingFor(new
[GitHub] merlimat opened a new pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead
merlimat opened a new pull request #2187: Removed the `Context.ack(byte[] messageId)` and expose `Record` instead URL: https://github.com/apache/incubator-pulsar/pull/2187 ### Motivation The `Context.ack(byte[] messageId)` is problematic when the function is consuming from multiple topics since we would have to keep track of the topic in the serialized message id. Since this API is just used in windowing code, replacing this by exposing "current" record which can be acked directly. Note: this is based on #2184 -- Only review 3bbc606408064e5f4c221da271747c9af6143fc9 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat closed pull request #2178: Inherit I/O streams in functions process runtime
merlimat closed pull request #2178: Inherit I/O streams in functions process
runtime
URL: https://github.com/apache/incubator-pulsar/pull/2178
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
index 8054ff8bb0..f3ed170600 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
@@ -415,7 +415,7 @@ public void
onSuccess(InstanceCommunication.HealthCheckResult t) {
private void startProcess() {
deathException = null;
try {
-ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
+ProcessBuilder processBuilder = new
ProcessBuilder(processArgs).inheritIO();
log.info("ProcessBuilder starting the process with args {}",
String.join(" ", processBuilder.command()));
process = processBuilder.start();
} catch (Exception ex) {
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[incubator-pulsar] branch master updated: Inherit I/O streams in functions process runtime (#2178)
This is an automated email from the ASF dual-hosted git repository.
mmerli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 918e38c Inherit I/O streams in functions process runtime (#2178)
918e38c is described below
commit 918e38c73c241c6707ffe41bf98f6b765506dc22
Author: Ivan Kelly
AuthorDate: Tue Jul 17 22:55:13 2018 +0100
Inherit I/O streams in functions process runtime (#2178)
When running processes with the local runner, the output to stderr and
stdout go nowhere. This makes it hard to debug startup issues.
This patch modifies the process runtime to make the child process
inherit the streams from the parent process, so the output will be
visible to the runner.
---
.../main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
index 8054ff8..f3ed170 100644
---
a/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
+++
b/pulsar-functions/runtime/src/main/java/org/apache/pulsar/functions/runtime/ProcessRuntime.java
@@ -415,7 +415,7 @@ class ProcessRuntime implements Runtime {
private void startProcess() {
deathException = null;
try {
-ProcessBuilder processBuilder = new ProcessBuilder(processArgs);
+ProcessBuilder processBuilder = new
ProcessBuilder(processArgs).inheritIO();
log.info("ProcessBuilder starting the process with args {}",
String.join(" ", processBuilder.command()));
process = processBuilder.start();
} catch (Exception ex) {
[GitHub] grantwwu commented on a change in pull request #2174: V2 doc changes
grantwwu commented on a change in pull request #2174: V2 doc changes
URL: https://github.com/apache/incubator-pulsar/pull/2174#discussion_r203193031
##
File path: site/docs/latest/admin-api/namespaces.md
##
@@ -144,7 +144,7 @@ test-tenant/ns2
REST API
-{% endpoint GET /admin/namespaces/:tenant/:cluster %}
+{% endpoint GET /admin/v2/namespaces/:tenant/:cluster %}
Review comment:
So the entire "List namespaces within a cluster" section ought to be deleted?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
ivankelly commented on a change in pull request #2179: Issue 2081: Improve
documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
URL: https://github.com/apache/incubator-pulsar/pull/2179#discussion_r203138017
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
##
@@ -222,15 +222,27 @@ void processArguments() throws Exception {
description = "Path to the main Python file for the function
(if the function is written in Python)",
listConverter = StringConverter.class)
protected String pyFile;
-@Parameter(names = "--inputs", description = "The function's input
topic or topics (multiple topics can be specified as a comma-separated list)")
+@Parameter(
+names = "--inputs",
+description =
+"The function's input topic or topics (multiple topics can be
specified as a comma-separated list)."
++ " A default SerDe is used for serializing/deserializing
messages. If you want to use your own SerDe,"
++ " specify the topic and the SerDe class in
[--customSerDeInputs]. The total list of input topics is"
++ " a union of [--inputs] and
([--topicsPattern]/[--customSerDeInputs])."
++ " A topic should only be specified in either [--inputs] or
[--customSerDeInputs].")
protected String inputs;
-@Parameter(names = "--topicsPattern", description = "TopicsPattern to
consume from list of topics under a namespace that match the pattern. [--input]
and [--topicsPattern] are mutually exclusive. Add SerDe class name for a
pattern in --customSerdeInputs (supported for java fun only)")
+@Parameter(names = "--topicsPattern", description = "TopicsPattern to
consume from list of topics under a namespace that match the pattern.
[--inputs] and [--topicsPattern] are mutually exclusive. Add SerDe class name
for a pattern in --customSerdeInputs (supported for java fun only)")
protected String topicsPattern;
@Parameter(names = "--output", description = "The function's output
topic")
protected String output;
@Parameter(names = "--logTopic", description = "The topic to which the
function's logs are produced")
protected String logTopic;
-@Parameter(names = "--customSerdeInputs", description = "The map of
input topics to SerDe class names (as a JSON string)")
+@Parameter(
+names = "--customSerdeInputs",
+description =
+"The map of input topics to SerDe class names (as a JSON
string). The total list of input topics is"
++ " a union of [--inputs] and
([--topicsPattern]/[--customSerDeInputs])."
++ " A topic should only be specified in either [--inputs] or
[--customSerDeInputs].")
Review comment:
As above.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
ivankelly commented on a change in pull request #2179: Issue 2081: Improve
documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
URL: https://github.com/apache/incubator-pulsar/pull/2179#discussion_r203137034
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
##
@@ -222,15 +222,27 @@ void processArguments() throws Exception {
description = "Path to the main Python file for the function
(if the function is written in Python)",
listConverter = StringConverter.class)
protected String pyFile;
-@Parameter(names = "--inputs", description = "The function's input
topic or topics (multiple topics can be specified as a comma-separated list)")
+@Parameter(
+names = "--inputs",
+description =
+"The function's input topic or topics (multiple topics can be
specified as a comma-separated list)."
++ " A default SerDe is used for serializing/deserializing
messages. If you want to use your own SerDe,"
++ " specify the topic and the SerDe class in
[--customSerDeInputs]. The total list of input topics is"
Review comment:
specify the topic(s) and the serDe class(es)
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
ivankelly commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions URL: https://github.com/apache/incubator-pulsar/pull/2179#discussion_r203138152 ## File path: site/_data/cli/pulsar-admin.yaml ## @@ -177,12 +177,25 @@ commands: description: The disk space to allocate to each function instance (in bytes) Review comment: We need to generate this file automatically. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
ivankelly commented on a change in pull request #2179: Issue 2081: Improve
documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
URL: https://github.com/apache/incubator-pulsar/pull/2179#discussion_r203137923
##
File path:
pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdFunctions.java
##
@@ -222,15 +222,27 @@ void processArguments() throws Exception {
description = "Path to the main Python file for the function
(if the function is written in Python)",
listConverter = StringConverter.class)
protected String pyFile;
-@Parameter(names = "--inputs", description = "The function's input
topic or topics (multiple topics can be specified as a comma-separated list)")
+@Parameter(
+names = "--inputs",
+description =
+"The function's input topic or topics (multiple topics can be
specified as a comma-separated list)."
++ " A default SerDe is used for serializing/deserializing
messages. If you want to use your own SerDe,"
++ " specify the topic and the SerDe class in
[--customSerDeInputs]. The total list of input topics is"
++ " a union of [--inputs] and
([--topicsPattern]/[--customSerDeInputs])."
++ " A topic should only be specified in either [--inputs] or
[--customSerDeInputs].")
Review comment:
This line doesn't make sense. Is it saying that a topic can't be in both?
Where does that leave --topicsPattern?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie opened a new pull request #2186: Issue 1288: Provide documentation for running BookKeeper auto-recovery
sijie opened a new pull request #2186: Issue 1288: Provide documentation for running BookKeeper auto-recovery URL: https://github.com/apache/incubator-pulsar/pull/2186 ### Motivation Fixes #1288. We mentioned auto-recovery in DCOS deployment. but we don't have any instructions to run auto-recovery in bare-mental deployment. ### Changes Update the documentation in bare-mental deployment. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie opened a new pull request #2185: Clean up sidebar menu under `Deployment`
sijie opened a new pull request #2185: Clean up sidebar menu under `Deployment` URL: https://github.com/apache/incubator-pulsar/pull/2185 *Motivation* There's "Deploy on Amazon Web Services" and "Pulsar on AWS" in the deployment section of the site. These are confused. *Changes* Remove sub-sections of "kubernetes" deployment from sidebard and only leave "Pulsar on Kubernetes" there. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat opened a new pull request #2184: Moved Record interface as part of functions api
merlimat opened a new pull request #2184: Moved Record interface as part of functions api URL: https://github.com/apache/incubator-pulsar/pull/2184 ### Motivation Allow to include `Record` as part of functions API (through `Context`) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie opened a new pull request #2183: Issue 2121: Improve logging around "Namespace not served by this instance"
sijie opened a new pull request #2183: Issue 2121: Improve logging around "Namespace not served by this instance" URL: https://github.com/apache/incubator-pulsar/pull/2183 ### Motivation Fixes #2121. Improve the logging around "namespace not served by this instance" to make things clearer. ### Changes Improve the logging to say "namespace bundle for topic not served by", rahter than "namespace not served by" This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie opened a new pull request #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1
sijie opened a new pull request #2182: Issue 2110: get-retention returns retentionSizeInMB as 0 when set to -1 URL: https://github.com/apache/incubator-pulsar/pull/2182 ### Motivation Fixes #2110. infinite retention is not correctly propagated. ### Changes Fix the command tool to set infinite retention correctly. Update command line description and website to reflect the fact retention size less than 1MB is treated as no retention. Add an integration test to verify the CLI change works. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2177: Allow construction of c++ builtin auth plugins via factory
sijie commented on issue #2177: Allow construction of c++ builtin auth plugins via factory URL: https://github.com/apache/incubator-pulsar/pull/2177#issuecomment-405704664 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on a change in pull request #2181: augmenting protoschema with info for parsing
sijie commented on a change in pull request #2181: augmenting protoschema with
info for parsing
URL: https://github.com/apache/incubator-pulsar/pull/2181#discussion_r203154033
##
File path:
pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/ProtobufSchemaTest.java
##
@@ -67,4 +78,33 @@ public void testSchema() {
Assert.assertEquals(schema.toString(), EXPECTED_SCHEMA_JSON);
}
+
+@Test
+public void testGenericOf() {
+try {
+
ProtobufSchema
protobufSchema
+=
ProtobufSchema.ofGenericClass(org.apache.pulsar.client.schema.proto.Test.TestMessage.class,
+Collections.emptyMap());
+} catch (Exception e) {
+Assert.fail();
Review comment:
nit: put a message in the fail method
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on a change in pull request #2181: augmenting protoschema with info for parsing
sijie commented on a change in pull request #2181: augmenting protoschema with
info for parsing
URL: https://github.com/apache/incubator-pulsar/pull/2181#discussion_r203153997
##
File path:
pulsar-client-schema/src/test/java/org/apache/pulsar/client/schema/ProtobufSchemaTest.java
##
@@ -67,4 +78,33 @@ public void testSchema() {
Assert.assertEquals(schema.toString(), EXPECTED_SCHEMA_JSON);
}
+
+@Test
+public void testGenericOf() {
+try {
+
ProtobufSchema
protobufSchema
+=
ProtobufSchema.ofGenericClass(org.apache.pulsar.client.schema.proto.Test.TestMessage.class,
+Collections.emptyMap());
+} catch (Exception e) {
+Assert.fail();
+}
+
+try {
+
ProtobufSchema
protobufSchema
+= ProtobufSchema.ofGenericClass(String.class,
+Collections.emptyMap());
+Assert.fail();
Review comment:
nit: put a meaningful message in fail method
e.g.
`fail("Should not construct a ProtobufShema over a non-protobuf-generated
class")`
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects
sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects URL: https://github.com/apache/incubator-pulsar/pull/2180#discussion_r203152828 ## File path: tests/integration-tests-base/pom.xml ## @@ -34,7 +34,7 @@ integration-tests-base pom - Apache Pulsar :: Tests :: Base module for Arquillian based integration tests + Apache Pulsar :: Tests :: Base module for TestContainers based integration tests Review comment: if you combine all integration modules into one integration module, I would prefer calling it "Apache Pulsar :: Tests :: Integration Tests". there is no need to call out "testcontainers". just like we used testng for testing, but we don't call out testng in any module's name or description. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects
sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects URL: https://github.com/apache/incubator-pulsar/pull/2180#discussion_r203152315 ## File path: tests/pom.xml ## @@ -33,7 +33,6 @@ Apache Pulsar :: Tests docker-images -integration-tests-utils integration-tests-topologies Review comment: better to combine these integration modules into one. since there is no needed to separate them anymore. that would reduce pulsar's build time. also please update tests README https://github.com/apache/incubator-pulsar/blob/master/tests/README.md This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects
sijie commented on a change in pull request #2180: [WIP] Cleanup Arquillian consolidate projects URL: https://github.com/apache/incubator-pulsar/pull/2180#discussion_r203152315 ## File path: tests/pom.xml ## @@ -33,7 +33,6 @@ Apache Pulsar :: Tests docker-images -integration-tests-utils integration-tests-topologies Review comment: better to compile these integration modules into one. since there is no needed to separate them anymore. that would reduce pulsar's build time. also please update tests README https://github.com/apache/incubator-pulsar/blob/master/tests/README.md This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] jerrypeng opened a new pull request #2181: augmenting protoschema with info for parsing
jerrypeng opened a new pull request #2181: augmenting protoschema with info for parsing URL: https://github.com/apache/incubator-pulsar/pull/2181 ### Motivation Explain here the context, and why you're making that change. What is the problem you're trying to solve. ### Modifications Describe the modifications you've done. ### Result After your change, what will change. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] aahmed-se opened a new pull request #2180: [WIP] Cleanup Arquillian consolidate projects
aahmed-se opened a new pull request #2180: [WIP] Cleanup Arquillian consolidate projects URL: https://github.com/apache/incubator-pulsar/pull/2180 ### Motivation This is the last in the series of PR's to migrate to Test Containers, here we remove arquillian resources and consolidate test projects. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie opened a new pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions
sijie opened a new pull request #2179: Issue 2081: Improve documentation about `--inputs` and `--customSerdeInputs` for pulsar functions URL: https://github.com/apache/incubator-pulsar/pull/2179 ### Motivation Fixes #2081. flags `--inputs` and `--customSerdeInputs` are confused. ### Changes - Update the descriptions of `--inputs` and `--customSerdeInputs` in functions admin command cli - Update the documentation in website This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2174: V2 doc changes
sijie commented on issue #2174: V2 doc changes URL: https://github.com/apache/incubator-pulsar/pull/2174#issuecomment-405674548 retest this please // (trigger ci) This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2175: Don't dirty the tree when building in CI
sijie commented on issue #2175: Don't dirty the tree when building in CI URL: https://github.com/apache/incubator-pulsar/pull/2175#issuecomment-405673648 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie closed issue #1433: Expose batch flushAsync() and flush() methods in Producer
sijie closed issue #1433: Expose batch flushAsync() and flush() methods in Producer URL: https://github.com/apache/incubator-pulsar/issues/1433 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #1433: Expose batch flushAsync() and flush() methods in Producer
sijie commented on issue #1433: Expose batch flushAsync() and flush() methods in Producer URL: https://github.com/apache/incubator-pulsar/issues/1433#issuecomment-405667599 This is fixed by #2103 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat closed issue #2141: Race condition in reconnection after managed ledger is fenced
merlimat closed issue #2141: Race condition in reconnection after managed ledger is fenced URL: https://github.com/apache/incubator-pulsar/issues/2141 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] merlimat commented on issue #2141: Race condition in reconnection after managed ledger is fenced
merlimat commented on issue #2141: Race condition in reconnection after managed ledger is fenced URL: https://github.com/apache/incubator-pulsar/issues/2141#issuecomment-405667315 This was fixed by #2148 and #2149 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2135: Allow user to get topic from consumer message (#2007)
sijie commented on issue #2135: Allow user to get topic from consumer message (#2007) URL: https://github.com/apache/incubator-pulsar/pull/2135#issuecomment-405666547 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[incubator-pulsar] branch master updated: Removed GPL deps on gnu crypto from aerospike connector (#2173)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new d5536cc Removed GPL deps on gnu crypto from aerospike connector
(#2173)
d5536cc is described below
commit d5536cc38a5811c8e02d580165b8e8d0861a8b4d
Author: Matteo Merli
AuthorDate: Tue Jul 17 10:41:29 2018 -0700
Removed GPL deps on gnu crypto from aerospike connector (#2173)
### Motivation
There's an optional dependency on gnu-crypto for aerospike connector. We
need to remove it to avoid license issue at release.
---
pulsar-io/aerospike/pom.xml | 6 ++
1 file changed, 6 insertions(+)
diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml
index 1a2d546..5f678be 100644
--- a/pulsar-io/aerospike/pom.xml
+++ b/pulsar-io/aerospike/pom.xml
@@ -52,6 +52,12 @@
com.aerospike
aerospike-client
${aerospike-client.version}
+
+
+ org.gnu
+ gnu-crypto
+
+
[GitHub] sijie closed pull request #2173: Removed GPL deps on gnu crypto from aerospike connector
sijie closed pull request #2173: Removed GPL deps on gnu crypto from aerospike
connector
URL: https://github.com/apache/incubator-pulsar/pull/2173
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/pulsar-io/aerospike/pom.xml b/pulsar-io/aerospike/pom.xml
index 1a2d5460ac..5f678be013 100644
--- a/pulsar-io/aerospike/pom.xml
+++ b/pulsar-io/aerospike/pom.xml
@@ -52,6 +52,12 @@
com.aerospike
aerospike-client
${aerospike-client.version}
+
+
+ org.gnu
+ gnu-crypto
+
+
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
sijie commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api URL: https://github.com/apache/incubator-pulsar/pull/2065#issuecomment-405666096 I know it is approved. let's not merge this until I cut 2.1 release. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] sijie commented on issue #2170: Add integration test for kafka sink
sijie commented on issue #2170: Add integration test for kafka sink URL: https://github.com/apache/incubator-pulsar/pull/2170#issuecomment-405661013 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] ivankelly opened a new pull request #2178: Inherit I/O streams in functions process runtime
ivankelly opened a new pull request #2178: Inherit I/O streams in functions process runtime URL: https://github.com/apache/incubator-pulsar/pull/2178 When running processes with the local runner, the output to stderr and stdout go nowhere. This makes it hard to debug startup issues. This patch modifies the process runtime to make the child process inherit the streams from the parent process, so the output will be visible to the runner. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2177: Allow construction of c++ builtin auth plugins via factory
ivankelly commented on a change in pull request #2177: Allow construction of
c++ builtin auth plugins via factory
URL: https://github.com/apache/incubator-pulsar/pull/2177#discussion_r203085975
##
File path: pulsar-client-cpp/lib/auth/AuthAthenz.h
##
@@ -25,6 +25,9 @@
namespace pulsar {
+const std::string ATHENZ_PLUGIN_NAME = "tls";
Review comment:
oops, fixed
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] maskit commented on a change in pull request #2177: Allow construction of c++ builtin auth plugins via factory
maskit commented on a change in pull request #2177: Allow construction of c++
builtin auth plugins via factory
URL: https://github.com/apache/incubator-pulsar/pull/2177#discussion_r203084783
##
File path: pulsar-client-cpp/lib/auth/AuthAthenz.h
##
@@ -25,6 +25,9 @@
namespace pulsar {
+const std::string ATHENZ_PLUGIN_NAME = "tls";
Review comment:
athenz?
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhaijack edited a comment on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
zhaijack edited a comment on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api URL: https://github.com/apache/incubator-pulsar/pull/2065#issuecomment-405614086 retest this please for C++/Python Tests build error This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
zhaijack commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api URL: https://github.com/apache/incubator-pulsar/pull/2065#issuecomment-405614086 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] djKooks commented on issue #2166: 'java.lang.NoSuchMethodError' when running standalone
djKooks commented on issue #2166: 'java.lang.NoSuchMethodError' when running standalone URL: https://github.com/apache/incubator-pulsar/issues/2166#issuecomment-405612703 @srkukarni @merlimat seems working. Thanks for help! This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] djKooks closed issue #2166: 'java.lang.NoSuchMethodError' when running standalone
djKooks closed issue #2166: 'java.lang.NoSuchMethodError' when running standalone URL: https://github.com/apache/incubator-pulsar/issues/2166 This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] zhaijack commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
zhaijack commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202986600
##
File path:
pulsar-broker/src/main/java/org/apache/pulsar/broker/offload/impl/ManagedLedgerOffloader.java
##
@@ -96,23 +122,66 @@ public static S3ManagedLedgerOffloader
create(ServiceConfiguration conf,
throw new
PulsarServerException("s3ManagedLedgerOffloadMaxBlockSizeInBytes cannot be less
than 5MB");
}
-AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
+return new ManagedLedgerOffloader(driver, bucket, scheduler,
maxBlockSize, readBufferSize, endpoint, region);
+}
+
+// build context for jclouds BlobStoreContext
+ManagedLedgerOffloader(String driver, String container, OrderedScheduler
scheduler,
+ int maxBlockSize, int readBufferSize, String
endpoint, String region) {
+this.scheduler = scheduler;
+this.readBufferSize = readBufferSize;
+
+this.bucket = container;
+this.maxBlockSize = maxBlockSize;
+
+Properties overrides = new Properties();
+// This property controls the number of parts being uploaded in
parallel.
+overrides.setProperty("jclouds.mpu.parallel.degree", "1");
+overrides.setProperty("jclouds.mpu.parts.size",
Integer.toString(maxBlockSize));
+overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
+overrides.setProperty(Constants.PROPERTY_MAX_RETRIES,
Integer.toString(100));
+
+ContextBuilder contextBuilder = ContextBuilder.newBuilder(driver);
+
+AWSCredentials credentials = null;
+try {
+DefaultAWSCredentialsProviderChain creds =
DefaultAWSCredentialsProviderChain.getInstance();
+credentials = creds.getCredentials();
+} catch (Exception e) {
+log.error("Exception when get credentials for s3 ", e);
+}
+
+String id = "accesskey";
Review comment:
Thanks. will keep this default value. empty string will meet error in
jclouds.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] zhaijack commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
zhaijack commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202986135
##
File path:
pulsar-broker/src/main/java/org/apache/pulsar/broker/offload/impl/BackedReadHandleImpl.java
##
@@ -42,28 +36,28 @@
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.impl.LedgerEntriesImpl;
import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
-
+import org.apache.pulsar.broker.offload.BackedInputStream;
import org.apache.pulsar.broker.offload.OffloadIndexBlock;
import org.apache.pulsar.broker.offload.OffloadIndexBlockBuilder;
import org.apache.pulsar.broker.offload.OffloadIndexEntry;
-import org.apache.pulsar.broker.offload.BackedInputStream;
-import
org.apache.pulsar.broker.offload.impl.S3ManagedLedgerOffloader.VersionCheck;
-
+import
org.apache.pulsar.broker.offload.impl.ManagedLedgerOffloader.VersionCheck;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class S3BackedReadHandleImpl implements ReadHandle {
-private static final Logger log =
LoggerFactory.getLogger(S3BackedReadHandleImpl.class);
+public class BackedReadHandleImpl implements ReadHandle {
Review comment:
Thanks, will change it, if you insist on this.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202977793 ## File path: jclouds-shaded/pom.xml ## @@ -0,0 +1,105 @@ + + +http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd; + xmlns="http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;> + 4.0.0 + +org.apache.pulsar +pulsar +2.2.0-incubating-SNAPSHOT +.. + + + jclouds-shaded + Apache Pulsar :: Jclouds shaded + + + + com.google.code.gson + gson + 2.5 + + + org.apache.jclouds + jclouds-allblobstore + 2.2.0-SNAPSHOT Review comment: Ok, it'll have to be updated before an actual release. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
ivankelly commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202979539
##
File path:
pulsar-broker/src/main/java/org/apache/pulsar/broker/offload/impl/BackedReadHandleImpl.java
##
@@ -42,28 +36,28 @@
import org.apache.bookkeeper.client.api.ReadHandle;
import org.apache.bookkeeper.client.impl.LedgerEntriesImpl;
import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
-
+import org.apache.pulsar.broker.offload.BackedInputStream;
import org.apache.pulsar.broker.offload.OffloadIndexBlock;
import org.apache.pulsar.broker.offload.OffloadIndexBlockBuilder;
import org.apache.pulsar.broker.offload.OffloadIndexEntry;
-import org.apache.pulsar.broker.offload.BackedInputStream;
-import
org.apache.pulsar.broker.offload.impl.S3ManagedLedgerOffloader.VersionCheck;
-
+import
org.apache.pulsar.broker.offload.impl.ManagedLedgerOffloader.VersionCheck;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.domain.Blob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class S3BackedReadHandleImpl implements ReadHandle {
-private static final Logger log =
LoggerFactory.getLogger(S3BackedReadHandleImpl.class);
+public class BackedReadHandleImpl implements ReadHandle {
Review comment:
This should be BlobStoreBackedReadHandleImpl. Basically, anywhere you
removed S3 we should have BlobStore.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
ivankelly commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202980509
##
File path:
pulsar-broker/src/main/java/org/apache/pulsar/broker/offload/impl/ManagedLedgerOffloader.java
##
@@ -96,23 +122,66 @@ public static S3ManagedLedgerOffloader
create(ServiceConfiguration conf,
throw new
PulsarServerException("s3ManagedLedgerOffloadMaxBlockSizeInBytes cannot be less
than 5MB");
}
-AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
+return new ManagedLedgerOffloader(driver, bucket, scheduler,
maxBlockSize, readBufferSize, endpoint, region);
+}
+
+// build context for jclouds BlobStoreContext
+ManagedLedgerOffloader(String driver, String container, OrderedScheduler
scheduler,
+ int maxBlockSize, int readBufferSize, String
endpoint, String region) {
+this.scheduler = scheduler;
+this.readBufferSize = readBufferSize;
+
+this.bucket = container;
+this.maxBlockSize = maxBlockSize;
+
+Properties overrides = new Properties();
+// This property controls the number of parts being uploaded in
parallel.
+overrides.setProperty("jclouds.mpu.parallel.degree", "1");
+overrides.setProperty("jclouds.mpu.parts.size",
Integer.toString(maxBlockSize));
+overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
+overrides.setProperty(Constants.PROPERTY_MAX_RETRIES,
Integer.toString(100));
+
+ContextBuilder contextBuilder = ContextBuilder.newBuilder(driver);
+
+AWSCredentials credentials = null;
+try {
+DefaultAWSCredentialsProviderChain creds =
DefaultAWSCredentialsProviderChain.getInstance();
+credentials = creds.getCredentials();
+} catch (Exception e) {
+log.error("Exception when get credentials for s3 ", e);
+}
+
+String id = "accesskey";
Review comment:
default these to ""
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
ivankelly commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202980698
##
File path:
pulsar-broker/src/main/java/org/apache/pulsar/broker/offload/impl/ManagedLedgerOffloader.java
##
@@ -47,37 +40,70 @@
import org.apache.pulsar.broker.offload.OffloadIndexBlock;
import org.apache.pulsar.broker.offload.OffloadIndexBlockBuilder;
import org.apache.pulsar.utils.PulsarBrokerVersionStringUtils;
+import org.jclouds.Constants;
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.BlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.domain.MultipartPart;
+import org.jclouds.blobstore.domain.MultipartUpload;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.domain.Location;
+import org.jclouds.domain.LocationBuilder;
+import org.jclouds.domain.LocationScope;
+import org.jclouds.io.Payload;
+import org.jclouds.io.Payloads;
+import org.jclouds.s3.reference.S3Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class S3ManagedLedgerOffloader implements LedgerOffloader {
-private static final Logger log =
LoggerFactory.getLogger(S3ManagedLedgerOffloader.class);
+public class ManagedLedgerOffloader implements LedgerOffloader {
Review comment:
rename to BlobStoreManagedLedgerOffloader
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] ivankelly commented on a change in pull request #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
ivankelly commented on a change in pull request #2065: GCS offload support(2):
replace `s3client` api with `jclouds` related api
URL: https://github.com/apache/incubator-pulsar/pull/2065#discussion_r202982115
##
File path: pulsar-broker/pom.xml
##
@@ -268,6 +268,12 @@
${project.version}
test
Review comment:
We shouldn't need the full s3 dependency now, just whatever AWSCredentials
is in.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] sijie closed pull request #2101: Migrate compaction and s3 offload test to testcontainers
sijie closed pull request #2101: Migrate compaction and s3 offload test to
testcontainers
URL: https://github.com/apache/incubator-pulsar/pull/2101
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
new file mode 100644
index 00..9efee214c9
--- /dev/null
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.containers;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * S3 simulation container
+ */
+@Slf4j
+public class S3Container extends ChaosContainer {
+
+public static final String NAME = "s3";
+private static final String IMAGE_NAME = "apachepulsar/s3mock:latest";
+private final String hostname;
+
+public S3Container(String clusterName, String hostname) {
+super(clusterName, IMAGE_NAME);
+this.hostname = hostname;
+this.withEnv("initialBuckets", "pulsar-integtest");
+}
+
+@Override
+public String getContainerName() {
+return clusterName + "-" + hostname;
+}
+
+@Override
+public void start() {
+this.withCreateContainerCmdModifier(createContainerCmd -> {
+createContainerCmd.withHostName(hostname);
+createContainerCmd.withName(getContainerName());
+});
+
+super.start();
+log.info("Start s3 service");
+}
+}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
index 040d59dad1..0f0bcffec5 100644
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
@@ -20,6 +20,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.pulsar.tests.containers.PulsarContainer.CS_PORT;
+import static org.apache.pulsar.tests.containers.PulsarContainer.ZK_PORT;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -32,6 +33,7 @@
import java.util.function.Function;
import java.util.stream.Stream;
+import com.google.common.collect.Streams;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.tests.containers.BKContainer;
@@ -53,9 +55,10 @@
public static final String ADMIN_SCRIPT = "/pulsar/bin/pulsar-admin";
public static final String CLIENT_SCRIPT = "/pulsar/bin/pulsar-client";
+public static final String PULSAR_COMMAND_SCRIPT = "/pulsar/bin/pulsar";
/**
- * Pulsar Cluster Spec.
+ * Pulsar Cluster Spec
*
* @param spec pulsar cluster spec.
* @return the built pulsar cluster
@@ -106,6 +109,31 @@ private PulsarCluster(PulsarClusterSpec spec) {
.withEnv("zookeeperServers", ZKContainer.NAME)
.withEnv("configurationStoreServers", CSContainer.NAME + ":" +
CS_PORT)
.withEnv("clusterName", clusterName);
+
+// create bookies
+bookieContainers.putAll(
+runNumContainers("bookie", spec.numBookies(), (name) -> new
BKContainer(clusterName, name)
+.withNetwork(network)
+.withNetworkAliases(name)
+.withEnv("zkServers", ZKContainer.NAME)
+.withEnv("useHostNameAsBookieID", "true")
+.withEnv("clusterName", clusterName)
+)
+);
+
+// create brokers
+brokerContainers.putAll(
+runNumContainers("broker", spec.numBrokers(), (name) -> new
[incubator-pulsar] branch master updated: Migrate compaction, s3offload to test containers (#2101)
This is an automated email from the ASF dual-hosted git repository.
sijie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 7b59248 Migrate compaction, s3offload to test containers (#2101)
7b59248 is described below
commit 7b59248ad87de2c9b96f56c542ff87a87a44e0f2
Author: Ali Ahmed
AuthorDate: Tue Jul 17 02:42:25 2018 -0700
Migrate compaction, s3offload to test containers (#2101)
###Motivation
This is part of the migration effort from arquillian framework to
testcontainers.
---
.../pulsar/tests/containers/S3Container.java | 54 +
.../pulsar/tests/topologies/PulsarCluster.java | 122 +++
.../pulsar/tests/topologies/PulsarClusterSpec.java | 5 +-
.../tests/topologies/PulsarClusterTestBase.java| 61 +++---
tests/integration/compaction/pom.xml | 17 +-
.../pulsar/tests/integration/TestCompaction.java | 140 ++---
.../compaction/src/test/resources/arquillian.xml | 32 ---
tests/integration/s3-offload/pom.xml | 11 +-
.../pulsar/tests/integration/TestS3Offload.java| 228 ++---
.../s3-offload/src/test/resources/arquillian.xml | 32 ---
10 files changed, 364 insertions(+), 338 deletions(-)
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
new file mode 100644
index 000..9efee21
--- /dev/null
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/containers/S3Container.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.tests.containers;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * S3 simulation container
+ */
+@Slf4j
+public class S3Container extends ChaosContainer {
+
+public static final String NAME = "s3";
+private static final String IMAGE_NAME = "apachepulsar/s3mock:latest";
+private final String hostname;
+
+public S3Container(String clusterName, String hostname) {
+super(clusterName, IMAGE_NAME);
+this.hostname = hostname;
+this.withEnv("initialBuckets", "pulsar-integtest");
+}
+
+@Override
+public String getContainerName() {
+return clusterName + "-" + hostname;
+}
+
+@Override
+public void start() {
+this.withCreateContainerCmdModifier(createContainerCmd -> {
+createContainerCmd.withHostName(hostname);
+createContainerCmd.withName(getContainerName());
+});
+
+super.start();
+log.info("Start s3 service");
+}
+}
diff --git
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
index 040d59d..0f0bcff 100644
---
a/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
+++
b/tests/integration-tests-topologies/src/main/java/org/apache/pulsar/tests/topologies/PulsarCluster.java
@@ -20,6 +20,7 @@ package org.apache.pulsar.tests.topologies;
import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.pulsar.tests.containers.PulsarContainer.CS_PORT;
+import static org.apache.pulsar.tests.containers.PulsarContainer.ZK_PORT;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -32,6 +33,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.function.Function;
import java.util.stream.Stream;
+import com.google.common.collect.Streams;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.pulsar.tests.containers.BKContainer;
@@ -53,9 +55,10 @@ public class PulsarCluster {
public static final String ADMIN_SCRIPT = "/pulsar/bin/pulsar-admin";
public static final String CLIENT_SCRIPT = "/pulsar/bin/pulsar-client";
+public static final String PULSAR_COMMAND_SCRIPT = "/pulsar/bin/pulsar";
/**
- *
[GitHub] ivankelly opened a new pull request #2177: Allow construction of c++ builtin auth plugins via factory
ivankelly opened a new pull request #2177: Allow construction of c++ builtin auth plugins via factory URL: https://github.com/apache/incubator-pulsar/pull/2177 Previously, to create the TLS or athenz plugin you had to create the auth plugin explicitly. Some application need to configure auth based on user provided parameters, so they need a factory like method to select the plugin. There was already a factory, but it only took a library path as first parameter. Thus it was impossible to use it with TLS or Athenz. This patch adds handling to the factory, that if "tls", "athenz" or the names of their respective java plugins are passed in, the correct builtin plugin will be created. This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[incubator-pulsar] branch asf-site updated: Updated site at revision 5b72eec
This is an automated email from the ASF dual-hosted git repository. mmerli pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/incubator-pulsar.git The following commit(s) were added to refs/heads/asf-site by this push: new 5d6625d Updated site at revision 5b72eec 5d6625d is described below commit 5d6625d3482df4fa0f88322c2dd4b53f7fbbede9 Author: jenkins AuthorDate: Tue Jul 17 08:43:15 2018 + Updated site at revision 5b72eec --- content/api/client/allclasses-frame.html | 2 -- content/api/client/allclasses-noframe.html | 2 -- content/api/client/index-all.html | 34 +++--- .../apache/pulsar/client/api/MessageBuilder.html | 6 ++-- .../org/apache/pulsar/client/api/PulsarClient.html | 18 ++-- .../apache/pulsar/client/api/ReaderListener.html | 4 +-- .../client/api/SubscriptionInitialPosition.html| 4 +-- .../apache/pulsar/client/api/package-frame.html| 2 -- .../apache/pulsar/client/api/package-summary.html | 12 +--- .../org/apache/pulsar/client/api/package-tree.html | 6 content/api/client/overview-tree.html | 6 content/api/client/serialized-form.html| 5 .../docs/latest/adaptors/PulsarSpark/index.html| 10 +++ .../docs/latest/adaptors/PulsarStorm/index.html| 8 ++--- content/docs/latest/admin-api/overview/index.html | 18 ++-- content/docs/latest/clients/Cpp/index.html | 8 ++--- content/docs/latest/clients/Java/index.html| 16 +- content/docs/latest/clients/Python/index.html | 10 +++ content/docs/latest/clients/WebSocket/index.html | 8 ++--- content/docs/latest/clients/go/index.html | 6 ++-- .../docs/latest/cookbooks/Encryption/index.html| 6 ++-- .../latest/cookbooks/PartitionedTopics/index.html | 16 +- .../latest/cookbooks/RetentionExpiry/index.html| 16 +- .../docs/latest/cookbooks/compaction/index.html| 8 ++--- .../cookbooks/message-deduplication/index.html | 12 .../docs/latest/cookbooks/message-queue/index.html | 16 +- .../latest/cookbooks/tiered-storage/index.html | 6 ++-- .../docs/latest/deployment/Kubernetes/index.html | 4 +-- .../docs/latest/deployment/aws-cluster/index.html | 6 ++-- content/docs/latest/deployment/cluster/index.html | 6 ++-- content/docs/latest/deployment/instance/index.html | 6 ++-- .../ConceptsAndArchitecture/index.html | 4 +-- .../latest/getting-started/LocalCluster/index.html | 4 +-- .../latest/getting-started/Pulsar-2.0/index.html | 2 +- .../docs/latest/getting-started/docker/index.html | 4 +-- .../docs/latest/project/BinaryProtocol/index.html | 4 +-- content/docs/latest/project/CompileCpp/index.html | 8 ++--- .../docs/latest/project/SimulationTools/index.html | 2 +- .../docs/latest/project/schema-storage/index.html | 4 +-- content/docs/latest/reference/CliTools/index.html | 22 +++--- content/docs/latest/reference/RestApi/index.html | 4 +-- .../docs/latest/security/authorization/index.html | 14 - content/docs/latest/security/encryption/index.html | 6 ++-- content/ja/adaptors/PulsarSpark/index.html | 8 ++--- content/ja/adaptors/PulsarStorm/index.html | 6 ++-- content/ja/admin/AdminInterface/index.html | 12 content/ja/admin/Authz/index.html | 12 content/ja/admin/ClustersBrokers/index.html| 6 ++-- content/ja/admin/PropertiesNamespaces/index.html | 6 ++-- content/ja/advanced/PartitionedTopics/index.html | 12 content/ja/advanced/RetentionExpiry/index.html | 12 content/ja/clients/Cpp/index.html | 6 ++-- content/ja/clients/Java/index.html | 8 ++--- content/ja/clients/Python/index.html | 8 ++--- content/ja/clients/WebSocket/index.html| 8 ++--- content/ja/deployment/InstanceSetup/index.html | 6 ++-- content/ja/deployment/Kubernetes/index.html| 4 +-- .../ConceptsAndArchitecture/index.html | 2 +- content/ja/getting-started/LocalCluster/index.html | 4 +-- content/ja/project/BinaryProtocol/index.html | 4 +-- content/ja/project/SimulationTools/index.html | 2 +- content/ja/reference/CliTools/index.html | 18 ++-- content/ja/reference/RestApi/index.html| 4 +-- 63 files changed, 227 insertions(+), 286 deletions(-) diff --git a/content/api/client/allclasses-frame.html b/content/api/client/allclasses-frame.html index 7a93024..82392ca 100644 --- a/content/api/client/allclasses-frame.html +++ b/content/api/client/allclasses-frame.html @@ -74,8 +74,6 @@ ReaderBuilder ReaderConfiguration ReaderListener -Schema -SchemaSerializationException SubscriptionInitialPosition SubscriptionType TopicMetadata diff --git a/content/api/client/allclasses-noframe.html
[GitHub] zhaijack commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api
zhaijack commented on issue #2065: GCS offload support(2): replace `s3client` api with `jclouds` related api URL: https://github.com/apache/incubator-pulsar/pull/2065#issuecomment-405471599 retest this please This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
