jerrypeng closed pull request #1749: [WIP] Pulsar IO documentation
URL: https://github.com/apache/incubator-pulsar/pull/1749
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
index 04e73bffb7..63dac5a619 100644
---
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
+++
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSinks.java
@@ -59,7 +59,7 @@
@Slf4j
@Getter
-@Parameters(commandDescription = "Interface for managing Pulsar Sinks (Egress
data from Pulsar)")
+@Parameters(commandDescription = "Interface for managing Pulsar IO sinks
(egress data from Pulsar)")
public class CmdSinks extends CmdBase {
private final CreateSink createSink;
@@ -94,7 +94,7 @@ void processArguments() throws Exception {
abstract void runCmd() throws Exception;
}
- @Parameters(commandDescription = "Run the Pulsar sink locally (rather than
deploying it to the Pulsar cluster)")
+ @Parameters(commandDescription = "Run the Pulsar IO sink locally (rather
than deploying it to the Pulsar cluster)")
class LocalSinkRunner extends CreateSink {
@Parameter(names = "--brokerServiceUrl", description = "The URL for
the Pulsar broker")
@@ -107,7 +107,7 @@ void runCmd() throws Exception {
}
}
- @Parameters(commandDescription = "Create Pulsar sink connectors")
+ @Parameters(commandDescription = "Create a Pulsar IO sink connector")
class CreateSink extends BaseCommand {
@Parameter(names = "--tenant", description = "The sink's tenant")
protected String tenant;
@@ -121,7 +121,7 @@ void runCmd() throws Exception {
protected String inputs;
@Parameter(names = "--customSerdeInputs", description = "The map of
input topics to SerDe class names (as a JSON string)")
protected String customSerdeInputString;
- @Parameter(names = "--processingGuarantees", description = "The
processing guarantees (aka delivery semantics) applied to the Sink")
+ @Parameter(names = "--processingGuarantees", description = "The
processing guarantees (aka delivery semantics) applied to the sink")
protected FunctionConfig.ProcessingGuarantees processingGuarantees;
@Parameter(names = "--parallelism", description = "")
protected String parallelism;
@@ -134,11 +134,11 @@ void runCmd() throws Exception {
@Parameter(names = "--sinkConfigFile", description = "The path to a
YAML config file specifying the "
+ "sink's configuration")
protected String sinkConfigFile;
- @Parameter(names = "--cpu", description = "The cpu in cores that need
to be allocated per function instance(applicable only to docker runtime)")
+ @Parameter(names = "--cpu", description = "The CPU (in cores) that
needs to be allocated per sink instance (applicable only to the Docker
runtime)")
protected Double cpu;
- @Parameter(names = "--ram", description = "The ram in bytes that need
to be allocated per function instance(applicable only to process/docker
runtime)")
+ @Parameter(names = "--ram", description = "The RAM (in bytes) that
needs to be allocated per sink instance (applicable only to the process and
Docker runtimes)")
protected Long ram;
- @Parameter(names = "--disk", description = "The disk in bytes that
need to be allocated per function instance(applicable only to docker runtime)")
+ @Parameter(names = "--disk", description = "The disk (in bytes) that
need to be allocated per sink instance (applicable only to Docker runtime)")
protected Long disk;
@Parameter(names = "--sinkConfig", description = "Sink config
key/values")
protected String sinkConfigString;
@@ -320,7 +320,7 @@ protected FunctionDetails createSinkConfig(SinkConfig
sinkConfig) {
}
}
- @Parameters(commandDescription = "Stops a Pulsar sink or source")
+ @Parameters(commandDescription = "Stops a Pulsar IO sink")
class DeleteSink extends BaseCommand {
@Parameter(names = "--tenant", description = "The tenant of the sink")
diff --git
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
index 1583589055..6e12a8b995 100644
---
a/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
+++
b/pulsar-client-tools/src/main/java/org/apache/pulsar/admin/cli/CmdSources.java
@@ -59,7 +59,7 @@
@Slf4j
@Getter
-@Parameters(commandDescription = "Interface for managing Pulsar Source
(Ingress data to Pulsar)")
+@Parameters(commandDescription = "Interface for managing Pulsar IO sources
(ingress data into Pulsar)")
public class CmdSources extends CmdBase {
private final CreateSource createSource;
@@ -107,7 +107,7 @@ void runCmd() throws Exception {
}
}
- @Parameters(commandDescription = "Create Pulsar source connectors")
+ @Parameters(commandDescription = "Create a Pulsar source connector")
class CreateSource extends BaseCommand {
@Parameter(names = "--tenant", description = "The source's tenant")
protected String tenant;
@@ -119,9 +119,9 @@ void runCmd() throws Exception {
protected FunctionConfig.ProcessingGuarantees processingGuarantees;
@Parameter(names = "--className", description = "The source's class
name")
protected String className;
- @Parameter(names = "--destinationTopicName", description = "Pulsar
topic to ingress data to")
+ @Parameter(names = "--destinationTopicName", description = "The Pulsar
topic to which data is sent")
protected String destinationTopicName;
- @Parameter(names = "--deserializationClassName", description = "The
classname for SerDe class for the source")
+ @Parameter(names = "--deserializationClassName", description = "The
SerDe classname for the source")
protected String deserializationClassName;
@Parameter(names = "--parallelism", description = "Number of instances
of the source")
protected String parallelism;
diff --git a/site/_data/cli/pulsar-admin.yaml b/site/_data/cli/pulsar-admin.yaml
index 21806199f9..1263f530ea 100644
--- a/site/_data/cli/pulsar-admin.yaml
+++ b/site/_data/cli/pulsar-admin.yaml
@@ -477,6 +477,164 @@ commands:
argument: cluster-name
- name: delete
description: Delete namespace isolation policy of a cluster. This
operation requires superuser privileges.
+- name: sink
+ description: An interface for managing Pulsar IO sinks (egree data from
Pulsar)
+ subcommands:
+ - name: create
+ description: Create a Pulsar IO sink connector
+ options:
+ - flags: --className
+ description: The sink's Java class name
+ - flags: --cpu
+ description: The CPU (in cores) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --customSerdeInputs
+ description: The map of input topics to SerDe class names (as a JSON
string)
+ - flags: --disk
+ description: The disk (in bytes) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --inputs
+ description: The sink's input topic(s) (multiple topics can be specified
as a comma-separated list)
+ - flags: --jar
+ description: Path to the Java jar file for the sink
+ - flags: --name
+ description: The sink's name
+ - flags: --namespace
+ description: The sink's namespace
+ - flags: --parallelism
+ description: |
+ "The sink's parallelism factor (i.e. the number of sink instances to
run)."
+ - flags: --processingGuarantees
+ description: |
+ "The processing guarantees (aka delivery semantics) applied to the
sink. Available values: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`."
+ - flags: --ram
+ description: The RAM (in bytes) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --sinkConfig
+ description: Sink config key/values
+ - flags: --sinkConfigFile
+ description: The path to a YAML config file specifying the sink's
configuration
+ - flags: --tenant
+ description: The sink's tenant
+ - name: delete
+ description: Stops a Pulsar IO sink
+ options:
+ - flags: --name
+ description: The name of the sink
+ - flags: --namespace
+ description: The namespace of the sink
+ - flags: --tenant
+ description: The tenant of the sink
+ - name: localrun
+ description: Run the Pulsar sink locally (rather than in the Pulsar
cluster)
+ options:
+ - flags: --brokerServiceUrl
+ description: The URL for the Pulsar broker
+ - flags: --className
+ description: The sink's Java class name
+ - flags: --cpu
+ description: The CPU (in cores) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --customSerdeInputs
+ description: The map of input topics to SerDe class names (as a JSON
string)
+ - flags: --disk
+ description: The disk (in bytes) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --inputs
+ description: The sink's input topic(s) (multiple topics can be specified
as a comma-separated list)
+ - flags: --jar
+ description: Path to the Java jar file for the sink
+ - flags: --name
+ description: The sink's name
+ - flags: --namespace
+ description: The sink's namespace
+ - flags: --parallelism
+ description: |
+ "The sink's parallelism factor (i.e. the number of sink instances to
run)."
+ - flags: --processingGuarantees
+ description: |
+ "The processing guarantees (aka delivery semantics) applied to the
sink. Available values: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`."
+ - flags: --ram
+ description: The RAM (in bytes) that needs to be allocated per sink
instance (applicable only to the Docker runtime)
+ - flags: --sinkConfig
+ description: Sink config key/values
+ - flags: --sinkConfigFile
+ description: The path to a YAML config file specifying the sink's
configuration
+ - flags: --tenant
+ description: The sink's tenant
+- name: source
+ description: An interface for managing Pulsar IO sources (ingress data into
Pulsar)
+ subcommands:
+ - name: create
+ description: Create a Pulsar IO source connector
+ options:
+ - flags: --className
+ description: The source's Java class name
+ - flags: --cpu
+ description: The CPU (in cores) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --deserializationClassName
+ description: The SerDe classname for the source
+ - flags: --destinationTopicName
+ description: The Pulsar topic to which data is sent
+ - flags: --disk
+ description: The disk (in bytes) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --jar
+ description: Path to the Java jar file for the source
+ - flags: --name
+ description: The source's name
+ - flags: --namespace
+ description: The source's namespace
+ - flags: --parallelism
+ description: |
+ The source's parallelism factor (i.e. the number of source instances
to run).
+ - flags: --processingGuarantees
+ description: |
+ "The processing guarantees (aka delivery semantics) applied to the
source. Available values: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`."
+ - flags: --ram
+ description: The RAM (in bytes) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --sourceConfig
+ description: Source config key/values
+ - flags: --sourceConfigFile
+ description: The path to a YAML config file specifying the source's
configuration
+ - flags: --tenant
+ description: The source's tenant
+ - name: delete
+ description: Stops a Pulsar IO source
+ options:
+ - flags: --name
+ description: The name of the source
+ - flags: --namespace
+ description: The namespace of the source
+ - flags: --tenant
+ description: The tenant of the source
+ - name: localrun
+ description: Run the Pulsar source locally (rather than in the Pulsar
cluster)
+ options:
+ - flags: --className
+ description: The source's Java class name
+ - flags: --cpu
+ description: The CPU (in cores) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --deserializationClassName
+ description: The SerDe classname for the source
+ - flags: --destinationTopicName
+ description: The Pulsar topic to which data is sent
+ - flags: --disk
+ description: The disk (in bytes) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --jar
+ description: Path to the Java jar file for the source
+ - flags: --name
+ description: The source's name
+ - flags: --namespace
+ description: The source's namespace
+ - flags: --parallelism
+ description: |
+ The source's parallelism factor (i.e. the number of source instances
to run).
+ - flags: --processingGuarantees
+ description: |
+ "The processing guarantees (aka delivery semantics) applied to the
source. Available values: `ATLEAST_ONCE`, `ATMOST_ONCE`, `EFFECTIVELY_ONCE`."
+ - flags: --ram
+ description: The RAM (in bytes) that needs to be allocated per source
instance (applicable only to the Docker runtime)
+ - flags: --sourceConfig
+ description: Source config key/values
+ - flags: --sourceConfigFile
+ description: The path to a YAML config file specifying the source's
configuration
+ - flags: --tenant
+ description: The source's tenant
- name: topics
description: Operations for managing Pulsar topics (both persistent and non
persistent)
subcommands:
diff --git a/site/_data/connectors.yaml b/site/_data/connectors.yaml
new file mode 100644
index 0000000000..1b4789f15a
--- /dev/null
+++ b/site/_data/connectors.yaml
@@ -0,0 +1,30 @@
+- name: Aerospike sink
+ url: https://www.aerospike.com/
+ class:
+ group: aerospike
+ name: AerospikeSink
+- name: Cassandra sink
+ url: https://cassandra.apache.org
+ class:
+ group: cassandra
+ name: CassandraSink
+- name: Kafka source
+ url: https://kafka.apache.org
+ class:
+ group: kafka
+ name: KafkaSource
+- name: Kafka sink
+ url: https://kafka.apache.org
+ class:
+ group: kafka
+ name: KafkaSink
+- name: RabbitMQ source
+ url: https://www.rabbitmq.com
+ class:
+ group: rabbitmq
+ name: RabbitMQSource
+- name: Twitter Firehose source
+ url: https://developer.twitter.com/en/docs
+ class:
+ group: twitter
+ name: TwitterFireHose
\ No newline at end of file
diff --git a/site/_data/sidebar.yaml b/site/_data/sidebar.yaml
index 04d68aa039..27b186933d 100644
--- a/site/_data/sidebar.yaml
+++ b/site/_data/sidebar.yaml
@@ -134,6 +134,8 @@ groups:
- title: Cookbooks
dir: cookbooks
docs:
+ - title: Pulsar IO
+ endpoint: pulsar-io
- title: Message deduplication
endpoint: message-deduplication
- title: Non-persistent messaging
diff --git a/site/_includes/connectors.html b/site/_includes/connectors.html
new file mode 100644
index 0000000000..8b881051e9
--- /dev/null
+++ b/site/_includes/connectors.html
@@ -0,0 +1,24 @@
+<table>
+ <thead>
+ <tr>
+ <th>Name</th>
+ <th>Java class</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for connector in site.data.connectors %}
+ <tr>
+ <td>
+ <a href="{{ connector.url }}">
+ {{ connector.name }}
+ </a>
+ </td>
+ <td>
+ <a
href="https://github.com/apache/incubator-pulsar/blob/master/pulsar-io/{{
connector.class.group }}/src/main/java/org/apache/pulsar/connect/{{
connector.class.group }}/{{ connector.class.name }}.java">
+ <code>org.apache.pulsar.io.{{ connector.class.group }}.{{
connector.class.name }}</code>
+ </a>
+ </td>
+ </tr>
+ {% endfor %}
+ </tbody>
+</table>
\ No newline at end of file
diff --git a/site/_includes/figure.html b/site/_includes/figure.html
index cfda529ec7..600857dea4 100644
--- a/site/_includes/figure.html
+++ b/site/_includes/figure.html
@@ -18,4 +18,11 @@
under the License.
-->
-<img src="{{ include.src }}"{{ if include.width }} width="{{ include.width
}}%"{{ endif }}{{ if include.alt }} alt="{{ include.alt }}"{{ endif }}>
\ No newline at end of file
+<figure>
+ <img src="{{ include.src }}"{% if include.width %} width="{{ include.width
}}%"{% endif %}{% if include.alt %} alt="{{ include.alt }}"{% endif %}>
+ {% if include.caption %}
+ <figcaption>
+ {{ include.caption }}
+ </figcaption>
+ {% endif %}
+</figure>
\ No newline at end of file
diff --git a/site/_sass/_docs.scss b/site/_sass/_docs.scss
index f5ee27867e..5144b14190 100644
--- a/site/_sass/_docs.scss
+++ b/site/_sass/_docs.scss
@@ -162,6 +162,23 @@
margin-right: 2px;
}
+ figure {
+ margin: 2rem 0;
+
+ img {
+ display: block;
+ max-width: 100%;
+ margin: 0 auto;
+ }
+
+ figcaption {
+ text-align: center;
+ margin-top: 1rem;
+ font-size: 1.5rem;
+ font-weight: 500;
+ }
+ }
+
p img {
display: block;
margin: 20px auto;
diff --git a/site/docs/latest/cookbooks/pulsar-io.md
b/site/docs/latest/cookbooks/pulsar-io.md
new file mode 100644
index 0000000000..7bbce5b18c
--- /dev/null
+++ b/site/docs/latest/cookbooks/pulsar-io.md
@@ -0,0 +1,51 @@
+---
+title: The Pulsar IO cookbook
+---
+
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+[Pulsar IO](../../getting-started/ConceptsAndArchitecture#pulsar-io) is a
feature of Pulsar that enables you to easily create and manage **connectors**
that interface with external systems, such as databases and other messaging
systems.
+
+## Setup
+
+In order to run Pulsar IO connectors, you'll need to install a source
distribution.
+
+## Managing connectors
+
+Pulsar connectors can be managed using the
[`source`](../../reference/CliTools#pulsar-admin-source) and
[`sink`](../../reference/CliTools#pulsar-admin-sink) commands of the
[`pulsar-admin`](../../reference/CliTools#pulsar-admin) CLI tool.
+
+### Running sources
+
+You can use the [`create`](../../reference/CliTools#pulsar-admin-source-create)
+
+This command, for example, would create a new connector in a Pulsar cluster:
+
+```bash
+$ bin/pulsar-admin source create \
+ --name
+```
+
+## Available connectors
+
+At the moment, the following connectors are available for Pulsar:
+
+{% include connectors.html %}
\ No newline at end of file
diff --git a/site/docs/latest/getting-started/ConceptsAndArchitecture.md
b/site/docs/latest/getting-started/ConceptsAndArchitecture.md
index 2965f072d1..b895c59697 100644
--- a/site/docs/latest/getting-started/ConceptsAndArchitecture.md
+++ b/site/docs/latest/getting-started/ConceptsAndArchitecture.md
@@ -534,6 +534,34 @@ MessageId id = MessageId.fromByteArray(msgIdBytes);
Reader reader = pulsarClient.createReader(topic, id, new
ReaderConfiguration());
```
+## Pulsar IO
+
+Messaging systems are most powerful when you can easily use them in
conjunction with external systems like databases and other messaging systems.
**Pulsar IO** is a feature of Pulsar that enables you to easily create, deploy,
and manage Pulsar **connectors** that interact with external systems, such as
[Apache Cassandra](https://cassandra.apache.org),
[Aerospike](https://www.aerospike.com), and many others.
+
+{% include admonition.html type="info" title="Pulsar IO and Pulsar Functions"
+ content="Under the hood, Pulsar IO connectors are specialized [Pulsar
Functions](#pulsar-functions) purpose-built to interface with external systems.
The [administrative interface](../../cookbooks/pulsar-io) for Pulsar IO is, in
fact, quite similar to that of Pulsar Functions." %}
+
+### Sources and sinks
+
+Pulsar IO connectors come in two types:
+
+* **Sources** feed data *into* Pulsar from other systems. Common sources
include other messaging systems and "firehose"-style data pipeline APIs.
+* **Sinks** are fed data *from* Pulsar. Common sinks include other messaging
systems and SQL and NoSQL databases.
+
+This diagram illustrates the relationship between sources, sinks, and Pulsar:
+
+{% include figure.html src="/img/pulsar-io.png" alt="Pulsar IO diagram"
caption="Pulsar IO connectors (sources and sinks)" width="80" %}
+
+### Working with connectors
+
+Pulsar IO connectors can be managed via the
[`pulsar-admin`](../../reference/CliTools#pulsar-admin) CLI tool, in particular
the [`source`](../../reference/CliTools#pulsar-admin-source) and
[`sink`](../../reference/CliTools#pulsar-admin-sink) commands.
+
+{% include admonition.html type="info" content="For a guide to managing
connectors in your Pulsar installation, see the [Pulsar IO
cookbook](../../cookbooks/pulsar-io#managing-connectors)." %}
+
+The following connectors are currently available for Pulsar:
+
+{% include connectors.html %}
+
## Schema registry
Type safety is extremely important in any application built around a message
bus like Pulsar. {% popover Producers %} and {% popover consumers %} need some
kind of mechanism for coordinating types at the {% popover topic %} level lest
a wide variety of potential problems arise (for example serialization and
deserialization issues). Applications typically adopt one of two basic
approaches to type safety in messaging:
@@ -603,4 +631,4 @@ For usage instructions, see the documentation for your
preferred client library:
* [Java](../../clients/Java#schemas)
-{% include admonition.html type="success" content="Support for other schema
formats will be added in future releases of Pulsar." %}
\ No newline at end of file
+{% include admonition.html type="success" content="Support for other schema
formats will be added in future releases of Pulsar." %}
diff --git a/site/img/pulsar-io.png b/site/img/pulsar-io.png
new file mode 100644
index 0000000000..3e74d4bab7
Binary files /dev/null and b/site/img/pulsar-io.png differ
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
