jerrypeng closed pull request #2479: Initial SQL documentation and DataGeneratorSource URL: https://github.com/apache/incubator-pulsar/pull/2479
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/bin/pulsar b/bin/pulsar index 80f2d3da87..c2def39863 100755 --- a/bin/pulsar +++ b/bin/pulsar @@ -321,7 +321,7 @@ elif [ $COMMAND == "compact-topic" ]; then elif [ $COMMAND == "sql" ]; then exec $JAVA -cp "${PRESTO_HOME}/lib/*" com.facebook.presto.cli.Presto --server localhost:8081 $@ elif [ $COMMAND == "sql-worker" ]; then - exec ${PRESTO_HOME}/bin/launcher --etc-dir ${PULSAR_PRESTO_CONF} run $@ + exec ${PRESTO_HOME}/bin/launcher --etc-dir ${PULSAR_PRESTO_CONF} $@ elif [ $COMMAND == "help" ]; then pulsar_help; else diff --git a/conf/presto/catalog/pulsar.properties b/conf/presto/catalog/pulsar.properties index 91919e0204..23b945e3b3 100644 --- a/conf/presto/catalog/pulsar.properties +++ b/conf/presto/catalog/pulsar.properties @@ -17,8 +17,13 @@ # under the License. # +# name of the connector to be displayed in the catalog connector.name=pulsar +# the url of Pulsar broker service pulsar.broker-service-url=http://localhost:8080 +# URI of Zookeeper cluster pulsar.zookeeper-uri=localhost:2181 +# minimum number of entries to read at a single time pulsar.entry-read-batch-size=100 +# default number of splits to use per query pulsar.target-num-splits=4 diff --git a/distribution/io/src/assemble/io.xml b/distribution/io/src/assemble/io.xml index 28f9ad6bbb..8cf7fce208 100644 --- a/distribution/io/src/assemble/io.xml +++ b/distribution/io/src/assemble/io.xml @@ -74,5 +74,10 @@ <outputDirectory>connectors</outputDirectory> <fileMode>644</fileMode> </file> + <file> + <source>${basedir}/../../pulsar-io/data-genenator/target/pulsar-io-data-generator-${project.version}.nar</source> + <outputDirectory>connectors</outputDirectory> + <fileMode>644</fileMode> + </file> </files> </assembly> diff --git a/pulsar-io/data-genenator/pom.xml b/pulsar-io/data-genenator/pom.xml new file mode 100644 index 0000000000..f01a80d50d --- /dev/null +++ b/pulsar-io/data-genenator/pom.xml @@ -0,0 +1,57 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.pulsar</groupId> + <artifactId>pulsar-io</artifactId> + <version>2.2.0-incubating-SNAPSHOT</version> + </parent> + + <artifactId>pulsar-io-data-generator</artifactId> + <name>Pulsar IO :: Data Generator</name> + + <dependencies> + + <dependency> + <groupId>${project.groupId}</groupId> + <artifactId>pulsar-io-core</artifactId> + <version>${project.version}</version> + </dependency> + + <dependency> + <groupId>io.codearte.jfairy</groupId> + <artifactId>jfairy</artifactId> + <version>0.5.9</version> + </dependency> + + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-nar-maven-plugin</artifactId> + </plugin> + </plugins> + </build> +</project> diff --git a/pulsar-io/data-genenator/src/main/java/org/apache/pulsar/io/datagenerator/DataGeneratorSource.java b/pulsar-io/data-genenator/src/main/java/org/apache/pulsar/io/datagenerator/DataGeneratorSource.java new file mode 100644 index 0000000000..6087747784 --- /dev/null +++ b/pulsar-io/data-genenator/src/main/java/org/apache/pulsar/io/datagenerator/DataGeneratorSource.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.io.datagenerator; + +import io.codearte.jfairy.Fairy; +import io.codearte.jfairy.producer.person.Person; +import org.apache.pulsar.functions.api.Record; +import org.apache.pulsar.io.core.Source; +import org.apache.pulsar.io.core.SourceContext; + +import java.util.Map; +import java.util.Optional; + + +public class DataGeneratorSource implements Source<Person> { + + @Override + public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception { + + } + + @Override + public Record<Person> read() throws Exception { + Thread.sleep(50); + Fairy fairy = Fairy.create(); + return new Record<Person>() { + @Override + public Optional<String> getKey() { + return Optional.empty(); + } + + @Override + public Person getValue() { + return fairy.person(); + } + }; + } + + @Override + public void close() throws Exception { + + } +} diff --git a/pulsar-io/data-genenator/src/main/resources/META-INF/services/pulsar-io.yaml b/pulsar-io/data-genenator/src/main/resources/META-INF/services/pulsar-io.yaml new file mode 100644 index 0000000000..d0fd7a69c7 --- /dev/null +++ b/pulsar-io/data-genenator/src/main/resources/META-INF/services/pulsar-io.yaml @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: data-generator +description: Test data generator source +sourceClass: org.apache.pulsar.io.datagenerator.DataGeneratorSource diff --git a/pulsar-io/pom.xml b/pulsar-io/pom.xml index 67aec42a27..f1494db6e7 100644 --- a/pulsar-io/pom.xml +++ b/pulsar-io/pom.xml @@ -38,6 +38,7 @@ <module>kafka</module> <module>rabbitmq</module> <module>kinesis</module> + <module>data-genenator</module> </modules> </project> diff --git a/site2/docs/assets/pulsar-sql-arch-1.png b/site2/docs/assets/pulsar-sql-arch-1.png new file mode 100755 index 0000000000..0441d343ac Binary files /dev/null and b/site2/docs/assets/pulsar-sql-arch-1.png differ diff --git a/site2/docs/assets/pulsar-sql-arch-2.png b/site2/docs/assets/pulsar-sql-arch-2.png new file mode 100755 index 0000000000..36a136d817 Binary files /dev/null and b/site2/docs/assets/pulsar-sql-arch-2.png differ diff --git a/site2/docs/sql-deployment-configurations.md b/site2/docs/sql-deployment-configurations.md new file mode 100644 index 0000000000..45d44bb214 --- /dev/null +++ b/site2/docs/sql-deployment-configurations.md @@ -0,0 +1,150 @@ +--- +id: sql-deployment-configuration +title: Pulsar SQl Deployment and Configuration +sidebar_label: Deployment and Configuration +--- + +## Presto Pulsar Connector Configurations +There are several configurations for the Presto Pulsar Connector. The properties file that contain these configurations can be found at ```${project.root}/conf/presto/catalog/pulsar.properties```. +The configurations for the connector and its default values are discribed below. + +```properties +# name of the connector to be displayed in the catalog +connector.name=pulsar + +# the url of Pulsar broker service +pulsar.broker-service-url=http://localhost:8080 + +# URI of Zookeeper cluster +pulsar.zookeeper-uri=localhost:2181 + +# minimum number of entries to read at a single time +pulsar.entry-read-batch-size=100 + +# default number of splits to use per query +pulsar.target-num-splits=4 +``` + +## Query Pulsar from Existing Presto Cluster + +If you already have an existing Presto cluster, you can copy Presto Pulsar connector plugin to your existing cluster. You can download the archived plugin package via: + +```bash +$ wget pulsar:binary_release_url +``` + +## Deploying a new cluster + +Please note that the [Getting Started](sql-getting-started.md) guide shows you how to easily setup a standalone single node enviroment to experiment with. + +Pulsar SQL is powered by [Presto](https://prestodb.io) thus many of the configurations for deployment is the same for the Pulsar SQL worker. + +You can use the same CLI args as the Presto launcher: + +```bash +$ ./bin/pulsar sql-worker --help +Usage: launcher [options] command + +Commands: run, start, stop, restart, kill, status + +Options: + -h, --help show this help message and exit + -v, --verbose Run verbosely + --etc-dir=DIR Defaults to INSTALL_PATH/etc + --launcher-config=FILE + Defaults to INSTALL_PATH/bin/launcher.properties + --node-config=FILE Defaults to ETC_DIR/node.properties + --jvm-config=FILE Defaults to ETC_DIR/jvm.config + --config=FILE Defaults to ETC_DIR/config.properties + --log-levels-file=FILE + Defaults to ETC_DIR/log.properties + --data-dir=DIR Defaults to INSTALL_PATH + --pid-file=FILE Defaults to DATA_DIR/var/run/launcher.pid + --launcher-log-file=FILE + Defaults to DATA_DIR/var/log/launcher.log (only in + daemon mode) + --server-log-file=FILE + Defaults to DATA_DIR/var/log/server.log (only in + daemon mode) + -D NAME=VALUE Set a Java system property + +``` + +There is a set of default configs for the cluster located in ```${project.root}/conf/presto``` that will be used by default. You can change them to customize your deployment + +You can also set the worker to read from a different configuration directory as well as set a different directory for writing its data: + +```bash +$ ./bin/pulsar sql-worker run --etc-dir /tmp/incubator-pulsar/conf/presto --data-dir /tmp/presto-1 +``` + +You can also start the worker as daemon process: + +```bash +$ ./bin sql-worker start +``` + +### Deploying to a 3 node cluster + +For example, if I wanted to deploy a Pulsar SQL/Presto cluster on 3 nodes, you can do the following: + +First, copy the Pulsar binary distribution to all three nodes. + +The first node, will run the Presto coordinator. The mininal configuration in ```${project.root}/conf/presto/config.properties``` can be the following + +```properties +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +discovery-server.enabled=true +discovery.uri=<coordinator-url> +``` + +Also, modify ```pulsar.broker-service-url``` and ```pulsar.zookeeper-uri``` configs in ```${project.root}/conf/presto/catalog/pulsar.properties``` on those nodes accordingly + +Afterwards, you can start the coordinator by just running + +```$ ./bin/pulsar sql-worker run``` + +For the other two nodes that will only serve as worker nodes, the configurations can be the following: + +```properties +coordinator=false +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +discovery.uri=<coordinator-url> + +``` + +Also, modify ```pulsar.broker-service-url``` and ```pulsar.zookeeper-uri``` configs in ```${project.root}/conf/presto/catalog/pulsar.properties``` accordingly + +You can also start the worker by just running: + +```$ ./bin/pulsar sql-worker run``` + +You can check the status of your cluster from the SQL CLI. To start the SQL CLI: + +```bash +$ ./bin/pulsar sql --server <coordinate_url> + +``` + +You can then run the following command to check the status of your nodes: + +```bash +presto> SELECT * FROM system.runtime.nodes; + node_id | http_uri | node_version | coordinator | state +---------+-------------------------+--------------+-------------+-------- + 1 | http://192.168.2.1:8081 | testversion | true | active + 3 | http://192.168.2.2:8081 | testversion | false | active + 2 | http://192.168.2.3:8081 | testversion | false | active +``` + + +For more information about deployment in Presto, please reference: + +[Deploying Presto](https://prestodb.io/docs/current/installation/deployment.html) + diff --git a/site2/docs/sql-getting-started.md b/site2/docs/sql-getting-started.md new file mode 100644 index 0000000000..cbadbedef1 --- /dev/null +++ b/site2/docs/sql-getting-started.md @@ -0,0 +1,142 @@ +--- +id: sql-getting-started +title: Pulsar SQL Overview +sidebar_label: Overview +--- + +It is super easy to start query data in Pulsar. + +## Requirements +1. **Pulsar distribution** + * If you haven't install Pulsar, please reference [Installing Pulsar](io-quickstart.md#installing-pulsar) +2. **Pulsar built-in connectors** + * If you haven't installed the built-in connectors, please reference [Installing Builtin Connectors](io-quickstart.md#installing-builtin-connectors) + +First, start a Pulsar standalone cluster: + +```bash +./bin/pulsar standalone +``` + +Next, start a Pulsar SQL worker: +```bash +./bin/pulsar sql-worker run +``` + +After both the Pulsar standalone cluster and the SQL worker are done initializing, run the SQL CLI: +```bash +./bin/pulsar sql +``` + +You can now start typing some SQL commands: + + +```bash +presto> show catalogs; + Catalog +--------- + pulsar + system +(2 rows) + +Query 20180829_211752_00004_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [0 rows, 0B] [0 rows/s, 0B/s] + + +presto> show schemas in pulsar; + Schema +----------------------- + information_schema + public/default + public/functions + sample/standalone/ns1 +(4 rows) + +Query 20180829_211818_00005_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [4 rows, 89B] [21 rows/s, 471B/s] + + +presto> show tables in pulsar."public/default"; + Table +------- +(0 rows) + +Query 20180829_211839_00006_7qpwh, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:00 [0 rows, 0B] [0 rows/s, 0B/s] + +``` + +Currently, there is no data in Pulsar that we can query. Lets start the built-in connector _DataGeneratorSource_ to ingest some mock data for us to query: + +```bash +./bin/pulsar-admin source create --tenant test-tenant --namespace test-namespace --name generator --destinationTopicName generator_test --source-type data-generator +``` + +Afterwards, the will be a topic with can query in the namespace "public/default": + +```bash +presto> show tables in pulsar."public/default"; + Table +---------------- + generator_test +(1 row) + +Query 20180829_213202_00000_csyeu, FINISHED, 1 node +Splits: 19 total, 19 done (100.00%) +0:02 [1 rows, 38B] [0 rows/s, 17B/s] +``` + +We can now query the data within the topic "generator_test": + +```bash +presto> select * from pulsar."public/default".generator_test; + + firstname | middlename | lastname | email | username | password | telephonenumber | age | companyemail | nationalidentitycardnumber | +-------------+-------------+-------------+----------------------------------+--------------+----------+-----------------+-----+-----------------------------------------------+----------------------------+ + Genesis | Katherine | Wiley | genesis.wi...@gmail.com | genesisw | y9D2dtU3 | 959-197-1860 | 71 | genesis.wi...@interdemconsulting.eu | 880-58-9247 | + Brayden | | Stanton | brayden.stan...@yahoo.com | braydens | ZnjmhXik | 220-027-867 | 81 | brayden.stan...@supermemo.eu | 604-60-7069 | + Benjamin | Julian | Velasquez | benjamin.velasq...@yahoo.com | benjaminv | 8Bc7m3eb | 298-377-0062 | 21 | benjamin.velasq...@hostesltd.biz | 213-32-5882 | + Michael | Thomas | Donovan | dono...@mail.com | michaeld | OqBm9MLs | 078-134-4685 | 55 | michael.dono...@memortech.eu | 443-30-3442 | + Brooklyn | Avery | Roach | brooklynro...@yahoo.com | broach | IxtBLafO | 387-786-2998 | 68 | brooklyn.ro...@warst.biz | 085-88-3973 | + Skylar | | Bradshaw | skylarbrads...@yahoo.com | skylarb | p6eC6cKy | 210-872-608 | 96 | skylar.brads...@flyhigh.eu | 453-46-0334 | +. +. +. +``` + +Now, you have some mock data to query and play around with! + +If you want to try to ingest some of your own data to play around with, you can write a simple producer to write custom defined data to Pulsar. + +For example: + +```java +public class Test { + + public static class Foo { + private int field1 = 1; + private String field2; + private long field3; + } + + public static void main(String[] args) throws Exception { + PulsarClient pulsarClient = PulsarClient.builder().serviceUrl("pulsar://localhost:6650").build(); + Producer<Foo> producer = pulsarClient.newProducer(AvroSchema.of(Foo.class)).topic("test_topic").create(); + + for (int i = 0; i < 1000; i++) { + Foo foo = new Foo(); + foo.setField1(i); + foo.setField2("foo" + i); + foo.setField3(System.currentTimeMillis()); + producer.newMessage().value(foo).send(); + } + producer.close(); + pulsarClient.close(); + } +} +``` + +Afterwards, you should be able query the data you just wrote. diff --git a/site2/docs/sql-overview.md b/site2/docs/sql-overview.md new file mode 100644 index 0000000000..1df9533e87 --- /dev/null +++ b/site2/docs/sql-overview.md @@ -0,0 +1,24 @@ +--- +id: sql-overview +title: Pulsar SQL Overview +sidebar_label: Overview +--- + +One of the common use cases of Pulsar is storing streams of event data. Often the event data is structured which predefined fields. There is tremendous value for users to be able to query the existing data that is already stored in Pulsar topics. With the implementation of the [Schema Registry](concepts-schema-registry.md), structured data can be stored in Pulsar and allows for the potential to query that data via SQL language. + +By leveraging [Presto](https://prestodb.io/), we have created a method for users to be able to query structured data stored within Pulsar in a very efficient and scalable manner. We will discuss why this very efficient and scalable in the [Performance](#performance) section below. + +At the core of this Pulsar SQL is the Presto Pulsar connector which allows Presto workers within a Presto cluster to query data from Pulsar. + + + + + +## Performance + +The reason why query performance is very efficient and highly scalable because of Pulsar's [two level segment based architecture](concepts-architecture-overview.md#apache-bookkeeper). + +Topics in Pulsar are stored as segments in [Apache Bookkeeper](https://bookkeeper.apache.org/). Each topic segment is also replicated to a configurable (default 3) number of Bookkeeper nodes which allows for concurrent reads and high read throughput. In the Presto Pulsar connector, we read data directly from Bookkeeper to take advantage of the Pulsar's segment based architecture. Thus, Presto workers can read concurrently from horizontally scalable number bookkeeper nodes. + + + diff --git a/site2/website/scripts/replace.js b/site2/website/scripts/replace.js index a46f0aa0e5..d0cc0fcc56 100644 --- a/site2/website/scripts/replace.js +++ b/site2/website/scripts/replace.js @@ -29,6 +29,10 @@ function connectorReleaseUrl(version) { return `https://archive.apache.org/dist/incubator/pulsar/pulsar-${version}/apache-pulsar-io-connectors-${version}-bin.tar.gz` } +function prestoPulsarReleaseUrl(version) { + return `https://archive.apache.org/dist/incubator/pulsar/pulsar-${version}/pulsar-presto-connector-${version}.tar.gz` +} + function rpmReleaseUrl(version, type) { rpmVersion = version.replace('incubating', '1_incubating'); return `https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=incubator/pulsar/pulsar-${version}/RPMS/apache-pulsar-client${type}-${rpmVersion}.x86_64.rpm` @@ -62,6 +66,7 @@ const from = [ /{{pulsar:version}}/g, /pulsar:binary_release_url/g, /pulsar:connector_release_url/g, + /pulsar:presto_pulsar_connector_release_url/g, /pulsar:download_page_url/g, /{{pulsar:rpm:client}}/g, /{{pulsar:rpm:client-debuginfo}}/g, @@ -81,7 +86,8 @@ const options = { `${latestVersionWithoutIncubating}`, `${latestVersion}`, binaryReleaseUrl(`${latestVersion}`), - connectorReleaseUrl(`${latestVersion}`), + connectorReleaseUrl(`${latestVersion}`), + prestoPulsarReleaseUrl(`${latestVersion}`) downloadPageUrl(), rpmReleaseUrl(`${latestVersion}`, ""), rpmReleaseUrl(`${latestVersion}`, "-debuginfo"), @@ -112,6 +118,7 @@ for (v of versions) { `${v}`, binaryReleaseUrl(`${v}`), connectorReleaseUrl(`${v}`), + prestoPulsarReleaseUrl(`${latestVersion}`) downloadPageUrl(), rpmReleaseUrl(`${v}`, ""), rpmReleaseUrl(`${v}`, "-debuginfo"), diff --git a/site2/website/sidebars.json b/site2/website/sidebars.json index f321d30b43..5afda90a11 100644 --- a/site2/website/sidebars.json +++ b/site2/website/sidebars.json @@ -34,6 +34,11 @@ "io-connectors", "io-develop" ], + "Pulsar SQL": [ + "sql-overview", + "sql-getting-started", + "sql-deployment-configurations" + ], "Deployment": [ "deploy-aws", "deploy-kubernetes", ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services