This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch docker-quickstart
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 5941b13466f0efdd77b685cb5742abd3ba216747
Author: Xiang Fu <fx19880...@gmail.com>
AuthorDate: Sun Sep 1 18:45:06 2019 -0700
Use docker compose for pinot quickstart
---
docker/Dockerfile | 5 +-
docker/README.md | 27 ++++++-
docker/docker-build.sh | 11 ++-
docker/docker-compose.yml | 82 ++++++++++++++++++++++
.../docker/airlineStats_offline_table_config.json | 21 ++++++
.../docker/airlineStats_realtime_table_config.json | 38 ++++++++++
6 files changed, 179 insertions(+), 5 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 86bb47c..63fd097 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -22,8 +22,9 @@ FROM openjdk:8
LABEL MAINTAINER=d...@pinot.apache.org
ARG PINOT_BRANCH=master
+ARG KAFKA_VERSION=0.9
ARG PINOT_GIT_URL="https://github.com/apache/incubator-pinot.git";
-RUN echo "Trying to build Pinot from [ ${PINOT_GIT_URL} ] on branch [
${PINOT_BRANCH} ]"
+RUN echo "Trying to build Pinot from [ ${PINOT_GIT_URL} ] on branch [
${PINOT_BRANCH} ] with Kafka version [ ${KAFKA_VERSION} ]"
ENV PINOT_HOME=/opt/pinot
ENV PINOT_BUILD_DIR=/opt/pinot-build
@@ -42,7 +43,7 @@ RUN apt-get update && \
RUN git clone ${PINOT_GIT_URL} ${PINOT_BUILD_DIR} && \
cd ${PINOT_BUILD_DIR} && \
git checkout ${PINOT_BRANCH} && \
- mvn install package -DskipTests -Pbin-dist -Pbuild-shaded-jar && \
+ mvn install package -DskipTests -Pbin-dist -Pbuild-shaded-jar
-Dkafka.version=${KAFKA_VERSION} && \
mkdir -p ${PINOT_HOME}/configs && \
mkdir -p ${PINOT_HOME}/data && \
cp -r pinot-distribution/target/apache-pinot-*-bin/apache-pinot-*-bin/*
${PINOT_HOME}/. && \
diff --git a/docker/README.md b/docker/README.md
index d0d23d3..ec364f3 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -113,4 +113,29 @@ docker run -p 8099:8099 pinot:release-0.1.0 StartBroker
-zkAddress 172.17.0.2:21
Example of bring up a local server:
```SHELL
docker run -p 8098:8098 pinot:release-0.1.0 StartServer -zkAddress
172.17.0.2:2181
-```
\ No newline at end of file
+```
+
+### Use docker compose to bring up Pinot stack
+Below is a script to use docker compose to bring up
zookeeper/kafka/pinot-controller/pinot-broker/pinot-server
+```SHELL
+docker-compose -f docker-compose.yml up
+```
+
+### Quick Start
+
+Below is the script to ingest airplane stats data to Kafka
+```SHELL
+docker run --network=docker_default pinot:kafka2 StreamAvroIntoKafka -avroFile
sample_data/airlineStats_data.avro -kafkaTopic flights-realtime
-kafkaBrokerList kafka:9092 -zkAddress zookeeper:2181
+```
+
+Below is the script to upload airlineStats schema
+```SHELL
+docker run --network=docker_default pinot:kafka2 AddSchema -controllerHost
pinot-controller -controllerPort 9000 -schemaFile
sample_data/airlineStats_schema.json -exec
+```
+
+Below is the script to create airlineStats table
+```SHELL
+docker run --network=docker_default pinot:kafka2 AddTable -filePath
sample_data/docker/airlineStats_realtime_table_config.json -controllerHost
pinot-controller -controllerPort 9000 -exec
+```
+
+In order to query pinot, try to open `localhost:9000/query` from your browser.
diff --git a/docker/docker-build.sh b/docker/docker-build.sh
index 9fda3e5..4de3362 100755
--- a/docker/docker-build.sh
+++ b/docker/docker-build.sh
@@ -42,6 +42,13 @@ else
PINOT_GIT_URL="https://github.com/apache/incubator-pinot.git";
fi
-echo "Trying to build Pinot docker image from Git URL: [ ${PINOT_GIT_URL} ] on
branch: [ ${PINOT_BRANCH} ] and tag it as: [ ${DOCKER_TAG} ]"
+if [[ "$#" -gt 3 ]]
+then
+ KAFKA_VERSION=$4
+else
+ KAFKA_VERSION=0.9
+fi
+
+echo "Trying to build Pinot docker image from Git URL: [ ${PINOT_GIT_URL} ] on
branch: [ ${PINOT_BRANCH} ] and tag it as: [ ${DOCKER_TAG} ]. Kafka
Dependencies: [ ${KAFKA_VERSION} ]."
-docker build --no-cache -t ${DOCKER_TAG} --build-arg
PINOT_BRANCH=${PINOT_BRANCH} --build-arg PINOT_GIT_URL=${PINOT_GIT_URL} -f
Dockerfile .
+docker build --no-cache -t ${DOCKER_TAG} --build-arg
PINOT_BRANCH=${PINOT_BRANCH} --build-arg PINOT_GIT_URL=${PINOT_GIT_URL}
--build-arg KAFKA_VERSION=${KAFKA_VERSION} -f Dockerfile .
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
new file mode 100644
index 0000000..7e8bbfe
--- /dev/null
+++ b/docker/docker-compose.yml
@@ -0,0 +1,82 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+version: '3'
+services:
+ zookeeper:
+ image: zookeeper:latest
+ hostname: zookeeper
+ ports:
+ - "2181:2181"
+ environment:
+ ZOO_MY_ID: 1
+ ZOO_PORT: 2181
+ ZOO_SERVERS: server.1=zookeeper:2888:3888
+ volumes:
+ - ./pinot-docker-demo/zookeeper/data:/data
+ - ./pinot-docker-demo/zookeeper/datalog:/datalog
+ kafka:
+ image: confluentinc/cp-kafka:5.3.0
+ hostname: kafka
+ ports:
+ - "9092:9092"
+ environment:
+ KAFKA_ADVERTISED_LISTENERS:
LISTENER_DOCKER_INTERNAL://kafka:19092,LISTENER_DOCKER_EXTERNAL://kafka:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+ KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+ KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181/kafka"
+ KAFKA_BROKER_ID: 1
+ KAFKA_LOG4J_LOGGERS:
"kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ volumes:
+ - ./pinot-docker-demo/kafka/data:/var/lib/kafka/data
+ depends_on:
+ - zookeeper
+ pinot-controller:
+ image: pinot:kafka2
+ hostname: pinot-controller
+ volumes:
+ - ./pinot-docker-demo/pinot/controller:/tmp/data/controller
+ ports:
+ - "9000:9000"
+ command: StartController -zkAddress zookeeper:2181
+ depends_on:
+ - kafka
+ pinot-broker:
+ image: pinot:kafka2
+ hostname: pinot-broker
+ ports:
+ - "8099:8099"
+ command: StartBroker -zkAddress zookeeper:2181
+ depends_on:
+ - zookeeper
+ - kafka
+ - pinot-controller
+ pinot-server:
+ image: pinot:kafka2
+ hostname: pinot-server
+ volumes:
+ - ./pinot-docker-demo/pinot/server:/tmp/data/server
+ ports:
+ - "8098:8098"
+ command: StartServer -zkAddress zookeeper:2181
+ depends_on:
+ - zookeeper
+ - kafka
+ - pinot-controller
diff --git
a/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json
b/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json
new file mode 100644
index 0000000..de73b4c
--- /dev/null
+++
b/pinot-tools/src/main/resources/sample_data/docker/airlineStats_offline_table_config.json
@@ -0,0 +1,21 @@
+{
+ "tableName": "airlineStats",
+ "tableType": "OFFLINE",
+ "segmentsConfig": {
+ "timeColumnName": "DaysSinceEpoch",
+ "timeType": "DAYS",
+ "segmentPushType": "APPEND",
+ "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
+ "replication": "1"
+ },
+ "tenants": {
+ "broker": "airline_broker",
+ "server": "airline"
+ },
+ "tableIndexConfig": {
+ "loadMode": "MMAP"
+ },
+ "metadata": {
+ "customConfigs": {}
+ }
+}
diff --git
a/pinot-tools/src/main/resources/sample_data/docker/airlineStats_realtime_table_config.json
b/pinot-tools/src/main/resources/sample_data/docker/airlineStats_realtime_table_config.json
new file mode 100644
index 0000000..e3ffd19
--- /dev/null
+++
b/pinot-tools/src/main/resources/sample_data/docker/airlineStats_realtime_table_config.json
@@ -0,0 +1,38 @@
+{
+ "tableName": "airlineStats",
+ "tableType": "REALTIME",
+ "segmentsConfig": {
+ "timeColumnName": "DaysSinceEpoch",
+ "timeType": "DAYS",
+ "retentionTimeUnit": "DAYS",
+ "retentionTimeValue": "5",
+ "segmentPushType": "APPEND",
+ "segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
+ "schemaName": "airlineStats",
+ "replication": "1",
+ "replicasPerPartition": "1"
+ },
+ "tenants": {
+ "broker": "airline_broker",
+ "server": "airline"
+ },
+ "tableIndexConfig": {
+ "loadMode": "MMAP",
+ "streamConfigs": {
+ "streamType": "kafka",
+ "stream.kafka.consumer.type": "simple",
+ "stream.kafka.topic.name": "flights-realtime",
+ "stream.kafka.decoder.class.name":
"org.apache.pinot.core.realtime.impl.kafka.KafkaJSONMessageDecoder",
+ "stream.kafka.consumer.factory.class.name":
"org.apache.pinot.core.realtime.impl.kafka2.KafkaConsumerFactory",
+ "stream.kafka.hlc.zk.connect.string": "zookeeper:2181/kafka",
+ "stream.kafka.zk.broker.url": "zookeeper:2181/kafka",
+ "stream.kafka.broker.list": "kafka:9092",
+ "realtime.segment.flush.threshold.time": "3600000",
+ "realtime.segment.flush.threshold.size": "50000",
+ "stream.kafka.consumer.prop.auto.offset.reset": "smallest"
+ }
+ },
+ "metadata": {
+ "customConfigs": {}
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
