This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch pinot-presto-docker-image in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit b55aa4929761d4e70825b3b38c2dc0cc080cf6f7 Author: Xiang Fu <[email protected]> AuthorDate: Mon Nov 11 20:59:31 2019 -0800 Adding pinot presto docker image --- docker/images/pinot-presto/Dockerfile | 59 +++++++++ docker/images/pinot-presto/README.md | 61 +++++++++ docker/images/pinot-presto/docker-build.sh | 47 +++++++ docker/images/pinot-presto/etc/config.properties | 27 ++++ docker/images/pinot-presto/etc/jvm.config | 8 ++ docker/images/pinot-presto/etc/log.properties | 20 +++ docker/images/pinot-presto/etc/node.properties | 22 ++++ kubernetes/examples/helm/pinot-presto-cli.sh | 31 +++++ kubernetes/examples/helm/presto-coordinator.yaml | 158 +++++++++++++++++++++++ kubernetes/examples/helm/superset.yaml | 2 +- 10 files changed, 434 insertions(+), 1 deletion(-) diff --git a/docker/images/pinot-presto/Dockerfile b/docker/images/pinot-presto/Dockerfile new file mode 100644 index 0000000..23965f9 --- /dev/null +++ b/docker/images/pinot-presto/Dockerfile @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM openjdk:8 + +LABEL [email protected] + +ARG PRESTO_BRANCH=master +ARG PRESTO_GIT_URL="https://github.com/prestodb/presto.git"; +RUN echo "Trying to build Presto image from [ ${PRESTO_GIT_URL} ] on branch [ ${PRESTO_BRANCH} ]" +ENV PRESTO_HOME=/home/presto +ENV PRESTO_BUILD_DIR=/home/presto/build + +# extra dependency for running launcher +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + vim wget curl git maven && \ + rm -rf /var/lib/apt/lists/* + +RUN groupadd -g 999 presto && \ + useradd -r -u 999 -g presto --create-home --shell /bin/bash presto +USER presto + +RUN git clone ${PRESTO_GIT_URL} ${PRESTO_BUILD_DIR} && \ + cd ${PRESTO_BUILD_DIR} && \ + git checkout ${PRESTO_BRANCH} && \ + ./mvnw clean install -DskipTests && \ + mkdir -p ${PRESTO_HOME}/data && \ + cp -r presto-server/target/presto-server-*/presto-server-*/* ${PRESTO_HOME} && \ + mkdir -p ${PRESTO_HOME}/plugin/pinot && cp -r presto-pinot/target/presto-pinot-*/* ${PRESTO_HOME}/plugin/pinot/ && \ + mvn dependency:purge-local-repository -DactTransitively=false -DreResolve=false --fail-at-end && \ + rm -rf ${PRESTO_BUILD_DIR} + +COPY etc ${PRESTO_HOME}/etc +EXPOSE 8080 + +VOLUME ["${PRESTO_HOME}/etc", "${PRESTO_HOME}/data"] + +WORKDIR ${PRESTO_HOME} + +ENTRYPOINT ["./bin/launcher"] + +CMD ["run"] \ No newline at end of file diff --git a/docker/images/pinot-presto/README.md b/docker/images/pinot-presto/README.md new file mode 100644 index 0000000..8a39b75 --- /dev/null +++ b/docker/images/pinot-presto/README.md @@ -0,0 +1,61 @@ +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +# Presto + +Docker image for [Presto](https://github.com/prestodb/presto) with Pinot integration. + +This docker build project is specialized for Pinot. + +## How to build + +```bash +./docker-build.sh +``` + +You can also build directly with `docker build` command by setting arguments: +```bash +docker build \ + --build-arg PRESTO_BRANCH=master \ + --tag fx19880617/pinot-presto:0.1 \ + --target build . +``` +## How to push + +```bash +docker push fx19880617/pinot-presto:0.1 +``` + +## Configuration + +Follow the [instructions](https://prestodb.io/docs/current/installation/deployment.html) provided by Presto for writing your own configuration files under `etc` directory. + +## Volumes + +The image defines two data volumes: one for mounting configuration into the container, and one for data. + +The configuration volume is located alternatively at `/home/presto/etc`, which contains all the configuration and plugins. + +The data volume is located at `/home/presto/data`. + +## Kubernetes Examples + +Please refer to [`presto-coordinator.yaml`](../../../kubernetes/examples/helm/prest-coordinator.yaml) as k8s deployment example. diff --git a/docker/images/pinot-presto/docker-build.sh b/docker/images/pinot-presto/docker-build.sh new file mode 100755 index 0000000..c84f22e --- /dev/null +++ b/docker/images/pinot-presto/docker-build.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +if [[ "$#" -gt 0 ]] +then + DOCKER_TAG=$1 +else + DOCKER_TAG="pinot-presto:latest" + echo "Not specified a Docker Tag, using default tag: ${DOCKER_TAG}." +fi + +if [[ "$#" -gt 1 ]] +then + PRESTO_BRANCH=$2 +else + PRESTO_BRANCH=master + echo "Not specified a Presto branch to build, using default branch: ${PRESTO_BRANCH}." +fi + +if [[ "$#" -gt 2 ]] +then + PRESTO_GIT_URL=$3 +else + PRESTO_GIT_URL="https://github.com/prestodb/presto.git"; +fi + +echo "Trying to build Pinot Presto docker image from Git URL: [ ${PRESTO_GIT_URL} ] on branch: [ ${PRESTO_BRANCH} ] and tag it as: [ ${DOCKER_TAG} ]." + +docker build --no-cache -t ${DOCKER_TAG} --build-arg PRESTO_BRANCH=${PRESTO_BRANCH} --build-arg PRESTO_GIT_URL=${PRESTO_GIT_URL} -f Dockerfile . diff --git a/docker/images/pinot-presto/etc/config.properties b/docker/images/pinot-presto/etc/config.properties new file mode 100644 index 0000000..5a553e3 --- /dev/null +++ b/docker/images/pinot-presto/etc/config.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery-server.enabled=true +discovery.uri=http://presto-coordinator:8080 \ No newline at end of file diff --git a/docker/images/pinot-presto/etc/jvm.config b/docker/images/pinot-presto/etc/jvm.config new file mode 100644 index 0000000..5985b68 --- /dev/null +++ b/docker/images/pinot-presto/etc/jvm.config @@ -0,0 +1,8 @@ +-server +-Xmx16G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError \ No newline at end of file diff --git a/docker/images/pinot-presto/etc/log.properties b/docker/images/pinot-presto/etc/log.properties new file mode 100644 index 0000000..7ae1016 --- /dev/null +++ b/docker/images/pinot-presto/etc/log.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +com.facebook.presto=INFO \ No newline at end of file diff --git a/docker/images/pinot-presto/etc/node.properties b/docker/images/pinot-presto/etc/node.properties new file mode 100644 index 0000000..7da7c40 --- /dev/null +++ b/docker/images/pinot-presto/etc/node.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +node.environment=production +node.id=presto-coordinator +node.data-dir=/home/presto/data \ No newline at end of file diff --git a/kubernetes/examples/helm/pinot-presto-cli.sh b/kubernetes/examples/helm/pinot-presto-cli.sh new file mode 100755 index 0000000..9195a77 --- /dev/null +++ b/kubernetes/examples/helm/pinot-presto-cli.sh @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if [[ ! -f "/tmp/presto-cli" ]]; then + curl -L https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/0.228/presto-cli-0.228-executable.jar -o /tmp/presto-cli + chmod +x /tmp/presto-cli +fi + +if [[ $(nc -z localhost 18080) != 0 ]]; then + kubectl port-forward service/presto-coordinator 18080:8080 -n pinot-quickstart > /dev/null & +fi + +/tmp/presto-cli --server localhost:18080 --catalog pinot --schema default +pkill -f "kubectl port-forward service/presto-coordinator 18080:8080 -n pinot-quickstart" + diff --git a/kubernetes/examples/helm/presto-coordinator.yaml b/kubernetes/examples/helm/presto-coordinator.yaml new file mode 100644 index 0000000..7edd084 --- /dev/null +++ b/kubernetes/examples/helm/presto-coordinator.yaml @@ -0,0 +1,158 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: presto-catalog + namespace: pinot-quickstart +data: + pinot.properties: |- + connector.name=pinot + pinot.controller-urls=pinot-controller:9000 + pinot.controller-rest-service=pinot-controller:9000 + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: presto-etc + namespace: pinot-quickstart +data: + config.properties: |- + coordinator=true + node-scheduler.include-coordinator=true + http-server.http.port=8080 + query.max-memory=5GB + query.max-memory-per-node=1GB + query.max-total-memory-per-node=2GB + discovery-server.enabled=true + discovery.uri=http://presto-coordinator:8080 + + jvm.config: |- + -server + -Xmx16G + -XX:+UseG1GC + -XX:G1HeapRegionSize=32M + -XX:+UseGCOverheadLimit + -XX:+ExplicitGCInvokesConcurrent + -XX:+HeapDumpOnOutOfMemoryError + -XX:+ExitOnOutOfMemoryError + + log.properties: |- + com.facebook.presto=INFO + + node.properties: |- + node.environment=production + node.id=presto-coordinator + node.data-dir=/home/presto/data + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: presto-coordinator + namespace: pinot-quickstart +spec: + selector: + matchLabels: + app: presto-coordinator + serviceName: presto-coordinator + replicas: 1 + updateStrategy: + type: RollingUpdate + podManagementPolicy: Parallel + template: + metadata: + labels: + app: presto-coordinator + spec: + terminationGracePeriodSeconds: 30 + securityContext: + runAsGroup: 1000 + fsGroup: 1000 + containers: + - image: fx19880617/pinot-presto:latest + imagePullPolicy: Always + name: presto-coordinator + args: [ "run" ] + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + memory: 4Gi + volumeMounts: + - name: presto-data + mountPath: "/home/presto/data" + - name: presto-catalog + mountPath: "/home/presto/etc/catalog" + readOnly: true + - name: presto-etc + mountPath: "/home/presto/etc/config.properties" + subPath: config.properties + readOnly: true + - name: presto-etc + mountPath: "/home/presto/etc/log.properties" + subPath: log.properties + readOnly: true + - name: presto-etc + mountPath: "/home/presto/etc/node.properties" + subPath: node.properties + readOnly: true + - name: presto-etc + mountPath: "/home/presto/etc/jvm.config" + subPath: jvm.config + readOnly: true + nodeSelector: {} + restartPolicy: Always + volumes: + - name: presto-catalog + configMap: + name: presto-catalog + - name: presto-etc + configMap: + name: presto-etc + volumeClaimTemplates: + - metadata: + name: presto-data + annotations: + pv.beta.kubernetes.io/gid: "1000" + pv.beta.kubernetes.io/groups: "1000" + spec: + accessModes: + - ReadWriteOnce + storageClassName: "default" + resources: + requests: + storage: 5Gi + +--- +apiVersion: v1 +kind: Service +metadata: + name: presto-coordinator + namespace: pinot-quickstart +spec: + ports: + # [podname].presto-coordinator.pinot-quickstart.svc.cluster.local + - port: 8080 + clusterIP: None + selector: + app: presto-coordinator diff --git a/kubernetes/examples/helm/superset.yaml b/kubernetes/examples/helm/superset.yaml index 91e4630..fa8137a 100644 --- a/kubernetes/examples/helm/superset.yaml +++ b/kubernetes/examples/helm/superset.yaml @@ -2302,7 +2302,7 @@ spec: spec: accessModes: - ReadWriteOnce - storageClassName: "standard" + storageClassName: "default" resources: requests: storage: 1Gi --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
