This is an automated email from the ASF dual-hosted git repository. dzamo pushed a commit to branch 1.20 in repository https://gitbox.apache.org/repos/asf/drill.git
commit bb8524116a788be01301956b3eac28e2d0b15f60 Author: James Turton <[email protected]> AuthorDate: Fri Sep 9 07:12:52 2022 +0200 DRILL-8293: Add a docker-compose file to run Drill in cluster mode (#2640) --- distribution/docker-cluster-mode/Dockerfile | 26 ++++++++++++ distribution/docker-cluster-mode/README | 25 +++++++++++ .../docker-cluster-mode/docker-compose.yml | 48 ++++++++++++++++++++++ .../docker-cluster-mode/drill-override.conf | 28 +++++++++++++ distribution/docker-cluster-mode/logback.xml | 39 ++++++++++++++++++ 5 files changed, 166 insertions(+) diff --git a/distribution/docker-cluster-mode/Dockerfile b/distribution/docker-cluster-mode/Dockerfile new file mode 100644 index 0000000000..d4e360865d --- /dev/null +++ b/distribution/docker-cluster-mode/Dockerfile @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM apache/drill:latest + +ENV DRILL_LOG_LEVEL=info + +COPY drill-override.conf $DRILL_HOME/conf/drill-override.conf +COPY logback.xml $DRILL_HOME/conf/logback.xml + +ENTRYPOINT $DRILL_HOME/bin/drillbit.sh run diff --git a/distribution/docker-cluster-mode/README b/distribution/docker-cluster-mode/README new file mode 100644 index 0000000000..bd8bb6d040 --- /dev/null +++ b/distribution/docker-cluster-mode/README @@ -0,0 +1,25 @@ +This directory contains source code artifacts to launch Drill in cluster mode +along with a ZooKeeper. The Drill image is based on a minor customisation of +the official Drill image that switches it from an embedded to a cluster mode +launch. Logging is redirected to stdout. + +In the docker-cluster-mode directory: + +1. docker build -t apache/drill-cluster-mode +2. docker-compose up + +Then access the web UI at http://localhost:8047 or connect a JDBC client to +jdbc:drill:drillbit=localhost or jdbc:drill:zk=localhost but note that you +will need to make the drillbit container hostnames resolvable from the host +to use a ZooKeeper JDBC URL. + +To launch a cluster of 3 Drillbits + +3. docker-compose up --scale drillbit=3 + +but first note that to use docker-compose's "scale" feature to run multiple +Drillbit containers on a single host you will need to remove the host port +mappings from the compose file to prevent collisions (see the comments +on the relevant lines in that file). Once the Drillbits are launched run +`docker-compose ps` to list the ephemeral ports that have been allocated on +the host. diff --git a/distribution/docker-cluster-mode/docker-compose.yml b/distribution/docker-cluster-mode/docker-compose.yml new file mode 100644 index 0000000000..d43bfa9e6f --- /dev/null +++ b/distribution/docker-cluster-mode/docker-compose.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.9" +services: + zookeeper: + container_name: zookeeper + hostname: zookeeper + image: zookeeper:latest + volumes: + - zk_data:/data + - zk_datalog:/datalog + - zk_logs:/logs + ports: + - "2181:2181" + + drillbit: + # container_name not set so that --scale can be used + image: apache/drill-cluster-mode:latest + volumes: + - drill_data:/data + ports: + - "8047:8047" # user HTTP, replace with just "8047" for --scale + - "31010:31010" # user RPC and JDBC, replace with just "31010" for --scale + depends_on: ["zookeeper"] + environment: + - DRILL_MAX_PROC_MEM=8G + +volumes: + zk_data: + zk_datalog: + zk_logs: + drill_data: diff --git a/distribution/docker-cluster-mode/drill-override.conf b/distribution/docker-cluster-mode/drill-override.conf new file mode 100644 index 0000000000..c65652a72b --- /dev/null +++ b/distribution/docker-cluster-mode/drill-override.conf @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This file tells Drill to consider this module when class path scanning. +# This file can also include any supplementary configuration information. +# This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information. + +# See 'drill-override-example.conf' for example configurations + +drill.exec: { + cluster-id: "drillbits1", + zk.connect: "zookeeper:2181" +} diff --git a/distribution/docker-cluster-mode/logback.xml b/distribution/docker-cluster-mode/logback.xml new file mode 100644 index 0000000000..c1cdbfdd27 --- /dev/null +++ b/distribution/docker-cluster-mode/logback.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<configuration> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> + </encoder> + </appender> + <logger name="org.apache.drill" additivity="false"> + <level value="${DRILL_LOG_LEVEL:-info}" /> + <appender-ref ref="STDOUT" /> + </logger> + <logger name="query.logger" additivity="false"> + <level value="${DRILL_LOG_LEVEL:-info}" /> + <appender-ref ref="STDOUT" /> + </logger> + <root> + <level value="${DRILL_LOG_LEVEL:-info}" /> + <appender-ref ref="STDOUT" /> + </root> +</configuration>
