This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 91539f77aa feat(docker): support running multiple Superset instances
simultaneously (#36751)
91539f77aa is described below
commit 91539f77aa5d10c0c61b6a28fce2db8dd808e64c
Author: Evan Rusackas <[email protected]>
AuthorDate: Thu Dec 18 17:04:58 2025 -0800
feat(docker): support running multiple Superset instances simultaneously
(#36751)
Co-authored-by: Claude Opus 4.5 <[email protected]>
---
.envrc.example | 41 +++++
Makefile | 21 ++-
docker-compose.yml | 26 ++--
docker/.env | 9 ++
docker/.env-local.example | 39 +++++
docker/README.md | 28 ++++
.../contributing/development-setup.md | 33 ++++
scripts/docker-compose-up.sh | 171 +++++++++++++++++++++
8 files changed, 350 insertions(+), 18 deletions(-)
diff --git a/.envrc.example b/.envrc.example
new file mode 100644
index 0000000000..1d1f80ff3f
--- /dev/null
+++ b/.envrc.example
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Auto-configure Docker Compose for multi-instance support
+# Requires direnv: https://direnv.net/
+#
+# Install: brew install direnv (or apt install direnv)
+# Setup: Add 'eval "$(direnv hook bash)"' to ~/.bashrc (or ~/.zshrc)
+# Allow: Run 'direnv allow' in this directory once
+
+# Generate unique project name from directory
+export COMPOSE_PROJECT_NAME=$(basename "$PWD" | tr '[:upper:]' '[:lower:]' |
sed 's/[^a-z0-9]/-/g')
+
+# Find available ports sequentially to avoid collisions
+_is_free() { ! lsof -i ":$1" &>/dev/null 2>&1; }
+
+_p=80; while ! _is_free $_p; do ((_p++)); done; export NGINX_PORT=$_p
+_p=8088; while ! _is_free $_p; do ((_p++)); done; export SUPERSET_PORT=$_p
+_p=9000; while ! _is_free $_p; do ((_p++)); done; export NODE_PORT=$_p
+_p=8080; while ! _is_free $_p || [ $_p -eq $NGINX_PORT ]; do ((_p++)); done;
export WEBSOCKET_PORT=$_p
+_p=8081; while ! _is_free $_p || [ $_p -eq $WEBSOCKET_PORT ]; do ((_p++));
done; export CYPRESS_PORT=$_p
+_p=5432; while ! _is_free $_p; do ((_p++)); done; export DATABASE_PORT=$_p
+_p=6379; while ! _is_free $_p; do ((_p++)); done; export REDIS_PORT=$_p
+
+unset _p _is_free
+
+echo "🐳 Superset configured: http://localhost:$SUPERSET_PORT (dev:
localhost:$NODE_PORT)"
diff --git a/Makefile b/Makefile
index 4a7121fd34..9d0a73c9e7 100644
--- a/Makefile
+++ b/Makefile
@@ -18,7 +18,7 @@
# Python version installed; we need 3.10-3.11
PYTHON=`command -v python3.11 || command -v python3.10`
-.PHONY: install superset venv pre-commit
+.PHONY: install superset venv pre-commit up down logs ps nuke
install: superset pre-commit
@@ -112,3 +112,22 @@ report-celery-beat:
admin-user:
superset fab create-admin
+
+# Docker Compose with auto-assigned ports (for running multiple instances)
+up:
+ ./scripts/docker-compose-up.sh
+
+up-detached:
+ ./scripts/docker-compose-up.sh -d
+
+down:
+ ./scripts/docker-compose-up.sh down
+
+logs:
+ ./scripts/docker-compose-up.sh logs -f
+
+ps:
+ ./scripts/docker-compose-up.sh ps
+
+nuke:
+ ./scripts/docker-compose-up.sh nuke
diff --git a/docker-compose.yml b/docker-compose.yml
index 050c6b22ef..5930951776 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -54,10 +54,9 @@ services:
- path: docker/.env-local # optional override
required: false
image: nginx:latest
- container_name: superset_nginx
restart: unless-stopped
ports:
- - "80:80"
+ - "${NGINX_PORT:-80}:80"
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
@@ -66,10 +65,9 @@ services:
redis:
image: redis:7
- container_name: superset_cache
restart: unless-stopped
ports:
- - "127.0.0.1:6379:6379"
+ - "127.0.0.1:${REDIS_PORT:-6379}:6379"
volumes:
- redis:/data
@@ -80,10 +78,9 @@ services:
- path: docker/.env-local # optional override
required: false
image: postgres:16
- container_name: superset_db
restart: unless-stopped
ports:
- - "127.0.0.1:5432:5432"
+ - "127.0.0.1:${DATABASE_PORT:-5432}:5432"
volumes:
- db_home:/var/lib/postgresql/data
- ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
@@ -96,13 +93,12 @@ services:
required: false
build:
<<: *common-build
- container_name: superset_app
command: ["/app/docker/docker-bootstrap.sh", "app"]
restart: unless-stopped
ports:
- - 8088:8088
+ - ${SUPERSET_PORT:-8088}:8088
# When in cypress-mode ->
- - 8081:8081
+ - ${CYPRESS_PORT:-8081}:8081
extra_hosts:
- "host.docker.internal:host-gateway"
user: *superset-user
@@ -114,10 +110,9 @@ services:
SUPERSET__SQLALCHEMY_EXAMPLES_URI: "duckdb:////app/data/examples.duckdb"
superset-websocket:
- container_name: superset_websocket
build: ./superset-websocket
ports:
- - 8080:8080
+ - ${WEBSOCKET_PORT:-8080}:8080
extra_hosts:
- "host.docker.internal:host-gateway"
depends_on:
@@ -149,7 +144,6 @@ services:
superset-init:
build:
<<: *common-build
- container_name: superset_init
command: ["/app/docker/docker-init.sh"]
env_file:
- path: docker/.env # default
@@ -186,9 +180,10 @@ services:
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
# configuring the dev-server to use the host.docker.internal to connect
to the backend
superset: "http://superset:8088";
+ # Bind to all interfaces so Docker port mapping works
+ WEBPACK_DEVSERVER_HOST: "0.0.0.0"
ports:
- - "127.0.0.1:9000:9000" # exposing the dynamic webpack dev server
- container_name: superset_node
+ - "127.0.0.1:${NODE_PORT:-9000}:9000" # exposing the dynamic webpack
dev server
command: ["/app/docker/docker-frontend.sh"]
env_file:
- path: docker/.env # default
@@ -200,7 +195,6 @@ services:
superset-worker:
build:
<<: *common-build
- container_name: superset_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
env_file:
- path: docker/.env # default
@@ -226,7 +220,6 @@ services:
superset-worker-beat:
build:
<<: *common-build
- container_name: superset_worker_beat
command: ["/app/docker/docker-bootstrap.sh", "beat"]
env_file:
- path: docker/.env # default
@@ -244,7 +237,6 @@ services:
superset-tests-worker:
build:
<<: *common-build
- container_name: superset_tests_worker
command: ["/app/docker/docker-bootstrap.sh", "worker"]
env_file:
- path: docker/.env # default
diff --git a/docker/.env b/docker/.env
index a0cbb47deb..def1be0d25 100644
--- a/docker/.env
+++ b/docker/.env
@@ -21,6 +21,15 @@ PYTHONUNBUFFERED=1
COMPOSE_PROJECT_NAME=superset
DEV_MODE=true
+# Port configuration (override in .env-local for multiple instances)
+# NGINX_PORT=80
+# SUPERSET_PORT=8088
+# NODE_PORT=9000
+# WEBSOCKET_PORT=8080
+# CYPRESS_PORT=8081
+# DATABASE_PORT=5432
+# REDIS_PORT=6379
+
# database configurations (do not modify)
DATABASE_DB=superset
DATABASE_HOST=db
diff --git a/docker/.env-local.example b/docker/.env-local.example
new file mode 100644
index 0000000000..433c316091
--- /dev/null
+++ b/docker/.env-local.example
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------
+# Example .env-local file for running multiple Superset instances
+# Copy this file to .env-local and customize for your setup
+# -----------------------------------------------------------------------
+
+# Unique project name prevents container/volume conflicts between clones
+# Each clone should have a different name (e.g., superset-pr123,
superset-feature-x)
+COMPOSE_PROJECT_NAME=superset-dev2
+
+# Port offsets for running multiple instances simultaneously
+# Instance 1 (default): 80, 8088, 9000, 8080, 8081, 5432, 6379
+# Instance 2 example: 81, 8089, 9001, 8082, 8083, 5433, 6380
+NGINX_PORT=81
+SUPERSET_PORT=8089
+NODE_PORT=9001
+WEBSOCKET_PORT=8082
+CYPRESS_PORT=8083
+DATABASE_PORT=5433
+REDIS_PORT=6380
+
+# For verbose logging during development:
+# SUPERSET_LOG_LEVEL=debug
diff --git a/docker/README.md b/docker/README.md
index eeb2a8f359..1ef46aa13a 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -77,6 +77,34 @@ To run the container, simply run: `docker compose up`
After waiting several minutes for Superset initialization to finish, you can
open a browser and view [`http://localhost:8088`](http://localhost:8088)
to start your journey.
+### Running Multiple Instances
+
+If you need to run multiple Superset instances simultaneously (e.g., different
branches or clones), use the make targets which automatically find available
ports:
+
+```bash
+make up
+```
+
+This automatically:
+- Generates a unique project name from your directory
+- Finds available ports (incrementing from defaults if in use)
+- Displays the assigned URLs before starting
+
+Available commands (run from repo root):
+
+| Command | Description |
+|---------|-------------|
+| `make up` | Start services (foreground) |
+| `make up-detached` | Start services (background) |
+| `make down` | Stop all services |
+| `make ps` | Show running containers |
+| `make logs` | Follow container logs |
+| `make nuke` | Stop, remove volumes & local images |
+
+From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up`
+
+**Important**: Always use these commands instead of plain `docker compose
down`, which won't know the correct project name.
+
## Developing
While running, the container server will reload on modification of the
Superset Python and JavaScript source code.
diff --git a/docs/developer_portal/contributing/development-setup.md
b/docs/developer_portal/contributing/development-setup.md
index a25cca8dbc..464c70a73c 100644
--- a/docs/developer_portal/contributing/development-setup.md
+++ b/docs/developer_portal/contributing/development-setup.md
@@ -139,6 +139,39 @@ docker volume rm superset_db_home
docker-compose up
```
+### Running multiple instances
+
+If you need to run multiple Superset clones simultaneously (e.g., testing
different branches),
+use `make up` instead of `docker compose up`:
+
+```bash
+make up
+```
+
+This automatically:
+- Generates a unique project name from your directory name
+- Finds available ports (incrementing from 8088, 9000, etc. if already in use)
+- Displays the assigned URLs before starting
+
+Each clone gets isolated containers and volumes, so you can run them
side-by-side without conflicts.
+
+Available commands (run from repo root):
+
+| Command | Description |
+|---------|-------------|
+| `make up` | Start services (foreground) |
+| `make up-detached` | Start services (background) |
+| `make down` | Stop all services |
+| `make ps` | Show running containers |
+| `make logs` | Follow container logs |
+| `make nuke` | Stop, remove volumes & local images |
+
+From a subdirectory, use: `make -C $(git rev-parse --show-toplevel) up`
+
+:::warning
+Always use these commands instead of plain `docker compose down`, which won't
know the correct project name for your instance.
+:::
+
## GitHub Codespaces (Cloud Development)
GitHub Codespaces provides a complete, pre-configured development environment
in the cloud. This is ideal for:
diff --git a/scripts/docker-compose-up.sh b/scripts/docker-compose-up.sh
new file mode 100755
index 0000000000..525d07e3c2
--- /dev/null
+++ b/scripts/docker-compose-up.sh
@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -----------------------------------------------------------------------
+# Smart docker-compose wrapper for running multiple Superset instances
+#
+# Features:
+# - Auto-generates unique project name from directory
+# - Finds available ports automatically
+# - No manual .env-local editing needed
+#
+# Usage:
+# ./scripts/docker-compose-up.sh [docker-compose args...]
+#
+# Examples:
+# ./scripts/docker-compose-up.sh # Start all services
+# ./scripts/docker-compose-up.sh -d # Start detached
+# ./scripts/docker-compose-up.sh down # Stop services
+# -----------------------------------------------------------------------
+
+set -e
+
+# Get the repo root directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Generate project name from directory name (sanitized for Docker)
+DIR_NAME=$(basename "$REPO_ROOT")
+PROJECT_NAME=$(echo "$DIR_NAME" | tr '[:upper:]' '[:lower:]' | sed
's/[^a-z0-9]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//')
+
+# Function to check if a port is available
+is_port_available() {
+ local port=$1
+ if command -v lsof &> /dev/null; then
+ ! lsof -i ":$port" &> /dev/null
+ elif command -v netstat &> /dev/null; then
+ ! netstat -tuln 2>/dev/null | grep -q ":$port "
+ elif command -v ss &> /dev/null; then
+ ! ss -tuln 2>/dev/null | grep -q ":$port "
+ else
+ # If no tool available, assume port is available
+ return 0
+ fi
+}
+
+# Track ports we've already claimed in this session
+CLAIMED_PORTS=""
+
+# Function to check if port is available and not already claimed
+is_port_free() {
+ local port=$1
+ # Check not already claimed by us
+ if [[ " $CLAIMED_PORTS " =~ " $port " ]]; then
+ return 1
+ fi
+ # Check not in use on system
+ is_port_available $port
+}
+
+# Function to find and claim next available port starting from base
+# Sets the result in the variable named by $2
+find_and_claim_port() {
+ local base_port=$1
+ local var_name=$2
+ local max_attempts=100
+ local port=$base_port
+
+ for ((i=0; i<max_attempts; i++)); do
+ if is_port_free $port; then
+ CLAIMED_PORTS="$CLAIMED_PORTS $port"
+ eval "$var_name=$port"
+ return 0
+ fi
+ ((port++))
+ done
+
+ echo "ERROR: Could not find available port starting from $base_port" >&2
+ return 1
+}
+
+# Base ports (defaults from docker-compose.yml)
+BASE_NGINX=80
+BASE_SUPERSET=8088
+BASE_NODE=9000
+BASE_WEBSOCKET=8080
+BASE_CYPRESS=8081
+BASE_DATABASE=5432
+BASE_REDIS=6379
+
+# Find available ports (no subshells - claims persist correctly)
+echo "🔍 Finding available ports..."
+find_and_claim_port $BASE_NGINX NGINX_PORT
+find_and_claim_port $BASE_SUPERSET SUPERSET_PORT
+find_and_claim_port $BASE_NODE NODE_PORT
+find_and_claim_port $BASE_WEBSOCKET WEBSOCKET_PORT
+find_and_claim_port $BASE_CYPRESS CYPRESS_PORT
+find_and_claim_port $BASE_DATABASE DATABASE_PORT
+find_and_claim_port $BASE_REDIS REDIS_PORT
+
+# Export for docker-compose
+export COMPOSE_PROJECT_NAME="$PROJECT_NAME"
+export NGINX_PORT
+export SUPERSET_PORT
+export NODE_PORT
+export WEBSOCKET_PORT
+export CYPRESS_PORT
+export DATABASE_PORT
+export REDIS_PORT
+
+echo ""
+echo "🐳 Starting Superset with:"
+echo " Project: $PROJECT_NAME"
+echo " Superset: http://localhost:$SUPERSET_PORT";
+echo " Dev Server: http://localhost:$NODE_PORT";
+echo " Nginx: http://localhost:$NGINX_PORT";
+echo " WebSocket: localhost:$WEBSOCKET_PORT"
+echo " Database: localhost:$DATABASE_PORT"
+echo " Redis: localhost:$REDIS_PORT"
+echo ""
+
+# Change to repo root
+cd "$REPO_ROOT"
+
+# Handle special commands
+case "${1:-}" in
+ --dry-run)
+ echo "✅ Dry run complete. To start, run without --dry-run"
+ exit 0
+ ;;
+ --env)
+ # Output as sourceable environment variables
+ echo "export COMPOSE_PROJECT_NAME='$PROJECT_NAME'"
+ echo "export NGINX_PORT=$NGINX_PORT"
+ echo "export SUPERSET_PORT=$SUPERSET_PORT"
+ echo "export NODE_PORT=$NODE_PORT"
+ echo "export WEBSOCKET_PORT=$WEBSOCKET_PORT"
+ echo "export CYPRESS_PORT=$CYPRESS_PORT"
+ echo "export DATABASE_PORT=$DATABASE_PORT"
+ echo "export REDIS_PORT=$REDIS_PORT"
+ exit 0
+ ;;
+ down|stop|logs|ps|exec|restart)
+ # Pass through to docker compose
+ docker compose "$@"
+ ;;
+ nuke)
+ # Nuclear option: remove everything (containers, volumes, local images)
+ echo "💥 Nuking all containers, volumes, and locally-built images for
$PROJECT_NAME..."
+ docker compose down -v --rmi local
+ echo "✅ Done. Run 'make up' or './scripts/docker-compose-up.sh' to
start fresh."
+ ;;
+ *)
+ # Default: start services
+ docker compose up "$@"
+ ;;
+esac
