This is an automated email from the ASF dual-hosted git repository. elizabeth pushed a commit to branch elizabeth/fix-resize-bug in repository https://gitbox.apache.org/repos/asf/superset.git
commit 1e806a77b212e63978c46de29f32d49a0c82e54b Author: Maxime Beauchemin <[email protected]> AuthorDate: Wed Jul 30 13:57:54 2025 -0700 feat(codespaces): auto-setup Python venv with dependencies (#34409) Co-authored-by: Claude <[email protected]> --- .devcontainer/Dockerfile | 20 ++++++++ .devcontainer/README.md | 11 +++++ .devcontainer/bashrc-additions | 62 +++++++++++++++++++++++++ .devcontainer/build-and-push-image.sh | 20 ++++++++ .devcontainer/devcontainer.json | 26 ++++++++--- .devcontainer/setup-dev.sh | 84 ++++++++++++++++++++++++++-------- .devcontainer/start-superset.sh | 71 +++++++++++++++++++++++----- docker/frontend-mem-nag.sh | 44 +++++++++++++++--- docs/docs/contributing/development.mdx | 2 +- 9 files changed, 297 insertions(+), 43 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..eacef7e716 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,20 @@ +# Keep this in sync with the base image in the main Dockerfile (ARG PY_VER) +FROM python:3.11.13-bookworm AS base + +# Install system dependencies that Superset needs +# This layer will be cached across Codespace sessions +RUN apt-get update && apt-get install -y \ + libsasl2-dev \ + libldap2-dev \ + libpq-dev \ + tmux \ + gh \ + && rm -rf /var/lib/apt/lists/* + +# Install uv for fast Python package management +# This will also be cached in the image +RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ + echo 'export PATH="/root/.cargo/bin:$PATH"' >> /etc/bash.bashrc + +# Set the cargo/bin directory in PATH for all users +ENV PATH="/root/.cargo/bin:${PATH}" diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 6b24183edc..e5dda78fe3 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -3,3 +3,14 @@ For complete documentation on using GitHub Codespaces with Apache Superset, please see: **[Setting up a Development Environment - GitHub Codespaces](https://superset.apache.org/docs/contributing/development#github-codespaces-cloud-development)** + +## Pre-installed Development Environment + +When you create a new Codespace from this repository, it automatically: + +1. **Creates a Python virtual environment** using `uv venv` +2. **Installs all development dependencies** via `uv pip install -r requirements/development.txt` +3. **Sets up pre-commit hooks** with `pre-commit install` +4. **Activates the virtual environment** automatically in all terminals + +The virtual environment is located at `/workspaces/{repository-name}/.venv` and is automatically activated through environment variables set in the devcontainer configuration. diff --git a/.devcontainer/bashrc-additions b/.devcontainer/bashrc-additions new file mode 100644 index 0000000000..28a65b51e6 --- /dev/null +++ b/.devcontainer/bashrc-additions @@ -0,0 +1,62 @@ +# Superset Codespaces environment setup +# This file is appended to ~/.bashrc during Codespace setup + +# Find the workspace directory (handles both 'superset' and 'superset-2' names) +WORKSPACE_DIR=$(find /workspaces -maxdepth 1 -name "superset*" -type d | head -1) + +if [ -n "$WORKSPACE_DIR" ]; then + # Check if virtual environment exists + if [ -d "$WORKSPACE_DIR/.venv" ]; then + # Activate the virtual environment + source "$WORKSPACE_DIR/.venv/bin/activate" + echo "โ Python virtual environment activated" + + # Verify pre-commit is installed and set up + if command -v pre-commit &> /dev/null; then + echo "โ pre-commit is available ($(pre-commit --version))" + # Install git hooks if not already installed + if [ -d "$WORKSPACE_DIR/.git" ] && [ ! -f "$WORKSPACE_DIR/.git/hooks/pre-commit" ]; then + echo "๐ช Installing pre-commit hooks..." + cd "$WORKSPACE_DIR" && pre-commit install + fi + else + echo "โ ๏ธ pre-commit not found. Run: pip install pre-commit" + fi + else + echo "โ ๏ธ Python virtual environment not found at $WORKSPACE_DIR/.venv" + echo " Run: cd $WORKSPACE_DIR && .devcontainer/setup-dev.sh" + fi + + # Always cd to the workspace directory for convenience + cd "$WORKSPACE_DIR" +fi + +# Add helpful aliases for Superset development +alias start-superset="$WORKSPACE_DIR/.devcontainer/start-superset.sh" +alias setup-dev="$WORKSPACE_DIR/.devcontainer/setup-dev.sh" + +# Show helpful message on login +echo "" +echo "๐ Superset Codespaces Environment" +echo "==================================" + +# Check if Superset is running +if docker ps 2>/dev/null | grep -q "superset"; then + echo "โ Superset is running!" + echo " - Check the 'Ports' tab for your live Superset URL" + echo " - Initial startup takes 10-20 minutes" + echo " - Login: admin/admin" +else + echo "โ ๏ธ Superset is not running. Use: start-superset" + # Check if there's a startup log + if [ -f "/tmp/superset-startup.log" ]; then + echo " ๐ Startup log found: cat /tmp/superset-startup.log" + fi +fi + +echo "" +echo "Quick commands:" +echo " start-superset - Start Superset with Docker Compose" +echo " setup-dev - Set up Python environment (if not already done)" +echo " pre-commit run - Run pre-commit checks on staged files" +echo "" diff --git a/.devcontainer/build-and-push-image.sh b/.devcontainer/build-and-push-image.sh new file mode 100755 index 0000000000..84fa0e4dbf --- /dev/null +++ b/.devcontainer/build-and-push-image.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Script to build and push the devcontainer image to GitHub Container Registry +# This allows caching the image between Codespace sessions + +# You'll need to run this with appropriate GitHub permissions +# gh auth login --scopes write:packages + +REGISTRY="ghcr.io" +OWNER="apache" +REPO="superset" +TAG="devcontainer-base" + +echo "Building devcontainer image..." +docker build -t $REGISTRY/$OWNER/$REPO:$TAG .devcontainer/ + +echo "Pushing to GitHub Container Registry..." +docker push $REGISTRY/$OWNER/$REPO:$TAG + +echo "Done! Update .devcontainer/devcontainer.json to use:" +echo " \"image\": \"$REGISTRY/$OWNER/$REPO:$TAG\"" diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c73a69e6c4..5f47e964d3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,15 @@ { "name": "Apache Superset Development", - // Keep this in sync with the base image in Dockerfile (ARG PY_VER) - // Using the same base as Dockerfile, but non-slim for dev tools - "image": "python:3.11.13-bookworm", + // Option 1: Use pre-built image directly + // "image": "ghcr.io/apache/superset:devcontainer-base", + + // Option 2: Build from Dockerfile with cache (current approach) + "build": { + "dockerfile": "Dockerfile", + "context": ".", + // Cache from the Apache registry image + "cacheFrom": ["ghcr.io/apache/superset:devcontainer-base"] + }, "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": { @@ -32,10 +39,17 @@ }, // Run commands after container is created - "postCreateCommand": "chmod +x .devcontainer/setup-dev.sh && .devcontainer/setup-dev.sh", + "postCreateCommand": "bash .devcontainer/setup-dev.sh || echo 'โ ๏ธ Setup had issues - run .devcontainer/setup-dev.sh manually'", - // Auto-start Superset on Codespace resume - "postStartCommand": ".devcontainer/start-superset.sh", + // Auto-start Superset after ensuring Docker is ready + // Run in foreground to see any errors, but don't block on failures + "postStartCommand": "bash -c 'echo \"Waiting 30s for services to initialize...\"; sleep 30; .devcontainer/start-superset.sh || echo \"โ ๏ธ Auto-start failed - run start-superset manually\"'", + + // Set environment variables + "remoteEnv": { + // Removed automatic venv activation to prevent startup issues + // The setup script will handle this + }, // VS Code customizations "customizations": { diff --git a/.devcontainer/setup-dev.sh b/.devcontainer/setup-dev.sh index f852118900..91482551be 100755 --- a/.devcontainer/setup-dev.sh +++ b/.devcontainer/setup-dev.sh @@ -3,30 +3,76 @@ echo "๐ง Setting up Superset development environment..." -# The universal image has most tools, just need Superset-specific libs -echo "๐ฆ Installing Superset-specific dependencies..." -sudo apt-get update -sudo apt-get install -y \ - libsasl2-dev \ - libldap2-dev \ - libpq-dev \ - tmux \ - gh - -# Install uv for fast Python package management -echo "๐ฆ Installing uv..." -curl -LsSf https://astral.sh/uv/install.sh | sh - -# Add cargo/bin to PATH for uv -echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.bashrc -echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> ~/.zshrc +# System dependencies and uv are now pre-installed in the Docker image +# This speeds up Codespace creation significantly! + +# Create virtual environment using uv +echo "๐ Creating Python virtual environment..." +if ! uv venv; then + echo "โ Failed to create virtual environment" + exit 1 +fi + +# Install Python dependencies +echo "๐ฆ Installing Python dependencies..." +if ! uv pip install -r requirements/development.txt; then + echo "โ Failed to install Python dependencies" + echo "๐ก You may need to run this manually after the Codespace starts" + exit 1 +fi + +# Install pre-commit hooks +echo "๐ช Installing pre-commit hooks..." +if source .venv/bin/activate && pre-commit install; then + echo "โ Pre-commit hooks installed" +else + echo "โ ๏ธ Pre-commit hooks installation failed (non-critical)" +fi # Install Claude Code CLI via npm echo "๐ค Installing Claude Code..." -npm install -g @anthropic-ai/claude-code +if npm install -g @anthropic-ai/claude-code; then + echo "โ Claude Code installed" +else + echo "โ ๏ธ Claude Code installation failed (non-critical)" +fi # Make the start script executable chmod +x .devcontainer/start-superset.sh +# Add bashrc additions for automatic venv activation +echo "๐ง Setting up automatic environment activation..." +if [ -f ~/.bashrc ]; then + # Check if we've already added our additions + if ! grep -q "Superset Codespaces environment setup" ~/.bashrc; then + echo "" >> ~/.bashrc + cat .devcontainer/bashrc-additions >> ~/.bashrc + echo "โ Added automatic venv activation to ~/.bashrc" + else + echo "โ Bashrc additions already present" + fi +else + # Create bashrc if it doesn't exist + cat .devcontainer/bashrc-additions > ~/.bashrc + echo "โ Created ~/.bashrc with automatic venv activation" +fi + +# Also add to zshrc since that's the default shell +if [ -f ~/.zshrc ] || [ -n "$ZSH_VERSION" ]; then + if ! grep -q "Superset Codespaces environment setup" ~/.zshrc; then + echo "" >> ~/.zshrc + cat .devcontainer/bashrc-additions >> ~/.zshrc + echo "โ Added automatic venv activation to ~/.zshrc" + fi +fi + echo "โ Development environment setup complete!" -echo "๐ Run '.devcontainer/start-superset.sh' to start Superset" +echo "" +echo "๐ The virtual environment will be automatically activated in new terminals" +echo "" +echo "๐ To activate in this terminal, run:" +echo " source ~/.bashrc" +echo "" +echo "๐ To start Superset:" +echo " start-superset" +echo "" diff --git a/.devcontainer/start-superset.sh b/.devcontainer/start-superset.sh index db41aceecf..6ba990cae1 100755 --- a/.devcontainer/start-superset.sh +++ b/.devcontainer/start-superset.sh @@ -1,6 +1,11 @@ #!/bin/bash # Startup script for Superset in Codespaces +# Log to a file for debugging +LOG_FILE="/tmp/superset-startup.log" +echo "[$(date)] Starting Superset startup script" >> "$LOG_FILE" +echo "[$(date)] User: $(whoami), PWD: $(pwd)" >> "$LOG_FILE" + echo "๐ Starting Superset in Codespaces..." echo "๐ Frontend will be available at port 9001" @@ -13,10 +18,37 @@ else echo "๐ Using current directory: $(pwd)" fi -# Check if docker is running -if ! docker info > /dev/null 2>&1; then - echo "โณ Waiting for Docker to start..." - sleep 5 +# Wait for Docker to be available +echo "โณ Waiting for Docker to start..." +echo "[$(date)] Waiting for Docker..." >> "$LOG_FILE" +max_attempts=30 +attempt=0 +while ! docker info > /dev/null 2>&1; do + if [ $attempt -eq $max_attempts ]; then + echo "โ Docker failed to start after $max_attempts attempts" + echo "[$(date)] Docker failed to start after $max_attempts attempts" >> "$LOG_FILE" + echo "๐ Please restart the Codespace or run this script manually later" + exit 1 + fi + echo " Attempt $((attempt + 1))/$max_attempts..." + echo "[$(date)] Docker check attempt $((attempt + 1))/$max_attempts" >> "$LOG_FILE" + sleep 2 + attempt=$((attempt + 1)) +done +echo "โ Docker is ready!" +echo "[$(date)] Docker is ready" >> "$LOG_FILE" + +# Check if Superset containers are already running +if docker ps | grep -q "superset"; then + echo "โ Superset containers are already running!" + echo "" + echo "๐ To access Superset:" + echo " 1. Click the 'Ports' tab at the bottom of VS Code" + echo " 2. Find port 9001 and click the globe icon to open" + echo " 3. Wait 10-20 minutes for initial startup" + echo "" + echo "๐ Login credentials: admin/admin" + exit 0 fi # Clean up any existing containers @@ -24,16 +56,33 @@ echo "๐งน Cleaning up existing containers..." docker-compose -f docker-compose-light.yml down # Start services -echo "๐๏ธ Building and starting services..." +echo "๐๏ธ Starting Superset in background (daemon mode)..." +echo "" + +# Start in detached mode +docker-compose -f docker-compose-light.yml up -d + +echo "" +echo "โ Docker Compose started successfully!" +echo "" +echo "๐ Important information:" +echo "โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ" +echo "โฑ๏ธ Initial startup takes 10-20 minutes" +echo "๐ Check the 'Ports' tab for your Superset URL (port 9001)" +echo "๐ค Login: admin / admin" echo "" -echo "๐ Once started, login with:" -echo " Username: admin" -echo " Password: admin" +echo "๐ Useful commands:" +echo " docker-compose -f docker-compose-light.yml logs -f # Follow logs" +echo " docker-compose -f docker-compose-light.yml ps # Check status" +echo " docker-compose -f docker-compose-light.yml down # Stop services" +echo "โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ" echo "" -echo "๐ Running in foreground with live logs (Ctrl+C to stop)..." +echo "๐ค Keeping terminal open for 60 seconds to test persistence..." +sleep 60 +echo "โ Test complete - check if this terminal is still visible!" -# Run docker-compose and capture exit code -docker-compose -f docker-compose-light.yml up +# Show final status +docker-compose -f docker-compose-light.yml ps EXIT_CODE=$? # If it failed, provide helpful instructions diff --git a/docker/frontend-mem-nag.sh b/docker/frontend-mem-nag.sh index 40517216b6..8dd658e566 100755 --- a/docker/frontend-mem-nag.sh +++ b/docker/frontend-mem-nag.sh @@ -23,25 +23,57 @@ MIN_MEM_FREE_GB=3 MIN_MEM_FREE_KB=$(($MIN_MEM_FREE_GB*1000000)) echo_mem_warn() { - MEM_FREE_KB=$(awk '/MemFree/ { printf "%s \n", $2 }' /proc/meminfo) - MEM_FREE_GB=$(awk '/MemFree/ { printf "%s \n", $2/1024/1024 }' /proc/meminfo) + # Check if running in Codespaces first + if [[ -n "${CODESPACES}" ]]; then + echo "Memory available: Codespaces managed" + return + fi - if [[ "${MEM_FREE_KB}" -lt "${MIN_MEM_FREE_KB}" ]]; then + # Check platform and get memory accordingly + if [[ -f /proc/meminfo ]]; then + # Linux + if grep -q MemAvailable /proc/meminfo; then + MEM_AVAIL_KB=$(awk '/MemAvailable/ { printf "%s \n", $2 }' /proc/meminfo) + MEM_AVAIL_GB=$(awk '/MemAvailable/ { printf "%s \n", $2/1024/1024 }' /proc/meminfo) + else + MEM_AVAIL_KB=$(awk '/MemFree/ { printf "%s \n", $2 }' /proc/meminfo) + MEM_AVAIL_GB=$(awk '/MemFree/ { printf "%s \n", $2/1024/1024 }' /proc/meminfo) + fi + elif [[ "$(uname)" == "Darwin" ]]; then + # macOS - use vm_stat to get free memory + # vm_stat reports in pages, typically 4096 bytes per page + PAGE_SIZE=$(pagesize) + FREE_PAGES=$(vm_stat | awk '/Pages free:/ {print $3}' | tr -d '.') + INACTIVE_PAGES=$(vm_stat | awk '/Pages inactive:/ {print $3}' | tr -d '.') + # Free + inactive pages give us available memory (similar to MemAvailable on Linux) + AVAIL_PAGES=$((FREE_PAGES + INACTIVE_PAGES)) + MEM_AVAIL_KB=$((AVAIL_PAGES * PAGE_SIZE / 1024)) + MEM_AVAIL_GB=$(echo "scale=2; $MEM_AVAIL_KB / 1024 / 1024" | bc) + else + # Other platforms + echo "Memory available: Unable to determine" + return + fi + + if [[ "${MEM_AVAIL_KB}" -lt "${MIN_MEM_FREE_KB}" ]]; then cat <<EOF =============================================== ======== Memory Insufficient Warning ========= =============================================== - It looks like you only have ${MEM_FREE_GB}GB of - memory free. Please increase your Docker + It looks like you only have ${MEM_AVAIL_GB}GB of + memory ${MEM_TYPE}. Please increase your Docker resources to at least ${MIN_MEM_FREE_GB}GB + Note: During builds, available memory may be + temporarily low due to caching and compilation. + =============================================== ======== Memory Insufficient Warning ========= =============================================== EOF else - echo "Memory check Ok [${MEM_FREE_GB}GB free]" + echo "Memory available: ${MEM_AVAIL_GB} GB" fi } diff --git a/docs/docs/contributing/development.mdx b/docs/docs/contributing/development.mdx index fe8995d28a..f50c4e724c 100644 --- a/docs/docs/contributing/development.mdx +++ b/docs/docs/contributing/development.mdx @@ -137,7 +137,7 @@ contributing to Apache Superset more accessible to developers worldwide. 1. **Create a Codespace**: Use this pre-configured link that sets up everything you need: - [**Launch Superset Codespace โ**](https://github.com/codespaces/new?skip_quickstart=true&machine=standardLinux32gb&repo=39464018&ref=codespaces&geo=UsWest&devcontainer_path=.devcontainer%2Fdevcontainer.json) + [**Launch Superset Codespace โ**](https://github.com/codespaces/new?skip_quickstart=true&machine=standardLinux32gb&repo=39464018&ref=master&devcontainer_path=.devcontainer%2Fdevcontainer.json&geo=UsWest) :::caution **Important**: You must select at least the **4 CPU / 16GB RAM** machine type (pre-selected in the link above).
