This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch flask_config in repository https://gitbox.apache.org/repos/asf/superset.git
commit 29b4c480f32c385f40880af8f9eec423913fa535 Author: Maxime Beauchemin <[email protected]> AuthorDate: Thu Jul 17 10:43:06 2025 -0700 docs --- docs/docs/configuration/configuring-superset.mdx | 213 ++++++++++++++++++++++- docs/package.json | 5 +- superset/cli/config.py | 65 ++++++- superset/config_defaults.py | 3 +- superset/config_extensions.py | 26 +++ superset/daos/settings.py | 2 +- 6 files changed, 297 insertions(+), 17 deletions(-) diff --git a/docs/docs/configuration/configuring-superset.mdx b/docs/docs/configuration/configuring-superset.mdx index 1a453adfe3..67e3861b65 100644 --- a/docs/docs/configuration/configuring-superset.mdx +++ b/docs/docs/configuration/configuring-superset.mdx @@ -7,13 +7,30 @@ version: 1 # Configuring Superset -## superset_config.py +## Configuration Overview + +Superset provides a flexible, multi-layered configuration system that supports: + +1. **File-based configuration** - Traditional Python configuration files +2. **Environment variables** - For containerized deployments and CI/CD +3. **Database-backed settings** - Runtime configuration changes (coming soon) +4. **Structured metadata** - Rich documentation and validation schemas + +### Configuration Priority + +Configuration values are loaded in the following order (later values override earlier ones): + +1. **Default configuration** - Built-in defaults from `superset/config_defaults.py` +2. **Environment variables** - Values prefixed with `SUPERSET__` +3. **User configuration file** - Your custom `superset_config.py` file + +### superset_config.py Superset exposes hundreds of configurable parameters through its -[config.py module](https://github.com/apache/superset/blob/master/superset/config.py). The +[config_defaults.py module](https://github.com/apache/superset/blob/master/superset/config_defaults.py). The variables and objects exposed act as a public interface of the bulk of what you may want to configure, alter and interface with. In this python module, you'll find all these -parameters, sensible defaults, as well as rich documentation in the form of comments +parameters, sensible defaults, as well as structured metadata documentation. To configure your application, you need to create your own configuration module, which will allow you to override few or many of these parameters. Instead of altering the core module, @@ -77,12 +94,12 @@ MAPBOX_API_KEY = '' :::tip Note that it is typical to copy and paste [only] the portions of the -core [superset/config.py](https://github.com/apache/superset/blob/master/superset/config.py) that +core [superset/config_defaults.py](https://github.com/apache/superset/blob/master/superset/config_defaults.py) that you want to alter, along with the related comments into your own `superset_config.py` file. ::: All the parameters and default values defined -in [superset/config.py](https://github.com/apache/superset/blob/master/superset/config.py) +in [superset/config_defaults.py](https://github.com/apache/superset/blob/master/superset/config_defaults.py) can be altered in your local `superset_config.py`. Administrators will want to read through the file to understand what can be configured locally as well as the default values in place. @@ -97,6 +114,102 @@ for more information on how to configure it. At the very least, you'll want to change `SECRET_KEY` and `SQLALCHEMY_DATABASE_URI`. Continue reading for more about each of these. +## Environment Variables Configuration + +For containerized deployments and CI/CD pipelines, Superset supports configuration through environment variables. This is particularly useful for: + +- **Docker deployments** - Configure containers without rebuilding images +- **Kubernetes environments** - Use ConfigMaps and Secrets +- **CI/CD pipelines** - Set configuration dynamically based on environment +- **Development workflows** - Override settings locally without changing files + +### Environment Variable Format + +All Superset environment variables must use the `SUPERSET__` prefix (note the double underscore): + +```bash +# Basic settings +export SUPERSET__ROW_LIMIT=100000 +export SUPERSET__SQLLAB_TIMEOUT=60 + +# Database configuration +export SUPERSET__SQLALCHEMY_DATABASE_URI="postgresql://user:pass@localhost/superset" + +# Secret key +export SUPERSET__SECRET_KEY="your-secret-key-here" +``` + +### JSON and Complex Values + +Environment variables automatically parse JSON values for complex configuration: + +```bash +# Feature flags as JSON +export SUPERSET__FEATURE_FLAGS='{"ENABLE_TEMPLATE_PROCESSING": true, "ENABLE_EXPLORE_DRAG_AND_DROP": true}' + +# Database configuration +export SUPERSET__DATABASE_CONFIG='{"timeout": 60, "pool_size": 10}' +``` + +### Nested Configuration + +For nested configuration objects, use triple underscores (`___`) to separate levels: + +```bash +# This sets FEATURE_FLAGS["ENABLE_TEMPLATE_PROCESSING"] = true +export SUPERSET__FEATURE_FLAGS__ENABLE_TEMPLATE_PROCESSING=true + +# This sets THEME_DEFAULT["token"]["colorPrimary"] = "#ff0000" +export SUPERSET__THEME_DEFAULT__token__colorPrimary="#ff0000" +``` + +### Environment Variable Examples + +You can view examples of environment variable configuration: + +```bash +# Show all available environment variable examples +superset config env-examples + +# Show current configuration and sources +superset config show --verbose + +# Get a specific configuration value +superset config get ROW_LIMIT +``` + +### Docker Environment Variables + +When using Docker, you can set environment variables in your `docker-compose.yml`: + +```yaml +services: + superset: + image: apache/superset:latest + environment: + - SUPERSET__ROW_LIMIT=100000 + - SUPERSET__SQLLAB_TIMEOUT=60 + - SUPERSET__FEATURE_FLAGS__ENABLE_TEMPLATE_PROCESSING=true + # ... other configuration +``` + +Or use an environment file: + +```bash +# .env file +SUPERSET__ROW_LIMIT=100000 +SUPERSET__SQLLAB_TIMEOUT=60 +SUPERSET__SECRET_KEY=your-secret-key-here +``` + +```yaml +services: + superset: + image: apache/superset:latest + env_file: + - .env +``` + ## Specifying a SECRET_KEY ### Adding an initial SECRET_KEY @@ -546,3 +659,93 @@ FEATURE_FLAGS = { ``` A current list of feature flags can be found in [RESOURCES/FEATURE_FLAGS.md](https://github.com/apache/superset/blob/master/RESOURCES/FEATURE_FLAGS.md). + +## Configuration Introspection + +Superset provides CLI commands to inspect and understand your current configuration: + +### View Current Configuration + +```bash +# Show all configuration as YAML +superset config show + +# Filter configuration by pattern +superset config show --filter "ROW_LIMIT" + +# Show configuration with sources (where each value comes from) +superset config show --verbose +``` + +### Get Specific Configuration Values + +```bash +# Get a specific configuration value with source information +superset config get ROW_LIMIT + +# Output shows both the value and where it came from: +# ROW_LIMIT: +# source: environment (SUPERSET__ROW_LIMIT) +# value: 100000 +``` + +### Environment Variable Examples + +```bash +# Show examples of environment variables for all documented settings +superset config env-examples + +# This shows: +# - Basic environment variable syntax +# - JSON formatting examples +# - Nested configuration examples +# - All documented settings with their metadata +``` + +### Configuration Sources + +The CLI will show you where each configuration value comes from: + +- **`environment (SUPERSET__KEY)`** - Value set via environment variable +- **`superset_config.py`** - Value set in your custom configuration file +- **`config_defaults.py`** - Default value from Superset's built-in configuration + +This helps you understand the configuration precedence and troubleshoot configuration issues. + +## Configuration Reference + +The following table shows all documented configuration settings with their metadata: + +import ConfigurationTable from '@site/src/components/ConfigurationTable'; + +<ConfigurationTable showEnvironmentVariables={true} /> + +## Environment Variables Examples + +Here are ready-to-use environment variable examples: + +import EnvironmentVariablesExample from '@site/src/components/EnvironmentVariablesExample'; + +<EnvironmentVariablesExample /> + +## Configuration Metadata and Documentation + +Superset's configuration system includes rich metadata for many settings, providing: + +- **Type information** - Whether a setting expects an integer, string, boolean, or object +- **Validation rules** - Minimum/maximum values, allowed options +- **Documentation** - Detailed descriptions of what each setting does +- **Impact levels** - How significant changes to this setting are +- **Restart requirements** - Whether changing this setting requires a restart + +This metadata is used for: +- **CLI documentation** - The `superset config env-examples` command shows this information +- **Future admin UI** - Settings management interface (coming soon) +- **Validation** - Ensuring configuration values are valid +- **API documentation** - Automatic generation of configuration schemas + +You can also access this information via CLI: + +```bash +superset config env-examples +``` diff --git a/docs/package.json b/docs/package.json index 121088a113..15ba8dd901 100644 --- a/docs/package.json +++ b/docs/package.json @@ -6,8 +6,9 @@ "scripts": { "docusaurus": "docusaurus", "_init": "cat src/intro_header.txt ../README.md > docs/intro.md", - "start": "yarn run _init && docusaurus start", - "build": "yarn run _init && DEBUG=docusaurus:* docusaurus build", + "_update-config": "bash scripts/update_docs.sh", + "start": "yarn run _init && yarn run _update-config && docusaurus start", + "build": "yarn run _init && yarn run _update-config && DEBUG=docusaurus:* docusaurus build", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/superset/cli/config.py b/superset/cli/config.py index 85f9069516..88011b85fa 100644 --- a/superset/cli/config.py +++ b/superset/cli/config.py @@ -46,6 +46,13 @@ def serialize_config_value(value: Any) -> Any: def get_config_source(key: str) -> str: """Determine where a config value comes from.""" + import os + + # Check if it's from environment variables (with double underscore prefix) + env_key = f"SUPERSET__{key}" + if env_key in os.environ: + return f"environment ({env_key})" + # Check if it's from superset_config.py user override try: import superset_config @@ -55,13 +62,6 @@ def get_config_source(key: str) -> str: except ImportError: pass - # Check if it's from environment variables - env_key = f"SUPERSET__{key}" - import os - - if env_key in os.environ: - return f"environment ({env_key})" - # Otherwise it's from defaults return "config_defaults.py" @@ -118,3 +118,54 @@ def get(key: str) -> None: result = {key: {"value": serialized_value, "source": source}} print(yaml.dump(result, default_flow_style=False)) + + [email protected]() +@with_appcontext +def env_examples() -> None: + """Show example environment variables for configuration.""" + from superset.config_extensions import SupersetConfig + + examples = [ + "# Superset configuration via environment variables", + "# All environment variables must start with SUPERSET__ prefix " + "(note double underscore)", + "", + "# Basic settings", + "export SUPERSET__ROW_LIMIT=100000", + "export SUPERSET__SAMPLES_ROW_LIMIT=10000", + "export SUPERSET__SQLLAB_TIMEOUT=60", + "", + "# Feature flags (JSON format)", + 'export SUPERSET__FEATURE_FLAGS=\'{"ENABLE_TEMPLATE_PROCESSING": true, ' + '"ENABLE_EXPLORE_DRAG_AND_DROP": true}\'', + "", + "# Or use triple underscore for nested values", + "export SUPERSET__FEATURE_FLAGS__ENABLE_TEMPLATE_PROCESSING=true", + "export SUPERSET__FEATURE_FLAGS__ENABLE_EXPLORE_DRAG_AND_DROP=true", + "", + "# Theme configuration", + 'export SUPERSET__THEME_DEFAULT=\'{"colors": ' + '{"primary": {"base": "#1985a1"}}}\'', + "", + "# Lists and complex types", + 'export SUPERSET__FAB_ROLES=\'["Admin", "Alpha", "Gamma"]\'', + "", + ] + + for line in examples: + click.echo(line) + + # Show documented settings if using SupersetConfig + if isinstance(app.config, SupersetConfig): + click.echo("\n# Documented settings with metadata:") + for key, schema in app.config.DATABASE_SETTINGS_SCHEMA.items(): + click.echo(f"\n# {schema.get('title', key)}") + click.echo(f"# {schema.get('description', '')}") + click.echo(f"# Type: {schema.get('type', 'unknown')}") + if "minimum" in schema or "maximum" in schema: + click.echo( + f"# Range: {schema.get('minimum', 'N/A')} - " + "{schema.get('maximum', 'N/A')}" + ) + click.echo(f"export SUPERSET__{key}={schema.get('default', '...')}") diff --git a/superset/config_defaults.py b/superset/config_defaults.py index cf27c36f23..d77eab5434 100644 --- a/superset/config_defaults.py +++ b/superset/config_defaults.py @@ -26,7 +26,6 @@ at the end of this file. from __future__ import annotations import importlib.util -import json import logging import os import re @@ -58,7 +57,7 @@ from superset.stats_logger import DummyStatsLogger from superset.superset_typing import CacheConfig from superset.tasks.types import ExecutorType from superset.themes.types import Theme, ThemeSettings -from superset.utils import core as utils +from superset.utils import core as utils, json from superset.utils.core import NO_TIME_RANGE, parse_boolean_string, QuerySource from superset.utils.encrypt import SQLAlchemyUtilsAdapter from superset.utils.log import DBEventLogger diff --git a/superset/config_extensions.py b/superset/config_extensions.py index 025f2ec393..39e70a5100 100644 --- a/superset/config_extensions.py +++ b/superset/config_extensions.py @@ -239,6 +239,32 @@ class SupersetConfig(Config): # For now, this is a placeholder pass + def load_from_environment(self, prefix: str = "SUPERSET_") -> bool: + """Load configuration from environment variables. + + Uses Flask's built-in from_prefixed_env method to load environment + variables with the SUPERSET__ prefix (note the double underscore). + This provides automatic JSON parsing and nested dictionary support. + + The double underscore clearly separates the system prefix from the + configuration key name. + + Examples: + SUPERSET__ROW_LIMIT=100000 + SUPERSET__SQLLAB_TIMEOUT=60 + SUPERSET__FEATURE_FLAGS='{"ENABLE_TEMPLATE_PROCESSING": true}' + SUPERSET__FEATURE_FLAGS__ENABLE_TEMPLATE_PROCESSING=true + + Args: + prefix: The environment variable prefix (default: "SUPERSET_") + + Returns: + bool: True if any values were loaded + """ + # Use Flask's built-in method which handles JSON parsing automatically + # Note: Flask will add one more underscore, so SUPERSET_ becomes SUPERSET__ + return self.from_prefixed_env(prefix) + def export_settings(self) -> Dict[str, Any]: """Export current settings with metadata.""" result = {} diff --git a/superset/daos/settings.py b/superset/daos/settings.py index 30dfdc8429..1cbc3c15d4 100644 --- a/superset/daos/settings.py +++ b/superset/daos/settings.py @@ -16,11 +16,11 @@ # under the License. from __future__ import annotations -import json from typing import Any, Dict, Optional from superset.daos.base import BaseDAO from superset.models.core import Settings +from superset.utils import json class SettingsDAO(BaseDAO[Settings]):
