This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch explorable in repository https://gitbox.apache.org/repos/asf/superset.git
commit 3bf3be44320d3d4f561af6013cd76636df9e5a8a Author: Beto Dealmeida <[email protected]> AuthorDate: Wed Oct 15 14:29:00 2025 -0400 Dynamic configuration --- superset/semantic-layer-demo/.gitignore | 8 + superset/semantic-layer-demo/README.md | 240 +++++++ superset/semantic-layer-demo/app.py | 136 ++++ superset/semantic-layer-demo/models.py | 283 ++++++++ superset/semantic-layer-demo/requirements.txt | 5 + superset/semantic-layer-demo/templates/index.html | 766 ++++++++++++++++++++++ superset/semantic_layers/snowflake_.py | 57 +- 7 files changed, 1487 insertions(+), 8 deletions(-) diff --git a/superset/semantic-layer-demo/.gitignore b/superset/semantic-layer-demo/.gitignore new file mode 100644 index 0000000000..9b4395c6ba --- /dev/null +++ b/superset/semantic-layer-demo/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +*.egg-info/ +dist/ +build/ diff --git a/superset/semantic-layer-demo/README.md b/superset/semantic-layer-demo/README.md new file mode 100644 index 0000000000..5b9effc9b0 --- /dev/null +++ b/superset/semantic-layer-demo/README.md @@ -0,0 +1,240 @@ +# Dynamic Schema Demo - Snowflake Configuration + +This is a self-contained demo showing how to build dynamic forms using Pydantic, OpenAPI/JSON Schema, and JSONForms. It demonstrates the pattern used in `superset/semantic_layers/snowflake_.py` for dynamic configuration forms. + +## Key Features + +- **Dynamic Fields**: Fields marked with `x-dynamic: true` in the JSON schema +- **Dependencies**: Fields specify their dependencies via `x-dependsOn` +- **Automatic Updates**: When dependencies are satisfied, the backend is queried for updated schema with actual options +- **Real Snowflake Integration**: Connects to actual Snowflake accounts to fetch databases and schemas + +## How It Works + +1. **Initial Schema**: The frontend loads an initial schema with empty dynamic fields +2. **User Input**: As the user fills in the form (e.g., account identifier and auth) +3. **Dependency Check**: Frontend detects when dependencies are satisfied +4. **Schema Update**: Frontend sends current data to backend +5. **Enriched Schema**: Backend returns updated schema with actual options (e.g., list of databases) +6. **Form Refresh**: Form updates to show the new dropdown options + +## Project Structure + +``` +semantic-layer-demo/ +├── app.py # Flask server with API endpoints +├── models.py # Pydantic models with dynamic schema logic +├── templates/ +│ └── index.html # Frontend with JSONForms integration +├── requirements.txt # Python dependencies +└── README.md # This file +``` + +## Setup + +1. Install dependencies: + +```bash +cd superset/semantic-layer-demo +pip install -r requirements.txt +``` + +2. Run the server: + +```bash +python app.py +``` + +3. Open your browser to http://localhost:5001 + +## Usage + +### Configuration Form + +1. **Account Identifier**: Enter your Snowflake account identifier (e.g., `abc12345` or `orgname-accountname`) +2. **Role** (optional): Enter a role name as freeform text +3. **Warehouse** (optional): Enter a warehouse name as freeform text +4. **Authentication**: Select either "Username and password" or "Private key" from the dropdown, then fill in credentials +5. **Database**: Once account and auth are provided, this dropdown will **automatically populate** with your real Snowflake databases +6. **Schema**: Once database is selected, this dropdown will **automatically populate** with schemas in that database + +### Real Snowflake Integration + +The demo connects to your Snowflake account using: +- `SHOW DATABASES` - to fetch available databases +- `INFORMATION_SCHEMA.SCHEMATA` - to fetch schemas for a selected database + +The dynamic schema refresh happens automatically when you fill in the required fields! + +### Runtime Form + +After completing the configuration form: + +1. Click "Get Runtime Schema" button +2. The runtime form appears on the right side +3. Shows fields that need to be provided at runtime (database/schema if not specified in config or if changing is allowed) + +**Dynamic Runtime Schema:** +If you didn't specify a database/schema in the configuration (or allowed changing them), the runtime form will be dynamic: +- First, you'll see a database dropdown (populated from your Snowflake account) +- After selecting a database, the **schema dropdown will automatically populate** with schemas from that database +- This uses the same dynamic schema pattern as the configuration form! + +## API Endpoints + +### GET /api/schema/configuration + +Returns the initial configuration schema with empty dynamic fields. + +### POST /api/schema/configuration + +Send partial configuration data to get an enriched schema with populated dynamic fields. + +**Request Body:** +```json +{ + "account_identifier": "abc12345", + "auth": { + "auth_type": "user_password", + "username": "myuser", + "password": "mypass" + } +} +``` + +**Response:** +```json +{ + "properties": { + "database": { + "enum": ["SAMPLE_DATA", "PRODUCTION", "ANALYTICS", "DEV"], + "x-dynamic": true, + "x-dependsOn": ["account_identifier", "auth"] + }, + ... + } +} +``` + +### POST /api/schema/runtime + +Get the runtime schema based on configuration and optional runtime data. + +**Request Body (Initial):** +```json +{ + "configuration": { + "account_identifier": "abc12345", + "auth": { ... }, + "allow_changing_database": true, + "allow_changing_schema": true + }, + "runtime_data": null +} +``` + +**Response (Initial):** +```json +{ + "properties": { + "database": { + "enum": ["SAMPLE_DATA", "PRODUCTION", ...], + "type": "string" + }, + "schema": { + "enum": [], + "type": "string", + "x-dynamic": true, + "x-dependsOn": ["database"] + } + } +} +``` + +**Request Body (After database selected):** +```json +{ + "configuration": { ... }, + "runtime_data": { + "database": "SAMPLE_DATA" + } +} +``` + +**Response (Updated):** +```json +{ + "properties": { + "database": { + "enum": ["SAMPLE_DATA", "PRODUCTION", ...], + "type": "string" + }, + "schema": { + "enum": ["PUBLIC", "TPCDS_SF10TCL", "INFORMATION_SCHEMA"], + "type": "string", + "x-dynamic": true, + "x-dependsOn": ["database"] + } + } +} +``` + +## Implementation Details + +### Custom JSON Schema Fields + +The demo uses custom JSON schema extensions: + +- `x-dynamic`: Boolean flag indicating this field's options are fetched dynamically +- `x-dependsOn`: Array of field names that must be filled before this field can be populated + +Example from `models.py`: + +```python +database: str | None = Field( + default=None, + description="The default database to use.", + json_schema_extra={ + "examples": ["testdb"], + "x-dynamic": True, + "x-dependsOn": ["account_identifier", "auth"], + }, +) +``` + +### Frontend Logic + +The frontend (`index.html`) implements: + +1. **Dependency Tracking**: Parses `x-dependsOn` from schema +2. **Change Detection**: Debounced onChange handler (500ms) +3. **Dependency Satisfaction Check**: Validates all dependencies have non-empty values +4. **Schema Refresh**: Fetches updated schema and re-renders form + +### Backend Logic + +The backend (`app.py` and `models.py`) implements: + +1. **Partial Validation**: Accepts incomplete configurations using `model_construct()` +2. **Dependency Checking**: Uses `getattr()` to check if dependencies are satisfied +3. **Option Fetching**: Connects to Snowflake and runs queries to fetch databases and schemas +4. **Schema Enrichment**: Updates `enum` fields in the schema with actual options from Snowflake + +## Production Considerations + +To use this pattern in production: + +1. Add connection pooling for better performance +2. Add better error handling and user feedback for connection failures +3. Add authentication/authorization for the API endpoints +4. Consider caching schema results to reduce database queries +5. Add rate limiting to prevent excessive Snowflake queries + +## Related Files + +This demo is based on the pattern in: +- `superset/semantic_layers/snowflake_.py` - Full Snowflake semantic layer implementation + +## License + +This demo is part of Apache Superset and follows the same license. diff --git a/superset/semantic-layer-demo/app.py b/superset/semantic-layer-demo/app.py new file mode 100644 index 0000000000..0dcf59ae9e --- /dev/null +++ b/superset/semantic-layer-demo/app.py @@ -0,0 +1,136 @@ +""" +Flask application demonstrating dynamic form generation. + +This server provides endpoints to: +1. Get the initial configuration schema +2. Get an updated configuration schema based on partial configuration +3. Get the runtime schema based on full configuration +""" + +from __future__ import annotations + +import json + +from flask import Flask, jsonify, make_response, render_template, request +from flask_cors import CORS +from models import ( + get_configuration_schema, + get_runtime_schema, + SnowflakeConfiguration, +) +from pydantic import ValidationError + +app = Flask(__name__) +app.config["JSON_SORT_KEYS"] = False # Preserve key order from Pydantic +CORS(app) # Enable CORS for development + + +def json_response(data, status=200): + """ + Return JSON response with preserved key order. + + Flask's jsonify() may sort keys even with JSON_SORT_KEYS=False, + so we use json.dumps() directly with sort_keys=False. + """ + response = make_response(json.dumps(data, sort_keys=False), status) + response.headers['Content-Type'] = 'application/json' + return response + + [email protected]("/") +def index(): + """Serve the main page.""" + return render_template("index.html") + + [email protected]("/api/schema/configuration", methods=["GET"]) +def get_initial_configuration_schema(): + """ + Get the initial configuration schema with empty dynamic fields. + """ + schema = get_configuration_schema(configuration=None) + return json_response(schema) + + [email protected]("/api/schema/configuration", methods=["POST"]) +def get_updated_configuration_schema(): + """ + Get an updated configuration schema based on partial configuration. + + The frontend sends the current form data, and we return an enriched schema + with options for dynamic fields whose dependencies are satisfied. + """ + try: + # Get the partial configuration from request + data = request.json or {} + + # Try to validate it (will fail if required fields are missing, but that's ok) + try: + configuration = SnowflakeConfiguration.model_validate(data) + except ValidationError: + # Partial validation - create a configuration with available fields + # We'll use construct to bypass validation + configuration = SnowflakeConfiguration.model_construct(**data) + + # Get the enriched schema + schema = get_configuration_schema(configuration=configuration) + return json_response(schema) + + except Exception as e: + return json_response({"error": str(e)}, 400) + + [email protected]("/api/schema/runtime", methods=["POST"]) +def get_runtime_schema_endpoint(): + """ + Get the runtime schema based on a configuration and optional runtime data. + + This is called: + 1. Initially after the user has completed the configuration form + 2. When runtime data changes (e.g., database selected) to get updated schema + + Request body should contain: + - configuration: The full configuration object + - runtime_data: (optional) The current runtime data for dynamic updates + """ + try: + data = request.json or {} + + # Extract configuration and runtime data + if "configuration" in data: + # New format: separate configuration and runtime_data + config_data = data["configuration"] + runtime_data = data.get("runtime_data") + else: + # Legacy format: just configuration + config_data = data + runtime_data = None + + configuration = SnowflakeConfiguration.model_validate(config_data) + schema = get_runtime_schema(configuration, runtime_data) + return json_response(schema) + + except ValidationError as e: + return json_response({"error": "Invalid configuration", "details": e.errors()}, 400) + except Exception as e: + return json_response({"error": str(e)}, 500) + + [email protected]("/api/validate/configuration", methods=["POST"]) +def validate_configuration(): + """ + Validate a configuration and return any errors. + """ + try: + data = request.json or {} + configuration = SnowflakeConfiguration.model_validate(data) + return json_response({"valid": True, "data": configuration.model_dump(mode="json")}) + + except ValidationError as e: + return json_response({"valid": False, "errors": e.errors()}, 400) + + +if __name__ == "__main__": + print("Starting demo server...") + print("Open http://localhost:5001 in your browser") + app.run(debug=True, port=5001) diff --git a/superset/semantic-layer-demo/models.py b/superset/semantic-layer-demo/models.py new file mode 100644 index 0000000000..a162458d35 --- /dev/null +++ b/superset/semantic-layer-demo/models.py @@ -0,0 +1,283 @@ +""" +Pydantic models demonstrating dynamic schema generation with x-dynamic and x-dependsOn. + +This is a simplified version of the Snowflake semantic layer configuration, +using mock data instead of actual Snowflake connections. +""" + +from __future__ import annotations + +from typing import Any, Literal, Union + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from pydantic import BaseModel, ConfigDict, create_model, Field, SecretStr +from snowflake.connector import connect + + +class UserPasswordAuth(BaseModel): + """Username and password authentication.""" + + model_config = ConfigDict(title="Username and password") + + auth_type: Literal["user_password"] = "user_password" + username: str = Field(description="The username to authenticate as.") + password: SecretStr = Field( + description="The password to authenticate with.", + repr=False, + ) + + +class PrivateKeyAuth(BaseModel): + """Private key authentication.""" + + model_config = ConfigDict(title="Private key") + + auth_type: Literal["private_key"] = "private_key" + private_key: SecretStr = Field( + description="The private key to authenticate with, in PEM format.", + repr=False, + ) + private_key_password: SecretStr = Field( + description="The password to decrypt the private key with.", + repr=False, + ) + + +class SnowflakeConfiguration(BaseModel): + """Parameters needed to connect to Snowflake.""" + + account_identifier: str = Field( + description="The Snowflake account identifier.", + json_schema_extra={"examples": ["abc12345"]}, + ) + + role: str | None = Field( + default=None, + description="The default role to use.", + json_schema_extra={"examples": ["myrole"]}, + ) + warehouse: str | None = Field( + default=None, + description="The default warehouse to use.", + json_schema_extra={"examples": ["testwh"]}, + ) + + auth: Union[UserPasswordAuth, PrivateKeyAuth] = Field( + discriminator="auth_type", + description="Authentication method", + ) + + database: str | None = Field( + default=None, + description="The default database to use.", + json_schema_extra={ + "examples": ["testdb"], + "x-dynamic": True, + "x-dependsOn": ["account_identifier", "auth"], + }, + ) + allow_changing_database: bool = Field( + default=False, + description="Allow changing the default database.", + ) + schema_: str | None = Field( + default=None, + description="The default schema to use.", + json_schema_extra={ + "examples": ["public"], + "x-dynamic": True, + "x-dependsOn": ["account_identifier", "auth", "database"], + }, + alias="schema", + ) + allow_changing_schema: bool = Field( + default=False, + description="Allow changing the default schema.", + ) + + +def get_connection_parameters(configuration: SnowflakeConfiguration) -> dict[str, Any]: + """Convert the configuration to connection parameters for the Snowflake connector.""" + params = { + "account": configuration.account_identifier, + "application": "Superset Semantic Layer Demo", + "paramstyle": "qmark", + "insecure_mode": True, + } + + if configuration.role: + params["role"] = configuration.role + if configuration.warehouse: + params["warehouse"] = configuration.warehouse + if configuration.database: + params["database"] = configuration.database + if configuration.schema_: + params["schema"] = configuration.schema_ + + auth = configuration.auth + if isinstance(auth, UserPasswordAuth): + params["user"] = auth.username + params["password"] = auth.password.get_secret_value() + elif isinstance(auth, PrivateKeyAuth): + pem_private_key = serialization.load_pem_private_key( + auth.private_key.get_secret_value().encode(), + password=( + auth.private_key_password.get_secret_value().encode() + if auth.private_key_password + else None + ), + backend=default_backend(), + ) + params["private_key"] = pem_private_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + else: + raise ValueError("Unsupported authentication method") + + return params + + +def fetch_databases(configuration: SnowflakeConfiguration) -> list[str]: + """Fetch the list of databases available in the Snowflake account.""" + try: + connection_parameters = get_connection_parameters(configuration) + with connect(**connection_parameters) as connection: + cursor = connection.cursor() + cursor.execute("SHOW DATABASES") + return sorted([row[1] for row in cursor]) + except Exception as e: + print(f"Error fetching databases: {e}") + return [] + + +def fetch_schemas( + configuration: SnowflakeConfiguration, database: str | None +) -> list[str]: + """Fetch the list of schemas available in a given database.""" + if not database: + return [] + + try: + connection_parameters = get_connection_parameters(configuration) + # Override the database in connection params to query the specific database + connection_parameters["database"] = database + + with connect(**connection_parameters) as connection: + cursor = connection.cursor() + query = """ + SELECT SCHEMA_NAME + FROM INFORMATION_SCHEMA.SCHEMATA + WHERE CATALOG_NAME = ? + ORDER BY SCHEMA_NAME + """ + cursor.execute(query, (database,)) + return [row[0] for row in cursor] + except Exception as e: + print(f"Error fetching schemas: {e}") + return [] + + +def get_configuration_schema( + configuration: SnowflakeConfiguration | None = None, +) -> dict[str, Any]: + """ + Get the JSON schema for the configuration. + + When a partial configuration is provided, this function enriches the schema + with actual options for dynamic fields (database, schema). + """ + schema = SnowflakeConfiguration.model_json_schema() + properties = schema["properties"] + + if configuration is None: + # Initial state - set these to empty arrays + properties["database"]["enum"] = [] + properties["schema"]["enum"] = [] + return schema + + # Check if we can populate database options + database_depends_on = properties["database"].get("x-dependsOn", []) + if all(getattr(configuration, dep, None) for dep in database_depends_on): + # Fetch real databases from Snowflake + databases = fetch_databases(configuration) + properties["database"]["enum"] = databases + + # Check if we can populate schema options + schema_depends_on = properties["schema"].get("x-dependsOn", []) + if all(getattr(configuration, dep, None) for dep in schema_depends_on): + # Fetch real schemas from Snowflake + if configuration.database: + schemas = fetch_schemas(configuration, configuration.database) + properties["schema"]["enum"] = schemas + + return schema + + +def get_runtime_schema( + configuration: SnowflakeConfiguration, runtime_data: dict[str, Any] | None = None +) -> dict[str, Any]: + """ + Get the JSON schema for runtime parameters. + + This creates a dynamic schema based on what the user needs to provide at runtime. + If database/schema weren't specified in config, or if changing is allowed, + they become required runtime parameters. + + The schema can be enriched with actual values when runtime_data is provided. + """ + fields: dict[str, tuple[type, Field]] = {} + + # If database not specified or changing is allowed, add it to runtime schema + if not configuration.database or configuration.allow_changing_database: + databases = fetch_databases(configuration) + if databases: + fields["database"] = ( + Literal[tuple(databases)], # type: ignore + Field(description="The database to use."), + ) + + # If schema not specified or changing is allowed, add it to runtime schema + if not configuration.schema_ or configuration.allow_changing_schema: + # Get schemas based on the database (from config or runtime data) + db = configuration.database or (runtime_data.get("database") if runtime_data else None) + + # Determine if schema field should be dynamic + is_dynamic = "database" in fields or not configuration.database + + if db: + schemas = fetch_schemas(configuration, db) + if schemas: + fields["schema_"] = ( + Literal[tuple(schemas)], # type: ignore + Field( + description="The schema to use.", + alias="schema", + json_schema_extra={ + "x-dynamic": True, + "x-dependsOn": ["database"], + } if is_dynamic else {}, + ), + ) + else: + # Database not provided yet, add schema as empty (will be populated dynamically) + fields["schema_"] = ( + str | None, + Field( + default=None, + description="The schema to use.", + alias="schema", + json_schema_extra={ + "x-dynamic": True, + "x-dependsOn": ["database"], + }, + ), + ) + + if not fields: + # No runtime parameters needed + return {"type": "object", "properties": {}} + + return create_model("RuntimeParameters", **fields).model_json_schema() diff --git a/superset/semantic-layer-demo/requirements.txt b/superset/semantic-layer-demo/requirements.txt new file mode 100644 index 0000000000..0665ed5b24 --- /dev/null +++ b/superset/semantic-layer-demo/requirements.txt @@ -0,0 +1,5 @@ +Flask==3.0.0 +flask-cors==4.0.0 +pydantic==2.5.0 +snowflake-connector-python==3.6.0 +cryptography==41.0.7 diff --git a/superset/semantic-layer-demo/templates/index.html b/superset/semantic-layer-demo/templates/index.html new file mode 100644 index 0000000000..4374c70715 --- /dev/null +++ b/superset/semantic-layer-demo/templates/index.html @@ -0,0 +1,766 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>Dynamic Schema Demo - Snowflake Configuration</title> + + <style> + body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + max-width: 1400px; + margin: 0 auto; + padding: 20px; + background-color: #f5f5f5; + } + + h1 { + color: #333; + border-bottom: 2px solid #4CAF50; + padding-bottom: 10px; + } + + .container { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 20px; + margin-top: 20px; + } + + .panel { + background: white; + border-radius: 8px; + padding: 20px; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + } + + .panel h2 { + margin-top: 0; + color: #555; + border-bottom: 1px solid #eee; + padding-bottom: 10px; + } + + .info-box { + background-color: #e3f2fd; + border-left: 4px solid #2196F3; + padding: 15px; + margin: 20px 0; + border-radius: 4px; + } + + .info-box h3 { + margin-top: 0; + color: #1976D2; + } + + .status { + padding: 10px; + margin: 10px 0; + border-radius: 4px; + font-size: 14px; + } + + .status.info { + background-color: #e3f2fd; + color: #1976D2; + } + + .status.success { + background-color: #e8f5e9; + color: #2e7d32; + } + + .status.error { + background-color: #ffebee; + color: #c62828; + } + + .json-output { + background-color: #f5f5f5; + border: 1px solid #ddd; + border-radius: 4px; + padding: 15px; + margin-top: 10px; + overflow-x: auto; + } + + .json-output pre { + margin: 0; + font-family: 'Courier New', monospace; + font-size: 12px; + } + + .form-group { + margin-bottom: 20px; + } + + .form-group label { + display: block; + font-weight: 600; + margin-bottom: 5px; + color: #333; + } + + .form-group .description { + font-size: 13px; + color: #666; + margin-bottom: 5px; + } + + .form-group input, + .form-group select { + width: 100%; + padding: 8px 12px; + border: 1px solid #ddd; + border-radius: 4px; + font-size: 14px; + box-sizing: border-box; + } + + .form-group input:focus, + .form-group select:focus { + outline: none; + border-color: #4CAF50; + box-shadow: 0 0 0 2px rgba(76, 175, 80, 0.2); + } + + .form-group input[type="checkbox"] { + width: auto; + margin-right: 5px; + } + + .nested-form { + border-left: 3px solid #e0e0e0; + padding-left: 15px; + margin-left: 10px; + } + + .discriminator-selector { + background-color: #f9f9f9; + padding: 10px; + border-radius: 4px; + margin-bottom: 10px; + } + + button { + background-color: #4CAF50; + color: white; + border: none; + padding: 10px 20px; + border-radius: 4px; + cursor: pointer; + font-size: 14px; + margin-top: 10px; + margin-right: 10px; + } + + button:hover { + background-color: #45a049; + } + + button:disabled { + background-color: #ccc; + cursor: not-allowed; + } + + .dynamic-field { + position: relative; + } + + .dynamic-field::after { + content: '⚡'; + position: absolute; + right: 30px; + top: 32px; + font-size: 18px; + opacity: 0.5; + } + + .dynamic-field.loading::after { + content: '⟳'; + animation: spin 1s linear infinite; + } + + @keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } + } + + .form-group select:disabled, + .form-group input:disabled { + background-color: #f5f5f5; + cursor: not-allowed; + opacity: 0.6; + } + </style> +</head> +<body> + <h1>Dynamic Schema Demo - Snowflake Configuration</h1> + + <div class="info-box"> + <h3>How it works</h3> + <p> + This demo shows dynamic form generation with Pydantic and JSON Schema. + As you fill in the form, fields marked with <code>x-dynamic: true</code> + will automatically populate with options when their dependencies (specified in + <code>x-dependsOn</code>) are satisfied. + </p> + <p> + <strong>Try it:</strong> Enter an account identifier (try <code>abc12345</code> or <code>xyz67890</code>), + fill in authentication, and watch the database dropdown populate (marked with ⚡). Then select a database + to see schema options. + </p> + </div> + + <div class="container"> + <div class="panel"> + <h2>Configuration Form</h2> + <div id="status-config" class="status info"> + Fill in the configuration. Dynamic fields will update automatically. + </div> + <form id="configuration-form"></form> + <button id="validate-config">Validate Configuration</button> + <button id="get-runtime-schema">Get Runtime Schema</button> + <div id="config-data" class="json-output" style="display: none;"> + <strong>Current Configuration Data:</strong> + <pre id="config-json"></pre> + </div> + </div> + + <div class="panel"> + <h2>Runtime Form</h2> + <div id="status-runtime" class="status info"> + Complete the configuration first, then click "Get Runtime Schema". If database/schema weren't configured, you'll select them here (dynamically). + </div> + <form id="runtime-form"></form> + <div id="runtime-data" class="json-output" style="display: none;"> + <strong>Runtime Parameters:</strong> + <pre id="runtime-json"></pre> + </div> + </div> + </div> + + <div class="panel" style="margin-top: 20px;"> + <h2>Schema Details</h2> + <div id="schema-info" class="json-output"> + <strong>Current Configuration Schema:</strong> + <pre id="schema-json"></pre> + </div> + </div> + + <script> + // State management + let configSchema = null; + let configData = {}; + + let runtimeSchema = null; + let runtimeData = {}; + + // Helper function to resolve $ref in JSON Schema + function resolveRef(refPath, schema = configSchema) { + if (!refPath || !refPath.startsWith('#/')) return null; + + // Remove the leading '#/' and split by '/' + const parts = refPath.substring(2).split('/'); + + // Navigate through the schema + let result = schema; + for (const part of parts) { + if (result && typeof result === 'object') { + result = result[part]; + } else { + return null; + } + } + + return result; + } + + // Form rendering utilities + function renderForm(schema, data, formId, onDataChange) { + const form = document.getElementById(formId); + form.innerHTML = ''; + + if (!schema || !schema.properties) { + form.innerHTML = '<p>No schema available</p>'; + return; + } + + // Handle discriminated unions (like auth) + const properties = schema.properties; + const required = schema.required || []; + + // Object.entries preserves insertion order (ES2015+), so field order from Pydantic is maintained + Object.entries(properties).forEach(([key, prop]) => { + const fieldName = prop.alias || key; + const isRequired = required.includes(key); + const isDynamic = prop['x-dynamic'] || false; + + // Only treat as discriminated union if it has an actual discriminator + // Simple nullable fields with anyOf should be handled as regular fields + if (prop.discriminator) { + renderDiscriminatedUnion(form, key, prop, data, onDataChange, isDynamic); + } else { + renderField(form, key, fieldName, prop, data[key], isRequired, onDataChange, isDynamic); + } + }); + } + + function renderField(container, key, displayName, prop, value, required, onChange, isDynamic = false) { + const group = document.createElement('div'); + group.className = 'form-group' + (isDynamic ? ' dynamic-field' : ''); + + const label = document.createElement('label'); + label.textContent = displayName + (required ? ' *' : ''); + label.htmlFor = key; + group.appendChild(label); + + if (prop.description) { + const desc = document.createElement('div'); + desc.className = 'description'; + desc.textContent = prop.description; + group.appendChild(desc); + } + + // Determine the actual type (handle anyOf pattern) + let actualType = prop.type; + if (!actualType && prop.anyOf) { + // Find the non-null type in anyOf + const nonNullType = prop.anyOf.find(t => t.type !== 'null'); + actualType = nonNullType?.type; + } + + // Handle different field types + if (prop.enum && prop.enum.length > 0) { + const select = document.createElement('select'); + select.id = key; + select.name = key; + + const emptyOption = document.createElement('option'); + emptyOption.value = ''; + emptyOption.textContent = '-- Select --'; + select.appendChild(emptyOption); + + prop.enum.forEach(option => { + const opt = document.createElement('option'); + opt.value = option; + opt.textContent = option; + if (value === option) opt.selected = true; + select.appendChild(opt); + }); + + select.addEventListener('change', (e) => onChange(key, e.target.value)); + group.appendChild(select); + } else if (actualType === 'boolean') { + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.id = key; + checkbox.name = key; + checkbox.checked = value || false; + checkbox.addEventListener('change', (e) => onChange(key, e.target.checked)); + group.appendChild(checkbox); + } else if (actualType === 'string' || actualType === 'integer' || actualType === 'number' || !actualType) { + const input = document.createElement('input'); + input.type = prop.format === 'password' ? 'password' : 'text'; + input.id = key; + input.name = key; + input.value = value || ''; + input.placeholder = prop.examples ? prop.examples[0] : ''; + input.addEventListener('input', (e) => onChange(key, e.target.value)); + group.appendChild(input); + } + + container.appendChild(group); + } + + function renderDiscriminatedUnion(container, key, prop, data, onDataChange, isDynamic = false) { + const group = document.createElement('div'); + group.className = 'form-group' + (isDynamic ? ' dynamic-field' : ''); + + const label = document.createElement('label'); + label.textContent = key.charAt(0).toUpperCase() + key.slice(1) + ' *'; + group.appendChild(label); + + if (prop.description) { + const desc = document.createElement('div'); + desc.className = 'description'; + desc.textContent = prop.description; + group.appendChild(desc); + } + + // Get the discriminator field - it's an object with propertyName + const discriminatorObj = prop.discriminator || {}; + const discriminatorField = discriminatorObj.propertyName || 'type'; + const mapping = discriminatorObj.mapping || {}; + const oneOf = prop.oneOf || prop.anyOf || []; + + // If no mapping but we have oneOf, we can't render this properly + if (Object.keys(mapping).length === 0 && oneOf.length === 0) { + console.warn('Cannot render discriminated union without mapping or oneOf'); + return; + } + + // Create discriminator selector + const selectorDiv = document.createElement('div'); + selectorDiv.className = 'discriminator-selector'; + + const select = document.createElement('select'); + select.id = key + '_type'; + + const emptyOption = document.createElement('option'); + emptyOption.value = ''; + emptyOption.textContent = '-- Select Type --'; + select.appendChild(emptyOption); + + // Build options from mapping + Object.entries(mapping).forEach(([typeValue, refPath]) => { + const opt = document.createElement('option'); + opt.value = typeValue; + + // Try to get title from the referenced schema + const schema = resolveRef(refPath); + opt.textContent = schema?.title || typeValue; + + if (data[key] && data[key][discriminatorField] === typeValue) { + opt.selected = true; + } + select.appendChild(opt); + }); + + selectorDiv.appendChild(select); + group.appendChild(selectorDiv); + + // Create container for nested fields + const nestedContainer = document.createElement('div'); + nestedContainer.className = 'nested-form'; + nestedContainer.id = key + '_fields'; + group.appendChild(nestedContainer); + + // Render nested fields based on selected type + const renderNestedFields = (selectedType) => { + nestedContainer.innerHTML = ''; + if (!selectedType) return; + + // Get the schema for this type from mapping + const refPath = mapping[selectedType]; + const selectedSchema = resolveRef(refPath); + + if (selectedSchema && selectedSchema.properties) { + const nestedData = data[key] || {}; + Object.entries(selectedSchema.properties).forEach(([nestedKey, nestedProp]) => { + if (nestedKey === discriminatorField) return; // Skip discriminator field + + renderField( + nestedContainer, + key + '.' + nestedKey, + nestedKey, + nestedProp, + nestedData[nestedKey], + selectedSchema.required?.includes(nestedKey), + (fullKey, value) => { + const actualKey = fullKey.split('.')[1]; + if (!data[key]) data[key] = {}; + data[key][actualKey] = value; + onDataChange(key, data[key]); + } + ); + }); + } + }; + + // Initial render + if (data[key]) { + renderNestedFields(data[key][discriminatorField]); + } + + // Handle type selection + select.addEventListener('change', (e) => { + const selectedType = e.target.value; + if (selectedType) { + data[key] = { [discriminatorField]: selectedType }; + onDataChange(key, data[key]); + renderNestedFields(selectedType); + } else { + delete data[key]; + onDataChange(key, undefined); + nestedContainer.innerHTML = ''; + } + }); + + container.appendChild(group); + } + + // Configuration form management + async function initConfigForm() { + try { + const response = await fetch('/api/schema/configuration'); + configSchema = await response.json(); + updateSchemaDisplay(configSchema); + renderConfigurationForm(); + updateStatus('status-config', 'Fill in the configuration. Dynamic fields will update automatically.', 'info'); + } catch (error) { + console.error('Error loading schema:', error); + updateStatus('status-config', 'Error loading schema: ' + error.message, 'error'); + } + } + + function renderConfigurationForm() { + renderForm(configSchema, configData, 'configuration-form', handleConfigChange); + } + + let updateTimer = null; + function handleConfigChange(key, value) { + configData[key] = value; + + // Update the JSON display + document.getElementById('config-data').style.display = 'block'; + document.getElementById('config-json').textContent = JSON.stringify(configData, null, 2); + + // Debounce the schema update request + clearTimeout(updateTimer); + updateTimer = setTimeout(async () => { + await updateConfigSchema(configData); + }, 500); + } + + async function updateConfigSchema(data) { + try { + const dynamicFields = getDynamicFields(configSchema); + let shouldUpdate = false; + const fieldsToUpdate = []; + + // Check if any dynamic field's dependencies are now satisfied + for (const [field, dependencies] of Object.entries(dynamicFields)) { + if (areDependenciesSatisfied(dependencies, data)) { + shouldUpdate = true; + fieldsToUpdate.push(field); + } + } + + if (shouldUpdate) { + // Add loading state to dynamic fields + fieldsToUpdate.forEach(field => { + const fieldElement = document.getElementById(field); + const formGroup = fieldElement?.closest('.form-group'); + if (formGroup) { + formGroup.classList.add('loading'); + if (fieldElement) { + fieldElement.disabled = true; + } + } + }); + + const response = await fetch('/api/schema/configuration', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(data) + }); + + if (response.ok) { + const newSchema = await response.json(); + + // Update and re-render with the new schema + configSchema = newSchema; + updateSchemaDisplay(configSchema); + renderConfigurationForm(); + updateStatus('status-config', 'Schema updated with new options!', 'success'); + } + + // Remove loading state (form will be re-rendered anyway, but this ensures cleanup) + fieldsToUpdate.forEach(field => { + const fieldElement = document.getElementById(field); + const formGroup = fieldElement?.closest('.form-group'); + if (formGroup) { + formGroup.classList.remove('loading'); + if (fieldElement) { + fieldElement.disabled = false; + } + } + }); + } + } catch (error) { + console.error('Error updating schema:', error); + // Remove loading state on error + document.querySelectorAll('.form-group.loading').forEach(el => { + el.classList.remove('loading'); + }); + } + } + + // Track dependencies for dynamic fields + function getDynamicFields(schema) { + const dynamicFields = {}; + if (schema && schema.properties) { + Object.entries(schema.properties).forEach(([key, prop]) => { + if (prop['x-dynamic']) { + dynamicFields[key] = prop['x-dependsOn'] || []; + } + }); + } + return dynamicFields; + } + + // Check if dependencies are satisfied + function areDependenciesSatisfied(dependencies, data) { + return dependencies.every(dep => { + const value = data[dep]; + if (value === null || value === undefined || value === '') { + return false; + } + if (typeof value === 'object' && Object.keys(value).length === 0) { + return false; + } + return true; + }); + } + + // Validate configuration + async function validateConfiguration() { + try { + const response = await fetch('/api/validate/configuration', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(configData) + }); + + if (response.ok) { + updateStatus('status-config', 'Configuration is valid!', 'success'); + } else { + const error = await response.json(); + updateStatus('status-config', 'Validation errors: ' + JSON.stringify(error.errors, null, 2), 'error'); + } + } catch (error) { + updateStatus('status-config', 'Error validating: ' + error.message, 'error'); + } + } + + // Runtime form management + async function getRuntimeSchema() { + try { + const response = await fetch('/api/schema/runtime', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + configuration: configData, + runtime_data: null + }) + }); + + if (response.ok) { + runtimeSchema = await response.json(); + runtimeData = {}; + renderRuntimeForm(); + updateStatus('status-runtime', 'Runtime schema loaded. Fill in runtime parameters.', 'info'); + } else { + const error = await response.json(); + updateStatus('status-runtime', 'Error: ' + error.error, 'error'); + } + } catch (error) { + updateStatus('status-runtime', 'Error loading runtime schema: ' + error.message, 'error'); + } + } + + function renderRuntimeForm() { + renderForm(runtimeSchema, runtimeData, 'runtime-form', handleRuntimeChange); + } + + let runtimeUpdateTimer = null; + function handleRuntimeChange(key, value) { + runtimeData[key] = value; + document.getElementById('runtime-data').style.display = 'block'; + document.getElementById('runtime-json').textContent = JSON.stringify(runtimeData, null, 2); + + // Debounce the schema update request + clearTimeout(runtimeUpdateTimer); + runtimeUpdateTimer = setTimeout(async () => { + await updateRuntimeSchema(runtimeData); + }, 500); + } + + async function updateRuntimeSchema(data) { + try { + const dynamicFields = getDynamicFields(runtimeSchema); + let shouldUpdate = false; + const fieldsToUpdate = []; + + // Check if any dynamic field's dependencies are now satisfied + for (const [field, dependencies] of Object.entries(dynamicFields)) { + if (areDependenciesSatisfied(dependencies, data)) { + shouldUpdate = true; + fieldsToUpdate.push(field); + } + } + + if (shouldUpdate) { + // Add loading state to dynamic fields + fieldsToUpdate.forEach(field => { + const fieldElement = document.getElementById(field); + const formGroup = fieldElement?.closest('.form-group'); + if (formGroup) { + formGroup.classList.add('loading'); + if (fieldElement) { + fieldElement.disabled = true; + } + } + }); + + const response = await fetch('/api/schema/runtime', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + configuration: configData, + runtime_data: data + }) + }); + + if (response.ok) { + const newSchema = await response.json(); + + // Update and re-render with the new schema + runtimeSchema = newSchema; + renderRuntimeForm(); + updateStatus('status-runtime', 'Runtime schema updated with new options!', 'success'); + } + + // Remove loading state + fieldsToUpdate.forEach(field => { + const fieldElement = document.getElementById(field); + const formGroup = fieldElement?.closest('.form-group'); + if (formGroup) { + formGroup.classList.remove('loading'); + if (fieldElement) { + fieldElement.disabled = false; + } + } + }); + } + } catch (error) { + console.error('Error updating runtime schema:', error); + // Remove loading state on error + document.querySelectorAll('#runtime-form .form-group.loading').forEach(el => { + el.classList.remove('loading'); + }); + } + } + + // Utility functions + function updateStatus(elementId, message, type) { + const statusEl = document.getElementById(elementId); + statusEl.textContent = message; + statusEl.className = 'status ' + type; + } + + function updateSchemaDisplay(schema) { + document.getElementById('schema-json').textContent = JSON.stringify(schema, null, 2); + } + + // Event listeners + document.getElementById('validate-config').addEventListener('click', validateConfiguration); + document.getElementById('get-runtime-schema').addEventListener('click', getRuntimeSchema); + + // Initialize on page load + initConfigForm(); + </script> +</body> +</html> diff --git a/superset/semantic_layers/snowflake_.py b/superset/semantic_layers/snowflake_.py index 40b8cf52b6..9887074d5b 100644 --- a/superset/semantic_layers/snowflake_.py +++ b/superset/semantic_layers/snowflake_.py @@ -252,9 +252,12 @@ class SnowflakeSemanticLayer: options = cls._fetch_databases(connection) properties["database"]["enum"] = list(options) - if all( - getattr(configuration, dependency) - for dependency in properties["schema"].get("x-dependsOn", []) + if ( + all( + getattr(configuration, dependency) + for dependency in properties["schema"].get("x-dependsOn", []) + ) + and configuration.database ): options = cls._fetch_schemas(connection, configuration.database) properties["schema"]["enum"] = list(options) @@ -265,12 +268,21 @@ class SnowflakeSemanticLayer: def get_runtime_schema( cls, configuration: SnowflakeConfiguration, + runtime_data: dict[str, Any] | None = None, ) -> dict[str, Any]: """ Get the JSON schema for the runtime parameters needed to load explorables. + + The schema can be enriched with actual values when `runtime_data` is provided, + enabling dynamic schema updates (e.g., populating schema dropdown after + database is selected). """ fields: dict[str, tuple[type, Field]] = {} + # update configuration with runtime data, for example, to select a schema after + # the database has been selected + configuration = configuration.model_copy(update=runtime_data) + connection_parameters = get_connection_parameters(configuration) with connect(**connection_parameters) as connection: if not configuration.database or configuration.allow_changing_database: @@ -281,11 +293,38 @@ class SnowflakeSemanticLayer: ) if not configuration.schema_ or configuration.allow_changing_schema: - options = cls._fetch_schemas(connection, configuration.database) - fields["schema_"] = ( - Literal[*options], - Field(description="The default schema to use.", alias="schema"), - ) + if configuration.database: + options = cls._fetch_schemas(connection, configuration.database) + fields["schema_"] = ( + Literal[*options], + Field( + description="The default schema to use.", + alias="schema", + json_schema_extra=( + { + "x-dynamic": True, + "x-dependsOn": ["database"], + } + if "database" in fields + else {} + ), + ), + ) + else: + # Database not provided yet, add schema as empty + # (will be populated dynamically) + fields["schema_"] = ( + str | None, + Field( + default=None, + description="The default schema to use.", + alias="schema", + json_schema_extra={ + "x-dynamic": True, + "x-dependsOn": ["database"], + }, + ), + ) return create_model("RuntimeParameters", **fields).model_json_schema() @@ -310,6 +349,8 @@ class SnowflakeSemanticLayer: ) -> set[str]: """ Fetch the list of schemas available in a given database. + + The connection should already have the database set in its context. """ if not database: return set()
