villebro commented on code in PR #39171:
URL: https://github.com/apache/superset/pull/39171#discussion_r3539606790
##########
superset-frontend/packages/superset-core/src/common/index.ts:
##########
@@ -227,8 +227,6 @@ export interface Extension {
id: string;
/** Human-readable name of the extension */
name: string;
- /** URL or path to the extension's remote entry point */
- remoteEntry: string;
Review Comment:
It took me a while to re-remember that `LoadedExtension` extends
`Extension`, and has `remoteEntry`. As a bycatch, maybe we could also update
the description to explain that this is the base metadata, rather than only
explaining what an extension is (the description for `LoadedExtension` was very
descriptive for a developer working on the frontend codebase).
##########
superset/extensions/storage/ephemeral.py:
##########
@@ -0,0 +1,181 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Host implementation for Ephemeral State (Tier 2 Storage).
+
+Provides the concrete cache-backed implementation that is injected into
+superset_core.extensions.storage.ephemeral at startup.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from flask import g
+from superset_core.extensions.storage.ephemeral import (
+ EphemeralSetOptions,
+ EphemeralState as CoreEphemeralState,
+)
+
+from superset.extensions import cache_manager
+from superset.extensions.context import get_current_extension_context
+
+# Key separator
+SEPARATOR = ":"
+
+# Key prefix for extension ephemeral state
+KEY_PREFIX = "superset-ext"
+
+
+def _get_extension_id() -> str:
+ """Get the current extension ID from context."""
+ context = get_current_extension_context()
+ if context is None:
+ raise RuntimeError(
+ "ephemeral_state can only be used within an extension context. "
+ "Ensure this code is being executed during extension loading or "
+ "within an extension API request handler."
+ )
+ return context.manifest.id
+
+
+def _get_current_user_id() -> int:
+ """Get the current authenticated user's ID."""
+ user = getattr(g, "user", None)
+ if user is None or not hasattr(user, "id"):
+ raise RuntimeError(
+ "ephemeral_state requires an authenticated user. "
+ "Ensure the request has been authenticated."
+ )
+ return user.id
+
+
+def _build_cache_key(*parts: Any) -> str:
+ """Build a namespaced cache key from parts."""
+ return SEPARATOR.join(str(part) for part in parts)
Review Comment:
The fact that this method is duplicated in `api.py` shows that this should
at a minimum be relocated to `utils.py` for DRYing
##########
docs/developer_docs/extensions/storage.md:
##########
@@ -0,0 +1,348 @@
+---
+title: Storage
+sidebar_position: 8
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Storage
+
+Superset Extensions have access to a managed storage API for persisting data.
The storage system provides multiple tiers with different persistence
characteristics, allowing extensions to choose the right storage for their
needs.
+
+Each extension receives its own isolated storage namespace. When Superset
loads your extension, it binds storage to your extension's unique identifier,
ensuring data privacy—two extensions using the same key will never collide, and
extensions cannot access each other's data.
+
+Storage is always accessed through the extension context via `getContext()`.
This binding to the context is what ties every operation to the current
extension's namespace.
+
+## Storage Tiers
+
+| Tier | Storage Type | Context Property | Use
Case |
+| ---- | ----------------- | ------------------------------------------ |
-------------------------------------- |
+| 1 | Browser storage | `ctx.storage.local`, `ctx.storage.session` | UI
state, wizard progress, draft forms |
+| 2 | Server-side cache | `ctx.storage.ephemeral` | Job
progress, temporary results |
+| 3 | Database | `ctx.storage.persistent` |
Saved artifacts, durable config |
+
+## Tier 1: Local State
+
+Browser-based storage that persists on the user's device. Use this for UI
state and settings that don't need to sync across devices.
+
+### Why Use the API Instead of localStorage Directly?
+
+You might wonder why extensions should use `ctx.storage.local` instead of
directly accessing `window.localStorage`. The managed API provides several
benefits:
+
+- **Automatic namespacing**: Each extension's data is isolated. Two extensions
using the same key name won't collide.
+- **User isolation**: By default, data is scoped to the current user,
preventing data leakage between users on shared devices.
+- **Clean uninstall**: When an extension is uninstalled, all its data can be
cleanly removed using prefix-based deletion.
+- **Future sandboxing**: The async API is designed for a future sandboxed
execution model where extensions run in isolated contexts without direct DOM
access.
+- **Consistent patterns**: The same API shape works across all storage tiers,
making it easy to switch between them.
+
+### localState
+
+Data persists across browser sessions until explicitly deleted or the user
clears browser storage.
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+// Save sidebar state
+await ctx.storage.local.set('sidebar_collapsed', true);
+
+// Retrieve it later
+const isCollapsed = await ctx.storage.local.get('sidebar_collapsed');
+
+// Remove it
+await ctx.storage.local.remove('sidebar_collapsed');
+```
+
+### sessionState
+
+Data is cleared when the browser tab is closed. Use for transient state within
a single session.
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+// Save wizard progress (lost when tab closes)
+await ctx.storage.session.set('wizard_step', 3);
+await ctx.storage.session.set('unsaved_form', { name: 'Draft' });
+
+// Retrieve on page reload within same tab
+const step = await ctx.storage.session.get('wizard_step');
+```
+
+Tier 1 has no `shared` accessor. Browser storage is per-device, so a "shared"
value would only be visible to other users of that same browser, not to other
users of the extension generally — see `.shared` on [Tier
2](#tier-2-ephemeral-state) or [Tier 3](#tier-3-persistent-state) for storage
that's actually shared across users.
+
+### When to Use Tier 1
+
+- UI state (sidebar collapsed, panel sizes)
+- Recently used items
+- Draft form values
+- Any data acceptable to lose if user clears browser
+
+### Limitations
+
+- Per-browser, per-device (not shared across devices)
+- Subject to browser storage quotas (~5-10 MB)
+- Not accessible from backend code
+
+## Tier 2: Ephemeral State
+
+Server-side cache storage with automatic TTL expiration. Use for temporary
data that needs to be shared between frontend and backend, or persist across
page reloads.
+
+TTL is required for every `set` call. The maximum allowed TTL is controlled by
`MAX_TTL` in the server configuration.
+
+### Frontend Usage
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+// Store with a 5-minute TTL
+await ctx.storage.ephemeral.set('job_progress', { pct: 42, status: 'running'
}, { ttl: 300 });
+
+// Retrieve
+const progress = await ctx.storage.ephemeral.get('job_progress');
+
+// Remove
+await ctx.storage.ephemeral.remove('job_progress');
+```
+
+### Backend Usage
+
+```python
+from superset_core.extensions.context import get_context
+from superset_core.extensions.storage.ephemeral import EphemeralSetOptions
+
+ctx = get_context()
+
+# Store job progress with a 5-minute TTL
+ctx.storage.ephemeral.set(
+ 'job_progress', {'pct': 42, 'status': 'running'},
EphemeralSetOptions(ttl=300)
+)
+
+# Retrieve
+progress = ctx.storage.ephemeral.get('job_progress')
+
+# Remove
+ctx.storage.ephemeral.remove('job_progress')
+```
+
+### Shared State
+
+For data that needs to be visible to all users:
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+await ctx.storage.ephemeral.shared.set('shared_result', { data: [1, 2, 3] }, {
ttl: 3600 });
+const result = await ctx.storage.ephemeral.shared.get('shared_result');
+```
+
+```python
+from superset_core.extensions.context import get_context
+from superset_core.extensions.storage.ephemeral import EphemeralSetOptions
+
+ctx = get_context()
+
+ctx.storage.ephemeral.shared.set(
+ 'shared_result', {'data': [1, 2, 3]}, EphemeralSetOptions(ttl=3600)
+)
+result = ctx.storage.ephemeral.shared.get('shared_result')
+```
+
+### When to Use Tier 2
+
+- Background job progress indicators
+- Cross-request intermediate state
+- Query result previews
+- Temporary computation results
+- Any data that can be recomputed if lost
+
+### Limitations
+
+- Not guaranteed to survive server restarts
+- Subject to cache eviction under memory pressure
+- TTL-based expiration (data disappears after timeout)
+- TTL is required on every `set` call
+
+## Tier 3: Persistent State
+
+Database-backed storage that survives server restarts, cache evictions, and
browser clears. Use for any data that must not be lost.
+
+### Frontend Usage
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+// Store a saved SQL snippet
+await ctx.storage.persistent.set('snippet:top_customers', { sql: 'SELECT ...'
});
+
+// Retrieve
+const snippet = await ctx.storage.persistent.get('snippet:top_customers');
+
+// Remove
+await ctx.storage.persistent.remove('snippet:top_customers');
+```
+
+### Backend Usage
+
+```python
+from superset_core.extensions.context import get_context
+
+ctx = get_context()
+
+# Store a saved SQL snippet
+ctx.storage.persistent.set('snippet:top_customers', {'sql': 'SELECT ...'})
+
+# Retrieve
+snippet = ctx.storage.persistent.get('snippet:top_customers')
+
+# Remove
+ctx.storage.persistent.remove('snippet:top_customers')
+```
+
+### Shared State
+
+For data that should be visible to all users of the extension:
+
+```typescript
+import { extensions } from '@apache-superset/core';
+
+const ctx = extensions.getContext();
+
+await ctx.storage.persistent.shared.set('global_config', { version: 2 });
+const config = await ctx.storage.persistent.shared.get('global_config');
+```
+
+```python
+from superset_core.extensions.context import get_context
+
+ctx = get_context()
+
+ctx.storage.persistent.shared.set('global_config', {'version': 2})
+config = ctx.storage.persistent.shared.get('global_config')
+```
+
+### Enumerating and Managing Storage from the Backend
+
+The `ctx.storage.persistent` accessor is for single-key reads and writes.
Backend extension code that needs to enumerate its own rows in bulk — for
example, a cleanup routine that removes storage linked to resources that no
longer exist — can use
`superset_core.extensions.storage.dao.ExtensionStorageDAO` instead:
+
+```python
+from superset_core.extensions.storage.dao import ExtensionStorageDAO
+
+# All entries this extension has linked to a given resource type
+entries = ExtensionStorageDAO.filter_by(resource_type="my-resource-type")
+
+# Bulk-delete rows for resources that no longer exist
+orphaned = [e for e in entries if e.resource_uuid not in
my_extension_live_resource_ids()]
+ExtensionStorageDAO.delete(orphaned)
+```
+
+`ExtensionStorageDAO` is automatically scoped to the calling extension's own
rows — `extension_id` is never a parameter you pass, so there is no way to
reach another extension's storage through this API. It supports the standard
DAO operations (`find_all`, `find_one_or_none`, `filter_by`, `query`, `create`,
`update`, `delete`), and, unlike `ctx.storage.persistent`, gives direct access
to each row's scope columns (`resource_type`, `resource_uuid`, `category`,
`description`) for entries created with those set.
+
+### When to Use Tier 3
+
+- Extension configuration that must survive restarts
+- User-specific saved artifacts that need to roam across devices and browsers
(e.g. saved SQL snippets, annotations)
+- Any data where loss is unacceptable
+
+### Limitations
+
+- Higher latency than Tiers 1–2 (database round-trip per operation)
+- Subject to the 16 MB value size limit
+- Subject to the per-extension quota configured via
`EXTENSIONS_PERSISTENT_STORAGE.QUOTA_PER_EXTENSION` (see [Quotas](#quotas))
+- Requires a database migration when first deployed
+
+## Key Patterns
+
+All storage keys are automatically namespaced:
+
+| Scope | Key Pattern |
+| ---------------------- | ---------------------------------------------------
|
+| User-scoped | `superset-ext:{extension_id}:user:{user_id}:{key}` |
+| Shared (Tiers 2 and 3) | `superset-ext:{extension_id}:shared:{key}` |
Review Comment:
nit, typical AI indentation error:
```suggestion
| Scope | Key Pattern
|
| ---------------------- |
-------------------------------------------------- |
| User-scoped |
`superset-ext:{extension_id}:user:{user_id}:{key}` |
| Shared (Tiers 2 and 3) | `superset-ext:{extension_id}:shared:{key}`
|
```
##########
superset/extensions/storage/persistent_model.py:
##########
@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid as uuid_module
+
+from flask_appbuilder import Model
+from sqlalchemy import (
+ Boolean,
+ Column,
+ ForeignKey,
+ Index,
+ Integer,
+ LargeBinary,
+ String,
+ Text,
+ UniqueConstraint,
+)
+from sqlalchemy.orm import backref, relationship
+from sqlalchemy_utils import UUIDType
+from superset_core.extensions.storage.models import (
+ ExtensionStorageEntry as CoreExtensionStorageEntry,
+)
+
+from superset.models.helpers import AuditMixinNullable
+
+# 16 MB — matches the KeyValue store limit.
+EXTENSION_STORAGE_MAX_SIZE = 2**24 - 1
+
+
+class ExtensionStorage(CoreExtensionStorageEntry, AuditMixinNullable, Model):
+ """Generic persistent key-value storage for extensions (Tier 3).
+
+ Each row is identified by (extension_id, user_fk, resource_type,
+ resource_uuid, key):
+
+ * Global scope — user_fk IS NULL, resource_type IS NULL
+ * User scope — user_fk set, resource_type IS NULL
+ * Resource scope — resource_type + resource_uuid set (user_fk optional)
+
+ The payload is stored as raw bytes (value) with a MIME-type hint
+ (value_type). When is_encrypted is True the value has been encrypted
+ at the DAO layer using Fernet and must be decrypted before use.
+ """
+
+ __tablename__ = "extension_storage"
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ uuid = Column(
+ UUIDType(binary=True),
+ default=uuid_module.uuid4,
+ unique=True,
+ nullable=False,
+ )
+
+ # Extension identity
+ extension_id = Column(String(255), nullable=False)
+
+ # Scope discriminators — all nullable; NULLs define the scope (see
docstring)
+ user_fk = Column(
+ Integer,
+ ForeignKey(
+ "ab_user.id",
+ ondelete="SET NULL",
+ name="fk_extension_storage_user_fk_ab_user",
+ ),
+ nullable=True,
+ )
+ resource_type = Column(String(64), nullable=True)
+ resource_uuid = Column(String(36), nullable=True)
Review Comment:
Maybe add a comment here why FK enforcement isn't possible (the reference
table is variable)
##########
superset/extensions/storage/persistent_dao.py:
##########
@@ -0,0 +1,313 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import logging
+
+from flask import current_app
+from flask_babel import gettext as _
+from sqlalchemy import and_, func, LargeBinary
+
+from superset import db
+from superset.daos.base import BaseDAO
+from superset.exceptions import SupersetGenericErrorException
+from superset.extensions.storage.filters import ExtensionStorageFilter
+from superset.extensions.storage.persistent_model import ExtensionStorage
+from superset.utils.encrypt import (
+ DEFAULT_ENCRYPTION_ENGINE_NAME,
+ EncryptedType,
+ resolve_encryption_engine,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ExtensionStorageQuotaExceeded(SupersetGenericErrorException):
+ """Raised when a write would exceed an extension's persistent storage
quota."""
+
+ status = 413
+
+ def __init__(self, extension_id: str, quota: int) -> None:
+ super().__init__(
+ message=_(
+ "Extension '%(extension_id)s' has exceeded its persistent "
+ "storage quota of %(quota)d bytes.",
+ extension_id=extension_id,
+ quota=quota,
+ ),
+ )
+
+
+def _derive_key(secret: str, user_fk: int) -> str:
+ """Derive a per-user encryption key from the master secret and user ID.
+
+ Uses HMAC-SHA256 so the derived key is cryptographically bound to both.
+ Ciphertext encrypted for one user cannot be decrypted as another.
+ """
+ return hmac.new(
+ secret.encode(),
+ str(user_fk).encode(),
+ hashlib.sha256,
+ ).hexdigest()
+
+
+def _enc_type(user_fk: int | None) -> EncryptedType:
+ """Return an EncryptedType for the given scope.
+
+ User-scoped rows use a key derived from SECRET_KEY and the user ID, so
+ ciphertext written for one user cannot be decrypted as another.
+ Shared rows (user_fk=None) use SECRET_KEY directly.
+ """
+ secret = current_app.config["SECRET_KEY"]
+ key = _derive_key(secret, user_fk) if user_fk is not None else secret
+ engine_name = current_app.config.get(
+ "SQLALCHEMY_ENCRYPTED_FIELD_ENGINE", DEFAULT_ENCRYPTION_ENGINE_NAME
+ )
+ return EncryptedType(
+ LargeBinary,
+ key=key,
+ engine=resolve_encryption_engine(engine_name),
+ )
+
+
+def _get_quota() -> int | None:
+ """Return the configured per-extension persistent storage quota in bytes.
+
+ Returns None when no quota is configured (unlimited).
+ """
+ config = current_app.config.get("EXTENSIONS_PERSISTENT_STORAGE", {})
+ return config.get("QUOTA_PER_EXTENSION")
+
+
+def _check_quota(extension_id: str, new_size: int, existing_size: int = 0) ->
None:
+ """Raise if writing `new_size` bytes would push the extension over quota.
+
+ `existing_size` is the size of the row being replaced (0 for inserts),
+ subtracted from current usage so overwriting a key doesn't double-count
+ it against the quota it already occupies.
+ """
+ quota = _get_quota()
+ if quota is None:
+ return
+ current_usage = (
+ db.session.query(
+ func.coalesce(func.sum(func.length(ExtensionStorage.value)), 0)
+ )
+ .filter(ExtensionStorage.extension_id == extension_id)
+ .scalar()
+ )
+ if current_usage - existing_size + new_size > quota:
+ raise ExtensionStorageQuotaExceeded(extension_id, quota)
Review Comment:
Assuming we're indexing by extension this `SUM` operation should be very
efficient, but I'd still like to call out that this can get expensive if an
extension places lots of data in persistent storage.
##########
superset/extensions/storage/api.py:
##########
@@ -0,0 +1,499 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+REST API for extension storage.
+
+Provides HTTP endpoints for frontend extensions to access server-side
+ephemeral storage without direct backend code.
+
+All operations are user-scoped by default. Use `?shared=true` query param
+to access shared state visible to all users.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from flask import current_app, g, request
+from flask.wrappers import Response
+from flask_appbuilder.api import BaseApi, expose, protect, safe
+
+from superset.extensions import cache_manager
+from superset.extensions.storage.persistent_dao import (
+ ExtensionStorageDAO,
+ ExtensionStorageQuotaExceeded,
+)
+from superset.extensions.types import LoadedExtension
+from superset.extensions.utils import get_extensions
+from superset.utils import json
+from superset.utils.decorators import transaction
+
+# Key separator
+SEPARATOR = ":"
+
+# Key prefix for extension ephemeral state
+KEY_PREFIX = "superset-ext"
+
+
+def _build_cache_key(*parts: Any) -> str:
+ """Build a namespaced cache key from parts."""
+ return SEPARATOR.join(str(part) for part in parts)
+
+
+def _get_extension_or_404(extension_id: str) -> LoadedExtension | None:
+ """Get extension by ID or return None if not found."""
+ extensions = get_extensions()
+ return extensions.get(extension_id)
+
+
+def _parse_ttl(body: dict[str, Any]) -> tuple[int | None, str | None]:
+ """Parse and validate TTL from request body.
+
+ Returns:
+ (ttl, error_message) - error_message is set if the value is missing or
invalid.
+ """
+ if "ttl" not in body:
+ return None, "Field 'ttl' is required"
+ try:
+ ttl = int(body["ttl"])
+ except (TypeError, ValueError):
+ return None, "Field 'ttl' must be a positive integer"
+ if ttl <= 0:
+ return None, "Field 'ttl' must be a positive integer"
+ max_ttl = current_app.config.get("EXTENSIONS_EPHEMERAL_STORAGE",
{}).get("MAX_TTL")
+ if max_ttl is not None and ttl > max_ttl:
+ return None, f"Field 'ttl' must not exceed {max_ttl} seconds"
+ return ttl, None
+
+
+def _build_storage_key(extension_id: str, key: str, shared: bool) -> str:
+ """Build the cache key based on scope (user or shared)."""
+ if shared:
+ return _build_cache_key(KEY_PREFIX, extension_id, "shared", key)
+ user_id = g.user.id
+ return _build_cache_key(KEY_PREFIX, extension_id, "user", user_id, key)
Review Comment:
As these are super critical to the storage API, should we not move these to
a separate `superset/extensions/storage/utils.py` and add a few simple
parametrized tests to both document their typical usage patterns and protect
against regressions?
##########
superset/extensions/storage/persistent_dao.py:
##########
@@ -0,0 +1,313 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import logging
+
+from flask import current_app
+from flask_babel import gettext as _
+from sqlalchemy import and_, func, LargeBinary
+
+from superset import db
+from superset.daos.base import BaseDAO
+from superset.exceptions import SupersetGenericErrorException
+from superset.extensions.storage.filters import ExtensionStorageFilter
+from superset.extensions.storage.persistent_model import ExtensionStorage
+from superset.utils.encrypt import (
+ DEFAULT_ENCRYPTION_ENGINE_NAME,
+ EncryptedType,
+ resolve_encryption_engine,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class ExtensionStorageQuotaExceeded(SupersetGenericErrorException):
+ """Raised when a write would exceed an extension's persistent storage
quota."""
+
+ status = 413
+
+ def __init__(self, extension_id: str, quota: int) -> None:
+ super().__init__(
+ message=_(
+ "Extension '%(extension_id)s' has exceeded its persistent "
+ "storage quota of %(quota)d bytes.",
+ extension_id=extension_id,
+ quota=quota,
+ ),
+ )
+
+
+def _derive_key(secret: str, user_fk: int) -> str:
+ """Derive a per-user encryption key from the master secret and user ID.
+
+ Uses HMAC-SHA256 so the derived key is cryptographically bound to both.
+ Ciphertext encrypted for one user cannot be decrypted as another.
+ """
+ return hmac.new(
+ secret.encode(),
+ str(user_fk).encode(),
+ hashlib.sha256,
+ ).hexdigest()
+
+
+def _enc_type(user_fk: int | None) -> EncryptedType:
+ """Return an EncryptedType for the given scope.
+
+ User-scoped rows use a key derived from SECRET_KEY and the user ID, so
+ ciphertext written for one user cannot be decrypted as another.
+ Shared rows (user_fk=None) use SECRET_KEY directly.
+ """
+ secret = current_app.config["SECRET_KEY"]
+ key = _derive_key(secret, user_fk) if user_fk is not None else secret
+ engine_name = current_app.config.get(
+ "SQLALCHEMY_ENCRYPTED_FIELD_ENGINE", DEFAULT_ENCRYPTION_ENGINE_NAME
+ )
+ return EncryptedType(
+ LargeBinary,
+ key=key,
+ engine=resolve_encryption_engine(engine_name),
+ )
+
+
+def _get_quota() -> int | None:
+ """Return the configured per-extension persistent storage quota in bytes.
+
+ Returns None when no quota is configured (unlimited).
+ """
+ config = current_app.config.get("EXTENSIONS_PERSISTENT_STORAGE", {})
Review Comment:
Nit: For keys in `config.py` we generally don't do `config.get()` but rather
`config[]`.
##########
superset/extensions/context.py:
##########
@@ -16,75 +16,124 @@
# under the License.
"""
-Extension Context Management - provides ambient context during extension
loading.
+Extension Context Management - provides ambient context for extensions.
-This module provides a thread-local context system that allows decorators to
-automatically detect whether they are being called in host or extension code
-during extension loading.
+This module provides a context system using Python's contextvars that allows
+extensions to access their context (metadata and scoped resources) via
get_context().
+
+The context is set during extension loading and when extension callbacks are
invoked.
+Uses ContextVar for thread-safe and async-safe context management with
automatic
+save/restore for nested contexts.
"""
from __future__ import annotations
-import contextlib
-from threading import local
-from typing import Any, Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Any, Iterator
from superset_core.extensions.types import Manifest
-# Thread-local storage for extension context
-_extension_context: local = local()
-
-class ExtensionContext:
- """Manages ambient extension context during loading."""
+class ExtensionStorage:
+ """Extension storage with all available tiers."""
- def __init__(self, manifest: Manifest):
- self.manifest = manifest
+ @property
+ def ephemeral(self) -> Any:
+ from superset.extensions.storage.ephemeral import EphemeralState
- def __enter__(self) -> "ExtensionContext":
- if getattr(_extension_context, "current", None) is not None:
- current_extension = _extension_context.current.manifest.id
- raise RuntimeError(
- f"Cannot initialize extension {self.manifest.id} while
extension "
- f"{current_extension} is already being initialized. "
- f"Nested extension initialization is not supported."
- )
+ return EphemeralState
- _extension_context.current = self
- return self
+ @property
+ def persistent(self) -> Any:
+ from superset.extensions.storage.persistent import PersistentState
- def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
- # Clear the current context
- _extension_context.current = None
+ return PersistentState
-class ExtensionContextWrapper:
- """Wrapper for extension context with extensible properties."""
+class ConcreteExtensionContext:
+ """Concrete implementation of ExtensionContext for the host."""
def __init__(self, manifest: Manifest):
self._manifest = manifest
+ self._storage = ExtensionStorage()
+
+ @property
+ def extension(self) -> Manifest:
+ """Extension metadata (new API)."""
+ return self._manifest
@property
def manifest(self) -> Manifest:
- """Get the extension manifest."""
+ """Extension manifest (for backward compatibility)."""
Review Comment:
Should we set a deprecation version for this field?
##########
superset/extensions/storage/persistent.py:
##########
@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Host implementation for Persistent State (Tier 3 Storage).
+
+Provides the concrete database-backed implementation that is injected into
+superset_core.extensions.storage.persistent at startup.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from flask import g
+from superset_core.extensions.storage.persistent import (
+ PersistentSetOptions,
+ PersistentState as CorePersistentState,
+)
+
+from superset.extensions.context import get_current_extension_context
+from superset.extensions.storage.persistent_dao import ExtensionStorageDAO
+from superset.utils import json
+from superset.utils.decorators import transaction
+
+
+def _get_extension_id() -> str:
+ """Get the current extension ID from context."""
+ context = get_current_extension_context()
+ if context is None:
+ raise RuntimeError(
+ "persistent_state can only be used within an extension context. "
+ "Ensure this code is being executed during extension loading or "
+ "within an extension API request handler."
+ )
+ return context.manifest.id
+
+
+def _get_current_user_id() -> int:
+ """Get the current authenticated user's ID."""
+ user = getattr(g, "user", None)
Review Comment:
we already have `get_user()` in `superset/utils/core.py`, so we should reuse
that at a minimum.
##########
superset/extensions/storage/persistent_model.py:
##########
@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid as uuid_module
+
+from flask_appbuilder import Model
+from sqlalchemy import (
+ Boolean,
+ Column,
+ ForeignKey,
+ Index,
+ Integer,
+ LargeBinary,
+ String,
+ Text,
+ UniqueConstraint,
+)
+from sqlalchemy.orm import backref, relationship
+from sqlalchemy_utils import UUIDType
+from superset_core.extensions.storage.models import (
+ ExtensionStorageEntry as CoreExtensionStorageEntry,
+)
+
+from superset.models.helpers import AuditMixinNullable
+
+# 16 MB — matches the KeyValue store limit.
+EXTENSION_STORAGE_MAX_SIZE = 2**24 - 1
+
+
+class ExtensionStorage(CoreExtensionStorageEntry, AuditMixinNullable, Model):
+ """Generic persistent key-value storage for extensions (Tier 3).
+
+ Each row is identified by (extension_id, user_fk, resource_type,
+ resource_uuid, key):
+
+ * Global scope — user_fk IS NULL, resource_type IS NULL
+ * User scope — user_fk set, resource_type IS NULL
+ * Resource scope — resource_type + resource_uuid set (user_fk optional)
+
+ The payload is stored as raw bytes (value) with a MIME-type hint
+ (value_type). When is_encrypted is True the value has been encrypted
+ at the DAO layer using Fernet and must be decrypted before use.
+ """
+
+ __tablename__ = "extension_storage"
+
+ id = Column(Integer, primary_key=True, autoincrement=True)
+ uuid = Column(
+ UUIDType(binary=True),
+ default=uuid_module.uuid4,
+ unique=True,
+ nullable=False,
+ )
+
+ # Extension identity
+ extension_id = Column(String(255), nullable=False)
+
+ # Scope discriminators — all nullable; NULLs define the scope (see
docstring)
+ user_fk = Column(
+ Integer,
+ ForeignKey(
+ "ab_user.id",
+ ondelete="SET NULL",
+ name="fk_extension_storage_user_fk_ab_user",
+ ),
+ nullable=True,
+ )
+ resource_type = Column(String(64), nullable=True)
+ resource_uuid = Column(String(36), nullable=True)
+
+ # Storage key within the scope
+ key = Column(String(255), nullable=False)
+
+ # Optional metadata
+ category = Column(String(64), nullable=True)
+ description = Column(Text, nullable=True)
+
+ # Payload
+ value = Column(LargeBinary(EXTENSION_STORAGE_MAX_SIZE), nullable=False)
+ value_type = Column(String(255), nullable=False,
default="application/json")
+ is_encrypted = Column(Boolean, nullable=False, default=False)
+
+ user = relationship(
+ "User",
+ backref=backref("extension_storage_entries", cascade="all,
delete-orphan"),
Review Comment:
As we have `ondelete="SET NULL"` on the foreign key definition, isn't
`cascade="all, delete-orphan"` incorrect here?
##########
superset/extensions/storage/persistent.py:
##########
@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Host implementation for Persistent State (Tier 3 Storage).
+
+Provides the concrete database-backed implementation that is injected into
+superset_core.extensions.storage.persistent at startup.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from flask import g
+from superset_core.extensions.storage.persistent import (
+ PersistentSetOptions,
+ PersistentState as CorePersistentState,
+)
+
+from superset.extensions.context import get_current_extension_context
+from superset.extensions.storage.persistent_dao import ExtensionStorageDAO
+from superset.utils import json
+from superset.utils.decorators import transaction
+
+
+def _get_extension_id() -> str:
+ """Get the current extension ID from context."""
+ context = get_current_extension_context()
+ if context is None:
+ raise RuntimeError(
+ "persistent_state can only be used within an extension context. "
+ "Ensure this code is being executed during extension loading or "
+ "within an extension API request handler."
+ )
+ return context.manifest.id
+
+
+def _get_current_user_id() -> int:
+ """Get the current authenticated user's ID."""
+ user = getattr(g, "user", None)
+ if user is None or not hasattr(user, "id"):
+ raise RuntimeError(
+ "persistent_state requires an authenticated user. "
+ "Ensure the request has been authenticated."
+ )
+ return user.id
+
+
+def _decode(raw: bytes | None) -> Any:
+ """Decode stored bytes back to a Python value."""
+ if raw is None:
+ return None
+ return json.loads(raw)
+
+
+def _encode(value: Any) -> bytes:
+ """Encode a Python value for database storage."""
+ return json.dumps(value).encode()
Review Comment:
I think it's probably fine to default to JSON, but I think it would be very
valuable to make the Codec overridable as we do in KeyValue where we support
multiple different formats (JSON, Pickle, Marshmallow).
##########
superset/extensions/context.py:
##########
@@ -16,75 +16,124 @@
# under the License.
"""
-Extension Context Management - provides ambient context during extension
loading.
+Extension Context Management - provides ambient context for extensions.
-This module provides a thread-local context system that allows decorators to
-automatically detect whether they are being called in host or extension code
-during extension loading.
+This module provides a context system using Python's contextvars that allows
+extensions to access their context (metadata and scoped resources) via
get_context().
+
+The context is set during extension loading and when extension callbacks are
invoked.
+Uses ContextVar for thread-safe and async-safe context management with
automatic
+save/restore for nested contexts.
"""
from __future__ import annotations
-import contextlib
-from threading import local
-from typing import Any, Generator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Any, Iterator
from superset_core.extensions.types import Manifest
-# Thread-local storage for extension context
-_extension_context: local = local()
-
-class ExtensionContext:
- """Manages ambient extension context during loading."""
+class ExtensionStorage:
+ """Extension storage with all available tiers."""
- def __init__(self, manifest: Manifest):
- self.manifest = manifest
+ @property
+ def ephemeral(self) -> Any:
+ from superset.extensions.storage.ephemeral import EphemeralState
- def __enter__(self) -> "ExtensionContext":
- if getattr(_extension_context, "current", None) is not None:
- current_extension = _extension_context.current.manifest.id
- raise RuntimeError(
- f"Cannot initialize extension {self.manifest.id} while
extension "
- f"{current_extension} is already being initialized. "
- f"Nested extension initialization is not supported."
- )
+ return EphemeralState
- _extension_context.current = self
- return self
+ @property
+ def persistent(self) -> Any:
+ from superset.extensions.storage.persistent import PersistentState
- def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
- # Clear the current context
- _extension_context.current = None
+ return PersistentState
-class ExtensionContextWrapper:
- """Wrapper for extension context with extensible properties."""
+class ConcreteExtensionContext:
+ """Concrete implementation of ExtensionContext for the host."""
def __init__(self, manifest: Manifest):
self._manifest = manifest
+ self._storage = ExtensionStorage()
+
+ @property
+ def extension(self) -> Manifest:
+ """Extension metadata (new API)."""
Review Comment:
Let's not call out new APIs, but rather only call out deprecated ones:
```suggestion
"""Extension metadata."""
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
