This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch backport-77211-v3-4-test-stacked in repository https://gitbox.apache.org/repos/asf/airflow.git
commit bd9ab5964b4011d2572299776e5c778d1a69724d Author: Shahar Epstein <[email protected]> AuthorDate: Thu Nov 19 00:41:13 2026 +0200 UI: Show partition status per asset in a Partitions tab (#77211) * UI: Show partition status per asset in a Partitions tab Partition status is visible only per consuming Dag, as pending runs, and as a "Partition Key" line on an asset event. Nothing answers "which partitions of this asset are materialized, in progress or failed", so a failed or running producer run is invisible from the asset page: it leaves no event. The status is derived from existing tables, with no new schema. A "missing" state is left out on purpose: an asset does not declare its partitions, so absent keys cannot be told from keys that never existed. * UI: Sort the asset Partitions tab and link its consumer runs Review of the tab asked for less guessing and more to act on. The day grid assumed daily keys and drew only the current page, so it is gone until a calendar that covers other cadences exists. A bare count of consumer runs gave nothing to follow, and a table with a fixed order could not bring failed or stale partitions to the top. A partition that is in progress also stayed stale until a reload. Counts and page now come from one statement, so they describe the same snapshot. * UI: Filter asset partitions by state from the filter bar The tab had two filter controls: the status counts, which toggled a filter on click, and the filter bar, which held only the partition key. Other pages filter state through the filter bar alone, and reset and saved filters only see what is in it. The counts stay as a summary of the asset and no longer act as buttons. (cherry picked from commit c58ebe61fb060ae58ccd5cddb788ad336030f6cc) Conflicts resolved for v3-4-test: - router.tsx: v3-4-test has eager routes, so the Partitions route is added in that style. - Regenerated OpenAPI spec and UI client: DagRunAssetReference is still a StrictBaseModel here. - AssetPartitions.test.tsx: the multiselect pill shows raw values on v3-4-test, without #76567. --- .../api_fastapi/core_api/datamodels/ui/assets.py | 44 ++ .../api_fastapi/core_api/openapi/_private_ui.yaml | 263 ++++++++++++ .../api_fastapi/core_api/routes/ui/assets.py | 291 ++++++++++++- .../src/airflow/ui/openapi-gen/queries/common.ts | 11 + .../ui/openapi-gen/queries/ensureQueryData.ts | 21 + .../src/airflow/ui/openapi-gen/queries/prefetch.ts | 21 + .../src/airflow/ui/openapi-gen/queries/queries.ts | 21 + .../src/airflow/ui/openapi-gen/queries/suspense.ts | 21 + .../airflow/ui/openapi-gen/requests/schemas.gen.ts | 148 +++++++ .../ui/openapi-gen/requests/services.gen.ts | 37 +- .../airflow/ui/openapi-gen/requests/types.gen.ts | 79 +++- .../airflow/ui/public/i18n/locales/en/assets.json | 9 + .../src/airflow/ui/src/constants/filterConfigs.tsx | 21 + .../src/airflow/ui/src/constants/searchParams.ts | 1 + .../src/airflow/ui/src/pages/Asset/AssetLayout.tsx | 3 +- .../Asset/AssetPartitions/AssetPartitions.test.tsx | 185 ++++++++ .../Asset/AssetPartitions/AssetPartitions.tsx | 178 ++++++++ .../ui/src/pages/Asset/AssetPartitions/index.ts | 20 + .../pages/Asset/AssetPartitions/partitionStatus.ts | 36 ++ airflow-core/src/airflow/ui/src/router.tsx | 2 + .../src/airflow/ui/src/utils/useFiltersHandler.ts | 1 + .../api_fastapi/core_api/routes/ui/test_assets.py | 474 +++++++++++++++++++++ 22 files changed, 1878 insertions(+), 9 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/assets.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/assets.py index 7296257205b..3c7513b2e91 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/assets.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/assets.py @@ -17,11 +17,16 @@ from __future__ import annotations from datetime import datetime +from typing import Literal from pydantic import Field from airflow.api_fastapi.core_api.base import BaseModel +from airflow.api_fastapi.core_api.datamodels.assets import DagRunAssetReference from airflow.api_fastapi.core_api.datamodels.common import MaybeAssetExpression +from airflow.utils.state import DagRunState + +AssetPartitionStatus = Literal["materialized", "in_progress", "failed"] class NextRunAssetEventResponse(BaseModel): @@ -59,3 +64,42 @@ class NextRunAssetsResponse(BaseModel): caller's permissions; this count does not. It reveals nothing new — the redacted ``asset_expression`` already carries one slot per asset.""" pending_partition_count: int | None = None + + +class AssetPartitionProducerRunResponse(BaseModel): + """The latest run of a producing Dag for one partition key.""" + + dag_id: str + run_id: str + state: DagRunState + start_date: datetime | None + end_date: datetime | None + + +class AssetPartitionResponse(BaseModel): + """Status of one partition key of an asset.""" + + partition_key: str + status: AssetPartitionStatus + last_event_id: int | None + last_event_timestamp: datetime | None + last_producer_run: AssetPartitionProducerRunResponse | None + consumer_runs: list[DagRunAssetReference] + """Dag runs that consumed the last event of the key.""" + + +class AssetPartitionStatusCounts(BaseModel): + """Number of partition keys per status.""" + + materialized: int = 0 + in_progress: int = 0 + failed: int = 0 + + +class AssetPartitionCollectionResponse(BaseModel): + """Partition keys of an asset with their status.""" + + partitions: list[AssetPartitionResponse] + total_entries: int + status_counts: AssetPartitionStatusCounts + """Counts for the keys that match ``partition_key_pattern``, whatever the ``status`` filter.""" diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 8b8034c8c89..0bf7fbc85e3 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -326,6 +326,93 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /ui/assets/{asset_id}/partitions: + get: + tags: + - Asset + summary: Get Asset Partitions + description: Return the partition keys of an asset with the status of each. + operationId: get_asset_partitions + security: + - OAuth2PasswordBearer: [] + - HTTPBearer: [] + parameters: + - name: asset_id + in: path + required: true + schema: + type: integer + title: Asset Id + - name: status + in: query + required: false + schema: + anyOf: + - type: array + items: + enum: + - materialized + - in_progress + - failed + type: string + - type: 'null' + title: Status + - name: partition_key_pattern + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + title: Partition Key Pattern + - name: order_by + in: query + required: false + schema: + type: string + default: -partition_key + title: Order By + - name: limit + in: query + required: false + schema: + type: integer + minimum: 0 + default: 50 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + minimum: 0 + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/AssetPartitionCollectionResponse' + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Bad Request + '404': + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPExceptionResponse' + description: Not Found + '422': + description: Validation Error + content: + application/json: + schema: + $ref: '#/components/schemas/HTTPValidationError' /ui/dags/{dag_id}/dagRuns/{dag_run_id}/stats: get: tags: @@ -3144,6 +3231,115 @@ components: title: AssetExpressionRef description: 'An unresolved asset reference leaf: ``{"asset_ref": {"name": ...}}`` or ``{"asset_ref": {"uri": ...}}``.' + AssetPartitionCollectionResponse: + properties: + partitions: + items: + $ref: '#/components/schemas/AssetPartitionResponse' + type: array + title: Partitions + total_entries: + type: integer + title: Total Entries + status_counts: + $ref: '#/components/schemas/AssetPartitionStatusCounts' + type: object + required: + - partitions + - total_entries + - status_counts + title: AssetPartitionCollectionResponse + description: Partition keys of an asset with their status. + AssetPartitionProducerRunResponse: + properties: + dag_id: + type: string + title: Dag Id + run_id: + type: string + title: Run Id + state: + $ref: '#/components/schemas/DagRunState' + start_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Start Date + end_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: End Date + type: object + required: + - dag_id + - run_id + - state + - start_date + - end_date + title: AssetPartitionProducerRunResponse + description: The latest run of a producing Dag for one partition key. + AssetPartitionResponse: + properties: + partition_key: + type: string + title: Partition Key + status: + type: string + enum: + - materialized + - in_progress + - failed + title: Status + last_event_id: + anyOf: + - type: integer + - type: 'null' + title: Last Event Id + last_event_timestamp: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Last Event Timestamp + last_producer_run: + anyOf: + - $ref: '#/components/schemas/AssetPartitionProducerRunResponse' + - type: 'null' + consumer_runs: + items: + $ref: '#/components/schemas/DagRunAssetReference' + type: array + title: Consumer Runs + type: object + required: + - partition_key + - status + - last_event_id + - last_event_timestamp + - last_producer_run + - consumer_runs + title: AssetPartitionResponse + description: Status of one partition key of an asset. + AssetPartitionStatusCounts: + properties: + materialized: + type: integer + title: Materialized + default: 0 + in_progress: + type: integer + title: In Progress + default: 0 + failed: + type: integer + title: Failed + default: 0 + type: object + title: AssetPartitionStatusCounts + description: Number of partition keys per status. AssetResponse: properties: id: @@ -4055,6 +4251,73 @@ components: title: DagFolderResponse description: A distinct Dag folder (directory of ``relative_fileloc``) within a bundle. + DagRunAssetReference: + properties: + run_id: + type: string + title: Run Id + dag_id: + type: string + title: Dag Id + logical_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Logical Date + start_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Start Date + end_date: + anyOf: + - type: string + format: date-time + - type: 'null' + title: End Date + state: + type: string + title: State + data_interval_start: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Data Interval Start + data_interval_end: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Data Interval End + partition_key: + anyOf: + - type: string + - type: 'null' + title: Partition Key + triggering: + type: boolean + title: Triggering + description: Whether this asset event triggered the referenced dag run. + Only a run's most recent consumed asset event triggers it; earlier consumed + events are included in the run but did not trigger it. + additionalProperties: false + type: object + required: + - run_id + - dag_id + - logical_date + - start_date + - end_date + - state + - data_interval_start + - data_interval_end + - partition_key + - triggering + title: DagRunAssetReference + description: DagRun serializer for asset responses. DagRunState: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/assets.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/assets.py index 25c74fae942..1997963c039 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/assets.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/assets.py @@ -16,15 +16,19 @@ # under the License. from __future__ import annotations -from typing import TYPE_CHECKING, Annotated, Any, cast +from typing import TYPE_CHECKING, Annotated, Any, cast, get_args import structlog -from fastapi import Depends, HTTPException, status -from sqlalchemy import ColumnElement, and_, case, exists, func, select, true +from fastapi import Depends, HTTPException, Query, status +from sqlalchemy import ColumnElement, and_, case, exists, func, or_, select, true, union +from sqlalchemy.orm import aliased, selectinload from airflow.api_fastapi.common.asset_expression import redact_asset_expression -from airflow.api_fastapi.common.db.assets import generate_assets_with_last_event_query -from airflow.api_fastapi.common.db.common import SessionDep, paginated_select +from airflow.api_fastapi.common.db.assets import ( + generate_assets_with_last_event_query, + resolve_triggering_event_refs, +) +from airflow.api_fastapi.common.db.common import SessionDep, apply_filters_to_select, paginated_select from airflow.api_fastapi.common.parameters import ( QueryAssetDagIdPatternSearch, QueryAssetGroupPatternSearch, @@ -43,8 +47,17 @@ from airflow.api_fastapi.common.parameters import ( ) from airflow.api_fastapi.common.partition_helpers import load_partitioned_timetable from airflow.api_fastapi.common.router import AirflowRouter -from airflow.api_fastapi.core_api.datamodels.assets import AssetCollectionResponse, AssetResponse +from airflow.api_fastapi.core_api.datamodels.assets import ( + AssetCollectionResponse, + AssetResponse, + DagRunAssetReference, +) from airflow.api_fastapi.core_api.datamodels.ui.assets import ( + AssetPartitionCollectionResponse, + AssetPartitionProducerRunResponse, + AssetPartitionResponse, + AssetPartitionStatus, + AssetPartitionStatusCounts, NextRunAssetEventResponse, NextRunAssetsResponse, ) @@ -66,9 +79,14 @@ from airflow.models.asset import ( AssetPartitionDagRun, DagScheduleAssetReference, PartitionedAssetKeyLog, + TaskOutletAssetReference, ) +from airflow.models.dagrun import DagRun +from airflow.utils.state import DagRunState if TYPE_CHECKING: + from sqlalchemy import Subquery + from airflow.partition_mappers.base import RollupMapper log = structlog.get_logger(logger_name=__name__) @@ -390,3 +408,264 @@ def next_run_assets( "pending_partition_count": pending_partition_count, } return NextRunAssetsResponse.model_validate(model_data) + + +def _build_asset_partition_status_subquery(asset_id: int, readable_dag_ids: set[str] | None) -> Subquery: + """ + Classify every partition key of an asset. + + A producer run belongs to its own ``partition_key``, which is the key its asset + events get unless the task overrides it. A run that emitted this asset, but never + under its own key, is known to have overridden the key and is left out. For each + key, ``E`` is the newest event and ``R`` the newest attempt among the runs of the + Dags that declare the asset as an outlet: + + - ``R`` queued or running, and no ``E`` since the attempt began: ``in_progress`` + - ``R`` failed, and no ``E`` since the attempt began: ``failed`` + - otherwise, ``E`` exists: ``materialized`` + - otherwise the key is left out + + An attempt begins at ``queued_at``, which is also reset when a task of the run is + cleared, whether or not the run has finished. A cleared run is therefore the newest + attempt and counts as not having produced an event yet. + + Runs and events of Dags the caller may not read are left out before anything is + aggregated, so neither the rows nor the counts reveal them. + """ + event_filters = [AssetEvent.asset_id == asset_id, AssetEvent.partition_key.is_not(None)] + run_filters = [ + DagRun.dag_id.in_( + select(TaskOutletAssetReference.dag_id).where(TaskOutletAssetReference.asset_id == asset_id) + ), + DagRun.partition_key.is_not(None), + ] + if readable_dag_ids is not None: + # Events created through the API or by a watcher have no source Dag to authorize on. + event_filters.append( + or_(AssetEvent.source_dag_id.in_(readable_dag_ids), AssetEvent.source_dag_id.is_(None)) + ) + run_filters.append(DagRun.dag_id.in_(readable_dag_ids)) + + ranked_events = ( + select( + AssetEvent.partition_key.label("partition_key"), + AssetEvent.id.label("event_id"), + AssetEvent.timestamp.label("event_timestamp"), + func.row_number() + .over( + partition_by=AssetEvent.partition_key, + order_by=(AssetEvent.timestamp.desc(), AssetEvent.id.desc()), + ) + .label("position"), + ) + .where(*event_filters) + .subquery("ranked_events") + ) + last_events = select(ranked_events).where(ranked_events.c.position == 1).cte("last_events") + + # One pass over the asset's events, rather than a lookup per run. + source_run = aliased(DagRun) + emitted_under_own_key = func.sum(case((AssetEvent.partition_key == source_run.partition_key, 1), else_=0)) + key_overriding_runs = ( + select(AssetEvent.source_dag_id.label("dag_id"), AssetEvent.source_run_id.label("run_id")) + .join( + source_run, + and_( + source_run.dag_id == AssetEvent.source_dag_id, + source_run.run_id == AssetEvent.source_run_id, + ), + ) + .where(AssetEvent.asset_id == asset_id, AssetEvent.partition_key.is_not(None)) + .group_by(AssetEvent.source_dag_id, AssetEvent.source_run_id) + .having(emitted_under_own_key == 0) + .cte("key_overriding_runs") + ) + attempt_began_at = func.coalesce(DagRun.queued_at, DagRun.start_date, DagRun.run_after) + ranked_runs = ( + select( + DagRun.partition_key.label("partition_key"), + DagRun.dag_id.label("run_dag_id"), + DagRun.run_id.label("run_id"), + DagRun.state.label("run_state"), + DagRun.start_date.label("run_start_date"), + DagRun.end_date.label("run_end_date"), + attempt_began_at.label("attempt_began_at"), + func.row_number() + .over( + partition_by=DagRun.partition_key, + order_by=(attempt_began_at.desc(), DagRun.id.desc()), + ) + .label("position"), + ) + .outerjoin( + key_overriding_runs, + and_( + key_overriding_runs.c.dag_id == DagRun.dag_id, + key_overriding_runs.c.run_id == DagRun.run_id, + ), + ) + .where(*run_filters, key_overriding_runs.c.run_id.is_(None)) + .subquery("ranked_runs") + ) + last_runs = select(ranked_runs).where(ranked_runs.c.position == 1).cte("last_runs") + keys = union(select(last_events.c.partition_key), select(last_runs.c.partition_key)).cte("partition_keys") + + no_event_since_attempt_began = or_( + last_events.c.event_id.is_(None), + last_events.c.event_timestamp < last_runs.c.attempt_began_at, + ) + partition_status = case( + ( + and_( + last_runs.c.run_state.in_([DagRunState.QUEUED, DagRunState.RUNNING]), + no_event_since_attempt_began, + ), + "in_progress", + ), + (and_(last_runs.c.run_state == DagRunState.FAILED, no_event_since_attempt_began), "failed"), + (last_events.c.event_id.is_not(None), "materialized"), + else_=None, + ) + return ( + select( + keys.c.partition_key, + partition_status.label("status"), + last_events.c.event_id.label("last_event_id"), + last_events.c.event_timestamp.label("last_event_timestamp"), + last_runs.c.run_dag_id, + last_runs.c.run_id, + last_runs.c.run_state, + last_runs.c.run_start_date, + last_runs.c.run_end_date, + ) + .select_from(keys) + .outerjoin(last_events, last_events.c.partition_key == keys.c.partition_key) + .outerjoin(last_runs, last_runs.c.partition_key == keys.c.partition_key) + .subquery("asset_partitions") + ) + + +PARTITION_SORT_COLUMNS = { + "partition_key": "partition_key", + "status": "status", + "last_event_timestamp": "last_event_timestamp", + "last_producer_run": "run_start_date", +} + + +def _build_partition_order(columns: Any, order_by: str) -> list[ColumnElement]: + """Order by an allow-listed column, NULLs last in both directions, with the unique key as tiebreak.""" + sort_name = order_by.removeprefix("-") + column = columns.c[PARTITION_SORT_COLUMNS[sort_name]] + sort_key = [ + case((column.is_(None), 1), else_=0), + column.desc() if order_by.startswith("-") else column.asc(), + ] + return sort_key if sort_name == "partition_key" else [*sort_key, columns.c.partition_key.asc()] + + +@assets_router.get( + "/assets/{asset_id}/partitions", + responses=create_openapi_http_exception_doc([status.HTTP_400_BAD_REQUEST, status.HTTP_404_NOT_FOUND]), + dependencies=[Depends(requires_access_asset(method="GET"))], +) +def get_asset_partitions( + asset_id: int, + limit: QueryLimit, + offset: QueryOffset, + readable_dags_filter: ReadableDagsFilterDep, + session: SessionDep, + partition_status: Annotated[list[AssetPartitionStatus] | None, Query(alias="status")] = None, + partition_key_pattern: str | None = None, + order_by: str = "-partition_key", +) -> AssetPartitionCollectionResponse: + """Return the partition keys of an asset with the status of each.""" + sort_name = order_by.removeprefix("-") + if sort_name not in PARTITION_SORT_COLUMNS: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + f"Ordering with '{order_by}' is disallowed or the attribute does not exist on the model", + ) + if session.scalar(select(AssetModel.id).where(AssetModel.id == asset_id)) is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, f"The Asset with ID: `{asset_id}` was not found") + + readable_dag_ids = readable_dags_filter.value + partitions = _build_asset_partition_status_subquery(asset_id, readable_dag_ids) + filters: list[ColumnElement[bool]] = [partitions.c.status.is_not(None)] + if partition_key_pattern: + filters.append(partitions.c.partition_key.icontains(partition_key_pattern, autoescape=True)) + + selected = set(partition_status or get_args(AssetPartitionStatus)) + filtered = select(partitions).where(*filters).cte("filtered_partitions") + counts = ( + select( + *[ + func.coalesce(func.sum(case((filtered.c.status == name, 1), else_=0)), 0).label( + f"{name}_count" + ) + for name in get_args(AssetPartitionStatus) + ] + ) + .select_from(filtered) + .cte("partition_counts") + ) + page = apply_filters_to_select( + statement=select(filtered) + .where(filtered.c.status.in_(selected)) + .order_by(*_build_partition_order(filtered, order_by)), + filters=[offset, limit], + ).cte("partition_page") + # The counts row is kept by the outer join, so an empty page still carries the counts. + all_rows = session.execute( + select(*counts.c, *page.c) + .select_from(counts.outerjoin(page, true())) + .order_by(*_build_partition_order(page, order_by)) + ).all() + status_counts = AssetPartitionStatusCounts.model_validate( + {name: getattr(all_rows[0], f"{name}_count") for name in get_args(AssetPartitionStatus)} + ) + total_entries = sum(getattr(status_counts, name) for name in selected) + rows = [row for row in all_rows if row.partition_key is not None] + + consumer_runs: dict[int, list[DagRunAssetReference]] = {} + if last_event_ids := [row.last_event_id for row in rows if row.last_event_id is not None]: + events = session.scalars( + select(AssetEvent) + .where(AssetEvent.id.in_(last_event_ids)) + .options(selectinload(AssetEvent.created_dagruns)) + ).all() + triggering_refs = resolve_triggering_event_refs(events, session=session) + for event in events: + runs = [] + for run in event.created_dagruns: + # In this route ``None`` means every Dag is readable. + if readable_dag_ids is None or run.dag_id in readable_dag_ids: + run.triggering = (event.id, run.dag_id, run.run_id) in triggering_refs + runs.append(DagRunAssetReference.model_validate(run)) + consumer_runs[event.id] = runs + + return AssetPartitionCollectionResponse( + partitions=[ + AssetPartitionResponse( + partition_key=row.partition_key, + status=row.status, + last_event_id=row.last_event_id, + last_event_timestamp=row.last_event_timestamp, + last_producer_run=( + None + if row.run_dag_id is None + else AssetPartitionProducerRunResponse( + dag_id=row.run_dag_id, + run_id=row.run_id, + state=row.run_state, + start_date=row.run_start_date, + end_date=row.run_end_date, + ) + ), + consumer_runs=consumer_runs.get(row.last_event_id, []), + ) + for row in rows + ], + total_entries=total_entries, + status_counts=status_counts, + ) diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index 10681aaf352..755187ecc45 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -113,6 +113,17 @@ export const useAssetServiceNextRunAssetsKey = "AssetServiceNextRunAssets"; export const UseAssetServiceNextRunAssetsKeyFn = ({ dagId }: { dagId: string; }, queryKey?: Array<unknown>) => [useAssetServiceNextRunAssetsKey, ...(queryKey ?? [{ dagId }])]; +export type AssetServiceGetAssetPartitionsDefaultResponse = Awaited<ReturnType<typeof AssetService.getAssetPartitions>>; +export type AssetServiceGetAssetPartitionsQueryResult<TData = AssetServiceGetAssetPartitionsDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>; +export const useAssetServiceGetAssetPartitionsKey = "AssetServiceGetAssetPartitions"; +export const UseAssetServiceGetAssetPartitionsKeyFn = ({ assetId, limit, offset, orderBy, partitionKeyPattern, status }: { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string; + status?: ("materialized" | "in_progress" | "failed")[]; +}, queryKey?: Array<unknown>) => [useAssetServiceGetAssetPartitionsKey, ...(queryKey ?? [{ assetId, limit, offset, orderBy, partitionKeyPattern, status }])]; export type BackfillServiceListBackfillsDefaultResponse = Awaited<ReturnType<typeof BackfillService.listBackfills>>; export type BackfillServiceListBackfillsQueryResult<TData = BackfillServiceListBackfillsDefaultResponse, TError = unknown> = UseQueryResult<TData, TError>; export const useBackfillServiceListBackfillsKey = "BackfillServiceListBackfills"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index 0b154dc75b7..6947aba7e41 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -214,6 +214,27 @@ export const ensureUseAssetServiceNextRunAssetsData = (queryClient: QueryClient, dagId: string; }) => queryClient.ensureQueryData({ queryKey: Common.UseAssetServiceNextRunAssetsKeyFn({ dagId }), queryFn: () => AssetService.nextRunAssets({ dagId }) }); /** +* Get Asset Partitions +* Return the partition keys of an asset with the status of each. +* @param data The data for the request. +* @param data.assetId +* @param data.status +* @param data.partitionKeyPattern +* @param data.orderBy +* @param data.limit +* @param data.offset +* @returns AssetPartitionCollectionResponse Successful Response +* @throws ApiError +*/ +export const ensureUseAssetServiceGetAssetPartitionsData = (queryClient: QueryClient, { assetId, limit, offset, orderBy, partitionKeyPattern, status }: { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string; + status?: ("materialized" | "in_progress" | "failed")[]; +}) => queryClient.ensureQueryData({ queryKey: Common.UseAssetServiceGetAssetPartitionsKeyFn({ assetId, limit, offset, orderBy, partitionKeyPattern, status }), queryFn: () => AssetService.getAssetPartitions({ assetId, limit, offset, orderBy, partitionKeyPattern, status }) }); +/** * List Backfills * @param data The data for the request. * @param data.dagId diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index a7cff8201de..4050ad9bcb6 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -214,6 +214,27 @@ export const prefetchUseAssetServiceNextRunAssets = (queryClient: QueryClient, { dagId: string; }) => queryClient.prefetchQuery({ queryKey: Common.UseAssetServiceNextRunAssetsKeyFn({ dagId }), queryFn: () => AssetService.nextRunAssets({ dagId }) }); /** +* Get Asset Partitions +* Return the partition keys of an asset with the status of each. +* @param data The data for the request. +* @param data.assetId +* @param data.status +* @param data.partitionKeyPattern +* @param data.orderBy +* @param data.limit +* @param data.offset +* @returns AssetPartitionCollectionResponse Successful Response +* @throws ApiError +*/ +export const prefetchUseAssetServiceGetAssetPartitions = (queryClient: QueryClient, { assetId, limit, offset, orderBy, partitionKeyPattern, status }: { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string; + status?: ("materialized" | "in_progress" | "failed")[]; +}) => queryClient.prefetchQuery({ queryKey: Common.UseAssetServiceGetAssetPartitionsKeyFn({ assetId, limit, offset, orderBy, partitionKeyPattern, status }), queryFn: () => AssetService.getAssetPartitions({ assetId, limit, offset, orderBy, partitionKeyPattern, status }) }); +/** * List Backfills * @param data The data for the request. * @param data.dagId diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index 1ae9ee003dd..3889c3830ab 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -214,6 +214,27 @@ export const useAssetServiceNextRunAssets = <TData = Common.AssetServiceNextRunA dagId: string; }, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseAssetServiceNextRunAssetsKeyFn({ dagId }, queryKey), queryFn: () => AssetService.nextRunAssets({ dagId }) as TData, ...options }); /** +* Get Asset Partitions +* Return the partition keys of an asset with the status of each. +* @param data The data for the request. +* @param data.assetId +* @param data.status +* @param data.partitionKeyPattern +* @param data.orderBy +* @param data.limit +* @param data.offset +* @returns AssetPartitionCollectionResponse Successful Response +* @throws ApiError +*/ +export const useAssetServiceGetAssetPartitions = <TData = Common.AssetServiceGetAssetPartitionsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ assetId, limit, offset, orderBy, partitionKeyPattern, status }: { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string; + status?: ("materialized" | "in_progress" | "failed")[]; +}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useQuery<TData, TError>({ queryKey: Common.UseAssetServiceGetAssetPartitionsKeyFn({ assetId, limit, offset, orderBy, partitionKeyPattern, status }, queryKey), queryFn: () => AssetService.getAssetPartitions({ assetId, limit, offset, orderBy, partitionKeyPattern, status }) as TData, ...options }); +/** * List Backfills * @param data The data for the request. * @param data.dagId diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index 5d72da28b64..6d17db89303 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -214,6 +214,27 @@ export const useAssetServiceNextRunAssetsSuspense = <TData = Common.AssetService dagId: string; }, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseAssetServiceNextRunAssetsKeyFn({ dagId }, queryKey), queryFn: () => AssetService.nextRunAssets({ dagId }) as TData, ...options }); /** +* Get Asset Partitions +* Return the partition keys of an asset with the status of each. +* @param data The data for the request. +* @param data.assetId +* @param data.status +* @param data.partitionKeyPattern +* @param data.orderBy +* @param data.limit +* @param data.offset +* @returns AssetPartitionCollectionResponse Successful Response +* @throws ApiError +*/ +export const useAssetServiceGetAssetPartitionsSuspense = <TData = Common.AssetServiceGetAssetPartitionsDefaultResponse, TError = unknown, TQueryKey extends Array<unknown> = unknown[]>({ assetId, limit, offset, orderBy, partitionKeyPattern, status }: { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string; + status?: ("materialized" | "in_progress" | "failed")[]; +}, queryKey?: TQueryKey, options?: Omit<UseQueryOptions<TData, TError>, "queryKey" | "queryFn">) => useSuspenseQuery<TData, TError>({ queryKey: Common.UseAssetServiceGetAssetPartitionsKeyFn({ assetId, limit, offset, orderBy, partitionKeyPattern, status }, queryKey), queryFn: () => AssetService.getAssetPartitions({ assetId, limit, offset, orderBy, partitionKeyPattern, status }) as TData, ...options }); +/** * List Backfills * @param data The data for the request. * @param data.dagId diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 08807f6db11..c4be2450de3 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -10298,6 +10298,154 @@ export const $XComUpdateBody = { description: 'Payload serializer for updating an XCom entry.' } as const; +export const $AssetPartitionCollectionResponse = { + properties: { + partitions: { + items: { + '$ref': '#/components/schemas/AssetPartitionResponse' + }, + type: 'array', + title: 'Partitions' + }, + total_entries: { + type: 'integer', + title: 'Total Entries' + }, + status_counts: { + '$ref': '#/components/schemas/AssetPartitionStatusCounts' + } + }, + type: 'object', + required: ['partitions', 'total_entries', 'status_counts'], + title: 'AssetPartitionCollectionResponse', + description: 'Partition keys of an asset with their status.' +} as const; + +export const $AssetPartitionProducerRunResponse = { + properties: { + dag_id: { + type: 'string', + title: 'Dag Id' + }, + run_id: { + type: 'string', + title: 'Run Id' + }, + state: { + '$ref': '#/components/schemas/DagRunState' + }, + start_date: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Start Date' + }, + end_date: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'End Date' + } + }, + type: 'object', + required: ['dag_id', 'run_id', 'state', 'start_date', 'end_date'], + title: 'AssetPartitionProducerRunResponse', + description: 'The latest run of a producing Dag for one partition key.' +} as const; + +export const $AssetPartitionResponse = { + properties: { + partition_key: { + type: 'string', + title: 'Partition Key' + }, + status: { + type: 'string', + enum: ['materialized', 'in_progress', 'failed'], + title: 'Status' + }, + last_event_id: { + anyOf: [ + { + type: 'integer' + }, + { + type: 'null' + } + ], + title: 'Last Event Id' + }, + last_event_timestamp: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Last Event Timestamp' + }, + last_producer_run: { + anyOf: [ + { + '$ref': '#/components/schemas/AssetPartitionProducerRunResponse' + }, + { + type: 'null' + } + ] + }, + consumer_runs: { + items: { + '$ref': '#/components/schemas/DagRunAssetReference' + }, + type: 'array', + title: 'Consumer Runs' + } + }, + type: 'object', + required: ['partition_key', 'status', 'last_event_id', 'last_event_timestamp', 'last_producer_run', 'consumer_runs'], + title: 'AssetPartitionResponse', + description: 'Status of one partition key of an asset.' +} as const; + +export const $AssetPartitionStatusCounts = { + properties: { + materialized: { + type: 'integer', + title: 'Materialized', + default: 0 + }, + in_progress: { + type: 'integer', + title: 'In Progress', + default: 0 + }, + failed: { + type: 'integer', + title: 'Failed', + default: 0 + } + }, + type: 'object', + title: 'AssetPartitionStatusCounts', + description: 'Number of partition keys per status.' +} as const; + export const $AuthenticatedMeResponse = { properties: { id: { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index 5326f95df0f..495e9d7856a 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, Dele [...] +import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, Dele [...] export class AssetService { /** @@ -483,6 +483,41 @@ export class AssetService { }); } + /** + * Get Asset Partitions + * Return the partition keys of an asset with the status of each. + * @param data The data for the request. + * @param data.assetId + * @param data.status + * @param data.partitionKeyPattern + * @param data.orderBy + * @param data.limit + * @param data.offset + * @returns AssetPartitionCollectionResponse Successful Response + * @throws ApiError + */ + public static getAssetPartitions(data: GetAssetPartitionsData): CancelablePromise<GetAssetPartitionsResponse> { + return __request(OpenAPI, { + method: 'GET', + url: '/ui/assets/{asset_id}/partitions', + path: { + asset_id: data.assetId + }, + query: { + status: data.status, + partition_key_pattern: data.partitionKeyPattern, + order_by: data.orderBy, + limit: data.limit, + offset: data.offset + }, + errors: { + 400: 'Bad Request', + 404: 'Not Found', + 422: 'Validation Error' + } + }); + } + } export class BackfillService { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index a17c7ac102d..5031b528018 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -2650,6 +2650,49 @@ export type XComUpdateBody = { region_index?: number | null; }; +/** + * Partition keys of an asset with their status. + */ +export type AssetPartitionCollectionResponse = { + partitions: Array<AssetPartitionResponse>; + total_entries: number; + status_counts: AssetPartitionStatusCounts; +}; + +/** + * The latest run of a producing Dag for one partition key. + */ +export type AssetPartitionProducerRunResponse = { + dag_id: string; + run_id: string; + state: DagRunState; + start_date: string | null; + end_date: string | null; +}; + +/** + * Status of one partition key of an asset. + */ +export type AssetPartitionResponse = { + partition_key: string; + status: 'materialized' | 'in_progress' | 'failed'; + last_event_id: number | null; + last_event_timestamp: string | null; + last_producer_run: AssetPartitionProducerRunResponse | null; + consumer_runs: Array<DagRunAssetReference>; +}; + +export type status = 'materialized' | 'in_progress' | 'failed'; + +/** + * Number of partition keys per status. + */ +export type AssetPartitionStatusCounts = { + materialized?: number; + in_progress?: number; + failed?: number; +}; + /** * Authenticated user information serializer for responses. */ @@ -3237,7 +3280,7 @@ export type LoopRunSummary = { loop_region_id?: string | null; }; -export type status = 'running' | 'stopped_early' | 'ran_to_cap' | 'failed' | 'skipped' | 'removed'; +export type status2 = 'running' | 'stopped_early' | 'ran_to_cap' | 'failed' | 'skipped' | 'removed'; /** * Runtime state of one loop invocation. @@ -3705,6 +3748,17 @@ export type NextRunAssetsData = { export type NextRunAssetsResponse2 = NextRunAssetsResponse; +export type GetAssetPartitionsData = { + assetId: number; + limit?: number; + offset?: number; + orderBy?: string; + partitionKeyPattern?: string | null; + status?: Array<('materialized' | 'in_progress' | 'failed')> | null; +}; + +export type GetAssetPartitionsResponse = AssetPartitionCollectionResponse; + export type ListBackfillsData = { dagId: string; limit?: number; @@ -6103,6 +6157,29 @@ export type $OpenApiTs = { }; }; }; + '/ui/assets/{asset_id}/partitions': { + get: { + req: GetAssetPartitionsData; + res: { + /** + * Successful Response + */ + 200: AssetPartitionCollectionResponse; + /** + * Bad Request + */ + 400: HTTPExceptionResponse; + /** + * Not Found + */ + 404: HTTPExceptionResponse; + /** + * Validation Error + */ + 422: HTTPValidationError; + }; + }; + }; '/api/v2/backfills': { get: { req: ListBackfillsData; diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/assets.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/assets.json index bc0bb75f282..480ff96b3cb 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/assets.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/assets.json @@ -55,6 +55,15 @@ "group": "Group", "lastAssetEvent": "Last Asset Event", "name": "Name", + "partitions": { + "consumerRuns": "Consumer Runs", + "consumerRunsHelp": "Dag runs that consumed the last asset event of the partition", + "lastProducerRun": "Last Producer Run", + "status": { + "in_progress": "In Progress", + "materialized": "Materialized" + } + }, "producingTasks": "Producing Tasks", "scheduledDags": "Scheduled Dags", "scheduling": "Scheduling", diff --git a/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx b/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx index 2dc992ccb38..c89add8bf91 100644 --- a/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx +++ b/airflow-core/src/airflow/ui/src/constants/filterConfigs.tsx @@ -405,6 +405,27 @@ export const useFilterConfigs = () => { supportsAdvancedSearch: true, type: FilterTypes.TEXT, }, + [SearchParamsKeys.PARTITION_STATUS]: { + icon: <MdCheckCircle />, + label: translate("common:state"), + options: [ + { + label: ( + <StateBadge state="success">{translate("assets:partitions.status.materialized")}</StateBadge> + ), + value: "materialized", + }, + { + label: <StateBadge state="running">{translate("assets:partitions.status.in_progress")}</StateBadge>, + value: "in_progress", + }, + { + label: <StateBadge state="failed">{translate("common:states.failed")}</StateBadge>, + value: "failed", + }, + ], + type: FilterTypes.MULTISELECT, + }, [SearchParamsKeys.PAUSED]: { fromSearchParams: pausedStateFromSearchParams, icon: <MdPause />, diff --git a/airflow-core/src/airflow/ui/src/constants/searchParams.ts b/airflow-core/src/airflow/ui/src/constants/searchParams.ts index ca604eefc14..5e4aba20440 100644 --- a/airflow-core/src/airflow/ui/src/constants/searchParams.ts +++ b/airflow-core/src/airflow/ui/src/constants/searchParams.ts @@ -104,6 +104,7 @@ export enum SearchParamsKeys { OWNERS = "owners", PARTITION_KEY = "partition_key", PARTITION_KEY_PATTERN = "partition_key_pattern", + PARTITION_STATUS = "partition_status", PAUSED = "paused", POOL = "pool", POOL_NAME_PATTERN = "pool_name_pattern", diff --git a/airflow-core/src/airflow/ui/src/pages/Asset/AssetLayout.tsx b/airflow-core/src/airflow/ui/src/pages/Asset/AssetLayout.tsx index 7bd53037598..e75ec9c4be7 100644 --- a/airflow-core/src/airflow/ui/src/pages/Asset/AssetLayout.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Asset/AssetLayout.tsx @@ -21,7 +21,7 @@ import { useState } from "react"; import { Box, Code, HStack, Text } from "@chakra-ui/react"; import { useReactFlow } from "@xyflow/react"; import { useTranslation } from "react-i18next"; -import { MdOutlineStorage, MdTimeline } from "react-icons/md"; +import { MdOutlineStorage, MdOutlineViewModule, MdTimeline } from "react-icons/md"; import { Group, Panel, Separator, useDefaultLayout } from "react-resizable-panels"; import { Outlet, useParams } from "react-router-dom"; @@ -66,6 +66,7 @@ export const AssetLayout = () => { const tabs = [ { icon: <MdTimeline />, label: translate("assets:events"), value: "" }, + { icon: <MdOutlineViewModule />, label: translate("common:partition_other"), value: "partitions" }, { icon: <MdOutlineStorage />, label: translate("assets:assetStateStore.title"), diff --git a/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.test.tsx b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.test.tsx new file mode 100644 index 00000000000..a6ff5f76829 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.test.tsx @@ -0,0 +1,185 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import "@testing-library/jest-dom/vitest"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import type * as ReactI18Next from "react-i18next"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { AssetService } from "openapi/requests"; +import type { AssetPartitionResponse, AssetPartitionStatusCounts } from "openapi/requests/types.gen"; + +import { TimezoneProvider } from "src/context/timezone"; +import { BaseWrapper } from "src/utils/Wrapper"; + +import { AssetPartitions } from "./AssetPartitions"; + +vi.mock("src/queries/useConfig", () => ({ useConfig: () => 25 })); + +vi.mock("react-i18next", async (importOriginal) => ({ + ...(await importOriginal<typeof ReactI18Next>()), + useTranslation: () => ({ + i18n: { language: "en" }, + // eslint-disable-next-line id-length + t: (key: string) => key, + }), +})); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + localStorage.clear(); +}); + +const partition = ( + key: string, + status: AssetPartitionResponse["status"], + extra: Partial<AssetPartitionResponse> = {}, +): AssetPartitionResponse => ({ + consumer_runs: [], + last_event_id: null, + last_event_timestamp: null, + last_producer_run: null, + partition_key: key, + status, + ...extra, +}); + +const renderPartitions = ( + partitions: Array<AssetPartitionResponse>, + statusCounts: AssetPartitionStatusCounts = { failed: 1, in_progress: 0, materialized: 2 }, + initialEntry = "/assets/1/partitions", +) => { + const getAssetPartitions = vi.spyOn(AssetService, "getAssetPartitions").mockResolvedValue({ + partitions, + status_counts: statusCounts, + total_entries: partitions.length, + }); + + render( + <BaseWrapper> + <MemoryRouter initialEntries={[initialEntry]}> + <TimezoneProvider> + <Routes> + <Route element={<AssetPartitions />} path="/assets/:assetId/partitions" /> + </Routes> + </TimezoneProvider> + </MemoryRouter> + </BaseWrapper>, + ); + + return getAssetPartitions; +}; + +describe("AssetPartitions", () => { + it("shows the count of each status and one row per key", async () => { + renderPartitions([ + partition("2026-01-02", "failed", { + consumer_runs: [ + { + dag_id: "consumer", + data_interval_end: null, + data_interval_start: null, + end_date: null, + logical_date: null, + partition_key: "2026-01-02", + run_id: "consumed__2026-01-02", + start_date: null, + state: "success", + triggering: true, + }, + ], + last_producer_run: { + dag_id: "producer", + end_date: null, + run_id: "scheduled__2026-01-02", + start_date: null, + state: "failed", + }, + }), + partition("2026-01-01", "materialized"), + ]); + + expect(await screen.findByRole("link", { name: "producer" })).toHaveAttribute( + "href", + "/dags/producer/runs/scheduled__2026-01-02", + ); + expect(screen.getByRole("link", { name: "consumer" })).toHaveAttribute( + "href", + "/dags/consumer/runs/consumed__2026-01-02", + ); + expect(screen.getByTestId("partition-status-materialized")).toHaveTextContent("2"); + expect(screen.getByTestId("partition-status-failed")).toHaveTextContent("1"); + expect(screen.getByTestId("partition-status-in_progress")).toHaveTextContent("0"); + }); + + it("filters by the statuses in the URL and does not filter from the counts", async () => { + const getAssetPartitions = renderPartitions( + [partition("2026-01-01", "failed")], + undefined, + "/assets/1/partitions?partition_status=failed&partition_status=in_progress", + ); + + await waitFor(() => + expect(getAssetPartitions).toHaveBeenLastCalledWith( + expect.objectContaining({ status: ["failed", "in_progress"] }), + ), + ); + expect(screen.getByText("failed, in_progress")).toBeInTheDocument(); + + const requests = getAssetPartitions.mock.calls.length; + + fireEvent.click(screen.getByTestId("partition-status-materialized")); + expect(getAssetPartitions).toHaveBeenCalledTimes(requests); + }); + + it("sorts by the clicked column", async () => { + const getAssetPartitions = renderPartitions([partition("2026-01-01", "materialized")]); + + expect(await screen.findByText("2026-01-01")).toBeInTheDocument(); + expect(getAssetPartitions).toHaveBeenLastCalledWith( + expect.objectContaining({ orderBy: "-partition_key" }), + ); + + fireEvent.click(screen.getByText("assets:lastAssetEvent")); + await waitFor(() => + expect(getAssetPartitions).toHaveBeenLastCalledWith( + expect.objectContaining({ orderBy: "last_event_timestamp" }), + ), + ); + }); + + it("refreshes only while a partition is in progress", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const getAssetPartitions = renderPartitions([], { failed: 0, in_progress: 1, materialized: 0 }); + + await waitFor(() => expect(getAssetPartitions).toHaveBeenCalledTimes(1)); + await vi.advanceTimersByTimeAsync(26_000); + await waitFor(() => expect(getAssetPartitions.mock.calls.length).toBeGreaterThan(1)); + }); + + it("does not refresh when no partition is in progress", async () => { + vi.useFakeTimers({ shouldAdvanceTime: true }); + const getAssetPartitions = renderPartitions([], { failed: 1, in_progress: 0, materialized: 0 }); + + await waitFor(() => expect(getAssetPartitions).toHaveBeenCalledTimes(1)); + await vi.advanceTimersByTimeAsync(60_000); + expect(getAssetPartitions).toHaveBeenCalledTimes(1); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.tsx b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.tsx new file mode 100644 index 00000000000..fb1803f95d8 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/AssetPartitions.tsx @@ -0,0 +1,178 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, HStack, Text } from "@chakra-ui/react"; +import type { ColumnDef } from "@tanstack/react-table"; +import { useTranslation } from "react-i18next"; +import { useParams, useSearchParams } from "react-router-dom"; + +import { useAssetServiceGetAssetPartitions } from "openapi/queries"; +import type { AssetPartitionResponse } from "openapi/requests/types.gen"; + +import { RouterLink, Tooltip } from "src/system-components"; + +import { TriggeredRuns } from "src/components/Assets/TriggeredRuns"; +import { DataTable } from "src/components/DataTable"; +import type { DataTableFeatures } from "src/components/DataTable/features"; +import { useTableURLState } from "src/components/DataTable/useTableUrlState"; +import { ErrorAlert } from "src/components/ErrorAlert"; +import { FilterBar } from "src/components/FilterBar"; +import { StateBadge } from "src/components/StateBadge"; +import Time from "src/components/Time"; + +import { SearchParamsKeys } from "src/constants/searchParams"; +import { formatNumber, useAutoRefresh } from "src/utils"; +import { useFiltersHandler, type FilterableSearchParamsKeys } from "src/utils/useFiltersHandler"; + +import { + PARTITION_STATUSES, + type PartitionStatus, + RUN_STATE_BY_STATUS, + STATUS_LABEL_KEY, +} from "./partitionStatus"; + +const FILTER_KEYS: Array<FilterableSearchParamsKeys> = [ + SearchParamsKeys.PARTITION_KEY, + SearchParamsKeys.PARTITION_STATUS, +]; + +const isPartitionStatus = (value: string): value is PartitionStatus => + (PARTITION_STATUSES as Array<string>).includes(value); + +const getColumns = ( + translate: (key: string) => string, +): Array<ColumnDef<DataTableFeatures, AssetPartitionResponse>> => [ + { + accessorKey: "partition_key", + header: translate("common:dagRun.partitionKey"), + }, + { + accessorKey: "status", + cell: ({ row: { original } }) => ( + <StateBadge state={RUN_STATE_BY_STATUS[original.status]}> + {translate(STATUS_LABEL_KEY[original.status])} + </StateBadge> + ), + header: translate("common:state"), + }, + { + accessorKey: "last_producer_run", + cell: ({ row: { original } }) => { + const run = original.last_producer_run; + + return run === null ? undefined : ( + <HStack gap={2}> + <StateBadge state={run.state} /> + <RouterLink + overflowWrap="anywhere" + to={`/dags/${run.dag_id}/runs/${encodeURIComponent(run.run_id)}`} + > + {run.dag_id} + </RouterLink> + </HStack> + ); + }, + header: translate("assets:partitions.lastProducerRun"), + }, + { + accessorKey: "last_event_timestamp", + cell: ({ row: { original } }) => + original.last_event_timestamp === null ? undefined : ( + <Time datetime={original.last_event_timestamp} whiteSpace="nowrap" /> + ), + header: translate("assets:lastAssetEvent"), + }, + { + accessorKey: "consumer_runs", + cell: ({ row: { original } }) => <TriggeredRuns dagRuns={original.consumer_runs} />, + enableSorting: false, + header: () => ( + <Tooltip content={translate("assets:partitions.consumerRunsHelp")}> + <span>{translate("assets:partitions.consumerRuns")}</span> + </Tooltip> + ), + }, +]; + +export const AssetPartitions = () => { + const { i18n, t: translate } = useTranslation(["assets", "common"]); + const { assetId: rawAssetId } = useParams(); + const assetId = rawAssetId === undefined ? 0 : parseInt(rawAssetId, 10); + const { setTableURLState, tableURLState } = useTableURLState(); + const { pagination, sorting } = tableURLState; + const [sort] = sorting; + const orderBy = sort ? `${sort.desc ? "-" : ""}${sort.id}` : "-partition_key"; + const [searchParams] = useSearchParams(); + const { filterConfigs, handleFiltersChange, initialValues } = useFiltersHandler(FILTER_KEYS); + const refetchInterval = useAutoRefresh({}); + + const selectedStatuses = searchParams.getAll(SearchParamsKeys.PARTITION_STATUS).filter(isPartitionStatus); + + const { data, error, isLoading } = useAssetServiceGetAssetPartitions( + { + assetId, + limit: pagination.pageSize, + offset: pagination.pageIndex * pagination.pageSize, + orderBy, + partitionKeyPattern: searchParams.get(SearchParamsKeys.PARTITION_KEY) ?? undefined, + status: selectedStatuses.length === 0 ? undefined : selectedStatuses, + }, + undefined, + { + enabled: assetId !== 0, + // Counted before the status filter and the page, so a hidden in-progress key keeps this alive. + refetchInterval: (query) => (query.state.data?.status_counts.in_progress ?? 0) > 0 && refetchInterval, + }, + ); + + const columns = getColumns(translate); + + return ( + <Box display="flex" flex={1} flexDirection="column" gap={3} minH={0} overflow="auto" px={4} py={2}> + <ErrorAlert error={error} /> + <HStack flexWrap="wrap" gap={4}> + {PARTITION_STATUSES.map((status) => ( + <HStack data-testid={`partition-status-${status}`} gap={2} key={status}> + <StateBadge fontSize="md" state={RUN_STATE_BY_STATUS[status]}> + {data === undefined ? " " : formatNumber(data.status_counts[status] ?? 0, i18n.language)} + </StateBadge> + <Text fontSize="sm">{translate(STATUS_LABEL_KEY[status])}</Text> + </HStack> + ))} + </HStack> + <DataTable + columns={columns} + data={data?.partitions ?? []} + displayMode="table" + filterActions={ + <FilterBar + configs={filterConfigs} + initialValues={initialValues} + onFiltersChange={handleFiltersChange} + /> + } + hideRowCountHeading + initialState={tableURLState} + isLoading={isLoading} + modelName="common:partition" + onStateChange={setTableURLState} + total={data?.total_entries ?? 0} + /> + </Box> + ); +}; diff --git a/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/index.ts b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/index.ts new file mode 100644 index 00000000000..058bc3de13b --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/index.ts @@ -0,0 +1,20 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from "./AssetPartitions"; diff --git a/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/partitionStatus.ts b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/partitionStatus.ts new file mode 100644 index 00000000000..0001ad8cbb2 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/Asset/AssetPartitions/partitionStatus.ts @@ -0,0 +1,36 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import type { AssetPartitionResponse, TaskInstanceState } from "openapi/requests/types.gen"; + +export type PartitionStatus = AssetPartitionResponse["status"]; + +export const PARTITION_STATUSES: Array<PartitionStatus> = ["materialized", "in_progress", "failed"]; + +// A partition has no state of its own: it borrows the colour and icon of the run state it stands for. +export const RUN_STATE_BY_STATUS: Record<PartitionStatus, TaskInstanceState> = { + failed: "failed", + in_progress: "running", + materialized: "success", +}; + +export const STATUS_LABEL_KEY: Record<PartitionStatus, string> = { + failed: "common:states.failed", + in_progress: "assets:partitions.status.in_progress", + materialized: "assets:partitions.status.materialized", +}; diff --git a/airflow-core/src/airflow/ui/src/router.tsx b/airflow-core/src/airflow/ui/src/router.tsx index b7d43ad4115..2269f903e4e 100644 --- a/airflow-core/src/airflow/ui/src/router.tsx +++ b/airflow-core/src/airflow/ui/src/router.tsx @@ -27,6 +27,7 @@ import { DagsLayout } from "src/layouts/DagsLayout"; import { Asset } from "src/pages/Asset"; import { AssetEvents } from "src/pages/Asset/AssetEvents"; +import { AssetPartitions } from "src/pages/Asset/AssetPartitions"; import { AssetStateStore } from "src/pages/Asset/AssetStateStore"; import { AssetsList } from "src/pages/AssetsList"; import { Configs } from "src/pages/Configs"; @@ -150,6 +151,7 @@ export const routerConfig = [ { children: [ { element: <AssetEvents />, index: true }, + { element: <AssetPartitions />, path: "partitions" }, { element: <AssetStateStore />, path: "asset-state-store" }, pluginRoute, ], diff --git a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts index 99786b38a91..eacbda58341 100644 --- a/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts +++ b/airflow-core/src/airflow/ui/src/utils/useFiltersHandler.ts @@ -98,6 +98,7 @@ export type FilterableSearchParamsKeys = | SearchParamsKeys.OWNERS | SearchParamsKeys.PARTITION_KEY | SearchParamsKeys.PARTITION_KEY_PATTERN + | SearchParamsKeys.PARTITION_STATUS | SearchParamsKeys.PAUSED | SearchParamsKeys.POOL_NAME_PATTERN | SearchParamsKeys.QUEUE_NAME_PATTERN diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_assets.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_assets.py index f964cd7a8ed..847644f719d 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_assets.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_assets.py @@ -39,6 +39,7 @@ from airflow.models.asset import ( TaskOutletAssetReference, ) from airflow.models.dagbundle import DagBundleModel +from airflow.models.dagrun import DagRun from airflow.models.team import Team from airflow.models.trigger import Trigger from airflow.partition_mappers.base import RollupMapper @@ -47,6 +48,8 @@ from airflow.partition_mappers.window import HourWindow from airflow.providers.standard.operators.empty import EmptyOperator from airflow.sdk.definitions.asset import Asset from airflow.sdk.definitions.timetables.assets import PartitionedAssetTimetable +from airflow.utils.state import DagRunState +from airflow.utils.types import DagRunType from tests_common.test_utils.asserts import assert_queries_count from tests_common.test_utils.config import conf_vars @@ -56,6 +59,7 @@ from tests_common.test_utils.db import ( clear_db_dag_bundles, clear_db_dags, clear_db_pakl, + clear_db_runs, clear_db_serialized_dags, clear_db_teams, ) @@ -1101,3 +1105,473 @@ class TestGetAssetsUi: assets_by_name = {asset["name"]: asset for asset in response.json()["assets"]} assert assets_by_name["asset0"]["scheduled_dags"][0]["team_name"] == "owning-team" assert assets_by_name["asset0"]["producing_tasks"][0]["team_name"] == "owning-team" + + +RUN_START = pendulum.datetime(2026, 1, 2, 10, tz="UTC") +BEFORE_RUN_START = RUN_START.subtract(hours=1) +AFTER_RUN_START = RUN_START.add(minutes=30) + + +class TestGetAssetPartitions: + @pytest.fixture(autouse=True) + def cleanup_assets(self): + clear_db_assets() + clear_db_runs() + + yield + + clear_db_assets() + clear_db_runs() + + @pytest.fixture + def asset(self, session): + asset = AssetModel(name="partitioned", uri="s3://bucket/partitioned", group="asset") + session.add_all([asset, DagBundleModel(name="testing")]) + session.flush() + return asset + + @staticmethod + def _add_producer(session, asset, dag_id="producer"): + session.add_all( + [ + DagModel(dag_id=dag_id, bundle_name="testing"), + TaskOutletAssetReference(dag_id=dag_id, task_id="produce", asset_id=asset.id), + ] + ) + + @staticmethod + def _add_run(session, key, state, start_date=RUN_START, dag_id="producer", queued_at=None): + run = DagRun( + dag_id=dag_id, + run_id=f"run_{key}_{state}", + run_type=DagRunType.SCHEDULED, + state=state, + start_date=start_date, + partition_key=key, + queued_at=queued_at, + run_after=RUN_START, + ) + session.add(run) + session.flush() + return run + + @staticmethod + def _add_event(session, asset, key, timestamp=BEFORE_RUN_START, source_run=None): + event = AssetEvent( + asset_id=asset.id, + partition_key=key, + timestamp=timestamp, + source_dag_id=source_run and source_run.dag_id, + source_run_id=source_run and source_run.run_id, + ) + session.add(event) + session.flush() + return event + + def test_should_respond_401(self, unauthenticated_test_client): + response = unauthenticated_test_client.get("/assets/1/partitions") + assert response.status_code == 401 + + def test_should_respond_403(self, unauthorized_test_client): + response = unauthorized_test_client.get("/assets/1/partitions") + assert response.status_code == 403 + + def test_should_respond_404(self, test_client): + response = test_client.get("/assets/9999/partitions") + assert response.status_code == 404 + + @pytest.mark.parametrize( + ("run_state", "run_start_date", "event_timestamp", "expected_status"), + [ + pytest.param(None, None, BEFORE_RUN_START, "materialized", id="event-without-run"), + pytest.param(DagRunState.RUNNING, RUN_START, None, "in_progress", id="running-without-event"), + pytest.param(DagRunState.QUEUED, None, BEFORE_RUN_START, "in_progress", id="cleared-run"), + pytest.param(DagRunState.FAILED, RUN_START, None, "failed", id="failed-without-event"), + pytest.param(DagRunState.FAILED, None, BEFORE_RUN_START, "failed", id="failed-before-start"), + pytest.param( + DagRunState.FAILED, RUN_START, BEFORE_RUN_START, "failed", id="old-event-then-failed-run" + ), + pytest.param( + DagRunState.FAILED, RUN_START, AFTER_RUN_START, "materialized", id="event-before-run-fails" + ), + pytest.param( + DagRunState.RUNNING, RUN_START, AFTER_RUN_START, "materialized", id="event-of-running-run" + ), + pytest.param( + DagRunState.SUCCESS, RUN_START, BEFORE_RUN_START, "materialized", id="old-event-then-success" + ), + pytest.param(DagRunState.SUCCESS, RUN_START, None, None, id="success-without-event"), + ], + ) + def test_status_of_a_key( + self, test_client, session, asset, run_state, run_start_date, event_timestamp, expected_status + ): + self._add_producer(session, asset) + if run_state is not None: + self._add_run(session, "2026-01-01", run_state, run_start_date) + if event_timestamp is not None: + self._add_event(session, asset, "2026-01-01", event_timestamp) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + assert response.status_code == 200 + statuses = [partition["status"] for partition in response.json()["partitions"]] + assert statuses == ([expected_status] if expected_status else []) + + def test_newest_run_across_producers_decides(self, test_client, session, asset): + self._add_producer(session, asset, "producer_a") + self._add_producer(session, asset, "producer_b") + self._add_run(session, "2026-01-01", DagRunState.FAILED, dag_id="producer_a") + newest = self._add_run(session, "2026-01-01", DagRunState.RUNNING, dag_id="producer_b") + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + assert response.json()["partitions"] == [ + { + "partition_key": "2026-01-01", + "status": "in_progress", + "last_event_id": None, + "last_event_timestamp": None, + "last_producer_run": { + "dag_id": "producer_b", + "run_id": newest.run_id, + "state": "running", + "start_date": "2026-01-02T10:00:00Z", + "end_date": None, + }, + "consumer_runs": [], + } + ] + + def test_cleared_older_run_is_the_newest_attempt(self, test_client, session, asset): + self._add_producer(session, asset) + cleared = self._add_run(session, "2026-01-01", DagRunState.QUEUED, None, queued_at=AFTER_RUN_START) + newer_row = self._add_run(session, "2026-01-01", DagRunState.SUCCESS, queued_at=BEFORE_RUN_START) + self._add_event(session, asset, "2026-01-01", RUN_START, source_run=newer_row) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + [partition] = response.json()["partitions"] + assert partition["status"] == "in_progress" + assert partition["last_producer_run"]["run_id"] == cleared.run_id + + def test_run_that_emitted_under_another_key_is_not_tied_to_its_own_key(self, test_client, session, asset): + self._add_producer(session, asset) + run = self._add_run(session, "batch-42", DagRunState.FAILED) + self._add_event(session, asset, "emea", AFTER_RUN_START, source_run=run) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + assert [(p["partition_key"], p["status"]) for p in response.json()["partitions"]] == [ + ("emea", "materialized") + ] + + def test_run_that_also_emitted_under_its_own_key_stays_tied_to_it(self, test_client, session, asset): + self._add_producer(session, asset, "producer_a") + self._add_producer(session, asset, "producer_b") + self._add_run( + session, "2026-01-01", DagRunState.QUEUED, None, dag_id="producer_a", queued_at=BEFORE_RUN_START + ) + newest = self._add_run(session, "2026-01-01", DagRunState.FAILED, dag_id="producer_b") + self._add_event(session, asset, "2026-01-01", AFTER_RUN_START, source_run=newest) + self._add_event(session, asset, "emea", AFTER_RUN_START, source_run=newest) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions", params={"partition_key_pattern": "2026"}) + + [partition] = response.json()["partitions"] + assert partition["status"] == "materialized" + assert partition["last_producer_run"]["dag_id"] == "producer_b" + + def test_clearing_a_task_of_a_running_run_makes_its_event_stale(self, test_client, session, asset): + self._add_producer(session, asset) + # Clearing resets queued_at; a run that is still running keeps its start_date. + run = self._add_run( + session, "2026-01-01", DagRunState.RUNNING, BEFORE_RUN_START, queued_at=AFTER_RUN_START + ) + self._add_event(session, asset, "2026-01-01", RUN_START, source_run=run) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + [partition] = response.json()["partitions"] + assert partition["status"] == "in_progress" + + def test_ignores_runs_of_dags_that_do_not_produce_the_asset(self, test_client, session, asset): + session.add(DagModel(dag_id="unrelated", bundle_name="testing")) + self._add_run(session, "2026-01-01", DagRunState.FAILED, dag_id="unrelated") + self._add_producer(session, asset) + self._add_run(session, None, DagRunState.FAILED) + self._add_event(session, asset, None) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + assert response.json() == { + "partitions": [], + "total_entries": 0, + "status_counts": {"materialized": 0, "in_progress": 0, "failed": 0}, + } + + def test_latest_event_is_the_one_with_the_latest_timestamp(self, test_client, session, asset): + self._add_producer(session, asset) + self._add_run(session, "2026-01-01", DagRunState.FAILED) + latest = self._add_event(session, asset, "2026-01-01", AFTER_RUN_START) + # Written later, but for an earlier moment: a higher id is not a later event. + self._add_event(session, asset, "2026-01-01", BEFORE_RUN_START) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + [partition] = response.json()["partitions"] + assert partition["status"] == "materialized" + assert partition["last_event_id"] == latest.id + assert partition["last_event_timestamp"] == "2026-01-02T10:30:00Z" + + @pytest.fixture + def three_statuses(self, session, asset): + self._add_producer(session, asset) + self._add_event(session, asset, "2026-01-01") + self._add_event(session, asset, "2026-01-02") + self._add_run(session, "2026-01-03", DagRunState.FAILED) + self._add_run(session, "2026-01-04", DagRunState.RUNNING) + self._add_event(session, asset, "100%") + session.commit() + + @pytest.mark.parametrize( + ("params", "expected_keys", "expected_total", "expected_counts"), + [ + pytest.param( + {}, + ["2026-01-04", "2026-01-03", "2026-01-02", "2026-01-01", "100%"], + 5, + {"materialized": 3, "in_progress": 1, "failed": 1}, + id="all-newest-key-first", + ), + pytest.param( + {"status": ["failed", "in_progress"]}, + ["2026-01-04", "2026-01-03"], + 2, + {"materialized": 3, "in_progress": 1, "failed": 1}, + id="status-filter-keeps-counts", + ), + pytest.param( + {"partition_key_pattern": "01-0"}, + ["2026-01-04", "2026-01-03", "2026-01-02", "2026-01-01"], + 4, + {"materialized": 2, "in_progress": 1, "failed": 1}, + id="pattern-narrows-counts", + ), + pytest.param( + {"partition_key_pattern": "%"}, + ["100%"], + 1, + {"materialized": 1, "in_progress": 0, "failed": 0}, + id="pattern-is-literal", + ), + pytest.param( + {"status": ["in_progress"], "partition_key_pattern": "2026-01-0"}, + ["2026-01-04"], + 1, + {"materialized": 2, "in_progress": 1, "failed": 1}, + id="single-status", + ), + pytest.param( + {"status": ["failed"], "partition_key_pattern": "01-01"}, + [], + 0, + {"materialized": 1, "in_progress": 0, "failed": 0}, + id="selected-status-without-keys-keeps-counts", + ), + pytest.param( + {"limit": 2, "offset": 10}, + [], + 5, + {"materialized": 3, "in_progress": 1, "failed": 1}, + id="offset-past-the-end-keeps-counts", + ), + pytest.param( + {"limit": 2, "offset": 1}, + ["2026-01-03", "2026-01-02"], + 5, + {"materialized": 3, "in_progress": 1, "failed": 1}, + id="pagination", + ), + ], + ) + @pytest.mark.usefixtures("three_statuses") + def test_filters_counts_and_pagination( + self, test_client, asset, params, expected_keys, expected_total, expected_counts + ): + response = test_client.get(f"/assets/{asset.id}/partitions", params=params) + + assert response.status_code == 200 + body = response.json() + assert [partition["partition_key"] for partition in body["partitions"]] == expected_keys + assert body["total_entries"] == expected_total + assert body["status_counts"] == expected_counts + + @pytest.mark.usefixtures("three_statuses") + def test_should_respond_422_for_unknown_status(self, test_client, asset): + response = test_client.get(f"/assets/{asset.id}/partitions", params={"status": "missing"}) + assert response.status_code == 422 + + @pytest.mark.parametrize( + ("fixture_name", "expected_queries"), + [ + pytest.param("three_statuses", 6, id="no-consumer-runs"), + pytest.param("consumed_events", 7, id="consumer-runs"), + ], + ) + def test_number_of_queries_does_not_grow_with_keys( + self, test_client, asset, request, fixture_name, expected_queries + ): + # The asset lookup, counts with the page, the last events and their consumer runs, plus + # two for the access check and the caller's readable Dags. A further query ranks the events + # a consumer run consumed, but only when there is one. Nothing runs once per key or per run. + request.getfixturevalue(fixture_name) + with assert_queries_count(expected_queries): + test_client.get(f"/assets/{asset.id}/partitions") + + @pytest.fixture + def consumed_events(self, session, asset): + """An old and a latest event of one key, each consumed by runs of two consumer Dags.""" + self._add_producer(session, asset) + old = self._add_event(session, asset, "2026-01-01", BEFORE_RUN_START) + latest = self._add_event(session, asset, "2026-01-01", AFTER_RUN_START) + session.add_all( + DagModel(dag_id=dag_id, bundle_name="testing") for dag_id in ("consumer_a", "consumer_b") + ) + consumptions = [ + (event, key, dag_id) + for event, key in ((old, "old"), (latest, "latest")) + for dag_id in ("consumer_a", "consumer_b") + ] + for event, key, dag_id in [*consumptions, (old, "again", "consumer_a")]: + run = self._add_run(session, key, DagRunState.SUCCESS, dag_id=dag_id) + run.consumed_asset_events.append(event) + session.commit() + + @pytest.mark.usefixtures("consumed_events") + def test_consumer_runs_are_those_of_the_latest_event(self, test_client, asset): + response = test_client.get(f"/assets/{asset.id}/partitions") + + [partition] = response.json()["partitions"] + assert sorted((run["dag_id"], run["run_id"]) for run in partition["consumer_runs"]) == [ + ("consumer_a", "run_latest_success"), + ("consumer_b", "run_latest_success"), + ] + + @mock.patch("airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids") + @pytest.mark.usefixtures("consumed_events") + def test_consumer_runs_leave_out_dags_the_caller_may_not_read( + self, mock_get_authorized_dag_ids, test_client, asset + ): + mock_get_authorized_dag_ids.return_value = {"consumer_b"} + + response = test_client.get(f"/assets/{asset.id}/partitions") + + [partition] = response.json()["partitions"] + assert [run["dag_id"] for run in partition["consumer_runs"]] == ["consumer_b"] + + def test_only_the_newest_consumed_event_triggers_a_consumer_run(self, test_client, session, asset): + self._add_producer(session, asset) + session.add(DagModel(dag_id="consumer", bundle_name="testing")) + older = self._add_event(session, asset, "2026-01-01", BEFORE_RUN_START) + newer = self._add_event(session, asset, "2026-01-02", AFTER_RUN_START) + run = self._add_run(session, "consumed", DagRunState.SUCCESS, dag_id="consumer") + run.consumed_asset_events.extend([older, newer]) + session.commit() + + response = test_client.get(f"/assets/{asset.id}/partitions") + + triggering_by_key = { + partition["partition_key"]: [r["triggering"] for r in partition["consumer_runs"]] + for partition in response.json()["partitions"] + } + assert triggering_by_key == {"2026-01-02": [True], "2026-01-01": [False]} + + @pytest.fixture + def sortable_keys(self, session, asset): + """Keys b and d tie on the event time; c has no event, d has no producer run.""" + self._add_producer(session, asset) + self._add_run(session, "a", DagRunState.FAILED, start_date=RUN_START.add(hours=2)) + self._add_run(session, "b", DagRunState.RUNNING, start_date=RUN_START.add(hours=1)) + self._add_run(session, "c", DagRunState.RUNNING, start_date=RUN_START) + self._add_event(session, asset, "a", BEFORE_RUN_START) + self._add_event(session, asset, "b", AFTER_RUN_START.add(hours=3)) + self._add_event(session, asset, "d", AFTER_RUN_START.add(hours=3)) + session.commit() + + @pytest.mark.parametrize( + ("order_by", "expected_keys"), + [ + pytest.param("partition_key", ["a", "b", "c", "d"], id="partition_key-asc"), + pytest.param("-partition_key", ["d", "c", "b", "a"], id="partition_key-desc"), + pytest.param("status", ["a", "c", "b", "d"], id="status-asc"), + pytest.param("-status", ["b", "d", "c", "a"], id="status-desc"), + pytest.param("last_event_timestamp", ["a", "b", "d", "c"], id="last_event_timestamp-asc"), + pytest.param("-last_event_timestamp", ["b", "d", "a", "c"], id="last_event_timestamp-desc"), + pytest.param("last_producer_run", ["c", "b", "a", "d"], id="last_producer_run-asc"), + pytest.param("-last_producer_run", ["a", "b", "c", "d"], id="last_producer_run-desc"), + ], + ) + @pytest.mark.usefixtures("sortable_keys") + def test_order_by(self, test_client, asset, order_by, expected_keys): + response = test_client.get(f"/assets/{asset.id}/partitions", params={"order_by": order_by}) + + assert response.status_code == 200 + assert [p["partition_key"] for p in response.json()["partitions"]] == expected_keys + + @pytest.mark.parametrize(("offset", "expected_key"), [(0, "b"), (1, "d")]) + @pytest.mark.usefixtures("sortable_keys") + def test_order_by_splits_a_tie_across_pages_of_a_status(self, test_client, asset, offset, expected_key): + response = test_client.get( + f"/assets/{asset.id}/partitions", + params={ + "order_by": "-last_event_timestamp", + "status": "materialized", + "limit": 1, + "offset": offset, + }, + ) + + body = response.json() + assert [p["partition_key"] for p in body["partitions"]] == [expected_key] + assert body["total_entries"] == 2 + assert body["status_counts"] == {"materialized": 2, "in_progress": 1, "failed": 1} + + @pytest.mark.parametrize("order_by", ["missing", "-missing", "consumer_runs", ""]) + @pytest.mark.usefixtures("sortable_keys") + def test_should_respond_400_for_unknown_order_by(self, test_client, asset, order_by): + response = test_client.get(f"/assets/{asset.id}/partitions", params={"order_by": order_by}) + + assert response.status_code == 400 + + @mock.patch("airflow.api_fastapi.auth.managers.base_auth_manager.BaseAuthManager.get_authorized_dag_ids") + def test_leaves_out_runs_and_events_of_dags_the_caller_may_not_read( + self, mock_get_authorized_dag_ids, test_client, session, asset + ): + self._add_producer(session, asset, "hidden") + self._add_producer(session, asset, "readable") + self._add_run(session, "hidden-run", DagRunState.FAILED, dag_id="hidden") + hidden_source = self._add_run(session, "hidden-event", DagRunState.SUCCESS, dag_id="hidden") + self._add_event(session, asset, "hidden-event", AFTER_RUN_START, source_run=hidden_source) + self._add_run(session, "readable-run", DagRunState.FAILED, dag_id="readable") + self._add_event(session, asset, "without-source") + session.commit() + mock_get_authorized_dag_ids.return_value = {"readable"} + + response = test_client.get(f"/assets/{asset.id}/partitions") + + body = response.json() + assert [(p["partition_key"], p["status"]) for p in body["partitions"]] == [ + ("without-source", "materialized"), + ("readable-run", "failed"), + ] + assert body["total_entries"] == 2 + assert body["status_counts"] == {"materialized": 1, "in_progress": 0, "failed": 1}
