This is an automated email from the ASF dual-hosted git repository.
rahulvats pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e850b4eb362 Fix flaky backfill E2E tests with stable locators (#62913)
e850b4eb362 is described below
commit e850b4eb3626663f5cd3e53eb15285557ee84bb0
Author: Yeonguk Choo <[email protected]>
AuthorDate: Sat Mar 7 17:56:43 2026 +0900
Fix flaky backfill E2E tests with stable locators (#62913)
* Fix flaky backfill E2E tests with stable locators
* fix 409 flaky
* feat: add support for maxActiveRuns and createPausedBackfillViaApi for
race with scheduler
---
.../src/airflow/ui/tests/e2e/pages/BackfillPage.ts | 612 +++++++++++----------
.../airflow/ui/tests/e2e/specs/backfill.spec.ts | 349 ++++++------
2 files changed, 480 insertions(+), 481 deletions(-)
diff --git a/airflow-core/src/airflow/ui/tests/e2e/pages/BackfillPage.ts
b/airflow-core/src/airflow/ui/tests/e2e/pages/BackfillPage.ts
index 100f940f9a5..288c3d40b58 100644
--- a/airflow-core/src/airflow/ui/tests/e2e/pages/BackfillPage.ts
+++ b/airflow-core/src/airflow/ui/tests/e2e/pages/BackfillPage.ts
@@ -18,48 +18,60 @@
*/
import { expect } from "@playwright/test";
import type { Locator, Page } from "@playwright/test";
+import { testConfig } from "playwright.config";
import { BasePage } from "tests/e2e/pages/BasePage";
-export type ReprocessBehavior = "All Runs" | "Missing and Errored Runs" |
"Missing Runs";
+export const REPROCESS_API_TO_UI = {
+ completed: "All Runs",
+ failed: "Missing and Errored Runs",
+ none: "Missing Runs",
+} as const;
-export type CreateBackfillOptions = {
+export type ReprocessBehaviorApi = keyof typeof REPROCESS_API_TO_UI;
+
+type BackfillDetails = {
+ completedAt: string;
+ createdAt: string;
fromDate: string;
- reprocessBehavior: ReprocessBehavior;
+ reprocessBehavior: string;
toDate: string;
};
-export type BackfillDetails = {
- createdAt: string;
+type BackfillRowIdentifier = {
fromDate: string;
- reprocessBehavior: string;
toDate: string;
};
-export type BackfillRowIdentifier = {
+type CreateBackfillOptions = {
fromDate: string;
+ maxActiveRuns?: number;
+ reprocessBehavior?: ReprocessBehaviorApi;
toDate: string;
};
-function normalizeDate(dateString: string): string {
- const trimmed = dateString.trim();
+type BackfillApiResponse = {
+ completed_at: string | null;
+ id: number;
+};
- if (trimmed.includes("T")) {
- const parts = trimmed.split("T");
+type BackfillListApiResponse = {
+ backfills: Array<BackfillApiResponse>;
+};
- return parts[0] ?? trimmed;
- }
+const {
+ connection: { baseUrl },
+} = testConfig;
- if (trimmed.includes(" ")) {
- const parts = trimmed.split(" ");
+function getColumnIndex(columnMap: Map<string, number>, name: string): number {
+ const index = columnMap.get(name);
- return parts[0] ?? trimmed;
- }
+ if (index === undefined) {
+ const available = [...columnMap.keys()].join(", ");
- return trimmed;
-}
+ throw new Error(`Column "${name}" not found. Available columns:
${available}`);
+ }
-function datesMatch(date1: string, date2: string): boolean {
- return normalizeDate(date1) === normalizeDate(date2);
+ return index;
}
export class BackfillPage extends BasePage {
@@ -72,32 +84,25 @@ export class BackfillPage extends BasePage {
public readonly cancelButton: Locator;
public readonly pauseButton: Locator;
public readonly triggerButton: Locator;
+ public readonly unpauseButton: Locator;
+
+ public get pauseOrUnpauseButton(): Locator {
+ return this.pauseButton.or(this.unpauseButton);
+ }
public constructor(page: Page) {
super(page);
- this.triggerButton = page.locator('button[aria-label="Trigger
Dag"]:has-text("Trigger")');
+ this.triggerButton = page.getByTestId("trigger-dag-button");
+ // Chakra UI radio cards: target the label directly since <input> is
hidden.
this.backfillModeRadio = page.locator('label:has-text("Backfill")');
- this.backfillFromDateInput =
page.locator('input[type="datetime-local"]').first();
- this.backfillToDateInput =
page.locator('input[type="datetime-local"]').nth(1);
- this.backfillRunButton = page.locator('button:has-text("Run Backfill")');
- this.backfillsTable = page.locator("table");
- this.backfillDateError = page.locator('text="Start Date must be before the
End Date"');
- this.cancelButton = page.locator('button[aria-label="Cancel backfill"]');
- this.pauseButton = page.locator(
- 'button[aria-label="Pause backfill"], button[aria-label="Unpause
backfill"]',
- );
- }
-
- public static findColumnIndex(columnMap: Map<string, number>, possibleNames:
Array<string>): number {
- for (const name of possibleNames) {
- const index = columnMap.get(name);
-
- if (index !== undefined) {
- return index;
- }
- }
-
- return -1;
+ this.backfillFromDateInput = page.getByTestId("datetime-input").first();
+ this.backfillToDateInput = page.getByTestId("datetime-input").nth(1);
+ this.backfillRunButton = page.getByRole("button", { name: "Run Backfill"
});
+ this.backfillsTable = page.getByTestId("table-list");
+ this.backfillDateError = page.getByText("Start Date must be before the End
Date");
+ this.cancelButton = page.getByRole("button", { name: "Cancel backfill" });
+ this.pauseButton = page.getByRole("button", { name: "Pause backfill" });
+ this.unpauseButton = page.getByRole("button", { name: "Unpause backfill"
});
}
public static getBackfillsUrl(dagName: string): string {
@@ -108,204 +113,252 @@ export class BackfillPage extends BasePage {
return `/dags/${dagName}`;
}
- public async clickCancelButton(): Promise<void> {
- await this.waitForBackdropClosed();
- await expect(this.cancelButton).toBeVisible({ timeout: 10_000 });
- await this.cancelButton.click();
- await expect(this.cancelButton).not.toBeVisible({ timeout: 30_000 });
- }
+ public async cancelAllActiveBackfillsViaApi(dagId: string): Promise<void> {
+ const response = await
this.page.request.get(`${baseUrl}/api/v2/backfills?dag_id=${dagId}&limit=100`, {
+ timeout: 30_000,
+ });
- public async clickPauseButton(): Promise<void> {
- await this.pauseButton.waitFor({ state: "visible", timeout: 30_000 });
- await this.pauseButton.scrollIntoViewIfNeeded();
- await expect(this.pauseButton).toBeEnabled({ timeout: 10_000 });
- const wasPaused = await this.isBackfillPaused();
+ expect(response.ok()).toBe(true);
+ const data = (await response.json()) as BackfillListApiResponse;
- const responsePromise = this.page
- .waitForResponse(
- (response) => {
- const url = response.url();
- const status = response.status();
-
- return url.includes("/backfills/") && (status === 200 || status ===
204);
- },
- { timeout: 10_000 },
- )
- .catch(() => undefined);
+ for (const backfill of data.backfills) {
+ if (backfill.completed_at === null) {
+ await this.cancelBackfillViaApi(backfill.id);
+ }
+ }
+ }
- await this.pauseButton.click();
- await responsePromise;
+ public async cancelBackfillViaApi(backfillId: number): Promise<void> {
+ const response = await
this.page.request.put(`${baseUrl}/api/v2/backfills/${backfillId}/cancel`, {
+ timeout: 30_000,
+ });
- // Wait for aria-label to change
- const expectedLabel = wasPaused ? "Pause backfill" : "Unpause backfill";
+ expect([200, 409]).toContain(response.status());
+ }
- await
expect(this.page.locator(`button[aria-label="${expectedLabel}"]`)).toBeVisible({
timeout: 10_000 });
+ public async clickCancelButton(): Promise<void> {
+ await this.cancelButton.click({ timeout: 10_000 });
+ await expect(this.cancelButton).not.toBeVisible({ timeout: 15_000 });
}
- public async createBackfill(dagName: string, options:
CreateBackfillOptions): Promise<void> {
- const { fromDate, reprocessBehavior, toDate } = options;
+ /** Create a backfill through the UI dialog. Returns the backfill ID. Caller
must ensure no active backfills exist. */
+ public async createBackfill(dagName: string, options:
CreateBackfillOptions): Promise<number> {
+ const { fromDate, reprocessBehavior = "none", toDate } = options;
+
+ const uiFromDate = fromDate.slice(0, 16);
+ const uiToDate = toDate.slice(0, 16);
await this.navigateToDagDetail(dagName);
- await this.waitForNoActiveBackfill();
await this.openBackfillDialog();
await this.backfillFromDateInput.click();
- await this.backfillFromDateInput.fill(fromDate);
- await this.backfillFromDateInput.dispatchEvent("change");
+ await this.backfillFromDateInput.fill(uiFromDate);
+ await this.backfillFromDateInput.press("Tab");
await this.backfillToDateInput.click();
- await this.backfillToDateInput.fill(toDate);
- await this.backfillToDateInput.dispatchEvent("change");
-
- await this.page.waitForTimeout(500);
+ await this.backfillToDateInput.fill(uiToDate);
+ await this.backfillToDateInput.press("Tab");
await this.selectReprocessBehavior(reprocessBehavior);
- const runsWillBeTriggered = this.page.locator("text=/\\d+ runs? will be
triggered/");
- const noRunsMatching = this.page.locator("text=/No runs matching/");
+ const runsWillBeTriggered = this.page.getByText(/\d+ runs? will be
triggered/);
+ const noRunsMatching = this.page.getByText(/No runs matching/);
await expect(runsWillBeTriggered.or(noRunsMatching)).toBeVisible({
timeout: 20_000 });
- let previousText = "";
- let stableCount = 0;
-
- while (stableCount < 3) {
- await this.page.waitForTimeout(500);
- const currentText = (await
runsWillBeTriggered.or(noRunsMatching).textContent()) ?? "";
-
- if (currentText === previousText) {
- stableCount++;
- } else {
- stableCount = 0;
- previousText = currentText;
- }
- }
-
- const hasRuns = await runsWillBeTriggered.isVisible();
-
- if (!hasRuns) {
+ if (await noRunsMatching.isVisible()) {
await this.page.keyboard.press("Escape");
- return;
+ throw new Error(
+ `No runs matching: dag=${dagName}, from=${fromDate}, to=${toDate},
reprocess=${reprocessBehavior}`,
+ );
}
- await expect(this.backfillRunButton).toBeVisible({ timeout: 20_000 });
await expect(this.backfillRunButton).toBeEnabled({ timeout: 10_000 });
- await this.backfillRunButton.scrollIntoViewIfNeeded();
const responsePromise = this.page.waitForResponse(
(res) =>
res.url().includes("/backfills") &&
!res.url().includes("/dry_run") &&
res.request().method() === "POST",
- { timeout: 60_000 },
+ { timeout: 30_000 },
);
- await this.backfillRunButton.click({ timeout: 20_000 });
+ await this.backfillRunButton.click();
const apiResponse = await responsePromise;
const status = apiResponse.status();
- if (status === 409) {
- await this.page.keyboard.press("Escape");
- await this.waitForBackdropClosed();
- await this.waitForNoActiveBackfill();
-
- return this.createBackfill(dagName, options);
- }
-
if (status < 200 || status >= 300) {
- const body = await apiResponse.text().catch(() => "unknown error");
+ const body = await apiResponse.text().catch(() => "unknown");
+
+ await this.page.keyboard.press("Escape");
throw new Error(`Backfill creation failed with status ${status}:
${body}`);
}
- const isClosed = await this.backfillRunButton
- .waitFor({ state: "hidden", timeout: 60_000 })
- .then(() => true)
- .catch(() => false);
+ const data = (await apiResponse.json()) as BackfillApiResponse;
- if (!isClosed) {
- await this.page.keyboard.press("Escape");
- }
+ return data.id;
+ }
- await this.waitForBackdropClosed();
+ /** Create a backfill via API. On 409, cancels active backfills and retries
once. */
+ public async createBackfillViaApi(dagId: string, options:
CreateBackfillOptions): Promise<number> {
+ const { fromDate, maxActiveRuns, reprocessBehavior = "none", toDate } =
options;
- return;
- }
+ const body: Record<string, unknown> = {
+ dag_id: dagId,
+ from_date: fromDate,
+ reprocess_behavior: reprocessBehavior,
+ to_date: toDate,
+ };
- public async findBackfillRowByDateRange(
- identifier: BackfillRowIdentifier,
- timeout: number = 180_000,
- ): Promise<Locator> {
- const { fromDate: expectedFrom, toDate: expectedTo } = identifier;
+ if (maxActiveRuns !== undefined) {
+ body.max_active_runs = maxActiveRuns;
+ }
- await this.backfillsTable.waitFor({ state: "visible", timeout: 10_000 });
- await this.waitForTableDataLoaded();
+ const response = await
this.page.request.post(`${baseUrl}/api/v2/backfills`, {
+ data: body,
+ headers: { "Content-Type": "application/json" },
+ timeout: 30_000,
+ });
- const columnMap = await this.getColumnIndexMap();
- const fromIndex = BackfillPage.findColumnIndex(columnMap, ["From",
"table.from"]);
- const toIndex = BackfillPage.findColumnIndex(columnMap, ["To",
"table.to"]);
+ if (response.status() === 409) {
+ await this.cancelAllActiveBackfillsViaApi(dagId);
+ await this.waitForNoActiveBackfillViaApi(dagId, 30_000);
+ const retryResponse = await
this.page.request.post(`${baseUrl}/api/v2/backfills`, {
+ data: body,
+ headers: { "Content-Type": "application/json" },
+ timeout: 30_000,
+ });
- if (fromIndex === -1 || toIndex === -1) {
- const availableColumns = [...columnMap.keys()].join(", ");
+ expect(retryResponse.ok()).toBe(true);
- throw new Error(
- `Required columns "From" and/or "To" not found. Available columns:
[${availableColumns}]`,
- );
+ return ((await retryResponse.json()) as BackfillApiResponse).id;
}
- let foundRow: Locator | undefined;
+ expect(response.ok()).toBe(true);
- await expect(async () => {
- await this.page.reload();
- await this.backfillsTable.waitFor({ state: "visible", timeout: 10_000 });
- await this.waitForTableDataLoaded();
+ return ((await response.json()) as BackfillApiResponse).id;
+ }
- const rows = this.page.locator("table tbody tr");
- const rowCount = await rows.count();
+ /**
+ * Create a backfill and immediately pause it. Retries the full create+pause
+ * cycle up to 3 times to handle the race where the scheduler completes the
+ * backfill before the pause call lands.
+ */
+ public async createPausedBackfillViaApi(dagId: string, options:
CreateBackfillOptions): Promise<number> {
+ for (let attempt = 0; attempt < 3; attempt++) {
+ const backfillId = await this.createBackfillViaApi(dagId, {
+ ...options,
+ maxActiveRuns: 1,
+ });
+
+ const paused = await this.pauseBackfillViaApi(backfillId);
+
+ if (paused) {
+ return backfillId;
+ }
- for (let i = 0; i < rowCount; i++) {
- const row = rows.nth(i);
- const cells = row.locator("td");
- const fromCell = (await cells.nth(fromIndex).textContent()) ?? "";
- const toCell = (await cells.nth(toIndex).textContent()) ?? "";
+ // Backfill completed before we could pause — cancel and retry.
+ await this.cancelAllActiveBackfillsViaApi(dagId);
+ await this.waitForNoActiveBackfillViaApi(dagId, 30_000);
+ }
- if (datesMatch(fromCell, expectedFrom) && datesMatch(toCell,
expectedTo)) {
- foundRow = row;
+ throw new Error(`Failed to create a paused backfill for ${dagId} after 3
attempts`);
+ }
- return;
- }
- }
+ public async findBackfillRowByDateRange(
+ identifier: BackfillRowIdentifier,
+ timeout: number = 120_000,
+ ): Promise<{ columnMap: Map<string, number>; row: Locator }> {
+ const { fromDate: expectedFrom, toDate: expectedTo } = identifier;
+
+ let foundRow: Locator | undefined;
+ let foundColumnMap: Map<string, number> | undefined;
+
+ await expect
+ .poll(
+ async () => {
+ try {
+ if (!(await this.backfillsTable.isVisible())) {
+ return false;
+ }
+
+ const headers = this.backfillsTable.locator("thead th");
+
+ if (!(await headers.first().isVisible())) {
+ return false;
+ }
+
+ const headerTexts = await headers.allTextContents();
+ const columnMap = new Map(headerTexts.map((text, index) =>
[text.trim(), index]));
+ const fromIndex = columnMap.get("From");
+ const toIndex = columnMap.get("To");
+
+ if (fromIndex === undefined || toIndex === undefined) {
+ return false;
+ }
+
+ const rows = this.backfillsTable.locator("tbody tr");
+ const rowCount = await rows.count();
+
+ for (let i = 0; i < rowCount; i++) {
+ const row = rows.nth(i);
+ const cells = row.locator("td");
+ const fromCell = ((await cells.nth(fromIndex).textContent()) ??
"").slice(0, 10);
+ const toCell = ((await cells.nth(toIndex).textContent()) ??
"").slice(0, 10);
+
+ if (fromCell === expectedFrom.slice(0, 10) && toCell ===
expectedTo.slice(0, 10)) {
+ foundRow = row;
+ foundColumnMap = columnMap;
+
+ return true;
+ }
+ }
+
+ return false;
+ } catch (error: unknown) {
+ console.warn("findBackfillRowByDateRange poll error:", error);
+
+ return false;
+ }
+ },
+ {
+ intervals: [2000, 5000],
+ message: `Backfill row with dates ${expectedFrom} ~ ${expectedTo}
not found in table`,
+ timeout,
+ },
+ )
+ .toBeTruthy();
- throw new Error("Backfill not yet visible");
- }).toPass({ timeout });
+ if (!foundRow || !foundColumnMap) {
+ throw new Error(`Backfill row with dates ${expectedFrom} ~ ${expectedTo}
not found`);
+ }
- // toPass() guarantees foundRow is set when it succeeds
- return foundRow as Locator;
+ return { columnMap: foundColumnMap, row: foundRow };
}
public async getBackfillDetailsByDateRange(identifier:
BackfillRowIdentifier): Promise<BackfillDetails> {
- const row = await this.findBackfillRowByDateRange(identifier);
+ const { columnMap, row } = await
this.findBackfillRowByDateRange(identifier);
const cells = row.locator("td");
- const columnMap = await this.getColumnIndexMap();
-
- const fromIndex = BackfillPage.findColumnIndex(columnMap, ["From",
"table.from"]);
- const toIndex = BackfillPage.findColumnIndex(columnMap, ["To",
"table.to"]);
- const reprocessIndex = BackfillPage.findColumnIndex(columnMap, [
- "Reprocess Behavior",
- "backfill.reprocessBehavior",
- ]);
- const createdAtIndex = BackfillPage.findColumnIndex(columnMap, ["Created
at", "table.createdAt"]);
- const [fromDate, toDate, reprocessBehavior, createdAt] = await
Promise.all([
- cells.nth(fromIndex === -1 ? 0 : fromIndex).textContent(),
- cells.nth(toIndex === -1 ? 1 : toIndex).textContent(),
- cells.nth(reprocessIndex === -1 ? 2 : reprocessIndex).textContent(),
- cells.nth(createdAtIndex === -1 ? 3 : createdAtIndex).textContent(),
+ const fromIndex = getColumnIndex(columnMap, "From");
+ const toIndex = getColumnIndex(columnMap, "To");
+ const reprocessIndex = getColumnIndex(columnMap, "Reprocess Behavior");
+ const createdAtIndex = getColumnIndex(columnMap, "Created at");
+ const completedAtIndex = getColumnIndex(columnMap, "Completed at");
+
+ const [fromDate, toDate, reprocessBehavior, createdAt, completedAt] =
await Promise.all([
+ cells.nth(fromIndex).textContent(),
+ cells.nth(toIndex).textContent(),
+ cells.nth(reprocessIndex).textContent(),
+ cells.nth(createdAtIndex).textContent(),
+ cells.nth(completedAtIndex).textContent(),
]);
return {
+ completedAt: (completedAt ?? "").trim(),
createdAt: (createdAt ?? "").trim(),
fromDate: (fromDate ?? "").trim(),
reprocessBehavior: (reprocessBehavior ?? "").trim(),
@@ -313,164 +366,129 @@ export class BackfillPage extends BasePage {
};
}
- public async getBackfillsTableRows(): Promise<number> {
- const rows = this.page.locator("table tbody tr");
-
- try {
- await rows.first().waitFor({ state: "visible", timeout: 5000 });
- } catch {
- return 0;
- }
-
- return await rows.count();
- }
-
- public async getBackfillStatus(rowIndex: number = 0): Promise<string> {
- const row = this.page.locator("table tbody tr").nth(rowIndex);
-
- await expect(row).toBeVisible({ timeout: 10_000 });
-
- const headers = this.page.locator("table thead th");
- const headerTexts = await headers.allTextContents();
- const completedAtIndex = headerTexts.findIndex((text) =>
text.toLowerCase().includes("completed"));
-
- if (completedAtIndex !== -1) {
- const completedCell = row.locator("td").nth(completedAtIndex);
- const completedText = ((await completedCell.textContent()) ?? "").trim();
-
- return completedText ? "Completed" : "Running";
- }
-
- return "Running";
- }
-
public getColumnHeader(columnName: string): Locator {
- return this.page.locator(`th:has-text("${columnName}")`);
- }
-
- public async getColumnIndexMap(): Promise<Map<string, number>> {
- const headers = this.page.locator("table thead th");
-
- await headers.first().waitFor({ state: "visible", timeout: 10_000 });
- const headerTexts = await headers.allTextContents();
-
- return new Map(headerTexts.map((text, index) => [text.trim(), index]));
+ return this.backfillsTable.getByRole("columnheader", { name: columnName });
}
public getFilterButton(): Locator {
- return this.page.locator(
- 'button[aria-label*="Filter table columns"], button:has-text("Filter
table columns")',
- );
+ return this.page.getByRole("button", { name: /filter table columns/i });
}
public async getTableColumnCount(): Promise<number> {
- const headers = this.page.locator("table thead th");
-
- return await headers.count();
- }
-
- public async isBackfillPaused(): Promise<boolean> {
- await expect(this.pauseButton).toBeVisible({ timeout: 10_000 });
- const ariaLabel = await this.pauseButton.getAttribute("aria-label");
-
- return Boolean(ariaLabel?.toLowerCase().includes("unpause"));
+ return this.backfillsTable.locator("thead th").count();
}
public async navigateToBackfillsTab(dagName: string): Promise<void> {
await this.navigateTo(BackfillPage.getBackfillsUrl(dagName));
- await expect(this.backfillsTable).toBeVisible({ timeout: 20_000 });
+ await expect(this.backfillsTable).toBeVisible({ timeout: 15_000 });
}
public async navigateToDagDetail(dagName: string): Promise<void> {
await this.navigateTo(BackfillPage.getDagDetailUrl(dagName));
- await expect(this.triggerButton).toBeVisible({ timeout: 30_000 });
+ await expect(this.triggerButton).toBeVisible({ timeout: 15_000 });
}
public async openBackfillDialog(): Promise<void> {
- await expect(this.triggerButton).toBeVisible({ timeout: 20_000 });
- await this.triggerButton.click();
-
- await expect(this.backfillModeRadio).toBeVisible({ timeout: 20_000 });
- await this.backfillModeRadio.click();
-
- await expect(this.backfillFromDateInput).toBeVisible({ timeout: 20_000 });
+ await this.triggerButton.click({ timeout: 15_000 });
+ await this.backfillModeRadio.click({ timeout: 10_000 });
+ await expect(this.backfillFromDateInput).toBeVisible({ timeout: 10_000 });
}
public async openFilterMenu(): Promise<void> {
- const filterButton = this.getFilterButton();
-
- await filterButton.click();
-
- const filterMenu = this.page.locator('[role="menu"]');
-
- await filterMenu.waitFor({ state: "visible", timeout: 5000 });
- }
-
- public async selectReprocessBehavior(behavior: ReprocessBehavior):
Promise<void> {
- const behaviorLabels: Record<ReprocessBehavior, string> = {
- "All Runs": "All Runs",
- "Missing and Errored Runs": "Missing and Errored Runs",
- "Missing Runs": "Missing Runs",
- };
-
- const label = behaviorLabels[behavior];
- const radioItem = this.page.locator(`label:has-text("${label}")`).first();
-
- await radioItem.waitFor({ state: "visible", timeout: 5000 });
- await radioItem.click();
- }
-
- public async toggleColumn(columnName: string): Promise<void> {
- const menuItem =
this.page.locator(`[role="menuitem"]:has-text("${columnName}")`);
-
- await menuItem.click();
+ await this.getFilterButton().click();
+ await expect(this.page.getByRole("menu")).toBeVisible({ timeout: 5000 });
}
- public async waitForBackdropClosed(timeout: number = 30_000): Promise<void> {
- const backdrop = this.page.locator('[data-part="backdrop"]');
+ public async pauseBackfillViaApi(backfillId: number): Promise<boolean> {
+ // Retry: the server may not have fully initialized the backfill yet.
+ for (let attempt = 0; attempt < 5; attempt++) {
+ const response = await
this.page.request.put(`${baseUrl}/api/v2/backfills/${backfillId}/pause`, {
+ timeout: 30_000,
+ });
- await expect(async () => {
- const count = await backdrop.count();
+ if (response.ok()) {
+ return true;
+ }
- if (count === 0) {
- return;
+ // 409 means the backfill already completed — not retriable.
+ if (response.status() === 409) {
+ return false;
}
- const state = await backdrop.getAttribute("data-state");
+ await this.page.waitForTimeout(2000);
+ }
- if (state !== "closed") {
- throw new Error("Backdrop still open");
- }
- }).toPass({ timeout });
+ return false;
}
- public async waitForNoActiveBackfill(timeout: number = 300_000):
Promise<void> {
- const { page, triggerButton } = this;
- const backfillInProgress = page.locator('text="Backfill in progress:"');
+ public async selectReprocessBehavior(behavior: ReprocessBehaviorApi):
Promise<void> {
+ const label = REPROCESS_API_TO_UI[behavior];
- await expect(async () => {
- await page.reload();
- await page.waitForLoadState("domcontentloaded", { timeout: 30_000 });
- await triggerButton.waitFor({ state: "visible", timeout: 30_000 });
-
- const isVisible = await backfillInProgress.isVisible();
+ await this.page.locator(`label:has-text("${label}")`).first().click({
timeout: 5000 });
+ }
- if (isVisible) {
- throw new Error("Backfill still in progress");
- }
- }).toPass({ intervals: [5000, 10_000, 15_000], timeout });
+ public async toggleColumn(columnName: string): Promise<void> {
+ await this.page.getByRole("menuitem", { name: columnName }).click();
}
- public async waitForTableDataLoaded(): Promise<void> {
- const firstCell = this.page.locator("table tbody tr:first-child
td:first-child");
+ public async togglePauseState(): Promise<void> {
+ await this.pauseOrUnpauseButton.click({ timeout: 10_000 });
+ }
- await expect(firstCell).toBeVisible({ timeout: 30_000 });
- await expect(async () => {
- const text = await firstCell.textContent();
+ public async waitForBackfillComplete(backfillId: number, timeout: number =
120_000): Promise<void> {
+ await expect
+ .poll(
+ async () => {
+ try {
+ const response = await
this.page.request.get(`${baseUrl}/api/v2/backfills/${backfillId}`, {
+ timeout: 30_000,
+ });
+
+ if (!response.ok()) {
+ return false;
+ }
+ const data = (await response.json()) as BackfillApiResponse;
+
+ return data.completed_at !== null;
+ } catch {
+ return false;
+ }
+ },
+ {
+ intervals: [2000, 5000, 10_000],
+ message: `Backfill ${backfillId} did not complete within
${timeout}ms`,
+ timeout,
+ },
+ )
+ .toBeTruthy();
+ }
- if (text === null || text.trim() === "") {
- throw new Error("Table data still loading");
- }
- }).toPass({ timeout: 30_000 });
+ public async waitForNoActiveBackfillViaApi(dagId: string, timeout: number =
120_000): Promise<void> {
+ await expect
+ .poll(
+ async () => {
+ try {
+ const response = await this.page.request.get(
+ `${baseUrl}/api/v2/backfills?dag_id=${dagId}&limit=100`,
+ { timeout: 30_000 },
+ );
+
+ if (!response.ok()) {
+ return false;
+ }
+ const data = (await response.json()) as BackfillListApiResponse;
+
+ return data.backfills.every((b) => b.completed_at !== null);
+ } catch {
+ return false;
+ }
+ },
+ {
+ intervals: [2000, 5000, 10_000],
+ message: `Active backfills for Dag ${dagId} did not clear within
${timeout}ms`,
+ timeout,
+ },
+ )
+ .toBeTruthy();
}
}
diff --git a/airflow-core/src/airflow/ui/tests/e2e/specs/backfill.spec.ts
b/airflow-core/src/airflow/ui/tests/e2e/specs/backfill.spec.ts
index f9abad46ab3..8babe2f9513 100644
--- a/airflow-core/src/airflow/ui/tests/e2e/specs/backfill.spec.ts
+++ b/airflow-core/src/airflow/ui/tests/e2e/specs/backfill.spec.ts
@@ -18,253 +18,234 @@
*/
import { test, expect } from "@playwright/test";
import { testConfig, AUTH_FILE } from "playwright.config";
-import { BackfillPage } from "tests/e2e/pages/BackfillPage";
-
-const getPastDate = (daysAgo: number): string => {
- const date = new Date();
-
- date.setDate(date.getDate() - daysAgo);
- date.setHours(0, 0, 0, 0);
-
- return date.toISOString().slice(0, 16);
+import { BackfillPage, REPROCESS_API_TO_UI } from
"tests/e2e/pages/BackfillPage";
+import type { ReprocessBehaviorApi } from "tests/e2e/pages/BackfillPage";
+
+// Fixed past dates avoid non-determinism from relative date calculations.
+// Controls tests use wide, non-overlapping ranges so the scheduler cannot
+// complete the backfill before the test interacts with it.
+const FIXED_DATES = {
+ controls: {
+ cancel: { from: "2014-01-01T00:00:00Z", to: "2015-01-01T00:00:00Z" },
+ cancelledNoResume: { from: "2016-01-01T00:00:00Z", to:
"2017-01-01T00:00:00Z" },
+ resumePause: { from: "2012-01-01T00:00:00Z", to: "2013-01-01T00:00:00Z" },
+ },
+ set1: { from: "2020-01-01T00:00:00Z", to: "2020-01-02T00:00:00Z" },
+ set2: { from: "2020-02-01T00:00:00Z", to: "2020-02-03T00:00:00Z" },
+ set3: { from: "2020-03-01T00:00:00Z", to: "2020-03-04T00:00:00Z" },
};
-test.describe("Backfill creation and validation", () => {
- // Serial mode ensures all tests run on one worker, preventing parallel
beforeAll conflicts
+// All blocks share the same Dag, so they must run serially to avoid
cross-block interference.
+test.describe("Backfill", () => {
test.describe.configure({ mode: "serial" });
- test.setTimeout(240_000);
- const testDagId = testConfig.testDag.id;
+ test.describe("Backfill creation and validation", () => {
+ test.setTimeout(120_000);
- const backfillConfigs = [
- { behavior: "All Runs" as const, fromDate: getPastDate(5), toDate:
getPastDate(4) },
- { behavior: "Missing Runs" as const, fromDate: getPastDate(8), toDate:
getPastDate(6) },
- { behavior: "Missing and Errored Runs" as const, fromDate:
getPastDate(12), toDate: getPastDate(10) },
- ];
+ const testDagId = testConfig.testDag.id;
- test.beforeAll(async ({ browser }) => {
- test.setTimeout(600_000);
+ const backfillConfigs: Array<{
+ behavior: ReprocessBehaviorApi;
+ dates: { from: string; to: string };
+ }> = [
+ { behavior: "completed", dates: FIXED_DATES.set1 },
+ { behavior: "none", dates: FIXED_DATES.set2 },
+ { behavior: "failed", dates: FIXED_DATES.set3 },
+ ];
- const context = await browser.newContext({ storageState: AUTH_FILE });
- const page = await context.newPage();
- const setupBackfillPage = new BackfillPage(page);
+ test.beforeAll(async ({ browser }) => {
+ test.setTimeout(300_000);
- for (const config of backfillConfigs) {
- await setupBackfillPage.createBackfill(testDagId, {
- fromDate: config.fromDate,
- reprocessBehavior: config.behavior,
- toDate: config.toDate,
- });
+ const context = await browser.newContext({ storageState: AUTH_FILE });
+ const page = await context.newPage();
+ const setupPage = new BackfillPage(page);
- // Wait for backfill to complete on Dag detail page (where the banner is
visible)
- await setupBackfillPage.navigateToDagDetail(testDagId);
- await setupBackfillPage.waitForNoActiveBackfill();
- }
+ await setupPage.navigateToDagDetail(testDagId);
+ await setupPage.cancelAllActiveBackfillsViaApi(testDagId);
- await context.close();
- });
+ for (const config of backfillConfigs) {
+ const backfillId = await setupPage.createBackfill(testDagId, {
+ fromDate: config.dates.from,
+ reprocessBehavior: config.behavior,
+ toDate: config.dates.to,
+ });
- test("verify backfill with 'all runs' behavior", async ({ page }) => {
- const backfillPage = new BackfillPage(page);
+ await setupPage.waitForBackfillComplete(backfillId);
+ }
- await backfillPage.navigateToBackfillsTab(testDagId);
-
- const config = backfillConfigs[0]!; // All Runs
-
- const backfillDetails = await backfillPage.getBackfillDetailsByDateRange({
- fromDate: config.fromDate,
- toDate: config.toDate,
+ await context.close();
});
- expect(backfillDetails.fromDate.slice(0,
10)).toEqual(config.fromDate.slice(0, 10));
- expect(backfillDetails.toDate.slice(0, 10)).toEqual(config.toDate.slice(0,
10));
-
- expect(backfillDetails.createdAt).not.toEqual("");
- expect(backfillDetails.reprocessBehavior).toEqual("All Runs");
- const status = await backfillPage.getBackfillStatus();
-
- expect(status).not.toEqual("");
- });
-
- test("verify backfill with 'missing runs' behavior", async ({ page }) => {
- const backfillPage = new BackfillPage(page);
+ test.afterAll(async ({ browser }) => {
+ const context = await browser.newContext({ storageState: AUTH_FILE });
+ const page = await context.newPage();
+ const cleanupPage = new BackfillPage(page);
- await backfillPage.navigateToBackfillsTab(testDagId);
-
- const config = backfillConfigs[1]!;
-
- const backfillDetails = await backfillPage.getBackfillDetailsByDateRange({
- fromDate: config.fromDate,
- toDate: config.toDate,
+ await cleanupPage.cancelAllActiveBackfillsViaApi(testDagId);
+ await context.close();
});
- expect(backfillDetails.fromDate.slice(0,
10)).toEqual(config.fromDate.slice(0, 10));
- expect(backfillDetails.toDate.slice(0, 10)).toEqual(config.toDate.slice(0,
10));
-
- expect(backfillDetails.createdAt).not.toEqual("");
- expect(backfillDetails.reprocessBehavior).toEqual("Missing Runs");
- const status = await backfillPage.getBackfillStatus();
-
- expect(status).not.toEqual("");
- });
+ for (const config of backfillConfigs) {
+ test(`verify backfill with '${REPROCESS_API_TO_UI[config.behavior]}'
behavior`, async ({ page }) => {
+ const backfillPage = new BackfillPage(page);
- test("verify backfill with 'missing and errored runs' behavior", async ({
page }) => {
- const backfillPage = new BackfillPage(page);
+ await backfillPage.navigateToBackfillsTab(testDagId);
- await backfillPage.navigateToBackfillsTab(testDagId);
+ const details = await backfillPage.getBackfillDetailsByDateRange({
+ fromDate: config.dates.from,
+ toDate: config.dates.to,
+ });
- const config = backfillConfigs[2]!;
+ expect(details.fromDate.slice(0,
10)).toEqual(config.dates.from.slice(0, 10));
+ expect(details.toDate.slice(0, 10)).toEqual(config.dates.to.slice(0,
10));
+ expect(details.createdAt).not.toEqual("");
+ expect(details.completedAt).not.toEqual("");
+
expect(details.reprocessBehavior).toEqual(REPROCESS_API_TO_UI[config.behavior]);
+ });
+ }
- const backfillDetails = await backfillPage.getBackfillDetailsByDateRange({
- fromDate: config.fromDate,
- toDate: config.toDate,
- });
+ test("Verify backfill table filters", async ({ page }) => {
+ const backfillPage = new BackfillPage(page);
- expect(backfillDetails.fromDate.slice(0,
10)).toEqual(config.fromDate.slice(0, 10));
- expect(backfillDetails.toDate.slice(0, 10)).toEqual(config.toDate.slice(0,
10));
+ await backfillPage.navigateToBackfillsTab(testDagId);
- expect(backfillDetails.createdAt).not.toEqual("");
- expect(backfillDetails.reprocessBehavior).toEqual("Missing and Errored
Runs");
- const status = await backfillPage.getBackfillStatus();
+ const initialColumnCount = await backfillPage.getTableColumnCount();
- expect(status).not.toEqual("");
- });
+ expect(initialColumnCount).toBeGreaterThan(0);
+ await expect(backfillPage.getFilterButton()).toBeVisible();
- test("Verify backfill table filters", async ({ page }) => {
- const backfillPage = new BackfillPage(page);
+ await backfillPage.openFilterMenu();
- await backfillPage.navigateToBackfillsTab(testDagId);
+ const filterMenuItems = page.getByRole("menuitem");
+ const filterMenuCount = await filterMenuItems.count();
- const initialColumnCount = await backfillPage.getTableColumnCount();
+ expect(filterMenuCount).toBeGreaterThan(0);
- expect(initialColumnCount).toBeGreaterThan(0);
- await expect(backfillPage.getFilterButton()).toBeVisible();
+ const firstMenuItem = filterMenuItems.first();
+ const columnToToggle = (await firstMenuItem.textContent())?.trim() ?? "";
- await backfillPage.openFilterMenu();
+ expect(columnToToggle).not.toBe("");
- const filterMenuItems = backfillPage.page.locator('[role="menuitem"]');
- const filterMenuCount = await filterMenuItems.count();
+ await backfillPage.toggleColumn(columnToToggle);
+ await page.keyboard.press("Escape");
- expect(filterMenuCount).toBeGreaterThan(0);
+ await
expect(backfillPage.getColumnHeader(columnToToggle)).not.toBeVisible();
- const firstMenuItem = filterMenuItems.first();
- const columnToToggle = (await firstMenuItem.textContent())?.trim() ?? "";
+ const newColumnCount = await backfillPage.getTableColumnCount();
- expect(columnToToggle).not.toBe("");
+ expect(newColumnCount).toBeLessThan(initialColumnCount);
- await backfillPage.toggleColumn(columnToToggle);
- await backfillPage.backfillsTable.click({ position: { x: 5, y: 5 } });
+ await backfillPage.openFilterMenu();
+ await backfillPage.toggleColumn(columnToToggle);
+ await page.keyboard.press("Escape");
- await
expect(backfillPage.getColumnHeader(columnToToggle)).not.toBeVisible();
+ await expect(backfillPage.getColumnHeader(columnToToggle)).toBeVisible();
- const newColumnCount = await backfillPage.getTableColumnCount();
+ const finalColumnCount = await backfillPage.getTableColumnCount();
- expect(newColumnCount).toBeLessThan(initialColumnCount);
+ expect(finalColumnCount).toBe(initialColumnCount);
+ });
+ });
- await backfillPage.openFilterMenu();
- await backfillPage.toggleColumn(columnToToggle);
- await backfillPage.backfillsTable.click({ position: { x: 5, y: 5 } });
+ test.describe("validate date range", () => {
+ test.setTimeout(30_000);
- await expect(backfillPage.getColumnHeader(columnToToggle)).toBeVisible();
+ const testDagId = testConfig.testDag.id;
- const finalColumnCount = await backfillPage.getTableColumnCount();
+ test("verify date range selection (start date, end date)", async ({ page
}) => {
+ const backfillPage = new BackfillPage(page);
- expect(finalColumnCount).toBe(initialColumnCount);
+ await backfillPage.navigateToDagDetail(testDagId);
+ await backfillPage.openBackfillDialog();
+ await backfillPage.backfillFromDateInput.fill("2025-01-10T00:00");
+ await backfillPage.backfillToDateInput.fill("2025-01-01T00:00");
+ await expect(backfillPage.backfillDateError).toBeVisible();
+ });
});
-});
-test.describe("validate date range", () => {
- test.setTimeout(60_000);
+ test.describe("Backfill pause, resume, and cancel controls", () => {
+ test.describe.configure({ mode: "serial" });
+ test.setTimeout(120_000);
- const testDagId = testConfig.testDag.id;
+ const testDagId = testConfig.testDag.id;
- test("verify date range selection (start date, end date)", async ({ page })
=> {
- const fromDate = getPastDate(1);
- const toDate = getPastDate(7);
- const backfillPage = new BackfillPage(page);
+ let backfillPage: BackfillPage;
- await backfillPage.navigateToDagDetail(testDagId);
- await backfillPage.openBackfillDialog();
- await backfillPage.backfillFromDateInput.fill(fromDate);
- await backfillPage.backfillToDateInput.fill(toDate);
- await expect(backfillPage.backfillDateError).toBeVisible();
- });
-});
+ test.beforeEach(async ({ page }) => {
+ backfillPage = new BackfillPage(page);
+ await backfillPage.cancelAllActiveBackfillsViaApi(testDagId);
+ await backfillPage.waitForNoActiveBackfillViaApi(testDagId, 30_000);
+ });
-test.describe("Backfill pause, resume, and cancel controls", () => {
- test.describe.configure({ mode: "serial" });
- test.setTimeout(180_000);
+ test.afterEach(async () => {
+ await backfillPage.cancelAllActiveBackfillsViaApi(testDagId);
+ });
- const testDagId = testConfig.testDag.id;
- const controlFromDate = getPastDate(90);
- const controlToDate = getPastDate(30);
+ test("verify pause and resume backfill", async () => {
+ const dates = FIXED_DATES.controls.resumePause;
- let backfillPage: BackfillPage;
+ // Create + pause atomically to eliminate race with scheduler.
+ await backfillPage.createPausedBackfillViaApi(testDagId, {
+ fromDate: dates.from,
+ reprocessBehavior: "completed",
+ toDate: dates.to,
+ });
- test.beforeEach(async ({ page }) => {
- backfillPage = new BackfillPage(page);
- await backfillPage.navigateToDagDetail(testDagId);
+ // Navigate to verify UI reflects the paused state, then test toggle
cycle.
+ await backfillPage.navigateToDagDetail(testDagId);
+ await expect(backfillPage.unpauseButton).toBeVisible({ timeout: 15_000
});
- if (await backfillPage.cancelButton.isVisible({ timeout: 5000 }).catch(()
=> false)) {
- await backfillPage.clickCancelButton();
- }
+ await backfillPage.togglePauseState();
+ await expect(backfillPage.pauseButton).toBeVisible({ timeout: 10_000 });
- await backfillPage.createBackfill(testDagId, {
- fromDate: controlFromDate,
- reprocessBehavior: "All Runs",
- toDate: controlToDate,
+ await backfillPage.togglePauseState();
+ await expect(backfillPage.unpauseButton).toBeVisible({ timeout: 10_000
});
});
- await expect(async () => {
- await backfillPage.page.reload();
- await expect(backfillPage.triggerButton).toBeVisible({ timeout: 10_000
});
- await expect(backfillPage.pauseButton).toBeVisible({ timeout: 5000 });
- }).toPass({ timeout: 60_000 });
- });
-
- test.afterEach(async () => {
- await backfillPage.navigateToDagDetail(testDagId);
- if (await backfillPage.cancelButton.isVisible({ timeout: 5000 }).catch(()
=> false)) {
- await backfillPage.clickCancelButton();
- }
- });
+ test("verify cancel backfill", async () => {
+ const dates = FIXED_DATES.controls.cancel;
- test("verify pause and resume backfill", async () => {
- await backfillPage.clickPauseButton();
- expect(await backfillPage.isBackfillPaused()).toBe(true);
+ // Create + pause atomically to eliminate race with scheduler.
+ await backfillPage.createPausedBackfillViaApi(testDagId, {
+ fromDate: dates.from,
+ reprocessBehavior: "completed",
+ toDate: dates.to,
+ });
- await backfillPage.clickPauseButton();
- expect(await backfillPage.isBackfillPaused()).toBe(false);
- });
+ await backfillPage.navigateToDagDetail(testDagId);
+ await expect(backfillPage.unpauseButton).toBeVisible({ timeout: 15_000
});
- test("verify cancel backfill", async () => {
- await backfillPage.clickCancelButton();
- await expect(backfillPage.pauseButton).not.toBeVisible({ timeout: 10_000
});
- await expect(backfillPage.cancelButton).not.toBeVisible({ timeout: 10_000
});
- });
+ await backfillPage.clickCancelButton();
+ await expect(backfillPage.pauseOrUnpauseButton).not.toBeVisible({
timeout: 10_000 });
+ await expect(backfillPage.cancelButton).not.toBeVisible({ timeout:
10_000 });
+ });
- test("verify cancelled backfill cannot be resumed", async () => {
- await backfillPage.clickCancelButton();
- await expect(backfillPage.pauseButton).not.toBeVisible({ timeout: 10_000
});
+ test("verify cancelled backfill cannot be resumed", async () => {
+ const dates = FIXED_DATES.controls.cancelledNoResume;
- await backfillPage.page.reload();
- await expect(backfillPage.triggerButton).toBeVisible({ timeout: 30_000 });
- await expect(backfillPage.pauseButton).not.toBeVisible({ timeout: 10_000
});
+ // Setup via API: create and cancel directly (UI cancel is tested above).
+ const backfillId = await backfillPage.createBackfillViaApi(testDagId, {
+ fromDate: dates.from,
+ maxActiveRuns: 1,
+ reprocessBehavior: "completed",
+ toDate: dates.to,
+ });
- await backfillPage.navigateToBackfillsTab(testDagId);
+ await backfillPage.cancelBackfillViaApi(backfillId);
- const row = await backfillPage.findBackfillRowByDateRange({
- fromDate: controlFromDate,
- toDate: controlToDate,
- });
+ // Verify UI: no pause/resume controls visible after cancel.
+ await backfillPage.navigateToDagDetail(testDagId);
+ await expect(backfillPage.pauseOrUnpauseButton).not.toBeVisible({
timeout: 10_000 });
- await expect(row).toBeVisible();
+ // Verify: completedAt is set in backfills table.
+ await backfillPage.navigateToBackfillsTab(testDagId);
- const columnMap = await backfillPage.getColumnIndexMap();
- const completedAtIndex = BackfillPage.findColumnIndex(columnMap,
["Completed at", "table.completedAt"]);
- const completedAtCell = row.locator("td").nth(completedAtIndex);
- const completedAtText = await completedAtCell.textContent();
+ const details = await backfillPage.getBackfillDetailsByDateRange({
+ fromDate: dates.from,
+ toDate: dates.to,
+ });
- expect(completedAtText?.trim()).not.toBe("");
+ expect(details.completedAt).not.toBe("");
+ });
});
});
