Author: Theodore Brown (theodorejb) Committer: Derick Rethans (derickr) Date: 2026-06-12T08:56:41+01:00
Commit: https://github.com/php/web-php/commit/cf7c6d2b3ea48b157b8991a85fea010db77fb5ed Raw diff: https://github.com/php/web-php/commit/cf7c6d2b3ea48b157b8991a85fea010db77fb5ed.diff Remove pixel-perfect screenshot tests Per feedback from other maintainers, these tests will just be a waste of time as we make numerous necessary design fixes and improvements. Changed paths: D .github/workflows/update-screenshots.yaml D tests/Visual/SearchModal.css D tests/Visual/SearchModal.spec.ts D tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png D tests/Visual/SmokeTest.spec.ts D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png M .github/workflows/preview-tests.yml M Makefile M playwright.config.ts Diff: diff --git a/.github/workflows/preview-tests.yml b/.github/workflows/preview-tests.yml index a9e69c6e0b..4a9c5ede8c 100644 --- a/.github/workflows/preview-tests.yml +++ b/.github/workflows/preview-tests.yml @@ -4,8 +4,8 @@ on: types: [ labeled ] jobs: - tests_visual: - name: "Visual Tests" + tests_e2e: + name: "E2E Tests" runs-on: "ubuntu-latest" if: "github.repository_owner == 'php' && contains(github.event.pull_request.labels.*.name, 'Status: Preview Allowed')" @@ -77,8 +77,8 @@ jobs: - name: "Install Playwright" run: "npx playwright install" - - name: "Run visual tests" - run: "make tests_visual" + - name: "Run E2E tests" + run: "make tests_e2e" - name: Upload playwright report uses: actions/upload-artifact@v7 diff --git a/.github/workflows/update-screenshots.yaml b/.github/workflows/update-screenshots.yaml deleted file mode 100644 index f62197aaca..0000000000 --- a/.github/workflows/update-screenshots.yaml +++ /dev/null @@ -1,74 +0,0 @@ -# https://docs.github.com/en/actions - -name: "Update screenshots" - -on: - workflow_dispatch: - -jobs: - tests_update_snapshots: - name: "Update Tests snapshots" - - runs-on: "ubuntu-latest" - - strategy: - matrix: - php-version: - - "8.2" - node-version: - - "22.x" - - env: - HTTP_HOST: "localhost:8080" - - steps: - - name: "Checkout" - uses: "actions/checkout@v6" - - - name: "Set up PHP" - uses: "shivammathur/setup-php@v2" - with: - coverage: "none" - extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter, iconv" - php-version: "${{ matrix.php-version }}" - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v6 - with: - node-version: ${{ matrix.node-version }} - - - name: "Set up problem matchers for PHP" - run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\"" - - - name: "Set up problem matchers for phpunit/phpunit" - run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\"" - - - name: "Determine composer cache directory" - run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV" - - - name: "Cache dependencies installed with composer" - uses: "actions/cache@v5" - with: - path: "${{ env.COMPOSER_CACHE_DIR }}" - key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}" - restore-keys: "php-${{ matrix.php-version }}-composer-" - - - name: "Install dependencies with composer" - run: "composer install --ansi --no-interaction --no-progress" - - - name: "Install dependencies" - run: "yarn install" - - - name: "Install Playwright" - run: "npx playwright install" - - - name: "Run visual tests" - run: "make tests_update_snapshots" - - - name: Update snapshots - uses: test-room-7/action-update-file@v2 - if: ${{ !cancelled() }} - with: - file-path: tests/Visual/**/* - commit-msg: Update snapshots - github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index 2d6e571924..12de398fe1 100644 --- a/Makefile +++ b/Makefile @@ -26,14 +26,9 @@ tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop -tests_visual: +tests_e2e: tests/server start; - npx playwright test --workers=$(CORES) - tests/server stop - -tests_update_snapshots: - tests/server start; - npx playwright test --update-snapshots + npx playwright test tests/server stop vendor: composer.json composer.lock diff --git a/playwright.config.ts b/playwright.config.ts index 0675282122..adf514e93b 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -30,11 +30,6 @@ export default defineConfig({ timeout: 0, projects: [ - { - name: 'chromium', - use: {...devices['Desktop Chrome']}, - testDir: './tests/Visual', - }, { name: 'End-to-End Chromium', use: {...devices['Desktop Chrome']}, diff --git a/tests/Visual/SearchModal.css b/tests/Visual/SearchModal.css deleted file mode 100644 index fa3a03bc6f..0000000000 --- a/tests/Visual/SearchModal.css +++ /dev/null @@ -1,3 +0,0 @@ -.hero__versions { - visibility: hidden !important; -} diff --git a/tests/Visual/SearchModal.spec.ts b/tests/Visual/SearchModal.spec.ts deleted file mode 100644 index 4a753c9f0a..0000000000 --- a/tests/Visual/SearchModal.spec.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { test, expect } from "@playwright/test"; -import path from "path"; - -const httpHost = process.env.HTTP_HOST; - -if (typeof httpHost !== "string") { - throw new Error('Environment variable "HTTP_HOST" is not set.'); -} - -test.beforeEach(async ({ page }) => { - await page.goto(httpHost); -}); - -const openSearchModal = async (page) => { - await page.getByRole("button", { name: "Search" }).click(); - const modal = await page.getByRole("dialog", { name: "Search modal" }); - - // Wait for the modal animation to finish - await expect(page.locator("#search-modal__backdrop.show")).not.toHaveClass( - "showing", - ); - - expect(modal).toBeVisible(); - return modal; -}; - -test("should match search modal visual snapshot", async ({ page }) => { - const modal = await openSearchModal(page); - await modal.getByRole("searchbox").fill("array"); - await expect(page).toHaveScreenshot(`tests/screenshots/search-modal.png`, { - // Cannot use mask as it ignores z-index - // See https://github.com/microsoft/playwright/issues/19002 - stylePath: path.join(__dirname, "SearchModal.css"), - }); -}); diff --git a/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png b/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png deleted file mode 100644 index 089cd07b76..0000000000 Binary files a/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts b/tests/Visual/SmokeTest.spec.ts deleted file mode 100644 index 7d8979edcc..0000000000 --- a/tests/Visual/SmokeTest.spec.ts +++ /dev/null @@ -1,76 +0,0 @@ -import {expect, test} from '@playwright/test'; - -export type TestPageOptions = { - path: string - options?: object - evaluate?: () => any - mask?: string[] -} - -const items: TestPageOptions[] = [ - { - path: 'index.php', - evaluate: () => { - const selector = document.querySelector('.elephpants'); - selector.remove() - }, - options: { - fullPage: true, - timeout: 10000, - }, - mask: ['.hero__versions'], - }, - { - path: 'archive/1998.php', - evaluate: () => { - const selector = document.querySelector('.elephpants'); - selector.remove() - } - }, - {path: 'releases/8_3_6.php'}, - {path: 'releases/8.0/index.php'}, - {path: 'releases/8.1/index.php'}, - {path: 'releases/8.2/index.php'}, - {path: 'releases/8.3/index.php'}, - {path: 'manual/index.php'}, - {path: 'manual/php5.php'}, - { - path: 'conferences/index.php', - options: { - fullPage: false, - } - }, -] - -for (const item of items) { - test(`testing with ${item.path}`, async ({page}, testInfo) => { - testInfo.snapshotSuffix = ''; - - const httpHost = process.env.HTTP_HOST - - if (typeof httpHost !== 'string') { - throw new Error('Environment variable "HTTP_HOST" is not set.') - } - - await page.goto(`${httpHost}/${item.path}`) - - if (typeof item.evaluate === 'function') { - await page.evaluate(item.evaluate) - } - - if (typeof item.mask === 'object') { - item.options = { - ...item.options, - mask: item.mask.map((selector) => page.locator(selector)) - } - } - - await expect(page).toHaveScreenshot( - `tests/screenshots/${item.path}.png`, - item.options ?? { - fullPage: true, - timeout: 10000, - } - ) - }) -} diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png deleted file mode 100644 index 847b1bae90..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png deleted file mode 100644 index 4c299f7b51..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png deleted file mode 100644 index 43ff6b6ca7..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png deleted file mode 100644 index 8ea58d12ca..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png deleted file mode 100644 index 4c615c577e..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png deleted file mode 100644 index 0c08cc4f3b..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png deleted file mode 100644 index 8aa25dd736..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png deleted file mode 100644 index 078fc5f7d8..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png deleted file mode 100644 index 26c6ff70c9..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png and /dev/null differ diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png deleted file mode 100644 index 8cb5b81304..0000000000 Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png and /dev/null differ
