From be524a8a7a261c208c8debcbd1355f6c26f59e51 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 21:10:30 +0000 Subject: [PATCH 1/8] Increase Playwright test reliability --- playwright.config.js | 21 +++- tests/events.spec.ts | 86 +++++++++++------ tests/filters.spec.ts | 216 ++++++++++++++++++------------------------ tests/theme.spec.ts | 97 +++++++++---------- 4 files changed, 215 insertions(+), 205 deletions(-) diff --git a/playwright.config.js b/playwright.config.js index 881c9f64..26d9b181 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -10,8 +10,8 @@ export default defineConfig({ // Fail the build on CI if you accidentally left test.only in the source code. forbidOnly: !!process.env.CI, - // Retry on CI only. - retries: process.env.CI ? 2 : 0, + // Increase retries for better reliability + retries: process.env.CI ? 3 : 1, // Opt out of parallel tests on CI. workers: process.env.CI ? 1 : undefined, @@ -25,6 +25,22 @@ export default defineConfig({ // Collect trace when retrying the failed test. trace: 'on-first-retry', + + // Add global timeout settings + actionTimeout: 15000, + navigationTimeout: 30000, + + // Enable screenshot on failure + screenshot: 'only-on-failure', + + // Enable video recording for failed tests + video: 'retain-on-failure', + + // Add viewport size + viewport: { width: 1280, height: 720 }, + + // Add automatic waiting + waitForNavigation: 'networkidle' }, // Configure projects for major browsers. projects: [ @@ -48,6 +64,7 @@ export default defineConfig({ command: 'netlify dev', port: 8888, reuseExistingServer: !process.env.CI, + timeout: 120000, // Increase server startup timeout } : undefined, }); diff --git a/tests/events.spec.ts b/tests/events.spec.ts index 3132d897..3c7105a6 100644 --- a/tests/events.spec.ts +++ b/tests/events.spec.ts @@ -2,41 +2,65 @@ import { test, expect } from '@playwright/test'; import AxeBuilder from '@axe-core/playwright'; -test.beforeEach(async ({ page, baseURL }) => { - await page.goto(baseURL); - // await page.waitForLoadState('networkidle'); - const filterDrawer = page.locator('#filter-drawer'); - const isVisible = await filterDrawer.isVisible(); - if (isVisible) { - await page.keyboard.press('Escape'); - await expect(filterDrawer).not.toBeVisible(); - } - await page.waitForSelector('#upcoming-events'); -}); +test.describe('Events page', () => { + test.beforeEach(async ({ page, baseURL }) => { + await page.goto(baseURL); + await page.waitForLoadState('networkidle'); + + // Clear any open drawers + const filterDrawer = page.locator('#filter-drawer'); + await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); + const isVisible = await filterDrawer.isVisible(); + if (isVisible) { + await page.keyboard.press('Escape'); + await expect(filterDrawer).not.toBeVisible(); + } -test('has title', async ({ page }) => { - await expect(page).toHaveTitle(/Eventua11y/); -}); + // Wait for main content + await Promise.all([ + page.waitForSelector('#upcoming-events', { state: 'visible' }), + page.waitForSelector('#global-header', { state: 'visible' }), + page.waitForSelector('#global-footer', { state: 'visible' }) + ]); + }); -test('header is visible', async ({ page }) => { - await expect(page.locator('#global-header')).toBeVisible(); -}); + test('has title', async ({ page }) => { + await expect(page).toHaveTitle(/Eventua11y/, { timeout: 10000 }); + }); -test('Today heading is visible', async ({ page }) => { - await expect(page.getByRole('heading', { name: 'Today' })).toBeVisible(); -}); + test('header is visible', async ({ page }) => { + const header = page.locator('#global-header'); + await expect(header).toBeVisible(); + await expect(header).toBeInViewport(); + }); -test('footer is visible', async ({ page }) => { - await expect(page.locator('#global-footer')).toBeVisible(); -}); + test('Today heading is visible', async ({ page }) => { + const heading = page.getByRole('heading', { name: 'Today' }); + await expect(heading).toBeVisible(); + await expect(heading).toBeInViewport(); + }); -test('has no accessibility violations', async ({ page }) => { - const accessibilityScanResults = await new AxeBuilder({ page }).analyze(); - expect(accessibilityScanResults.violations).toEqual([]); -}); + test('footer is visible', async ({ page }) => { + const footer = page.locator('#global-footer'); + await expect(footer).toBeVisible(); + }); + + test('has no accessibility violations', async ({ page }) => { + // Wait for dynamic content to load + await page.waitForLoadState('networkidle'); + await page.waitForSelector('#upcoming-events .event', { state: 'visible' }); + + const accessibilityScanResults = await new AxeBuilder({ page }) + .exclude('#filter-drawer') // Exclude drawer to avoid false positives + .analyze(); + + expect(accessibilityScanResults.violations).toEqual([]); + }); -test('has at least one upcoming event', async ({ page }) => { - const upcomingEvents = page.locator('.event'); - const countOfEvents = await upcomingEvents.count(); - await expect(countOfEvents).toBeGreaterThan(0); + test('has at least one upcoming event', async ({ page }) => { + const upcomingEvents = page.locator('.event'); + await upcomingEvents.first().waitFor({ state: 'visible' }); + const countOfEvents = await upcomingEvents.count(); + await expect(countOfEvents).toBeGreaterThan(0); + }); }); diff --git a/tests/filters.spec.ts b/tests/filters.spec.ts index 3e109eb5..a79fc424 100644 --- a/tests/filters.spec.ts +++ b/tests/filters.spec.ts @@ -1,125 +1,97 @@ import { test, expect } from '@playwright/test'; -test.beforeEach(async ({ page, baseURL }) => { - await page.goto(baseURL); - // Wait for hydration indicators - await page.waitForSelector('#upcoming-events'); - await page.waitForSelector('#filters'); - await page.waitForSelector('.filters__count:not(:empty)'); - await page.waitForSelector('#open-filter-drawer:not([disabled])'); - await page.waitForSelector('#upcoming-events'); - await page.waitForSelector('#filters'); -}); - -test('filter button is visible', async ({ page }) => { - // Wait for page ready - await page.waitForLoadState('domcontentloaded'); - - // Locate and verify button - const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible', timeout: 5000 }); - await expect(filterButton).toBeVisible(); - await expect(filterButton).toBeEnabled(); -}); - -test('filter drawer opens when filter button is clicked', async ({ page }) => { - // Wait for initial page load - await page.waitForLoadState('domcontentloaded'); - - // Get filter button and wait for it to be ready - const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible', timeout: 5000 }); - - // Click and wait for drawer - await filterButton.click(); - - // Get drawer and verify state - const drawer = page.locator('#filter-drawer'); - await expect(drawer).toHaveAttribute('open', ''); - await expect(drawer).toBeVisible(); - - // Optional: Wait for transition - await page.waitForTimeout(300); -}); - -test('filter drawer closes when close button is clicked', async ({ page }) => { - // Open drawer - const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible' }); - await filterButton.click({ force: true }); - - // Wait for drawer to be visible - const drawer = page.locator('#filter-drawer'); - await expect(drawer).toBeVisible(); - - // Close drawer - const closeButton = page.getByRole('button', { name: 'Close' }); - await closeButton.waitFor({ state: 'visible' }); - await closeButton.click({ force: true }); - - // Verify drawer is closed - await expect(drawer).not.toBeVisible(); - - // Optional: Wait for animation - await page.waitForTimeout(300); -}); - -test('filter drawer closes when esc key is pressed', async ({ page }) => { - // Initial page load - await page.waitForLoadState('domcontentloaded'); - - // Get and click filter button - const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible' }); - await filterButton.click(); - - // Verify drawer opens - const drawer = page.locator('#filter-drawer'); - await expect(drawer).toHaveAttribute('open', ''); - await expect(drawer).toBeVisible(); - - // Press escape and wait for transition - await page.keyboard.press('Escape'); - await page.waitForTimeout(300); - - // Verify drawer closes - await expect(drawer).not.toHaveAttribute('open'); - await expect(drawer).not.toBeVisible(); -}); - -test('reset button appears when filters are applied', async ({ page }) => { - // Setup - await page.waitForLoadState('domcontentloaded'); - - // Open drawer - const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible' }); - await filterButton.click(); - - // Verify drawer opens - const drawer = page.locator('#filter-drawer'); - await expect(drawer).toBeVisible(); - - // Verify reset button initially hidden - const resetButton = page.getByRole('button', { name: 'Reset Filters' }); - await expect(resetButton).not.toBeVisible(); - - // Change filter and wait for update - const onlineCheckbox = page.getByRole('checkbox', { name: 'Online' }); - await onlineCheckbox.waitFor({ state: 'visible' }); - await onlineCheckbox.check({ force: true }); - await page.waitForTimeout(300); - - // Verify reset button appears - await expect(resetButton.first()).toBeVisible({ timeout: 5000 }); -}); - -// Reset button clears filters -test('reset button clears filters', async ({ page }) => { - await page.getByRole('button', { name: 'Filter' }).click(); - await page - .getByRole('checkbox', { name: 'Not accepting talks' }, { exact: true }) - .check({ force: true }); - await page.getByTestId('drawer-reset').click({ force: true }); - await expect(page.getByTestId('drawer-reset')).not.toBeVisible(); +test.describe('Filters functionality', () => { + test.beforeEach(async ({ page, baseURL }) => { + await page.goto(baseURL); + await page.waitForLoadState('networkidle'); + + // Wait for critical elements and web components to be ready + await Promise.all([ + page.waitForSelector('#upcoming-events', { state: 'visible' }), + page.waitForSelector('#filters', { state: 'visible' }), + page.waitForSelector('#open-filter-drawer:not([disabled])', { state: 'visible' }) + ]); + }); + + const openFilterDrawer = async (page) => { + const filterButton = page.getByRole('button', { name: 'Filter' }); + await filterButton.waitFor({ state: 'visible' }); + await filterButton.click(); + const drawer = page.locator('#filter-drawer'); + await drawer.waitFor({ state: 'visible' }); + // Wait for drawer animation and content + await page.waitForTimeout(300); + return drawer; + }; + + test('filter button is visible and interactive', async ({ page }) => { + const filterButton = page.getByRole('button', { name: 'Filter' }); + await expect(filterButton).toBeVisible(); + await expect(filterButton).toBeEnabled(); + }); + + test('filter drawer opens when filter button is clicked', async ({ page }) => { + const drawer = await openFilterDrawer(page); + await expect(drawer).toHaveAttribute('open', ''); + await expect(drawer).toBeVisible(); + }); + + test('filter drawer closes when close button is clicked', async ({ page }) => { + const drawer = await openFilterDrawer(page); + + const closeButton = page.getByRole('button', { name: /Show \d+ of \d+ events/ }); + await closeButton.waitFor({ state: 'visible' }); + await closeButton.click(); + + // Wait for drawer animation + await page.waitForTimeout(300); + await expect(drawer).not.toBeVisible(); + await expect(drawer).not.toHaveAttribute('open'); + }); + + test('filter drawer closes when esc key is pressed', async ({ page }) => { + const drawer = await openFilterDrawer(page); + + await page.keyboard.press('Escape'); + // Wait for drawer animation + await page.waitForTimeout(300); + await expect(drawer).not.toBeVisible(); + await expect(drawer).not.toHaveAttribute('open'); + }); + + test('reset button appears when filters are applied', async ({ page }) => { + await openFilterDrawer(page); + + const resetButton = page.getByTestId('drawer-reset'); + await expect(resetButton).not.toBeVisible(); + + // Use radio button text content to find and click it + await page.getByText('Online', { exact: true }).first().click(); + await page.waitForTimeout(100); // Wait for state update + + await resetButton.scrollIntoViewIfNeeded(); + await expect(resetButton).toBeVisible(); + }); + + test('reset button clears filters', async ({ page }) => { + await openFilterDrawer(page); + + // Click the "Not accepting talks" radio using text content + await page.getByText('Not accepting talks', { exact: true }).first().click(); + await page.waitForTimeout(100); // Wait for state update + + const resetButton = page.getByTestId('drawer-reset'); + await resetButton.scrollIntoViewIfNeeded(); + await resetButton.waitFor({ state: 'visible' }); + await resetButton.click(); + + // Wait for button to disappear (confirms reset completed) + await expect(resetButton).not.toBeVisible(); + + // Wait for and verify that all radio groups show "No preference" as selected + const noPreferenceRadios = page.locator('sl-radio[value="any"]'); + for (const radio of await noPreferenceRadios.all()) { + await expect(radio).toHaveAttribute('aria-checked', 'true'); + } + }); }); diff --git a/tests/theme.spec.ts b/tests/theme.spec.ts index 402e2e99..84ee3140 100644 --- a/tests/theme.spec.ts +++ b/tests/theme.spec.ts @@ -2,25 +2,35 @@ import { test, expect } from '@playwright/test'; test.describe('Theme Switching', () => { test.beforeEach(async ({ context, page }) => { - // Set initial state + // Reset storage state await context.addInitScript(() => { window.localStorage.clear(); }); await context.clearCookies(); - - // Reset system preference + + // Reset system preference and load page await page.emulateMedia({ colorScheme: 'light' }); - - // Load page await page.goto('/'); + await page.waitForLoadState('networkidle'); + + // Ensure clean state const filterDrawer = page.locator('#filter-drawer'); - const isVisible = await filterDrawer.isVisible(); - if (isVisible) { + await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); + if (await filterDrawer.isVisible()) { await page.keyboard.press('Escape'); await expect(filterDrawer).not.toBeVisible(); } + + // Wait for theme elements + await page.waitForSelector('#theme-selector-button', { state: 'visible' }); }); + const switchTheme = async (page, themeId) => { + await page.click('#theme-selector-button'); + await page.waitForSelector('#theme-selector sl-menu', { state: 'visible' }); + await page.click(`#${themeId}`); + }; + test('should start with system theme', async ({ page }) => { await expect(page.locator('html')).toHaveAttribute('data-theme', 'light'); await expect( @@ -28,86 +38,73 @@ test.describe('Theme Switching', () => { ).toBeVisible(); }); - test('should switch to light theme and persist', async ({ - context, - page, - }) => { - // Switch theme - await page.click('#theme-selector-button'); - await page.click('#light-mode'); - - // Verify initial change + test('should switch to light theme and persist', async ({ context, page }) => { + await switchTheme(page, 'light-mode'); + await expect(page.locator('html')).toHaveAttribute('data-theme', 'light'); await expect( page.locator('#theme-selector-button sl-icon[label="Light mode"]') ).toBeVisible(); - // Verify persistence + // Test persistence in new page const newPage = await context.newPage(); await newPage.goto('/'); - await expect(newPage.locator('html')).toHaveAttribute( - 'data-theme', - 'light' - ); + await newPage.waitForLoadState('networkidle'); + + await expect(newPage.locator('html')).toHaveAttribute('data-theme', 'light'); + await newPage.close(); }); - test('should switch to dark theme and persist', async ({ - context, - page, - browser, - }) => { - // Switch theme - await page.click('#theme-selector-button'); - await page.click('#dark-mode'); - - // Verify initial change + test('should switch to dark theme and persist', async ({ context, page, browser }) => { + await switchTheme(page, 'dark-mode'); + await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark'); await expect( page.locator('#theme-selector-button sl-icon[label="Dark mode"]') ).toBeVisible(); - // Get storage state + // Test persistence with storage state const storageState = await context.storageState(); - - // Create new context with storage state const newContext = await browser.newContext({ storageState }); const newPage = await newContext.newPage(); - + await newPage.goto('/'); + await newPage.waitForLoadState('networkidle'); + await expect(newPage.locator('html')).toHaveAttribute('data-theme', 'dark'); await newContext.close(); }); - test('should respect system preference when set to system theme', async ({ - page, - }) => { + test('should respect system preference when set to system theme', async ({ page }) => { // Test dark preference + await switchTheme(page, 'system-default'); await page.emulateMedia({ colorScheme: 'dark' }); - await page.click('#theme-selector-button'); - await page.click('#system-default'); + await page.reload(); + await page.waitForLoadState('networkidle'); await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark'); // Test light preference await page.emulateMedia({ colorScheme: 'light' }); await page.reload(); + await page.waitForLoadState('networkidle'); await expect(page.locator('html')).toHaveAttribute('data-theme', 'light'); }); - test('should handle theme selector interactions correctly', async ({ - page, - }) => { + test('should handle theme selector interactions correctly', async ({ page }) => { + const menu = page.locator('#theme-selector sl-menu'); + // Test menu opening await page.click('#theme-selector-button'); - await expect(page.locator('#theme-selector sl-menu')).toBeVisible(); - + await expect(menu).toBeVisible(); + // Test click outside - await page.click('body'); - await expect(page.locator('#theme-selector sl-menu')).not.toBeVisible(); - + await page.click('body', { position: { x: 0, y: 0 } }); + await expect(menu).not.toBeVisible(); + // Test keyboard interaction await page.click('#theme-selector-button'); - await expect(page.locator('#theme-selector sl-menu')).toBeVisible(); + await expect(menu).toBeVisible(); await page.keyboard.press('Escape'); - await expect(page.locator('#theme-selector sl-menu')).not.toBeVisible(); + await expect(menu).not.toBeVisible(); }); }); From 15c6d0f46b366b3c785e297d34e6b3ae228cabd8 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 21:19:20 +0000 Subject: [PATCH 2/8] Formatting --- playwright.config.js | 2 +- tests/events.spec.ts | 8 ++++---- tests/filters.spec.ts | 41 +++++++++++++++++++++++++--------------- tests/theme.spec.ts | 44 ++++++++++++++++++++++++++++--------------- 4 files changed, 60 insertions(+), 35 deletions(-) diff --git a/playwright.config.js b/playwright.config.js index 26d9b181..62b51714 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -40,7 +40,7 @@ export default defineConfig({ viewport: { width: 1280, height: 720 }, // Add automatic waiting - waitForNavigation: 'networkidle' + waitForNavigation: 'networkidle', }, // Configure projects for major browsers. projects: [ diff --git a/tests/events.spec.ts b/tests/events.spec.ts index 3c7105a6..f27b3afc 100644 --- a/tests/events.spec.ts +++ b/tests/events.spec.ts @@ -6,7 +6,7 @@ test.describe('Events page', () => { test.beforeEach(async ({ page, baseURL }) => { await page.goto(baseURL); await page.waitForLoadState('networkidle'); - + // Clear any open drawers const filterDrawer = page.locator('#filter-drawer'); await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); @@ -20,7 +20,7 @@ test.describe('Events page', () => { await Promise.all([ page.waitForSelector('#upcoming-events', { state: 'visible' }), page.waitForSelector('#global-header', { state: 'visible' }), - page.waitForSelector('#global-footer', { state: 'visible' }) + page.waitForSelector('#global-footer', { state: 'visible' }), ]); }); @@ -49,11 +49,11 @@ test.describe('Events page', () => { // Wait for dynamic content to load await page.waitForLoadState('networkidle'); await page.waitForSelector('#upcoming-events .event', { state: 'visible' }); - + const accessibilityScanResults = await new AxeBuilder({ page }) .exclude('#filter-drawer') // Exclude drawer to avoid false positives .analyze(); - + expect(accessibilityScanResults.violations).toEqual([]); }); diff --git a/tests/filters.spec.ts b/tests/filters.spec.ts index a79fc424..ecef8740 100644 --- a/tests/filters.spec.ts +++ b/tests/filters.spec.ts @@ -9,7 +9,9 @@ test.describe('Filters functionality', () => { await Promise.all([ page.waitForSelector('#upcoming-events', { state: 'visible' }), page.waitForSelector('#filters', { state: 'visible' }), - page.waitForSelector('#open-filter-drawer:not([disabled])', { state: 'visible' }) + page.waitForSelector('#open-filter-drawer:not([disabled])', { + state: 'visible', + }), ]); }); @@ -30,19 +32,25 @@ test.describe('Filters functionality', () => { await expect(filterButton).toBeEnabled(); }); - test('filter drawer opens when filter button is clicked', async ({ page }) => { + test('filter drawer opens when filter button is clicked', async ({ + page, + }) => { const drawer = await openFilterDrawer(page); await expect(drawer).toHaveAttribute('open', ''); await expect(drawer).toBeVisible(); }); - test('filter drawer closes when close button is clicked', async ({ page }) => { + test('filter drawer closes when close button is clicked', async ({ + page, + }) => { const drawer = await openFilterDrawer(page); - - const closeButton = page.getByRole('button', { name: /Show \d+ of \d+ events/ }); + + const closeButton = page.getByRole('button', { + name: /Show \d+ of \d+ events/, + }); await closeButton.waitFor({ state: 'visible' }); await closeButton.click(); - + // Wait for drawer animation await page.waitForTimeout(300); await expect(drawer).not.toBeVisible(); @@ -51,7 +59,7 @@ test.describe('Filters functionality', () => { test('filter drawer closes when esc key is pressed', async ({ page }) => { const drawer = await openFilterDrawer(page); - + await page.keyboard.press('Escape'); // Wait for drawer animation await page.waitForTimeout(300); @@ -61,33 +69,36 @@ test.describe('Filters functionality', () => { test('reset button appears when filters are applied', async ({ page }) => { await openFilterDrawer(page); - + const resetButton = page.getByTestId('drawer-reset'); await expect(resetButton).not.toBeVisible(); - + // Use radio button text content to find and click it await page.getByText('Online', { exact: true }).first().click(); await page.waitForTimeout(100); // Wait for state update - + await resetButton.scrollIntoViewIfNeeded(); await expect(resetButton).toBeVisible(); }); test('reset button clears filters', async ({ page }) => { await openFilterDrawer(page); - + // Click the "Not accepting talks" radio using text content - await page.getByText('Not accepting talks', { exact: true }).first().click(); + await page + .getByText('Not accepting talks', { exact: true }) + .first() + .click(); await page.waitForTimeout(100); // Wait for state update - + const resetButton = page.getByTestId('drawer-reset'); await resetButton.scrollIntoViewIfNeeded(); await resetButton.waitFor({ state: 'visible' }); await resetButton.click(); - + // Wait for button to disappear (confirms reset completed) await expect(resetButton).not.toBeVisible(); - + // Wait for and verify that all radio groups show "No preference" as selected const noPreferenceRadios = page.locator('sl-radio[value="any"]'); for (const radio of await noPreferenceRadios.all()) { diff --git a/tests/theme.spec.ts b/tests/theme.spec.ts index 84ee3140..f49c6383 100644 --- a/tests/theme.spec.ts +++ b/tests/theme.spec.ts @@ -7,12 +7,12 @@ test.describe('Theme Switching', () => { window.localStorage.clear(); }); await context.clearCookies(); - + // Reset system preference and load page await page.emulateMedia({ colorScheme: 'light' }); await page.goto('/'); await page.waitForLoadState('networkidle'); - + // Ensure clean state const filterDrawer = page.locator('#filter-drawer'); await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); @@ -38,9 +38,12 @@ test.describe('Theme Switching', () => { ).toBeVisible(); }); - test('should switch to light theme and persist', async ({ context, page }) => { + test('should switch to light theme and persist', async ({ + context, + page, + }) => { await switchTheme(page, 'light-mode'); - + await expect(page.locator('html')).toHaveAttribute('data-theme', 'light'); await expect( page.locator('#theme-selector-button sl-icon[label="Light mode"]') @@ -50,14 +53,21 @@ test.describe('Theme Switching', () => { const newPage = await context.newPage(); await newPage.goto('/'); await newPage.waitForLoadState('networkidle'); - - await expect(newPage.locator('html')).toHaveAttribute('data-theme', 'light'); + + await expect(newPage.locator('html')).toHaveAttribute( + 'data-theme', + 'light' + ); await newPage.close(); }); - test('should switch to dark theme and persist', async ({ context, page, browser }) => { + test('should switch to dark theme and persist', async ({ + context, + page, + browser, + }) => { await switchTheme(page, 'dark-mode'); - + await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark'); await expect( page.locator('#theme-selector-button sl-icon[label="Dark mode"]') @@ -67,15 +77,17 @@ test.describe('Theme Switching', () => { const storageState = await context.storageState(); const newContext = await browser.newContext({ storageState }); const newPage = await newContext.newPage(); - + await newPage.goto('/'); await newPage.waitForLoadState('networkidle'); - + await expect(newPage.locator('html')).toHaveAttribute('data-theme', 'dark'); await newContext.close(); }); - test('should respect system preference when set to system theme', async ({ page }) => { + test('should respect system preference when set to system theme', async ({ + page, + }) => { // Test dark preference await switchTheme(page, 'system-default'); await page.emulateMedia({ colorScheme: 'dark' }); @@ -90,17 +102,19 @@ test.describe('Theme Switching', () => { await expect(page.locator('html')).toHaveAttribute('data-theme', 'light'); }); - test('should handle theme selector interactions correctly', async ({ page }) => { + test('should handle theme selector interactions correctly', async ({ + page, + }) => { const menu = page.locator('#theme-selector sl-menu'); - + // Test menu opening await page.click('#theme-selector-button'); await expect(menu).toBeVisible(); - + // Test click outside await page.click('body', { position: { x: 0, y: 0 } }); await expect(menu).not.toBeVisible(); - + // Test keyboard interaction await page.click('#theme-selector-button'); await expect(menu).toBeVisible(); From 6936e0d742cc7971455c15ee48ee9e69e6fedc20 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 21:31:10 +0000 Subject: [PATCH 3/8] Increase timeout for end-to-end tests on Netlify PR preview --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5e6de09d..0242e715 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: needs: tests_e2e_netlify_prepare name: Run end-to-end tests on Netlify PR preview runs-on: ubuntu-latest - timeout-minutes: 3 + timeout-minutes: 6 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 From ec84fe32b9012607240f539451382b5a7ef09421 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 21:57:17 +0000 Subject: [PATCH 4/8] Increase action and navigation timeout in Playwright configuration --- playwright.config.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playwright.config.js b/playwright.config.js index 62b51714..e1047546 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -27,8 +27,8 @@ export default defineConfig({ trace: 'on-first-retry', // Add global timeout settings - actionTimeout: 15000, - navigationTimeout: 30000, + actionTimeout: 60000, + navigationTimeout: 60000, // Enable screenshot on failure screenshot: 'only-on-failure', From ce8a1a9a2035993b24f3e48fc1af708fa77a59f1 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 22:08:04 +0000 Subject: [PATCH 5/8] Increase timeouts and improve reliability for Playwright tests --- .github/workflows/tests.yml | 8 +++++--- playwright.config.js | 9 ++++++--- tests/filters.spec.ts | 19 ++++++++++++------- tests/theme.spec.ts | 29 +++++++++++++++++++---------- 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0242e715..3dd0ea2b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: needs: tests_e2e_netlify_prepare name: Run end-to-end tests on Netlify PR preview runs-on: ubuntu-latest - timeout-minutes: 6 + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -44,7 +44,9 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps - name: Run tests - run: npx playwright test + run: npx playwright test --retries=5 env: PLAYWRIGHT_TEST_BASE_URL: 'https://deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app/' - DEBUG: pw:api + # Enable detailed debug logging + DEBUG: "pw:api,pw:browser*,pw:protocol*" + PWDEBUG: 1 diff --git a/playwright.config.js b/playwright.config.js index e1047546..289123e9 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -26,9 +26,12 @@ export default defineConfig({ // Collect trace when retrying the failed test. trace: 'on-first-retry', - // Add global timeout settings - actionTimeout: 60000, - navigationTimeout: 60000, + // Increase timeouts for CI environment + actionTimeout: process.env.CI ? 90000 : 60000, + navigationTimeout: process.env.CI ? 90000 : 60000, + + // Add explicit test timeout + testTimeout: process.env.CI ? 120000 : 60000, // Enable screenshot on failure screenshot: 'only-on-failure', diff --git a/tests/filters.spec.ts b/tests/filters.spec.ts index ecef8740..0268bf80 100644 --- a/tests/filters.spec.ts +++ b/tests/filters.spec.ts @@ -3,26 +3,31 @@ import { test, expect } from '@playwright/test'; test.describe('Filters functionality', () => { test.beforeEach(async ({ page, baseURL }) => { await page.goto(baseURL); - await page.waitForLoadState('networkidle'); - - // Wait for critical elements and web components to be ready + + // Wait for all critical components and network activity await Promise.all([ + page.waitForLoadState('networkidle'), + page.waitForLoadState('domcontentloaded'), page.waitForSelector('#upcoming-events', { state: 'visible' }), page.waitForSelector('#filters', { state: 'visible' }), page.waitForSelector('#open-filter-drawer:not([disabled])', { state: 'visible', + timeout: 10000 }), ]); }); const openFilterDrawer = async (page) => { const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible' }); + await filterButton.waitFor({ state: 'visible', timeout: 10000 }); await filterButton.click(); + const drawer = page.locator('#filter-drawer'); - await drawer.waitFor({ state: 'visible' }); - // Wait for drawer animation and content - await page.waitForTimeout(300); + await drawer.waitFor({ state: 'visible', timeout: 10000 }); + + // Wait for drawer animation and ensure content is actually visible + await page.waitForSelector('#filter-drawer[open]', { state: 'visible', timeout: 10000 }); + await page.waitForTimeout(500); // Increased animation wait time for CI return drawer; }; diff --git a/tests/theme.spec.ts b/tests/theme.spec.ts index f49c6383..1c4fae4e 100644 --- a/tests/theme.spec.ts +++ b/tests/theme.spec.ts @@ -11,18 +11,27 @@ test.describe('Theme Switching', () => { // Reset system preference and load page await page.emulateMedia({ colorScheme: 'light' }); await page.goto('/'); - await page.waitForLoadState('networkidle'); - - // Ensure clean state + + // Wait for critical components to be ready + await Promise.all([ + page.waitForLoadState('networkidle'), + page.waitForLoadState('domcontentloaded'), + page.waitForSelector('#upcoming-events', { state: 'visible' }), + page.waitForSelector('#theme-selector-button', { state: 'visible' }) + ]); + + // Ensure clean state with more reliable drawer handling const filterDrawer = page.locator('#filter-drawer'); - await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); - if (await filterDrawer.isVisible()) { - await page.keyboard.press('Escape'); - await expect(filterDrawer).not.toBeVisible(); + try { + await filterDrawer.waitFor({ state: 'attached', timeout: 10000 }); + if (await filterDrawer.isVisible()) { + await page.keyboard.press('Escape'); + await filterDrawer.waitFor({ state: 'hidden', timeout: 5000 }); + } + } catch (e) { + // If drawer timeout occurs, it's likely already hidden + console.log('Filter drawer not found or already hidden'); } - - // Wait for theme elements - await page.waitForSelector('#theme-selector-button', { state: 'visible' }); }); const switchTheme = async (page, themeId) => { From fee87ba363433f112987b47efbd0cd8aa739616c Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 22:16:24 +0000 Subject: [PATCH 6/8] Add system dependencies and environment info to GitHub Actions workflow --- .github/workflows/tests.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3dd0ea2b..80225f79 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,12 +39,23 @@ jobs: - uses: actions/setup-node@v4 with: node-version: lts/* + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y xvfb fonts-liberation fonts-noto-color-emoji libfontconfig1 libfreetype6 - name: Install dependencies run: npm ci - name: Install Playwright browsers run: npx playwright install --with-deps + - name: Show environment info + run: | + echo "Node version: $(node -v)" + echo "NPM version: $(npm -v)" + echo "Display: $DISPLAY" + echo "XDG_RUNTIME_DIR: $XDG_RUNTIME_DIR" + xvfb-run --auto-servernum ls -l /tmp/.X* - name: Run tests - run: npx playwright test --retries=5 + run: xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" npx playwright test --retries=5 env: PLAYWRIGHT_TEST_BASE_URL: 'https://deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app/' # Enable detailed debug logging From c2366c5c8610eae51e67875d17940435e80954ee Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 22:40:42 +0000 Subject: [PATCH 7/8] Refactor CI configuration and improve timeout settings for Playwright tests --- .github/workflows/tests.yml | 26 +++++++++++--------------- playwright.config.js | 31 +++++++++++++++++++++++++------ tests/filters.spec.ts | 20 ++++++++++---------- tests/theme.spec.ts | 6 +++--- 4 files changed, 49 insertions(+), 34 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 80225f79..e0e2398d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,31 +33,27 @@ jobs: needs: tests_e2e_netlify_prepare name: Run end-to-end tests on Netlify PR preview runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/playwright:v1.40.0-jammy timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: lts/* - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y xvfb fonts-liberation fonts-noto-color-emoji libfontconfig1 libfreetype6 - name: Install dependencies run: npm ci - - name: Install Playwright browsers - run: npx playwright install --with-deps - - name: Show environment info + - name: Verify network connectivity run: | - echo "Node version: $(node -v)" - echo "NPM version: $(npm -v)" - echo "Display: $DISPLAY" - echo "XDG_RUNTIME_DIR: $XDG_RUNTIME_DIR" - xvfb-run --auto-servernum ls -l /tmp/.X* + echo "Testing connection to Netlify preview..." + curl -v https://deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app/ + echo "Testing DNS resolution..." + nslookup deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app - name: Run tests - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" npx playwright test --retries=5 env: PLAYWRIGHT_TEST_BASE_URL: 'https://deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app/' - # Enable detailed debug logging DEBUG: "pw:api,pw:browser*,pw:protocol*" - PWDEBUG: 1 + CI: 'true' + run: | + echo "Starting Playwright tests..." + npx playwright test --project=chromium --retries=5 --reporter=list diff --git a/playwright.config.js b/playwright.config.js index 289123e9..75caf2b4 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -26,12 +26,12 @@ export default defineConfig({ // Collect trace when retrying the failed test. trace: 'on-first-retry', - // Increase timeouts for CI environment - actionTimeout: process.env.CI ? 90000 : 60000, - navigationTimeout: process.env.CI ? 90000 : 60000, + // More reasonable timeouts for CI + actionTimeout: process.env.CI ? 45000 : 30000, + navigationTimeout: process.env.CI ? 45000 : 30000, // Add explicit test timeout - testTimeout: process.env.CI ? 120000 : 60000, + testTimeout: process.env.CI ? 90000 : 60000, // Enable screenshot on failure screenshot: 'only-on-failure', @@ -44,12 +44,31 @@ export default defineConfig({ // Add automatic waiting waitForNavigation: 'networkidle', + + // CI-specific browser launch options + launchOptions: process.env.CI ? { + args: ['--no-sandbox', '--disable-setuid-sandbox'], + } : undefined, + + // Use persistent context in CI to reduce browser startup overhead + contextOptions: process.env.CI ? { + acceptDownloads: false, + strictSelectors: true, + } : undefined, }, // Configure projects for major browsers. projects: [ { name: 'chromium', - use: { ...devices['Desktop Chrome'] }, + use: { + ...devices['Desktop Chrome'], + // Additional chromium-specific settings for CI + launchOptions: process.env.CI ? { + executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu'], + headless: true + } : undefined, + }, }, { name: 'firefox', @@ -67,7 +86,7 @@ export default defineConfig({ command: 'netlify dev', port: 8888, reuseExistingServer: !process.env.CI, - timeout: 120000, // Increase server startup timeout + timeout: 60000, // Reduced server startup timeout } : undefined, }); diff --git a/tests/filters.spec.ts b/tests/filters.spec.ts index 0268bf80..a19cf78c 100644 --- a/tests/filters.spec.ts +++ b/tests/filters.spec.ts @@ -12,22 +12,22 @@ test.describe('Filters functionality', () => { page.waitForSelector('#filters', { state: 'visible' }), page.waitForSelector('#open-filter-drawer:not([disabled])', { state: 'visible', - timeout: 10000 + timeout: 5000 }), ]); }); const openFilterDrawer = async (page) => { const filterButton = page.getByRole('button', { name: 'Filter' }); - await filterButton.waitFor({ state: 'visible', timeout: 10000 }); + await filterButton.waitFor({ state: 'visible', timeout: 5000 }); await filterButton.click(); const drawer = page.locator('#filter-drawer'); - await drawer.waitFor({ state: 'visible', timeout: 10000 }); + await drawer.waitFor({ state: 'visible', timeout: 5000 }); // Wait for drawer animation and ensure content is actually visible - await page.waitForSelector('#filter-drawer[open]', { state: 'visible', timeout: 10000 }); - await page.waitForTimeout(500); // Increased animation wait time for CI + await page.waitForSelector('#filter-drawer[open]', { state: 'visible', timeout: 5000 }); + await page.waitForTimeout(300); // Standard animation duration return drawer; }; @@ -53,13 +53,13 @@ test.describe('Filters functionality', () => { const closeButton = page.getByRole('button', { name: /Show \d+ of \d+ events/, }); - await closeButton.waitFor({ state: 'visible' }); + await closeButton.waitFor({ state: 'visible', timeout: 5000 }); await closeButton.click(); // Wait for drawer animation await page.waitForTimeout(300); await expect(drawer).not.toBeVisible(); - await expect(drawer).not.toHaveAttribute('open'); + await expect(drawer).not toHaveAttribute('open'); }); test('filter drawer closes when esc key is pressed', async ({ page }) => { @@ -69,14 +69,14 @@ test.describe('Filters functionality', () => { // Wait for drawer animation await page.waitForTimeout(300); await expect(drawer).not.toBeVisible(); - await expect(drawer).not.toHaveAttribute('open'); + await expect(drawer).not toHaveAttribute('open'); }); test('reset button appears when filters are applied', async ({ page }) => { await openFilterDrawer(page); const resetButton = page.getByTestId('drawer-reset'); - await expect(resetButton).not.toBeVisible(); + await expect(resetButton).not toBeVisible(); // Use radio button text content to find and click it await page.getByText('Online', { exact: true }).first().click(); @@ -102,7 +102,7 @@ test.describe('Filters functionality', () => { await resetButton.click(); // Wait for button to disappear (confirms reset completed) - await expect(resetButton).not.toBeVisible(); + await expect(resetButton).not toBeVisible(); // Wait for and verify that all radio groups show "No preference" as selected const noPreferenceRadios = page.locator('sl-radio[value="any"]'); diff --git a/tests/theme.spec.ts b/tests/theme.spec.ts index 1c4fae4e..af507984 100644 --- a/tests/theme.spec.ts +++ b/tests/theme.spec.ts @@ -20,13 +20,13 @@ test.describe('Theme Switching', () => { page.waitForSelector('#theme-selector-button', { state: 'visible' }) ]); - // Ensure clean state with more reliable drawer handling + // Ensure clean state with more reasonable drawer handling const filterDrawer = page.locator('#filter-drawer'); try { - await filterDrawer.waitFor({ state: 'attached', timeout: 10000 }); + await filterDrawer.waitFor({ state: 'attached', timeout: 5000 }); if (await filterDrawer.isVisible()) { await page.keyboard.press('Escape'); - await filterDrawer.waitFor({ state: 'hidden', timeout: 5000 }); + await filterDrawer.waitFor({ state: 'hidden', timeout: 2000 }); } } catch (e) { // If drawer timeout occurs, it's likely already hidden From 76ed7d54063f57f60f1670c79ec79132f6a3ee02 Mon Sep 17 00:00:00 2001 From: Matt Obee Date: Sun, 9 Feb 2025 22:48:00 +0000 Subject: [PATCH 8/8] Formatting --- .github/workflows/tests.yml | 2 +- playwright.config.js | 36 +++++++++++++++++++++++------------- tests/theme.spec.ts | 4 ++-- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e0e2398d..39c2f8e3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,7 +52,7 @@ jobs: - name: Run tests env: PLAYWRIGHT_TEST_BASE_URL: 'https://deploy-preview-${{ env.GITHUB_PR_NUMBER }}--eventua11y.netlify.app/' - DEBUG: "pw:api,pw:browser*,pw:protocol*" + DEBUG: 'pw:api,pw:browser*,pw:protocol*' CI: 'true' run: | echo "Starting Playwright tests..." diff --git a/playwright.config.js b/playwright.config.js index 75caf2b4..b6e8d5a0 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -46,28 +46,38 @@ export default defineConfig({ waitForNavigation: 'networkidle', // CI-specific browser launch options - launchOptions: process.env.CI ? { - args: ['--no-sandbox', '--disable-setuid-sandbox'], - } : undefined, + launchOptions: process.env.CI + ? { + args: ['--no-sandbox', '--disable-setuid-sandbox'], + } + : undefined, // Use persistent context in CI to reduce browser startup overhead - contextOptions: process.env.CI ? { - acceptDownloads: false, - strictSelectors: true, - } : undefined, + contextOptions: process.env.CI + ? { + acceptDownloads: false, + strictSelectors: true, + } + : undefined, }, // Configure projects for major browsers. projects: [ { name: 'chromium', - use: { + use: { ...devices['Desktop Chrome'], // Additional chromium-specific settings for CI - launchOptions: process.env.CI ? { - executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu'], - headless: true - } : undefined, + launchOptions: process.env.CI + ? { + executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-gpu', + ], + headless: true, + } + : undefined, }, }, { diff --git a/tests/theme.spec.ts b/tests/theme.spec.ts index af507984..6ba255a6 100644 --- a/tests/theme.spec.ts +++ b/tests/theme.spec.ts @@ -11,13 +11,13 @@ test.describe('Theme Switching', () => { // Reset system preference and load page await page.emulateMedia({ colorScheme: 'light' }); await page.goto('/'); - + // Wait for critical components to be ready await Promise.all([ page.waitForLoadState('networkidle'), page.waitForLoadState('domcontentloaded'), page.waitForSelector('#upcoming-events', { state: 'visible' }), - page.waitForSelector('#theme-selector-button', { state: 'visible' }) + page.waitForSelector('#theme-selector-button', { state: 'visible' }), ]); // Ensure clean state with more reasonable drawer handling