From 907606684d22b19c7555a2e7539c31b1c8687935 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Sep 2024 13:54:53 +0200 Subject: [PATCH 1/3] Add Playwright-based tests Playwright (https://playwright.dev/) is a quite complete framework to perform UI tests of websites. We use this framework here to verify that: - the search works - the correct download is suggested, based on the current browser - a subset of the GUIs are shown based on the platform indicated by the current browser - switching between languages works - following links from a translated manual page fall back to English if the linked-to manual page has no translation - navigating between book sections colorizes the correct links in the drop-down A new feature of the Hugo/Pagefind-based site is that the search facility is language-aware. For this reason, we also verify that: - search results are language-dependent These tests can be run by first installing the `@playwright/test` dependency, via `npm install`, and then invoking `npx playwright test`. Signed-off-by: Johannes Schindelin --- .gitignore | 4 ++ package.json | 10 +++ playwright.config.js | 79 ++++++++++++++++++++ tests/git-scm.spec.js | 162 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 255 insertions(+) create mode 100644 package.json create mode 100644 playwright.config.js create mode 100644 tests/git-scm.spec.js diff --git a/.gitignore b/.gitignore index 9f7ce75d3d..52f2f3d341 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ /.hugo_build.lock /public/ /resources/_gen/ +/package-lock.json +/node_modules/ +/test-results/ +/playwright-report/ diff --git a/package.json b/package.json new file mode 100644 index 0000000000..7bc247a84f --- /dev/null +++ b/package.json @@ -0,0 +1,10 @@ +{ + "name": "git-scm.com", + "version": "0.0.0", + "description": "This is the Git home page.", + "license": "MIT", + "devDependencies": { + "@playwright/test": "^1.47.0", + "@types/node": "^22.5.4" + } +} diff --git a/playwright.config.js b/playwright.config.js new file mode 100644 index 0000000000..4208666d32 --- /dev/null +++ b/playwright.config.js @@ -0,0 +1,79 @@ +// @ts-check +const { defineConfig, devices } = require('@playwright/test'); + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config({ path: path.resolve(__dirname, '.env') }); + +/** + * @see https://playwright.dev/docs/test-configuration + */ +module.exports = defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + // baseURL: 'http://127.0.0.1:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); + diff --git a/tests/git-scm.spec.js b/tests/git-scm.spec.js new file mode 100644 index 0000000000..a3c182139a --- /dev/null +++ b/tests/git-scm.spec.js @@ -0,0 +1,162 @@ +const { test, expect, selectors } = require('@playwright/test') + +const url = 'https://git.github.io/git-scm.com/' + +async function pretendPlatform(page, platform) { + await page.context().addInitScript({ + content: `Object.defineProperty(navigator, 'platform', { get: () => '${platform}' })` + }) +} + +test.describe('Windows', () => { + test.use({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + }) + + test('download/GUI links', async ({ page }) => { + pretendPlatform(page, 'Windows') + await page.goto(url) + await expect(page.getByRole('link', { name: 'Download for Windows' })).toBeVisible() + + await expect(page.getByRole('link', { name: 'Graphical UIs' })).not.toBeVisible() + const windowsGUIs = page.getByRole('link', { name: 'Windows GUIs' }) + await expect(windowsGUIs).toBeVisible() + await expect(windowsGUIs).toHaveAttribute('href', /\/download\/gui\/windows$/) + + // navigate to Windows GUIs + await windowsGUIs.click() + const windowsButton = page.getByRole('link', { name: 'Windows' }) + await expect(windowsButton).toBeVisible() + await expect(windowsButton).toHaveClass(/selected/) + + const allButton = page.getByRole('link', { name: 'All' }) + await expect(allButton).not.toHaveClass(/selected/) + + const thumbnails = page.locator('.gui-thumbnails li:visible') + const count = await thumbnails.count() + await allButton.click() + await expect.poll(() => thumbnails.count()).toBeGreaterThan(count) + }) +}) + +test.describe('macOS', () => { + test.use({ + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15', + }) + + test('download/GUI links', async ({ page }) => { + pretendPlatform(page, 'Mac OS X') + await page.goto(url) + await expect(page.getByRole('link', { name: 'Download for Mac' })).toBeVisible() + + await expect(page.getByRole('link', { name: 'Graphical UIs' })).not.toBeVisible() + await expect(page.getByRole('link', { name: 'Mac GUIs' })).toBeVisible() + }) +}) + +test.describe('Linux', () => { + test.use({ + userAgent: 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:130.0) Gecko/20100101 Firefox/130.0', + }) + + test('download/GUI links', async ({ page }) => { + pretendPlatform(page, 'Linux') + await page.goto(url) + await expect(page.getByRole('link', { name: 'Download for Linux' })).toBeVisible() + + await expect(page.getByRole('link', { name: 'Graphical UIs' })).not.toBeVisible() + await expect(page.getByRole('link', { name: 'Linux GUIs' })).toBeVisible() + }) +}) + +test('search', async ({ page }) => { + await page.goto(url) + + // Search for "commit" + const searchBox = page.getByPlaceholder('Type / to search entire site…') + await searchBox.fill('commit') + await searchBox.press('Shift') + + // Expect the div to show up + const showAllResults = page.getByText('Show all results...') + await expect(showAllResults).toBeVisible() + + // Expect the first search result to be "git-commit" + const searchResults = page.locator('#search-results') + await expect(searchResults.getByRole("link")).not.toHaveCount(0) + await expect(searchResults.getByRole("link").nth(0)).toHaveText('git-commit') + + // On localized pages, the search results should be localized as well + await page.goto(`${url}docs/git-commit/fr`) + await searchBox.fill('add') + await searchBox.press('Shift') + await expect(searchResults.getByRole("link").nth(0)).toHaveAttribute('href', /\/docs\/git-add\/fr(\.html)?$/) + + // pressing the Enter key should navigate to the full search results page + await searchBox.press('Enter') + await expect(page).toHaveURL(/\/search.*language=fr/) +}) + +test('manual pages', async ({ page }) => { + await page.goto(`${url}docs/git-config`) + + // The summary follows immediately after the heading "NAME", which is the first heading on the page + const summary = page.locator('xpath=//h2/following-sibling::*[1]').first() + await expect(summary).toHaveText('git-config - Get and set repository or global options') + await expect(summary).toBeVisible() + + // Verify that the drop-downs are shown when clicked + const previousVersionDropdown = page.locator('#previous-versions-dropdown') + await expect(previousVersionDropdown).not.toBeVisible() + await page.getByRole('link', { name: 'Latest version' }).click() + await expect(previousVersionDropdown).toBeVisible() + + const topicsDropdown = page.locator('#topics-dropdown') + await expect(topicsDropdown).not.toBeVisible() + await page.getByRole('link', { name: 'Topics' }).click() + await expect(topicsDropdown).toBeVisible() + await expect(previousVersionDropdown).not.toBeVisible() + + const languageDropdown = page.locator('#l10n-versions-dropdown') + await expect(languageDropdown).not.toBeVisible() + await page.getByRole('link', { name: 'English' }).click() + await expect(languageDropdown).toBeVisible() + await expect(topicsDropdown).not.toBeVisible() + await expect(previousVersionDropdown).not.toBeVisible() + + // Verify that the language is changed when a different language is selected + await page.getByRole('link', { name: 'Français' }).click() + await expect(summary).toHaveText('git-config - Lire et écrire les options du dépôt et les options globales') + await expect(summary).not.toHaveText('git-config - Get and set repository or global options') + + // links to other manual pages should stay within the language when possible, + // but fall back to English if the page was not yet translated + const gitRevisionsLink = page.getByRole('link', { name: 'gitrevisions[7]' }) + await expect(gitRevisionsLink).toBeVisible() + await expect(gitRevisionsLink).toHaveAttribute('href', /\/docs\/gitrevisions\/fr$/) + gitRevisionsLink.click() + await expect(page).toHaveURL(/\/docs\/gitrevisions$/) +}) + +test('book', async ({ page }) => { + await page.goto(`${url}book`) + + // Navigate to the first section + await page.getByRole('link', { name: 'Getting Started' }).click() + await expect(page).toHaveURL(/Getting-Started-About-Version-Control/) + + // Verify that the drop-down is shown when clicked + const chaptersDropdown = page.locator('#chapters-dropdown') + await expect(chaptersDropdown).not.toBeVisible() + await page.getByRole('link', { name: 'Chapters' }).click() + await expect(chaptersDropdown).toBeVisible() + + // Only the current section is marked as active + await expect(chaptersDropdown.getByRole('link', { name: /About Version Control/ })).toHaveClass(/active/) + await expect(chaptersDropdown.locator('.active')).toHaveCount(1) + + // Navigate to the French translation + await page.getByRole('link', { name: 'Français' }).click() + await expect(page).toHaveURL(/book\/fr/) + await expect(page.getByRole('link', { name: 'Démarrage rapide' })).toBeVisible() +}) \ No newline at end of file From 0fe01913b3838a53b19082bd5a91f68dbfee37d0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Sep 2024 16:16:43 +0200 Subject: [PATCH 2/3] playwright: let the UI tests pass with the Rails app Currently, the Git home page is implemented as a Rails app. For various reasons, this app behaves a bit different than the upcoming Hugo/Pagefind-based home page. Let's document the differences via the UI tests. Signed-off-by: Johannes Schindelin --- tests/git-scm.spec.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/git-scm.spec.js b/tests/git-scm.spec.js index a3c182139a..8c22d1b535 100644 --- a/tests/git-scm.spec.js +++ b/tests/git-scm.spec.js @@ -1,6 +1,8 @@ const { test, expect, selectors } = require('@playwright/test') -const url = 'https://git.github.io/git-scm.com/' +const url = 'https://git-scm.com/' +// const url = 'https://git.github.io/git-scm.com/' +const isRailsApp = url === 'https://git-scm.com/' async function pretendPlatform(page, platform) { await page.context().addInitScript({ @@ -90,11 +92,19 @@ test('search', async ({ page }) => { await page.goto(`${url}docs/git-commit/fr`) await searchBox.fill('add') await searchBox.press('Shift') - await expect(searchResults.getByRole("link").nth(0)).toHaveAttribute('href', /\/docs\/git-add\/fr(\.html)?$/) + if (isRailsApp) { + await expect(searchResults.getByRole("link").nth(0)).toHaveAttribute('href', /\/docs\/git-add$/) + } else { + await expect(searchResults.getByRole("link").nth(0)).toHaveAttribute('href', /\/docs\/git-add\/fr(\.html)?$/) + } // pressing the Enter key should navigate to the full search results page await searchBox.press('Enter') - await expect(page).toHaveURL(/\/search.*language=fr/) + if (isRailsApp) { + await expect(page).toHaveURL(/\/search/) + } else { + await expect(page).toHaveURL(/\/search.*language=fr/) + } }) test('manual pages', async ({ page }) => { @@ -108,7 +118,11 @@ test('manual pages', async ({ page }) => { // Verify that the drop-downs are shown when clicked const previousVersionDropdown = page.locator('#previous-versions-dropdown') await expect(previousVersionDropdown).not.toBeVisible() - await page.getByRole('link', { name: 'Latest version' }).click() + if (isRailsApp) { + await page.getByRole('link', { name: /Version \d+\.\d+\.\d+/ }).click() + } else { + await page.getByRole('link', { name: 'Latest version' }).click() + } await expect(previousVersionDropdown).toBeVisible() const topicsDropdown = page.locator('#topics-dropdown') From b43a1240b5095a42d0d3858d8ae887dd9a9d05e3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Sep 2024 16:26:23 +0200 Subject: [PATCH 3/3] WIP: ci: run Playwright-based UI tests The UI tests are most beneficial when they are run regularly ;-) To this end, let's run them on every PR, and let's allow running it manually with the deployed website. Signed-off-by: Johannes Schindelin --- .github/workflows/playwright.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/playwright.yml diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 0000000000..f2ed463b5d --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,22 @@ +name: Playwright Tests +on: + pull_request: + workflow_dispatch: +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Run Playwright tests + run: npx playwright test + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report + path: playwright-report/