add e2e testing (#777)

* docs(test): add playwright modes test plan

* test(e2e): add playwright phase-1 mode coverage

* test(e2e): add classic standalone no-server regression

* ci(docker): gate image build on playwright e2e

* ci(e2e): run on master push and allow manual dispatch

---------

Co-authored-by: Stefan Stidl <stefan.stidl@ffg.at>

.github/workflows/docker-publish.yml

@@ -24,7 +24,30 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Install Playwright Chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Run Playwright tests
+        run: npm run test:e2e
+
   build:
+    needs: e2e
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

.github/workflows/playwright.yml

@@ -0,0 +1,30 @@
+name: Playwright E2E
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Install Playwright Chromium
+        run: npx playwright install --with-deps chromium
+
+      - name: Run Playwright tests
+        run: npm run test:e2e

.gitignore

@@ -4,6 +4,8 @@ db-dir/
 .vscode/
 node_modules/
 package-lock.json
+playwright-report/
+test-results/
 results/speedtest_telemetry.db
 results/speedtest_telemetry.db-shm
 results/speedtest_telemetry.db-wal

package.json

@@ -5,6 +5,8 @@
   "main": "speedtest.js",
   "scripts": {
     "test": "echo \"No automated tests configured yet\" && exit 0",
+    "test:e2e": "playwright test",
+    "test:e2e:headed": "playwright test --headed",
     "lint": "eslint speedtest.js speedtest_worker.js",
     "lint:fix": "eslint --fix speedtest.js speedtest_worker.js",
     "format": "prettier --write \"*.js\"",
@@ -37,6 +39,7 @@
   },
   "homepage": "https://github.com/librespeed/speedtest#readme",
   "devDependencies": {
+    "@playwright/test": "^1.55.0",
     "eslint": "^8.57.0",
     "prettier": "^3.1.1"
   },

playwright.config.js

@@ -0,0 +1,27 @@
+const { defineConfig, devices } = require('@playwright/test');
+
+module.exports = defineConfig({
+  testDir: './tests/e2e',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 1 : 0,
+  workers: process.env.CI ? 2 : undefined,
+  timeout: 30_000,
+  expect: {
+    timeout: 7_500,
+  },
+  reporter: process.env.CI ? [['github'], ['html', { open: 'never' }]] : [['list']],
+  use: {
+    trace: 'on-first-retry',
+    screenshot: 'only-on-failure',
+    video: 'off',
+  },
+  globalSetup: require.resolve('./tests/e2e/global-setup.js'),
+  globalTeardown: require.resolve('./tests/e2e/global-teardown.js'),
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});

tests/PLAYWRIGHT_MODES_PLAN.md

@@ -0,0 +1,120 @@
+# Playwright Plan: Test All Runtime and UI Modes
+
+## Objective
+Build a deterministic Playwright test suite that validates LibreSpeed behavior across all supported deployment modes and UI design modes, without asserting real network throughput values.
+
+## Modes to Cover
+
+### Docker runtime modes
+- `standalone`
+- `backend`
+- `frontend`
+- `dual`
+
+### UI design modes
+- Classic (`index-classic.html`)
+- Modern (`index-modern.html`)
+- Switcher behavior from `index.html`:
+  - default from `config.json` (`useNewDesign`)
+  - `?design=new` override
+  - `?design=old` override
+
+## Test Strategy
+
+### 1. Keep assertions deterministic
+Do not assert real bandwidth numbers. Focus on:
+- HTTP availability of expected files/endpoints
+- correct redirects/switching behavior
+- expected UI controls rendered
+- expected server-list loading behavior
+- ability to initiate/abort flow at UI level
+
+### 2. Separate test types
+- **Mode smoke tests** (fast, always-run): verify each runtime mode serves the right surfaces.
+- **UI mode tests**: verify classic/modern pages and switcher rules.
+- **Optional flow tests** (later): mock `Speedtest` in browser to simulate state changes and verify UI updates.
+
+### 3. Use Docker Compose as the environment contract
+Run Playwright against containers started with explicit `MODE` values to mirror production entrypoint behavior.
+
+## Proposed Test Matrix
+
+### A) `standalone`
+Expectations:
+- `GET /` responds and serves UI (classic by default unless overridden)
+- `GET /backend/empty.php`, `GET /backend/garbage.php`, `GET /backend/getIP.php` available
+- `GET /results/telemetry.php` reachable (even if telemetry disabled behavior differs)
+- `GET /index.html?design=new` resolves to modern page
+- `GET /index.html?design=old` resolves to classic page
+
+### B) `backend`
+Expectations:
+- backend endpoints return success (`/backend/empty.php`, `/backend/garbage.php`, `/backend/getIP.php`)
+- tests only assert local backend endpoint contracts in this mode
+
+### C) `frontend`
+Expectations:
+- UI entrypoint available
+- server list loads from `/servers.json` (or `SERVER_LIST_URL` if set)
+- backend test endpoints should not be treated as local testpoint contract in this mode
+- selecting server and pressing start does not crash UI shell
+
+### D) `dual`
+Expectations:
+- combines frontend + local backend availability
+- UI can load multi-server list
+- backend endpoints available locally
+
+## Playwright Architecture
+
+### Files
+- `playwright.config.ts`
+- `tests/e2e/modes.spec.ts` (runtime-mode smoke)
+- `tests/e2e/design-switch.spec.ts` (classic/modern/switch overrides)
+- `tests/e2e/helpers/env.ts` (base URLs + mode metadata)
+- `tests/e2e/helpers/ui.ts` (shared selectors, start/abort helpers)
+
+### Environment boot
+- `docker compose -f tests/docker-compose-playwright.yml up -d --build`
+- dedicate one service per runtime mode on separate ports
+- for `frontend` and `dual`, mount a stable `servers.json`
+
+### Selector policy
+Use role/text selectors anchored on stable labels and IDs already in pages; avoid brittle CSS-path selectors.
+
+## Phased Rollout
+
+### Phase 1 (recommended first PR)
+- Add Playwright scaffolding and CI job
+- Add smoke coverage for 4 Docker runtime modes
+- Add design switch tests (`index.html`, `?design=new`, `?design=old`)
+- No full speed measurement assertions
+
+### Phase 2
+- Add deterministic UI flow tests with mocked Speedtest state updates
+- Validate button states (`Start` -> running -> abort/end)
+- Validate result widgets receive simulated values
+
+### Phase 3
+- Add telemetry-enabled scenario tests (`TELEMETRY=true`) for link visibility and stats exposure
+- Add negative tests (missing/invalid `servers.json` in frontend/dual)
+
+## Risks and Mitigations
+- Flaky speed measurements due to host/network variance
+  - Mitigation: avoid throughput assertions; use mocked state for UI behavior.
+- Divergence between local static run and Docker entrypoint behavior
+  - Mitigation: run all mode tests against Docker services.
+- Selector drift between classic and modern UIs
+  - Mitigation: maintain per-design helper selectors with minimal coupling.
+
+## CI Proposal
+- Trigger on PR + main branch
+- Build test image once, run mode services in parallel ports
+- Playwright retries: `1` in CI, `0` locally
+- Upload traces/screenshots on failure only
+- Browser scope for v1: Chromium only
+
+## Confirmed Decisions
+1. Browser scope for v1: Chromium only.
+2. Telemetry checks are deferred to Phase 3.
+3. `backend` mode tests assert backend endpoint contracts only.

tests/docker-compose-playwright.yml

@@ -0,0 +1,68 @@
+services:
+  backend-testpoint:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    environment:
+      - MODE=backend
+      - WEBPORT=8080
+
+  standalone:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    environment:
+      - MODE=standalone
+      - WEBPORT=8080
+      - USE_NEW_DESIGN=false
+    ports:
+      - "18180:8080"
+
+  standalone-new:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    environment:
+      - MODE=standalone
+      - WEBPORT=8080
+      - USE_NEW_DESIGN=true
+    ports:
+      - "18185:8080"
+
+  backend:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    environment:
+      - MODE=backend
+      - WEBPORT=8080
+    ports:
+      - "18181:8080"
+
+  frontend:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    depends_on:
+      - backend-testpoint
+    environment:
+      - MODE=frontend
+      - WEBPORT=8080
+    volumes:
+      - ./e2e/fixtures/servers-frontend.json:/servers.json:ro
+    ports:
+      - "18182:8080"
+
+  dual:
+    build:
+      context: ..
+      dockerfile: Dockerfile
+    depends_on:
+      - backend-testpoint
+    environment:
+      - MODE=dual
+      - WEBPORT=8080
+    volumes:
+      - ./e2e/fixtures/servers-dual.json:/servers.json:ro
+    ports:
+      - "18183:8080"

tests/e2e/classic-standalone-regression.spec.js

@@ -0,0 +1,16 @@
+const { test, expect } = require('@playwright/test');
+const { baseUrls } = require('./helpers/env');
+const { classicStartButton } = require('./helpers/ui');
+
+test.describe('Classic standalone regression coverage', () => {
+  test('standalone classic does not get stuck on "No servers available"', async ({ page }) => {
+    await page.goto(`${baseUrls.standalone}/index-classic.html`);
+
+    // In standalone mode, classic UI should show the test wrapper directly.
+    await expect(page.locator('#testWrapper')).toHaveClass(/visible/, { timeout: 10_000 });
+    await expect(page.locator('#loading')).toHaveClass(/hidden/, { timeout: 10_000 });
+
+    await expect(page.locator('#message')).not.toContainText(/No servers available/i);
+    await expect(classicStartButton(page)).toBeVisible();
+  });
+});

tests/e2e/design-switch.spec.js

@@ -0,0 +1,29 @@
+const { test, expect } = require('@playwright/test');
+const { baseUrls } = require('./helpers/env');
+const { classicStartButton, modernStartButton } = require('./helpers/ui');
+
+test.describe('Design switch behavior', () => {
+  test('index.html defaults to classic when useNewDesign=false', async ({ page }) => {
+    await page.goto(`${baseUrls.standalone}/index.html`);
+    await expect(page).toHaveURL(/index-classic\.html/);
+    await expect(classicStartButton(page)).toBeVisible();
+  });
+
+  test('index.html defaults to modern when useNewDesign=true', async ({ page }) => {
+    await page.goto(`${baseUrls.standaloneNew}/index.html`);
+    await expect(page).toHaveURL(/index-modern\.html/);
+    await expect(modernStartButton(page)).toBeVisible();
+  });
+
+  test('query override design=new forces modern', async ({ page }) => {
+    await page.goto(`${baseUrls.standalone}/index.html?design=new`);
+    await expect(page).toHaveURL(/index-modern\.html\?design=new/);
+    await expect(modernStartButton(page)).toBeVisible();
+  });
+
+  test('query override design=old forces classic', async ({ page }) => {
+    await page.goto(`${baseUrls.standaloneNew}/index.html?design=old`);
+    await expect(page).toHaveURL(/index-classic\.html\?design=old/);
+    await expect(classicStartButton(page)).toBeVisible();
+  });
+});

tests/e2e/fixtures/servers-dual.json

@@ -0,0 +1,20 @@
+[
+  {
+    "name": "Local dual backend",
+    "server": "/",
+    "dlURL": "backend/garbage.php",
+    "ulURL": "backend/empty.php",
+    "pingURL": "backend/empty.php",
+    "getIpURL": "backend/getIP.php",
+    "id": 1
+  },
+  {
+    "name": "External backend testpoint",
+    "server": "http://backend-testpoint/",
+    "dlURL": "garbage.php",
+    "ulURL": "empty.php",
+    "pingURL": "empty.php",
+    "getIpURL": "getIP.php",
+    "id": 2
+  }
+]

tests/e2e/fixtures/servers-frontend.json

@@ -0,0 +1,11 @@
+[
+  {
+    "name": "Backend testpoint",
+    "server": "http://backend-testpoint/",
+    "dlURL": "garbage.php",
+    "ulURL": "empty.php",
+    "pingURL": "empty.php",
+    "getIpURL": "getIP.php",
+    "id": 1
+  }
+]

tests/e2e/global-setup.js

@@ -0,0 +1,39 @@
+const { execSync } = require('node:child_process');
+
+const COMPOSE_FILE = 'tests/docker-compose-playwright.yml';
+
+async function waitForReady(name, url, timeoutMs) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      const response = await fetch(url, { redirect: 'manual' });
+      if (response.ok) {
+        return;
+      }
+    } catch {
+      // Retry until timeout.
+    }
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+  throw new Error(`Timed out waiting for ${name} at ${url}`);
+}
+
+module.exports = async () => {
+  try {
+    execSync(`docker compose -f ${COMPOSE_FILE} up -d --build`, {
+      stdio: 'inherit',
+    });
+  } catch (error) {
+    throw new Error(
+      `Failed to start Docker test stack from ${COMPOSE_FILE}. ` +
+        `Ensure Docker is running. Original error: ${error.message}`
+    );
+  }
+
+  const timeoutMs = 180_000;
+  await waitForReady('standalone', 'http://127.0.0.1:18180/index.html', timeoutMs);
+  await waitForReady('standalone-new', 'http://127.0.0.1:18185/index.html', timeoutMs);
+  await waitForReady('backend', 'http://127.0.0.1:18181/empty.php', timeoutMs);
+  await waitForReady('frontend', 'http://127.0.0.1:18182/index-modern.html', timeoutMs);
+  await waitForReady('dual', 'http://127.0.0.1:18183/index-modern.html', timeoutMs);
+};

tests/e2e/global-teardown.js

@@ -0,0 +1,9 @@
+const { execSync } = require('node:child_process');
+
+const COMPOSE_FILE = 'tests/docker-compose-playwright.yml';
+
+module.exports = async () => {
+  execSync(`docker compose -f ${COMPOSE_FILE} down --remove-orphans`, {
+    stdio: 'inherit',
+  });
+};

tests/e2e/helpers/env.js

@@ -0,0 +1,11 @@
+const baseUrls = {
+  standalone: 'http://127.0.0.1:18180',
+  backend: 'http://127.0.0.1:18181',
+  frontend: 'http://127.0.0.1:18182',
+  dual: 'http://127.0.0.1:18183',
+  standaloneNew: 'http://127.0.0.1:18185',
+};
+
+module.exports = {
+  baseUrls,
+};

tests/e2e/helpers/ui.js

@@ -0,0 +1,12 @@
+function modernStartButton(page) {
+  return page.locator('#start-button');
+}
+
+function classicStartButton(page) {
+  return page.locator('#startStopBtn');
+}
+
+module.exports = {
+  modernStartButton,
+  classicStartButton,
+};

tests/e2e/modes.spec.js

@@ -0,0 +1,56 @@
+const { test, expect } = require('@playwright/test');
+const { baseUrls } = require('./helpers/env');
+const { modernStartButton } = require('./helpers/ui');
+
+test.describe('Runtime mode smoke coverage', () => {
+  test('standalone exposes UI and local backend endpoints', async ({ page, request }) => {
+    const root = await request.get(`${baseUrls.standalone}/`);
+    expect(root.ok()).toBeTruthy();
+
+    const index = await request.get(`${baseUrls.standalone}/index.html`);
+    expect(index.ok()).toBeTruthy();
+    await expect(await index.text()).toContain('design-switch.js');
+
+    for (const endpoint of ['/backend/empty.php', '/backend/garbage.php', '/backend/getIP.php']) {
+      const response = await request.get(`${baseUrls.standalone}${endpoint}`);
+      expect(response.ok()).toBeTruthy();
+    }
+
+    await page.goto(`${baseUrls.standalone}/index-modern.html`);
+    await expect(modernStartButton(page)).toBeVisible();
+  });
+
+  test('backend exposes only local backend contract endpoints', async ({ request }) => {
+    for (const endpoint of ['/empty.php', '/garbage.php', '/getIP.php']) {
+      const response = await request.get(`${baseUrls.backend}${endpoint}`);
+      expect(response.ok()).toBeTruthy();
+    }
+  });
+
+  test('frontend serves UI and server list without local backend contract', async ({ page, request }) => {
+    const serverList = await request.get(`${baseUrls.frontend}/server-list.json`);
+    expect(serverList.ok()).toBeTruthy();
+    await expect(await serverList.text()).toContain('Backend testpoint');
+
+    const localBackendEndpoint = await request.get(`${baseUrls.frontend}/backend/empty.php`);
+    expect(localBackendEndpoint.status()).toBe(404);
+
+    await page.goto(`${baseUrls.frontend}/index-modern.html`);
+    await expect(modernStartButton(page)).toBeVisible();
+    await expect(page.locator('#selected-server')).not.toHaveText(/searching nearest server/i);
+  });
+
+  test('dual combines frontend and local backend availability', async ({ page, request }) => {
+    const serverList = await request.get(`${baseUrls.dual}/server-list.json`);
+    expect(serverList.ok()).toBeTruthy();
+    await expect(await serverList.text()).toContain('Local dual backend');
+
+    for (const endpoint of ['/backend/empty.php', '/backend/garbage.php', '/backend/getIP.php']) {
+      const response = await request.get(`${baseUrls.dual}${endpoint}`);
+      expect(response.ok()).toBeTruthy();
+    }
+
+    await page.goto(`${baseUrls.dual}/index-modern.html`);
+    await expect(modernStartButton(page)).toBeVisible();
+  });
+});