Merge pull request #3016 from cdr/jsjoeio/refactor-e2e

dev(testing): add jest-playwright and reduce flakiness of e2e tests

.eslintrc.yaml

@@ -15,7 +15,12 @@ extends:
   - plugin:import/recommended
   - plugin:import/typescript
   - plugin:prettier/recommended
-  - prettier # Removes eslint rules that conflict with prettier.
+  # Recommended by jest-playwright
+  # https://github.com/playwright-community/jest-playwright#globals
+  - plugin:jest-playwright/recommended
+  # Prettier should always be last
+  # Removes eslint rules that conflict with prettier.
+  - prettier
 
 rules:
   # Sometimes you need to add args to implement a function signature even

.github/workflows/ci.yaml

@@ -63,7 +63,7 @@ jobs:
       - uses: microsoft/playwright-github-action@v1
       - name: Install dependencies and run end-to-end tests
         run: |
-          ./release-packages/code-server*-linux-amd64/bin/code-server &
+          ./release-packages/code-server*-linux-amd64/bin/code-server --log trace &
           yarn --frozen-lockfile
           yarn test:e2e
       - name: Upload test artifacts

ci/dev/test-e2e.sh

@@ -15,7 +15,7 @@ main() {
     echo -e "\n"
     exit 1
   fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
 }
 
 main "$@"

ci/steps/test-e2e.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 main() {
   cd "$(dirname "$0")/../.."
 
-  "./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
+  "./release-packages/code-server*-linux-amd64/bin/code-server" &
   yarn --frozen-lockfile
   yarn test:e2e
 }

lib/vscode/src/vs/platform/extensionManagement/node/extensionDownloader.ts

@@ -76,7 +76,7 @@ export class ExtensionsDownloader extends Disposable {
 	private async cleanUp(): Promise<void> {
 		try {
 			if (!(await this.fileService.exists(this.extensionsDownloadDir))) {
-				this.logService.trace('Extension VSIX downlads cache dir does not exist');
+				this.logService.trace('Extension VSIX downloads cache dir does not exist');
 				return;
 			}
 			const folderStat = await this.fileService.resolve(this.extensionsDownloadDir, { resolveMetadata: true });

package.json

@@ -61,6 +61,7 @@
     "eslint-config-prettier": "^8.1.0",
     "eslint-import-resolver-alias": "^1.1.2",
     "eslint-plugin-import": "^2.18.2",
+    "eslint-plugin-jest-playwright": "^0.2.1",
     "eslint-plugin-prettier": "^3.1.0",
     "istanbul-badges-readme": "^1.2.0",
     "leaked-handles": "^5.2.0",

test/e2e/browser.test.ts

+/// <reference types="jest-playwright-preset" />
+
+// This test is for nothing more than to make sure
+// tests are running in multiple browsers
+describe("Browser gutcheck", () => {
+  beforeEach(async () => {
+    await jestPlaywright.resetBrowser({
+      logger: {
+        isEnabled: (name) => name === "browser",
+        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+      },
+    })
+  })
+
+  test("should display correct browser based on userAgent", async () => {
+    const displayNames = {
+      chromium: "Chrome",
+      firefox: "Firefox",
+      webkit: "Safari",
+    }
+    const userAgent = await page.evaluate("navigator.userAgent")
+
+    if (browserName === "chromium") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "firefox") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+
+    if (browserName === "webkit") {
+      expect(userAgent).toContain(displayNames[browserName])
+    }
+  })
+})

test/e2e/e2e.test.ts

-import { chromium, Page, Browser } from "playwright"
-import { CODE_SERVER_ADDRESS } from "../utils/constants"
-
-let browser: Browser
-let page: Page
-
-beforeAll(async () => {
-  browser = await chromium.launch()
-})
-afterAll(async () => {
-  await browser.close()
-})
-beforeEach(async () => {
-  page = await browser.newPage()
-})
-afterEach(async () => {
-  await page.close()
-})
-
-it("should see the login page", async () => {
-  await page.goto(CODE_SERVER_ADDRESS)
-  // It should send us to the login page
-  expect(await page.title()).toBe("code-server login")
-})

test/e2e/globalSetup.test.ts

+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
+
+// This test is to make sure the globalSetup works as expected
+// meaning globalSetup ran and stored the storageState in STORAGE
+describe("globalSetup", () => {
+  beforeEach(async () => {
+    // Create a new context with the saved storage state
+    // so we don't have to logged in
+    const storageState = JSON.parse(STORAGE) || {}
+    await jestPlaywright.resetContext({
+      storageState,
+    })
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  })
+
+  it("should keep us logged in using the storageState", async () => {
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
+  })
+})

test/e2e/login.test.ts

-import { chromium, Page, Browser, BrowserContext } from "playwright"
+/// <reference types="jest-playwright-preset" />
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("login", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
-    context = await browser.newContext()
-  })
-
-  afterAll(async () => {
-    await browser.close()
-  })
-
   beforeEach(async () => {
-    page = await context.newPage()
-  })
-
-  afterEach(async () => {
-    await page.close()
-    // Remove password from local storage
-    await context.clearCookies()
+    await jestPlaywright.resetBrowser()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should be able to login", async () => {
-    await page.goto(CODE_SERVER_ADDRESS)
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
-    // See the editor
-    const codeServerEditor = await page.isVisible(".monaco-workbench")
-    expect(codeServerEditor).toBeTruthy()
+    await page.waitForLoadState("networkidle")
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
   })
 })

test/e2e/loginPage.test.ts

+/// <reference types="jest-playwright-preset" />
+
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
+
+describe("login page", () => {
+  beforeEach(async () => {
+    await jestPlaywright.resetContext({
+      logger: {
+        isEnabled: (name, severity) => name === "browser",
+        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+      },
+    })
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
+  })
+
+  it("should see the login page", async () => {
+    // It should send us to the login page
+    expect(await page.title()).toBe("code-server login")
+  })
+})

test/e2e/logout.test.ts

-import { chromium, Page, Browser, BrowserContext } from "playwright"
-import { CODE_SERVER_ADDRESS, PASSWORD, E2E_VIDEO_DIR } from "../utils/constants"
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("logout", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
-    context = await browser.newContext({
-      recordVideo: { dir: E2E_VIDEO_DIR },
-    })
-  })
-
-  afterAll(async () => {
-    await browser.close()
-  })
-
   beforeEach(async () => {
-    page = await context.newPage()
-  })
-
-  afterEach(async () => {
-    await page.close()
-    // Remove password from local storage
-    await context.clearCookies()
+    await jestPlaywright.resetBrowser()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should be able login and logout", async () => {
-    await page.goto(CODE_SERVER_ADDRESS)
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login
     await page.click(".submit")
-    // See the editor
-    const codeServerEditor = await page.isVisible(".monaco-workbench")
-    expect(codeServerEditor).toBeTruthy()
+    await page.waitForLoadState("networkidle")
+    // Make sure the editor actually loaded
+    expect(await page.isVisible("div.monaco-workbench"))
 
     // Click the Application menu
     await page.click("[aria-label='Application Menu']")
@@ -45,10 +24,16 @@ describe("logout", () => {
     expect(await page.isVisible(logoutButton))
 
     await page.hover(logoutButton)
-
-    await page.click(logoutButton)
-    // it takes a couple seconds to navigate
+    // TODO(@jsjoeio)
+    // Look into how we're attaching the handlers for the logout feature
+    // We need to see how it's done upstream and add logging to the
+    // handlers themselves.
+    // They may be attached too slowly, hence why we need this timeout
     await page.waitForTimeout(2000)
+
+    // Recommended by Playwright for async navigation
+    // https://github.com/microsoft/playwright/issues/1987#issuecomment-620182151
+    await Promise.all([page.waitForNavigation(), page.click(logoutButton)])
     const currentUrl = page.url()
     expect(currentUrl).toBe(`${CODE_SERVER_ADDRESS}/login`)
   })

test/e2e/openHelpAbout.test.ts

-import { chromium, Page, Browser, BrowserContext, Cookie } from "playwright"
-import { hash } from "../../src/node/util"
-import { CODE_SERVER_ADDRESS, PASSWORD, STORAGE, E2E_VIDEO_DIR } from "../utils/constants"
-import { createCookieIfDoesntExist } from "../utils/helpers"
+/// <reference types="jest-playwright-preset" />
+import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
 
 describe("Open Help > About", () => {
-  let browser: Browser
-  let page: Page
-  let context: BrowserContext
-
-  beforeAll(async () => {
-    browser = await chromium.launch()
+  beforeEach(async () => {
     // Create a new context with the saved storage state
+    // so we don't have to logged in
     const storageState = JSON.parse(STORAGE) || {}
-
-    const cookieToStore = {
-      sameSite: "Lax" as const,
-      name: "key",
-      value: hash(PASSWORD),
-      domain: "localhost",
-      path: "/",
-      expires: -1,
-      httpOnly: false,
-      secure: false,
-    }
-
-    // For some odd reason, the login method used in globalSetup.ts doesn't always work
-    // I don't know if it's on playwright clearing our cookies by accident
-    // or if it's our cookies disappearing.
-    // This means we need an additional check to make sure we're logged in.
-    // We do this by manually adding the cookie to the browser environment
-    // if it's not there at the time the test starts
-    const cookies: Cookie[] = storageState.cookies || []
-    // If the cookie exists in cookies then
-    // this will return the cookies with no changes
-    // otherwise if it doesn't exist, it will create it
-    // hence the name maybeUpdatedCookies
-    //
-    // TODO(@jsjoeio)
-    // The playwright storage thing sometimes works and sometimes doesn't. We should investigate this further
-    // at some point.
-    // See discussion: https://github.com/cdr/code-server/pull/2648#discussion_r575434946
-
-    const maybeUpdatedCookies = createCookieIfDoesntExist(cookies, cookieToStore)
-
-    context = await browser.newContext({
-      storageState: { cookies: maybeUpdatedCookies },
-      recordVideo: { dir: E2E_VIDEO_DIR },
+    await jestPlaywright.resetContext({
+      storageState,
     })
-  })
-
-  afterAll(async () => {
-    // Remove password from local storage
-    await context.clearCookies()
-
-    await context.close()
-    await browser.close()
-  })
-
-  beforeEach(async () => {
-    page = await context.newPage()
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   })
 
   it("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async () => {
-    // waitUntil: "domcontentloaded"
-    // In case the page takes a long time to load
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "domcontentloaded" })
-
     // Make sure the editor actually loaded
     expect(await page.isVisible("div.monaco-workbench"))
 

test/jest.e2e.config.ts

@@ -2,11 +2,28 @@
 import type { Config } from "@jest/types"
 
 const config: Config.InitialOptions = {
+  preset: "jest-playwright-preset",
   transform: {
     "^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
   },
   globalSetup: "<rootDir>/utils/globalSetup.ts",
-  testEnvironment: "node",
+  testEnvironmentOptions: {
+    "jest-playwright": {
+      // TODO(@jsjoeio) enable on webkit and firefox
+      // waiting on next playwright release
+      // - https://github.com/microsoft/playwright/issues/6009#event-4536210890
+      // - https://github.com/microsoft/playwright/issues/6020
+      browsers: ["chromium"],
+      // If there's a page error, we don't exit
+      // i.e. something logged in the console
+      exitOnPageError: false,
+      contextOptions: {
+        recordVideo: {
+          dir: "./test/e2e/videos",
+        },
+      },
+    },
+  },
   testPathIgnorePatterns: ["/node_modules/", "/lib/", "/out/", "test/unit"],
   testTimeout: 30000,
   modulePathIgnorePatterns: [

test/package.json

@@ -7,9 +7,13 @@
     "@types/node-fetch": "^2.5.8",
     "@types/supertest": "^2.0.10",
     "jest": "^26.6.3",
+    "jest-playwright-preset": "^1.5.1",
     "jsdom": "^16.4.0",
     "node-fetch": "^2.6.1",
     "playwright": "^1.8.0",
+    "playwright-chromium": "^1.10.0",
+    "playwright-firefox": "^1.10.0",
+    "playwright-webkit": "^1.10.0",
     "supertest": "^6.1.1",
     "ts-jest": "^26.4.4"
   }

test/unit/util.test.ts

@@ -11,10 +11,7 @@ import {
   trimSlashes,
   normalize,
 } from "../../src/common/util"
-import { Cookie as CookieEnum } from "../../src/node/routes/login"
-import { hash } from "../../src/node/util"
-import { PASSWORD } from "../utils/constants"
-import { checkForCookie, createCookieIfDoesntExist, loggerModule, Cookie } from "../utils/helpers"
+import { loggerModule } from "../utils/helpers"
 
 const dom = new JSDOM()
 global.document = dom.window.document
@@ -252,58 +249,4 @@ describe("util", () => {
       expect(loggerModule.logger.error).toHaveBeenCalledWith("api: oh no")
     })
   })
-
-  describe("checkForCookie", () => {
-    it("should check if the cookie exists and has a value", () => {
-      const fakeCookies: Cookie[] = [
-        {
-          name: CookieEnum.Key,
-          value: hash(PASSWORD),
-          domain: "localhost",
-          secure: false,
-          sameSite: "Lax",
-          httpOnly: false,
-          expires: 18000,
-          path: "/",
-        },
-      ]
-      expect(checkForCookie(fakeCookies, CookieEnum.Key)).toBe(true)
-    })
-    it("should return false if there are no cookies", () => {
-      const fakeCookies: Cookie[] = []
-      expect(checkForCookie(fakeCookies, "key")).toBe(false)
-    })
-  })
-
-  describe("createCookieIfDoesntExist", () => {
-    it("should create a cookie if it doesn't exist", () => {
-      const cookies: Cookie[] = []
-      const cookieToStore = {
-        name: CookieEnum.Key,
-        value: hash(PASSWORD),
-        domain: "localhost",
-        secure: false,
-        sameSite: "Lax" as const,
-        httpOnly: false,
-        expires: 18000,
-        path: "/",
-      }
-      expect(createCookieIfDoesntExist(cookies, cookieToStore)).toStrictEqual([cookieToStore])
-    })
-    it("should return the same cookies if the cookie already exists", () => {
-      const PASSWORD = "123supersecure"
-      const cookieToStore = {
-        name: CookieEnum.Key,
-        value: hash(PASSWORD),
-        domain: "localhost",
-        secure: false,
-        sameSite: "Lax" as const,
-        httpOnly: false,
-        expires: 18000,
-        path: "/",
-      }
-      const cookies: Cookie[] = [cookieToStore]
-      expect(createCookieIfDoesntExist(cookies, cookieToStore)).toStrictEqual(cookies)
-    })
-  })
 })

test/utils/constants.ts

 export const CODE_SERVER_ADDRESS = process.env.CODE_SERVER_ADDRESS || "http://localhost:8080"
 export const PASSWORD = process.env.PASSWORD || "e45432jklfdsab"
 export const STORAGE = process.env.STORAGE || ""
-export const E2E_VIDEO_DIR = "./test/e2e/videos"

test/utils/globalSetup.ts

@@ -2,33 +2,39 @@
 // so that it authenticates us into code-server
 // ensuring that we're logged in before we run any tests
 import { chromium } from "playwright"
-import { CODE_SERVER_ADDRESS, PASSWORD } from "./constants"
+import { hash } from "../../src/node/util"
+import { PASSWORD } from "./constants"
 import * as wtfnode from "./wtfnode"
 
+const cookieToStore = {
+  sameSite: "Lax" as const,
+  name: "key",
+  value: hash(PASSWORD),
+  domain: "localhost",
+  path: "/",
+  expires: -1,
+  httpOnly: false,
+  secure: false,
+}
+
 module.exports = async () => {
   console.log("\n🚨 Running Global Setup for Jest End-to-End Tests")
   console.log("     Please hang tight...")
   const browser = await chromium.launch()
-  const context = await browser.newContext()
-  const page = await context.newPage()
+  const page = await browser.newPage()
+  const storage = await page.context().storageState()
 
   if (process.env.WTF_NODE) {
     wtfnode.setup()
   }
 
-  await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "domcontentloaded" })
-  // Type in password
-  await page.fill(".password", PASSWORD)
-  // Click the submit button and login
-  await page.click(".submit")
+  storage.cookies = [cookieToStore]
 
   // Save storage state and store as an env variable
   // More info: https://playwright.dev/docs/auth?_highlight=authe#reuse-authentication-state
-  const storage = await context.storageState()
   process.env.STORAGE = JSON.stringify(storage)
-
   await page.close()
   await browser.close()
-  await context.close()
+
   console.log("✅ Global Setup for Jest End-to-End Tests is now complete.")
 }

test/utils/helpers.ts

-// Borrowed from playwright
-export interface Cookie {
-  name: string
-  value: string
-  domain: string
-  path: string
-  /**
-   * Unix time in seconds.
-   */
-  expires: number
-  httpOnly: boolean
-  secure: boolean
-  sameSite: "Strict" | "Lax" | "None"
-}
-
-/**
- * Checks if a cookie exists in array of cookies
- */
-export function checkForCookie(cookies: Array<Cookie>, key: string): boolean {
-  // Check for a cookie where the name is equal to key
-  return Boolean(cookies.find((cookie) => cookie.name === key))
-}
-
-/**
- * Creates a login cookie if one doesn't already exist
- */
-export function createCookieIfDoesntExist(cookies: Array<Cookie>, cookieToStore: Cookie): Array<Cookie> {
-  const cookieName = cookieToStore.name
-  const doesCookieExist = checkForCookie(cookies, cookieName)
-  if (!doesCookieExist) {
-    const updatedCookies = [...cookies, cookieToStore]
-    return updatedCookies
-  }
-  return cookies
-}
-
 export const loggerModule = {
   field: jest.fn(),
   level: 2,

yarn.lock

@@ -3164,6 +3164,11 @@ eslint-plugin-import@^2.18.2:
     resolve "^1.17.0"
     tsconfig-paths "^3.9.0"
 
+eslint-plugin-jest-playwright@^0.2.1:
+  version "0.2.1"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-jest-playwright/-/eslint-plugin-jest-playwright-0.2.1.tgz#8778fee9d5915132a03d94370d3eea0a7ddd08f3"
+  integrity sha512-BicKUJUpVPsLbHN8c5hYaZn6pv8PCMjBGHXUfvlY1p75fh4APVfX2gTK14HuiR8/Bv3fKBQu5MTaqCro4E3OHg==
+
 eslint-plugin-prettier@^3.1.0:
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-3.3.1.tgz#7079cfa2497078905011e6f82e8dd8453d1371b7"