Merge pull request #3133 from cdr/jsjoeio/migrate-to-playwright-test

refactor(testing): migrate to playwright-test from jest-playwright

.github/workflows/ci.yaml

@@ -350,11 +350,11 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v2
         with:
-          name: test-videos
-          path: ./test/e2e/videos
+          name: failed-test-videos
+          path: ./test/test-results
 
       - name: Remove release packages and test artifacts
-        run: rm -rf ./release-packages ./test/e2e/videos
+        run: rm -rf ./release-packages ./test/test-results
 
   docker-amd64:
     runs-on: ubuntu-latest

.gitignore

@@ -16,5 +16,5 @@ node-*
 .home
 coverage
 **/.DS_Store
-test/e2e/videos
-test/e2e/screenshots
+# Failed e2e test videos are saved here
+test/test-results

ci/dev/test-e2e.sh

@@ -3,19 +3,10 @@ set -euo pipefail
 
 main() {
   cd "$(dirname "$0")/../.."
-  # We must keep jest in a sub-directory. See ../../test/package.json for more
-  # information. We must also run it from the root otherwise coverage will not
-  # include our source files.
-  if [[ -z ${PASSWORD-} ]] || [[ -z ${CODE_SERVER_ADDRESS-} ]]; then
-    echo "The end-to-end testing suites rely on your local environment"
-    echo -e "\n"
-    echo "Please set the following environment variables locally:"
-    echo "  \$PASSWORD"
-    echo "  \$CODE_SERVER_ADDRESS"
-    echo -e "\n"
-    exit 1
-  fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
+  cd test
+  # We set these environment variables because they're used in the e2e tests
+  # they don't have to be these values, but these are the defaults
+  PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 yarn folio --config=config.ts --reporter=list "$@"
 }
 
 main "$@"

test/config.ts

@@ -0,0 +1,73 @@
+import {
+  ChromiumEnv,
+  FirefoxEnv,
+  WebKitEnv,
+  test,
+  setConfig,
+  PlaywrightOptions,
+  Config,
+  globalSetup,
+} from "@playwright/test"
+import * as crypto from "crypto"
+import path from "path"
+import { PASSWORD } from "./utils/constants"
+import * as wtfnode from "./utils/wtfnode"
+
+// Playwright doesn't like that ../src/node/util has an enum in it
+// so I had to copy hash in separately
+const hash = (str: string): string => {
+  return crypto.createHash("sha256").update(str).digest("hex")
+}
+
+const cookieToStore = {
+  sameSite: "Lax" as const,
+  name: "key",
+  value: hash(PASSWORD),
+  domain: "localhost",
+  path: "/",
+  expires: -1,
+  httpOnly: false,
+  secure: false,
+}
+
+globalSetup(async () => {
+  console.log("\n🚨 Running globalSetup for playwright end-to-end tests")
+  console.log("👋 Please hang tight...")
+
+  if (process.env.WTF_NODE) {
+    wtfnode.setup()
+  }
+
+  const storage = {
+    cookies: [cookieToStore],
+  }
+
+  // Save storage state and store as an env variable
+  // More info: https://playwright.dev/docs/auth?_highlight=authe#reuse-authentication-state
+  process.env.STORAGE = JSON.stringify(storage)
+  console.log("✅ globalSetup is now complete.")
+})
+
+const config: Config = {
+  testDir: path.join(__dirname, "e2e"), // Search for tests in this directory.
+  timeout: 30000, // Each test is given 30 seconds.
+  retries: 3, // Retry failing tests 2 times
+}
+
+if (process.env.CI) {
+  // In CI, retry failing tests 2 times
+  // in the event of flakiness
+  config.retries = 2
+}
+
+setConfig(config)
+
+const options: PlaywrightOptions = {
+  headless: true, // Run tests in headless browsers.
+  video: "retain-on-failure",
+}
+
+// Run tests in three browsers.
+test.runWith(new ChromiumEnv(options), { tag: "chromium" })
+test.runWith(new FirefoxEnv(options), { tag: "firefox" })
+test.runWith(new WebKitEnv(options), { tag: "webkit" })

test/e2e/browser.test.ts

@@ -1,35 +1,15 @@
-/// <reference types="jest-playwright-preset" />
+import { test, expect } from "@playwright/test"
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
 
-// This test is for nothing more than to make sure
-// tests are running in multiple browsers
-describe("Browser gutcheck", () => {
-  beforeEach(async () => {
-    await jestPlaywright.resetBrowser({
-      logger: {
-        isEnabled: (name) => name === "browser",
-        log: (name, severity, message, args) => console.log(`${name} ${message}`),
-      },
-    })
-  })
-
-  test("should display correct browser based on userAgent", async () => {
+// This is a "gut-check" test to make sure playwright is working as expected
+test("browser should display correct userAgent", async ({ page, browserName }) => {
   const displayNames = {
     chromium: "Chrome",
     firefox: "Firefox",
     webkit: "Safari",
   }
+  await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
   const userAgent = await page.evaluate("navigator.userAgent")
 
-    if (browserName === "chromium") {
   expect(userAgent).toContain(displayNames[browserName])
-    }
-
-    if (browserName === "firefox") {
-      expect(userAgent).toContain(displayNames[browserName])
-    }
-
-    if (browserName === "webkit") {
-      expect(userAgent).toContain(displayNames[browserName])
-    }
-  })
 })

test/e2e/globalSetup.test.ts

@@ -1,20 +1,24 @@
-/// <reference types="jest-playwright-preset" />
+import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
 
 // This test is to make sure the globalSetup works as expected
 // meaning globalSetup ran and stored the storageState in STORAGE
-describe("globalSetup", () => {
-  beforeEach(async () => {
+test.describe("globalSetup", () => {
   // Create a new context with the saved storage state
   // so we don't have to logged in
-    const storageState = JSON.parse(STORAGE) || {}
-    await jestPlaywright.resetContext({
-      storageState,
-    })
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
+  const options: any = {}
 
-  it("should keep us logged in using the storageState", async () => {
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
+    const storageState = JSON.parse(STORAGE) || {}
+    options.contextOptions = {
+      storageState,
+    }
+  }
+  test("should keep us logged in using the storageState", options, async ({ page }) => {
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Make sure the editor actually loaded
     expect(await page.isVisible("div.monaco-workbench"))
   })

test/e2e/login.test.ts

@@ -1,13 +1,17 @@
-/// <reference types="jest-playwright-preset" />
+import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
-describe("login", () => {
-  beforeEach(async () => {
-    await jestPlaywright.resetBrowser()
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
+test.describe("login", () => {
+  // Reset the browser so no cookies are persisted
+  // by emptying the storageState
+  const options = {
+    contextOptions: {
+      storageState: {},
+    },
+  }
 
-  it("should be able to login", async () => {
+  test("should be able to login", options, async ({ page }) => {
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login

test/e2e/loginPage.test.ts

@@ -1,19 +1,17 @@
-/// <reference types="jest-playwright-preset" />
-
+import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS } from "../utils/constants"
 
-describe("login page", () => {
-  beforeEach(async () => {
-    await jestPlaywright.resetContext({
-      logger: {
-        isEnabled: (name, severity) => name === "browser",
-        log: (name, severity, message, args) => console.log(`${name} ${message}`),
+test.describe("login page", () => {
+  // Reset the browser so no cookies are persisted
+  // by emptying the storageState
+  const options = {
+    contextOptions: {
+      storageState: {},
     },
-    })
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
+  }
 
-  it("should see the login page", async () => {
+  test("should see the login page", options, async ({ page }) => {
+    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
     // It should send us to the login page
     expect(await page.title()).toBe("code-server login")
   })

test/e2e/logout.test.ts

@@ -1,13 +1,16 @@
-/// <reference types="jest-playwright-preset" />
+import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
-describe("logout", () => {
-  beforeEach(async () => {
-    await jestPlaywright.resetBrowser()
+test.describe("logout", () => {
+  // Reset the browser so no cookies are persisted
+  // by emptying the storageState
+  const options = {
+    contextOptions: {
+      storageState: {},
+    },
+  }
+  test("should be able login and logout", options, async ({ page }) => {
     await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
-
-  it("should be able login and logout", async () => {
     // Type in password
     await page.fill(".password", PASSWORD)
     // Click the submit button and login

test/e2e/openHelpAbout.test.ts

@@ -1,18 +1,25 @@
-/// <reference types="jest-playwright-preset" />
+import { test, expect } from "@playwright/test"
 import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
 
-describe("Open Help > About", () => {
-  beforeEach(async () => {
+test.describe("Open Help > About", () => {
   // Create a new context with the saved storage state
   // so we don't have to logged in
+  const options: any = {}
+  // TODO@jsjoeio
+  // Fix this once https://github.com/microsoft/playwright-test/issues/240
+  // is fixed
+  if (STORAGE) {
     const storageState = JSON.parse(STORAGE) || {}
-    await jestPlaywright.resetContext({
+    options.contextOptions = {
       storageState,
-    })
-    await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
-  })
+    }
+  }
 
-  it("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async () => {
+  test(
+    "should see a 'Help' then 'About' button in the Application Menu that opens a dialog",
+    options,
+    async ({ page }) => {
+      await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
       // Make sure the editor actually loaded
       expect(await page.isVisible("div.monaco-workbench"))
 
@@ -34,5 +41,6 @@ describe("Open Help > About", () => {
 
       const codeServerText = "text=code-server"
       expect(await page.isVisible(codeServerText))
-  })
+    },
+  )
 })

test/jest.e2e.config.ts

@@ -1,39 +0,0 @@
-// jest.config.ts
-import type { Config } from "@jest/types"
-
-const config: Config.InitialOptions = {
-  preset: "jest-playwright-preset",
-  transform: {
-    "^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
-  },
-  globalSetup: "<rootDir>/utils/globalSetup.ts",
-  testEnvironmentOptions: {
-    "jest-playwright": {
-      // TODO(@jsjoeio) enable on webkit and firefox
-      // waiting on next playwright release
-      // - https://github.com/microsoft/playwright/issues/6009#event-4536210890
-      // - https://github.com/microsoft/playwright/issues/6020
-      browsers: ["chromium"],
-      // If there's a page error, we don't exit
-      // i.e. something logged in the console
-      exitOnPageError: false,
-      contextOptions: {
-        recordVideo: {
-          dir: "./test/e2e/videos",
-        },
-      },
-    },
-  },
-  testPathIgnorePatterns: ["/node_modules/", "/lib/", "/out/", "test/unit"],
-  testTimeout: 30000,
-  modulePathIgnorePatterns: [
-    "<rootDir>/../lib/vscode",
-    "<rootDir>/../release-packages",
-    "<rootDir>/../release",
-    "<rootDir>/../release-standalone",
-    "<rootDir>/../release-npm-package",
-    "<rootDir>/../release-gcp",
-    "<rootDir>/../release-images",
-  ],
-}
-export default config

test/package.json

@@ -2,19 +2,19 @@
   "license": "MIT",
   "#": "We must put jest in a sub-directory otherwise VS Code somehow picks up the types and generates conflicts with mocha.",
   "devDependencies": {
+    "@playwright/test": "^0.1101.0-alpha2",
     "@types/jest": "^26.0.20",
     "@types/jsdom": "^16.2.6",
     "@types/node-fetch": "^2.5.8",
     "@types/supertest": "^2.0.10",
     "jest": "^26.6.3",
-    "jest-playwright-preset": "^1.5.1",
     "jsdom": "^16.4.0",
     "node-fetch": "^2.6.1",
-    "playwright": "^1.8.0",
-    "playwright-chromium": "^1.10.0",
-    "playwright-firefox": "^1.10.0",
-    "playwright-webkit": "^1.10.0",
+    "playwright": "^1.11.0-next-alpha-apr-13-2021",
     "supertest": "^6.1.1",
     "ts-jest": "^26.4.4"
+  },
+  "resolutions": {
+    "@playwright/test/playwright": "^1.11.0-next-alpha-apr-13-2021"
   }
 }