Merge pull request #2852 from cdr/jsjoeio-2646-separate-testing

dev(testing): separate unit and e2e tests

.github/workflows/ci.yaml

@@ -27,7 +27,15 @@ jobs:
         with:
           args: ./ci/steps/lint.sh
 
-  test:
+  test-unit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Run unit tests
+        uses: ./ci/images/debian10
+        with:
+          args: ./ci/steps/test-unit.sh
+  test-e2e:
     needs: linux-amd64
     runs-on: ubuntu-latest
     env:
@@ -44,19 +52,19 @@ jobs:
         run: |
           cd release-packages && tar -xzf code-server*-linux-amd64.tar.gz
       - uses: microsoft/playwright-github-action@v1
-      - name: Install dependencies and run tests
+      - name: Install dependencies and run end-to-end tests
         run: |
           ./release-packages/code-server*-linux-amd64/bin/code-server --home $CODE_SERVER_ADDRESS/healthz &
           yarn --frozen-lockfile
-          yarn test
+          yarn test:e2e
       - name: Upload test artifacts
         if: always()
         uses: actions/upload-artifact@v2
         with:
           name: test-videos
-          path: ./test/videos
+          path: ./test/e2e/videos
       - name: Remove release packages and test artifacts
-        run: rm -rf ./release-packages ./test/videos
+        run: rm -rf ./release-packages ./test/e2e/videos
 
   release:
     runs-on: ubuntu-latest

.gitignore

@@ -16,5 +16,5 @@ node-*
 .home
 coverage
 **/.DS_Store
-test/videos
+test/e2e/videos
-test/screenshots
+test/e2e/screenshots

ci/README.md

@@ -52,7 +52,7 @@ Make sure you have `$GITHUB_TOKEN` set and [hub](https://github.com/github/hub)
 
 Currently, we run a command to manually generate the code coverage shield. Follow these steps:
 
-1. Run `yarn test` and make sure all the tests are passing
+1. Run `yarn test:unit` and make sure all the tests are passing
 2. Run `yarn badges`
 3. Go into the README and change the color from `red` to `green` in this line:
 
@@ -72,8 +72,10 @@ This directory contains scripts used for the development of code-server.
   - Runs formatters.
 - [./ci/dev/lint.sh](./dev/lint.sh) (`yarn lint`)
   - Runs linters.
-- [./ci/dev/test.sh](./dev/test.sh) (`yarn test`)
+- [./ci/dev/test-unit.sh](./dev/test-unit.sh) (`yarn test:unit`)
-  - Runs tests.
+  - Runs unit tests.
+- [./ci/dev/test-e2e.sh](./dev/test-e2e.sh) (`yarn test:e2e`)
+  - Runs end-to-end tests.
 - [./ci/dev/ci.sh](./dev/ci.sh) (`yarn ci`)
   - Runs `yarn fmt`, `yarn lint` and `yarn test`.
 - [./ci/dev/watch.ts](./dev/watch.ts) (`yarn watch`)
@@ -142,11 +144,13 @@ This directory contains the scripts used in CI.
 Helps avoid clobbering the CI configuration.
 
 - [./steps/fmt.sh](./steps/fmt.sh)
-  - Runs `yarn fmt` after ensuring VS Code is patched.
+  - Runs `yarn fmt`.
 - [./steps/lint.sh](./steps/lint.sh)
-  - Runs `yarn lint` after ensuring VS Code is patched.
+  - Runs `yarn lint`.
-- [./steps/test.sh](./steps/test.sh)
+- [./steps/test-unit.sh](./steps/test-unit.sh)
-  - Runs `yarn test` after ensuring VS Code is patched.
+  - Runs `yarn test:unit`.
+- [./steps/test-e2e.sh](./steps/test-e2e.sh)
+  - Runs `yarn test:e2e`.
 - [./steps/release.sh](./steps/release.sh)
   - Runs the release process.
   - Generates the npm package at `./release`.

ci/dev/ci.sh

@@ -6,7 +6,7 @@ main() {
 
   yarn fmt
   yarn lint
-  yarn test
+  yarn test:unit
 }
 
 main "$@"

ci/dev/test-e2e.sh

@@ -3,12 +3,9 @@ set -euo pipefail
 
 main() {
   cd "$(dirname "$0")/../.."
-  cd test/test-plugin
-  make -s out/index.js
   # We must keep jest in a sub-directory. See ../../test/package.json for more
   # information. We must also run it from the root otherwise coverage will not
   # include our source files.
-  cd "$OLDPWD"
   if [[ -z ${PASSWORD-} ]] || [[ -z ${CODE_SERVER_ADDRESS-} ]]; then
     echo "The end-to-end testing suites rely on your local environment"
     echo -e "\n"
@@ -21,7 +18,7 @@ main() {
     echo -e "\n"
     exit 1
   fi
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
 }
 
 main "$@"

ci/dev/test-unit.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+  cd "$(dirname "$0")/../.."
+  cd test/unit/test-plugin
+  make -s out/index.js
+  # We must keep jest in a sub-directory. See ../../test/package.json for more
+  # information. We must also run it from the root otherwise coverage will not
+  # include our source files.
+  cd "$OLDPWD"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+}
+
+main "$@"

ci/steps/test-e2e.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+main() {
+  cd "$(dirname "$0")/../.."
+
+  "./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
+  yarn --frozen-lockfile
+  yarn test:e2e
+}
+
+main "$@"

ci/steps/test-unit.sh

@@ -6,7 +6,7 @@ main() {
 
   yarn --frozen-lockfile
 
-  yarn test
+  yarn test:unit
 }
 
 main "$@"

package.json

@@ -16,7 +16,9 @@
     "release:standalone": "./ci/build/build-standalone-release.sh",
     "release:github-draft": "./ci/build/release-github-draft.sh",
     "release:github-assets": "./ci/build/release-github-assets.sh",
+    "test:e2e": "./ci/dev/test-e2e.sh",
     "test:standalone-release": "./ci/build/test-standalone-release.sh",
+    "test:unit": "./ci/dev/test-unit.sh",
     "package": "./ci/build/build-packages.sh",
     "postinstall": "./ci/dev/postinstall.sh",
     "update:vscode": "./ci/dev/update-vscode.sh",
@@ -124,7 +126,8 @@
     "testPathIgnorePatterns": [
       "node_modules",
       "lib",
-      "out"
+      "out",
+      "test/e2e"
     ],
     "collectCoverage": true,
     "collectCoverageFrom": [
@@ -144,8 +147,6 @@
         "lines": 40
       }
     },
-    "testTimeout": 30000,
-    "globalSetup": "<rootDir>/test/globalSetup.ts",
     "modulePathIgnorePatterns": [
       "<rootDir>/lib/vscode",
       "<rootDir>/release-packages",
@@ -156,7 +157,7 @@
       "<rootDir>/release-images"
     ],
     "moduleNameMapper": {
-      "^.+\\.(css|less)$": "<rootDir>/test/cssStub.ts"
+      "^.+\\.(css|less)$": "<rootDir>/test/utils/cssStub.ts"
     }
   }
 }

test/e2e/e2e.test.ts

@@ -1,5 +1,5 @@
 import { chromium, Page, Browser } from "playwright"
-import { CODE_SERVER_ADDRESS } from "./constants"
+import { CODE_SERVER_ADDRESS } from "../utils/constants"
 
 let browser: Browser
 let page: Page

test/e2e/goHome.test.ts

@@ -1,7 +1,7 @@
 import { chromium, Page, Browser, BrowserContext, Cookie } from "playwright"
-import { hash } from "../src/node/util"
+import { hash } from "../../src/node/util"
-import { CODE_SERVER_ADDRESS, PASSWORD, STORAGE } from "./constants"
+import { CODE_SERVER_ADDRESS, PASSWORD, STORAGE, E2E_VIDEO_DIR } from "../utils/constants"
-import { createCookieIfDoesntExist } from "./helpers"
+import { createCookieIfDoesntExist } from "../utils/helpers"
 
 describe("go home", () => {
   let browser: Browser
@@ -45,7 +45,7 @@ describe("go home", () => {
 
     context = await browser.newContext({
       storageState: { cookies: maybeUpdatedCookies },
-      recordVideo: { dir: "./test/videos/" },
+      recordVideo: { dir: E2E_VIDEO_DIR },
     })
   })
 

test/e2e/login.test.ts

@@ -1,5 +1,5 @@
 import { chromium, Page, Browser, BrowserContext } from "playwright"
-import { CODE_SERVER_ADDRESS, PASSWORD } from "./constants"
+import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
 
 describe("login", () => {
   let browser: Browser

test/jest.e2e.config.ts

@@ -0,0 +1,22 @@
+// jest.config.ts
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  transform: {
+    "^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
+  },
+  globalSetup: "<rootDir>/utils/globalSetup.ts",
+  testEnvironment: "node",
+  testPathIgnorePatterns: ["node_modules", "lib", "out", "test/unit"],
+  testTimeout: 30000,
+  modulePathIgnorePatterns: [
+    "<rootDir>/../lib/vscode",
+    "<rootDir>/../release-packages",
+    "<rootDir>/../release",
+    "<rootDir>/../release-standalone",
+    "<rootDir>/../release-npm-package",
+    "<rootDir>/../release-gcp",
+    "<rootDir>/../release-images",
+  ],
+}
+export default config

test/unit/cli.test.ts

@@ -3,8 +3,8 @@ import * as fs from "fs-extra"
 import * as net from "net"
 import * as os from "os"
 import * as path from "path"
-import { Args, parse, setDefaults, shouldOpenInExistingInstance } from "../src/node/cli"
+import { Args, parse, setDefaults, shouldOpenInExistingInstance } from "../../src/node/cli"
-import { paths, tmpdir } from "../src/node/util"
+import { paths, tmpdir } from "../../src/node/util"
 
 type Mutable<T> = {
   -readonly [P in keyof T]: T[P]

test/unit/constants.test.ts

@@ -1,8 +1,8 @@
-import { commit, getPackageJson, version } from "../src/node/constants"
+import { commit, getPackageJson, version } from "../../src/node/constants"
-import { loggerModule } from "./helpers"
+import { loggerModule } from "../utils/helpers"
 
 // jest.mock is hoisted above the imports so we must use `require` here.
-jest.mock("@coder/logger", () => require("./helpers").loggerModule)
+jest.mock("@coder/logger", () => require("../utils/helpers").loggerModule)
 
 describe("constants", () => {
   describe("getPackageJson", () => {

test/unit/emitter.test.ts

@@ -1,8 +1,8 @@
 // Note: we need to import logger from the root
 // because this is the logger used in logError in ../src/common/util
-import { logger } from "../node_modules/@coder/logger"
+import { logger } from "../../node_modules/@coder/logger"
 
-import { Emitter } from "../src/common/emitter"
+import { Emitter } from "../../src/common/emitter"
 
 describe("emitter", () => {
   let spy: jest.SpyInstance

test/unit/health.test.ts

@@ -1,5 +1,5 @@
-import * as httpserver from "./httpserver"
+import * as httpserver from "../utils/httpserver"
-import * as integration from "./integration"
+import * as integration from "../utils/integration"
 
 describe("health", () => {
   let codeServer: httpserver.HttpServer | undefined

test/unit/http.test.ts

@@ -1,4 +1,4 @@
-import { HttpCode, HttpError } from "../src/common/http"
+import { HttpCode, HttpError } from "../../src/common/http"
 
 describe("http", () => {
   describe("HttpCode", () => {

test/unit/plugin.test.ts

@@ -2,11 +2,11 @@ import { logger } from "@coder/logger"
 import * as express from "express"
 import * as fs from "fs"
 import * as path from "path"
-import { HttpCode } from "../src/common/http"
+import { HttpCode } from "../../src/common/http"
-import { AuthType } from "../src/node/cli"
+import { AuthType } from "../../src/node/cli"
-import { codeServer, PluginAPI } from "../src/node/plugin"
+import { codeServer, PluginAPI } from "../../src/node/plugin"
-import * as apps from "../src/node/routes/apps"
+import * as apps from "../../src/node/routes/apps"
-import * as httpserver from "./httpserver"
+import * as httpserver from "../utils/httpserver"
 const fsp = fs.promises
 
 // Jest overrides `require` so our usual override doesn't work.

test/unit/proxy.test.ts

@@ -1,7 +1,7 @@
 import bodyParser from "body-parser"
 import * as express from "express"
-import * as httpserver from "./httpserver"
+import * as httpserver from "../utils/httpserver"
-import * as integration from "./integration"
+import * as integration from "../utils/integration"
 
 describe("proxy", () => {
   const nhooyrDevServer = new httpserver.HttpServer()

test/unit/register.test.ts

@@ -1,5 +1,5 @@
 import { JSDOM } from "jsdom"
-import { loggerModule } from "./helpers"
+import { loggerModule } from "../utils/helpers"
 
 describe("register", () => {
   describe("when navigator and serviceWorker are defined", () => {
@@ -40,7 +40,7 @@ describe("register", () => {
 
     it("should register a ServiceWorker", () => {
       // Load service worker like you would in the browser
-      require("../src/browser/register")
+      require("../../src/browser/register")
       expect(mockRegisterFn).toHaveBeenCalled()
       expect(mockRegisterFn).toHaveBeenCalledTimes(1)
     })
@@ -54,7 +54,7 @@ describe("register", () => {
       })
 
       // Load service worker like you would in the browser
-      require("../src/browser/register")
+      require("../../src/browser/register")
 
       expect(mockRegisterFn).toHaveBeenCalled()
       expect(loggerModule.logger.error).toHaveBeenCalled()
@@ -78,7 +78,7 @@ describe("register", () => {
 
     it("should log an error to the console", () => {
       // Load service worker like you would in the browser
-      require("../src/browser/register")
+      require("../../src/browser/register")
       expect(spy).toHaveBeenCalled()
       expect(spy).toHaveBeenCalledTimes(1)
       expect(spy).toHaveBeenCalledWith("[Service Worker] navigator is undefined")

test/unit/serviceWorker.test.ts

@@ -58,7 +58,7 @@ describe("serviceWorker", () => {
   })
 
   it("should add 3 listeners: install, activate and fetch", () => {
-    require("../src/browser/serviceWorker.ts")
+    require("../../src/browser/serviceWorker.ts")
     const listenerEventNames = listeners.map((listener) => listener.event)
 
     expect(listeners).toHaveLength(3)
@@ -68,20 +68,20 @@ describe("serviceWorker", () => {
   })
 
   it("should call the proper callbacks for 'install'", async () => {
-    require("../src/browser/serviceWorker.ts")
+    require("../../src/browser/serviceWorker.ts")
     emit("install")
     expect(spy).toHaveBeenCalledWith("[Service Worker] installed")
     expect(spy).toHaveBeenCalledTimes(1)
   })
 
   it("should do nothing when 'fetch' is called", async () => {
-    require("../src/browser/serviceWorker.ts")
+    require("../../src/browser/serviceWorker.ts")
     emit("fetch")
     expect(spy).not.toHaveBeenCalled()
   })
 
   it("should call the proper callbacks for 'activate'", async () => {
-    require("../src/browser/serviceWorker.ts")
+    require("../../src/browser/serviceWorker.ts")
     emit("activate")
 
     // Activate serviceWorker

test/unit/socket.test.ts

@@ -3,9 +3,9 @@ import * as fs from "fs-extra"
 import * as net from "net"
 import * as path from "path"
 import * as tls from "tls"
-import { Emitter } from "../src/common/emitter"
+import { Emitter } from "../../src/common/emitter"
-import { SocketProxyProvider } from "../src/node/socket"
+import { SocketProxyProvider } from "../../src/node/socket"
-import { generateCertificate, tmpdir } from "../src/node/util"
+import { generateCertificate, tmpdir } from "../../src/node/util"
 
 describe("SocketProxyProvider", () => {
   const provider = new SocketProxyProvider()

test/unit/test-plugin/tsconfig.json

@@ -44,7 +44,7 @@
     // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
     "baseUrl": "./" /* Base directory to resolve non-absolute module names. */,
     "paths": {
-      "code-server": ["../../typings/pluginapi"]
+      "code-server": ["../../../typings/pluginapi"]
     } /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */,
     // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
     // "typeRoots": [],                       /* List of folders to include type definitions from. */

test/unit/update.test.ts

@@ -1,9 +1,9 @@
 import * as fs from "fs-extra"
 import * as http from "http"
 import * as path from "path"
-import { SettingsProvider, UpdateSettings } from "../src/node/settings"
+import { SettingsProvider, UpdateSettings } from "../../src/node/settings"
-import { LatestResponse, UpdateProvider } from "../src/node/update"
+import { LatestResponse, UpdateProvider } from "../../src/node/update"
-import { tmpdir } from "../src/node/util"
+import { tmpdir } from "../../src/node/util"
 
 describe.skip("update", () => {
   let version = "1.0.0"

test/unit/util.test.ts

@@ -10,11 +10,11 @@ import {
   split,
   trimSlashes,
   normalize,
-} from "../src/common/util"
+} from "../../src/common/util"
-import { Cookie as CookieEnum } from "../src/node/routes/login"
+import { Cookie as CookieEnum } from "../../src/node/routes/login"
-import { hash } from "../src/node/util"
+import { hash } from "../../src/node/util"
-import { PASSWORD } from "./constants"
+import { PASSWORD } from "../utils/constants"
-import { checkForCookie, createCookieIfDoesntExist, loggerModule, Cookie } from "./helpers"
+import { checkForCookie, createCookieIfDoesntExist, loggerModule, Cookie } from "../utils/helpers"
 
 const dom = new JSDOM()
 global.document = dom.window.document
@@ -22,7 +22,7 @@ global.document = dom.window.document
 type LocationLike = Pick<Location, "pathname" | "origin">
 
 // jest.mock is hoisted above the imports so we must use `require` here.
-jest.mock("@coder/logger", () => require("./helpers").loggerModule)
+jest.mock("@coder/logger", () => require("../utils/helpers").loggerModule)
 
 describe("util", () => {
   describe("normalize", () => {

test/utils/constants.ts

@@ -1,3 +1,4 @@
 export const CODE_SERVER_ADDRESS = process.env.CODE_SERVER_ADDRESS || "http://localhost:8080"
 export const PASSWORD = process.env.PASSWORD || "e45432jklfdsab"
 export const STORAGE = process.env.STORAGE || ""
+export const E2E_VIDEO_DIR = "./test/e2e/videos"

test/utils/globalSetup.ts

@@ -6,7 +6,7 @@ import { CODE_SERVER_ADDRESS, PASSWORD } from "./constants"
 import * as wtfnode from "./wtfnode"
 
 module.exports = async () => {
-  console.log("\n🚨 Running Global Setup for Jest Tests")
+  console.log("\n🚨 Running Global Setup for Jest End-to-End Tests")
   console.log("     Please hang tight...")
   const browser = await chromium.launch()
   const context = await browser.newContext()
@@ -30,5 +30,5 @@ module.exports = async () => {
   await page.close()
   await browser.close()
   await context.close()
-  console.log("✅ Global Setup for Jest Tests is now complete.")
+  console.log("✅ Global Setup for Jest End-to-End Tests is now complete.")
 }

test/utils/httpserver.ts

@@ -3,9 +3,9 @@ import * as http from "http"
 import * as net from "net"
 import * as nodeFetch from "node-fetch"
 import Websocket from "ws"
-import * as util from "../src/common/util"
+import * as util from "../../src/common/util"
-import { ensureAddress } from "../src/node/app"
+import { ensureAddress } from "../../src/node/app"
-import { handleUpgrade } from "../src/node/wsRouter"
+import { handleUpgrade } from "../../src/node/wsRouter"
 
 // Perhaps an abstraction similar to this should be used in app.ts as well.
 export class HttpServer {

test/utils/integration.ts

@@ -1,7 +1,7 @@
 import * as express from "express"
-import { createApp } from "../src/node/app"
+import { createApp } from "../../src/node/app"
-import { parse, setDefaults, parseConfigFile, DefaultedArgs } from "../src/node/cli"
+import { parse, setDefaults, parseConfigFile, DefaultedArgs } from "../../src/node/cli"
-import { register } from "../src/node/routes"
+import { register } from "../../src/node/routes"
 import * as httpserver from "./httpserver"
 
 export async function setup(