refactor: move integration tests to Jest (#5275)

* feat: add installExtension integration test

This adds a new helper function called `runCodeServerCommand` along with
a test for `--install-extension`. We can use this approach for writing
integration tests (i.e. testing a real code-server build, CLI commands,
etc).

* refactor: s/ test:standalone with test:integration

This replaces our integration approach to use Jest instead of a single
bash script. By doing this, we will be able to easily maintain and add
to our integration test suite.

* refactor: filter unit tests

Now that our integration tests also use Jest, we need to update our unit
test script to ignore `test/integration`.

* refactor: add SKIP_SUBMODULE_DEPS to postinstall

* refactor: add SKIP_SUBMODULE_DEPS to postinstall

* fixup!: skip submod deps

* refactor: move runCodeServerCommand into sep. file

When Jest runs a test, it loads all the files and imports for that test.
This means you might be "requiring" code that's unrelated to your tests.

This leads to unexpected errors depending on where the code runs.

Moved this file to avoid GLIBC and other errors relaed to argon2 when
running integration tests in CI.

* fizup: formatting

* fizup: increase timeout

* refactor: use fixture in installExtension test

Instead of relying on a network to install an extension, we use a
fixture - vsix file in the repo. This is also faster.

* feat: add integration test for listExtensions

* chore: ignore integration fixtures

* fixup: formatting

* fixup: remove custom-hacks.css

* fixup: formatting

* Update test/integration/installExtension.test.ts

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/listExtensions.test.ts

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/installExtension.test.ts

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/listExtensions.test.ts

Co-authored-by: Asher <ash@coder.com>

* fixup: contributing integration tests section

* fixup: update ci/readme

* fixup: use RELEASE_PATH in test-integration.sh

* refactor: unzip vsix for listExtensions

* refactor: use exec instead of spawn

* Update docs/CONTRIBUTING.md

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/listExtensions.test.ts

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/listExtensions.test.ts

Co-authored-by: Asher <ash@coder.com>

* Update test/integration/listExtensions.test.ts

Co-authored-by: Asher <ash@coder.com>

* refactor: use different default binary path

* fixup!: formatting

Co-authored-by: Asher <ash@coder.com>

.github/workflows/ci.yaml

@@ -288,8 +288,11 @@ jobs:
       - name: Build standalone release
         run: source scl_source enable devtoolset-9 && yarn release:standalone
 
-      - name: Sanity test standalone release
-        run: yarn test:standalone-release
+      - name: Install test dependencies
+        run: SKIP_SUBMODULE_DEPS=1 yarn install
+
+      - name: Run integration tests on standalone release
+        run: yarn test:integration
 
       - name: Build packages with nfpm
         run: yarn package
@@ -421,8 +424,11 @@ jobs:
       - name: Build standalone release
         run: yarn release:standalone
 
-      - name: Sanity test standalone release
-        run: yarn test:standalone-release
+      - name: Install test dependencies
+        run: SKIP_SUBMODULE_DEPS=1 yarn install
+
+      - name: Run integration tests on standalone release
+        run: yarn test:integration
 
       - name: Build packages with nfpm
         run: yarn package

.github/workflows/installer.yml

@@ -34,7 +34,7 @@ jobs:
         run: ./install.sh
 
       - name: Test code-server
-        run: yarn test:standalone-release code-server
+        run: CODE_SERVER_PATH="code-server" yarn test:integration
 
   alpine:
     name: Test installer on Alpine
@@ -66,4 +66,4 @@ jobs:
         run: ./install.sh
 
       - name: Test code-server
-        run: yarn test:standalone-release code-server
+        run: CODE_SERVER_PATH="code-server" yarn test:integration

ci/README.md

@@ -45,9 +45,6 @@ You can disable minification by setting `MINIFY=`.
   - Builds vscode into `./lib/vscode/out-vscode`.
 - [./ci/build/build-release.sh](./build/build-release.sh) (`yarn release`)
   - Bundles the output of the above two scripts into a single node module at `./release`.
-- [./ci/build/build-standalone-release.sh](./build/build-standalone-release.sh) (`yarn release:standalone`)
-  - Requires a node module already built into `./release` with the above script.
-  - Will build a standalone release with node and node_modules bundled into `./release-standalone`.
 - [./ci/build/clean.sh](./build/clean.sh) (`yarn clean`)
   - Removes all build artifacts.
   - Useful to do a clean build.
@@ -97,6 +94,8 @@ Helps avoid clobbering the CI configuration.
   - Runs `yarn lint`.
 - [./steps/test-unit.sh](./steps/test-unit.sh)
   - Runs `yarn test:unit`.
+- [./steps/test-integration.sh](./steps/test-integration.sh)
+  - Runs `yarn test:integration`.
 - [./steps/test-e2e.sh](./steps/test-e2e.sh)
   - Runs `yarn test:e2e`.
 - [./steps/release.sh](./steps/release.sh)

ci/build/test-standalone-release.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Make sure a code-server release works. You can pass in the path otherwise it
-# will use release-standalone in the current directory.
-#
-# This is to make sure we don't have Node version errors or any other
-# compilation-related errors.
-main() {
-  cd "$(dirname "${0}")/../.."
-
-  local EXTENSIONS_DIR
-  EXTENSIONS_DIR="$(mktemp -d)"
-
-  local path=${1:-./release-standalone/bin/code-server}
-
-  echo "Testing standalone release in $path."
-
-  # NOTE: using a basic theme extension because it doesn't update often and is more reliable for testing
-  "$path" --extensions-dir "$EXTENSIONS_DIR" --install-extension wesbos.theme-cobalt2
-  local installed_extensions
-  installed_extensions="$("$path" --extensions-dir "$EXTENSIONS_DIR" --list-extensions 2>&1)"
-  # We use grep as wesbos.theme-cobalt2 may have dependency extensions that change.
-  if ! echo "$installed_extensions" | grep -q "wesbos.theme-cobalt2"; then
-    echo "Unexpected output from listing extensions:"
-    echo "$installed_extensions"
-    exit 1
-  fi
-
-  echo "Standalone release works correctly."
-}
-
-main "$@"

ci/dev/postinstall.sh

@@ -29,7 +29,11 @@ main() {
 
   install-deps test
   install-deps test/e2e/extensions/test-extension
-  install-deps lib/vscode
+  # We don't need these when running the integration tests
+  # so you can pass SKIP_SUBMODULE_DEPS
+  if [[ ! ${SKIP_SUBMODULE_DEPS-} ]]; then
+    install-deps lib/vscode
+  fi
 }
 
 main "$@"

ci/dev/test-integration.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+help() {
+  echo >&2 "  You can build the standalone release with 'yarn release:standalone'"
+  echo >&2 "  Or you can pass in a custom path."
+  echo >&2 "  CODE_SERVER_PATH='/var/tmp/coder/code-server/bin/code-server' yarn test:integration"
+}
+
+# Make sure a code-server release works. You can pass in the path otherwise it
+# will look for release-standalone in the current directory.
+#
+# This is to make sure we don't have Node version errors or any other
+# compilation-related errors.
+main() {
+  cd "$(dirname "$0")/../.."
+
+  source ./ci/lib.sh
+
+  local path="$RELEASE_PATH-standalone/bin/code-server"
+  if [[ ! ${CODE_SERVER_PATH-} ]]; then
+    echo "Set CODE_SERVER_PATH to test another build of code-server"
+  else
+    path="$CODE_SERVER_PATH"
+  fi
+
+  echo "Running tests with code-server binary: '$path'"
+
+  if [[ ! -f $path ]]; then
+    echo >&2 "No code-server build detected"
+    echo >&2 "Looked in $path"
+    help
+    exit 1
+  fi
+
+  CODE_SERVER_PATH="$path" CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --coverage=false --testRegex "./test/integration" --testPathIgnorePatterns "./test/integration/fixtures"
+}
+
+main "$@"

ci/dev/test-unit.sh

@@ -30,7 +30,7 @@ main() {
   # We must keep jest in a sub-directory. See ../../test/package.json for more
   # information. We must also run it from the root otherwise coverage will not
   # include our source files.
-  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@"
+  CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --testRegex "./test/unit/.*ts" --testPathIgnorePatterns "./test/unit/node/test-plugin"
 }
 
 main "$@"

docs/CONTRIBUTING.md

@@ -147,7 +147,7 @@ Build the release packages (make sure that you run `yarn release` first):
 
 ```shell
 yarn release:standalone
-yarn test:standalone-release
+yarn test:integration
 yarn package
 ```
 
@@ -188,9 +188,8 @@ We use these to test anything related to our scripts (most of which live under `
 
 ### Integration tests
 
-These are a work in progress. We build code-server and run a script called
-[test-standalone-release.sh](../ci/build/test-standalone-release.sh), which
-ensures that code-server's CLI is working.
+These are a work in progress. We build code-server and run tests with `yarn test:integration`, which ensures that code-server builds work on their respective
+platforms.
 
 Our integration tests look at components that rely on one another. For example,
 testing the CLI requires us to build and package code-server.

package.json

@@ -18,8 +18,8 @@
     "release:github-assets": "./ci/build/release-github-assets.sh",
     "release:prep": "./ci/build/release-prep.sh",
     "test:e2e": "VSCODE_IPC_HOOK_CLI= ./ci/dev/test-e2e.sh",
-    "test:standalone-release": "./ci/build/test-standalone-release.sh",
     "test:unit": "./ci/dev/test-unit.sh --forceExit --detectOpenHandles",
+    "test:integration": "./ci/dev/test-integration.sh",
     "test:scripts": "./ci/dev/test-scripts.sh",
     "package": "./ci/build/build-packages.sh",
     "postinstall": "./ci/dev/postinstall.sh",

test/integration/installExtension.test.ts

@@ -0,0 +1,25 @@
+import { stat } from "fs/promises"
+import path from "path"
+import { clean, tmpdir } from "../utils/helpers"
+import { runCodeServerCommand } from "../utils/runCodeServerCommand"
+
+describe("--install-extension", () => {
+  const testName = "installExtension"
+  let tempDir: string
+  let setupFlags: string[]
+
+  beforeEach(async () => {
+    await clean(testName)
+    tempDir = await tmpdir(testName)
+    setupFlags = ["--extensions-dir", tempDir]
+  })
+  it("should install an extension", async () => {
+    const extName = `wesbos.theme-cobalt2-2.1.6`
+    const vsixFileName = "wesbos.theme-cobalt2-2.1.6.vsix"
+    const extensionFixture = path.resolve(`./test/integration/fixtures/${vsixFileName}`)
+    await runCodeServerCommand([...setupFlags, "--install-extension", extensionFixture])
+    const pathToExtFolder = path.join(tempDir, extName)
+    const statInfo = await stat(pathToExtFolder)
+    expect(statInfo.isDirectory()).toBe(true)
+  }, 20000)
+})

test/integration/listExtensions.test.ts

@@ -0,0 +1,30 @@
+import { rename } from "fs/promises"
+import path from "path"
+import extract from "extract-zip"
+import { clean, tmpdir } from "../utils/helpers"
+import { runCodeServerCommand } from "../utils/runCodeServerCommand"
+
+describe("--list-extensions", () => {
+  const testName = "listExtensions"
+  const extName = `wesbos.theme-cobalt2`
+  const extVersion = "2.1.6"
+  const vsixFileName = `${extName}-${extVersion}.vsix`
+  let tempDir: string
+  let setupFlags: string[]
+
+  beforeEach(async () => {
+    await clean(testName)
+    tempDir = await tmpdir(testName)
+    setupFlags = ["--extensions-dir", tempDir]
+    const extensionFixture = path.resolve(`./test/integration/fixtures/${vsixFileName}`)
+    // Make folder because this is where we'll move the extension
+    const pathToUnpackedExtension = path.join(tempDir, `${extName}-${extVersion}`)
+    const tempPathToUnpackedExtension = path.join(tempDir, `${extName}-temp`)
+    await extract(extensionFixture, { dir: tempPathToUnpackedExtension })
+    await rename(path.join(tempPathToUnpackedExtension, "extension", pathToUnpackedExtension))
+  })
+  it("should list installed extensions", async () => {
+    const { stdout } = await runCodeServerCommand([...setupFlags, "--list-extensions"])
+    expect(stdout).toMatch(extName)
+  }, 20000)
+})

test/package.json

@@ -10,6 +10,7 @@
     "@types/supertest": "^2.0.11",
     "@types/wtfnode": "^0.7.0",
     "argon2": "^0.28.0",
+    "extract-zip": "^2.0.1",
     "jest": "^27.3.1",
     "jest-fetch-mock": "^3.0.3",
     "jsdom": "^16.4.0",

test/utils/runCodeServerCommand.ts

@@ -0,0 +1,14 @@
+import { exec } from "child_process"
+import path from "path"
+import { promisify } from "util"
+
+/**
+ *
+ * A helper function for integration tests to run code-server commands.
+ */
+export async function runCodeServerCommand(argv: string[]): Promise<{ stdout: string; stderr: string }> {
+  const CODE_SERVER_COMMAND = process.env.CODE_SERVER_PATH || path.resolve("../../release-standalone/bin/code-server")
+  const { stdout, stderr } = await promisify(exec)(`${CODE_SERVER_COMMAND} ${argv.join(" ")}`)
+
+  return { stdout, stderr }
+}

test/yarn.lock

@@ -1690,9 +1690,9 @@ globals@^11.1.0:
   integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==
 
 graceful-fs@^4.2.4:
-  version "4.2.9"
-  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96"
-  integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==
+  version "4.2.10"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
+  integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
 
 has-flag@^3.0.0:
   version "3.0.0"