Shake CI and docs up

ci/Dockerfile

@@ -0,0 +1,37 @@
+FROM debian:10
+
+RUN apt-get update
+RUN apt-get install -y curl
+
+ARG DOWNLOAD_URL=https://github.com/cdr/code-server/releases/download/2.1698/code-server2.1698-vsc1.41.1-linux-x86_64.tar.gz
+
+RUN cd /tmp && curl -L "$DOWNLOAD_URL" | \
+  tar -xz && \
+  cp code-server*/code-server /usr/local/bin/code-server
+
+# https://wiki.debian.org/Locale#Manually
+RUN apt-get install -y locales
+RUN sed -i "s/# en_US.UTF-8/en_US.UTF-8/" /etc/locale.gen
+RUN locale-gen
+ENV LANG=en_US.UTF-8
+
+RUN chsh -s /bin/bash
+ENV SHELL=/bin/bash
+
+RUN apt-get install -y dumb-init sudo
+RUN apt-get install -y man procps vim nano htop ssh git
+
+RUN adduser --gecos '' --disabled-password coder && \
+	echo "coder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/nopasswd
+
+RUN curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.4/fixuid-0.4-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \
+    chown root:root /usr/local/bin/fixuid && \
+    chmod 4755 /usr/local/bin/fixuid && \
+    mkdir -p /etc/fixuid && \
+    printf "user: coder\ngroup: coder\n" > /etc/fixuid/config.yml
+
+RUN rm -rf /var/lib/apt/lists/*
+EXPOSE 8080
+USER coder
+WORKDIR /home/coder
+ENTRYPOINT ["dumb-init", "fixuid", "-q", "code-server", "--host", "0.0.0.0"]

ci/build.ts

@@ -0,0 +1,440 @@
+import { Binary } from "@coder/nbin"
+import * as cp from "child_process"
+import * as fs from "fs-extra"
+import * as os from "os"
+import Bundler from "parcel-bundler"
+import * as path from "path"
+import * as util from "util"
+
+enum Task {
+  Binary = "binary",
+  Package = "package",
+  Build = "build",
+  Watch = "watch",
+}
+
+class Builder {
+  private readonly rootPath = path.resolve(__dirname, "..")
+  private readonly vscodeSourcePath = path.join(this.rootPath, "lib/vscode")
+  private readonly binariesPath = path.join(this.rootPath, "binaries")
+  private readonly buildPath = path.join(this.rootPath, "build")
+  private readonly codeServerVersion: string
+  private _target?: "darwin" | "alpine" | "linux"
+  private currentTask?: Task
+
+  public constructor() {
+    this.ensureArgument("rootPath", this.rootPath)
+    this.codeServerVersion = this.ensureArgument(
+      "codeServerVersion",
+      process.env.VERSION || require(path.join(this.rootPath, "package.json")).version
+    )
+  }
+
+  public run(task: Task | undefined): void {
+    this.currentTask = task
+    this.doRun(task).catch((error) => {
+      console.error(error.message)
+      process.exit(1)
+    })
+  }
+
+  private async task<T>(message: string, fn: () => Promise<T>): Promise<T> {
+    const time = Date.now()
+    this.log(`${message}...`, true)
+    try {
+      const t = await fn()
+      process.stdout.write(`took ${Date.now() - time}ms\n`)
+      return t
+    } catch (error) {
+      process.stdout.write("failed\n")
+      throw error
+    }
+  }
+
+  /**
+   * Writes to stdout with an optional newline.
+   */
+  private log(message: string, skipNewline = false): void {
+    process.stdout.write(`[${this.currentTask || "default"}] ${message}`)
+    if (!skipNewline) {
+      process.stdout.write("\n")
+    }
+  }
+
+  private async doRun(task: Task | undefined): Promise<void> {
+    if (!task) {
+      throw new Error("No task provided")
+    }
+
+    const arch = this.ensureArgument("arch", os.arch().replace(/^x/, "x86_"))
+    const target = this.ensureArgument("target", await this.target())
+    const binaryName = `code-server-${this.codeServerVersion}-${target}-${arch}`
+
+    switch (task) {
+      case Task.Watch:
+        return this.watch()
+      case Task.Binary:
+        return this.binary(binaryName)
+      case Task.Package:
+        return this.package(binaryName)
+      case Task.Build:
+        return this.build()
+      default:
+        throw new Error(`No task matching "${task}"`)
+    }
+  }
+
+  /**
+   * Get the target of the system.
+   */
+  private async target(): Promise<"darwin" | "alpine" | "linux"> {
+    if (!this._target) {
+      if (os.platform() === "darwin" || (process.env.OSTYPE && /^darwin/.test(process.env.OSTYPE))) {
+        this._target = "darwin"
+      } else {
+        // Alpine's ldd doesn't have a version flag but if you use an invalid flag
+        // (like --version) it outputs the version to stderr and exits with 1.
+        const result = await util
+          .promisify(cp.exec)("ldd --version")
+          .catch((error) => ({ stderr: error.message, stdout: "" }))
+        if (/musl/.test(result.stderr) || /musl/.test(result.stdout)) {
+          this._target = "alpine"
+        } else {
+          this._target = "linux"
+        }
+      }
+    }
+    return this._target
+  }
+
+  /**
+   * Make sure the argument is set. Display the value if it is.
+   */
+  private ensureArgument(name: string, arg?: string): string {
+    if (!arg) {
+      throw new Error(`${name} is missing`)
+    }
+    this.log(`${name} is "${arg}"`)
+    return arg
+  }
+
+  /**
+   * Build VS Code and code-server.
+   */
+  private async build(): Promise<void> {
+    process.env.NODE_OPTIONS = "--max-old-space-size=32384 " + (process.env.NODE_OPTIONS || "")
+    process.env.NODE_ENV = "production"
+
+    await this.task("cleaning up old build", async () => {
+      if (!process.env.SKIP_VSCODE) {
+        return fs.remove(this.buildPath)
+      }
+      // If skipping VS Code, keep the existing build if any.
+      try {
+        const files = await fs.readdir(this.buildPath)
+        return Promise.all(files.filter((f) => f !== "lib").map((f) => fs.remove(path.join(this.buildPath, f))))
+      } catch (error) {
+        if (error.code !== "ENOENT") {
+          throw error
+        }
+      }
+    })
+
+    const commit = require(path.join(this.vscodeSourcePath, "build/lib/util")).getVersion(this.rootPath) as string
+    if (!process.env.SKIP_VSCODE) {
+      await this.buildVscode(commit)
+    } else {
+      this.log("skipping vs code build")
+    }
+    await this.buildCodeServer(commit)
+
+    this.log(`final build: ${this.buildPath}`)
+  }
+
+  private async buildCodeServer(commit: string): Promise<void> {
+    await this.task("building code-server", async () => {
+      return util.promisify(cp.exec)("tsc --outDir ./out-build --tsBuildInfoFile ./.prod.tsbuildinfo", {
+        cwd: this.rootPath,
+      })
+    })
+
+    await this.task("bundling code-server", async () => {
+      return this.createBundler("dist-build", commit).bundle()
+    })
+
+    await this.task("copying code-server into build directory", async () => {
+      await fs.mkdirp(this.buildPath)
+      await Promise.all([
+        fs.copy(path.join(this.rootPath, "out-build"), path.join(this.buildPath, "out")),
+        fs.copy(path.join(this.rootPath, "dist-build"), path.join(this.buildPath, "dist")),
+        // For source maps and images.
+        fs.copy(path.join(this.rootPath, "src"), path.join(this.buildPath, "src")),
+      ])
+    })
+
+    await this.copyDependencies("code-server", this.rootPath, this.buildPath, {
+      commit,
+      version: process.env.VERSION,
+    })
+  }
+
+  private async buildVscode(commit: string): Promise<void> {
+    await this.task("building vs code", () => {
+      return util.promisify(cp.exec)("yarn gulp compile-build", { cwd: this.vscodeSourcePath })
+    })
+
+    await this.task("building builtin extensions", async () => {
+      const exists = await fs.pathExists(path.join(this.vscodeSourcePath, ".build/extensions"))
+      if (exists) {
+        process.stdout.write("already built, skipping...")
+      } else {
+        await util.promisify(cp.exec)("yarn gulp compile-extensions-build", { cwd: this.vscodeSourcePath })
+      }
+    })
+
+    await this.task("optimizing vs code", async () => {
+      return util.promisify(cp.exec)("yarn gulp optimize --gulpfile ./coder.js", { cwd: this.vscodeSourcePath })
+    })
+
+    if (process.env.MINIFY) {
+      await this.task("minifying vs code", () => {
+        return util.promisify(cp.exec)("yarn gulp minify --gulpfile ./coder.js", { cwd: this.vscodeSourcePath })
+      })
+    }
+
+    const vscodeBuildPath = path.join(this.buildPath, "lib/vscode")
+    await this.task("copying vs code into build directory", async () => {
+      await fs.mkdirp(vscodeBuildPath)
+      await Promise.all([
+        fs.move(
+          path.join(this.vscodeSourcePath, `out-vscode${process.env.MINIFY ? "-min" : ""}`),
+          path.join(vscodeBuildPath, "out")
+        ),
+        fs.copy(path.join(this.vscodeSourcePath, ".build/extensions"), path.join(vscodeBuildPath, "extensions")),
+      ])
+    })
+
+    await this.copyDependencies("vs code", this.vscodeSourcePath, vscodeBuildPath, {
+      commit,
+      date: new Date().toISOString(),
+    })
+  }
+
+  private async copyDependencies(name: string, sourcePath: string, buildPath: string, merge: object): Promise<void> {
+    await this.task(`copying ${name} dependencies`, async () => {
+      return Promise.all(
+        ["node_modules", "package.json", "yarn.lock"].map((fileName) => {
+          return fs.copy(path.join(sourcePath, fileName), path.join(buildPath, fileName))
+        })
+      )
+    })
+
+    const fileName = name === "code-server" ? "package" : "product"
+    await this.task(`writing final ${name} ${fileName}.json`, async () => {
+      const json = JSON.parse(await fs.readFile(path.join(sourcePath, `${fileName}.json`), "utf8"))
+      return fs.writeFile(
+        path.join(buildPath, `${fileName}.json`),
+        JSON.stringify(
+          {
+            ...json,
+            ...merge,
+          },
+          null,
+          2
+        )
+      )
+    })
+
+    if (process.env.MINIFY) {
+      await this.task(`restricting ${name} to production dependencies`, async () => {
+        await util.promisify(cp.exec)("yarn --production --ignore-scripts", { cwd: buildPath })
+        if (name === "code-server") {
+          await util.promisify(cp.exec)("yarn postinstall", { cwd: buildPath })
+        }
+      })
+    }
+  }
+
+  /**
+   * Bundles the built code into a binary.
+   */
+  private async binary(binaryName: string): Promise<void> {
+    const bin = new Binary({
+      mainFile: path.join(this.buildPath, "out/node/entry.js"),
+      target: await this.target(),
+    })
+
+    bin.writeFiles(path.join(this.buildPath, "**"))
+
+    await fs.mkdirp(this.binariesPath)
+
+    const binaryPath = path.join(this.binariesPath, binaryName)
+    await fs.writeFile(binaryPath, await bin.build())
+    await fs.chmod(binaryPath, "755")
+
+    this.log(`binary: ${binaryPath}`)
+  }
+
+  /**
+   * Package the binary into a release archive.
+   */
+  private async package(binaryName: string): Promise<void> {
+    const releasePath = path.join(this.rootPath, "release")
+    const archivePath = path.join(releasePath, binaryName)
+
+    await fs.remove(archivePath)
+    await fs.mkdirp(archivePath)
+
+    await fs.copyFile(path.join(this.binariesPath, binaryName), path.join(archivePath, "code-server"))
+    await fs.copyFile(path.join(this.rootPath, "README.md"), path.join(archivePath, "README.md"))
+    await fs.copyFile(path.join(this.vscodeSourcePath, "LICENSE.txt"), path.join(archivePath, "LICENSE.txt"))
+    await fs.copyFile(
+      path.join(this.vscodeSourcePath, "ThirdPartyNotices.txt"),
+      path.join(archivePath, "ThirdPartyNotices.txt")
+    )
+
+    if ((await this.target()) === "darwin") {
+      await util.promisify(cp.exec)(`zip -r "${binaryName}.zip" "${binaryName}"`, { cwd: releasePath })
+      this.log(`archive: ${archivePath}.zip`)
+    } else {
+      await util.promisify(cp.exec)(`tar -czf "${binaryName}.tar.gz" "${binaryName}"`, { cwd: releasePath })
+      this.log(`archive: ${archivePath}.tar.gz`)
+    }
+  }
+
+  private async watch(): Promise<void> {
+    let server: cp.ChildProcess | undefined
+    const restartServer = (): void => {
+      if (server) {
+        server.kill()
+      }
+      const s = cp.fork(path.join(this.rootPath, "out/node/entry.js"), process.argv.slice(3))
+      console.log(`[server] spawned process ${s.pid}`)
+      s.on("exit", () => console.log(`[server] process ${s.pid} exited`))
+      server = s
+    }
+
+    const vscode = cp.spawn("yarn", ["watch"], { cwd: this.vscodeSourcePath })
+    const tsc = cp.spawn("tsc", ["--watch", "--pretty", "--preserveWatchOutput"], { cwd: this.rootPath })
+    const bundler = this.createBundler()
+
+    const cleanup = (code?: number | null): void => {
+      this.log("killing vs code watcher")
+      vscode.removeAllListeners()
+      vscode.kill()
+
+      this.log("killing tsc")
+      tsc.removeAllListeners()
+      tsc.kill()
+
+      if (server) {
+        this.log("killing server")
+        server.removeAllListeners()
+        server.kill()
+      }
+
+      this.log("killing bundler")
+      process.exit(code || 0)
+    }
+
+    process.on("SIGINT", () => cleanup())
+    process.on("SIGTERM", () => cleanup())
+
+    vscode.on("exit", (code) => {
+      this.log("vs code watcher terminated unexpectedly")
+      cleanup(code)
+    })
+    tsc.on("exit", (code) => {
+      this.log("tsc terminated unexpectedly")
+      cleanup(code)
+    })
+    const bundle = bundler.bundle().catch(() => {
+      this.log("parcel watcher terminated unexpectedly")
+      cleanup(1)
+    })
+    bundler.on("buildEnd", () => {
+      console.log("[parcel] bundled")
+    })
+    bundler.on("buildError", (error) => {
+      console.error("[parcel]", error)
+    })
+
+    vscode.stderr.on("data", (d) => process.stderr.write(d))
+    tsc.stderr.on("data", (d) => process.stderr.write(d))
+
+    // From https://github.com/chalk/ansi-regex
+    const pattern = [
+      "[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)",
+      "(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))",
+    ].join("|")
+    const re = new RegExp(pattern, "g")
+
+    /**
+     * Split stdout on newlines and strip ANSI codes.
+     */
+    const onLine = (proc: cp.ChildProcess, callback: (strippedLine: string, originalLine: string) => void): void => {
+      let buffer = ""
+      if (!proc.stdout) {
+        throw new Error("no stdout")
+      }
+      proc.stdout.setEncoding("utf8")
+      proc.stdout.on("data", (d) => {
+        const data = buffer + d
+        const split = data.split("\n")
+        const last = split.length - 1
+
+        for (let i = 0; i < last; ++i) {
+          callback(split[i].replace(re, ""), split[i])
+        }
+
+        // The last item will either be an empty string (the data ended with a
+        // newline) or a partial line (did not end with a newline) and we must
+        // wait to parse it until we get a full line.
+        buffer = split[last]
+      })
+    }
+
+    let startingVscode = false
+    onLine(vscode, (line, original) => {
+      console.log("[vscode]", original)
+      // Wait for watch-client since "Finished compilation" will appear multiple
+      // times before the client starts building.
+      if (!startingVscode && line.includes("Starting watch-client")) {
+        startingVscode = true
+      } else if (startingVscode && line.includes("Finished compilation") && process.env.AUTO_PATCH) {
+        cp.exec("yarn patch:generate", { cwd: this.rootPath }, (error, _, stderr) => {
+          if (error || stderr) {
+            console.error(error ? error.message : stderr)
+          }
+        })
+      }
+    })
+
+    onLine(tsc, (line, original) => {
+      // tsc outputs blank lines; skip them.
+      if (line !== "") {
+        console.log("[tsc]", original)
+      }
+      if (line.includes("Watching for file changes")) {
+        bundle.then(restartServer)
+      }
+    })
+  }
+
+  private createBundler(out = "dist", commit?: string): Bundler {
+    return new Bundler(path.join(this.rootPath, "src/browser/pages/app.ts"), {
+      cache: true,
+      cacheDir: path.join(this.rootPath, ".cache"),
+      detailedReport: true,
+      minify: !!process.env.MINIFY,
+      hmr: false,
+      logLevel: 1,
+      outDir: path.join(this.rootPath, out),
+      publicUrl: `/static-${commit}/dist`,
+      target: "browser",
+    })
+  }
+}
+
+const builder = new Builder()
+builder.run(process.argv[2] as Task)

ci/clean.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+main() {
+  git clean -xffd
+  git submodule foreach --recursive git clean -xffd
+  git submodule foreach --recursive git reset --hard
+}
+
+main "$@"

ci/fmt.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+main() {
+  shfmt -i 2 -w -s -sr $$(git ls-files "*.sh")
+  prettier --write --loglevel=warn $$(git ls-files "*.js" "*.ts" "*.tsx" "*.html" "*.json" "*.css" "*.md" "*.toml" "*.yaml" "*.yml")
+	if [[ "$CI" != "" && $$(git ls-files --other --modified --exclude-standard) != "" ]]; then
+	  echo "Files need generation or are formatted incorrectly:"
+	  git -c color.ui=always status | grep --color=no '\[31m'
+	  echo "Please run the following locally:"
+	  echo "  make fmt"
+	  exit 1
+	fi
+}
+
+main "$@"

ci/image/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:12-buster
+
+RUN apt-get update && apt-get install -y \
+	libxkbfile-dev \
+	libsecret-1-dev \
+	dumb-init
+
+RUN curl -L https://github.com/mvdan/sh/releases/download/v3.0.1/shfmt_v3.0.1_linux_amd64 > /usr/local/bin/shfmt && chmod +x /usr/local/bin/shfmt
+
+COPY entrypoint.sh /bin/entrypoint.sh
+
+ENTRYPOINT ["dumb-init", "/bin/entrypoint.sh"]

ci/image/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+eval "$@"

ci/lint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+main() {
+  eslint --max-warnings=0 --fix $$(git ls-files "*.ts" "*.tsx" "*.js")
+  stylelint --fix $$(git ls-files "*.css")
+}
+
+main "$@"

ci/release.bash

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# ci.bash -- Build code-server in the CI.
+
+set -euo pipefail
+
+# This script assumes that yarn has already ran.
+function main() {
+  cd "$(dirname "${0}")/.."
+
+  local code_server_version=${VERSION:-${TRAVIS_TAG:-${DRONE_TAG:-}}}
+  if [[ -z $code_server_version ]] ; then
+    code_server_version=$(grep version ./package.json | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[:space:]')
+  fi
+  export VERSION=$code_server_version
+
+  YARN_CACHE_FOLDER="$(pwd)/yarn-cache"
+  export YARN_CACHE_FOLDER
+
+  # Always minify and package on tags since that's when releases are pushed.
+  if [[ -n ${DRONE_TAG:-} || -n ${TRAVIS_TAG:-} ]] ; then
+    export MINIFY="true"
+    export PACKAGE="true"
+  fi
+
+  yarn build
+  yarn binary
+  if [[ -n ${PACKAGE:-} ]] ; then
+    yarn package
+  fi
+
+  cd binaries
+
+  if [[ -n ${STRIP_BIN_TARGET:-} ]] ; then
+    # In this case provide plainly named binaries.
+    for binary in code-server* ; do
+      echo "Moving $binary to code-server"
+      mv "$binary" code-server
+    done
+  elif [[ -n ${DRONE_TAG:-} || -n ${TRAVIS_TAG:-} ]] ; then
+    # Prepare directory for uploading binaries on release.
+    for binary in code-server* ; do
+      mkdir -p "../binary-upload"
+
+      local prefix="code-server-$code_server_version-"
+      local target="${binary#$prefix}"
+      if [[ $target == "linux-x86_64" ]] ; then
+        echo "Copying $binary to ../binary-upload/latest-linux"
+        cp "$binary" "../binary-upload/latest-linux"
+      fi
+
+      local gcp_dir
+      gcp_dir="../binary-upload/releases/$code_server_version/$target"
+      mkdir -p "$gcp_dir"
+
+      echo "Copying $binary to $gcp_dir/code-server"
+      cp "$binary" "$gcp_dir/code-server"
+    done
+  fi
+}
+
+main "$@"

ci/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends": "../tsconfig.json",
+  "include": [
+    "./**/*.ts"
+  ]
+}

ci/vscode.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# 1. Ensures VS Code is cloned.
+# 2. Patches it.
+# 3. Installs it.
+main() {
+  cd "$(dirname "$0")/.."
+
+  git submodule update --init
+
+  # If the patch fails to apply, then it's likely already applied
+  yarn vscode:patch &>/dev/null || true
+
+  # Install VS Code dependencies.
+  (cd lib/vscode && yarn)
+}
+
+main "$@"