Add build scripts

.gitignore

@@ -1 +1,3 @@
 node_modules
+build
+release

.travis.yml

@@ -8,20 +8,20 @@ matrix:
   - os: linux
     dist: trusty
     env:
-      - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="centos"
+      - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="centos"
   - os: linux
     dist: trusty
     env:
-      - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION" TARGET="alpine"
+      - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER" TARGET="alpine"
   - os: osx
     env:
-      - VSCODE_VERSION="1.33.1" MAJOR_VERSION="1" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER-vsc$VSCODE_VERSION"
+      - VSCODE_VERSION="1.36.0" MAJOR_VERSION="2" VERSION="$MAJOR_VERSION.$TRAVIS_BUILD_NUMBER"
 before_install:
-- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
+- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install -y libxkbfile-dev libsecret-1-dev; fi
-  libsecret-1-dev; fi
 - npm install -g yarn@1.12.3
+- npm install -g @coder/nbin
 script:
-- scripts/build.sh
+- scripts/ci.bash
 before_deploy:
 - echo "$VERSION" "$TRAVIS_COMMIT"
 - git config --local user.name "$USER_NAME"

README.md

@@ -51,13 +51,19 @@ Quickstart guides for [Google Cloud](doc/admin/install/google_cloud.md), [AWS](d
 How to [secure your setup](/doc/security/ssl.md).
 
 ## Build
-- Run `yarn build ${vscodeVersion}`in this directory (for example, `yarn build 1.35.0`).
+- If you also plan on developing, set the `OUT` environment variable: `
+  export OUT=/path/to/some/directory`. Otherwise it will build in this
+  directory which will cause issues because then `yarn watch` will try to
+  compile the build directory as well.
+- Run `yarn build ${vscodeVersion} ${target} ${arch}`in this directory (for example:
+  `yarn build 1.35.0 linux x64`).
 
 ## Development
 
 - Clone VS Code.
-- Clone code-server to `src/vs/server` in the VS Code source.
+- Run `yarn` in the VS Code root directory.
-- Run `yarn` in this directory (only need to do this once).
+- Clone this repository to `src/vs/server` in the VS Code source.
+- Run `yarn` in this directory.
 - Run `yarn watch` in this directory.
 - Wait for the initial compilation to complete.
 - Run `yarn start` in this directory.

main.js

@@ -1 +1,11 @@
+try {
+	const nbin = require("nbin");
+	const path = require("path");
+	const rootPath = path.resolve(__dirname, "../../..");
+	console.log("Shimming", rootPath);
+	nbin.shimNativeFs(rootPath);
+} catch (error) {
+	console.log("Not in the binary");
+}
+
 require("../../bootstrap-amd").load("vs/server/cli");

package.json

@@ -1,8 +1,13 @@
 {
+  "license": "MIT",
   "scripts": {
+    "postinstall": "rm -r node_modules/@types/node # I keep getting type conflicts",
     "start": "nodemon ../../../out/vs/server/main.js --watch ../../../out --verbose",
     "watch": "cd ../../../ && yarn watch-client",
-    "build": "echo TODO && exit 1"
+    "build": "bash ./scripts/tasks.bash build",
+    "package": "bash ./scripts/tasks.bash package",
+    "vstar": "bash ./scripts/tasks.bash vstar",
+    "binary": "bash ./scripts/tasks.bash binary"
   },
   "devDependencies": {
     "@types/tar-stream": "^1.6.1",

scripts/build.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-# Build using a Docker container using the specified image and version.
-function docker_build() {
-	local image="${1}" ; shift
-	local version="${1}" ; shift
-
-	local containerId
-	containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}")
-	docker start "${containerId}"
-	docker exec "${containerId}" mkdir -p /src
-
-	function docker_exec() {
-		docker exec "${containerId}" bash -c "$@"
-	}
-
-	docker cp ./. "${containerId}":/src
-	docker_exec "cd /src && yarn"
-	docker_exec "cd /src && npm rebuild"
-	docker_exec "cd /src && NODE_ENV=production VERSION=${version} yarn task build:server:binary"
-	docker_exec "cd /src && yarn task package ${version}"
-	docker cp "${containerId}":/src/release/. ./release/
-
-	docker stop "${containerId}"
-}
-
-function main() {
-	local version=${VERSION:-}
-	local ostype=${OSTYPE:-}
-
-	if [[ -z "${version}" ]] ; then
-		>&2 echo "Must set VERSION environment variable"
-		exit 1
-	fi
-
-	if [[ "${ostype}" == "darwin"* ]]; then
-		NODE_ENV=production VERSION="${version}" yarn task build:server:binary
-		yarn task package "${version}"
-	else
-		local image
-		if [[ "$TARGET" == "alpine" ]]; then
-			image="codercom/nbin-alpine"
-		else
-			image="codercom/nbin-centos"
-		fi
-		docker_build "${image}" "${version}"
-	fi
-}
-
-main "$@"

scripts/ci.bash

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -euo pipefail
+
+# Build using a Docker container using the specified image and version.
+function docker-build() {
+	local image="${1}" ; shift
+	local version="${1}" ; shift
+	local vscodeVersion="${1}" ; shift
+	local target="${1}" ; shift
+	local arch="${1}" ; shift
+
+	local containerId
+	containerId=$(docker create --network=host --rm -it -v "$(pwd)"/.cache:/src/.cache "${image}")
+	docker start "${containerId}"
+	docker exec "${containerId}" mkdir -p /src
+
+	function docker-exec() {
+		docker exec "${containerId}" bash -c "$@"
+	}
+
+	docker cp ./. "${containerId}":/src
+	docker-exec "cd /src && CI=true yarn build \"${vscodeVersion}\" \"${target}\" \"${arch}\""
+	docker-exec "cd /src && CI=true yarn binary \"${vscodeVersion}\" \"${target}\" \"${arch}\""
+	docker-exec "cd /src && CI=true yarn package \"${vscodeVersion}\" \"${target}\" \"${arch}\" \"${version}\""
+	docker cp "${containerId}":/src/release/. ./release/
+
+	docker stop "${containerId}"
+}
+
+# Build code-server in the CI.
+function main() {
+	local version="${VERSION:-}"
+	local vscodeVersion="${VSCODE_VERSION:-}"
+	local ostype="${OSTYPE:-}"
+	local target="${TARGET:-}"
+	local arch=x64
+
+	if [[ -z "${version}" ]] ; then
+		>&2 echo "Must set VERSION environment variable"; exit 1
+	fi
+
+	if [[ -z "${vscodeVersion}" ]] ; then
+		>&2 echo "Must set VSCODE_VERSION environment variable"; exit 1
+	fi
+
+	if [[ "${ostype}" == "darwin"* ]]; then
+		target=darwin
+		CI=true yarn build "${vscodeVersion}" "${target}" "${arch}"
+		CI=true yarn binary "${vscodeVersion}" "${target}" "${arch}"
+		CI=true yarn package "${vscodeVersion}" "${target}" "${arch}" "${version}"
+	else
+		local image
+		if [[ "${target}" == alpine ]]; then
+			image=codercom/nbin-alpine
+			target=musl
+		else
+			image=codercom/nbin-centos
+			target=linux
+		fi
+		docker-build "${image}" "${version}" "${vscodeVersion}" "${target}" "${arch}"
+	fi
+}
+
+main "$@"

scripts/nbin.js

@@ -0,0 +1,24 @@
+/* global require, __dirname, process */
+const { Binary } = require("@coder/nbin");
+const fs = require("fs");
+const path = require("path");
+
+const target = process.argv[2];
+const arch = process.argv[3];
+const source = process.argv[4];
+
+const bin = new Binary({
+	mainFile: path.join(source, "out/vs/server/main.js"),
+	target: target,
+});
+
+bin.writeFiles(path.join(source, "**"));
+
+bin.build().then((binaryData) => {
+	const outputPath = path.join(source, "code-server");
+	fs.writeFileSync(outputPath, binaryData);
+	fs.chmodSync(outputPath, "755");
+}).catch((ex) => {
+	console.error(ex);
+	process.exit(1);
+});

scripts/tasks.bash

@@ -0,0 +1,250 @@
+#!/bin/bash
+set -euo pipefail
+
+function log() {
+	local message="${1}" ; shift
+	local level="${1:-info}"
+	if [[ "${level}" == "error" ]] ; then
+		>&2 echo "${message}"
+	else
+		echo "${message}"
+	fi
+}
+
+function exit-if-ci() {
+	if [[ -n "${ci}" ]] ; then
+		log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} is incorrectly built" "error"
+		exit 1
+	fi
+}
+
+# Copy code-server into VS Code along with its dependencies.
+function copy-server() {
+	local serverPath="${vscodeSourcePath}/src/vs/server"
+	rm -rf "${serverPath}"
+	mkdir -p "${serverPath}"
+
+	log "Copying server code"
+
+	cp "${rootPath}"/*.{ts,js} "${serverPath}"
+	cp "${rootPath}/package.json" "${serverPath}"
+	cp "${rootPath}/yarn.lock" "${serverPath}"
+
+	if [[ -d "${rootPath}/node_modules" ]] ; then
+		log "Copying dependencies"
+		cp -r "${rootPath}/node_modules" "${serverPath}"
+	else
+		log "Installing dependencies"
+		cd "${serverPath}"
+		yarn
+		rm -r node_modules/@types/node # I keep getting type conflicts
+	fi
+}
+
+# Copy code-server into VS Code then build it.
+function build-code-server() {
+	copy-server
+
+	# TODO: look into making it do the full minified build for just our code
+	# (basically just want to skip extensions, target our server code, and get
+	# the same type of build you get with the vscode-linux-x64-min task).
+	# Something like: yarn gulp "vscode-server-${target}-${arch}-min"
+	cd "${vscodeSourcePath}"
+	yarn gulp compile-client
+
+	rm -rf "${codeServerBuildPath}"
+	mkdir -p "${codeServerBuildPath}"
+
+	cp -r "${vscodeBuildPath}/resources/app/extensions" "${codeServerBuildPath}"
+	cp -r "${vscodeBuildPath}/resources/app/"*.json "${codeServerBuildPath}"
+	cp -r "${vscodeSourcePath}/out" "${codeServerBuildPath}"
+	rm -rf "${codeServerBuildPath}/out/vs/server/node_modules"
+	cp -r "${vscodeSourcePath}/remote/node_modules" "${codeServerBuildPath}"
+
+	log "Final build: ${codeServerBuildPath}"
+}
+
+# Build VS Code if it hasn't already been built. If we're in the CI and it's
+# not fully built, error and exit.
+function build-vscode() {
+	if [[ ! -d "${vscodeSourcePath}" ]] ; then
+		exit-if-ci
+		log "${vscodeSourceName} does not exist, cloning"
+		git clone https://github.com/microsoft/vscode --quiet \
+			--branch "${vscodeVersion}" --single-branch --depth=1 \
+			"${vscodeSourcePath}"
+	else
+		log "${vscodeSourceName} already exists, skipping clone"
+	fi
+
+	cd "${vscodeSourcePath}"
+
+	if [[ ! -d "${vscodeSourcePath}/node_modules" ]] ; then
+		exit-if-ci
+		log "Installing VS Code dependencies"
+		yarn
+		# Not entirely sure why but there seem to be problems with native modules.
+		# Also vscode-ripgrep keeps complaining after the rebuild that the
+		# node_modules directory doesn't exist, so we're ignoring that for now.
+		npm rebuild || true
+
+		# Keep just what we need to keep the pre-built archive smaller.
+		rm -rf "${vscodeSourcePath}/.git"
+		rm -rf "${vscodeSourcePath}/test"
+	else
+		log "${vscodeSourceName}/node_modules already exists, skipping install"
+	fi
+
+	if [[ ! -d "${vscodeBuildPath}" ]] ; then
+		exit-if-ci
+		log "${vscodeBuildName} does not exist, building"
+		local builtPath="${buildPath}/VSCode-${target}-${arch}"
+		rm -rf "${builtPath}"
+		yarn gulp "vscode-${target}-${arch}-min" --max-old-space-size=32384
+		mkdir -p "${vscodeBuildPath}/resources/app"
+		# Copy just what we need to keep the pre-built archive smaller.
+		mv "${builtPath}/resources/app/extensions" "${vscodeBuildPath}/resources/app"
+		mv "${builtPath}/resources/app/"*.json "${vscodeBuildPath}/resources/app"
+		rm -rf "${builtPath}"
+	else
+		log "${vscodeBuildName} already exists, skipping build"
+	fi
+}
+
+# Download VS Code with either curl or wget depending on which is available.
+function download-vscode() {
+	cd "${buildPath}"
+	if command -v wget &> /dev/null ; then
+		log "Attempting to download ${tarName} with wget"
+		wget "${vsSourceUrl}" --quiet
+	else
+		log "Attempting to download ${tarName} with curl"
+		curl "${vsSourceUrl}" --silent --fail --output "${tarName}"
+	fi
+}
+
+# Download pre-built VS Code if necessary. Build if there is no available
+# download but not when in the CI. The pre-built package basically just
+# provides us the dependencies and extensions so we don't have to install and
+# build them respectively which takes a long time.
+function prepare-vscode() {
+	if [[ ! -d "${vscodeBuildPath}" || ! -d "${vscodeSourcePath}" ]] ; then
+		mkdir -p "${buildPath}"
+		local tarName="vstar-${vscodeVersion}-${target}-${arch}.tar.gz"
+		local vsSourceUrl="https://codesrv-ci.cdr.sh/${tarName}"
+		if download-vscode ; then
+			cd "${buildPath}"
+			rm -rf "${vscodeBuildPath}"
+			tar -xzf "${tarName}"
+			rm "${tarName}"
+		elif [[ -n "${ci}" ]] ; then
+			log "Pre-built VS Code ${vscodeVersion}-${target}-${arch} does not exist" "error"
+			exit 1
+		else
+			log "${tarName} does not exist, building"
+			build-vscode
+			return
+		fi
+	else
+		log "VS Code is already downloaded or built"
+	fi
+
+	log "Ensuring VS Code is fully built"
+	build-vscode
+}
+
+function build-task() {
+	prepare-vscode
+	build-code-server
+}
+
+function vstar-task() {
+	local archivePath="${releasePath}/vstar-${vscodeVersion}-${target}-${arch}.tar.gz"
+	rm -f "${archivePath}"
+	mkdir -p "${releasePath}"
+	tar -C "${buildPath}" -czf "${archivePath}" "${vscodeSourceName}" "${vscodeBuildName}"
+	log "Archive: ${archivePath}"
+}
+
+function package-task() {
+	local version="${1}" ; shift
+
+	log " version: ${version}"
+
+	local archiveName="code-server${version}-vsc${vscodeVersion}-${target}-${arch}"
+	local archivePath="${releasePath}/${archiveName}"
+	rm -rf "${archivePath}"
+	mkdir -p "${archivePath}"
+
+	cp "${buildPath}/code-server" "${archivePath}"
+	cp "${rootPath}/README.md" "${archivePath}"
+	cp "${vscodeSourcePath}/LICENSE.txt" "${archivePath}"
+	cp "${vscodeSourcePath}/ThirdPartyNotices.txt" "${archivePath}"
+
+	cd "${releasePath}"
+	if [[ "${target}" == "linux" ]] ; then
+		tar -czf "${archiveName}.tar.gz" "${archiveName}"
+	else
+		zip -r "${archiveName}.zip" "${archiveName}"
+	fi
+
+	log "Archive: ${archivePath}"
+}
+
+# Package built code into a binary.
+function binary-task() {
+	# I had trouble getting VS Code to build with the @coder/nbin dependency due
+	# to the types it installs (tons of conflicts), so for now it's a global
+	# dependency.
+	cd "${rootPath}"
+	npm link @coder/nbin
+	node "${rootPath}/scripts/nbin.js" "${target}" "${arch}" "${codeServerBuildPath}"
+	rm node_modules/@coder/nbin
+	mv "${codeServerBuildPath}/code-server" "${buildPath}"
+	log "Binary at ${buildPath}/code-server"
+}
+
+function main() {
+	local task="${1}" ; shift
+	local vscodeVersion="${1}" ; shift
+	local target="${1}" ; shift
+	local arch="${1}" ; shift
+	local ci="${CI:-}"
+
+	local relativeRootPath
+	local rootPath
+	relativeRootPath="$(dirname "${0}")/.."
+	rootPath="$(realpath "${relativeRootPath}")"
+
+	# This lets you build in a separate directory since building within this
+	# directory while developing makes it hard to keep developing since compiling
+	# will compile everything in the build directory as well.
+	local outPath="${OUT:-${rootPath}}"
+
+	local releasePath="${outPath}/release"
+	local buildPath="${outPath}/build"
+
+	local vscodeSourceName="vscode-${vscodeVersion}-source"
+	local vscodeBuildName="vscode-${vscodeVersion}-${target}-${arch}-built"
+	local vscodeSourcePath="${buildPath}/${vscodeSourceName}"
+	local vscodeBuildPath="${buildPath}/${vscodeBuildName}"
+
+	local codeServerBuildName="code-server-${target}-${arch}-built"
+	local codeServerBuildPath="${buildPath}/${codeServerBuildName}"
+
+	log "Running ${task} task"
+	log " rootPath: ${rootPath}"
+	log " outPath: ${outPath}"
+	log " vscodeVersion: ${vscodeVersion}"
+	log " target: ${target}"
+	log " arch: ${arch}"
+	if [[ -n "${ci}" ]] ; then
+		log " CI: yes"
+	else
+		log " CI: no"
+	fi
+
+	"${task}-task" "$@"
+}
+
+main "$@"

scripts/vstar.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-# Builds a tarfile containing vscode sourcefiles neccessary for CI.
-# Done outside the CI and uploaded to object storage to reduce CI time.
-
-branch=1.33.1
-dir=/tmp/vstar
-outfile=/tmp/vstar-$branch.tar.gz
-rm -rf $dir
-mkdir -p $dir
-
-cd $dir
-git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
-cd vscode
-
-yarn
-
-npx gulp vscode-linux-x64 --max-old-space-size=32384
-rm -rf extensions build out* test
-cd ..
-mv *-x64/resources/app/extensions ./extensions
-rm -rf *-x64
-tar -czvf $outfile .

yarn.lock

@@ -3,9 +3,9 @@
 
 
 "@types/node@*", "@types/node@^10.12.12":
-  version "10.14.10"
+  version "10.14.12"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.10.tgz#e491484c6060af8d461e12ec81c0da8a3282b8de"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-10.14.12.tgz#0eec3155a46e6c4db1f27c3e588a205f767d622f"
-  integrity sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q==
+  integrity sha512-QcAKpaO6nhHLlxWBvpc4WeLrTvPqlHOvaj0s5GriKkA1zq+bsFBPpfYCvQhLqLgYlIko8A9YrPdaMHCo5mBcpg==
 
 "@types/tar-stream@^1.6.1":
   version "1.6.1"