Improve CI caching (#416)

* Adjust linux distro to ubuntu 14.04

* Cache lib directory for speedy builds

* Fix path linking for default extensions

* Update reset

* Reset to head

* Improve caching

* Still run yarn in CI

* Update yarn before install

* Increase cache timeout

* Install vscode from vstar

* Undo data-dir changes to CLI, add back clean, remove unused CI func

* Remove additional flags added

* Remove unused dependency

* Reset vscode install dir so patching always works

.gitignore

@@ -4,3 +4,4 @@ dist
 out
 .DS_Store
 release
+.cache

.travis.yml

@@ -6,11 +6,12 @@ env:
 matrix:
   include:
   - os: linux
-    dist: ubuntu
+    dist: trusty
   - os: osx
 before_install:
 - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
   libsecret-1-dev; fi
+- npm install -g yarn@1.12.3
 script:
 - scripts/build.sh
 before_deploy:
@@ -35,4 +36,8 @@ deploy:
   on:
     repo: codercom/code-server
     branch: master
-cache: yarn
+cache:
+  yarn: true
+  timeout: 1000
+  directories:
+  - .cache

build/tasks.ts

@@ -4,21 +4,22 @@ import * as fse from "fs-extra";
 import * as os from "os";
 import * as path from "path";
 import * as zlib from "zlib";
+import * as https from "https";
+import * as tar from "tar";
 
 const isWin = os.platform() === "win32";
 const libPath = path.join(__dirname, "../lib");
 const vscodePath = path.join(libPath, "vscode");
+const defaultExtensionsPath = path.join(libPath, "extensions");
 const pkgsPath = path.join(__dirname, "../packages");
-const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions");
 const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0";
+const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`;
 
 const buildServerBinary = register("build:server:binary", async (runner) => {
 	await ensureInstalled();
-	await copyForDefaultExtensions();
 	await Promise.all([
 		buildBootstrapFork(),
 		buildWeb(),
-		buildDefaultExtensions(),
 		buildServerBundle(),
 		buildAppBrowser(),
 	]);
@@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => {
 	await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]);
 });
 
-const extDirPath = path.join("lib", "vscode-default-extensions");
-const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => {
-	if (!fs.existsSync(defaultExtensionsPath)) {
-		await ensureClean();
-		await ensureInstalled();
-		await new Promise((resolve, reject): void => {
-			fse.remove(extDirPath, (err) => {
-				if (err) {
-					return reject(err);
-				}
-
-				resolve();
-			});
-		});
-		await new Promise((resolve, reject): void => {
-			fse.copy(vscodePath, extDirPath, (err) => {
-				if (err) {
-					return reject(err);
-				}
-
-				resolve();
-			});
-		});
-	}
-});
-
-const buildDefaultExtensions = register("build:default-extensions", async (runner) => {
-	if (!fs.existsSync(defaultExtensionsPath)) {
-		await copyForDefaultExtensions();
-		runner.cwd = extDirPath;
-		const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]);
-		if (resp.exitCode !== 0) {
-			throw new Error(`Failed to build default extensions: ${resp.stderr}`);
-		}
-	}
-});
-
 const ensureInstalled = register("vscode:install", async (runner) => {
-	await ensureCloned();
+	runner.cwd = libPath;
 
-	runner.cwd = vscodePath;
+	if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) {
-	const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []);
+		const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version;
-	if (install.exitCode !== 0) {
+		if (pkgVersion === vscodeVersion) {
-		throw new Error(`Failed to install vscode dependencies: ${install.stderr}`);
+			runner.cwd = vscodePath;
-	}
-});
 
-const ensureCloned = register("vscode:clone", async (runner) => {
+			const reset = await runner.execute("git", ["reset", "--hard"]);
-	if (fs.existsSync(vscodePath)) {
+			if (reset.exitCode !== 0) {
-		await ensureClean();
+				throw new Error(`Failed to clean git repository: ${reset.stderr}`);
-	} else {
+			}
-		fse.mkdirpSync(libPath);
+
-		runner.cwd = libPath;
+			return;
-		const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]);
-		if (clone.exitCode !== 0) {
-			throw new Error(`Failed to clone: ${clone.exitCode}`);
 		}
 	}
 
-	runner.cwd = vscodePath;
+	fse.removeSync(libPath);
-	const checkout = await runner.execute("git", ["checkout", vscodeVersion]);
+	fse.mkdirpSync(libPath);
-	if (checkout.exitCode !== 0) {
-		throw new Error(`Failed to checkout: ${checkout.stderr}`);
-	}
-});
 
-const ensureClean = register("vscode:clean", async (runner) => {
+	await new Promise<void>((resolve, reject): void => {
-	runner.cwd = vscodePath;
+		https.get(vsSourceUrl, (res) => {
+			if (res.statusCode !== 200) {
+				return reject(res.statusMessage);
+			}
 
-	const status = await runner.execute("git", ["status", "--porcelain"]);
+			res.pipe(tar.x({
-	if (status.stdout.trim() !== "") {
+				C: libPath,
-		const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]);
+			}).on("finish", () => {
-		if (clean.exitCode !== 0) {
+				resolve();
-			throw new Error(`Failed to clean git repository: ${clean.stderr}`);
+			}).on("error", (err: Error) => {
-		}
+				reject(err);
-		const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]);
+			}));
-		if (removeUnstaged.exitCode !== 0) {
+		}).on("error", (err) => {
-			throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`);
+			reject(err);
-		}
+		});
-	}
+	});
-	const fetch = await runner.execute("git", ["fetch", "--prune"]);
-	if (fetch.exitCode !== 0) {
-		throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`);
-	}
 });
 
 const ensurePatched = register("vscode:patch", async (runner) => {
 	if (!fs.existsSync(vscodePath)) {
 		throw new Error("vscode must be cloned to patch");
 	}
-	await ensureClean();
+	await ensureInstalled();
 
 	runner.cwd = vscodePath;
 	const patchPath = path.join(__dirname, "../scripts/vscode.patch");

package.json

@@ -15,7 +15,9 @@
   "devDependencies": {
     "@types/fs-extra": "^5.0.4",
     "@types/node": "^10.12.18",
+    "@types/tar": "^4.0.0",
     "@types/trash": "^4.3.1",
+    "cache-loader": "^2.0.1",
     "cross-env": "^5.2.0",
     "crypto-browserify": "^3.12.0",
     "css-loader": "^2.1.0",
@@ -34,6 +36,8 @@
     "sass-loader": "^7.1.0",
     "string-replace-loader": "^2.1.1",
     "style-loader": "^0.23.1",
+    "tar": "^4.4.8",
+    "terser-webpack-plugin": "^1.2.3",
     "ts-loader": "^5.3.3",
     "ts-node": "^7.0.1",
     "tsconfig-paths": "^3.8.0",

scripts/vstar.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euxo pipefail
+
+# Builds a tarfile containing vscode sourcefiles neccessary for CI.
+# Done outside the CI and uploaded to object storage to reduce CI time.
+
+branch=1.32.0
+dir=/tmp/vstar
+outfile=/tmp/vstar-$branch.tar.gz
+rm -rf $dir
+mkdir -p $dir
+
+cd $dir
+git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
+cd vscode
+
+yarn
+
+npx gulp vscode-linux-x64 --max-old-space-size=32384
+rm -rf extensions build out* test
+cd ..
+mv *-x64/resources/app/extensions ./extensions
+rm -rf *-x64
+tar -czvf $outfile .

scripts/webpack.general.config.js

@@ -3,6 +3,7 @@ const os = require("os");
 const environment = process.env.NODE_ENV || "development";
 const HappyPack = require("happypack");
 const webpack = require("webpack");
+const TerserPlugin = require("terser-webpack-plugin");
 
 const root = path.join(__dirname, "..");
 
@@ -107,6 +108,11 @@ module.exports = (options = {}) => ({
 			id: "ts",
 			threads: Math.max(os.cpus().length - 1, 1),
 			loaders: [{
+				path: "cache-loader",
+				query: {
+					cacheDirectory: path.join(__dirname, "..", ".cache"),
+				},
+			}, {
 				path: "ts-loader",
 				query: {
 					happyPackMode: true,
@@ -121,6 +127,14 @@ module.exports = (options = {}) => ({
 			"process.env.VERSION": `"${process.env.VERSION || ""}"`,
 		}),
 	],
+	optimization: {
+		minimizer: [
+			new TerserPlugin({
+				cache: path.join(__dirname, "..", ".cache", "terser"),
+				parallel: true,
+			}),
+		],
+	},
 	stats: {
 		all: false, // Fallback for options not defined.
 		errors: true,

yarn.lock

@@ -24,6 +24,13 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-10.12.18.tgz#1d3ca764718915584fcd9f6344621b7672665c67"
   integrity sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==
 
+"@types/tar@^4.0.0":
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/@types/tar/-/tar-4.0.0.tgz#e3239d969eeb693a012200613860d0eb871c94f0"
+  integrity sha512-YybbEHNngcHlIWVCYsoj7Oo1JU9JqONuAlt1LlTH/lmL8BMhbzdFUgReY87a05rY1j8mfK47Del+TCkaLAXwLw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/trash@^4.3.1":
   version "4.3.1"
   resolved "https://registry.yarnpkg.com/@types/trash/-/trash-4.3.1.tgz#4880ff17c4eb467f1a26774ea6328428403b5c57"
@@ -723,6 +730,17 @@ cache-base@^1.0.1:
     union-value "^1.0.0"
     unset-value "^1.0.0"
 
+cache-loader@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/cache-loader/-/cache-loader-2.0.1.tgz#5758f41a62d7c23941e3c3c7016e6faeb03acb07"
+  integrity sha512-V99T3FOynmGx26Zom+JrVBytLBsmUCzVG2/4NnUKgvXN4bEV42R1ERl1IyiH/cvFIDA1Ytq2lPZ9tXDSahcQpQ==
+  dependencies:
+    loader-utils "^1.1.0"
+    mkdirp "^0.5.1"
+    neo-async "^2.6.0"
+    normalize-path "^3.0.0"
+    schema-utils "^1.0.0"
+
 camel-case@3.0.x:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/camel-case/-/camel-case-3.0.0.tgz#ca3c3688a4e9cf3a4cda777dc4dcbc713249cf73"
@@ -916,6 +934,11 @@ commander@^2.12.1, commander@^2.18.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.19.0.tgz#f6198aa84e5b83c46054b94ddedbfed5ee9ff12a"
   integrity sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==
 
+commander@^2.19.0:
+  version "2.20.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
+  integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
+
 commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@@ -3305,7 +3328,7 @@ negotiator@0.6.1:
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
   integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=
 
-neo-async@^2.5.0:
+neo-async@^2.5.0, neo-async@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835"
   integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==
@@ -3452,6 +3475,11 @@ normalize-path@^2.1.1:
   dependencies:
     remove-trailing-separator "^1.0.1"
 
+normalize-path@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
+  integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
+
 npm-bundled@^1.0.1:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.5.tgz#3c1732b7ba936b3a10325aef616467c0ccbcc979"
@@ -4625,6 +4653,14 @@ source-map-support@^0.5.6, source-map-support@~0.5.6:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
+source-map-support@~0.5.10:
+  version "0.5.11"
+  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.11.tgz#efac2ce0800355d026326a0ca23e162aeac9a4e2"
+  integrity sha512-//sajEx/fGL3iw6fltKMdPvy8kL3kJ2O3iuYlRoT3k9Kb4BjOoZ+BZzaNHeuaruSt+Kf3Zk9tnfAQg9/AJqUVQ==
+  dependencies:
+    buffer-from "^1.0.0"
+    source-map "^0.6.0"
+
 source-map-url@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
@@ -4924,7 +4960,7 @@ tar@^2.0.0:
     fstream "^1.0.2"
     inherits "2"
 
-tar@^4:
+tar@^4, tar@^4.4.8:
   version "4.4.8"
   resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.8.tgz#b19eec3fde2a96e64666df9fdb40c5ca1bc3747d"
   integrity sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==
@@ -4951,6 +4987,29 @@ terser-webpack-plugin@^1.1.0:
     webpack-sources "^1.1.0"
     worker-farm "^1.5.2"
 
+terser-webpack-plugin@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/terser-webpack-plugin/-/terser-webpack-plugin-1.2.3.tgz#3f98bc902fac3e5d0de730869f50668561262ec8"
+  integrity sha512-GOK7q85oAb/5kE12fMuLdn2btOS9OBZn4VsecpHDywoUC/jLhSAKOiYo0ezx7ss2EXPMzyEWFoE0s1WLE+4+oA==
+  dependencies:
+    cacache "^11.0.2"
+    find-cache-dir "^2.0.0"
+    schema-utils "^1.0.0"
+    serialize-javascript "^1.4.0"
+    source-map "^0.6.1"
+    terser "^3.16.1"
+    webpack-sources "^1.1.0"
+    worker-farm "^1.5.2"
+
+terser@^3.16.1:
+  version "3.17.0"
+  resolved "https://registry.yarnpkg.com/terser/-/terser-3.17.0.tgz#f88ffbeda0deb5637f9d24b0da66f4e15ab10cb2"
+  integrity sha512-/FQzzPJmCpjAH9Xvk2paiWrFq+5M6aVOf+2KRbwhByISDX/EujxsK+BAvrhb6H+2rtrLCHK9N01wO014vrIwVQ==
+  dependencies:
+    commander "^2.19.0"
+    source-map "~0.6.1"
+    source-map-support "~0.5.10"
+
 terser@^3.8.1:
   version "3.14.1"
   resolved "https://registry.yarnpkg.com/terser/-/terser-3.14.1.tgz#cc4764014af570bc79c79742358bd46926018a32"