Fix loading within the CLI (#27)

* Fix loading within the CLI

* Remove app

* Remove promise handle

* Fix requested changes

packages/server/src/cli.ts

@@ -8,6 +8,8 @@ import * as WebSocket from "ws";
 import { createApp } from "./server";
 import { requireModule } from "./vscode/bootstrapFork";
 import { SharedProcess, SharedProcessState } from "./vscode/sharedProcess";
+import { setup as setupNativeModules } from './modules';
+import { fillFs } from './fill';
 
 export class Entry extends Command {
 
@@ -49,12 +51,17 @@ export class Entry extends Command {
 			logger.warn("Failed to remove extracted dependency.", field("dependency", "spdlog"), field("error", ex.message));
 		}
 
+		if (process.env.CLI) {
+			fillFs();
+		}
+
 		const { args, flags } = this.parse(Entry);
 
 		if (flags.env) {
 			Object.assign(process.env, JSON.parse(flags.env));
 		}
 
+		const builtInExtensionsDir = path.join(process.env.BUILD_DIR || path.join(__dirname, ".."), "build/extensions");
 		if (flags["bootstrap-fork"]) {
 			const modulePath = flags["bootstrap-fork"];
 			if (!modulePath) {
@@ -62,7 +69,7 @@ export class Entry extends Command {
 				process.exit(1);
 			}
 
-			return requireModule(modulePath);
+			return requireModule(modulePath, builtInExtensionsDir);
 		}
 
 		const dataDir = flags["data-dir"] || path.join(os.homedir(), ".vscode-online");
@@ -73,6 +80,11 @@ export class Entry extends Command {
 			process.exit(1);
 		}
 
+		if (!fs.existsSync(dataDir)) {
+			fs.mkdirSync(dataDir);
+		}
+		setupNativeModules(dataDir);
+
 		const logDir = path.join(dataDir, "logs", new Date().toISOString().replace(/[-:.TZ]/g, ""));
 		process.env.VSCODE_LOGS = logDir;
 
@@ -80,7 +92,7 @@ export class Entry extends Command {
 		// TODO: fill in appropriate doc url
 		logger.info("Additional documentation: https://coder.com/docs");
 		logger.info("Initializing", field("data-dir", dataDir), field("working-dir", workingDir), field("log-dir", logDir));
-		const sharedProcess = new SharedProcess(dataDir);
+		const sharedProcess = new SharedProcess(dataDir, builtInExtensionsDir);
 		logger.info("Starting shared process...", field("socket", sharedProcess.socketPath));
 		const sendSharedProcessReady = (socket: WebSocket): void => {
 			const active = new SharedProcessActiveMessage();
@@ -120,6 +132,7 @@ export class Entry extends Command {
 				app.use(require("webpack-hot-middleware")(compiler));
 			}
 		}, {
+				builtInExtensionsDirectory: builtInExtensionsDir,
 				dataDirectory: dataDir,
 				workingDirectory: workingDir,
 			});

packages/server/src/fill.ts

@@ -0,0 +1,154 @@
+import * as fs from "fs";
+import * as util from "util";
+
+const oldAccess = fs.access;
+const existsWithinBinary = (path: fs.PathLike): Promise<boolean> => {
+	return new Promise<boolean>((resolve) => {
+		if (typeof path === "number") {
+			if (path < 0) {
+				return resolve(true);
+			}
+		}
+		oldAccess(path, fs.constants.F_OK, (err) => {
+			const exists = !err;
+			const es = fs.existsSync(path);
+			const res = !exists && es;
+			resolve(res);
+		});
+	});
+};
+
+export const fillFs = () => {
+	/**
+	 * Refer to https://github.com/nexe/nexe/blob/master/src/fs/patch.ts
+	 * For impls
+	 */
+
+	if (!process.env.CLI) {
+		throw new Error("Should not fill FS when not in CLI");
+	}
+
+	interface FD {
+		readonly path: string;
+		position: number;
+	}
+
+	let lastFd = Number.MIN_SAFE_INTEGER;
+	const fds = new Map<number, FD>();
+
+	const replaceNative = <T extends keyof typeof fs>(propertyName: T, func: (callOld: () => void, ...args: any[]) => any, customPromisify?: (...args: any[]) => Promise<any>): void => {
+		const oldFunc = (<any>fs)[propertyName];
+		fs[propertyName] = (...args: any[]) => {
+			try {
+				return func(() => {
+					return oldFunc(...args);
+				}, ...args);
+			} catch (ex) {
+				return oldFunc(...args);
+			}
+		};
+		if (customPromisify) {
+			(<any>fs[propertyName])[util.promisify.custom] = customPromisify;
+		}
+	};
+
+	replaceNative("access", (callNative, path, mode, callback) => {
+		existsWithinBinary(path).then((exists) => {
+			if (!exists) {
+				return callNative();
+			}
+
+			return callback();
+		});
+	});
+
+	replaceNative("exists", (callOld, path, callback) => {
+		existsWithinBinary(path).then((exists) => {
+			if (exists) {
+				return callback(true);
+			}
+
+			return callOld();
+		});
+	}, (path) => new Promise((res) => fs.exists(path, res)));
+
+	replaceNative("open", (callOld, path: fs.PathLike, flags: string | Number, mode: any, callback: any) => {
+		existsWithinBinary(path).then((exists) => {
+			if (!exists) {
+				return callOld();
+			}
+
+			if (typeof mode === "function") {
+				callback = mode;
+				mode = undefined;
+			}
+
+			if (path === process.execPath) {
+				return callOld();
+			}
+
+			const fd = lastFd++;
+			fds.set(fd, {
+				path: path.toString(),
+				position: 0,
+			});
+			callback(undefined, fd);
+		});
+	});
+
+	replaceNative("close", (callOld, fd: number, callback) => {
+		if (!fds.has(fd)) {
+			return callOld();
+		}
+
+		fds.delete(fd);
+		callback();
+	});
+
+	replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void, ) => {
+		if (!fds.has(fd)) {
+			return callOld();
+		}
+
+		const fileDesc = fds.get(fd)!;
+
+		/**
+		 * `readFile` is filled within nexe, but `read` is not
+		 * https://github.com/nexe/nexe/blob/master/src/fs/patch.ts#L199
+		 * We can simulate a real _read_ by reading the entire file.
+		 * Efficiency can be improved here by storing the entire file in memory
+		 * until it has been closed.
+		 */
+		return fs.readFile(fileDesc.path, (err, rb) => {
+			if (err) {
+				return callOld();
+			}
+
+			rb = rb.slice(position || fileDesc.position);
+			const sliced = rb.slice(0, length);
+			if (position === null) {
+				fileDesc.position += sliced.byteLength;
+			}
+			buffer.set(sliced, offset);
+			if (callback) {
+				callback(undefined!, sliced.byteLength, buffer);
+			}
+		});
+	}, (fd: number, buffer: Buffer, offset: number, length: number, position: number | null): Promise<{
+		bytesRead: number;
+		buffer: Buffer;
+	}> => {
+			return new Promise((res, rej) => {
+				fs.read(fd, buffer, offset, length, position, (err, bytesRead, buffer) => {
+					if (err) {
+						return rej(err);
+					}
+
+					res({
+						bytesRead,
+						buffer,
+					});
+				});
+			});
+		});
+};

packages/server/src/modules.ts

@@ -0,0 +1,41 @@
+import * as fs from "fs";
+import * as path from "path";
+
+declare var __non_webpack_require__: typeof require;
+
+/**
+ * Handling of native modules within the CLI
+ */
+export const setup = (dataDirectory: string): void => {
+	if (!process.env.CLI) {
+		return;
+	}
+
+	try {
+		fs.mkdirSync(path.join(dataDirectory, "modules"));
+	} catch (ex) {
+		if (ex.code !== "EEXIST") {
+			throw ex;
+		}
+	}
+
+	const unpackModule = (moduleName: string): void => {
+		const memFile = path.join(process.env.BUILD_DIR!, "build/modules", moduleName + ".node");
+		const diskFile = path.join(dataDirectory, "modules", moduleName + ".node");
+		if (!fs.existsSync(diskFile)) {
+			fs.writeFileSync(diskFile, fs.readFileSync(memFile));
+		}
+	};
+
+	/**
+	 * We need to unpack node-pty and patch its `loadNative` function to require our unpacked pty.node
+	 * If pty.node isn't unpacked a SIGSEGV is thrown and the application exits. The exact reasoning
+	 * for this is unknown ATM, but this patch works around it.
+	 */
+	unpackModule("pty");
+	const nodePtyUtils = require("../../protocol/node_modules/node-pty/lib/utils") as typeof import("../../protocol/node_modules/node-pty/src/utils");
+	nodePtyUtils.loadNative = (modName: string) => {
+		return __non_webpack_require__(path.join(dataDirectory, "modules", modName + ".node"));
+	};
+	require("../../protocol/node_modules/node-pty/lib/index") as typeof import("../../protocol/node_modules/node-pty/src/index");
+};

packages/server/src/server.ts

@@ -4,6 +4,8 @@ import { Server, ServerOptions } from "@coder/protocol/src/node/server";
 import { NewSessionMessage } from '@coder/protocol/src/proto';
 import { ChildProcess } from "child_process";
 import * as express from "express";
+//@ts-ignore
+import * as expressStaticGzip from "express-static-gzip";
 import * as fs from "fs";
 import * as http from "http";
 import * as mime from "mime-types";
@@ -67,8 +69,12 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 		} : undefined);
 	});
 
-	app.use(express.static(path.join(process.env.BUILD_DIR || path.join(__dirname, ".."), "build/web")));
-
+	const baseDir = process.env.BUILD_DIR || path.join(__dirname, "..");
+	if (process.env.CLI) {
+		app.use(expressStaticGzip(path.join(baseDir, "build/web")));
+	} else {
+		app.use(express.static(path.join(baseDir, "resources/web")));
+	}
 	app.get("/resource/:url(*)", async (req, res) => {
 		try {
 			const fullPath = `/${req.params.url}`;
@@ -76,7 +82,7 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 			// if (relative.startsWith("..")) {
 			// 	return res.status(403).end();
 			// }
-			const exists = await util.promisify(fs.exists)(fullPath);
+			const exists = fs.existsSync(fullPath);
 			if (!exists) {
 				res.status(404).end();
 				return;

packages/server/src/vscode/bootstrapFork.ts

@@ -1,19 +1,50 @@
 import * as cp from "child_process";
 import * as fs from "fs";
 import * as path from "path";
+import * as zlib from "zlib";
+import * as vm from "vm";
 
-export const requireModule = (modulePath: string): void => {
+export const requireModule = (modulePath: string, builtInExtensionsDir: string): void => {
 	process.env.AMD_ENTRYPOINT = modulePath;
-
 	const xml = require("xhr2");
-
 	// tslint:disable-next-line no-any this makes installing extensions work.
 	(global as any).XMLHttpRequest = xml.XMLHttpRequest;
 
+	const mod = require("module") as typeof import("module");
+	/**
+	 * Used for loading extensions. Using __non_webpack_require__ didn't work
+	 * as it was not resolving to the FS.
+	 */
+	(global as any).nativeNodeRequire = (id: string) => {
+		const customMod = new mod.Module(id);
+		customMod.filename = id;
+		customMod.paths = (<any>mod)._nodeModulePaths(path.dirname(id));
+
+		if (id.startsWith(builtInExtensionsDir)) {
+			customMod.loaded = true;
+			const req = vm.runInThisContext(mod.wrap(fs.readFileSync(id + ".js").toString()), {
+				displayErrors: true,
+				filename: id + ".js",
+			});
+			req(customMod.exports, customMod.require.bind(customMod), customMod, __filename, path.dirname(id));
+			return customMod.exports;
+		}
+
+		return customMod.require(id);
+	};
+
 	// Always do this so we can see console.logs.
 	// process.env.VSCODE_ALLOW_IO = "true";
 
-	const content = fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build/bootstrap-fork.js"));
+	let content: Buffer | undefined;
+	const readFile = (name: string): Buffer => {
+		return fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build", name));
+	};
+	if (process.env.CLI) {
+		content = zlib.gunzipSync(readFile("bootstrap-fork.js.gz"));
+	} else {
+		content = readFile("../resources/bootstrap-fork.js");
+	}
 	eval(content.toString());
 };
 
@@ -36,9 +67,9 @@ export const forkModule = (modulePath: string, env?: NodeJS.ProcessEnv): cp.Chil
 		stdio: [null, null, null, "ipc"],
 	};
 	if (process.env.CLI === "true") {
-		proc = cp.spawn(process.execPath, args, options);
+		proc = cp.execFile(process.execPath, args, options);
 	} else {
-		proc = cp.spawn(process.execArgv[0], ["-r", "tsconfig-paths/register", process.argv[1], ...args], options);
+		proc = cp.spawn(process.execPath, ["--require", "ts-node/register", "--require", "tsconfig-paths/register", process.argv[1], ...args], options);
 	}
 
 	return proc;

packages/server/src/vscode/sharedProcess.ts

@@ -31,6 +31,7 @@ export class SharedProcess {
 
 	public constructor(
 		private readonly userDataDir: string,
+		private readonly builtInExtensionsDir: string,
 	) {
 		this.onStateEmitter = new Emitter();
 		this.restart();
@@ -87,7 +88,7 @@ export class SharedProcess {
 				logLevel: LogLevel;
 			} = {
 				args: {
-					"builtin-extensions-dir": path.join(process.env.BUILD_DIR || path.join(__dirname, "../.."), "build/extensions"),
+					"builtin-extensions-dir": this.builtInExtensionsDir,
 					"user-data-dir": this.userDataDir,
 					"extensions-dir": extensionsDir,
 				} as any,