Getting the client to run (#12)

* Clean up workbench and integrate initialization data

* Uncomment Electron fill

* Run server & client together

* Clean up Electron fill & patch

* Bind fs methods

This makes them usable with the promise form:
`promisify(access)(...)`.

* Add space between tag and title to browser logger

* Add typescript dep to server and default __dirname for path

* Serve web files from server

* Adjust some dev options

* Rework workbench a bit to use a class and catch unexpected errors

* No mkdirs for now, fix util fill, use bash with exec

* More fills, make general client abstract

* More fills

* Fix cp.exec

* Fix require calls in fs fill being aliased

* Create data and storage dir

* Implement fs.watch

Using exec for now.

* Implement storage database fill

* Fix os export and homedir

* Add comment to use navigator.sendBeacon

* Fix fs callbacks (some args are optional)

* Make sure data directory exists when passing it back

* Update patch

* Target es5

* More fills

* Add APIs required for bootstrap-fork to function (#15)

* Add bootstrap-fork execution

* Add createConnection

* Bundle bootstrap-fork into cli

* Remove .node directory created from spdlog

* Fix npm start

* Remove unnecessary comment

* Add webpack-hot-middleware if CLI env is not set

* Add restarting to shared process

* Fix starting with yarn

packages/server/src/cli.ts

@@ -1,9 +1,12 @@
+import { SharedProcessInitMessage } from "@coder/protocol/src/proto";
 import { Command, flags } from "@oclif/command";
 import { logger, field } from "@coder/logger";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
-import { createApp } from './server';
+import { requireModule } from "./vscode/bootstrapFork";
+import { createApp } from "./server";
+import { SharedProcess } from './vscode/sharedProcess';
 
 export class Entry extends Command {
 
@@ -17,23 +20,65 @@ export class Entry extends Command {
 		open: flags.boolean({ char: "o", description: "Open in browser on startup" }),
 		port: flags.integer({ char: "p", default: 8080, description: "Port to bind on" }),
 		version: flags.version({ char: "v" }),
+
+		// Dev flags
+		"bootstrap-fork": flags.string({ hidden: true }),
 	};
 	public static args = [{
 		name: "workdir",
 		description: "Specify working dir",
-		default: () => process.cwd(),
+		default: (): string => process.cwd(),
 	}];
 
 	public async run(): Promise<void> {
+		try {
+			/**
+			 * Suuuper janky
+			 * Comes from - https://github.com/nexe/nexe/issues/524
+			 * Seems to cleanup by removing this path immediately
+			 * If any native module is added its assumed this pathname
+			 * will change.
+			 */
+			require("spdlog");
+			const nodePath = path.join(process.cwd(), "e91a410b");
+			fs.unlinkSync(path.join(nodePath, "spdlog.node"));
+			fs.rmdirSync(nodePath);
+		} catch (ex) {
+			logger.warn("Failed to remove extracted dependency.", field("dependency", "spdlog"), field("error", ex.message));
+		}
+
 		const { args, flags } = this.parse(Entry);
 
-		const dataDir = flags["data-dir"] || path.join(os.homedir(), `.vscode-online`);
+		if (flags["bootstrap-fork"]) {
+			const modulePath = flags["bootstrap-fork"];
+			if (!modulePath) {
+				logger.error("No module path specified to fork!");
+				process.exit(1);
+			}
+
+			requireModule(modulePath);
+			return;
+		}
+
+		const dataDir = flags["data-dir"] || path.join(os.homedir(), ".vscode-online");
 		const workingDir = args["workdir"];
 
 		logger.info("\u001B[1mvscode-remote v1.0.0");
 		// TODO: fill in appropriate doc url
 		logger.info("Additional documentation: https://coder.com/docs");
 		logger.info("Initializing", field("data-dir", dataDir), field("working-dir", workingDir));
+		const sharedProcess = new SharedProcess(dataDir);
+		logger.info("Starting shared process...", field("socket", sharedProcess.socketPath));
+		sharedProcess.onWillRestart(() => {
+			logger.info("Restarting shared process...");
+
+			sharedProcess.ready.then(() => {
+				logger.info("Shared process has restarted!");
+			});
+		});
+		sharedProcess.ready.then(() => {
+			logger.info("Shared process has started up!");
+		});
 
 		const app = createApp((app) => {
 			app.use((req, res, next) => {
@@ -43,17 +88,33 @@ export class Entry extends Command {
 
 				next();
 			});
+			if (process.env.CLI === "false" || !process.env.CLI) {
+				const webpackConfig = require(path.join(__dirname, "..", "..", "web", "webpack.dev.config.js"));
+				const compiler = require("webpack")(webpackConfig);
+				app.use(require("webpack-dev-middleware")(compiler, {
+					logger,
+					publicPath: webpackConfig.output.publicPath,
+					stats: webpackConfig.stats,
+				}));
+				app.use(require("webpack-hot-middleware")(compiler));
+			}
 		}, {
-			dataDirectory: dataDir,
-			workingDirectory: workingDir,
-		});
+				dataDirectory: dataDir,
+				workingDirectory: workingDir,
+			});
 
-		logger.info("Starting webserver...", field("host", flags.host), field("port", flags.port))
+		logger.info("Starting webserver...", field("host", flags.host), field("port", flags.port));
 		app.server.listen(flags.port, flags.host);
 		let clientId = 1;
 		app.wss.on("connection", (ws, req) => {
 			const id = clientId++;
-			logger.info(`WebSocket opened \u001B[0m${req.url}`, field("client", id), field("ip", req.socket.remoteAddress));
+			const spm = (<any>req).sharedProcessInit as SharedProcessInitMessage;
+			if (!spm) {
+				logger.warn("Received a socket without init data. Not sure how this happened.");
+
+				return;
+			}
+			logger.info(`WebSocket opened \u001B[0m${req.url}`, field("client", id), field("ip", req.socket.remoteAddress), field("window_id", spm.getWindowId()), field("log_directory", spm.getLogDirectory()));
 
 			ws.on("close", (code) => {
 				logger.info(`WebSocket closed \u001B[0m${req.url}`, field("client", id), field("code", code));
@@ -73,9 +134,10 @@ export class Entry extends Command {
 		logger.info(`http://localhost:${flags.port}/`);
 		logger.info(" ");
 	}
+
 }
 
 Entry.run(undefined, {
-	root: process.env.BUILD_DIR as string,
+	root: process.env.BUILD_DIR as string || __dirname,
 	//@ts-ignore
 }).catch(require("@oclif/errors/handle"));

packages/server/src/ipc.ts

@@ -0,0 +1,70 @@
+import { EventEmitter } from "events";
+import { ChildProcess } from "child_process";
+
+export interface IpcMessage {
+	readonly event: string;
+	readonly args: any[];
+}
+
+export class StdioIpcHandler extends EventEmitter {
+
+	private isListening: boolean = false;
+
+	public constructor(
+		private readonly childProcess?: ChildProcess,
+	) {
+		super();
+	}
+
+	public on(event: string, cb: (...args: any[]) => void): this {
+		this.listen();
+		return super.on(event, cb);
+	}
+
+	public once(event: string, cb: (...args: any[]) => void): this {
+		this.listen();
+		return super.once(event, cb);
+	}
+
+	public addListener(event: string, cb: (...args: any[]) => void): this {
+		this.listen();
+		return super.addListener(event, cb);
+	}
+
+	public send(event: string, ...args: any[]): void {
+		const msg: IpcMessage = {
+			event,
+			args,
+		};
+		const d = JSON.stringify(msg);
+		if (this.childProcess) {
+			this.childProcess.stdin.write(d + "\n");
+		} else {
+			process.stdout.write(d);
+		}
+	}
+
+	private listen(): void {
+		if (this.isListening) {
+			return;
+		}
+		const onData = (data: any) => {
+			try {
+				const d = JSON.parse(data.toString()) as IpcMessage;
+				this.emit(d.event, ...d.args);
+			} catch (ex) {
+				if (!this.childProcess) {
+					process.stderr.write(`Failed to parse incoming data: ${ex.message}`);
+				}
+			}
+		};
+		if (this.childProcess) {
+			this.childProcess.stdout.resume();
+			this.childProcess.stdout.on("data", onData);
+		} else {
+			process.stdin.resume();
+			process.stdin.on("data", onData);
+		}
+	}
+
+}

packages/server/src/server.ts

@@ -3,34 +3,67 @@ import { Server, ServerOptions } from "@coder/protocol/src/node/server";
 import * as express from "express";
 import * as http from "http";
 import * as ws from "ws";
+import * as url from "url";
+import { ClientMessage, SharedProcessInitMessage } from '@coder/protocol/src/proto';
 
 export const createApp = (registerMiddleware?: (app: express.Application) => void, options?: ServerOptions): {
 	readonly express: express.Application;
 	readonly server: http.Server;
 	readonly wss: ws.Server;
- } => {
+} => {
 	const app = express();
 	if (registerMiddleware) {
 		registerMiddleware(app);
 	}
 	const server = http.createServer(app);
 	const wss = new ws.Server({ server });
-	
-	wss.on("connection", (ws: WebSocket) => {
+
+	wss.shouldHandle = (req): boolean => {
+		if (typeof req.url === "undefined") {
+			return false;
+		}
+
+		const parsedUrl = url.parse(req.url, true);
+		const sharedProcessInit = parsedUrl.query["shared_process_init"];
+		if (typeof sharedProcessInit === "undefined" || Array.isArray(sharedProcessInit)) {
+			return false;
+		}
+
+		try {
+			const msg = ClientMessage.deserializeBinary(Buffer.from(sharedProcessInit, "base64"));
+			if (!msg.hasSharedProcessInit()) {
+				return false;
+			}
+			const spm = msg.getSharedProcessInit()!;
+			(<any>req).sharedProcessInit = spm;
+		} catch (ex) {
+			return false;
+		}
+
+		return true;
+	};
+
+	wss.on("connection", (ws: WebSocket, req) => {
+		const spm = (<any>req).sharedProcessInit as SharedProcessInitMessage;
+		if (!spm) {
+			ws.close();
+			return;
+		}
+
 		const connection: ReadWriteConnection = {
-			onMessage: (cb) => {
+			onMessage: (cb): void => {
 				ws.addEventListener("message", (event) => cb(event.data));
 			},
-			close: () => ws.close(),
-			send: (data) => ws.send(data),
-			onClose: (cb) => ws.addEventListener("close", () => cb()),
+			close: (): void => ws.close(),
+			send: (data): void => ws.send(data),
+			onClose: (cb): void => ws.addEventListener("close", () => cb()),
 		};
-	
+
 		const server = new Server(connection, options);
 	});
 
 	/**
-	 * We should static-serve the `web` package at this point
+	 * We should static-serve the `web` package at this point.
 	 */
 	app.get("/", (req, res, next) => {
 		res.write("Example! :)");

packages/server/src/vscode/bootstrapFork.ts

@@ -0,0 +1,29 @@
+import * as cp from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+
+declare var __non_webpack_require__: typeof require;
+
+export const requireModule = (modulePath: string): void => {
+	process.env.AMD_ENTRYPOINT = modulePath;
+	process.env.VSCODE_ALLOW_IO = "true";
+	const content = fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build/bootstrap-fork.js"));
+	eval(content.toString());
+};
+
+/**
+ * Uses the internal bootstrap-fork.js to load a module
+ * @example
+ * const cp = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain");
+ * cp.stdout.on("data", (data) => console.log(data.toString("utf8")));
+ * cp.stderr.on("data", (data) => console.log(data.toString("utf8")));
+ * @param modulePath
+ */
+export const forkModule = (modulePath: string): cp.ChildProcess => {
+	const args = ["--bootstrap-fork", modulePath];
+	if (process.env.CLI === "true") {
+		return cp.spawn(process.execPath, args);
+	} else {
+		return cp.spawn("npm", ["start", "--scripts-prepend-node-path", "--", ...args]);
+	}
+};

packages/server/src/vscode/sharedProcess.ts

@@ -0,0 +1,106 @@
+import { ChildProcess } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { forkModule } from "./bootstrapFork";
+import { StdioIpcHandler } from "../ipc";
+import { logger, field } from "@coder/logger/src";
+import { ParsedArgs } from "vs/platform/environment/common/environment";
+import { LogLevel } from "vs/platform/log/common/log";
+import { Emitter, Event } from '@coder/events/src';
+
+export class SharedProcess {
+	public readonly socketPath: string = path.join(os.tmpdir(), `.vscode-online${Math.random().toString()}`);
+	private _ready: Promise<void> | undefined;
+	private activeProcess: ChildProcess | undefined;
+	private ipcHandler: StdioIpcHandler | undefined;
+	private readonly willRestartEmitter: Emitter<void>;
+
+	public constructor(
+		private readonly userDataDir: string,
+	) {
+		this.willRestartEmitter = new Emitter();
+
+		this.restart();
+	}
+
+	public get onWillRestart(): Event<void> {
+		return this.willRestartEmitter.event;
+	}
+
+	public get ready(): Promise<void> {
+		return this._ready!;
+	}
+
+	public restart(): void {
+		if (this.activeProcess) {
+			this.willRestartEmitter.emit();
+		}
+
+		if (this.activeProcess && !this.activeProcess.killed) {
+			this.activeProcess.kill();
+		}
+
+		let resolve: () => void;
+		let reject: (err: Error) => void;
+
+		const extensionsDir = path.join(this.userDataDir, "extensions");
+		const mkdir = (dir: string): void => {
+			try {
+				fs.mkdirSync(dir);
+			} catch (ex) {
+				if (ex.code !== "EEXIST") {
+					throw ex;
+				}
+			}
+		};
+		mkdir(this.userDataDir);
+		mkdir(extensionsDir);
+
+		this._ready = new Promise<void>((res, rej) => {
+			resolve = res;
+			reject = rej;
+		});
+
+		let resolved: boolean = false;
+		this.activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain");
+		this.activeProcess.on("exit", () => {
+			this.restart();
+		});
+		this.ipcHandler = new StdioIpcHandler(this.activeProcess);
+		this.ipcHandler.once("handshake:hello", () => {
+			const data: {
+				sharedIPCHandle: string;
+				args: ParsedArgs;
+				logLevel: LogLevel;
+			} = {
+				args: {
+					"builtin-extensions-dir": path.join(process.env.BUILD_DIR || path.join(__dirname, "../.."), "build/extensions"),
+					"user-data-dir": this.userDataDir,
+					"extensions-dir": extensionsDir,
+				} as any,
+				logLevel: 0,
+				sharedIPCHandle: this.socketPath,
+			};
+			this.ipcHandler!.send("handshake:hey there", "", data);
+		});
+		this.ipcHandler.once("handshake:im ready", () => {
+			resolved = true;
+			resolve();
+		});
+		this.activeProcess.stderr.on("data", (data) => {
+			if (!resolved) {
+				reject(data.toString());
+			} else {
+				logger.named("SHRD PROC").debug("stderr", field("message", data.toString()));
+			}
+		});
+	}
+
+	public dispose(): void {
+		if (this.ipcHandler) {
+			this.ipcHandler.send("handshake:goodbye");
+		}
+		this.ipcHandler = undefined;
+	}
+}