Featureful (#31)

* Fix loading within the CLI

* Remove app

* Remove promise handle

* Add initial travis file

* Add libxkbfile dependency

* Add libxkbfile-dev

* Add build script

* Fix malformed bash statement

* Remove yarn from script

* Improve build script

* Extract upx before usage

* Only run upx if on linux

* Ensure resource directory exists

* Pack runnable binary

* Export binary with platform

* Improve build process

* Install upx before running install script

* Update typescript version before running nexe

* Add os.release() function for multi-platform support

* Update travis.yml to improve deployment

* Add on CI

* Update to v1.31.0

* Add libsecret

* Update build target

* Skip cleanup

* Fix built-in extensions

* Add basics for apps

* Create custom DNS server

* Fix forking within CLI. Fixes TS language features

* Fix filename resolve

* Fix default extensions path

* Add custom dialog

* Store workspace path

* Remove outfiles

* Cleanup

* Always authed outside of CLI

* Use location.host for client

* Remove useless app interface

* Remove debug file for building wordlist

* Use chromes tcp host

* Update patch

* Build browser app before packaging

* Replace all css containing file:// URLs, fix webviews

* Fix save

* Fix mkdir

packages/server/src/cli.ts

@@ -6,7 +6,7 @@ import * as os from "os";
 import * as path from "path";
 import * as WebSocket from "ws";
 import { createApp } from "./server";
-import { requireModule } from "./vscode/bootstrapFork";
+import { requireModule, requireFork } from "./vscode/bootstrapFork";
 import { SharedProcess, SharedProcessState } from "./vscode/sharedProcess";
 import { setup as setupNativeModules } from "./modules";
 import { fillFs } from "./fill";
@@ -26,6 +26,8 @@ export class Entry extends Command {
 
 		// Dev flags
 		"bootstrap-fork": flags.string({ hidden: true }),
+		"fork": flags.string({ hidden: true }),
+
 		env: flags.string({ hidden: true }),
 		args: flags.string({ hidden: true }),
 	};
@@ -76,6 +78,12 @@ export class Entry extends Command {
 			return requireModule(modulePath, builtInExtensionsDir);
 		}
 
+		if (flags["fork"]) {
+			const modulePath = flags["fork"];
+
+			return requireFork(modulePath, JSON.parse(flags.args!), builtInExtensionsDir);
+		}
+
 		const dataDir = flags["data-dir"] || path.join(os.homedir(), ".vscode-remote");
 		const workingDir = args["workdir"];
 
@@ -92,6 +100,38 @@ export class Entry extends Command {
 		const logDir = path.join(dataDir, "logs", new Date().toISOString().replace(/[-:.TZ]/g, ""));
 		process.env.VSCODE_LOGS = logDir;
 
+		const certPath = flags.cert;
+		const certKeyPath = flags["cert-key"];
+
+		if (certPath && !certKeyPath) {
+			logger.error("'--cert-key' flag is required when specifying a certificate!");
+			process.exit(1);
+		}
+
+		if (!certPath && certKeyPath) {
+			logger.error("'--cert' flag is required when specifying certificate key!");
+			process.exit(1);
+		}
+
+		let certData: Buffer | undefined;
+		let certKeyData: Buffer | undefined;
+
+		if (typeof certPath !== "undefined" && typeof certKeyPath !== "undefined") {
+			try {
+				certData = fs.readFileSync(certPath);
+			} catch (ex) {
+				logger.error(`Failed to read certificate: ${ex.message}`);
+				process.exit(1);
+			}
+
+			try {
+				certKeyData = fs.readFileSync(certKeyPath);
+			} catch (ex) {
+				logger.error(`Failed to read certificate key: ${ex.message}`);
+				process.exit(1);
+			}
+		}
+
 		logger.info("\u001B[1mvscode-remote v1.0.0");
 		// TODO: fill in appropriate doc url
 		logger.info("Additional documentation: https://coder.com/docs");
@@ -111,7 +151,9 @@ export class Entry extends Command {
 			}
 		});
 
-		const app = createApp((app) => {
+		const password = "023450wf0951";
+		const hasCustomHttps = certData && certKeyData;
+		const app = await createApp((app) => {
 			app.use((req, res, next) => {
 				res.on("finish", () => {
 					logger.trace(`\u001B[1m${req.method} ${res.statusCode} \u001B[0m${req.url}`, field("host", req.hostname), field("ip", req.ip));
@@ -132,10 +174,13 @@ export class Entry extends Command {
 				app.use(require("webpack-hot-middleware")(compiler));
 			}
 		}, {
-			builtInExtensionsDirectory: builtInExtensionsDir,
-			dataDirectory: dataDir,
-			workingDirectory: workingDir,
-		});
+				builtInExtensionsDirectory: builtInExtensionsDir,
+				dataDirectory: dataDir,
+				workingDirectory: workingDir,
+			}, password, hasCustomHttps ? {
+				key: certKeyData,
+				cert: certData,
+			} : undefined);
 
 		logger.info("Starting webserver...", field("host", flags.host), field("port", flags.port));
 		app.server.listen(flags.port, flags.host);
@@ -161,7 +206,7 @@ export class Entry extends Command {
 		}
 
 		logger.info(" ");
-		logger.info("Password:\u001B[1m 023450wf09");
+		logger.info(`Password:\u001B[1m ${password}`);
 		logger.info(" ");
 		logger.info("Started (click the link below to open):");
 		logger.info(`http://localhost:${flags.port}/`);

packages/server/src/fill.ts

@@ -49,7 +49,11 @@ export const fillFs = (): void => {
 			}
 		};
 		if (customPromisify) {
-			(<any>fs[propertyName])[util.promisify.custom] = customPromisify;
+			(<any>fs[propertyName])[util.promisify.custom] = (...args: any[]) => {
+				return customPromisify(...args).catch((ex) => {
+					throw ex;
+				});
+			};
 		}
 	};
 
@@ -113,13 +117,6 @@ export const fillFs = (): void => {
 
 		const fileDesc = fds.get(fd)!;
 
-		/**
-		 * `readFile` is filled within nexe, but `read` is not
-		 * https://github.com/nexe/nexe/blob/master/src/fs/patch.ts#L199
-		 * We can simulate a real _read_ by reading the entire file.
-		 * Efficiency can be improved here by storing the entire file in memory
-		 * until it has been closed.
-		 */
 		return fs.readFile(fileDesc.path, (err, rb) => {
 			if (err) {
 				return callOld();

packages/server/src/modules.ts

@@ -35,8 +35,10 @@ export const setup = (dataDirectory: string): void => {
 	 */
 	unpackModule("pty");
 	const nodePtyUtils = require("../../protocol/node_modules/node-pty/lib/utils") as typeof import("../../protocol/node_modules/node-pty/src/utils");
-	nodePtyUtils.loadNative = (modName: string) => {
+	// tslint:disable-next-line:no-any
+	nodePtyUtils.loadNative = (modName: string): any => {
 		return __non_webpack_require__(path.join(dataDirectory, "modules", modName + ".node"));
 	};
+	// tslint:disable-next-line:no-unused-expression
 	require("../../protocol/node_modules/node-pty/lib/index") as typeof import("../../protocol/node_modules/node-pty/src/index");
 };

packages/server/src/portScanner.ts

@@ -0,0 +1,105 @@
+//@ts-ignore
+import * as netstat from "node-netstat";
+import { Event, Emitter } from "@coder/events";
+
+export interface PortScanner {
+	readonly ports: ReadonlyArray<number>;
+
+	readonly onAdded: Event<ReadonlyArray<number>>;
+	readonly onRemoved: Event<ReadonlyArray<number>>;
+
+	dispose(): void;
+}
+
+/**
+ * Creates a disposable port scanner.
+ * Will scan local ports and emit events when ports are added or removed.
+ * Currently only scans TCP ports.
+ */
+export const createPortScanner = (scanInterval: number = 250): PortScanner => {
+	const ports = new Map<number, number>();
+
+	const addEmitter = new Emitter<number[]>();
+	const removeEmitter = new Emitter<number[]>();
+
+	const scan = (onCompleted: (err?: Error) => void): void => {
+		const scanTime = Date.now();
+		const added: number[] = [];
+		netstat({
+			done: (err: Error): void => {
+				const removed: number[] = [];
+				ports.forEach((value, key) => {
+					if (value !== scanTime) {
+						// Remove port
+						removed.push(key);
+						ports.delete(key);
+					}
+				});
+				if (removed.length > 0) {
+					removeEmitter.emit(removed);
+				}
+
+				if (added.length > 0) {
+					addEmitter.emit(added);
+				}
+
+				onCompleted(err);
+			},
+			filter: {
+				state: "LISTEN",
+			},
+		}, (data: {
+			readonly protocol: string;
+			readonly local: {
+				readonly port: number;
+				readonly address: string;
+			};
+		}) => {
+				// https://en.wikipedia.org/wiki/Registered_port
+				if (data.local.port <= 1023 || data.local.port >= 49151) {
+					return;
+				}
+				// Only forward TCP ports
+				if (!data.protocol.startsWith("tcp")) {
+					return;
+				}
+
+				if (!ports.has(data.local.port)) {
+					added.push(data.local.port);
+				}
+				ports.set(data.local.port, scanTime);
+			});
+	};
+
+	let lastTimeout: NodeJS.Timer | undefined;
+	let disposed: boolean = false;
+
+	const doInterval = (): void => {
+		scan(() => {
+			if (disposed) {
+				return;
+			}
+			lastTimeout = setTimeout(doInterval, scanInterval);
+		});
+	};
+
+	doInterval();
+
+	return {
+		get ports(): number[] {
+			return Array.from(ports.keys());
+		},
+		get onAdded(): Event<number[]> {
+			return addEmitter.event;
+		},
+		get onRemoved(): Event<number[]> {
+			return removeEmitter.event;
+		},
+		dispose(): void {
+			if (typeof lastTimeout !== "undefined") {
+				clearTimeout(lastTimeout);
+			}
+			disposed = true;
+		},
+	};
+};

packages/server/src/server.ts

@@ -1,4 +1,4 @@
-import { logger } from "@coder/logger";
+import { logger, field } from "@coder/logger";
 import { ReadWriteConnection } from "@coder/protocol";
 import { Server, ServerOptions } from "@coder/protocol/src/node/server";
 import * as express from "express";
@@ -6,30 +6,116 @@ import * as express from "express";
 import * as expressStaticGzip from "express-static-gzip";
 import * as fs from "fs";
 import * as http from "http";
+//@ts-ignore
+import * as httpolyglot from "httpolyglot";
+import * as https from "https";
 import * as mime from "mime-types";
+import * as net from "net";
 import * as path from "path";
+import * as pem from "pem";
 import * as util from "util";
 import * as ws from "ws";
-import { isCli, buildDir } from "./constants";
+import { TunnelCloseCode } from "@coder/tunnel/src/common";
+import { handle as handleTunnel } from "@coder/tunnel/src/server";
+import { createPortScanner } from "./portScanner";
+import { buildDir, isCli } from "./constants";
 
-export const createApp = (registerMiddleware?: (app: express.Application) => void, options?: ServerOptions): {
+export const createApp = async (registerMiddleware?: (app: express.Application) => void, options?: ServerOptions, password?: string, httpsOptions?: https.ServerOptions): Promise<{
 	readonly express: express.Application;
 	readonly server: http.Server;
 	readonly wss: ws.Server;
-} => {
+}> => {
+	const parseCookies = (req: http.IncomingMessage): { [key: string]: string } => {
+		const cookies: { [key: string]: string } = {};
+		const rc = req.headers.cookie;
+		if (rc) {
+			rc.split(";").forEach((cook) => {
+				const parts = cook.split("=");
+				cookies[parts.shift()!.trim()] = decodeURI(parts.join("="));
+			});
+		}
+
+		return cookies;
+	};
+
+	const isAuthed = (req: http.IncomingMessage): boolean => {
+		try {
+			if (!password || !isCli) {
+				return true;
+			}
+
+			// Try/catch placed here just in case
+			const cookies = parseCookies(req);
+			if (cookies.password && cookies.password === password) {
+				return true;
+			}
+		} catch (ex) {
+			logger.error("Failed to parse cookies", field("error", ex));
+		}
+
+		return false;
+	};
+
+	const isEncrypted = (socket: net.Socket): boolean => {
+		// tslint:disable-next-line:no-any
+		return (socket as any).encrypted;
+	};
+
 	const app = express();
 	if (registerMiddleware) {
 		registerMiddleware(app);
 	}
-	const server = http.createServer(app);
+
+	const certs = await new Promise<pem.CertificateCreationResult>((res, rej): void => {
+		pem.createCertificate({
+			selfSigned: true,
+		}, (err, result) => {
+			if (err) {
+				rej(err);
+
+				return;
+			}
+
+			res(result);
+		});
+	});
+
+	const server = httpolyglot.createServer({
+		key: certs.serviceKey,
+		cert: certs.certificate,
+	}, app) as http.Server;
 	const wss = new ws.Server({ server });
 
 	wss.shouldHandle = (req): boolean => {
-		// Should handle auth here
-		return true;
+		return isAuthed(req);
 	};
 
-	wss.on("connection", (ws) => {
+	const portScanner = createPortScanner();
+	wss.on("connection", (ws, req) => {
+		if (req.url && req.url.startsWith("/tunnel")) {
+			try {
+				const rawPort = req.url.split("/").pop();
+				const port = Number.parseInt(rawPort!, 10);
+
+				handleTunnel(ws, port);
+			} catch (ex) {
+				ws.close(TunnelCloseCode.Error, ex.toString());
+			}
+
+			return;
+		}
+
+		if (req.url && req.url.startsWith("/ports")) {
+			const onAdded = portScanner.onAdded((added) => ws.send(JSON.stringify({ added })));
+			const onRemoved = portScanner.onRemoved((removed) => ws.send(JSON.stringify({ removed })));
+			ws.on("close", () => {
+				onAdded.dispose();
+				onRemoved.dispose();
+			});
+
+			return ws.send(JSON.stringify({ ports: portScanner.ports }));
+		}
+
 		const connection: ReadWriteConnection = {
 			onMessage: (cb): void => {
 				ws.addEventListener("message", (event) => cb(event.data));
@@ -52,11 +138,17 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 	});
 
 	const baseDir = buildDir || path.join(__dirname, "..");
-	if (isCli) {
-		app.use(expressStaticGzip(path.join(baseDir, "build/web")));
-	} else {
-		app.use(express.static(path.join(baseDir, "resources/web")));
-	}
+	const authStaticFunc = expressStaticGzip(path.join(baseDir, "build/web/auth"));
+	const unauthStaticFunc = expressStaticGzip(path.join(baseDir, "build/web/unauth"));
+	app.use((req, res, next) => {
+		if (isAuthed(req)) {
+			// We can serve the actual VSCode bin
+			authStaticFunc(req, res, next);
+		} else {
+			// Serve only the unauthed version
+			unauthStaticFunc(req, res, next);
+		}
+	});
 	app.get("/resource/:url(*)", async (req, res) => {
 		try {
 			const fullPath = `/${req.params.url}`;
@@ -91,6 +183,28 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
 			res.end();
 		}
 	});
+	app.post("/resource/:url(*)", async (req, res) => {
+		try {
+			const fullPath = `/${req.params.url}`;
+
+			const data: string[] = [];
+			req.setEncoding("utf8");
+			req.on("data", (chunk) => {
+				data.push(chunk);
+			});
+			req.on("end", () => {
+				const body = data.join("");
+				fs.writeFileSync(fullPath, body);
+				logger.debug("Wrote resource", field("path", fullPath), field("content-length", body.length));
+				res.status(200);
+				res.end();
+			});
+		} catch (ex) {
+			res.write(ex.toString());
+			res.status(500);
+			res.end();
+		}
+	});
 
 	return {
 		express: app,

packages/server/src/vscode/bootstrapFork.ts

@@ -5,37 +5,104 @@ import * as zlib from "zlib";
 import * as vm from "vm";
 import { isCli } from "../constants";
 
+let ipcMsgBuffer: Buffer[] | undefined = [];
+let ipcMsgListener = process.send ? (d: Buffer): number => ipcMsgBuffer!.push(d) : undefined;
+if (ipcMsgListener) {
+	process.on("message", ipcMsgListener);
+}
+
+/**
+ * Requires a module from the filesystem.
+ *
+ * Will load from the CLI if file is included inside of the default extensions dir
+ */
+// tslint:disable-next-line:no-any
+const requireFilesystemModule = (id: string, builtInExtensionsDir: string): any => {
+	const mod = require("module") as typeof import("module");
+	const customMod = new mod.Module(id);
+	customMod.filename = id;
+	// tslint:disable-next-line:no-any
+	customMod.paths = [(<any>mod)._nodeModulePaths(path.dirname(id)), path.join(__dirname, "../../../../lib/vscode/node_modules")];
+
+	if (id.startsWith(builtInExtensionsDir)) {
+		customMod.loaded = true;
+		const fileName = id.endsWith(".js") ? id : `${id}.js`;
+		const req = vm.runInThisContext(mod.wrap(fs.readFileSync(fileName).toString()), {
+			displayErrors: true,
+			filename: id + fileName,
+		});
+		req(customMod.exports, customMod.require.bind(customMod), customMod, fileName, path.dirname(id));
+
+		return customMod.exports;
+	}
+
+	return customMod.require(id);
+};
+
+/**
+ * Called from forking a module
+ */
+export const requireFork = (modulePath: string, args: string[], builtInExtensionsDir: string): void => {
+	const Module = require("module") as typeof import("module");
+	const oldRequire = Module.prototype.require;
+	// tslint:disable-next-line:no-any
+	Module.prototype.require = (id: string): any => {
+		if (id === "typescript") {
+			return require("typescript");
+		}
+
+		return oldRequire(id);
+	};
+
+	if (!process.send) {
+		throw new Error("No IPC messaging initialized");
+	}
+
+	process.argv = ["", "", ...args];
+
+	requireFilesystemModule(modulePath, builtInExtensionsDir);
+
+	if (ipcMsgBuffer && ipcMsgListener) {
+		process.removeListener("message", ipcMsgListener);
+		// tslint:disable-next-line:no-any
+		ipcMsgBuffer.forEach((i) => process.emit("message" as any, i as any));
+		ipcMsgBuffer = undefined;
+		ipcMsgListener = undefined;
+	}
+};
+
 export const requireModule = (modulePath: string, builtInExtensionsDir: string): void => {
 	process.env.AMD_ENTRYPOINT = modulePath;
 	const xml = require("xhr2");
+	xml.XMLHttpRequest.prototype._restrictedHeaders["user-agent"] = false;
 	// tslint:disable-next-line no-any this makes installing extensions work.
 	(global as any).XMLHttpRequest = xml.XMLHttpRequest;
 
 	const mod = require("module") as typeof import("module");
+	const promiseFinally = require("promise.prototype.finally") as { shim: () => void };
+	promiseFinally.shim();
 	/**
 	 * Used for loading extensions. Using __non_webpack_require__ didn't work
 	 * as it was not resolving to the FS.
 	 */
-	(global as any).nativeNodeRequire = (id: string): any => {// tslint:disable-line no-any
-		const customMod = new mod.Module(id);
-		customMod.filename = id;
-		// tslint:disable-next-line no-any
-		customMod.paths = (mod as any)._nodeModulePaths(path.dirname(id));
-
-		if (id.startsWith(builtInExtensionsDir)) {
-			customMod.loaded = true;
-			const req = vm.runInThisContext(mod.wrap(fs.readFileSync(id + ".js").toString()), {
-				displayErrors: true,
-				filename: id + ".js",
-			});
-			req(customMod.exports, customMod.require.bind(customMod), customMod, __filename, path.dirname(id));
-
-			return customMod.exports;
-		}
-
-		return customMod.require(id);
+	// tslint:disable-next-line:no-any
+	(global as any).nativeNodeRequire = (id: string): any => {
+		return requireFilesystemModule(id, builtInExtensionsDir);
 	};
 
+	if (isCli) {
+		/**
+		 * Needed for properly forking external modules within the CLI
+		 */
+		// tslint:disable-next-line:no-any
+		(<any>cp).fork = (modulePath: string, args: ReadonlyArray<string> = [], options?: cp.ForkOptions): cp.ChildProcess => {
+			return cp.spawn(process.execPath, ["--fork", modulePath, "--args", JSON.stringify(args)], {
+				...options,
+				stdio: [null, null, null, "ipc"],
+			});
+		};
+	}
+
 	let content: Buffer | undefined;
 	const readFile = (name: string): Buffer => {
 		return fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build", name));
@@ -43,7 +110,7 @@ export const requireModule = (modulePath: string, builtInExtensionsDir: string):
 	if (isCli) {
 		content = zlib.gunzipSync(readFile("bootstrap-fork.js.gz"));
 	} else {
-		content = readFile("../resources/bootstrap-fork.js");
+		content = readFile("../../vscode/bin/bootstrap-fork.js");
 	}
 	eval(content.toString());
 };
@@ -71,7 +138,7 @@ export const forkModule = (modulePath: string, args: string[], options: cp.ForkO
 		stdio: [null, null, null, "ipc"],
 	};
 	if (isCli) {
-		proc = cp.execFile(process.execPath, forkArgs, forkOptions);
+		proc = cp.spawn(process.execPath, forkArgs, options);
 	} else {
 		proc = cp.spawn(process.execPath, ["--require", "ts-node/register", "--require", "tsconfig-paths/register", process.argv[1], ...forkArgs], forkOptions);
 	}