Refactor evaluations (#285)
* Replace evaluations with proxies and messages * Return proxies synchronously Otherwise events can be lost. * Ensure events cannot be missed * Refactor remaining fills * Use more up-to-date version of util For callbackify. * Wait for dispose to come back before removing This prevents issues with the "done" event not always being the last event fired. For example a socket might close and then end, but only if the caller called end. * Remove old node-pty tests * Fix emitting events twice on duplex streams * Preserve environment when spawning processes * Throw a better error if the proxy doesn't exist * Remove rimraf dependency from ide * Update net.Server.listening * Use exit event instead of killed Doesn't look like killed is even a thing. * Add response timeout to server * Fix trash * Require node-pty & spdlog after they get unpackaged This fixes an error when running in the binary. * Fix errors in down emitter preventing reconnecting * Fix disposing proxies when nothing listens to "error" event * Refactor event tests to use jest.fn() * Reject proxy call when disconnected Otherwise it'll wait for the timeout which is a waste of time since we already know the connection is dead. * Use nbin for binary packaging * Remove additional module requires * Attempt to remove require for local bootstrap-fork * Externalize fsevents
This commit is contained in:
Asher
Kyle Carberry
@ -9,7 +9,6 @@ import * as os from "os";
|
||||
import * as path from "path";
|
||||
import * as WebSocket from "ws";
|
||||
import { buildDir, cacheHome, dataHome, isCli, serveStatic } from "./constants";
|
||||
import { fillFs } from "./fill";
|
||||
import { setup as setupNativeModules } from "./modules";
|
||||
import { createApp } from "./server";
|
||||
import { forkModule, requireFork, requireModule } from "./vscode/bootstrapFork";
|
||||
@ -44,8 +43,10 @@ export class Entry extends Command {
|
||||
}];
|
||||
|
||||
public async run(): Promise<void> {
|
||||
Error.stackTraceLimit = Infinity;
|
||||
|
||||
if (isCli) {
|
||||
fillFs();
|
||||
require("nbin").shimNativeFs("/home/kyle/node/coder/code-server/packages/server");
|
||||
}
|
||||
|
||||
const { args, flags } = this.parse(Entry);
|
||||
@ -182,13 +183,13 @@ export class Entry extends Command {
|
||||
dataDirectory: dataDir,
|
||||
workingDirectory: workingDir,
|
||||
cacheDirectory: cacheHome,
|
||||
fork: (modulePath: string, args: string[], options: ForkOptions): ChildProcess => {
|
||||
fork: (modulePath: string, args?: string[], options?: ForkOptions): ChildProcess => {
|
||||
if (options && options.env && options.env.AMD_ENTRYPOINT) {
|
||||
return forkModule(options.env.AMD_ENTRYPOINT, args, options, dataDir);
|
||||
}
|
||||
|
||||
if (isCli) {
|
||||
return spawn(process.execPath, ["--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
|
||||
return spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), "--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
|
||||
...options,
|
||||
stdio: [null, null, null, "ipc"],
|
||||
});
|
||||
|
||||
@ -1,195 +0,0 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as util from "util";
|
||||
import { isCli, buildDir } from "./constants";
|
||||
|
||||
// tslint:disable:no-any
|
||||
const nativeFs = (<any>global).nativeFs as typeof fs || {};
|
||||
const oldAccess = fs.access;
|
||||
const existsWithinBinary = (path: fs.PathLike): Promise<boolean> => {
|
||||
return new Promise<boolean>((resolve): void => {
|
||||
if (typeof path === "number") {
|
||||
if (path < 0) {
|
||||
return resolve(true);
|
||||
}
|
||||
}
|
||||
oldAccess(path, fs.constants.F_OK, (err) => {
|
||||
const exists = !err;
|
||||
const es = fs.existsSync(path);
|
||||
const res = !exists && es;
|
||||
resolve(res);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export const fillFs = (): void => {
|
||||
/**
|
||||
* Refer to https://github.com/nexe/nexe/blob/master/src/fs/patch.ts
|
||||
* For impls
|
||||
*/
|
||||
|
||||
if (!isCli) {
|
||||
throw new Error("Should not fill FS when not in CLI");
|
||||
}
|
||||
|
||||
interface FD {
|
||||
readonly path: string;
|
||||
position: number;
|
||||
}
|
||||
|
||||
let lastFd = Number.MIN_SAFE_INTEGER;
|
||||
const fds = new Map<number, FD>();
|
||||
|
||||
const replaceNative = <T extends keyof typeof fs>(propertyName: T, func: (callOld: () => void, ...args: any[]) => any, customPromisify?: (...args: any[]) => Promise<any>): void => {
|
||||
const oldFunc = (<any>fs)[propertyName];
|
||||
fs[propertyName] = (...args: any[]): any => {
|
||||
try {
|
||||
return func(() => {
|
||||
return oldFunc(...args);
|
||||
}, ...args);
|
||||
} catch (ex) {
|
||||
return oldFunc(...args);
|
||||
}
|
||||
};
|
||||
if (customPromisify) {
|
||||
(<any>fs[propertyName])[util.promisify.custom] = (...args: any[]): any => {
|
||||
return customPromisify(...args).catch((ex) => {
|
||||
throw ex;
|
||||
});
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
replaceNative("access", (callNative, path, mode, callback) => {
|
||||
existsWithinBinary(path).then((exists) => {
|
||||
if (!exists) {
|
||||
return callNative();
|
||||
}
|
||||
|
||||
return callback();
|
||||
});
|
||||
});
|
||||
|
||||
replaceNative("exists", (callOld, path, callback) => {
|
||||
existsWithinBinary(path).then((exists) => {
|
||||
if (exists) {
|
||||
return callback(true);
|
||||
}
|
||||
|
||||
return callOld();
|
||||
});
|
||||
}, (path) => new Promise((res): void => fs.exists(path, res)));
|
||||
|
||||
replaceNative("open", (callOld, path: fs.PathLike, flags: string | Number, mode: any, callback: any) => {
|
||||
existsWithinBinary(path).then((exists) => {
|
||||
if (!exists) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
if (typeof mode === "function") {
|
||||
callback = mode;
|
||||
mode = undefined;
|
||||
}
|
||||
|
||||
if (path === process.execPath) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
const fd = lastFd++;
|
||||
fds.set(fd, {
|
||||
path: path.toString(),
|
||||
position: 0,
|
||||
});
|
||||
callback(undefined, fd);
|
||||
});
|
||||
});
|
||||
|
||||
replaceNative("close", (callOld, fd: number, callback) => {
|
||||
if (!fds.has(fd)) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
fds.delete(fd);
|
||||
callback();
|
||||
});
|
||||
|
||||
replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void) => {
|
||||
if (!fds.has(fd)) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
const fileDesc = fds.get(fd)!;
|
||||
|
||||
return fs.readFile(fileDesc.path, (err, rb) => {
|
||||
if (err) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
rb = rb.slice(position || fileDesc.position);
|
||||
const sliced = rb.slice(0, length);
|
||||
if (position === null) {
|
||||
fileDesc.position += sliced.byteLength;
|
||||
}
|
||||
buffer.set(sliced, offset);
|
||||
if (callback) {
|
||||
callback(undefined!, sliced.byteLength, buffer);
|
||||
}
|
||||
});
|
||||
}, (fd: number, buffer: Buffer, offset: number, length: number, position: number | null): Promise<{
|
||||
bytesRead: number;
|
||||
buffer: Buffer;
|
||||
}> => {
|
||||
return new Promise((res, rej): void => {
|
||||
fs.read(fd, buffer, offset, length, position, (err, bytesRead, buffer) => {
|
||||
if (err) {
|
||||
return rej(err);
|
||||
}
|
||||
|
||||
res({
|
||||
bytesRead,
|
||||
buffer,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
replaceNative("readdir", (callOld, directory: string, callback: (err: NodeJS.ErrnoException, paths: string[]) => void) => {
|
||||
const relative = path.relative(directory, buildDir!);
|
||||
if (relative.startsWith("..")) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
return nativeFs.readdir(directory, callback);
|
||||
});
|
||||
|
||||
const fillNativeFunc = <T extends keyof typeof fs>(propertyName: T): void => {
|
||||
replaceNative(propertyName, (callOld, newPath, ...args) => {
|
||||
if (typeof newPath !== "string") {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
const rel = path.relative(newPath, buildDir!);
|
||||
if (rel.startsWith("..")) {
|
||||
return callOld();
|
||||
}
|
||||
|
||||
const func = nativeFs[propertyName] as any;
|
||||
|
||||
return func(newPath, ...args);
|
||||
});
|
||||
};
|
||||
|
||||
const properties: Array<keyof typeof fs> = [
|
||||
"existsSync",
|
||||
"readFile",
|
||||
"readFileSync",
|
||||
"createReadStream",
|
||||
"readdir",
|
||||
"readdirSync",
|
||||
"statSync",
|
||||
"stat",
|
||||
"realpath",
|
||||
"realpathSync",
|
||||
];
|
||||
properties.forEach((p) => fillNativeFunc(p));
|
||||
};
|
||||
@ -1,11 +1,13 @@
|
||||
import { mkdirp } from "fs-extra";
|
||||
import { logger, field } from "@coder/logger";
|
||||
import { field, logger } from "@coder/logger";
|
||||
import { ReadWriteConnection } from "@coder/protocol";
|
||||
import { Server, ServerOptions } from "@coder/protocol/src/node/server";
|
||||
import { TunnelCloseCode } from "@coder/tunnel/src/common";
|
||||
import { handle as handleTunnel } from "@coder/tunnel/src/server";
|
||||
import * as express from "express";
|
||||
//@ts-ignore
|
||||
import * as expressStaticGzip from "express-static-gzip";
|
||||
import * as fs from "fs";
|
||||
import { mkdirp } from "fs-extra";
|
||||
import * as http from "http";
|
||||
//@ts-ignore
|
||||
import * as httpolyglot from "httpolyglot";
|
||||
@ -17,11 +19,9 @@ import * as path from "path";
|
||||
import * as pem from "pem";
|
||||
import * as util from "util";
|
||||
import * as ws from "ws";
|
||||
import safeCompare = require("safe-compare");
|
||||
import { TunnelCloseCode } from "@coder/tunnel/src/common";
|
||||
import { handle as handleTunnel } from "@coder/tunnel/src/server";
|
||||
import { createPortScanner } from "./portScanner";
|
||||
import { buildDir } from "./constants";
|
||||
import { createPortScanner } from "./portScanner";
|
||||
import safeCompare = require("safe-compare");
|
||||
|
||||
interface CreateAppOptions {
|
||||
registerMiddleware?: (app: express.Application) => void;
|
||||
@ -180,10 +180,13 @@ export const createApp = async (options: CreateAppOptions): Promise<{
|
||||
logger.error(error.message);
|
||||
}
|
||||
},
|
||||
onUp: (): void => undefined, // This can't come back up.
|
||||
onDown: (cb): void => ws.addEventListener("close", () => cb()),
|
||||
onClose: (cb): void => ws.addEventListener("close", () => cb()),
|
||||
};
|
||||
|
||||
const server = new Server(connection, options.serverOptions);
|
||||
// tslint:disable-next-line no-unused-expression
|
||||
new Server(connection, options.serverOptions);
|
||||
});
|
||||
|
||||
const baseDir = buildDir || path.join(__dirname, "..");
|
||||
@ -202,6 +205,10 @@ export const createApp = async (options: CreateAppOptions): Promise<{
|
||||
unauthStaticFunc(req, res, next);
|
||||
}
|
||||
});
|
||||
// @ts-ignore
|
||||
app.use((err, req, res, next) => {
|
||||
next();
|
||||
});
|
||||
app.get("/ping", (req, res) => {
|
||||
res.json({
|
||||
hostname: os.hostname(),
|
||||
@ -235,9 +242,11 @@ export const createApp = async (options: CreateAppOptions): Promise<{
|
||||
}
|
||||
const content = await util.promisify(fs.readFile)(fullPath);
|
||||
|
||||
res.header("Content-Type", mimeType as string);
|
||||
res.writeHead(200, {
|
||||
"Content-Type": mimeType,
|
||||
"Content-Length": content.byteLength,
|
||||
});
|
||||
res.write(content);
|
||||
res.status(200);
|
||||
res.end();
|
||||
} catch (ex) {
|
||||
res.write(ex.toString());
|
||||
|
||||
@ -1,9 +1,8 @@
|
||||
import * as cp from "child_process";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as zlib from "zlib";
|
||||
import * as vm from "vm";
|
||||
import { isCli } from "../constants";
|
||||
import { buildDir, isCli } from "../constants";
|
||||
|
||||
let ipcMsgBuffer: Buffer[] | undefined = [];
|
||||
let ipcMsgListener = process.send ? (d: Buffer): number => ipcMsgBuffer!.push(d) : undefined;
|
||||
@ -11,6 +10,8 @@ if (ipcMsgListener) {
|
||||
process.on("message", ipcMsgListener);
|
||||
}
|
||||
|
||||
declare var __non_webpack_require__: typeof require;
|
||||
|
||||
/**
|
||||
* Requires a module from the filesystem.
|
||||
*
|
||||
@ -29,7 +30,7 @@ const requireFilesystemModule = (id: string, builtInExtensionsDir: string): any
|
||||
const fileName = id.endsWith(".js") ? id : `${id}.js`;
|
||||
const req = vm.runInThisContext(mod.wrap(fs.readFileSync(fileName).toString()), {
|
||||
displayErrors: true,
|
||||
filename: id + fileName,
|
||||
filename: fileName,
|
||||
});
|
||||
req(customMod.exports, customMod.require.bind(customMod), customMod, fileName, path.dirname(id));
|
||||
|
||||
@ -46,6 +47,14 @@ export const requireFork = (modulePath: string, args: string[], builtInExtension
|
||||
const Module = require("module") as typeof import("module");
|
||||
const oldRequire = Module.prototype.require;
|
||||
// tslint:disable-next-line:no-any
|
||||
const oldLoad = (Module as any)._findPath;
|
||||
// @ts-ignore
|
||||
(Module as any)._findPath = function (request, parent, isMain): any {
|
||||
const lookupPaths = oldLoad.call(this, request, parent, isMain);
|
||||
|
||||
return lookupPaths;
|
||||
};
|
||||
// tslint:disable-next-line:no-any
|
||||
Module.prototype.require = function (id: string): any {
|
||||
if (id === "typescript") {
|
||||
return require("typescript");
|
||||
@ -96,23 +105,20 @@ export const requireModule = (modulePath: string, dataDir: string, builtInExtens
|
||||
*/
|
||||
// tslint:disable-next-line:no-any
|
||||
(<any>cp).fork = (modulePath: string, args: ReadonlyArray<string> = [], options?: cp.ForkOptions): cp.ChildProcess => {
|
||||
return cp.spawn(process.execPath, ["--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
|
||||
return cp.spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), "--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
|
||||
...options,
|
||||
stdio: [null, null, null, "ipc"],
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
let content: Buffer | undefined;
|
||||
const readFile = (name: string): Buffer => {
|
||||
return fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build", name));
|
||||
};
|
||||
const baseDir = path.join(buildDir, "build");
|
||||
if (isCli) {
|
||||
content = zlib.gunzipSync(readFile("bootstrap-fork.js.gz"));
|
||||
__non_webpack_require__(path.join(baseDir, "bootstrap-fork.js.gz"));
|
||||
} else {
|
||||
content = readFile("../../vscode/out/bootstrap-fork.js");
|
||||
// We need to check `isCli` here to confuse webpack.
|
||||
require(path.join(__dirname, isCli ? "" : "../../../vscode/out/bootstrap-fork.js"));
|
||||
}
|
||||
eval(content.toString());
|
||||
};
|
||||
|
||||
/**
|
||||
@ -123,12 +129,12 @@ export const requireModule = (modulePath: string, dataDir: string, builtInExtens
|
||||
* cp.stderr.on("data", (data) => console.log(data.toString("utf8")));
|
||||
* @param modulePath Path of the VS Code module to load.
|
||||
*/
|
||||
export const forkModule = (modulePath: string, args: string[], options: cp.ForkOptions, dataDir?: string): cp.ChildProcess => {
|
||||
export const forkModule = (modulePath: string, args?: string[], options?: cp.ForkOptions, dataDir?: string): cp.ChildProcess => {
|
||||
let proc: cp.ChildProcess;
|
||||
const forkOptions: cp.ForkOptions = {
|
||||
stdio: [null, null, null, "ipc"],
|
||||
};
|
||||
if (options.env) {
|
||||
if (options && options.env) {
|
||||
// This prevents vscode from trying to load original-fs from electron.
|
||||
delete options.env.ELECTRON_RUN_AS_NODE;
|
||||
forkOptions.env = options.env;
|
||||
@ -141,7 +147,7 @@ export const forkModule = (modulePath: string, args: string[], options: cp.ForkO
|
||||
forkArgs.push("--data-dir", dataDir);
|
||||
}
|
||||
if (isCli) {
|
||||
proc = cp.spawn(process.execPath, forkArgs, forkOptions);
|
||||
proc = cp.spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), ...forkArgs], forkOptions);
|
||||
} else {
|
||||
proc = cp.spawn(process.execPath, ["--require", "ts-node/register", "--require", "tsconfig-paths/register", process.argv[1], ...forkArgs], forkOptions);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user