OCram85/code-server
Archived
1
0
Fork 0
Code Pull Requests 2 Activity

Refactor evaluations (#285)

Browse Source 
* Replace evaluations with proxies and messages

* Return proxies synchronously

Otherwise events can be lost.

* Ensure events cannot be missed

* Refactor remaining fills

* Use more up-to-date version of util

For callbackify.

* Wait for dispose to come back before removing

This prevents issues with the "done" event not always being the last
event fired. For example a socket might close and then end, but only
if the caller called end.

* Remove old node-pty tests

* Fix emitting events twice on duplex streams

* Preserve environment when spawning processes

* Throw a better error if the proxy doesn't exist

* Remove rimraf dependency from ide

* Update net.Server.listening

* Use exit event instead of killed

Doesn't look like killed is even a thing.

* Add response timeout to server

* Fix trash

* Require node-pty & spdlog after they get unpackaged

This fixes an error when running in the binary.

* Fix errors in down emitter preventing reconnecting

* Fix disposing proxies when nothing listens to "error" event

* Refactor event tests to use jest.fn()

* Reject proxy call when disconnected

Otherwise it'll wait for the timeout which is a waste of time since we
already know the connection is dead.

* Use nbin for binary packaging

* Remove additional module requires

* Attempt to remove require for local bootstrap-fork

* Externalize fsevents
This commit is contained in:
Asher
2019-03-26 13:01:25 -05:00
committed by Kyle Carberry
parent d16c6aeb30
commit dc2253e718
75 changed files with 5866 additions and 6181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
packages/server/package.json
View File

@ -6,9 +6,10 @@
"scripts": {
"start": "node --max-old-space-size=32384 --require ts-node/register --require tsconfig-paths/register src/cli.ts",
"build": "rm -rf ./out && ../../node_modules/.bin/cross-env CLI=true UV_THREADPOOL_SIZE=100 node --max-old-space-size=32384 ../../node_modules/webpack/bin/webpack.js --config ./webpack.config.js",
"build:nexe": "node scripts/nexe.js"
"build:binary": "ts-node scripts/nbin.ts"
},
"dependencies": {
"@coder/nbin": "^1.0.0",
"@oclif/config": "^1.10.4",
"@oclif/errors": "^1.2.2",
"@oclif/plugin-help": "^2.1.4",
@ -32,7 +33,6 @@
"@types/safe-compare": "^1.1.0",
"@types/ws": "^6.0.1",
"fs-extra": "^7.0.1",
"nexe": "^2.0.0-rc.34",
"opn": "^5.4.0",
"string-replace-webpack-plugin": "^0.1.3",
"ts-node": "^7.0.1",

21
packages/server/scripts/nbin.ts Normal file
View File

@ -0,0 +1,21 @@
import { Binary } from "@coder/nbin";
import * as fs from "fs";
import * as os from "os";
import * as path from "path";
const target = `${os.platform()}-${os.arch()}`;
const rootDir = path.join(__dirname, "..");
const bin = new Binary({
mainFile: path.join(rootDir, "out", "cli.js"),
});
bin.writeFiles(path.join(rootDir, "build", "**"));
bin.writeFiles(path.join(rootDir, "out", "**"));
bin.build().then((binaryData) => {
const outputPath = path.join(__dirname, "..", `cli-${target}`);
fs.writeFileSync(outputPath, binaryData);
fs.chmodSync(outputPath, "755");
}).catch((ex) => {
// tslint:disable-next-line:no-console
console.error(ex);
process.exit(1);
});

31
packages/server/scripts/nexe.js
View File

@ -1,31 +0,0 @@
const fs = require("fs");
const fse = require("fs-extra");
const os = require("os");
const path = require("path");
const nexePath = require.resolve("nexe");
const shimPath = path.join(path.dirname(nexePath), "lib/steps/shim.js");
let shimContent = fs.readFileSync(shimPath).toString();
const replaceString = `global.nativeFs = { existsSync: originalExistsSync, readFile: originalReadFile, readFileSync: originalReadFileSync, createReadStream: originalCreateReadStream, readdir: originalReaddir, readdirSync: originalReaddirSync, statSync: originalStatSync, stat: originalStat, realpath: originalRealpath, realpathSync: originalRealpathSync };`;
shimContent = shimContent.replace(/compiler\.options\.resources\.length[\s\S]*wrap\("(.*\\n)"/g, (om, a) => {
return om.replace(a, `${a}${replaceString}`);
});
fs.writeFileSync(shimPath, shimContent);
const nexe = require("nexe");
const target = `${os.platform()}-${os.arch()}`;
nexe.compile({
debugBundle: true,
input: path.join(__dirname, "../out/cli.js"),
output: `cli-${target}`,
targets: [target],
/**
* To include native extensions, do NOT install node_modules for each one. They
* are not required as each extension is built using webpack.
*/
resources: [
path.join(__dirname, "../package.json"),
path.join(__dirname, "../build/**/*"),
],
});

9
packages/server/src/cli.ts
View File

@ -9,7 +9,6 @@ import * as os from "os";
import * as path from "path";
import * as WebSocket from "ws";
import { buildDir, cacheHome, dataHome, isCli, serveStatic } from "./constants";
import { fillFs } from "./fill";
import { setup as setupNativeModules } from "./modules";
import { createApp } from "./server";
import { forkModule, requireFork, requireModule } from "./vscode/bootstrapFork";
@ -44,8 +43,10 @@ export class Entry extends Command {
}];
public async run(): Promise<void> {
Error.stackTraceLimit = Infinity;
if (isCli) {
fillFs();
require("nbin").shimNativeFs("/home/kyle/node/coder/code-server/packages/server");
}
const { args, flags } = this.parse(Entry);
@ -182,13 +183,13 @@ export class Entry extends Command {
dataDirectory: dataDir,
workingDirectory: workingDir,
cacheDirectory: cacheHome,
fork: (modulePath: string, args: string[], options: ForkOptions): ChildProcess => {
fork: (modulePath: string, args?: string[], options?: ForkOptions): ChildProcess => {
if (options && options.env && options.env.AMD_ENTRYPOINT) {
return forkModule(options.env.AMD_ENTRYPOINT, args, options, dataDir);
}
if (isCli) {
return spawn(process.execPath, ["--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
return spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), "--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
...options,
stdio: [null, null, null, "ipc"],
});

195
packages/server/src/fill.ts
View File

@ -1,195 +0,0 @@
import * as fs from "fs";
import * as path from "path";
import * as util from "util";
import { isCli, buildDir } from "./constants";
// tslint:disable:no-any
const nativeFs = (<any>global).nativeFs as typeof fs || {};
const oldAccess = fs.access;
const existsWithinBinary = (path: fs.PathLike): Promise<boolean> => {
return new Promise<boolean>((resolve): void => {
if (typeof path === "number") {
if (path < 0) {
return resolve(true);
}
}
oldAccess(path, fs.constants.F_OK, (err) => {
const exists = !err;
const es = fs.existsSync(path);
const res = !exists && es;
resolve(res);
});
});
};
export const fillFs = (): void => {
/**
* Refer to https://github.com/nexe/nexe/blob/master/src/fs/patch.ts
* For impls
*/
if (!isCli) {
throw new Error("Should not fill FS when not in CLI");
}
interface FD {
readonly path: string;
position: number;
}
let lastFd = Number.MIN_SAFE_INTEGER;
const fds = new Map<number, FD>();
const replaceNative = <T extends keyof typeof fs>(propertyName: T, func: (callOld: () => void, ...args: any[]) => any, customPromisify?: (...args: any[]) => Promise<any>): void => {
const oldFunc = (<any>fs)[propertyName];
fs[propertyName] = (...args: any[]): any => {
try {
return func(() => {
return oldFunc(...args);
}, ...args);
} catch (ex) {
return oldFunc(...args);
}
};
if (customPromisify) {
(<any>fs[propertyName])[util.promisify.custom] = (...args: any[]): any => {
return customPromisify(...args).catch((ex) => {
throw ex;
});
};
}
};
replaceNative("access", (callNative, path, mode, callback) => {
existsWithinBinary(path).then((exists) => {
if (!exists) {
return callNative();
}
return callback();
});
});
replaceNative("exists", (callOld, path, callback) => {
existsWithinBinary(path).then((exists) => {
if (exists) {
return callback(true);
}
return callOld();
});
}, (path) => new Promise((res): void => fs.exists(path, res)));
replaceNative("open", (callOld, path: fs.PathLike, flags: string | Number, mode: any, callback: any) => {
existsWithinBinary(path).then((exists) => {
if (!exists) {
return callOld();
}
if (typeof mode === "function") {
callback = mode;
mode = undefined;
}
if (path === process.execPath) {
return callOld();
}
const fd = lastFd++;
fds.set(fd, {
path: path.toString(),
position: 0,
});
callback(undefined, fd);
});
});
replaceNative("close", (callOld, fd: number, callback) => {
if (!fds.has(fd)) {
return callOld();
}
fds.delete(fd);
callback();
});
replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void) => {
if (!fds.has(fd)) {
return callOld();
}
const fileDesc = fds.get(fd)!;
return fs.readFile(fileDesc.path, (err, rb) => {
if (err) {
return callOld();
}
rb = rb.slice(position || fileDesc.position);
const sliced = rb.slice(0, length);
if (position === null) {
fileDesc.position += sliced.byteLength;
}
buffer.set(sliced, offset);
if (callback) {
callback(undefined!, sliced.byteLength, buffer);
}
});
}, (fd: number, buffer: Buffer, offset: number, length: number, position: number | null): Promise<{
bytesRead: number;
buffer: Buffer;
}> => {
return new Promise((res, rej): void => {
fs.read(fd, buffer, offset, length, position, (err, bytesRead, buffer) => {
if (err) {
return rej(err);
}
res({
bytesRead,
buffer,
});
});
});
});
replaceNative("readdir", (callOld, directory: string, callback: (err: NodeJS.ErrnoException, paths: string[]) => void) => {
const relative = path.relative(directory, buildDir!);
if (relative.startsWith("..")) {
return callOld();
}
return nativeFs.readdir(directory, callback);
});
const fillNativeFunc = <T extends keyof typeof fs>(propertyName: T): void => {
replaceNative(propertyName, (callOld, newPath, ...args) => {
if (typeof newPath !== "string") {
return callOld();
}
const rel = path.relative(newPath, buildDir!);
if (rel.startsWith("..")) {
return callOld();
}
const func = nativeFs[propertyName] as any;
return func(newPath, ...args);
});
};
const properties: Array<keyof typeof fs> = [
"existsSync",
"readFile",
"readFileSync",
"createReadStream",
"readdir",
"readdirSync",
"statSync",
"stat",
"realpath",
"realpathSync",
];
properties.forEach((p) => fillNativeFunc(p));
};

27
packages/server/src/server.ts
View File

@ -1,11 +1,13 @@
import { mkdirp } from "fs-extra";
import { logger, field } from "@coder/logger";
import { field, logger } from "@coder/logger";
import { ReadWriteConnection } from "@coder/protocol";
import { Server, ServerOptions } from "@coder/protocol/src/node/server";
import { TunnelCloseCode } from "@coder/tunnel/src/common";
import { handle as handleTunnel } from "@coder/tunnel/src/server";
import * as express from "express";
//@ts-ignore
import * as expressStaticGzip from "express-static-gzip";
import * as fs from "fs";
import { mkdirp } from "fs-extra";
import * as http from "http";
//@ts-ignore
import * as httpolyglot from "httpolyglot";
@ -17,11 +19,9 @@ import * as path from "path";
import * as pem from "pem";
import * as util from "util";
import * as ws from "ws";
import safeCompare = require("safe-compare");
import { TunnelCloseCode } from "@coder/tunnel/src/common";
import { handle as handleTunnel } from "@coder/tunnel/src/server";
import { createPortScanner } from "./portScanner";
import { buildDir } from "./constants";
import { createPortScanner } from "./portScanner";
import safeCompare = require("safe-compare");
interface CreateAppOptions {
registerMiddleware?: (app: express.Application) => void;
@ -180,10 +180,13 @@ export const createApp = async (options: CreateAppOptions): Promise<{
logger.error(error.message);
}
},
onUp: (): void => undefined, // This can't come back up.
onDown: (cb): void => ws.addEventListener("close", () => cb()),
onClose: (cb): void => ws.addEventListener("close", () => cb()),
};
const server = new Server(connection, options.serverOptions);
// tslint:disable-next-line no-unused-expression
new Server(connection, options.serverOptions);
});
const baseDir = buildDir || path.join(__dirname, "..");
@ -202,6 +205,10 @@ export const createApp = async (options: CreateAppOptions): Promise<{
unauthStaticFunc(req, res, next);
}
});
// @ts-ignore
app.use((err, req, res, next) => {
next();
});
app.get("/ping", (req, res) => {
res.json({
hostname: os.hostname(),
@ -235,9 +242,11 @@ export const createApp = async (options: CreateAppOptions): Promise<{
}
const content = await util.promisify(fs.readFile)(fullPath);
res.header("Content-Type", mimeType as string);
res.writeHead(200, {
"Content-Type": mimeType,
"Content-Length": content.byteLength,
});
res.write(content);
res.status(200);
res.end();
} catch (ex) {
res.write(ex.toString());

34
packages/server/src/vscode/bootstrapFork.ts
View File

@ -1,9 +1,8 @@
import * as cp from "child_process";
import * as fs from "fs";
import * as path from "path";
import * as zlib from "zlib";
import * as vm from "vm";
import { isCli } from "../constants";
import { buildDir, isCli } from "../constants";
let ipcMsgBuffer: Buffer[] | undefined = [];
let ipcMsgListener = process.send ? (d: Buffer): number => ipcMsgBuffer!.push(d) : undefined;
@ -11,6 +10,8 @@ if (ipcMsgListener) {
process.on("message", ipcMsgListener);
}
declare var __non_webpack_require__: typeof require;
/**
* Requires a module from the filesystem.
*
@ -29,7 +30,7 @@ const requireFilesystemModule = (id: string, builtInExtensionsDir: string): any
const fileName = id.endsWith(".js") ? id : `${id}.js`;
const req = vm.runInThisContext(mod.wrap(fs.readFileSync(fileName).toString()), {
displayErrors: true,
filename: id + fileName,
filename: fileName,
});
req(customMod.exports, customMod.require.bind(customMod), customMod, fileName, path.dirname(id));
@ -46,6 +47,14 @@ export const requireFork = (modulePath: string, args: string[], builtInExtension
const Module = require("module") as typeof import("module");
const oldRequire = Module.prototype.require;
// tslint:disable-next-line:no-any
const oldLoad = (Module as any)._findPath;
// @ts-ignore
(Module as any)._findPath = function (request, parent, isMain): any {
const lookupPaths = oldLoad.call(this, request, parent, isMain);
return lookupPaths;
};
// tslint:disable-next-line:no-any
Module.prototype.require = function (id: string): any {
if (id === "typescript") {
return require("typescript");
@ -96,23 +105,20 @@ export const requireModule = (modulePath: string, dataDir: string, builtInExtens
*/
// tslint:disable-next-line:no-any
(<any>cp).fork = (modulePath: string, args: ReadonlyArray<string> = [], options?: cp.ForkOptions): cp.ChildProcess => {
return cp.spawn(process.execPath, ["--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
return cp.spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), "--fork", modulePath, "--args", JSON.stringify(args), "--data-dir", dataDir], {
...options,
stdio: [null, null, null, "ipc"],
});
};
}
let content: Buffer | undefined;
const readFile = (name: string): Buffer => {
return fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build", name));
};
const baseDir = path.join(buildDir, "build");
if (isCli) {
content = zlib.gunzipSync(readFile("bootstrap-fork.js.gz"));
__non_webpack_require__(path.join(baseDir, "bootstrap-fork.js.gz"));
} else {
content = readFile("../../vscode/out/bootstrap-fork.js");
// We need to check `isCli` here to confuse webpack.
require(path.join(__dirname, isCli ? "" : "../../../vscode/out/bootstrap-fork.js"));
}
eval(content.toString());
};
/**
@ -123,12 +129,12 @@ export const requireModule = (modulePath: string, dataDir: string, builtInExtens
* cp.stderr.on("data", (data) => console.log(data.toString("utf8")));
* @param modulePath Path of the VS Code module to load.
*/
export const forkModule = (modulePath: string, args: string[], options: cp.ForkOptions, dataDir?: string): cp.ChildProcess => {
export const forkModule = (modulePath: string, args?: string[], options?: cp.ForkOptions, dataDir?: string): cp.ChildProcess => {
let proc: cp.ChildProcess;
const forkOptions: cp.ForkOptions = {
stdio: [null, null, null, "ipc"],
};
if (options.env) {
if (options && options.env) {
// This prevents vscode from trying to load original-fs from electron.
delete options.env.ELECTRON_RUN_AS_NODE;
forkOptions.env = options.env;
@ -141,7 +147,7 @@ export const forkModule = (modulePath: string, args: string[], options: cp.ForkO
forkArgs.push("--data-dir", dataDir);
}
if (isCli) {
proc = cp.spawn(process.execPath, forkArgs, forkOptions);
proc = cp.spawn(process.execPath, [path.join(buildDir, "out", "cli.js"), ...forkArgs], forkOptions);
} else {
proc = cp.spawn(process.execPath, ["--require", "ts-node/register", "--require", "tsconfig-paths/register", process.argv[1], ...forkArgs], forkOptions);
}

7
packages/server/webpack.config.js
View File

@ -13,6 +13,7 @@ module.exports = merge(
path: path.join(__dirname, "out"),
libraryTarget: "commonjs",
},
mode: "production",
node: {
console: false,
global: false,
@ -27,7 +28,11 @@ module.exports = merge(
"node-pty": "node-pty-prebuilt",
},
},
externals: ["tslib"],
externals: {
"tslib": "commonjs tslib",
"nbin": "commonjs nbin",
"fsevents": "fsevents",
},
entry: "./packages/server/src/cli.ts",
plugins: [
new webpack.DefinePlugin({

2817
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff