OCram85/code-server
Archived
1
0
Fork 0
Code Pull Requests 2 Activity

Fix loading within the CLI (#27)

Browse Source 
* Fix loading within the CLI

* Remove app

* Remove promise handle

* Fix requested changes
This commit is contained in:
Kyle Carberry 2019-02-05 11:15:20 -06:00
parent a85af49c58
commit 797efe72fd
No known key found for this signature in database
GPG Key ID: A0409BDB6B0B3EDB
28 changed files with 477 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/logger/src/logger.ts
View File

@ -272,8 +272,9 @@ export class Logger {
if (name) {
this.nameColor = hashStringToColor(name);
}
if (process.env.LOG_LEVEL) {
switch (process.env.LOG_LEVEL) {
const envLevel = typeof global !== "undefined" && typeof global.process !== "undefined" ? global.process.env.LOG_LEVEL : process.env.LOG_LEVEL;
if (envLevel) {
switch (envLevel) {
case "debug": this.level = Level.Debug; break;
case "info": this.level = Level.Info; break;
case "warn": this.level = Level.Warn; break;

3
packages/protocol/package.json
View File

@ -4,7 +4,8 @@
"dependencies": {
"express": "^4.16.4",
"google-protobuf": "^3.6.1",
"node-pty": "^0.8.0",
"node-pty": "^0.8.1",
"tslib": "^1.9.3",
"ws": "^6.1.2"
},
"devDependencies": {

1
packages/protocol/src/browser/client.ts
View File

@ -327,6 +327,7 @@ export class Client {
workingDirectory: init.getWorkingDirectory(),
os: opSys,
shell: init.getShell(),
builtInExtensionsDirectory: init.getBuiltinExtensionsDir(),
};
this.initDataEmitter.emit(this._initData);
} else if (message.hasEvalDone()) {

59
packages/protocol/src/browser/modules/fs.ts
View File

@ -8,6 +8,7 @@ import { Client } from "../client";
// Use this to get around Webpack inserting our fills.
// TODO: is there a better way?
declare var _require: typeof require;
declare var _Buffer: typeof Buffer;
/**
* Implements the native fs module
@ -121,7 +122,7 @@ export class FS {
const ae = this.client.run((ae, path, options) => {
const fs = _require("fs") as typeof import("fs");
const str = fs.createWriteStream(path, options);
ae.on("write", (d, e) => str.write(Buffer.from(d, e)));
ae.on("write", (d, e) => str.write(_Buffer.from(d, "utf8")));
ae.on("close", () => str.close());
str.on("close", () => ae.emit("close"));
str.on("open", (fd) => ae.emit("open", fd));
@ -216,18 +217,18 @@ export class FS {
this.client.evaluate((fd) => {
const fs = _require("fs") as typeof import("fs");
const util = _require("util") as typeof import("util");
const tslib = _require("tslib") as typeof import("tslib");
return util.promisify(fs.fstat)(fd).then((stats) => {
return {
...stats,
_isBlockDevice: stats.isBlockDevice(),
_isCharacterDevice: stats.isCharacterDevice(),
return tslib.__assign(stats, {
_isBlockDevice: stats.isBlockDevice ? stats.isBlockDevice() : false,
_isCharacterDevice: stats.isCharacterDevice ? stats.isCharacterDevice() : false,
_isDirectory: stats.isDirectory(),
_isFIFO: stats.isFIFO(),
_isFIFO: stats.isFIFO ? stats.isFIFO() : false,
_isFile: stats.isFile(),
_isSocket: stats.isSocket(),
_isSymbolicLink: stats.isSymbolicLink(),
};
_isSocket: stats.isSocket ? stats.isSocket() : false,
_isSymbolicLink: stats.isSymbolicLink ? stats.isSymbolicLink() : false,
});
});
}, fd).then((stats) => {
callback(undefined!, new Stats(stats));
@ -322,18 +323,18 @@ export class FS {
this.client.evaluate((path) => {
const fs = _require("fs") as typeof import("fs");
const util = _require("util") as typeof import("util");
const tslib = _require("tslib") as typeof import("tslib");
return util.promisify(fs.lstat)(path).then((stats) => {
return {
...stats,
_isBlockDevice: stats.isBlockDevice(),
_isCharacterDevice: stats.isCharacterDevice(),
return tslib.__assign(stats, {
_isBlockDevice: stats.isBlockDevice ? stats.isBlockDevice() : false,
_isCharacterDevice: stats.isCharacterDevice ? stats.isCharacterDevice() : false,
_isDirectory: stats.isDirectory(),
_isFIFO: stats.isFIFO(),
_isFIFO: stats.isFIFO ? stats.isFIFO() : false,
_isFile: stats.isFile(),
_isSocket: stats.isSocket(),
_isSymbolicLink: stats.isSymbolicLink(),
};
_isSocket: stats.isSocket ? stats.isSocket() : false,
_isSymbolicLink: stats.isSymbolicLink ? stats.isSymbolicLink() : false,
});
});
}, path).then((stats) => {
callback(undefined!, new Stats(stats));
@ -397,7 +398,7 @@ export class FS {
this.client.evaluate((fd, length, position) => {
const fs = _require("fs") as typeof import("fs");
const util = _require("util") as typeof import("util");
const buffer = new Buffer(length);
const buffer = new _Buffer(length);
return util.promisify(fs.read)(fd, buffer, 0, length, position).then((resp) => {
return {
@ -513,18 +514,22 @@ export class FS {
this.client.evaluate((path) => {
const fs = _require("fs") as typeof import("fs");
const util = _require("util") as typeof import("util");
const tslib = _require("tslib") as typeof import("tslib");
return util.promisify(fs.stat)(path).then((stats) => {
return {
...stats,
_isBlockDevice: stats.isBlockDevice(),
_isCharacterDevice: stats.isCharacterDevice(),
return tslib.__assign(stats, {
/**
* We need to check if functions exist because nexe's implemented FS
* lib doesnt implement fs.stats properly
*/
_isBlockDevice: stats.isBlockDevice ? stats.isBlockDevice() : false,
_isCharacterDevice: stats.isCharacterDevice ? stats.isCharacterDevice() : false,
_isDirectory: stats.isDirectory(),
_isFIFO: stats.isFIFO(),
_isFIFO: stats.isFIFO ? stats.isFIFO() : false,
_isFile: stats.isFile(),
_isSocket: stats.isSocket(),
_isSymbolicLink: stats.isSymbolicLink(),
};
_isSocket: stats.isSocket ? stats.isSocket() : false,
_isSymbolicLink: stats.isSymbolicLink ? stats.isSymbolicLink() : false,
});
});
}, path).then((stats) => {
callback(undefined!, new Stats(stats));
@ -602,7 +607,7 @@ export class FS {
const fs = _require("fs") as typeof import("fs");
const util = _require("util") as typeof import("util");
return util.promisify(fs.write)(fd, Buffer.from(buffer, "utf8"), offset, length, position).then((resp) => {
return util.promisify(fs.write)(fd, _Buffer.from(buffer, "utf8"), offset, length, position).then((resp) => {
return {
bytesWritten: resp.bytesWritten,
content: resp.buffer.toString("utf8"),

1
packages/protocol/src/common/connection.ts
View File

@ -21,6 +21,7 @@ export interface InitData {
readonly homeDirectory: string;
readonly tmpDirectory: string;
readonly shell: string;
readonly builtInExtensionsDirectory: string;
}
export interface ISharedProcessData {

4
packages/protocol/src/node/command.ts
View File

@ -1,6 +1,5 @@
import * as cp from "child_process";
import * as net from "net";
import * as nodePty from "node-pty";
import * as stream from "stream";
import { TextEncoder } from "text-encoding";
import { Logger, logger, field } from "@coder/logger";
@ -44,6 +43,7 @@ export const handleNewSession = (connection: SendableConnection, newSession: New
});
if (newSession.getTtyDimensions()) {
// Spawn with node-pty
const nodePty = require("node-pty") as typeof import("node-pty");
const ptyProc = nodePty.spawn(newSession.getCommand(), newSession.getArgsList(), {
cols: newSession.getTtyDimensions()!.getWidth(),
rows: newSession.getTtyDimensions()!.getHeight(),
@ -56,7 +56,7 @@ export const handleNewSession = (connection: SendableConnection, newSession: New
processTitle = ptyProc.process;
const id = new IdentifySessionMessage();
id.setId(newSession.getId());
id.setTitle(processTitle);
id.setTitle(processTitle!);
const sm = new ServerMessage();
sm.setIdentifySession(id);
connection.send(sm.serializeBinary());

4
packages/protocol/src/node/evaluate.ts
View File

@ -76,7 +76,7 @@ export const evaluate = (connection: SendableConnection, message: NewEvalMessage
connection.send(serverMsg.serializeBinary());
},
} : undefined,
Buffer,
_Buffer: Buffer,
require: typeof __non_webpack_require__ !== "undefined" ? __non_webpack_require__ : require,
_require: typeof __non_webpack_require__ !== "undefined" ? __non_webpack_require__ : require,
tslib_1: require("tslib"), // TODO: is there a better way to do this?
@ -98,7 +98,7 @@ export const evaluate = (connection: SendableConnection, message: NewEvalMessage
onDispose();
}
} catch (ex) {
sendErr(EvalFailedMessage.Reason.EXCEPTION, ex.toString());
sendErr(EvalFailedMessage.Reason.EXCEPTION, ex.toString() + " " + ex.stack);
}
return eventEmitter ? {

11
packages/protocol/src/node/server.ts
View File

@ -14,6 +14,7 @@ import * as net from "net";
export interface ServerOptions {
readonly workingDirectory: string;
readonly dataDirectory: string;
readonly builtInExtensionsDirectory: string;
forkProvider?(message: NewSessionMessage): cp.ChildProcess;
}
@ -35,7 +36,10 @@ export class Server {
try {
this.handleMessage(ClientMessage.deserializeBinary(data));
} catch (ex) {
logger.error("Failed to handle client message", field("length", data.byteLength), field("exception", ex));
logger.error("Failed to handle client message", field("length", data.byteLength), field("exception", {
message: ex.message,
stack: ex.stack,
}));
}
});
connection.onClose(() => {
@ -80,6 +84,7 @@ export class Server {
const initMsg = new WorkingInitMessage();
initMsg.setDataDirectory(options.dataDirectory);
initMsg.setWorkingDirectory(options.workingDirectory);
initMsg.setBuiltinExtensionsDir(options.builtInExtensionsDirectory);
initMsg.setHomeDirectory(os.homedir());
initMsg.setTmpDirectory(os.tmpdir());
const platform = os.platform();
@ -98,8 +103,8 @@ export class Server {
throw new Error(`unrecognized platform "${platform}"`);
}
initMsg.setOperatingSystem(operatingSystem);
if (process.env.SHELL) {
initMsg.setShell(process.env.SHELL);
if (global.process.env.SHELL) {
initMsg.setShell(global.process.env.SHELL);
}
const srvMsg = new ServerMessage();
srvMsg.setInit(initMsg);

1
packages/protocol/src/proto/client.proto
View File

@ -63,4 +63,5 @@ message WorkingInitMessage {
}
OperatingSystem operating_system = 5;
string shell = 6;
string builtin_extensions_dir = 7;
}

4
packages/protocol/src/proto/client_pb.d.ts vendored
View File

@ -270,6 +270,9 @@ export class WorkingInitMessage extends jspb.Message {
getShell(): string;
setShell(value: string): void;
getBuiltinExtensionsDir(): string;
setBuiltinExtensionsDir(value: string): void;
serializeBinary(): Uint8Array;
toObject(includeInstance?: boolean): WorkingInitMessage.AsObject;
static toObject(includeInstance: boolean, msg: WorkingInitMessage): WorkingInitMessage.AsObject;
@ -288,6 +291,7 @@ export namespace WorkingInitMessage {
workingDirectory: string,
operatingSystem: WorkingInitMessage.OperatingSystem,
shell: string,
builtinExtensionsDir: string,
}
export enum OperatingSystem {

29
packages/protocol/src/proto/client_pb.js
View File

@ -1684,7 +1684,8 @@ proto.WorkingInitMessage.toObject = function(includeInstance, msg) {
dataDirectory: msg.getDataDirectory(),
workingDirectory: msg.getWorkingDirectory(),
operatingSystem: msg.getOperatingSystem(),
shell: msg.getShell()
shell: msg.getShell(),
builtinExtensionsDir: msg.getBuiltinExtensionsDir()
};
if (includeInstance) {
@ -1745,6 +1746,10 @@ proto.WorkingInitMessage.deserializeBinaryFromReader = function(msg, reader) {
var value = /** @type {string} */ (reader.readString());
msg.setShell(value);
break;
case 7:
var value = /** @type {string} */ (reader.readString());
msg.setBuiltinExtensionsDir(value);
break;
default:
reader.skipField();
break;
@ -1825,6 +1830,13 @@ proto.WorkingInitMessage.prototype.serializeBinaryToWriter = function (writer) {
f
);
}
f = this.getBuiltinExtensionsDir();
if (f.length > 0) {
writer.writeString(
7,
f
);
}
};
@ -1927,6 +1939,21 @@ proto.WorkingInitMessage.prototype.setShell = function(value) {
};
/**
* optional string builtin_extensions_dir = 7;
* @return {string}
*/
proto.WorkingInitMessage.prototype.getBuiltinExtensionsDir = function() {
return /** @type {string} */ (jspb.Message.getFieldProto3(this, 7, ""));
};
/** @param {string} value */
proto.WorkingInitMessage.prototype.setBuiltinExtensionsDir = function(value) {
jspb.Message.setField(this, 7, value);
};
/**
* @enum {number}
*/

13
packages/protocol/test/modules/fs.test.ts
View File

@ -536,7 +536,7 @@ describe("fs", () => {
});
describe("stat", () => {
it("should stat", (done) => {
it("should stat file", (done) => {
fs.stat(testFile, (err, stats) => {
expect(err).toBeUndefined();
expect(stats.size).toBeGreaterThan(0);
@ -546,6 +546,17 @@ describe("fs", () => {
});
});
it("should stat folder", (done) => {
const dir = tmpFile();
nativeFs.mkdirSync(dir);
fs.stat(dir, (err, stats) => {
expect(err).toBeUndefined();
expect(stats.isDirectory()).toBeTruthy();
done();
});
});
it("should fail to stat", (done) => {
fs.stat(path.join(__dirname, "no-exist"), (err, stats) => {
expect(err).toBeDefined();

23
packages/protocol/yarn.lock
View File

@ -236,22 +236,22 @@ ms@2.0.0:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
nan@2.10.0:
version "2.10.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f"
integrity sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==
nan@2.12.1:
version "2.12.1"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552"
integrity sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==
negotiator@0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=
node-pty@^0.8.0:
version "0.8.0"
resolved "https://registry.yarnpkg.com/node-pty/-/node-pty-0.8.0.tgz#08bccb633f49e2e3f7245eb56ea6b40f37ccd64f"
integrity sha512-g5ggk3gN4gLrDmAllee5ScFyX3YzpOC/U8VJafha4pE7do0TIE1voiIxEbHSRUOPD1xYqmY+uHhOKAd3avbxGQ==
node-pty@^0.8.1:
version "0.8.1"
resolved "https://registry.yarnpkg.com/node-pty/-/node-pty-0.8.1.tgz#94b457bec013e7a09b8d9141f63b0787fa25c23f"
integrity sha512-j+/g0Q5dR+vkELclpJpz32HcS3O/3EdPSGPvDXJZVJQLCvgG0toEbfmymxAEyQyZEpaoKHAcoL+PvKM+4N9nlw==
dependencies:
nan "2.10.0"
nan "2.12.1"
on-finished@~2.3.0:
version "2.3.0"
@ -364,6 +364,11 @@ ts-protoc-gen@^0.8.0:
dependencies:
google-protobuf "^3.6.1"
tslib@^1.9.3:
version "1.9.3"
resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.3.tgz#d7e4dd79245d85428c4d7e4822a79917954ca286"
integrity sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==
type-is@~1.6.16:
version "1.6.16"
resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.16.tgz#f89ce341541c672b25ee7ae3c73dee3b2be50194"

1
packages/server/.gitignore vendored
View File

@ -1,6 +1,7 @@
out
cli*
build
resources
# This file is generated when the binary is created.
# We want to use the parent tsconfig so we can ignore it.

13
packages/server/package.json
View File

@ -4,12 +4,12 @@
"bin": "./out/cli.js",
"files": [],
"scripts": {
"start": "ts-node -r tsconfig-paths/register src/cli.ts",
"start": "node --max-old-space-size=32384 --require ts-node/register --require tsconfig-paths/register src/cli.ts",
"build:webpack": "rm -rf ./out && export CLI=true && ../../node_modules/.bin/webpack --config ./webpack.config.js",
"build:nexe": "node scripts/nexe.js",
"build:bootstrap-fork": "cd ../vscode && npm run build:bootstrap-fork; cp ./bin/bootstrap-fork.js ../server/build/bootstrap-fork.js",
"build:default-extensions": "cd ../../lib/vscode && npx gulp vscode-linux-arm && cd ../.. && cp -r ./lib/VSCode-linux-arm/resources/app/extensions/* ./packages/server/build/extensions/",
"build:web": "cd ../web && npm run build; mkdir ../server/build/web && cp ./out/* ../server/build/web",
"build:bootstrap-fork": "cd ../vscode && npm run build:bootstrap-fork; mkdir -p ./packages/server/resources; cp ./bin/bootstrap-fork.js ../server/resources/bootstrap-fork.js",
"build:default-extensions": "cd ../../lib/vscode && npx gulp vscode-linux-arm && cd ../.. && mkdir -p ./packages/server/resources; cp -r ./lib/VSCode-linux-arm/resources/app/extensions/* ./packages/server/resources/extensions/",
"build:web": "cd ../web; rm -rf ./out; NODE_ENV=production npm run build; rm -rf ../server/resources/web; mkdir -p ../server/resources/web; cp -r ./out/* ../server/resources/web",
"build": "npm run build:bootstrap-fork && npm run build:webpack && npm run build:nexe"
},
"dependencies": {
@ -17,17 +17,20 @@
"@oclif/errors": "^1.2.2",
"@oclif/plugin-help": "^2.1.4",
"express": "^4.16.4",
"express-static-gzip": "^1.1.3",
"mime-types": "^2.1.21",
"nexe": "^2.0.0-rc.34",
"node-pty": "^0.8.1",
"segfault-handler": "^1.0.1",
"spdlog": "^0.7.2",
"ws": "^6.1.2",
"xhr2": "^0.1.4"
},
"devDependencies": {
"@types/express": "^4.16.0",
"@types/fs-extra": "^5.0.4",
"@types/mime-types": "^2.1.0",
"@types/ws": "^6.0.1",
"fs-extra": "^7.0.1",
"string-replace-webpack-plugin": "^0.1.3",
"ts-node": "^7.0.1",
"tsconfig-paths": "^3.7.0",

58
packages/server/scripts/nexe.js
View File

@ -1,7 +1,9 @@
const fs = require("fs");
const fse = require("fs-extra");
const os = require("os");
const path = require("path");
const nexe = require("nexe");
const zlib = require("zlib");
const nexeRoot = path.join(os.homedir(), ".nexe");
if (!fs.existsSync(nexeRoot)) {
@ -19,36 +21,52 @@ listed.forEach((list) => {
}
});
const tmpDir = path.join(__dirname, "../build");
if (fs.existsSync(tmpDir)) {
console.log("Removing old build dir...");
fse.removeSync(tmpDir);
}
console.log("Copying build files...");
fse.copySync(path.join(__dirname, "../resources"), tmpDir, {
recursive: true,
});
const modDir = path.join(tmpDir, "modules");
fs.mkdirSync(modDir);
fse.copySync(path.join(__dirname, "../../protocol/node_modules/node-pty/build/Release/pty.node"), path.join(modDir, "pty.node"));
const zipper = (p) => {
const stat = fs.statSync(p);
if (!stat.isDirectory()) {
fs.writeFileSync(p + ".gz", zlib.gzipSync(fs.readFileSync(p)));
fse.removeSync(p);
return;
}
const files = fs.readdirSync(p);
files.forEach((f) => zipper(path.join(p, f)));
};
zipper(path.join(tmpDir, "web"));
zipper(path.join(tmpDir, "bootstrap-fork.js"));
nexe.compile({
debugBundle: true,
input: path.join(__dirname, "../out/cli.js"),
output: 'cli',
native: {
"node-pty": {
additionalFiles: [
'./node_modules/node-pty/build/Release/pty',
],
},
"spdlog": {
additionalFiles: [
'spdlog.node',
],
},
},
targets: ["linux"],
native: {
spdlog: {
additionalFiles: [
'spdlog.node'
],
},
},
/**
* To include native extensions, do NOT install node_modules for each one. They
* are not required as each extension is built using webpack.
*/
resources: [
path.join(__dirname, "../package.json"),
path.join(__dirname, "../build/**"),
path.join(__dirname, "../build/**/*"),
],
});
/**
* Notes for tmrw
*
* `upx ~/.nexe/linux` <- node binary before compiling with nexe
* Use `testing.js` for bundling with nexe to build
*/

17
packages/server/src/cli.ts
View File

@ -8,6 +8,8 @@ import * as WebSocket from "ws";
import { createApp } from "./server";
import { requireModule } from "./vscode/bootstrapFork";
import { SharedProcess, SharedProcessState } from "./vscode/sharedProcess";
import { setup as setupNativeModules } from './modules';
import { fillFs } from './fill';
export class Entry extends Command {
@ -49,12 +51,17 @@ export class Entry extends Command {
logger.warn("Failed to remove extracted dependency.", field("dependency", "spdlog"), field("error", ex.message));
}
if (process.env.CLI) {
fillFs();
}
const { args, flags } = this.parse(Entry);
if (flags.env) {
Object.assign(process.env, JSON.parse(flags.env));
}
const builtInExtensionsDir = path.join(process.env.BUILD_DIR || path.join(__dirname, ".."), "build/extensions");
if (flags["bootstrap-fork"]) {
const modulePath = flags["bootstrap-fork"];
if (!modulePath) {
@ -62,7 +69,7 @@ export class Entry extends Command {
process.exit(1);
}
return requireModule(modulePath);
return requireModule(modulePath, builtInExtensionsDir);
}
const dataDir = flags["data-dir"] || path.join(os.homedir(), ".vscode-online");
@ -73,6 +80,11 @@ export class Entry extends Command {
process.exit(1);
}
if (!fs.existsSync(dataDir)) {
fs.mkdirSync(dataDir);
}
setupNativeModules(dataDir);
const logDir = path.join(dataDir, "logs", new Date().toISOString().replace(/[-:.TZ]/g, ""));
process.env.VSCODE_LOGS = logDir;
@ -80,7 +92,7 @@ export class Entry extends Command {
// TODO: fill in appropriate doc url
logger.info("Additional documentation: https://coder.com/docs");
logger.info("Initializing", field("data-dir", dataDir), field("working-dir", workingDir), field("log-dir", logDir));
const sharedProcess = new SharedProcess(dataDir);
const sharedProcess = new SharedProcess(dataDir, builtInExtensionsDir);
logger.info("Starting shared process...", field("socket", sharedProcess.socketPath));
const sendSharedProcessReady = (socket: WebSocket): void => {
const active = new SharedProcessActiveMessage();
@ -120,6 +132,7 @@ export class Entry extends Command {
app.use(require("webpack-hot-middleware")(compiler));
}
}, {
builtInExtensionsDirectory: builtInExtensionsDir,
dataDirectory: dataDir,
workingDirectory: workingDir,
});

154
packages/server/src/fill.ts Normal file
View File

@ -0,0 +1,154 @@
import * as fs from "fs";
import * as util from "util";
const oldAccess = fs.access;
const existsWithinBinary = (path: fs.PathLike): Promise<boolean> => {
return new Promise<boolean>((resolve) => {
if (typeof path === "number") {
if (path < 0) {
return resolve(true);
}
}
oldAccess(path, fs.constants.F_OK, (err) => {
const exists = !err;
const es = fs.existsSync(path);
const res = !exists && es;
resolve(res);
});
});
};
export const fillFs = () => {
/**
* Refer to https://github.com/nexe/nexe/blob/master/src/fs/patch.ts
* For impls
*/
if (!process.env.CLI) {
throw new Error("Should not fill FS when not in CLI");
}
interface FD {
readonly path: string;
position: number;
}
let lastFd = Number.MIN_SAFE_INTEGER;
const fds = new Map<number, FD>();
const replaceNative = <T extends keyof typeof fs>(propertyName: T, func: (callOld: () => void, ...args: any[]) => any, customPromisify?: (...args: any[]) => Promise<any>): void => {
const oldFunc = (<any>fs)[propertyName];
fs[propertyName] = (...args: any[]) => {
try {
return func(() => {
return oldFunc(...args);
}, ...args);
} catch (ex) {
return oldFunc(...args);
}
};
if (customPromisify) {
(<any>fs[propertyName])[util.promisify.custom] = customPromisify;
}
};
replaceNative("access", (callNative, path, mode, callback) => {
existsWithinBinary(path).then((exists) => {
if (!exists) {
return callNative();
}
return callback();
});
});
replaceNative("exists", (callOld, path, callback) => {
existsWithinBinary(path).then((exists) => {
if (exists) {
return callback(true);
}
return callOld();
});
}, (path) => new Promise((res) => fs.exists(path, res)));
replaceNative("open", (callOld, path: fs.PathLike, flags: string | Number, mode: any, callback: any) => {
existsWithinBinary(path).then((exists) => {
if (!exists) {
return callOld();
}
if (typeof mode === "function") {
callback = mode;
mode = undefined;
}
if (path === process.execPath) {
return callOld();
}
const fd = lastFd++;
fds.set(fd, {
path: path.toString(),
position: 0,
});
callback(undefined, fd);
});
});
replaceNative("close", (callOld, fd: number, callback) => {
if (!fds.has(fd)) {
return callOld();
}
fds.delete(fd);
callback();
});
replaceNative("read", (callOld, fd: number, buffer: Buffer, offset: number, length: number, position: number | null, callback?: (err: NodeJS.ErrnoException, bytesRead: number, buffer: Buffer) => void, ) => {
if (!fds.has(fd)) {
return callOld();
}
const fileDesc = fds.get(fd)!;
/**
* `readFile` is filled within nexe, but `read` is not
* https://github.com/nexe/nexe/blob/master/src/fs/patch.ts#L199
* We can simulate a real _read_ by reading the entire file.
* Efficiency can be improved here by storing the entire file in memory
* until it has been closed.
*/
return fs.readFile(fileDesc.path, (err, rb) => {
if (err) {
return callOld();
}
rb = rb.slice(position || fileDesc.position);
const sliced = rb.slice(0, length);
if (position === null) {
fileDesc.position += sliced.byteLength;
}
buffer.set(sliced, offset);
if (callback) {
callback(undefined!, sliced.byteLength, buffer);
}
});
}, (fd: number, buffer: Buffer, offset: number, length: number, position: number | null): Promise<{
bytesRead: number;
buffer: Buffer;
}> => {
return new Promise((res, rej) => {
fs.read(fd, buffer, offset, length, position, (err, bytesRead, buffer) => {
if (err) {
return rej(err);
}
res({
bytesRead,
buffer,
});
});
});
});
};

41
packages/server/src/modules.ts Normal file
View File

@ -0,0 +1,41 @@
import * as fs from "fs";
import * as path from "path";
declare var __non_webpack_require__: typeof require;
/**
* Handling of native modules within the CLI
*/
export const setup = (dataDirectory: string): void => {
if (!process.env.CLI) {
return;
}
try {
fs.mkdirSync(path.join(dataDirectory, "modules"));
} catch (ex) {
if (ex.code !== "EEXIST") {
throw ex;
}
}
const unpackModule = (moduleName: string): void => {
const memFile = path.join(process.env.BUILD_DIR!, "build/modules", moduleName + ".node");
const diskFile = path.join(dataDirectory, "modules", moduleName + ".node");
if (!fs.existsSync(diskFile)) {
fs.writeFileSync(diskFile, fs.readFileSync(memFile));
}
};
/**
* We need to unpack node-pty and patch its `loadNative` function to require our unpacked pty.node
* If pty.node isn't unpacked a SIGSEGV is thrown and the application exits. The exact reasoning
* for this is unknown ATM, but this patch works around it.
*/
unpackModule("pty");
const nodePtyUtils = require("../../protocol/node_modules/node-pty/lib/utils") as typeof import("../../protocol/node_modules/node-pty/src/utils");
nodePtyUtils.loadNative = (modName: string) => {
return __non_webpack_require__(path.join(dataDirectory, "modules", modName + ".node"));
};
require("../../protocol/node_modules/node-pty/lib/index") as typeof import("../../protocol/node_modules/node-pty/src/index");
};

12
packages/server/src/server.ts
View File

@ -4,6 +4,8 @@ import { Server, ServerOptions } from "@coder/protocol/src/node/server";
import { NewSessionMessage } from '@coder/protocol/src/proto';
import { ChildProcess } from "child_process";
import * as express from "express";
//@ts-ignore
import * as expressStaticGzip from "express-static-gzip";
import * as fs from "fs";
import * as http from "http";
import * as mime from "mime-types";
@ -67,8 +69,12 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
} : undefined);
});
app.use(express.static(path.join(process.env.BUILD_DIR || path.join(__dirname, ".."), "build/web")));
const baseDir = process.env.BUILD_DIR || path.join(__dirname, "..");
if (process.env.CLI) {
app.use(expressStaticGzip(path.join(baseDir, "build/web")));
} else {
app.use(express.static(path.join(baseDir, "resources/web")));
}
app.get("/resource/:url(*)", async (req, res) => {
try {
const fullPath = `/${req.params.url}`;
@ -76,7 +82,7 @@ export const createApp = (registerMiddleware?: (app: express.Application) => voi
// if (relative.startsWith("..")) {
// return res.status(403).end();
// }
const exists = await util.promisify(fs.exists)(fullPath);
const exists = fs.existsSync(fullPath);
if (!exists) {
res.status(404).end();
return;

43
packages/server/src/vscode/bootstrapFork.ts
View File

@ -1,19 +1,50 @@
import * as cp from "child_process";
import * as fs from "fs";
import * as path from "path";
import * as zlib from "zlib";
import * as vm from "vm";
export const requireModule = (modulePath: string): void => {
export const requireModule = (modulePath: string, builtInExtensionsDir: string): void => {
process.env.AMD_ENTRYPOINT = modulePath;
const xml = require("xhr2");
// tslint:disable-next-line no-any this makes installing extensions work.
(global as any).XMLHttpRequest = xml.XMLHttpRequest;
const mod = require("module") as typeof import("module");
/**
* Used for loading extensions. Using __non_webpack_require__ didn't work
* as it was not resolving to the FS.
*/
(global as any).nativeNodeRequire = (id: string) => {
const customMod = new mod.Module(id);
customMod.filename = id;
customMod.paths = (<any>mod)._nodeModulePaths(path.dirname(id));
if (id.startsWith(builtInExtensionsDir)) {
customMod.loaded = true;
const req = vm.runInThisContext(mod.wrap(fs.readFileSync(id + ".js").toString()), {
displayErrors: true,
filename: id + ".js",
});
req(customMod.exports, customMod.require.bind(customMod), customMod, __filename, path.dirname(id));
return customMod.exports;
}
return customMod.require(id);
};
// Always do this so we can see console.logs.
// process.env.VSCODE_ALLOW_IO = "true";
const content = fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build/bootstrap-fork.js"));
let content: Buffer | undefined;
const readFile = (name: string): Buffer => {
return fs.readFileSync(path.join(process.env.BUILD_DIR as string || path.join(__dirname, "../.."), "./build", name));
};
if (process.env.CLI) {
content = zlib.gunzipSync(readFile("bootstrap-fork.js.gz"));
} else {
content = readFile("../resources/bootstrap-fork.js");
}
eval(content.toString());
};
@ -36,9 +67,9 @@ export const forkModule = (modulePath: string, env?: NodeJS.ProcessEnv): cp.Chil
stdio: [null, null, null, "ipc"],
};
if (process.env.CLI === "true") {
proc = cp.spawn(process.execPath, args, options);
proc = cp.execFile(process.execPath, args, options);
} else {
proc = cp.spawn(process.execArgv[0], ["-r", "tsconfig-paths/register", process.argv[1], ...args], options);
proc = cp.spawn(process.execPath, ["--require", "ts-node/register", "--require", "tsconfig-paths/register", process.argv[1], ...args], options);
}
return proc;

3
packages/server/src/vscode/sharedProcess.ts
View File

@ -31,6 +31,7 @@ export class SharedProcess {
public constructor(
private readonly userDataDir: string,
private readonly builtInExtensionsDir: string,
) {
this.onStateEmitter = new Emitter();
this.restart();
@ -87,7 +88,7 @@ export class SharedProcess {
logLevel: LogLevel;
} = {
args: {
"builtin-extensions-dir": path.join(process.env.BUILD_DIR || path.join(__dirname, "../.."), "build/extensions"),
"builtin-extensions-dir": this.builtInExtensionsDir,
"user-data-dir": this.userDataDir,
"extensions-dir": extensionsDir,
} as any,

13
packages/server/webpack.config.js
View File

@ -20,6 +20,17 @@ module.exports = merge({
]
}),
},
{
test: /node\-pty\/lib\/index\.js/,
loader: StringReplacePlugin.replace({
replacements: [
{
pattern: /exports\.native.*;/,
replacement: () => "exports.native = null;",
}
]
}),
},
],
},
output: {
@ -36,7 +47,7 @@ module.exports = merge({
__dirname: false,
setImmediate: false
},
externals: ["node-pty", "spdlog"],
externals: ["spdlog", "tslib"],
entry: "./packages/server/src/cli.ts",
target: "node",
plugins: [

65
packages/server/yarn.lock
View File

@ -102,6 +102,13 @@
"@types/express-serve-static-core" "*"
"@types/serve-static" "*"
"@types/fs-extra@^5.0.4":
version "5.0.4"
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.0.4.tgz#b971134d162cc0497d221adde3dbb67502225599"
integrity sha512-DsknoBvD8s+RFfSGjmERJ7ZOP1HI0UZRA3FSI+Zakhrc/Gy26YQsLI+m5V5DHxroHRJqCDLKJp7Hixn8zyaF7g==
dependencies:
"@types/node" "*"
"@types/json5@^0.0.29":
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
@ -192,9 +199,9 @@ amdefine@>=0.0.4:
integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=
ansi-escapes@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.1.0.tgz#f73207bb81207d75fd6c83f125af26eea378ca30"
integrity sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==
version "3.2.0"
resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b"
integrity sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==
ansi-regex@^2.0.0:
version "2.1.1"
@ -423,6 +430,13 @@ binary-extensions@^1.0.0:
resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.12.0.tgz#c2d780f53d45bba8317a8902d4ceeaf3a6385b14"
integrity sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==
bindings@^1.2.1:
version "1.4.0"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.4.0.tgz#909efa49f2ebe07ecd3cb136778f665052040127"
integrity sha512-7znEVX22Djn+nYjxCWKDne0RRloa9XfYa84yk3s+HkE3LpDYZmhArYr9O9huBoHY3/oXispx5LorIX7Sl2CgSQ==
dependencies:
file-uri-to-path "1.0.0"
bindings@^1.3.0:
version "1.3.1"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.1.tgz#21fc7c6d67c18516ec5aaa2815b145ff77b26ea5"
@ -453,9 +467,9 @@ body-parser@1.18.3:
type-is "~1.6.16"
bowser@^2.0.0-beta.3:
version "2.0.0-beta.3"
resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.0.0-beta.3.tgz#1b74d4fd69199aa237bc0f1f4c5e80df711635d8"
integrity sha512-eHZmZrwbWb8pZn7XtIrtpni/EjHgU6DZEcvJUrLRrTtnhBY8Kbj379JY+GGKeJ7CctidvWP2ueBAAzGDenydCg==
version "2.1.0"
resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.1.0.tgz#76cc094f97578ba4858fb4359445ee1317d1be6f"
integrity sha512-tP90ci4QY8PRBQjU0+iTsoO3DMNYtXCM0aVxeKhjxXF8IH9xTXUmjcTECPN+y5v0BGeRDfMcSLeohPiUZuz37g==
brace-expansion@^1.1.7:
version "1.1.11"
@ -1081,6 +1095,13 @@ expand-range@^1.8.1:
dependencies:
fill-range "^2.1.0"
express-static-gzip@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/express-static-gzip/-/express-static-gzip-1.1.3.tgz#345ea02637d9d5865777d6fb57ccc0884abcda65"
integrity sha512-k8Q4Dx4PDpzEb8kth4uiPWrBeJWJYSgnWMzNdjQUOsEyXfYKbsyZDkU/uXYKcorRwOie5Vzp4RMEVrJLMfB6rA==
dependencies:
serve-static "^1.12.3"
express@^4.14.0, express@^4.16.4:
version "4.16.4"
resolved "https://registry.yarnpkg.com/express/-/express-4.16.4.tgz#fddef61926109e24c515ea97fd2f1bdbf62df12e"
@ -1258,6 +1279,11 @@ file-type@^6.1.0:
resolved "https://registry.yarnpkg.com/file-type/-/file-type-6.2.0.tgz#e50cd75d356ffed4e306dc4f5bcf52a79903a919"
integrity sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg==
file-uri-to-path@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
filename-regex@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
@ -1374,7 +1400,7 @@ fs-constants@^1.0.0:
resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
fs-extra@^7.0.0:
fs-extra@^7.0.0, fs-extra@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
integrity sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==
@ -2318,7 +2344,7 @@ mute-stream@0.0.7:
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"
integrity sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=
nan@2.12.1, nan@^2.8.0, nan@^2.9.2:
nan@^2.0.9, nan@^2.8.0, nan@^2.9.2:
version "2.12.1"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.12.1.tgz#7b1aa193e9aa86057e3c7bbd0ac448e770925552"
integrity sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==
@ -2396,13 +2422,6 @@ node-pre-gyp@^0.10.0:
semver "^5.3.0"
tar "^4"
node-pty@^0.8.1:
version "0.8.1"
resolved "https://registry.yarnpkg.com/node-pty/-/node-pty-0.8.1.tgz#94b457bec013e7a09b8d9141f63b0787fa25c23f"
integrity sha512-j+/g0Q5dR+vkELclpJpz32HcS3O/3EdPSGPvDXJZVJQLCvgG0toEbfmymxAEyQyZEpaoKHAcoL+PvKM+4N9nlw==
dependencies:
nan "2.12.1"
nopt@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d"
@ -3003,6 +3022,14 @@ seek-bzip@^1.0.5:
dependencies:
commander "~2.8.1"
segfault-handler@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/segfault-handler/-/segfault-handler-1.0.1.tgz#9466d8f77d8826cfbdfa811124ece02983fd6ad2"
integrity sha512-3koBV3F0IPxTYacnoL7WoFlaMndXsXj62tiVbbW9Kzp3K+F9Y6GWW5XmKSv7j+7nf2M+qjNzc4W1iZoa8vocjw==
dependencies:
bindings "^1.2.1"
nan "^2.0.9"
semver@^5.3.0, semver@^5.6.0:
version "5.6.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.6.0.tgz#7e74256fbaa49c75aa7c7a205cc22799cac80004"
@ -3027,7 +3054,7 @@ send@0.16.2:
range-parser "~1.2.0"
statuses "~1.4.0"
serve-static@1.13.2:
serve-static@1.13.2, serve-static@^1.12.3:
version "1.13.2"
resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.13.2.tgz#095e8472fd5b46237db50ce486a43f4b86c6cec1"
integrity sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==
@ -3191,9 +3218,9 @@ split-string@^3.0.1, split-string@^3.0.2:
extend-shallow "^3.0.0"
sshpk@^1.7.0:
version "1.16.0"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.0.tgz#1d4963a2fbffe58050aa9084ca20be81741c07de"
integrity sha512-Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ==
version "1.16.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==
dependencies:
asn1 "~0.2.3"
assert-plus "^1.0.0"

3
packages/vscode/webpack.config.bootstrap.js
View File

@ -16,7 +16,7 @@ module.exports = (env) => {
entry: path.join(root, "lib/vscode/src/bootstrap-fork.js"),
mode: "development",
target: "node",
externals: ["node-pty", "spdlog"],
externals: ["spdlog"],
output: {
chunkFilename: "[name].bundle.js",
path: path.resolve(__dirname, "./bin"),
@ -42,6 +42,7 @@ module.exports = (env) => {
"gc-signals": path.join(fills, "empty.ts"),
"native-keymap": path.join(fills, "native-keymap.ts"),
"windows-process-tree": path.resolve(fills, "empty.ts"),
"node-pty": path.resolve(fills, "empty.ts"),
"electron": path.join(vscodeFills, "stdioElectron.ts"),
"native-watchdog": path.join(vscodeFills, "native-watchdog.ts"),

2
packages/web/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@coder/web",
"scripts": {
"build": "../../node_modules/.bin/webpack --config ./webpack.dev.config.js"
"build": "UV_THREADPOOL_SIZE=100 node --max-old-space-size=32384 ../../node_modules/webpack/bin/webpack.js --config ./webpack.config.js"
}
}

5
packages/web/webpack.dev.config.js → packages/web/webpack.config.js
View File

@ -2,9 +2,12 @@ const path = require("path");
const webpack = require("webpack");
const merge = require("webpack-merge");
const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
const UglifyJsPlugin = require('uglifyjs-webpack-plugin');
const prod = process.env.NODE_ENV === "production";
module.exports = merge(require("./webpack.common.config.js"), {
devtool: "cheap-module-eval-source-map",
devtool: prod ? "source-map" : "cheap-module-eval-source-map",
mode: prod ? "production" : "development",
output: {
path: path.join(__dirname, "out"),
},

2
tslint.json
View File

@ -9,7 +9,7 @@
"class-name": true,
"eofline": true,
"import-spacing": true,
"no-angle-bracket-type-assertion": true,
"no-angle-bracket-type-assertion": false,
"no-bitwise": false,
"no-any": true,
"newline-before-return": true,