OCram85/code-server
Archived
1
0
Fork 0
Code Pull Requests 2 Activity

Improve retry

Browse Source 
Registering returns an instance that lets you retry and recover without
needing to keep passing the name everywhere.

Also refactored the shared process a little to make better use of the
retry and downgraded stderr messages to warnings because they aren't
critical.
This commit is contained in:
Asher
2019-04-01 13:31:34 -05:00
parent 3fec7f432c
commit 033ef151ca
4 changed files with 276 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
packages/ide/src/fill/client.ts
View File

@ -11,7 +11,7 @@ class WebsocketConnection implements ReadWriteConnection {
private activeSocket: WebSocket | undefined;
private readonly messageBuffer = <Uint8Array[]>[];
private readonly socketTimeoutDelay = 60 * 1000;
private readonly retryName = "Socket";
private readonly retry = retry.register("Socket", () => this.connect());
private isUp: boolean = false;
private closed: boolean = false;
@ -26,11 +26,14 @@ class WebsocketConnection implements ReadWriteConnection {
public readonly onMessage = this.messageEmitter.event;
public constructor() {
retry.register(this.retryName, () => this.connect());
retry.block(this.retryName);
retry.run(this.retryName);
this.retry.block();
this.retry.run();
}
/**
* Send data across the socket. If closed, will error. If connecting, will
* queue.
*/
public send(data: Buffer | Uint8Array): void {
if (this.closed) {
throw new Error("web socket is closed");
@ -42,6 +45,9 @@ class WebsocketConnection implements ReadWriteConnection {
}
}
/**
* Close socket connection.
*/
public close(): void {
this.closed = true;
this.dispose();
@ -75,8 +81,8 @@ class WebsocketConnection implements ReadWriteConnection {
field("wasClean", event.wasClean),
);
if (!this.closed) {
retry.block(this.retryName);
retry.run(this.retryName);
this.retry.block();
this.retry.run();
}
});
@ -108,15 +114,19 @@ class WebsocketConnection implements ReadWriteConnection {
}, this.socketTimeoutDelay);
await new Promise((resolve, reject): void => {
const onClose = (): void => {
const doReject = (): void => {
clearTimeout(socketWaitTimeout);
socket.removeEventListener("close", onClose);
socket.removeEventListener("error", doReject);
socket.removeEventListener("close", doReject);
reject();
};
socket.addEventListener("close", onClose);
socket.addEventListener("error", doReject);
socket.addEventListener("close", doReject);
socket.addEventListener("open", async () => {
socket.addEventListener("open", () => {
clearTimeout(socketWaitTimeout);
socket.removeEventListener("error", doReject);
socket.removeEventListener("close", doReject);
resolve();
});
});

145
packages/ide/src/retry.ts
View File

@ -1,14 +1,64 @@
import { logger, field } from "@coder/logger";
import { NotificationService, INotificationHandle, INotificationService, Severity } from "./fill/notification";
// tslint:disable no-any can have different return values
interface IRetryItem {
/**
* How many times this item has been retried.
*/
count?: number;
delay?: number; // In seconds.
end?: number; // In ms.
fn(): any | Promise<any>; // tslint:disable-line no-any can have different return values
/**
* In seconds.
*/
delay?: number;
/**
* In milliseconds.
*/
end?: number;
/**
* Function to run when retrying.
*/
fn(): any;
/**
* Timer for running this item.
*/
timeout?: number | NodeJS.Timer;
/**
* Whether the item is retrying or waiting to retry.
*/
running?: boolean;
showInNotification: boolean;
}
/**
* An retry-able instance.
*/
export interface RetryInstance {
/**
* Run this retry.
*/
run(error?: Error): void;
/**
* Block on this instance.
*/
block(): void;
}
/**
* A retry-able instance that doesn't use a promise so it must be manually
* ran again on failure and recovered on success.
*/
export interface ManualRetryInstance extends RetryInstance {
/**
* Mark this item as recovered.
*/
recover(): void;
}
/**
@ -21,7 +71,7 @@ interface IRetryItem {
* to the user explaining what is happening with an option to immediately retry.
*/
export class Retry {
private items = new Map<string, IRetryItem>();
private readonly items = new Map<string, IRetryItem>();
// Times are in seconds.
private readonly retryMinDelay = 1;
@ -50,13 +100,54 @@ export class Retry {
}
/**
* Block retries when we know they will fail (for example when starting Wush
* back up). If a name is passed, that service will still be allowed to retry
* Register a function to retry that starts/connects to a service.
*
* The service is automatically retried or recovered when the promise resolves
* or rejects. If the service dies after starting, it must be manually
* retried.
*/
public register(name: string, fn: () => Promise<any>): RetryInstance;
/**
* Register a function to retry that starts/connects to a service.
*
* Must manually retry if it fails to start again or dies after restarting and
* manually recover if it succeeds in starting again.
*/
public register(name: string, fn: () => any): ManualRetryInstance;
/**
* Register a function to retry that starts/connects to a service.
*/
public register(name: string, fn: () => any): RetryInstance | ManualRetryInstance {
if (this.items.has(name)) {
throw new Error(`"${name}" is already registered`);
}
this.items.set(name, { fn });
return {
block: (): void => this.block(name),
run: (error?: Error): void => this.run(name, error),
recover: (): void => this.recover(name),
};
}
/**
* Un-register a function to retry.
*/
public unregister(name: string): void {
if (!this.items.has(name)) {
throw new Error(`"${name}" is not registered`);
}
this.items.delete(name);
}
/**
* Block retries when we know they will fail (for example when the socket is
* down ). If a name is passed, that service will still be allowed to retry
* (unless we have already blocked).
*
* Blocking without a name will override a block with a name.
*/
public block(name?: string): void {
private block(name?: string): void {
if (!this.blocked || !name) {
this.blocked = name || true;
this.items.forEach((item) => {
@ -68,7 +159,7 @@ export class Retry {
/**
* Unblock retries and run any that are pending.
*/
public unblock(): void {
private unblock(): void {
this.blocked = false;
this.items.forEach((item, name) => {
if (item.running) {
@ -77,35 +168,10 @@ export class Retry {
});
}
/**
* Register a function to retry that starts/connects to a service.
*
* If the function returns a promise, it will automatically be retried,
* recover, & unblock after calling `run` once (otherwise they need to be
* called manually).
*/
// tslint:disable-next-line no-any can have different return values
public register(name: string, fn: () => any | Promise<any>, showInNotification: boolean = true): void {
if (this.items.has(name)) {
throw new Error(`"${name}" is already registered`);
}
this.items.set(name, { fn, showInNotification });
}
/**
* Unregister a function to retry.
*/
public unregister(name: string): void {
if (!this.items.has(name)) {
throw new Error(`"${name}" is not registered`);
}
this.items.delete(name);
}
/**
* Retry a service.
*/
public run(name: string, error?: Error): void {
private run(name: string, error?: Error): void {
if (!this.items.has(name)) {
throw new Error(`"${name}" is not registered`);
}
@ -149,7 +215,7 @@ export class Retry {
/**
* Reset a service after a successfully recovering.
*/
public recover(name: string): void {
private recover(name: string): void {
if (!this.items.has(name)) {
throw new Error(`"${name}" is not registered`);
}
@ -191,9 +257,9 @@ export class Retry {
if (this.blocked === name) {
this.unblock();
}
}).catch(() => {
}).catch((error) => {
endItem();
this.run(name);
this.run(name, error);
});
} else {
endItem();
@ -214,8 +280,7 @@ export class Retry {
const now = Date.now();
const items = Array.from(this.items.entries()).filter(([_, item]) => {
return item.showInNotification
&& typeof item.end !== "undefined"
return typeof item.end !== "undefined"
&& item.end > now
&& item.delay && item.delay >= this.notificationThreshold;
}).sort((a, b) => {

215
packages/server/src/vscode/sharedProcess.ts
View File

@ -1,5 +1,6 @@
import { ChildProcess } from "child_process";
import * as fs from "fs";
import * as fse from "fs-extra";
import * as os from "os";
import * as path from "path";
import { forkModule } from "./bootstrapFork";
@ -7,7 +8,7 @@ import { StdioIpcHandler } from "../ipc";
import { ParsedArgs } from "vs/platform/environment/common/environment";
import { Emitter } from "@coder/events/src";
import { retry } from "@coder/ide/src/retry";
import { logger, field, Level } from "@coder/logger";
import { logger, Level } from "@coder/logger";
export enum SharedProcessState {
Stopped,
@ -23,129 +24,143 @@ export type SharedProcessEvent = {
};
export class SharedProcess {
public readonly socketPath: string = os.platform() === "win32" ? path.join("\\\\?\\pipe", os.tmpdir(), `.code-server${Math.random().toString()}`) : path.join(os.tmpdir(), `.code-server${Math.random().toString()}`);
public readonly socketPath: string = os.platform() === "win32"
? path.join("\\\\?\\pipe", os.tmpdir(), `.code-server${Math.random().toString()}`)
: path.join(os.tmpdir(), `.code-server${Math.random().toString()}`);
private _state: SharedProcessState = SharedProcessState.Stopped;
private activeProcess: ChildProcess | undefined;
private ipcHandler: StdioIpcHandler | undefined;
private readonly onStateEmitter = new Emitter<SharedProcessEvent>();
public readonly onState = this.onStateEmitter.event;
private readonly retryName = "Shared process";
private readonly logger = logger.named("shared");
private readonly retry = retry.register("Shared process", () => this.connect());
private disposed: boolean = false;
public constructor(
private readonly userDataDir: string,
private readonly builtInExtensionsDir: string,
) {
retry.register(this.retryName, () => this.restart());
retry.run(this.retryName);
this.retry.run();
}
public get state(): SharedProcessState {
return this._state;
}
public restart(): void {
if (this.activeProcess && !this.activeProcess.killed) {
this.activeProcess.kill();
}
const extensionsDir = path.join(this.userDataDir, "extensions");
const mkdir = (dir: string): void => {
try {
fs.mkdirSync(dir);
} catch (ex) {
if (ex.code !== "EEXIST" && ex.code !== "EISDIR") {
throw ex;
}
}
};
mkdir(this.userDataDir);
mkdir(extensionsDir);
const backupsDir = path.join(this.userDataDir, "Backups");
mkdir(backupsDir);
const workspacesFile = path.join(backupsDir, "workspaces.json");
if (!fs.existsSync(workspacesFile)) {
fs.closeSync(fs.openSync(workspacesFile, "w"));
}
this.setState({
state: SharedProcessState.Starting,
});
let resolved: boolean = false;
const maybeStop = (error: string): void => {
if (resolved) {
return;
}
this.setState({
error,
state: SharedProcessState.Stopped,
});
if (!this.activeProcess) {
return;
}
this.activeProcess.kill();
};
this.activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain", [], {
env: {
VSCODE_ALLOW_IO: "true",
VSCODE_LOGS: process.env.VSCODE_LOGS,
},
}, this.userDataDir);
if (this.logger.level <= Level.Trace) {
this.activeProcess.stdout.on("data", (data) => {
this.logger.trace(() => ["stdout", field("data", data.toString())]);
});
}
this.activeProcess.on("error", (error) => {
this.logger.error("error", field("error", error));
maybeStop(error.message);
});
this.activeProcess.on("exit", (err) => {
if (this._state !== SharedProcessState.Stopped) {
this.setState({
error: `Exited with ${err}`,
state: SharedProcessState.Stopped,
});
}
retry.run(this.retryName, new Error(`Exited with ${err}`));
});
this.ipcHandler = new StdioIpcHandler(this.activeProcess);
this.ipcHandler.once("handshake:hello", () => {
const data: {
sharedIPCHandle: string;
args: Partial<ParsedArgs>;
logLevel: Level;
} = {
args: {
"builtin-extensions-dir": this.builtInExtensionsDir,
"user-data-dir": this.userDataDir,
"extensions-dir": extensionsDir,
},
logLevel: this.logger.level,
sharedIPCHandle: this.socketPath,
};
this.ipcHandler!.send("handshake:hey there", "", data);
});
this.ipcHandler.once("handshake:im ready", () => {
resolved = true;
retry.recover(this.retryName);
this.setState({
state: SharedProcessState.Ready,
});
});
this.activeProcess.stderr.on("data", (data) => {
this.logger.error("stderr", field("data", data.toString()));
maybeStop(data.toString());
});
}
/**
* Signal the shared process to terminate.
*/
public dispose(): void {
this.disposed = true;
if (this.ipcHandler) {
this.ipcHandler.send("handshake:goodbye");
}
this.ipcHandler = undefined;
}
/**
* Start and connect to the shared process.
*/
private async connect(): Promise<void> {
this.setState({ state: SharedProcessState.Starting });
const activeProcess = await this.restart();
activeProcess.stderr.on("data", (data) => {
// Warn instead of error to prevent panic. It's unlikely stderr here is
// about anything critical to the functioning of the editor.
logger.warn(data.toString());
});
activeProcess.on("exit", (exitCode) => {
const error = new Error(`Exited with ${exitCode}`);
this.setState({
error: error.message,
state: SharedProcessState.Stopped,
});
if (!this.disposed) {
this.retry.run(error);
}
});
this.setState({ state: SharedProcessState.Ready });
}
/**
* Restart the shared process. Kill existing process if running. Resolve when
* the shared process is ready and reject when it errors or dies before being
* ready.
*/
private async restart(): Promise<ChildProcess> {
if (this.activeProcess && !this.activeProcess.killed) {
this.activeProcess.kill();
}
const extensionsDir = path.join(this.userDataDir, "extensions");
const backupsDir = path.join(this.userDataDir, "Backups");
await Promise.all([
fse.mkdirp(extensionsDir),
fse.mkdirp(backupsDir),
]);
const workspacesFile = path.join(backupsDir, "workspaces.json");
if (!fs.existsSync(workspacesFile)) {
fs.appendFileSync(workspacesFile, "");
}
const activeProcess = forkModule("vs/code/electron-browser/sharedProcess/sharedProcessMain", [], {
env: {
VSCODE_ALLOW_IO: "true",
VSCODE_LOGS: process.env.VSCODE_LOGS,
},
}, this.userDataDir);
this.activeProcess = activeProcess;
await new Promise((resolve, reject): void => {
const doReject = (error: Error | number): void => {
if (typeof error === "number") {
error = new Error(`Exited with ${error}`);
}
activeProcess.removeAllListeners();
this.setState({
error: error.message,
state: SharedProcessState.Stopped,
});
reject(error);
};
activeProcess.on("error", doReject);
activeProcess.on("exit", doReject);
this.ipcHandler = new StdioIpcHandler(activeProcess);
this.ipcHandler.once("handshake:hello", () => {
const data: {
sharedIPCHandle: string;
args: Partial<ParsedArgs>;
logLevel: Level;
} = {
args: {
"builtin-extensions-dir": this.builtInExtensionsDir,
"user-data-dir": this.userDataDir,
"extensions-dir": extensionsDir,
},
logLevel: this.logger.level,
sharedIPCHandle: this.socketPath,
};
this.ipcHandler!.send("handshake:hey there", "", data);
});
this.ipcHandler.once("handshake:im ready", () => {
activeProcess.removeListener("error", doReject);
activeProcess.removeListener("exit", doReject);
resolve();
});
});
return activeProcess;
}
/**
* Set the internal shared process state and emit the state event.
*/
private setState(event: SharedProcessEvent): void {
this._state = event.state;
this.onStateEmitter.emit(event);