2019-02-07 01:11:31 +01:00
|
|
|
import { logger, field } from "@coder/logger";
|
2019-01-30 22:40:01 +01:00
|
|
|
import { NotificationService, INotificationHandle, INotificationService, Severity } from "./fill/notification";
|
2019-01-08 01:46:19 +01:00
|
|
|
|
2019-04-01 20:31:34 +02:00
|
|
|
// tslint:disable no-any can have different return values
|
|
|
|
|
2019-01-08 01:46:19 +01:00
|
|
|
interface IRetryItem {
|
2019-04-01 20:31:34 +02:00
|
|
|
/**
|
|
|
|
* How many times this item has been retried.
|
|
|
|
*/
|
2019-01-08 01:46:19 +01:00
|
|
|
count?: number;
|
2019-04-01 20:31:34 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
* In seconds.
|
|
|
|
*/
|
|
|
|
delay?: number;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* In milliseconds.
|
|
|
|
*/
|
|
|
|
end?: number;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Function to run when retrying.
|
|
|
|
*/
|
|
|
|
fn(): any;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Timer for running this item.
|
|
|
|
*/
|
2019-01-08 01:46:19 +01:00
|
|
|
timeout?: number | NodeJS.Timer;
|
2019-04-01 20:31:34 +02:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Whether the item is retrying or waiting to retry.
|
|
|
|
*/
|
2019-01-08 01:46:19 +01:00
|
|
|
running?: boolean;
|
2019-04-01 20:31:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* An retry-able instance.
|
|
|
|
*/
|
|
|
|
export interface RetryInstance {
|
|
|
|
/**
|
|
|
|
* Run this retry.
|
|
|
|
*/
|
|
|
|
run(error?: Error): void;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Block on this instance.
|
|
|
|
*/
|
|
|
|
block(): void;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A retry-able instance that doesn't use a promise so it must be manually
|
|
|
|
* ran again on failure and recovered on success.
|
|
|
|
*/
|
|
|
|
export interface ManualRetryInstance extends RetryInstance {
|
|
|
|
/**
|
|
|
|
* Mark this item as recovered.
|
|
|
|
*/
|
|
|
|
recover(): void;
|
2019-01-08 01:46:19 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Retry services. Handles multiple services so when a connection drops the
|
|
|
|
* user doesn't get a separate notification for each service.
|
|
|
|
*
|
|
|
|
* Attempts to restart services silently up to a maximum number of tries, then
|
|
|
|
* starts waiting for a delay that grows exponentially with each attempt with a
|
|
|
|
* cap on the delay. Once the delay is long enough, it will show a notification
|
|
|
|
* to the user explaining what is happening with an option to immediately retry.
|
|
|
|
*/
|
|
|
|
export class Retry {
|
2019-04-01 20:31:34 +02:00
|
|
|
private readonly items = new Map<string, IRetryItem>();
|
2019-01-08 01:46:19 +01:00
|
|
|
|
|
|
|
// Times are in seconds.
|
|
|
|
private readonly retryMinDelay = 1;
|
|
|
|
private readonly retryMaxDelay = 10;
|
|
|
|
private readonly maxImmediateRetries = 5;
|
|
|
|
private readonly retryExponent = 1.5;
|
|
|
|
private blocked: string | boolean | undefined;
|
|
|
|
|
|
|
|
private notificationHandle: INotificationHandle | undefined;
|
2019-02-06 18:53:23 +01:00
|
|
|
private readonly updateDelay = 1;
|
2019-01-08 01:46:19 +01:00
|
|
|
private updateTimeout: number | NodeJS.Timer | undefined;
|
2019-02-06 18:53:23 +01:00
|
|
|
private readonly notificationThreshold = 3;
|
2019-01-08 01:46:19 +01:00
|
|
|
|
|
|
|
// Time in milliseconds to wait before restarting a service. (See usage below
|
|
|
|
// for reasoning.)
|
2019-02-06 18:53:23 +01:00
|
|
|
private readonly waitDelay = 50;
|
2019-01-08 01:46:19 +01:00
|
|
|
|
2019-02-06 18:53:23 +01:00
|
|
|
public constructor(private _notificationService: INotificationService) {}
|
2019-01-08 01:46:19 +01:00
|
|
|
|
2019-01-30 22:40:01 +01:00
|
|
|
public set notificationService(service: INotificationService) {
|
|
|
|
this._notificationService = service;
|
|
|
|
}
|
|
|
|
|
|
|
|
public get notificationService(): INotificationService {
|
|
|
|
return this._notificationService;
|
2019-01-08 01:46:19 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2019-04-01 20:31:34 +02:00
|
|
|
* Register a function to retry that starts/connects to a service.
|
|
|
|
*
|
|
|
|
* The service is automatically retried or recovered when the promise resolves
|
|
|
|
* or rejects. If the service dies after starting, it must be manually
|
|
|
|
* retried.
|
|
|
|
*/
|
|
|
|
public register(name: string, fn: () => Promise<any>): RetryInstance;
|
|
|
|
/**
|
|
|
|
* Register a function to retry that starts/connects to a service.
|
|
|
|
*
|
|
|
|
* Must manually retry if it fails to start again or dies after restarting and
|
|
|
|
* manually recover if it succeeds in starting again.
|
|
|
|
*/
|
|
|
|
public register(name: string, fn: () => any): ManualRetryInstance;
|
|
|
|
/**
|
|
|
|
* Register a function to retry that starts/connects to a service.
|
|
|
|
*/
|
|
|
|
public register(name: string, fn: () => any): RetryInstance | ManualRetryInstance {
|
|
|
|
if (this.items.has(name)) {
|
|
|
|
throw new Error(`"${name}" is already registered`);
|
|
|
|
}
|
|
|
|
this.items.set(name, { fn });
|
|
|
|
|
|
|
|
return {
|
|
|
|
block: (): void => this.block(name),
|
|
|
|
run: (error?: Error): void => this.run(name, error),
|
|
|
|
recover: (): void => this.recover(name),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Un-register a function to retry.
|
|
|
|
*/
|
|
|
|
public unregister(name: string): void {
|
|
|
|
if (!this.items.has(name)) {
|
|
|
|
throw new Error(`"${name}" is not registered`);
|
|
|
|
}
|
|
|
|
this.items.delete(name);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Block retries when we know they will fail (for example when the socket is
|
|
|
|
* down ). If a name is passed, that service will still be allowed to retry
|
2019-01-08 01:46:19 +01:00
|
|
|
* (unless we have already blocked).
|
|
|
|
*
|
|
|
|
* Blocking without a name will override a block with a name.
|
|
|
|
*/
|
2019-04-06 01:49:29 +02:00
|
|
|
public block(name?: string): void {
|
2019-01-08 01:46:19 +01:00
|
|
|
if (!this.blocked || !name) {
|
|
|
|
this.blocked = name || true;
|
|
|
|
this.items.forEach((item) => {
|
|
|
|
this.stopItem(item);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Unblock retries and run any that are pending.
|
|
|
|
*/
|
2019-04-01 20:31:34 +02:00
|
|
|
private unblock(): void {
|
2019-01-08 01:46:19 +01:00
|
|
|
this.blocked = false;
|
|
|
|
this.items.forEach((item, name) => {
|
|
|
|
if (item.running) {
|
|
|
|
this.runItem(name, item);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Retry a service.
|
|
|
|
*/
|
2019-04-01 20:31:34 +02:00
|
|
|
private run(name: string, error?: Error): void {
|
2019-01-08 01:46:19 +01:00
|
|
|
if (!this.items.has(name)) {
|
|
|
|
throw new Error(`"${name}" is not registered`);
|
|
|
|
}
|
|
|
|
|
|
|
|
const item = this.items.get(name)!;
|
|
|
|
if (item.running) {
|
|
|
|
throw new Error(`"${name}" is already retrying`);
|
|
|
|
}
|
|
|
|
|
|
|
|
item.running = true;
|
|
|
|
// This timeout is for the case when the connection drops; this allows time
|
2019-02-07 01:11:31 +01:00
|
|
|
// for the socket service to come in and block everything because some other
|
2019-01-08 01:46:19 +01:00
|
|
|
// services might make it here first and try to restart, which will fail.
|
|
|
|
setTimeout(() => {
|
|
|
|
if (this.blocked && this.blocked !== name) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!item.count || item.count < this.maxImmediateRetries) {
|
2019-02-07 01:11:31 +01:00
|
|
|
return this.runItem(name, item, error);
|
2019-01-08 01:46:19 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!item.delay) {
|
|
|
|
item.delay = this.retryMinDelay;
|
|
|
|
} else {
|
|
|
|
item.delay = Math.ceil(item.delay * this.retryExponent);
|
|
|
|
if (item.delay > this.retryMaxDelay) {
|
|
|
|
item.delay = this.retryMaxDelay;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-02-07 01:11:31 +01:00
|
|
|
logger.info(`Retrying ${name.toLowerCase()} in ${item.delay}s`, error && field("error", error.message));
|
2019-01-08 01:46:19 +01:00
|
|
|
const itemDelayMs = item.delay * 1000;
|
|
|
|
item.end = Date.now() + itemDelayMs;
|
2019-02-07 01:11:31 +01:00
|
|
|
item.timeout = setTimeout(() => this.runItem(name, item, error), itemDelayMs);
|
2019-01-08 01:46:19 +01:00
|
|
|
|
|
|
|
this.updateNotification();
|
|
|
|
}, this.waitDelay);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Reset a service after a successfully recovering.
|
|
|
|
*/
|
2019-04-01 20:31:34 +02:00
|
|
|
private recover(name: string): void {
|
2019-01-08 01:46:19 +01:00
|
|
|
if (!this.items.has(name)) {
|
|
|
|
throw new Error(`"${name}" is not registered`);
|
|
|
|
}
|
|
|
|
|
|
|
|
const item = this.items.get(name)!;
|
|
|
|
if (typeof item.timeout === "undefined" && !item.running && typeof item.count !== "undefined") {
|
2019-01-18 22:46:40 +01:00
|
|
|
logger.info(`Connected to ${name.toLowerCase()}`);
|
2019-01-08 01:46:19 +01:00
|
|
|
item.delay = undefined;
|
|
|
|
item.count = undefined;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Run an item.
|
|
|
|
*/
|
2019-02-07 01:11:31 +01:00
|
|
|
private runItem(name: string, item: IRetryItem, error?: Error): void {
|
2019-01-08 01:46:19 +01:00
|
|
|
if (!item.count) {
|
|
|
|
item.count = 1;
|
|
|
|
} else {
|
|
|
|
++item.count;
|
|
|
|
}
|
|
|
|
|
|
|
|
const retryCountText = item.count <= this.maxImmediateRetries
|
|
|
|
? `[${item.count}/${this.maxImmediateRetries}]`
|
|
|
|
: `[${item.count}]`;
|
2019-02-07 01:11:31 +01:00
|
|
|
logger.info(`Starting ${name.toLowerCase()} ${retryCountText}...`, error && field("error", error.message));
|
2019-01-08 01:46:19 +01:00
|
|
|
|
|
|
|
const endItem = (): void => {
|
|
|
|
this.stopItem(item);
|
|
|
|
item.running = false;
|
|
|
|
};
|
|
|
|
|
|
|
|
try {
|
|
|
|
const maybePromise = item.fn();
|
|
|
|
if (maybePromise instanceof Promise) {
|
|
|
|
maybePromise.then(() => {
|
|
|
|
endItem();
|
|
|
|
this.recover(name);
|
|
|
|
if (this.blocked === name) {
|
|
|
|
this.unblock();
|
|
|
|
}
|
2019-04-01 20:31:34 +02:00
|
|
|
}).catch((error) => {
|
2019-01-08 01:46:19 +01:00
|
|
|
endItem();
|
2019-04-01 20:31:34 +02:00
|
|
|
this.run(name, error);
|
2019-01-08 01:46:19 +01:00
|
|
|
});
|
|
|
|
} else {
|
|
|
|
endItem();
|
|
|
|
}
|
|
|
|
} catch (error) {
|
|
|
|
// Prevent an exception from causing the item to never run again.
|
|
|
|
endItem();
|
|
|
|
throw error;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Update, close, or show the notification.
|
|
|
|
*/
|
|
|
|
private updateNotification(): void {
|
|
|
|
// tslint:disable-next-line no-any because NodeJS.Timer is valid.
|
|
|
|
clearTimeout(this.updateTimeout as any);
|
|
|
|
|
|
|
|
const now = Date.now();
|
|
|
|
const items = Array.from(this.items.entries()).filter(([_, item]) => {
|
2019-04-01 20:31:34 +02:00
|
|
|
return typeof item.end !== "undefined"
|
2019-01-08 01:46:19 +01:00
|
|
|
&& item.end > now
|
|
|
|
&& item.delay && item.delay >= this.notificationThreshold;
|
|
|
|
}).sort((a, b) => {
|
|
|
|
return a[1] < b[1] ? -1 : 1;
|
|
|
|
});
|
|
|
|
|
|
|
|
if (items.length === 0) {
|
|
|
|
if (this.notificationHandle) {
|
|
|
|
this.notificationHandle.close();
|
|
|
|
this.notificationHandle = undefined;
|
|
|
|
}
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
const join = (arr: string[]): string => {
|
|
|
|
const last = arr.pop()!; // Assume length > 0.
|
|
|
|
|
|
|
|
return arr.length > 0 ? `${arr.join(", ")} and ${last}` : last;
|
|
|
|
};
|
|
|
|
|
|
|
|
const servicesStr = join(items.map(([name, _]) => name.toLowerCase()));
|
|
|
|
const message = `Lost connection to ${servicesStr}. Retrying in ${
|
|
|
|
join(items.map(([_, item]) => `${Math.ceil((item.end! - now) / 1000)}s`))
|
|
|
|
}.`;
|
|
|
|
|
|
|
|
const buttons = [{
|
|
|
|
label: `Retry ${items.length > 1 ? "Services" : items[0][0]} Now`,
|
|
|
|
run: (): void => {
|
|
|
|
logger.info(`Forcing ${servicesStr} to restart now`);
|
|
|
|
items.forEach(([name, item]) => {
|
|
|
|
this.runItem(name, item);
|
|
|
|
});
|
|
|
|
this.updateNotification();
|
|
|
|
},
|
|
|
|
}];
|
|
|
|
|
|
|
|
if (!this.notificationHandle) {
|
|
|
|
this.notificationHandle = this.notificationService.prompt(
|
|
|
|
Severity.Info,
|
|
|
|
message,
|
|
|
|
buttons,
|
|
|
|
() => {
|
|
|
|
this.notificationHandle = undefined;
|
|
|
|
// tslint:disable-next-line no-any because NodeJS.Timer is valid.
|
|
|
|
clearTimeout(this.updateTimeout as any);
|
|
|
|
},
|
|
|
|
);
|
|
|
|
} else {
|
|
|
|
this.notificationHandle.updateMessage(message);
|
|
|
|
this.notificationHandle.updateButtons(buttons);
|
|
|
|
}
|
|
|
|
|
|
|
|
this.updateTimeout = setTimeout(() => this.updateNotification(), this.updateDelay * 1000);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Stop an item's timer.
|
|
|
|
*/
|
|
|
|
private stopItem(item: IRetryItem): void {
|
|
|
|
// tslint:disable-next-line no-any because NodeJS.Timer is valid.
|
|
|
|
clearTimeout(item.timeout as any);
|
|
|
|
item.timeout = undefined;
|
|
|
|
item.end = undefined;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-18 22:46:40 +01:00
|
|
|
// Global instance so we can block other retries when retrying the main
|
|
|
|
// connection.
|
2019-01-30 22:40:01 +01:00
|
|
|
export const retry = new Retry(new NotificationService());
|