Fix install from VSIX for TAR and ZIP formats (#245)

* Fix install from VSIX for TAR and ZIP formats

* Parse TAR before ZIP, when installing from VSIX

packages/package.json

@@ -12,6 +12,11 @@
     "xmlhttprequest": "1.8.0"
   },
   "jest": {
+    "globals": {
+      "ts-jest": {
+        "diagnostics": false
+      }
+    },
     "moduleFileExtensions": [
       "ts",
       "tsx",
@@ -26,7 +31,9 @@
       "@coder/ide/src/fill/evaluation": "<rootDir>/ide/src/fill/evaluation",
       "@coder/ide/src/fill/client": "<rootDir>/ide/src/fill/client",
       "@coder/(.*)/test": "<rootDir>/$1/test",
-      "@coder/(.*)": "<rootDir>/$1/src"
+      "@coder/(.*)": "<rootDir>/$1/src",
+      "vs/(.*)": "<rootDir>/../lib/vscode/src/vs/$1",
+      "vszip": "<rootDir>/../lib/vscode/src/vs/base/node/zip.ts"
     },
     "transform": {
       "^.+\\.tsx?$": "ts-jest"
@@ -37,4 +44,4 @@
     ],
     "testRegex": ".*\\.test\\.tsx?"
   }
-}
+}

packages/vscode/.gitignore

@@ -1 +1,2 @@
-bin
+bin
+test/.test*

packages/vscode/src/fill/zip.ts

@@ -4,11 +4,11 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as nls from "vs/nls";
+import * as vszip from "vszip";
 import * as fs from "fs";
 import * as path from "path";
 import * as tarStream from "tar-stream";
 import { promisify } from "util";
-import { ILogService } from "vs/platform/log/common/log";
 import { CancellationToken } from "vs/base/common/cancellation";
 import { mkdirp } from "vs/base/node/pfs";
 
@@ -16,8 +16,8 @@ export interface IExtractOptions {
 	overwrite?: boolean;
 
 	/**
-	 * Source path within the ZIP archive. Only the files contained in this
-	 * path will be extracted.
+	 * Source path within the TAR/ZIP archive. Only the files
+	 * contained in this path will be extracted.
 	 */
 	sourcePath?: string;
 }
@@ -28,11 +28,15 @@ export interface IFile {
 	localPath?: string;
 }
 
-export function zip(tarPath: string, files: IFile[]): Promise<string> {
-	return new Promise<string>((c, e) => {
+/**
+ * Override the standard VS Code behavior for zipping
+ * extensions to use the TAR format instead of ZIP.
+ */
+export const zip = (tarPath: string, files: IFile[]): Promise<string> => {
+	return new Promise<string>((c, e): void => {
 		const pack = tarStream.pack();
 		const chunks: Buffer[] = [];
-		const ended = new Promise<Buffer>((res, rej) => {
+		const ended = new Promise<Buffer>((res): void => {
 			pack.on("end", () => {
 				res(Buffer.concat(chunks));
 			});
@@ -56,132 +60,160 @@ export function zip(tarPath: string, files: IFile[]): Promise<string> {
 			e(ex);
 		});
 	});
-}
+};
 
-export async function extract(tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise<void> {
-	const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : '');
+/**
+ * Override the standard VS Code behavior for extracting
+ * archives, to first attempt to process the archive as a TAR
+ * and then fallback on the original implementation, for processing
+ * ZIPs.
+ */
+export const extract = (archivePath: string, extractPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise<void> => {
+	return new Promise<void>((c, e): void => {
+		extractTar(archivePath, extractPath, options, token).then(c).catch((ex) => {
+			if (!ex.toString().includes("Invalid tar header")) {
+				e(ex);
 
-	return new Promise<void>(async (c, e) => {
-		const buffer = await promisify(fs.readFile)(tarPath);
-		const extractor = tarStream.extract();
-		extractor.once('error', e);
-		extractor.on('entry', (header, stream, next) => {
-			const rawName = header.name;
-
-			const nextEntry = (): void => {
-				stream.resume();
-				next();
-			};
-
-			if (token.isCancellationRequested) {
-				return nextEntry();
-			}
-
-			if (!sourcePathRegex.test(rawName)) {
-				return nextEntry();
-			}
-
-			const fileName = rawName.replace(sourcePathRegex, '');
-
-			const targetFileName = path.join(targetPath, fileName);
-			if (/\/$/.test(fileName)) {
-				stream.resume();
-				mkdirp(targetFileName).then(() => {
-					next();
-				}, e);
 				return;
 			}
-
-			const dirName = path.dirname(fileName);
-			const targetDirName = path.join(targetPath, dirName);
-			if (targetDirName.indexOf(targetPath) !== 0) {
-				e(nls.localize('invalid file', "Error extracting {0}. Invalid file.", fileName));
-				return nextEntry();
-			}
-
-			mkdirp(targetDirName, void 0, token).then(() => {
-				const fstream = fs.createWriteStream(targetFileName, { mode: header.mode });
-				fstream.once('close', () => {
-					next();
-				});
-				fstream.once('error', (err) => {
-					e(err);
-				});
-				stream.pipe(fstream);
-				stream.resume();
-			});
+			vszip.extract(archivePath, extractPath, options, token).then(c).catch(e);
 		});
-		extractor.once('finish', () => {
-			c();
-		});
-		extractor.write(buffer);
-		extractor.end();
 	});
-}
+};
 
-export function buffer(tarPath: string, filePath: string): Promise<Buffer> {
-	return new Promise<Buffer>(async (c, e) => {
+/**
+ * Override the standard VS Code behavior for buffering
+ * archives, to first process the Buffer as a TAR and then
+ * fallback on the original implementation, for processing ZIPs.
+ */
+export const buffer = (targetPath: string, filePath: string): Promise<Buffer> => {
+	return new Promise<Buffer>((c, e): void => {
 		let done: boolean = false;
-		extractAssets(tarPath, new RegExp(filePath), (path: string, data: Buffer) => {
-			if (path === filePath) {
+		extractAssets(targetPath, new RegExp(filePath), (assetPath: string, data: Buffer) => {
+			if (path.normalize(assetPath) === path.normalize(filePath)) {
 				done = true;
 				c(data);
 			}
 		}).then(() => {
 			if (!done) {
-				e("couldnt find asset " + filePath);
+				e("couldn't find asset " + filePath);
 			}
 		}).catch((ex) => {
-			e(ex);
+			if (!ex.toString().includes("Invalid tar header")) {
+				e(ex);
+
+				return;
+			}
+			vszip.buffer(targetPath, filePath).then(c).catch(e);
 		});
 	});
-}
+};
 
-async function extractAssets(tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise<void> {
-	const buffer = await promisify(fs.readFile)(tarPath);
-	const extractor = tarStream.extract();
-	let callbackResolve: () => void;
-	let callbackReject: (ex?) => void;
-	const complete = new Promise<void>((r, rej) => {
-		callbackResolve = r;
-		callbackReject = rej;
-	});
-	extractor.once("error", (err) => {
-		callbackReject(err);
-	});
-	extractor.on("entry", (header, stream, next) => {
-		const name = header.name;
-		if (match.test(name)) {
-			extractData(stream).then((data) => {
-				callback(name, data);
-				next();
+/**
+ * Override the standard VS Code behavior for extracting assets
+ * from archive Buffers to use the TAR format instead of ZIP.
+ */
+export const extractAssets = (tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise<void> => {
+	return new Promise<void>(async (c, e): Promise<void> => {
+		try {
+			const buffer = await promisify(fs.readFile)(tarPath);
+			const extractor = tarStream.extract();
+			extractor.once("error", e);
+			extractor.on("entry", (header, stream, next) => {
+				const name = header.name;
+				if (match.test(name)) {
+					extractData(stream).then((data) => {
+						callback(name, data);
+						next();
+					}).catch(e);
+					stream.resume();
+				} else {
+					stream.on("end", () => {
+						next();
+					});
+					stream.resume();
+				}
 			});
-			stream.resume();
-		} else {
-			stream.on("end", () => {
-				next();
+			extractor.on("finish", () => {
+				c();
 			});
-			stream.resume();
+			extractor.write(buffer);
+			extractor.end();
+		} catch (ex) {
+			e(ex);
 		}
 	});
-	extractor.on("finish", () => {
-		callbackResolve();
-	});
-	extractor.write(buffer);
-	extractor.end();
-	return complete;
-}
+};
 
-async function extractData(stream: NodeJS.ReadableStream): Promise<Buffer> {
-	return new Promise<Buffer>((res, rej) => {
+const extractData = (stream: NodeJS.ReadableStream): Promise<Buffer> => {
+	return new Promise<Buffer>((c, e): void => {
 		const fileData: Buffer[] = [];
-		stream.on('data', (data) => fileData.push(data));
-		stream.on('end', () => {
+		stream.on("data", (data) => fileData.push(data));
+		stream.on("end", () => {
 			const fd = Buffer.concat(fileData);
-			res(fd);
-		});
-		stream.on('error', (err) => {
-			rej(err);
+			c(fd);
 		});
+		stream.on("error", e);
 	});
-}
+};
+
+const extractTar = (tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise<void> => {
+	return new Promise<void>(async (c, e): Promise<void> => {
+		try {
+			const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : "");
+			const buffer = await promisify(fs.readFile)(tarPath);
+			const extractor = tarStream.extract();
+			extractor.once("error", e);
+			extractor.on("entry", (header, stream, next) => {
+				const rawName = path.normalize(header.name);
+
+				const nextEntry = (): void => {
+					stream.resume();
+					next();
+				};
+
+				if (token.isCancellationRequested) {
+					return nextEntry();
+				}
+
+				if (!sourcePathRegex.test(rawName)) {
+					return nextEntry();
+				}
+
+				const fileName = rawName.replace(sourcePathRegex, "");
+				const targetFileName = path.join(targetPath, fileName);
+				if (/\/$/.test(fileName)) {
+					stream.resume();
+					mkdirp(targetFileName).then(() => {
+						next();
+					}, e);
+
+					return;
+				}
+
+				const dirName = path.dirname(fileName);
+				const targetDirName = path.join(targetPath, dirName);
+				if (targetDirName.indexOf(targetPath) !== 0) {
+					e(nls.localize("invalid file", "Error extracting {0}. Invalid file.", fileName));
+
+					return nextEntry();
+				}
+
+				return mkdirp(targetDirName, undefined, token).then(() => {
+					const fstream = fs.createWriteStream(targetFileName, { mode: header.mode });
+					fstream.once("close", () => {
+						next();
+					});
+					fstream.once("error", e);
+					stream.pipe(fstream);
+					stream.resume();
+				});
+			});
+			extractor.once("finish", c);
+			extractor.write(buffer);
+			extractor.end();
+		} catch (ex) {
+			e(ex);
+		}
+	});
+};

packages/vscode/test/zip.test.ts

@@ -0,0 +1,59 @@
+import * as zip from "../src/fill/zip";
+import * as path from "path";
+import * as fs from "fs";
+import * as cp from "child_process";
+import { CancellationToken } from "vs/base/common/cancellation";
+
+// tslint:disable-next-line:no-any
+jest.mock("vs/nls", () => ({ "localize": (...args: any): string => `${JSON.stringify(args)}` }));
+
+describe("zip", () => {
+	const tarPath = path.resolve(__dirname, "./test-extension.tar");
+	const vsixPath = path.resolve(__dirname, "./test-extension.vsix");
+	const extractPath = path.resolve(__dirname, "./.test-extension");
+
+	beforeEach(() => {
+		if (!fs.existsSync(extractPath) || path.dirname(extractPath) !== __dirname) {
+			return;
+		}
+		cp.execSync(`rm -rf '${extractPath}'`);
+	});
+
+	const resolveExtract = async (archivePath: string): Promise<void> => {
+		expect(fs.existsSync(archivePath)).toEqual(true);
+		await expect(zip.extract(
+			archivePath,
+			extractPath,
+			{ sourcePath: "extension", overwrite: true },
+			CancellationToken.None,
+		)).resolves.toBe(undefined);
+		expect(fs.existsSync(extractPath)).toEqual(true);
+	};
+
+	// tslint:disable-next-line:no-any
+	const extract = (archivePath: string): () => any => {
+		// tslint:disable-next-line:no-any
+		return async (): Promise<any> => {
+			await resolveExtract(archivePath);
+			expect(fs.existsSync(path.resolve(extractPath, ".vsixmanifest"))).toEqual(true);
+			expect(fs.existsSync(path.resolve(extractPath, "package.json"))).toEqual(true);
+		};
+	};
+	it("should extract from tarred VSIX", extract(tarPath), 2000);
+	it("should extract from zipped VSIX", extract(vsixPath), 2000);
+
+	// tslint:disable-next-line:no-any
+	const buffer = (archivePath: string): () => any => {
+		// tslint:disable-next-line:no-any
+		return async (): Promise<any> => {
+			await resolveExtract(archivePath);
+			const manifestPath = path.resolve(extractPath, ".vsixmanifest");
+			expect(fs.existsSync(manifestPath)).toEqual(true);
+			const manifestBuf = fs.readFileSync(manifestPath);
+			expect(manifestBuf.length).toBeGreaterThan(0);
+			await expect(zip.buffer(archivePath, "extension.vsixmanifest")).resolves.toEqual(manifestBuf);
+		};
+	};
+	it("should buffer tarred VSIX", buffer(tarPath), 2000);
+	it("should buffer zipped VSIX", buffer(vsixPath), 2000);
+});

packages/vscode/webpack.bootstrap.config.js

@@ -62,6 +62,7 @@ module.exports = merge(
 				"vs/platform/product/node/package": path.resolve(vsFills, "package.ts"),
 				"vs/platform/product/node/product": path.resolve(vsFills, "product.ts"),
 				"vs/base/node/zip": path.resolve(vsFills, "zip.ts"),
+				"vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"),
 				"vs": path.resolve(root, "lib/vscode/src/vs"),
 			},
 		},

packages/web/webpack.config.js

@@ -75,6 +75,7 @@ module.exports = merge(
 			"vs/platform/product/node/package": path.resolve(vsFills, "package.ts"),
 			"vs/platform/product/node/product": path.resolve(vsFills, "product.ts"),
 			"vs/base/node/zip": path.resolve(vsFills, "zip.ts"),
+			"vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"),
 			"vs": path.join(root, "lib", "vscode", "src", "vs"),
 		},
 	},