diff --git a/packages/package.json b/packages/package.json index 72ea26dc0..12e48d0ba 100644 --- a/packages/package.json +++ b/packages/package.json @@ -12,6 +12,11 @@ "xmlhttprequest": "1.8.0" }, "jest": { + "globals": { + "ts-jest": { + "diagnostics": false + } + }, "moduleFileExtensions": [ "ts", "tsx", @@ -26,7 +31,9 @@ "@coder/ide/src/fill/evaluation": "/ide/src/fill/evaluation", "@coder/ide/src/fill/client": "/ide/src/fill/client", "@coder/(.*)/test": "/$1/test", - "@coder/(.*)": "/$1/src" + "@coder/(.*)": "/$1/src", + "vs/(.*)": "/../lib/vscode/src/vs/$1", + "vszip": "/../lib/vscode/src/vs/base/node/zip.ts" }, "transform": { "^.+\\.tsx?$": "ts-jest" @@ -37,4 +44,4 @@ ], "testRegex": ".*\\.test\\.tsx?" } -} +} \ No newline at end of file diff --git a/packages/vscode/.gitignore b/packages/vscode/.gitignore index c5e82d745..8af4e3b6b 100644 --- a/packages/vscode/.gitignore +++ b/packages/vscode/.gitignore @@ -1 +1,2 @@ -bin \ No newline at end of file +bin +test/.test* \ No newline at end of file diff --git a/packages/vscode/src/fill/zip.ts b/packages/vscode/src/fill/zip.ts index 1f6e1cd62..b7aafb20e 100644 --- a/packages/vscode/src/fill/zip.ts +++ b/packages/vscode/src/fill/zip.ts @@ -4,11 +4,11 @@ *--------------------------------------------------------------------------------------------*/ import * as nls from "vs/nls"; +import * as vszip from "vszip"; import * as fs from "fs"; import * as path from "path"; import * as tarStream from "tar-stream"; import { promisify } from "util"; -import { ILogService } from "vs/platform/log/common/log"; import { CancellationToken } from "vs/base/common/cancellation"; import { mkdirp } from "vs/base/node/pfs"; @@ -16,8 +16,8 @@ export interface IExtractOptions { overwrite?: boolean; /** - * Source path within the ZIP archive. Only the files contained in this - * path will be extracted. + * Source path within the TAR/ZIP archive. Only the files + * contained in this path will be extracted. */ sourcePath?: string; } @@ -28,11 +28,15 @@ export interface IFile { localPath?: string; } -export function zip(tarPath: string, files: IFile[]): Promise { - return new Promise((c, e) => { +/** + * Override the standard VS Code behavior for zipping + * extensions to use the TAR format instead of ZIP. + */ +export const zip = (tarPath: string, files: IFile[]): Promise => { + return new Promise((c, e): void => { const pack = tarStream.pack(); const chunks: Buffer[] = []; - const ended = new Promise((res, rej) => { + const ended = new Promise((res): void => { pack.on("end", () => { res(Buffer.concat(chunks)); }); @@ -56,132 +60,160 @@ export function zip(tarPath: string, files: IFile[]): Promise { e(ex); }); }); -} +}; -export async function extract(tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise { - const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : ''); +/** + * Override the standard VS Code behavior for extracting + * archives, to first attempt to process the archive as a TAR + * and then fallback on the original implementation, for processing + * ZIPs. + */ +export const extract = (archivePath: string, extractPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise => { + return new Promise((c, e): void => { + extractTar(archivePath, extractPath, options, token).then(c).catch((ex) => { + if (!ex.toString().includes("Invalid tar header")) { + e(ex); - return new Promise(async (c, e) => { - const buffer = await promisify(fs.readFile)(tarPath); - const extractor = tarStream.extract(); - extractor.once('error', e); - extractor.on('entry', (header, stream, next) => { - const rawName = header.name; - - const nextEntry = (): void => { - stream.resume(); - next(); - }; - - if (token.isCancellationRequested) { - return nextEntry(); - } - - if (!sourcePathRegex.test(rawName)) { - return nextEntry(); - } - - const fileName = rawName.replace(sourcePathRegex, ''); - - const targetFileName = path.join(targetPath, fileName); - if (/\/$/.test(fileName)) { - stream.resume(); - mkdirp(targetFileName).then(() => { - next(); - }, e); return; } - - const dirName = path.dirname(fileName); - const targetDirName = path.join(targetPath, dirName); - if (targetDirName.indexOf(targetPath) !== 0) { - e(nls.localize('invalid file', "Error extracting {0}. Invalid file.", fileName)); - return nextEntry(); - } - - mkdirp(targetDirName, void 0, token).then(() => { - const fstream = fs.createWriteStream(targetFileName, { mode: header.mode }); - fstream.once('close', () => { - next(); - }); - fstream.once('error', (err) => { - e(err); - }); - stream.pipe(fstream); - stream.resume(); - }); + vszip.extract(archivePath, extractPath, options, token).then(c).catch(e); }); - extractor.once('finish', () => { - c(); - }); - extractor.write(buffer); - extractor.end(); }); -} +}; -export function buffer(tarPath: string, filePath: string): Promise { - return new Promise(async (c, e) => { +/** + * Override the standard VS Code behavior for buffering + * archives, to first process the Buffer as a TAR and then + * fallback on the original implementation, for processing ZIPs. + */ +export const buffer = (targetPath: string, filePath: string): Promise => { + return new Promise((c, e): void => { let done: boolean = false; - extractAssets(tarPath, new RegExp(filePath), (path: string, data: Buffer) => { - if (path === filePath) { + extractAssets(targetPath, new RegExp(filePath), (assetPath: string, data: Buffer) => { + if (path.normalize(assetPath) === path.normalize(filePath)) { done = true; c(data); } }).then(() => { if (!done) { - e("couldnt find asset " + filePath); + e("couldn't find asset " + filePath); } }).catch((ex) => { - e(ex); + if (!ex.toString().includes("Invalid tar header")) { + e(ex); + + return; + } + vszip.buffer(targetPath, filePath).then(c).catch(e); }); }); -} +}; -async function extractAssets(tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise { - const buffer = await promisify(fs.readFile)(tarPath); - const extractor = tarStream.extract(); - let callbackResolve: () => void; - let callbackReject: (ex?) => void; - const complete = new Promise((r, rej) => { - callbackResolve = r; - callbackReject = rej; - }); - extractor.once("error", (err) => { - callbackReject(err); - }); - extractor.on("entry", (header, stream, next) => { - const name = header.name; - if (match.test(name)) { - extractData(stream).then((data) => { - callback(name, data); - next(); +/** + * Override the standard VS Code behavior for extracting assets + * from archive Buffers to use the TAR format instead of ZIP. + */ +export const extractAssets = (tarPath: string, match: RegExp, callback: (path: string, data: Buffer) => void): Promise => { + return new Promise(async (c, e): Promise => { + try { + const buffer = await promisify(fs.readFile)(tarPath); + const extractor = tarStream.extract(); + extractor.once("error", e); + extractor.on("entry", (header, stream, next) => { + const name = header.name; + if (match.test(name)) { + extractData(stream).then((data) => { + callback(name, data); + next(); + }).catch(e); + stream.resume(); + } else { + stream.on("end", () => { + next(); + }); + stream.resume(); + } }); - stream.resume(); - } else { - stream.on("end", () => { - next(); + extractor.on("finish", () => { + c(); }); - stream.resume(); + extractor.write(buffer); + extractor.end(); + } catch (ex) { + e(ex); } }); - extractor.on("finish", () => { - callbackResolve(); - }); - extractor.write(buffer); - extractor.end(); - return complete; -} +}; -async function extractData(stream: NodeJS.ReadableStream): Promise { - return new Promise((res, rej) => { +const extractData = (stream: NodeJS.ReadableStream): Promise => { + return new Promise((c, e): void => { const fileData: Buffer[] = []; - stream.on('data', (data) => fileData.push(data)); - stream.on('end', () => { + stream.on("data", (data) => fileData.push(data)); + stream.on("end", () => { const fd = Buffer.concat(fileData); - res(fd); - }); - stream.on('error', (err) => { - rej(err); + c(fd); }); + stream.on("error", e); }); -} +}; + +const extractTar = (tarPath: string, targetPath: string, options: IExtractOptions = {}, token: CancellationToken): Promise => { + return new Promise(async (c, e): Promise => { + try { + const sourcePathRegex = new RegExp(options.sourcePath ? `^${options.sourcePath}` : ""); + const buffer = await promisify(fs.readFile)(tarPath); + const extractor = tarStream.extract(); + extractor.once("error", e); + extractor.on("entry", (header, stream, next) => { + const rawName = path.normalize(header.name); + + const nextEntry = (): void => { + stream.resume(); + next(); + }; + + if (token.isCancellationRequested) { + return nextEntry(); + } + + if (!sourcePathRegex.test(rawName)) { + return nextEntry(); + } + + const fileName = rawName.replace(sourcePathRegex, ""); + const targetFileName = path.join(targetPath, fileName); + if (/\/$/.test(fileName)) { + stream.resume(); + mkdirp(targetFileName).then(() => { + next(); + }, e); + + return; + } + + const dirName = path.dirname(fileName); + const targetDirName = path.join(targetPath, dirName); + if (targetDirName.indexOf(targetPath) !== 0) { + e(nls.localize("invalid file", "Error extracting {0}. Invalid file.", fileName)); + + return nextEntry(); + } + + return mkdirp(targetDirName, undefined, token).then(() => { + const fstream = fs.createWriteStream(targetFileName, { mode: header.mode }); + fstream.once("close", () => { + next(); + }); + fstream.once("error", e); + stream.pipe(fstream); + stream.resume(); + }); + }); + extractor.once("finish", c); + extractor.write(buffer); + extractor.end(); + } catch (ex) { + e(ex); + } + }); +}; diff --git a/packages/vscode/test/test-extension.tar b/packages/vscode/test/test-extension.tar new file mode 100644 index 000000000..bd1f69c39 Binary files /dev/null and b/packages/vscode/test/test-extension.tar differ diff --git a/packages/vscode/test/test-extension.vsix b/packages/vscode/test/test-extension.vsix new file mode 100644 index 000000000..3c133799f Binary files /dev/null and b/packages/vscode/test/test-extension.vsix differ diff --git a/packages/vscode/test/zip.test.ts b/packages/vscode/test/zip.test.ts new file mode 100644 index 000000000..e7685dc40 --- /dev/null +++ b/packages/vscode/test/zip.test.ts @@ -0,0 +1,59 @@ +import * as zip from "../src/fill/zip"; +import * as path from "path"; +import * as fs from "fs"; +import * as cp from "child_process"; +import { CancellationToken } from "vs/base/common/cancellation"; + +// tslint:disable-next-line:no-any +jest.mock("vs/nls", () => ({ "localize": (...args: any): string => `${JSON.stringify(args)}` })); + +describe("zip", () => { + const tarPath = path.resolve(__dirname, "./test-extension.tar"); + const vsixPath = path.resolve(__dirname, "./test-extension.vsix"); + const extractPath = path.resolve(__dirname, "./.test-extension"); + + beforeEach(() => { + if (!fs.existsSync(extractPath) || path.dirname(extractPath) !== __dirname) { + return; + } + cp.execSync(`rm -rf '${extractPath}'`); + }); + + const resolveExtract = async (archivePath: string): Promise => { + expect(fs.existsSync(archivePath)).toEqual(true); + await expect(zip.extract( + archivePath, + extractPath, + { sourcePath: "extension", overwrite: true }, + CancellationToken.None, + )).resolves.toBe(undefined); + expect(fs.existsSync(extractPath)).toEqual(true); + }; + + // tslint:disable-next-line:no-any + const extract = (archivePath: string): () => any => { + // tslint:disable-next-line:no-any + return async (): Promise => { + await resolveExtract(archivePath); + expect(fs.existsSync(path.resolve(extractPath, ".vsixmanifest"))).toEqual(true); + expect(fs.existsSync(path.resolve(extractPath, "package.json"))).toEqual(true); + }; + }; + it("should extract from tarred VSIX", extract(tarPath), 2000); + it("should extract from zipped VSIX", extract(vsixPath), 2000); + + // tslint:disable-next-line:no-any + const buffer = (archivePath: string): () => any => { + // tslint:disable-next-line:no-any + return async (): Promise => { + await resolveExtract(archivePath); + const manifestPath = path.resolve(extractPath, ".vsixmanifest"); + expect(fs.existsSync(manifestPath)).toEqual(true); + const manifestBuf = fs.readFileSync(manifestPath); + expect(manifestBuf.length).toBeGreaterThan(0); + await expect(zip.buffer(archivePath, "extension.vsixmanifest")).resolves.toEqual(manifestBuf); + }; + }; + it("should buffer tarred VSIX", buffer(tarPath), 2000); + it("should buffer zipped VSIX", buffer(vsixPath), 2000); +}); diff --git a/packages/vscode/webpack.bootstrap.config.js b/packages/vscode/webpack.bootstrap.config.js index abfda9bd3..0b50f54e5 100644 --- a/packages/vscode/webpack.bootstrap.config.js +++ b/packages/vscode/webpack.bootstrap.config.js @@ -62,6 +62,7 @@ module.exports = merge( "vs/platform/product/node/package": path.resolve(vsFills, "package.ts"), "vs/platform/product/node/product": path.resolve(vsFills, "product.ts"), "vs/base/node/zip": path.resolve(vsFills, "zip.ts"), + "vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"), "vs": path.resolve(root, "lib/vscode/src/vs"), }, }, diff --git a/packages/web/webpack.config.js b/packages/web/webpack.config.js index ecf027dd6..8c0316aca 100644 --- a/packages/web/webpack.config.js +++ b/packages/web/webpack.config.js @@ -75,6 +75,7 @@ module.exports = merge( "vs/platform/product/node/package": path.resolve(vsFills, "package.ts"), "vs/platform/product/node/product": path.resolve(vsFills, "product.ts"), "vs/base/node/zip": path.resolve(vsFills, "zip.ts"), + "vszip": path.resolve(root, "lib/vscode/src/vs/base/node/zip.ts"), "vs": path.join(root, "lib", "vscode", "src", "vs"), }, },