Archived
1
0

not finished

This commit is contained in:
Asher
2019-01-07 18:46:19 -06:00
committed by Kyle Carberry
parent 776bb227e6
commit 9cd81f73fa
79 changed files with 11015 additions and 0 deletions

View File

@ -0,0 +1,20 @@
{
"name": "requirefs",
"description": "",
"main": "src/index.ts",
"scripts": {
"benchmark": "ts-node ./test/*.bench.ts"
},
"dependencies": {
"jszip": "2.6.0",
"path": "0.12.7",
"resolve": "1.8.1"
},
"devDependencies": {
"@types/benchmark": "^1.0.31",
"@types/jszip": "3.1.4",
"@types/resolve": "0.0.8",
"benchmark": "^2.1.4",
"text-encoding": "0.6.4"
}
}

View File

@ -0,0 +1 @@
export * from "./requirefs";

View File

@ -0,0 +1,170 @@
import * as JSZip from "jszip";
import * as path from "path";
import * as resolve from "resolve";
import { Tar } from "./tarReader";
const textDecoder = new (typeof TextDecoder === "undefined" ? require("text-encoding").TextDecoder : TextDecoder)();
export interface IFileReader {
exists(path: string): boolean;
read(path: string): Uint8Array;
}
/**
* RequireFS allows users to require from a file system.
*/
export class RequireFS {
private readonly reader: IFileReader;
private readonly customModules: Map<string, { exports: object }>;
private readonly requireCache: Map<string, { exports: object }>;
private baseDir: string | undefined;
public constructor(reader: IFileReader) {
this.reader = reader;
this.customModules = new Map();
this.requireCache = new Map();
}
/**
* Add a base-directory to nest from.
*/
public basedir(path: string): void {
this.baseDir = path;
}
/**
* Provide custom modules to the require instance.
*/
// tslint:disable-next-line:no-any
public provide(module: string, value: any): void {
if (this.customModules.has(module)) {
throw new Error("custom module has already been registered with this name");
}
this.customModules.set(module, value);
}
public readFile(target: string, type?: "string"): string;
public readFile(target: string, type?: "buffer"): Buffer;
/**
* Read a file and returns its contents.
*/
public readFile(target: string, type?: "string" | "buffer"): string | Buffer {
target = path.normalize(target);
const read = this.reader.read(target);
return type === "string" ? textDecoder.decode(read) : Buffer.from(read);
}
/**
* Require a path from a file system.
*/
// tslint:disable-next-line:no-any
public require(target: string): any {
target = path.normalize(target);
return this.doRequire([target], `./${path.basename(target)}`);
}
/**
* Do require for a caller. Needed for resolving relative paths.
*/
private doRequire(callers: string[], resolvePath: string): object {
if (this.customModules.has(resolvePath)) {
return this.customModules.get(resolvePath)!.exports;
}
const caller = callers[callers.length - 1];
const reader = this.reader;
const newRelative = this.realizePath(caller, resolvePath);
if (this.requireCache.has(newRelative)) {
return this.requireCache.get(newRelative)!.exports;
}
const module = {
exports: {},
};
this.requireCache.set(newRelative, module);
const content = textDecoder.decode(reader.read(newRelative));
if (newRelative.endsWith(".json")) {
module.exports = JSON.parse(content);
} else {
eval("'use strict'; " + content);
}
return module.exports;
}
/**
* Attempts to find a module from a path
*/
private realizePath(caller: string, fullRelative: string): string {
const stripPrefix = (path: string): string => {
if (path.startsWith("/")) {
path = path.substr(1);
}
if (path.endsWith("/")) {
path = path.substr(0, path.length - 1);
}
return path;
};
const callerDirname = path.dirname(caller);
const resolvedPath = resolve.sync(fullRelative, {
basedir: this.baseDir ? callerDirname.startsWith(this.baseDir) ? callerDirname : path.join(this.baseDir, callerDirname) : callerDirname,
extensions: [".js"],
readFileSync: (file: string): string => {
return this.readFile(stripPrefix(file));
},
isFile: (file: string): boolean => {
return this.reader.exists(stripPrefix(file));
},
});
return stripPrefix(resolvedPath);
}
}
export const fromTar = (content: Uint8Array): RequireFS => {
const tar = Tar.fromUint8Array(content);
return new RequireFS({
exists: (path: string): boolean => {
return tar.files.has(path);
},
read: (path: string): Uint8Array => {
const file = tar.files.get(path);
if (!file) {
throw new Error(`file "${path}" not found`);
}
return file.read();
},
});
};
export const fromZip = (content: Uint8Array): RequireFS => {
const zip = new JSZip(content);
return new RequireFS({
exists: (fsPath: string): boolean => {
const file = zip.file(fsPath);
return typeof file !== "undefined" && file !== null;
},
read: (fsPath: string): Uint8Array => {
const file = zip.file(fsPath);
if (!file) {
throw new Error(`file "${fsPath}" not found`);
}
// TODO: Should refactor to allow a promise.
// tslint:disable-next-line no-any
return zip.file(fsPath).async("uint8array") as any;
},
});
};

View File

@ -0,0 +1,285 @@
import * as path from "path";
const textDecoder = new (typeof TextDecoder === "undefined" ? require("text-encoding").TextDecoder : TextDecoder)();
/**
* Tar represents a tar archive.
*/
export class Tar {
/**
* Return a tar object from a Uint8Array.
*/
public static fromUint8Array(array: Uint8Array): Tar {
const reader = new Reader(array);
const tar = new Tar();
while (true) {
try {
const file = TarFile.fromReader(reader);
if (file) {
tar._files.set(path.normalize(file.name), file);
}
} catch (e) {
if (e.message === "EOF") {
break;
}
throw e;
}
}
reader.unclamp();
return tar;
}
private readonly _files: Map<string, TarFile>;
private constructor() {
this._files = new Map();
}
public get files(): ReadonlyMap<string, TarFile> {
return this._files;
}
}
/**
* Represents a tar files location within a reader
*/
export class TarFile {
/**
* Locate a tar file from a reader.
*/
public static fromReader(reader: Reader): TarFile | undefined {
const firstByte = reader.peek(1)[0];
// If the first byte is nil, we know it isn't a filename
if (firstByte === 0x00) {
// The tar header is 512 bytes large. Its safe to skip here
// because we know this block is not a header
reader.skip(512);
return undefined;
}
let name = reader.readString(100);
reader.skip(8); // 100->108 mode
reader.skip(8); // 108->116 uid
reader.skip(8); // 116->124 gid
const rawSize = reader.read(12); // 124->136 size
reader.skip(12); // 136->148 mtime
if (reader.jump(345).readByte()) {
name = reader.jump(345).readString(155) + "/" + name;
}
const nums: number[] = [];
rawSize.forEach((a) => nums.push(a));
const parseSize = (): number => {
let offset = 0;
// While 48 (ASCII value of 0), the byte is nil and considered padding.
while (offset < rawSize.length && nums[offset] === 48) {
offset++;
}
const clamp = (index: number, len: number, defaultValue: number): number => {
if (typeof index !== "number") {
return defaultValue;
}
// Coerce index to an integer.
index = ~~index;
if (index >= len) {
return len;
}
if (index >= 0) {
return index;
}
index += len;
if (index >= 0) {
return index;
}
return 0;
};
// Checks for the index of the POSIX file-size terminating char.
// Falls back to GNU's tar format. If neither characters are found
// the index will default to the end of the file size buffer.
let i = nums.indexOf(32, offset);
if (i === -1) {
i = nums.indexOf(0, offset);
if (i === -1) {
i = rawSize.length - 1;
}
}
const end = clamp(i, rawSize.length, rawSize.length - 1);
if (end === offset) {
return 0;
}
return parseInt(textDecoder.decode(rawSize.slice(offset, end)), 8);
};
const size = parseSize();
const overflow = ((): number => {
let newSize = size;
newSize &= 511;
return newSize && 512 - newSize;
})();
reader.jump(512);
const offset = reader.offset;
reader.skip(overflow + size);
reader.clamp();
const tarFile = new TarFile(reader, {
offset,
name,
size,
});
return tarFile;
}
public constructor(
private readonly reader: Reader,
private readonly data: {
name: string;
size: number;
offset: number;
},
) { }
public get name(): string {
return this.data.name;
}
public get size(): number {
return this.data.size;
}
/**
* Check if the file type is a file.
*/
public isFile(): boolean {
throw new Error("not implemented");
}
/**
* Read the file as a string.
*/
public readAsString(): string {
return textDecoder.decode(this.read());
}
/**
* Read the file as Uint8Array.
*/
public read(): Uint8Array {
return this.reader.jump(this.data.offset).read(this.data.size);
}
}
/**
* Reads within a Uint8Array.
*/
export class Reader {
private array: Uint8Array;
private _offset: number;
private lastClamp: number;
public constructor(array: Uint8Array) {
this.array = array;
this._offset = 0;
this.lastClamp = 0;
}
public get offset(): number {
return this._offset;
}
/**
* Skip the specified amount of bytes.
*/
public skip(amount: number): boolean {
if (this._offset + amount > this.array.length) {
throw new Error("EOF");
}
this._offset += amount;
return true;
}
/**
* Clamp the reader at a position.
*/
public clamp(): void {
this.lastClamp = this._offset;
}
/**
* Unclamp the reader.
*/
public unclamp(): void {
this.lastClamp = 0;
}
/**
* Jump to a specific offset.
*/
public jump(offset: number): Reader {
this._offset = offset + this.lastClamp;
return this;
}
/**
* Peek the amount of bytes.
*/
public peek(amount: number): Uint8Array {
return this.array.slice(this.offset, this.offset + amount);
}
/**
* Read a string.
*/
public readString(amount: number): string {
// Replacing the 0s removes all nil bytes from the str
return textDecoder.decode(this.read(amount)).replace(/\0/g, "");
}
/**
* Read a byte in the array.
*/
public readByte(): number {
const data = this.array[this._offset];
this._offset++;
return data;
}
/**
* Read the amount of bytes.
*/
public read(amount: number): Uint8Array {
if (this._offset > this.array.length) {
throw new Error("EOF");
}
const data = this.array.slice(this._offset, this._offset + amount);
this._offset += amount;
return data;
}
}

3
packages/requirefs/test/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
!lib/node_modules
*.tar
*.zip

View File

@ -0,0 +1 @@
exports = require("./chained-2");

View File

@ -0,0 +1 @@
exports = require("./chained-3");

View File

@ -0,0 +1 @@
exports.text = "moo";

View File

@ -0,0 +1 @@
exports = require("donkey");

View File

@ -0,0 +1 @@
exports.frog = "hi";

View File

@ -0,0 +1,3 @@
const frogger = require("frogger");
exports = frogger;

View File

@ -0,0 +1 @@
exports.banana = "potato";

View File

@ -0,0 +1 @@
exports = coder.test;

View File

@ -0,0 +1 @@
exports.orangeColor = require("./subfolder/oranges").orange;

View File

@ -0,0 +1 @@
exports = require("../individual");

View File

@ -0,0 +1 @@
exports.orange = "blue";

View File

@ -0,0 +1,48 @@
import * as benchmark from "benchmark";
import { performance } from "perf_hooks";
import { TestCaseArray, isMac } from "./requirefs.util";
const files = [
"./individual.js", "./chained-1", "./subfolder",
"./subfolder/goingUp", "./nodeResolve",
];
const toBench = new TestCaseArray();
// Limits the amount of time taken for each test,
// but increases uncertainty.
benchmark.options.maxTime = 0.5;
let suite = new benchmark.Suite();
let _start = 0;
const addMany = (names: string[]): benchmark.Suite => {
for (let name of names) {
for (let file of files) {
suite = suite.add(`${name} -> ${file}`, async () => {
let rfs = await toBench.byName(name).rfs;
rfs.require(file);
});
}
}
_start = performance.now();
return suite;
}
// Returns mean time per operation, in microseconds (10^-6s).
const mean = (c: any): number => {
return Number((c.stats.mean * 10e+5).toFixed(5));
};
// Swap out the tar command for gtar, when on MacOS.
let testNames = ["zip", "bsdtar", isMac ? "gtar" : "tar"];
addMany(testNames).on("cycle", (event: benchmark.Event) => {
console.log(String(event.target) + ` (~${mean(event.target)} μs/op)`);
}).on("complete", () => {
const slowest = suite.filter("slowest").shift();
const fastest = suite.filter("fastest").shift();
console.log(`===\nFastest is ${fastest.name} with ~${mean(fastest)} μs/op`);
if (slowest.name !== fastest.name) {
console.log(`Slowest is ${slowest.name} with ~${mean(slowest)} μs/op`);
}
const d = ((performance.now() - _start)/1000).toFixed(2);
console.log(`Benchmark took ${d} s`);
})
.run({ "async": true });

View File

@ -0,0 +1,56 @@
import { RequireFS } from "../src/requirefs";
import { TestCaseArray, isMac } from "./requirefs.util";
const toTest = new TestCaseArray();
describe("requirefs", () => {
for (let i = 0; i < toTest.length(); i++) {
const testCase = toTest.byID(i);
if (!isMac && testCase.name === "gtar") {
break;
}
if (isMac && testCase.name === "tar") {
break;
}
describe(testCase.name, () => {
let rfs: RequireFS;
beforeAll(async () => {
rfs = await testCase.rfs;
});
it("should parse individual module", () => {
expect(rfs.require("./individual.js").frog).toEqual("hi");
});
it("should parse chained modules", () => {
expect(rfs.require("./chained-1").text).toEqual("moo");
});
it("should parse through subfolders", () => {
expect(rfs.require("./subfolder").orangeColor).toEqual("blue");
});
it("should be able to move up directories", () => {
expect(rfs.require("./subfolder/goingUp").frog).toEqual("hi");
});
it("should resolve node_modules", () => {
expect(rfs.require("./nodeResolve").banana).toEqual("potato");
});
it("should access global scope", () => {
// tslint:disable-next-line no-any for testing
(window as any).coder = {
test: "hi",
};
expect(rfs.require("./scope")).toEqual("hi");
});
it("should find custom module", () => {
rfs.provide("donkey", "ok");
expect(rfs.require("./customModule")).toEqual("ok");
});
});
}
});

View File

@ -0,0 +1,112 @@
import * as cp from "child_process";
import * as path from "path";
import * as fs from "fs";
import * as os from "os";
import { fromTar, RequireFS, fromZip } from "../src/requirefs";
export const isMac = os.platform() === "darwin";
/**
* Encapsulates a RequireFS Promise and the
* name of the test case it will be used in.
*/
interface TestCase {
rfs: Promise<RequireFS>;
name: string;
}
/**
* TestCaseArray allows tests and benchmarks to share
* test cases while limiting redundancy.
*/
export class TestCaseArray {
private cases: Array<TestCase> = [];
constructor(cases?: Array<TestCase>) {
if (!cases) {
this.cases = TestCaseArray.defaults();
return
}
this.cases = cases;
}
/**
* Returns default test cases. MacOS users need to have `gtar` binary
* in order to run GNU-tar tests and benchmarks.
*/
public static defaults(): Array<TestCase> {
let cases: Array<TestCase> = [
TestCaseArray.newCase("cd lib && zip -r ../lib.zip ./*", "lib.zip", async (c) => fromZip(c), "zip"),
TestCaseArray.newCase("cd lib && bsdtar cvf ../lib.tar ./*", "lib.tar", async (c) => fromTar(c), "bsdtar"),
];
if (isMac) {
const gtarInstalled: boolean = cp.execSync("which tar").length > 0;
if (gtarInstalled) {
cases.push(TestCaseArray.newCase("cd lib && gtar cvf ../lib.tar ./*", "lib.tar", async (c) => fromTar(c), "gtar"));
} else {
throw new Error("failed to setup gtar test case, gtar binary is necessary to test GNU-tar on MacOS");
}
} else {
cases.push(TestCaseArray.newCase("cd lib && tar cvf ../lib.tar ./*", "lib.tar", async (c) => fromTar(c), "tar"));
}
return cases;
};
/**
* Returns a test case prepared with the provided RequireFS Promise.
* @param command Command to run immediately. For setup.
* @param targetFile File to be read and handled by prepare function.
* @param prepare Run on target file contents before test.
* @param name Test case name.
*/
public static newCase(command: string, targetFile: string, prepare: (content: Uint8Array) => Promise<RequireFS>, name: string): TestCase {
cp.execSync(command, { cwd: __dirname });
const content = fs.readFileSync(path.join(__dirname, targetFile));
return {
name,
rfs: prepare(content),
};
}
/**
* Returns updated TestCaseArray instance, with a new test case.
* @see TestCaseArray.newCase
*/
public add(command: string, targetFile: string, prepare: (content: Uint8Array) => Promise<RequireFS>, name: string): TestCaseArray {
this.cases.push(TestCaseArray.newCase(command, targetFile, prepare, name));
return this;
};
/**
* Gets a test case by index.
* @param id Test case index.
*/
public byID(id: number): TestCase {
if (!this.cases[id]) {
if (id < 0 || id >= this.cases.length) {
throw new Error(`test case index "${id}" out of bounds`);
}
throw new Error(`test case at index "${id}" not found`);
}
return this.cases[id];
}
/**
* Gets a test case by name.
* @param name Test case name.
*/
public byName(name: string): TestCase {
let c = this.cases.find((c) => c.name === name);
if (!c) {
throw new Error(`test case "${name}" not found`);
}
return c;
}
/**
* Gets the number of test cases.
*/
public length(): number {
return this.cases.length;
}
}

View File

@ -0,0 +1,99 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1
"@types/benchmark@^1.0.31":
version "1.0.31"
resolved "https://registry.yarnpkg.com/@types/benchmark/-/benchmark-1.0.31.tgz#2dd3514e93396f362ba5551a7c9ff0da405c1d38"
integrity sha512-F6fVNOkGEkSdo/19yWYOwVKGvzbTeWkR/XQYBKtGBQ9oGRjBN9f/L4aJI4sDcVPJO58Y1CJZN8va9V2BhrZapA==
"@types/jszip@3.1.4":
version "3.1.4"
resolved "https://registry.yarnpkg.com/@types/jszip/-/jszip-3.1.4.tgz#9b81e3901a6988e9459ac27abf483e6b892251af"
integrity sha512-UaVbz4buRlBEolZYrxqkrGDOypugYlbqGNrUFB4qBaexrLypTH0jyvaF5jolNy5D+5C4kKV1WJ3Yx9cn/JH8oA==
dependencies:
"@types/node" "*"
"@types/node@*":
version "10.11.3"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.11.3.tgz#c055536ac8a5e871701aa01914be5731539d01ee"
integrity sha512-3AvcEJAh9EMatxs+OxAlvAEs7OTy6AG94mcH1iqyVDwVVndekLxzwkWQ/Z4SDbY6GO2oyUXyWW8tQ4rENSSQVQ==
"@types/resolve@0.0.8":
version "0.0.8"
resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-0.0.8.tgz#f26074d238e02659e323ce1a13d041eee280e194"
integrity sha512-auApPaJf3NPfe18hSoJkp8EbZzer2ISk7o8mCC3M9he/a04+gbMF97NkpD2S8riMGvm4BMRI59/SZQSaLTKpsQ==
dependencies:
"@types/node" "*"
benchmark@^2.1.4:
version "2.1.4"
resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629"
integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik=
dependencies:
lodash "^4.17.4"
platform "^1.3.3"
inherits@2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
jszip@2.6.0:
version "2.6.0"
resolved "https://registry.yarnpkg.com/jszip/-/jszip-2.6.0.tgz#7fb3e9c2f11c8a9840612db5dabbc8cf3a7534b7"
integrity sha1-f7PpwvEciphAYS212rvIzzp1NLc=
dependencies:
pako "~1.0.0"
lodash@^4.17.4:
version "4.17.11"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
pako@~1.0.0:
version "1.0.6"
resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.6.tgz#0101211baa70c4bca4a0f63f2206e97b7dfaf258"
integrity sha512-lQe48YPsMJAig+yngZ87Lus+NF+3mtu7DVOBu6b/gHO1YpKwIj5AWjZ/TOS7i46HD/UixzWb1zeWDZfGZ3iYcg==
path-parse@^1.0.5:
version "1.0.6"
resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c"
integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==
path@0.12.7:
version "0.12.7"
resolved "https://registry.yarnpkg.com/path/-/path-0.12.7.tgz#d4dc2a506c4ce2197eb481ebfcd5b36c0140b10f"
integrity sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=
dependencies:
process "^0.11.1"
util "^0.10.3"
platform@^1.3.3:
version "1.3.5"
resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.5.tgz#fb6958c696e07e2918d2eeda0f0bc9448d733444"
integrity sha512-TuvHS8AOIZNAlE77WUDiR4rySV/VMptyMfcfeoMgs4P8apaZM3JrnbzBiixKUv+XR6i+BXrQh8WAnjaSPFO65Q==
process@^0.11.1:
version "0.11.10"
resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI=
resolve@1.8.1:
version "1.8.1"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.8.1.tgz#82f1ec19a423ac1fbd080b0bab06ba36e84a7a26"
integrity sha512-AicPrAC7Qu1JxPCZ9ZgCZlY35QgFnNqc+0LtbRNxnVw4TXvjQ72wnuL9JQcEBgXkI9JM8MsT9kaQoHcpCRJOYA==
dependencies:
path-parse "^1.0.5"
text-encoding@0.6.4:
version "0.6.4"
resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19"
integrity sha1-45mpgiV6J22uQou5KEXLcb3CbRk=
util@^0.10.3:
version "0.10.4"
resolved "https://registry.yarnpkg.com/util/-/util-0.10.4.tgz#3aa0125bfe668a4672de58857d3ace27ecb76901"
integrity sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==
dependencies:
inherits "2.0.3"