From 3479bd5aaa0e4a04fbcf0b88dbd1ab10a580e874 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 23 Dec 2020 22:09:38 +0100 Subject: [PATCH] Add label-custom input (#35) Co-authored-by: CrazyMax --- .github/CONTRIBUTING.md | 18 +- .github/workflows/ci.yml | 19 + README.md | 45 +- __tests__/context.test.ts | 110 ++- __tests__/meta.test.ts | 36 + dist/index.js | 1461 ++++++++++++++++++++++++++++++++++++- package.json | 1 + src/context.ts | 30 +- src/meta.ts | 6 +- tsconfig.json | 8 +- yarn.lock | 5 + 11 files changed, 1671 insertions(+), 68 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 19bd949..329fef1 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,7 +2,8 @@ Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. -Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE). +Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) +to the public under the [project's open source license](LICENSE). ## Submitting a pull request @@ -16,21 +17,6 @@ Contributions to this project are [released](https://help.github.com/articles/gi 8. Push to your fork and [submit a pull request](https://github.com/crazy-max/ghaction-docker-meta/compare) 9. Pat your self on the back and wait for your pull request to be reviewed and merged. -## Container based developer flow - -If you don't want to maintain a Node developer environment that fits this project you can use containerized commands instead of invoking yarn directly. - -``` -# format code and build javascript artifacts -docker buildx bake pre-checkin - -# validate all code has correctly formatted and built -docker buildx bake validate - -# run tests -docker buildx bake test -``` - Here are a few things you can do that will increase the likelihood of your pull request being accepted: - Write tests. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2937c6..3c8063a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,25 @@ jobs: {{major}}.{{minor}}.{{patch}} tag-latest: ${{ matrix.tag-latest }} + label-custom: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Docker meta + uses: ./ + with: + images: | + ${{ env.DOCKER_IMAGE }} + ghcr.io/name/app + label-custom: | + maintainer=CrazyMax + org.opencontainers.image.title=MyCustomTitle + org.opencontainers.image.description=Another description + org.opencontainers.image.vendor=MyCompany + docker-push: runs-on: ubuntu-latest services: diff --git a/README.md b/README.md index 486270b..798cfb6 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ ___ * [Schedule tag](#schedule-tag) * [Overwrite labels](#overwrite-labels) * [Keep up-to-date with GitHub Dependabot](#keep-up-to-date-with-github-dependabot) -* [How can I help?](#how-can-i-help) +* [Contributing](#contributing) * [License](#license) ## Features @@ -235,6 +235,19 @@ jobs: Following inputs can be used as `step.with` keys +> `List` type is a newline-delimited string +> ```yaml +> label-custom: | +> org.opencontainers.image.title=MyCustomTitle +> org.opencontainers.image.description=Another description +> org.opencontainers.image.vendor=MyCompany +> ``` + +> `CSV` type is a comma-delimited string +> ```yaml +> images: name/app,ghcr.io/name/app +> ``` + | Name | Type | Description | |---------------------|----------|------------------------------------| | `images` | List/CSV | List of Docker images to use as base name for tags | @@ -248,11 +261,10 @@ Following inputs can be used as `step.with` keys | `tag-schedule` | String | [Template](#schedule-tag) to apply to schedule tag (default `nightly`) | | `tag-custom` | List/CSV | List of custom tags | | `tag-custom-only` | Bool | Only use `tag-custom` as Docker tags | +| `label-custom` | List | List of custom labels | | `sep-tags` | String | Separator to use for tags output (default `\n`) | | `sep-labels` | String | Separator to use for labels output (default `\n`) | -> List/CSV type can be a newline or comma delimited string - > `tag-semver` and `tag-match` are mutually exclusive ### outputs @@ -326,16 +338,13 @@ labels generated are not suitable, you can overwrite them like this: ```yaml - - name: Build and push - uses: docker/build-push-action@v2 + name: Docker meta + id: docker_meta + uses: crazy-max/ghaction-docker-meta@v1 with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64,linux/386 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: | - ${{ steps.docker_meta.outputs.labels }} + images: name/app + label-custom: | + maintainer=CrazyMax org.opencontainers.image.title=MyCustomTitle org.opencontainers.image.description=Another description org.opencontainers.image.vendor=MyCompany @@ -357,12 +366,14 @@ updates: interval: "daily" ``` -## How can I help? +# Contributing -All kinds of contributions are welcome :raised_hands:! The most basic way to show your support is to star :star2: -the project, or to raise issues :speech_balloon: You can also support this project by -[**becoming a sponsor on GitHub**](https://github.com/sponsors/crazy-max) :clap: or by making a -[Paypal donation](https://www.paypal.me/crazyws) to ensure this journey continues indefinitely! :rocket: +Want to contribute? Awesome! The most basic way to show your support is to star :star2: the project, +or to raise issues :speech_balloon:. If you want to open a pull request, please read the +[contributing guidelines](.github/CONTRIBUTING.md). + +You can also support this project by [**becoming a sponsor on GitHub**](https://github.com/sponsors/crazy-max) or by +making a [Paypal donation](https://www.paypal.me/crazyws) to ensure this journey continues indefinitely! Thanks again for your support, it is much appreciated! :pray: diff --git a/__tests__/context.test.ts b/__tests__/context.test.ts index 0b097eb..87d95db 100644 --- a/__tests__/context.test.ts +++ b/__tests__/context.test.ts @@ -1,54 +1,150 @@ import * as context from '../src/context'; describe('getInputList', () => { - it('handles single line correctly', async () => { + it('single line correctly', async () => { await setInput('foo', 'bar'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar']); }); - it('handles multiple lines correctly', async () => { + it('multiline correctly', async () => { setInput('foo', 'bar\nbaz'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz']); }); - it('remove empty lines correctly', async () => { + it('empty lines correctly', async () => { setInput('foo', 'bar\n\nbaz'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz']); }); - it('handles comma correctly', async () => { + it('comma correctly', async () => { setInput('foo', 'bar,baz'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz']); }); - it('remove empty result correctly', async () => { + it('empty result correctly', async () => { setInput('foo', 'bar,baz,'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz']); }); - it('handles different new lines correctly', async () => { + it('different new lines correctly', async () => { setInput('foo', 'bar\r\nbaz'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz']); }); - it('handles different new lines and comma correctly', async () => { + it('different new lines and comma correctly', async () => { setInput('foo', 'bar\r\nbaz,bat'); const res = await context.getInputList('foo'); console.log(res); expect(res).toEqual(['bar', 'baz', 'bat']); }); + + it('multiline and ignoring comma correctly', async () => { + setInput('cache-from', 'user/app:cache\ntype=local,src=path/to/dir'); + const res = await context.getInputList('cache-from', true); + console.log(res); + expect(res).toEqual(['user/app:cache', 'type=local,src=path/to/dir']); + }); + + it('different new lines and ignoring comma correctly', async () => { + setInput('cache-from', 'user/app:cache\r\ntype=local,src=path/to/dir'); + const res = await context.getInputList('cache-from', true); + console.log(res); + expect(res).toEqual(['user/app:cache', 'type=local,src=path/to/dir']); + }); + + it('multiline values', async () => { + setInput( + 'secrets', + `GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789 +"MYSECRET=aaaaaaaa +bbbbbbb +ccccccccc" +FOO=bar` + ); + const res = await context.getInputList('secrets', true); + console.log(res); + expect(res).toEqual([ + 'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789', + `MYSECRET=aaaaaaaa +bbbbbbb +ccccccccc`, + 'FOO=bar' + ]); + }); + + it('multiline values with empty lines', async () => { + setInput( + 'secrets', + `GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789 +"MYSECRET=aaaaaaaa +bbbbbbb +ccccccccc" +FOO=bar +"EMPTYLINE=aaaa + +bbbb +ccc"` + ); + const res = await context.getInputList('secrets', true); + console.log(res); + expect(res).toEqual([ + 'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789', + `MYSECRET=aaaaaaaa +bbbbbbb +ccccccccc`, + 'FOO=bar', + `EMPTYLINE=aaaa + +bbbb +ccc` + ]); + }); + + it('multiline values without quotes', async () => { + setInput( + 'secrets', + `GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789 +MYSECRET=aaaaaaaa +bbbbbbb +ccccccccc +FOO=bar` + ); + const res = await context.getInputList('secrets', true); + console.log(res); + expect(res).toEqual(['GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789', 'MYSECRET=aaaaaaaa', 'bbbbbbb', 'ccccccccc', 'FOO=bar']); + }); + + it('multiline values escape quotes', async () => { + setInput( + 'secrets', + `GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789 +"MYSECRET=aaaaaaaa +bbbb""bbb +ccccccccc" +FOO=bar` + ); + const res = await context.getInputList('secrets', true); + console.log(res); + expect(res).toEqual([ + 'GIT_AUTH_TOKEN=abcdefgh,ijklmno=0123456789', + `MYSECRET=aaaaaaaa +bbbb\"bbb +ccccccccc`, + 'FOO=bar' + ]); + }); }); describe('asyncForEach', () => { diff --git a/__tests__/meta.test.ts b/__tests__/meta.test.ts index c1aa051..aea3425 100644 --- a/__tests__/meta.test.ts +++ b/__tests__/meta.test.ts @@ -1009,6 +1009,42 @@ describe('latest', () => { "org.opencontainers.image.licenses=MIT" ] ], + [ + 'event_tag_v1.1.1.env', + { + images: ['org/app', 'ghcr.io/MyUSER/MyApp'], + tagLatest: false, + labelCustom: [ + "maintainer=CrazyMax", + "org.opencontainers.image.title=MyCustomTitle", + "org.opencontainers.image.description=Another description", + "org.opencontainers.image.vendor=MyCompany", + ], + } as Inputs, + { + main: 'v1.1.1', + partial: [], + latest: false + } as Version, + [ + 'org/app:v1.1.1', + 'ghcr.io/myuser/myapp:v1.1.1', + ], + [ + "org.opencontainers.image.title=Hello-World", + "org.opencontainers.image.description=This your first repo!", + "org.opencontainers.image.url=https://github.com/octocat/Hello-World", + "org.opencontainers.image.source=https://github.com/octocat/Hello-World", + "org.opencontainers.image.version=v1.1.1", + "org.opencontainers.image.created=2020-01-10T00:30:00.000Z", + "org.opencontainers.image.revision=90dd6032fac8bda1b6c4436a2e65de27961ed071", + "org.opencontainers.image.licenses=MIT", + "maintainer=CrazyMax", + "org.opencontainers.image.title=MyCustomTitle", + "org.opencontainers.image.description=Another description", + "org.opencontainers.image.vendor=MyCompany" + ] + ], ])('given %p event ', tagsLabelsTest); }); diff --git a/dist/index.js b/dist/index.js index 67851da..3a8f66e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -7,6 +7,25 @@ module.exports = "use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { @@ -16,9 +35,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.asyncForEach = exports.getInputList = exports.getInputs = void 0; -const core = __webpack_require__(2186); +const sync_1 = __importDefault(__webpack_require__(8750)); +const core = __importStar(__webpack_require__(2186)); function getInputs() { return { images: getInputList('images'), @@ -32,21 +55,35 @@ function getInputs() { tagSchedule: core.getInput('tag-schedule') || 'nightly', tagCustom: getInputList('tag-custom'), tagCustomOnly: /true/i.test(core.getInput('tag-custom-only') || 'false'), + labelCustom: getInputList('label-custom'), sepTags: core.getInput('sep-tags') || `\n`, sepLabels: core.getInput('sep-labels') || `\n`, githubToken: core.getInput('github-token') }; } exports.getInputs = getInputs; -function getInputList(name) { +function getInputList(name, ignoreComma) { + let res = []; const items = core.getInput(name); if (items == '') { - return []; + return res; } - return items - .split(/\r?\n/) - .filter(x => x) - .reduce((acc, line) => acc.concat(line.split(',').filter(x => x)).map(pat => pat.trim()), []); + for (let output of sync_1.default(items, { + columns: false, + relaxColumnCount: true, + skipLinesWithEmptyValues: true + })) { + if (output.length == 1) { + res.push(output[0]); + continue; + } + else if (!ignoreComma) { + res.push(...output); + continue; + } + res.push(output.join(',')); + } + return res.filter(item => item).map(pat => pat.trim()); } exports.getInputList = getInputList; exports.asyncForEach = (array, callback) => __awaiter(void 0, void 0, void 0, function* () { @@ -63,6 +100,25 @@ exports.asyncForEach = (array, callback) => __awaiter(void 0, void 0, void 0, fu "use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { @@ -74,7 +130,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.repo = exports.context = void 0; -const github = __webpack_require__(5438); +const github = __importStar(__webpack_require__(5438)); function context() { return github.context; } @@ -99,6 +155,25 @@ exports.repo = repo; "use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { @@ -110,9 +185,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }; Object.defineProperty(exports, "__esModule", ({ value: true })); const context_1 = __webpack_require__(3842); -const github = __webpack_require__(5928); +const github = __importStar(__webpack_require__(5928)); const meta_1 = __webpack_require__(3714); -const core = __webpack_require__(2186); +const core = __importStar(__webpack_require__(2186)); function run() { return __awaiter(this, void 0, void 0, function* () { try { @@ -164,16 +239,38 @@ run(); /***/ }), /***/ 3714: -/***/ ((__unused_webpack_module, exports, __webpack_require__) => { +/***/ (function(__unused_webpack_module, exports, __webpack_require__) { "use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; Object.defineProperty(exports, "__esModule", ({ value: true })); exports.Meta = void 0; -const handlebars = __webpack_require__(7492); -const moment = __webpack_require__(9623); -const semver = __webpack_require__(1383); -const core = __webpack_require__(2186); +const handlebars = __importStar(__webpack_require__(7492)); +const moment_1 = __importDefault(__webpack_require__(9623)); +const semver = __importStar(__webpack_require__(1383)); +const core = __importStar(__webpack_require__(2186)); class Meta { constructor(inputs, context, repo) { this.inputs = inputs; @@ -195,7 +292,7 @@ class Meta { if (/schedule/.test(this.context.eventName)) { version.main = handlebars.compile(this.inputs.tagSchedule)({ date: function (format) { - return moment(currentDate).utc().format(format); + return moment_1.default(currentDate).utc().format(format); } }); } @@ -287,7 +384,7 @@ class Meta { } labels() { var _a; - return [ + let labels = [ `org.opencontainers.image.title=${this.repo.name || ''}`, `org.opencontainers.image.description=${this.repo.description || ''}`, `org.opencontainers.image.url=${this.repo.html_url || ''}`, @@ -297,6 +394,8 @@ class Meta { `org.opencontainers.image.revision=${this.context.sha || ''}`, `org.opencontainers.image.licenses=${((_a = this.repo.license) === null || _a === void 0 ? void 0 : _a.spdx_id) || ''}` ]; + labels.push(...this.inputs.labelCustom); + return labels; } } exports.Meta = Meta; @@ -4006,6 +4105,1334 @@ function removeHook (state, name, method) { } +/***/ }), + +/***/ 6942: +/***/ ((module) => { + + + +class ResizeableBuffer{ + constructor(size=100){ + this.size = size + this.length = 0 + this.buf = Buffer.alloc(size) + } + prepend(val){ + if(Buffer.isBuffer(val)){ + const length = this.length + val.length + if(length >= this.size){ + this.resize() + if(length >= this.size){ + throw Error('INVALID_BUFFER_STATE') + } + } + const buf = this.buf + this.buf = Buffer.alloc(this.size) + val.copy(this.buf, 0) + buf.copy(this.buf, val.length) + this.length += val.length + }else{ + const length = this.length++ + if(length === this.size){ + this.resize() + } + const buf = this.clone() + this.buf[0] = val + buf.copy(this.buf,1, 0, length) + } + } + append(val){ + const length = this.length++ + if(length === this.size){ + this.resize() + } + this.buf[length] = val + } + clone(){ + return Buffer.from(this.buf.slice(0, this.length)) + } + resize(){ + const length = this.length + this.size = this.size * 2 + const buf = Buffer.alloc(this.size) + this.buf.copy(buf,0, 0, length) + this.buf = buf + } + toString(encoding){ + if(encoding){ + return this.buf.slice(0, this.length).toString(encoding) + }else{ + return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)) + } + } + toJSON(){ + return this.toString('utf8') + } + reset(){ + this.length = 0 + } +} + +module.exports = ResizeableBuffer + + +/***/ }), + +/***/ 2830: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + +/* +CSV Parse + +Please look at the [project documentation](https://csv.js.org/parse/) for +additional information. +*/ + +const { Transform } = __webpack_require__(2413) +const ResizeableBuffer = __webpack_require__(6942) + +const tab = 9 +const nl = 10 +const np = 12 +const cr = 13 +const space = 32 +const boms = { + // Note, the following are equals: + // Buffer.from("\ufeff") + // Buffer.from([239, 187, 191]) + // Buffer.from('EFBBBF', 'hex') + 'utf8': Buffer.from([239, 187, 191]), + // Note, the following are equals: + // Buffer.from "\ufeff", 'utf16le + // Buffer.from([255, 254]) + 'utf16le': Buffer.from([255, 254]) +} + +class Parser extends Transform { + constructor(opts = {}){ + super({...{readableObjectMode: true}, ...opts, encoding: null}) + this.__originalOptions = opts + this.__normalizeOptions(opts) + } + __normalizeOptions(opts){ + const options = {} + // Merge with user options + for(let opt in opts){ + options[underscore(opt)] = opts[opt] + } + // Normalize option `encoding` + // Note: defined first because other options depends on it + // to convert chars/strings into buffers. + if(options.encoding === undefined || options.encoding === true){ + options.encoding = 'utf8' + }else if(options.encoding === null || options.encoding === false){ + options.encoding = null + }else if(typeof options.encoding !== 'string' && options.encoding !== null){ + throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ + 'Invalid option encoding:', + 'encoding must be a string or null to return a buffer,', + `got ${JSON.stringify(options.encoding)}` + ], options) + } + // Normalize option `bom` + if(options.bom === undefined || options.bom === null || options.bom === false){ + options.bom = false + }else if(options.bom !== true){ + throw new CsvError('CSV_INVALID_OPTION_BOM', [ + 'Invalid option bom:', 'bom must be true,', + `got ${JSON.stringify(options.bom)}` + ], options) + } + // Normalize option `cast` + let fnCastField = null + if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ + options.cast = undefined + }else if(typeof options.cast === 'function'){ + fnCastField = options.cast + options.cast = true + }else if(options.cast !== true){ + throw new CsvError('CSV_INVALID_OPTION_CAST', [ + 'Invalid option cast:', 'cast must be true or a function,', + `got ${JSON.stringify(options.cast)}` + ], options) + } + // Normalize option `cast_date` + if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ + options.cast_date = false + }else if(options.cast_date === true){ + options.cast_date = function(value){ + const date = Date.parse(value) + return !isNaN(date) ? new Date(date) : value + } + }else if(typeof options.cast_date !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ + 'Invalid option cast_date:', 'cast_date must be true or a function,', + `got ${JSON.stringify(options.cast_date)}` + ], options) + } + // Normalize option `columns` + let fnFirstLineToHeaders = null + if(options.columns === true){ + // Fields in the first line are converted as-is to columns + fnFirstLineToHeaders = undefined + }else if(typeof options.columns === 'function'){ + fnFirstLineToHeaders = options.columns + options.columns = true + }else if(Array.isArray(options.columns)){ + options.columns = normalizeColumnsArray(options.columns) + }else if(options.columns === undefined || options.columns === null || options.columns === false){ + options.columns = false + }else{ + throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ + 'Invalid option columns:', + 'expect an object, a function or true,', + `got ${JSON.stringify(options.columns)}` + ], options) + } + // Normalize option `columns_duplicates_to_array` + if(options.columns_duplicates_to_array === undefined || options.columns_duplicates_to_array === null || options.columns_duplicates_to_array === false){ + options.columns_duplicates_to_array = false + }else if(options.columns_duplicates_to_array !== true){ + throw new CsvError('CSV_INVALID_OPTION_COLUMNS_DUPLICATES_TO_ARRAY', [ + 'Invalid option columns_duplicates_to_array:', + 'expect an boolean,', + `got ${JSON.stringify(options.columns_duplicates_to_array)}` + ], options) + } + // Normalize option `comment` + if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ + options.comment = null + }else{ + if(typeof options.comment === 'string'){ + options.comment = Buffer.from(options.comment, options.encoding) + } + if(!Buffer.isBuffer(options.comment)){ + throw new CsvError('CSV_INVALID_OPTION_COMMENT', [ + 'Invalid option comment:', + 'comment must be a buffer or a string,', + `got ${JSON.stringify(options.comment)}` + ], options) + } + } + // Normalize option `delimiter` + const delimiter_json = JSON.stringify(options.delimiter) + if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter] + if(options.delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options) + } + options.delimiter = options.delimiter.map(function(delimiter){ + if(delimiter === undefined || delimiter === null || delimiter === false){ + return Buffer.from(',', options.encoding) + } + if(typeof delimiter === 'string'){ + delimiter = Buffer.from(delimiter, options.encoding) + } + if( !Buffer.isBuffer(delimiter) || delimiter.length === 0){ + throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ + 'Invalid option delimiter:', + 'delimiter must be a non empty string or buffer or array of string|buffer,', + `got ${delimiter_json}` + ], options) + } + return delimiter + }) + // Normalize option `escape` + if(options.escape === undefined || options.escape === true){ + options.escape = Buffer.from('"', options.encoding) + }else if(typeof options.escape === 'string'){ + options.escape = Buffer.from(options.escape, options.encoding) + }else if (options.escape === null || options.escape === false){ + options.escape = null + } + if(options.escape !== null){ + if(!Buffer.isBuffer(options.escape)){ + throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`) + } + } + // Normalize option `from` + if(options.from === undefined || options.from === null){ + options.from = 1 + }else{ + if(typeof options.from === 'string' && /\d+/.test(options.from)){ + options.from = parseInt(options.from) + } + if(Number.isInteger(options.from)){ + if(options.from < 0){ + throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`) + } + }else{ + throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`) + } + } + // Normalize option `from_line` + if(options.from_line === undefined || options.from_line === null){ + options.from_line = 1 + }else{ + if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ + options.from_line = parseInt(options.from_line) + } + if(Number.isInteger(options.from_line)){ + if(options.from_line <= 0){ + throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`) + } + }else{ + throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`) + } + } + // Normalize option `info` + if(options.info === undefined || options.info === null || options.info === false){ + options.info = false + }else if(options.info !== true){ + throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`) + } + // Normalize option `max_record_size` + if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ + options.max_record_size = 0 + }else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0){ + // Great, nothing to do + }else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ + options.max_record_size = parseInt(options.max_record_size) + }else{ + throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`) + } + // Normalize option `objname` + if(options.objname === undefined || options.objname === null || options.objname === false){ + options.objname = undefined + }else if(Buffer.isBuffer(options.objname)){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty buffer`) + } + if(options.encoding === null){ + // Don't call `toString`, leave objname as a buffer + }else{ + options.objname = options.objname.toString(options.encoding) + } + }else if(typeof options.objname === 'string'){ + if(options.objname.length === 0){ + throw new Error(`Invalid Option: objname must be a non empty string`) + } + // Great, nothing to do + }else{ + throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`) + } + // Normalize option `on_record` + if(options.on_record === undefined || options.on_record === null){ + options.on_record = undefined + }else if(typeof options.on_record !== 'function'){ + throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ + 'Invalid option `on_record`:', + 'expect a function,', + `got ${JSON.stringify(options.on_record)}` + ], options) + } + // Normalize option `quote` + if(options.quote === null || options.quote === false || options.quote === ''){ + options.quote = null + }else{ + if(options.quote === undefined || options.quote === true){ + options.quote = Buffer.from('"', options.encoding) + }else if(typeof options.quote === 'string'){ + options.quote = Buffer.from(options.quote, options.encoding) + } + if(!Buffer.isBuffer(options.quote)){ + throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`) + } + } + // Normalize option `raw` + if(options.raw === undefined || options.raw === null || options.raw === false){ + options.raw = false + }else if(options.raw !== true){ + throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`) + } + // Normalize option `record_delimiter` + if(!options.record_delimiter){ + options.record_delimiter = [] + }else if(!Array.isArray(options.record_delimiter)){ + options.record_delimiter = [options.record_delimiter] + } + options.record_delimiter = options.record_delimiter.map( function(rd){ + if(typeof rd === 'string'){ + rd = Buffer.from(rd, options.encoding) + } + return rd + }) + // Normalize option `relax` + if(typeof options.relax === 'boolean'){ + // Great, nothing to do + }else if(options.relax === undefined || options.relax === null){ + options.relax = false + }else{ + throw new Error(`Invalid Option: relax must be a boolean, got ${JSON.stringify(options.relax)}`) + } + // Normalize option `relax_column_count` + if(typeof options.relax_column_count === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count === undefined || options.relax_column_count === null){ + options.relax_column_count = false + }else{ + throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`) + } + if(typeof options.relax_column_count_less === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ + options.relax_column_count_less = false + }else{ + throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`) + } + if(typeof options.relax_column_count_more === 'boolean'){ + // Great, nothing to do + }else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ + options.relax_column_count_more = false + }else{ + throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`) + } + // Normalize option `skip_empty_lines` + if(typeof options.skip_empty_lines === 'boolean'){ + // Great, nothing to do + }else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ + options.skip_empty_lines = false + }else{ + throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`) + } + // Normalize option `skip_lines_with_empty_values` + if(typeof options.skip_lines_with_empty_values === 'boolean'){ + // Great, nothing to do + }else if(options.skip_lines_with_empty_values === undefined || options.skip_lines_with_empty_values === null){ + options.skip_lines_with_empty_values = false + }else{ + throw new Error(`Invalid Option: skip_lines_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_lines_with_empty_values)}`) + } + // Normalize option `skip_lines_with_error` + if(typeof options.skip_lines_with_error === 'boolean'){ + // Great, nothing to do + }else if(options.skip_lines_with_error === undefined || options.skip_lines_with_error === null){ + options.skip_lines_with_error = false + }else{ + throw new Error(`Invalid Option: skip_lines_with_error must be a boolean, got ${JSON.stringify(options.skip_lines_with_error)}`) + } + // Normalize option `rtrim` + if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ + options.rtrim = false + }else if(options.rtrim !== true){ + throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`) + } + // Normalize option `ltrim` + if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ + options.ltrim = false + }else if(options.ltrim !== true){ + throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`) + } + // Normalize option `trim` + if(options.trim === undefined || options.trim === null || options.trim === false){ + options.trim = false + }else if(options.trim !== true){ + throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`) + } + // Normalize options `trim`, `ltrim` and `rtrim` + if(options.trim === true && opts.ltrim !== false){ + options.ltrim = true + }else if(options.ltrim !== true){ + options.ltrim = false + } + if(options.trim === true && opts.rtrim !== false){ + options.rtrim = true + }else if(options.rtrim !== true){ + options.rtrim = false + } + // Normalize option `to` + if(options.to === undefined || options.to === null){ + options.to = -1 + }else{ + if(typeof options.to === 'string' && /\d+/.test(options.to)){ + options.to = parseInt(options.to) + } + if(Number.isInteger(options.to)){ + if(options.to <= 0){ + throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`) + } + }else{ + throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`) + } + } + // Normalize option `to_line` + if(options.to_line === undefined || options.to_line === null){ + options.to_line = -1 + }else{ + if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ + options.to_line = parseInt(options.to_line) + } + if(Number.isInteger(options.to_line)){ + if(options.to_line <= 0){ + throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`) + } + }else{ + throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`) + } + } + this.info = { + comment_lines: 0, + empty_lines: 0, + invalid_field_length: 0, + lines: 1, + records: 0 + } + this.options = options + this.state = { + bomSkipped: false, + castField: fnCastField, + commenting: false, + // Current error encountered by a record + error: undefined, + enabled: options.from_line === 1, + escaping: false, + // escapeIsQuote: options.escape === options.quote, + escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, + expectedRecordLength: options.columns === null ? 0 : options.columns.length, + field: new ResizeableBuffer(20), + firstLineToHeaders: fnFirstLineToHeaders, + info: Object.assign({}, this.info), + needMoreDataSize: Math.max( + // Skip if the remaining buffer smaller than comment + options.comment !== null ? options.comment.length : 0, + // Skip if the remaining buffer can be delimiter + ...options.delimiter.map( (delimiter) => delimiter.length), + // Skip if the remaining buffer can be escape sequence + options.quote !== null ? options.quote.length : 0, + ), + previousBuf: undefined, + quoting: false, + stop: false, + rawBuffer: new ResizeableBuffer(100), + record: [], + recordHasError: false, + record_length: 0, + recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map( (v) => v.length)), + trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], + wasQuoting: false, + wasRowDelimiter: false + } + } + // Implementation of `Transform._transform` + _transform(buf, encoding, callback){ + if(this.state.stop === true){ + return + } + const err = this.__parse(buf, false) + if(err !== undefined){ + this.state.stop = true + } + callback(err) + } + // Implementation of `Transform._flush` + _flush(callback){ + if(this.state.stop === true){ + return + } + const err = this.__parse(undefined, true) + callback(err) + } + // Central parser implementation + __parse(nextBuf, end){ + const {bom, comment, escape, from_line, info, ltrim, max_record_size, quote, raw, relax, rtrim, skip_empty_lines, to, to_line} = this.options + let {record_delimiter} = this.options + const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state + let buf + if(previousBuf === undefined){ + if(nextBuf === undefined){ + // Handle empty string + this.push(null) + return + }else{ + buf = nextBuf + } + }else if(previousBuf !== undefined && nextBuf === undefined){ + buf = previousBuf + }else{ + buf = Buffer.concat([previousBuf, nextBuf]) + } + // Handle UTF BOM + if(bomSkipped === false){ + if(bom === false){ + this.state.bomSkipped = true + }else if(buf.length < 3){ + // No enough data + if(end === false){ + // Wait for more data + this.state.previousBuf = buf + return + } + }else{ + for(let encoding in boms){ + if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ + // Skip BOM + buf = buf.slice(boms[encoding].length) + // Renormalize original options with the new encoding + this.__normalizeOptions({...this.__originalOptions, encoding: encoding}) + break + } + } + this.state.bomSkipped = true + } + } + const bufLen = buf.length + let pos + for(pos = 0; pos < bufLen; pos++){ + // Ensure we get enough space to look ahead + // There should be a way to move this out of the loop + if(this.__needMoreData(pos, bufLen, end)){ + break + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++ + if(info === true && this.state.record.length === 0 && this.state.field.length === 0 && this.state.wasQuoting === false){ + this.state.info = Object.assign({}, this.info) + } + this.state.wasRowDelimiter = false + } + if(to_line !== -1 && this.info.lines > to_line){ + this.state.stop = true + this.push(null) + return + } + // Auto discovery of record_delimiter, unix, mac and windows supported + if(this.state.quoting === false && record_delimiter.length === 0){ + const record_delimiterCount = this.__autoDiscoverRowDelimiter(buf, pos) + if(record_delimiterCount){ + record_delimiter = this.options.record_delimiter + } + } + const chr = buf[pos] + if(raw === true){ + rawBuffer.append(chr) + } + if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false ){ + this.state.wasRowDelimiter = true + } + // Previous char was a valid escape char + // treat the current char as a regular char + if(this.state.escaping === true){ + this.state.escaping = false + }else{ + // Escape is only active inside quoted fields + // We are quoting, the char is an escape chr and there is a chr to escape + // if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){ + if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){ + if(escapeIsQuote){ + if(this.__isQuote(buf, pos+escape.length)){ + this.state.escaping = true + pos += escape.length - 1 + continue + } + }else{ + this.state.escaping = true + pos += escape.length - 1 + continue + } + } + // Not currently escaping and chr is a quote + // TODO: need to compare bytes instead of single char + if(this.state.commenting === false && this.__isQuote(buf, pos)){ + if(this.state.quoting === true){ + const nextChr = buf[pos+quote.length] + const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr) + const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr) + const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr) + const isNextChrRowDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRowDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length) + // Escape a quote + // Treat next char as a regular character + if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ + pos += escape.length - 1 + }else if(!nextChr || isNextChrDelimiter || isNextChrRowDelimiter || isNextChrComment || isNextChrTrimable){ + this.state.quoting = false + this.state.wasQuoting = true + pos += quote.length - 1 + continue + }else if(relax === false){ + const err = this.__error( + new CsvError('CSV_INVALID_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + `got "${String.fromCharCode(nextChr)}"`, + `at line ${this.info.lines}`, + 'instead of delimiter, row delimiter, trimable character', + '(if activated) or comment', + ], this.options, this.__context()) + ) + if(err !== undefined) return err + }else{ + this.state.quoting = false + this.state.wasQuoting = true + this.state.field.prepend(quote) + pos += quote.length - 1 + } + }else{ + if(this.state.field.length !== 0){ + // In relax mode, treat opening quote preceded by chrs as regular + if( relax === false ){ + const err = this.__error( + new CsvError('INVALID_OPENING_QUOTE', [ + 'Invalid Opening Quote:', + `a quote is found inside a field at line ${this.info.lines}`, + ], this.options, this.__context(), { + field: this.state.field, + }) + ) + if(err !== undefined) return err + } + }else{ + this.state.quoting = true + pos += quote.length - 1 + continue + } + } + } + if(this.state.quoting === false){ + let recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos) + if(recordDelimiterLength !== 0){ + // Do not emit comments which take a full line + const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0) + if(skipCommentLine){ + this.info.comment_lines++ + // Skip full comment line + }else{ + // Skip if line is empty and skip_empty_lines activated + if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ + this.info.empty_lines++ + pos += recordDelimiterLength - 1 + continue + } + // Activate records emition if above from_line + if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0 ) >= from_line){ + this.state.enabled = true + this.__resetField() + this.__resetRow() + pos += recordDelimiterLength - 1 + continue + }else{ + const errField = this.__onField() + if(errField !== undefined) return errField + const errRecord = this.__onRow() + if(errRecord !== undefined) return errRecord + } + if(to !== -1 && this.info.records >= to){ + this.state.stop = true + this.push(null) + return + } + } + this.state.commenting = false + pos += recordDelimiterLength - 1 + continue + } + if(this.state.commenting){ + continue + } + const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr) + if(commentCount !== 0){ + this.state.commenting = true + continue + } + let delimiterLength = this.__isDelimiter(buf, pos, chr) + if(delimiterLength !== 0){ + const errField = this.__onField() + if(errField !== undefined) return errField + pos += delimiterLength - 1 + continue + } + } + } + if(this.state.commenting === false){ + if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){ + const err = this.__error( + new CsvError('CSV_MAX_RECORD_SIZE', [ + 'Max Record Size:', + 'record exceed the maximum number of tolerated bytes', + `of ${max_record_size}`, + `at line ${this.info.lines}`, + ], this.options, this.__context()) + ) + if(err !== undefined) return err + } + } + + const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr) + // rtrim in non quoting is handle in __onField + const rappend = rtrim === false || this.state.wasQuoting === false + if( lappend === true && rappend === true ){ + this.state.field.append(chr) + }else if(rtrim === true && !this.__isCharTrimable(chr)){ + const err = this.__error( + new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [ + 'Invalid Closing Quote:', + 'found non trimable byte after quote', + `at line ${this.info.lines}`, + ], this.options, this.__context()) + ) + if(err !== undefined) return err + } + } + if(end === true){ + // Ensure we are not ending in a quoting state + if(this.state.quoting === true){ + const err = this.__error( + new CsvError('CSV_QUOTE_NOT_CLOSED', [ + 'Quote Not Closed:', + `the parsing is finished with an opening quote at line ${this.info.lines}`, + ], this.options, this.__context()) + ) + if(err !== undefined) return err + }else{ + // Skip last line if it has no characters + if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){ + const errField = this.__onField() + if(errField !== undefined) return errField + const errRecord = this.__onRow() + if(errRecord !== undefined) return errRecord + }else if(this.state.wasRowDelimiter === true){ + this.info.empty_lines++ + }else if(this.state.commenting === true){ + this.info.comment_lines++ + } + } + }else{ + this.state.previousBuf = buf.slice(pos) + } + if(this.state.wasRowDelimiter === true){ + this.info.lines++ + this.state.wasRowDelimiter = false + } + } + // Helper to test if a character is a space or a line delimiter + __isCharTrimable(chr){ + return chr === space || chr === tab || chr === cr || chr === nl || chr === np + } + __onRow(){ + const {columns, columns_duplicates_to_array, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_lines_with_empty_values} = this.options + const {enabled, record} = this.state + if(enabled === false){ + return this.__resetRow() + } + // Convert the first line into column names + const recordLength = record.length + if(columns === true){ + if(isRecordEmpty(record)){ + this.__resetRow() + return + } + return this.__firstLineToColumns(record) + } + if(columns === false && this.info.records === 0){ + this.state.expectedRecordLength = recordLength + } + if(recordLength !== this.state.expectedRecordLength){ + const err = columns === false ? + // Todo: rename CSV_INCONSISTENT_RECORD_LENGTH to + // CSV_RECORD_INCONSISTENT_FIELDS_LENGTH + new CsvError('CSV_INCONSISTENT_RECORD_LENGTH', [ + 'Invalid Record Length:', + `expect ${this.state.expectedRecordLength},`, + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__context(), { + record: record, + }) + : + // Todo: rename CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH to + // CSV_RECORD_INCONSISTENT_COLUMNS + new CsvError('CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH', [ + 'Invalid Record Length:', + `columns length is ${columns.length},`, // rename columns + `got ${recordLength} on line ${this.info.lines}`, + ], this.options, this.__context(), { + record: record, + }) + if(relax_column_count === true || + (relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || + (relax_column_count_more === true && recordLength > this.state.expectedRecordLength) ){ + this.info.invalid_field_length++ + this.state.error = err + // Error is undefined with skip_lines_with_error + }else{ + const finalErr = this.__error(err) + if(finalErr) return finalErr + } + } + if(skip_lines_with_empty_values === true){ + if(isRecordEmpty(record)){ + this.__resetRow() + return + } + } + if(this.state.recordHasError === true){ + this.__resetRow() + this.state.recordHasError = false + return + } + this.info.records++ + if(from === 1 || this.info.records >= from){ + if(columns !== false){ + const obj = {} + // Transform record array to an object + for(let i = 0, l = record.length; i < l; i++){ + if(columns[i] === undefined || columns[i].disabled) continue + // Turn duplicate columns into an array + if (columns_duplicates_to_array === true && obj[columns[i].name]) { + if (Array.isArray(obj[columns[i].name])) { + obj[columns[i].name] = obj[columns[i].name].concat(record[i]) + } else { + obj[columns[i].name] = [obj[columns[i].name], record[i]] + } + } else { + obj[columns[i].name] = record[i] + } + } + const {objname} = this.options + if(objname === undefined){ + if(raw === true || info === true){ + const err = this.__push(Object.assign( + {record: obj}, + (raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), + (info === true ? {info: this.state.info}: {}) + )) + if(err){ + return err + } + }else{ + const err = this.__push(obj) + if(err){ + return err + } + } + }else{ + if(raw === true || info === true){ + const err = this.__push(Object.assign( + {record: [obj[objname], obj]}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.state.info}: {} + )) + if(err){ + return err + } + }else{ + const err = this.__push([obj[objname], obj]) + if(err){ + return err + } + } + } + }else{ + if(raw === true || info === true){ + const err = this.__push(Object.assign( + {record: record}, + raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, + info === true ? {info: this.state.info}: {} + )) + if(err){ + return err + } + }else{ + const err = this.__push(record) + if(err){ + return err + } + } + } + } + this.__resetRow() + } + __firstLineToColumns(record){ + const {firstLineToHeaders} = this.state + try{ + const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record) + if(!Array.isArray(headers)){ + return this.__error( + new CsvError('CSV_INVALID_COLUMN_MAPPING', [ + 'Invalid Column Mapping:', + 'expect an array from column function,', + `got ${JSON.stringify(headers)}` + ], this.options, this.__context(), { + headers: headers, + }) + ) + } + const normalizedHeaders = normalizeColumnsArray(headers) + this.state.expectedRecordLength = normalizedHeaders.length + this.options.columns = normalizedHeaders + this.__resetRow() + return + }catch(err){ + return err + } + } + __resetRow(){ + if(this.options.raw === true){ + this.state.rawBuffer.reset() + } + this.state.error = undefined + this.state.record = [] + this.state.record_length = 0 + } + __onField(){ + const {cast, encoding, rtrim, max_record_size} = this.options + const {enabled, wasQuoting} = this.state + // Short circuit for the from_line options + if(enabled === false){ /* this.options.columns !== true && */ + return this.__resetField() + } + let field = this.state.field.toString(encoding) + if(rtrim === true && wasQuoting === false){ + field = field.trimRight() + } + if(cast === true){ + const [err, f] = this.__cast(field) + if(err !== undefined) return err + field = f + } + this.state.record.push(field) + // Increment record length if record size must not exceed a limit + if(max_record_size !== 0 && typeof field === 'string'){ + this.state.record_length += field.length + } + this.__resetField() + } + __resetField(){ + this.state.field.reset() + this.state.wasQuoting = false + } + __push(record){ + const {on_record} = this.options + if(on_record !== undefined){ + const context = this.__context() + try{ + record = on_record.call(null, record, context) + }catch(err){ + return err + } + if(record === undefined || record === null){ return } + } + this.push(record) + } + // Return a tuple with the error and the casted value + __cast(field){ + const {columns, relax_column_count} = this.options + const isColumns = Array.isArray(columns) + // Dont loose time calling cast + // because the final record is an object + // and this field can't be associated to a key present in columns + if( isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length ){ + return [undefined, undefined] + } + const context = this.__context() + if(this.state.castField !== null){ + try{ + return [undefined, this.state.castField.call(null, field, context)] + }catch(err){ + return [err] + } + } + if(this.__isFloat(field)){ + return [undefined, parseFloat(field)] + }else if(this.options.cast_date !== false){ + return [undefined, this.options.cast_date.call(null, field, context)] + } + return [undefined, field] + } + // Keep it in case we implement the `cast_int` option + // __isInt(value){ + // // return Number.isInteger(parseInt(value)) + // // return !isNaN( parseInt( obj ) ); + // return /^(\-|\+)?[1-9][0-9]*$/.test(value) + // } + __isFloat(value){ + return (value - parseFloat( value ) + 1) >= 0 // Borrowed from jquery + } + __compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ + if(sourceBuf[0] !== firstByte) return 0 + const sourceLength = sourceBuf.length + for(let i = 1; i < sourceLength; i++){ + if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0 + } + return sourceLength + } + __needMoreData(i, bufLen, end){ + if(end) return false + const {quote} = this.options + const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state + const numOfCharLeft = bufLen - i - 1 + const requiredLength = Math.max( + needMoreDataSize, + // Skip if the remaining buffer smaller than record delimiter + recordDelimiterMaxLength, + // Skip if the remaining buffer can be row delimiter following the closing quote + // 1 is for quote.length + quoting ? (quote.length + recordDelimiterMaxLength) : 0, + ) + return numOfCharLeft < requiredLength + } + __isDelimiter(buf, pos, chr){ + const {delimiter} = this.options + loop1: for(let i = 0; i < delimiter.length; i++){ + const del = delimiter[i] + if(del[0] === chr){ + for(let j = 1; j < del.length; j++){ + if(del[j] !== buf[pos+j]) continue loop1 + } + return del.length + } + } + return 0 + } + __isRecordDelimiter(chr, buf, pos){ + const {record_delimiter} = this.options + const recordDelimiterLength = record_delimiter.length + loop1: for(let i = 0; i < recordDelimiterLength; i++){ + const rd = record_delimiter[i] + const rdLength = rd.length + if(rd[0] !== chr){ + continue + } + for(let j = 1; j < rdLength; j++){ + if(rd[j] !== buf[pos+j]){ + continue loop1 + } + } + return rd.length + } + return 0 + } + __isEscape(buf, pos, chr){ + const {escape} = this.options + if(escape === null) return false + const l = escape.length + if(escape[0] === chr){ + for(let i = 0; i < l; i++){ + if(escape[i] !== buf[pos+i]){ + return false + } + } + return true + } + return false + } + __isQuote(buf, pos){ + const {quote} = this.options + if(quote === null) return false + const l = quote.length + for(let i = 0; i < l; i++){ + if(quote[i] !== buf[pos+i]){ + return false + } + } + return true + } + __autoDiscoverRowDelimiter(buf, pos){ + const {encoding} = this.options + const chr = buf[pos] + if(chr === cr){ + if(buf[pos+1] === nl){ + this.options.record_delimiter.push(Buffer.from('\r\n', encoding)) + this.state.recordDelimiterMaxLength = 2 + return 2 + }else{ + this.options.record_delimiter.push(Buffer.from('\r', encoding)) + this.state.recordDelimiterMaxLength = 1 + return 1 + } + }else if(chr === nl){ + this.options.record_delimiter.push(Buffer.from('\n', encoding)) + this.state.recordDelimiterMaxLength = 1 + return 1 + } + return 0 + } + __error(msg){ + const {skip_lines_with_error} = this.options + const err = typeof msg === 'string' ? new Error(msg) : msg + if(skip_lines_with_error){ + this.state.recordHasError = true + this.emit('skip', err) + return undefined + }else{ + return err + } + } + __context(){ + const {columns} = this.options + const isColumns = Array.isArray(columns) + return { + column: isColumns === true ? + ( columns.length > this.state.record.length ? + columns[this.state.record.length].name : + null + ) : + this.state.record.length, + empty_lines: this.info.empty_lines, + error: this.state.error, + header: columns === true, + index: this.state.record.length, + invalid_field_length: this.info.invalid_field_length, + quoting: this.state.wasQuoting, + lines: this.info.lines, + records: this.info.records + } + } +} + +const parse = function(){ + let data, options, callback + for(let i in arguments){ + const argument = arguments[i] + const type = typeof argument + if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){ + data = argument + }else if(options === undefined && isObject(argument)){ + options = argument + }else if(callback === undefined && type === 'function'){ + callback = argument + }else{ + throw new CsvError('CSV_INVALID_ARGUMENT', [ + 'Invalid argument:', + `got ${JSON.stringify(argument)} at index ${i}` + ], this.options) + } + } + const parser = new Parser(options) + if(callback){ + const records = options === undefined || options.objname === undefined ? [] : {} + parser.on('readable', function(){ + let record + while((record = this.read()) !== null){ + if(options === undefined || options.objname === undefined){ + records.push(record) + }else{ + records[record[0]] = record[1] + } + } + }) + parser.on('error', function(err){ + callback(err, undefined, parser.info) + }) + parser.on('end', function(){ + callback(undefined, records, parser.info) + }) + } + if(data !== undefined){ + // Give a chance for events to be registered later + if(typeof setImmediate === 'function'){ + setImmediate(function(){ + parser.write(data) + parser.end() + }) + }else{ + parser.write(data) + parser.end() + } + } + return parser +} + +class CsvError extends Error { + constructor(code, message, options, ...contexts) { + if(Array.isArray(message)) message = message.join(' ') + super(message) + if(Error.captureStackTrace !== undefined){ + Error.captureStackTrace(this, CsvError) + } + this.code = code + for(const context of contexts){ + for(const key in context){ + const value = context[key] + this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)) + } + } + } +} + +parse.Parser = Parser + +parse.CsvError = CsvError + +module.exports = parse + +const underscore = function(str){ + return str.replace(/([A-Z])/g, function(_, match){ + return '_' + match.toLowerCase() + }) +} + +const isObject = function(obj){ + return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)) +} + +const isRecordEmpty = function(record){ + return record.every( (field) => field == null || field.toString && field.toString().trim() === '' ) +} + +const normalizeColumnsArray = function(columns){ + const normalizedColumns = []; + for(let i = 0, l = columns.length; i < l; i++){ + const column = columns[i] + if(column === undefined || column === null || column === false){ + normalizedColumns[i] = { disabled: true } + }else if(typeof column === 'string'){ + normalizedColumns[i] = { name: column } + }else if(isObject(column)){ + if(typeof column.name !== 'string'){ + throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ + 'Option columns missing name:', + `property "name" is required at position ${i}`, + 'when column is an object literal' + ]) + } + normalizedColumns[i] = column + }else{ + throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ + 'Invalid column definition:', + 'expect a string or a literal object,', + `got ${JSON.stringify(column)} at position ${i}` + ]) + } + } + return normalizedColumns; +} + + +/***/ }), + +/***/ 8750: +/***/ ((module, __unused_webpack_exports, __webpack_require__) => { + + +const parse = __webpack_require__(2830) + +module.exports = function(data, options={}){ + if(typeof data === 'string'){ + data = Buffer.from(data) + } + const records = options && options.objname ? {} : [] + const parser = new parse.Parser(options) + parser.push = function(record){ + if(record === null){ + return + } + if(options.objname === undefined) + records.push(record) + else{ + records[record[0]] = record[1] + } + } + const err1 = parser.__parse(data, false) + if(err1 !== undefined) throw err1 + const err2 = parser.__parse(undefined, true) + if(err2 !== undefined) throw err2 + return records +} + + /***/ }), /***/ 8932: diff --git a/package.json b/package.json index 60023b5..cd2c302 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "dependencies": { "@actions/core": "^1.2.6", "@actions/github": "^4.0.0", + "csv-parse": "^4.14.2", "handlebars": "^4.7.6", "moment": "^2.29.1", "semver": "^7.3.4" diff --git a/src/context.ts b/src/context.ts index 8252565..68eb5b9 100644 --- a/src/context.ts +++ b/src/context.ts @@ -1,3 +1,4 @@ +import csvparse from 'csv-parse/lib/sync'; import * as core from '@actions/core'; export interface Inputs { @@ -12,6 +13,7 @@ export interface Inputs { tagSchedule: string; tagCustom: string[]; tagCustomOnly: boolean; + labelCustom: string[]; sepTags: string; sepLabels: string; githubToken: string; @@ -30,21 +32,37 @@ export function getInputs(): Inputs { tagSchedule: core.getInput('tag-schedule') || 'nightly', tagCustom: getInputList('tag-custom'), tagCustomOnly: /true/i.test(core.getInput('tag-custom-only') || 'false'), + labelCustom: getInputList('label-custom'), sepTags: core.getInput('sep-tags') || `\n`, sepLabels: core.getInput('sep-labels') || `\n`, githubToken: core.getInput('github-token') }; } -export function getInputList(name: string): string[] { +export function getInputList(name: string, ignoreComma?: boolean): string[] { + let res: Array = []; + const items = core.getInput(name); if (items == '') { - return []; + return res; } - return items - .split(/\r?\n/) - .filter(x => x) - .reduce((acc, line) => acc.concat(line.split(',').filter(x => x)).map(pat => pat.trim()), []); + + for (let output of csvparse(items, { + columns: false, + relaxColumnCount: true, + skipLinesWithEmptyValues: true + }) as Array) { + if (output.length == 1) { + res.push(output[0]); + continue; + } else if (!ignoreComma) { + res.push(...output); + continue; + } + res.push(output.join(',')); + } + + return res.filter(item => item).map(pat => pat.trim()); } export const asyncForEach = async (array, callback) => { diff --git a/src/meta.ts b/src/meta.ts index 2377d16..b87203e 100644 --- a/src/meta.ts +++ b/src/meta.ts @@ -1,5 +1,5 @@ import * as handlebars from 'handlebars'; -import * as moment from 'moment'; +import moment from 'moment'; import * as semver from 'semver'; import {Inputs} from './context'; import * as core from '@actions/core'; @@ -130,7 +130,7 @@ export class Meta { } public labels(): Array { - return [ + let labels: Array = [ `org.opencontainers.image.title=${this.repo.name || ''}`, `org.opencontainers.image.description=${this.repo.description || ''}`, `org.opencontainers.image.url=${this.repo.html_url || ''}`, @@ -140,5 +140,7 @@ export class Meta { `org.opencontainers.image.revision=${this.context.sha || ''}`, `org.opencontainers.image.licenses=${this.repo.license?.spdx_id || ''}` ]; + labels.push(...this.inputs.labelCustom); + return labels; } } diff --git a/tsconfig.json b/tsconfig.json index 095620a..7f2af30 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,9 +11,11 @@ "rootDir": "./src", "strict": true, "noImplicitAny": false, - "esModuleInterop": false, - "resolveJsonModule": true, + "esModuleInterop": true, "sourceMap": true }, - "exclude": ["node_modules", "**/*.test.ts"] + "exclude": [ + "node_modules", + "**/*.test.ts" + ] } diff --git a/yarn.lock b/yarn.lock index a6d97b7..c67d972 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1176,6 +1176,11 @@ cssstyle@^2.2.0: dependencies: cssom "~0.3.6" +csv-parse@^4.14.2: + version "4.14.2" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.14.2.tgz#c1329cff95a99b8773a92c4e62f8bff114b34726" + integrity sha512-YE2xlTKtM035/94llhgsp9qFQxGi47EkQJ1pZ+mLT/98GpIsbjkMGAb7Rmu9hNxVfYFOLf10hP+rPVqnoccLgw== + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"