Merge commit 'be3e8236086165e5e45a5a10783823874b3f3ebd' as 'lib/vscode'

lib/vscode/build/npm/postinstall.js

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const cp = require('child_process');
+const path = require('path');
+const fs = require('fs');
+const yarn = process.platform === 'win32' ? 'yarn.cmd' : 'yarn';
+
+/**
+ * @param {string} location
+ * @param {*} [opts]
+ */
+function yarnInstall(location, opts) {
+	opts = opts || { env: process.env };
+	opts.cwd = location;
+	opts.stdio = 'inherit';
+
+	const raw = process.env['npm_config_argv'] || '{}';
+	const argv = JSON.parse(raw);
+	const original = argv.original || [];
+	const args = original.filter(arg => arg === '--ignore-optional' || arg === '--frozen-lockfile');
+
+	console.log(`Installing dependencies in ${location}...`);
+	console.log(`$ yarn ${args.join(' ')}`);
+	const result = cp.spawnSync(yarn, args, opts);
+
+	if (result.error || result.status !== 0) {
+		process.exit(1);
+	}
+}
+
+yarnInstall('extensions'); // node modules shared by all extensions
+
+if (!(process.platform === 'win32' && (process.arch === 'arm64' || process.env['npm_config_arch'] === 'arm64'))) {
+	yarnInstall('remote'); // node modules used by vscode server
+	yarnInstall('remote/web'); // node modules used by vscode web
+}
+
+const allExtensionFolders = fs.readdirSync('extensions');
+const extensions = allExtensionFolders.filter(e => {
+	try {
+		let packageJSON = JSON.parse(fs.readFileSync(path.join('extensions', e, 'package.json')).toString());
+		return packageJSON && (packageJSON.dependencies || packageJSON.devDependencies);
+	} catch (e) {
+		return false;
+	}
+});
+
+extensions.forEach(extension => yarnInstall(`extensions/${extension}`));
+
+function yarnInstallBuildDependencies() {
+	// make sure we install the deps of build/lib/watch for the system installed
+	// node, since that is the driver of gulp
+	const watchPath = path.join(path.dirname(__dirname), 'lib', 'watch');
+	const yarnrcPath = path.join(watchPath, '.yarnrc');
+
+	const disturl = 'https://nodejs.org/download/release';
+	const target = process.versions.node;
+	const runtime = 'node';
+
+	const yarnrc = `disturl "${disturl}"
+target "${target}"
+runtime "${runtime}"`;
+
+	fs.writeFileSync(yarnrcPath, yarnrc, 'utf8');
+	yarnInstall(watchPath);
+}
+
+yarnInstall(`build`); // node modules required for build
+yarnInstall('test/automation'); // node modules required for smoketest
+yarnInstall('test/smoke'); // node modules required for smoketest
+yarnInstall('test/integration/browser'); // node modules required for integration
+yarnInstallBuildDependencies(); // node modules for watching, specific to host node version, not electron
+
+cp.execSync('git config pull.rebase true');

lib/vscode/build/npm/preinstall.js

@@ -0,0 +1,34 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+let err = false;
+
+const majorNodeVersion = parseInt(/^(\d+)\./.exec(process.versions.node)[1]);
+
+if (majorNodeVersion < 10 || majorNodeVersion >= 13) {
+	console.error('\033[1;31m*** Please use node >=10 and <=12.\033[0;0m');
+	err = true;
+}
+
+const cp = require('child_process');
+const yarnVersion = cp.execSync('yarn -v', { encoding: 'utf8' }).trim();
+const parsedYarnVersion = /^(\d+)\.(\d+)\./.exec(yarnVersion);
+const majorYarnVersion = parseInt(parsedYarnVersion[1]);
+const minorYarnVersion = parseInt(parsedYarnVersion[2]);
+
+if (majorYarnVersion < 1 || minorYarnVersion < 10) {
+	console.error('\033[1;31m*** Please use yarn >=1.10.1.\033[0;0m');
+	err = true;
+}
+
+if (!/yarn[\w-.]*\.js$|yarnpkg$/.test(process.env['npm_execpath'])) {
+	console.error('\033[1;31m*** Please use yarn to install dependencies.\033[0;0m');
+	err = true;
+}
+
+if (err) {
+	console.error('');
+	process.exit(1);
+}

lib/vscode/build/npm/update-all-grammars.js

@@ -0,0 +1,46 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const cp = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {string} location
+ */
+function updateGrammar(location) {
+	const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+	const result = cp.spawnSync(npm, ['run', 'update-grammar'], {
+		cwd: location,
+		stdio: 'inherit'
+	});
+
+	if (result.error || result.status !== 0) {
+		process.exit(1);
+	}
+}
+
+const allExtensionFolders = fs.readdirSync('extensions');
+const extensions = allExtensionFolders.filter(e => {
+	try {
+		let packageJSON = JSON.parse(fs.readFileSync(path.join('extensions', e, 'package.json')).toString());
+		return packageJSON && packageJSON.scripts && packageJSON.scripts['update-grammar'];
+	} catch (e) {
+		return false;
+	}
+});
+
+console.log(`Updating ${extensions.length} grammars...`);
+
+extensions.forEach(extension => updateGrammar(`extensions/${extension}`));
+
+// run integration tests
+
+if (process.platform === 'win32') {
+	cp.spawn('.\\scripts\\test-integration.bat', [], { env: process.env, stdio: 'inherit' });
+} else {
+	cp.spawn('/bin/bash', ['./scripts/test-integration.sh'], { env: process.env, stdio: 'inherit' });
+}
+

lib/vscode/build/npm/update-distro.js

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const cp = require('child_process');
+const path = require('path');
+const fs = require('fs');
+
+const rootPath = path.dirname(path.dirname(path.dirname(__dirname)));
+const vscodePath = path.join(rootPath, 'vscode');
+const distroPath = path.join(rootPath, 'vscode-distro');
+const commit = cp.execSync('git rev-parse HEAD', { cwd: distroPath, encoding: 'utf8' }).trim();
+const packageJsonPath = path.join(vscodePath, 'package.json');
+const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+
+packageJson.distro = commit;
+fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2));

lib/vscode/build/npm/update-grammar.js

@@ -0,0 +1,176 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var plist = require('fast-plist');
+var cson = require('cson-parser');
+var https = require('https');
+var url = require('url');
+
+let commitDate = '0000-00-00';
+
+/**
+ * @param {string} urlString
+ */
+function getOptions(urlString) {
+	var _url = url.parse(urlString);
+	var headers = {
+		'User-Agent': 'VSCode'
+	};
+	var token = process.env['GITHUB_TOKEN'];
+	if (token) {
+		headers['Authorization'] = 'token ' + token;
+	}
+	return {
+		protocol: _url.protocol,
+		host: _url.host,
+		port: _url.port,
+		path: _url.path,
+		headers: headers
+	};
+}
+
+/**
+ * @param {string} url
+ * @param {number} redirectCount
+ */
+function download(url, redirectCount) {
+	return new Promise((c, e) => {
+		var content = '';
+		https.get(getOptions(url), function (response) {
+			response.on('data', function (data) {
+				content += data.toString();
+			}).on('end', function () {
+				if (response.statusCode === 403 && response.headers['x-ratelimit-remaining'] === '0') {
+					e('GitHub API rate exceeded. Set GITHUB_TOKEN environment variable to increase rate limit.');
+					return;
+				}
+				let count = redirectCount || 0;
+				if (count < 5 && response.statusCode >= 300 && response.statusCode <= 303 || response.statusCode === 307) {
+					let location = response.headers['location'];
+					if (location) {
+						console.log("Redirected " + url + " to " + location);
+						download(location, count + 1).then(c, e);
+						return;
+					}
+				}
+				c(content);
+			});
+		}).on('error', function (err) {
+			e(err.message);
+		});
+	});
+}
+
+function getCommitSha(repoId, repoPath) {
+	var commitInfo = 'https://api.github.com/repos/' + repoId + '/commits?path=' + repoPath;
+	return download(commitInfo).then(function (content) {
+		try {
+			let lastCommit = JSON.parse(content)[0];
+			return Promise.resolve({
+				commitSha: lastCommit.sha,
+				commitDate: lastCommit.commit.author.date
+			});
+		} catch (e) {
+			return Promise.reject(new Error("Failed extracting the SHA: " + content));
+		}
+	});
+}
+
+exports.update = function (repoId, repoPath, dest, modifyGrammar, version = 'master', packageJsonPathOverride = '') {
+	var contentPath = 'https://raw.githubusercontent.com/' + repoId + `/${version}/` + repoPath;
+	console.log('Reading from ' + contentPath);
+	return download(contentPath).then(function (content) {
+		var ext = path.extname(repoPath);
+		var grammar;
+		if (ext === '.tmLanguage' || ext === '.plist') {
+			grammar = plist.parse(content);
+		} else if (ext === '.cson') {
+			grammar = cson.parse(content);
+		} else if (ext === '.json' || ext === '.JSON-tmLanguage') {
+			grammar = JSON.parse(content);
+		} else {
+			return Promise.reject(new Error('Unknown file extension: ' + ext));
+		}
+		if (modifyGrammar) {
+			modifyGrammar(grammar);
+		}
+		return getCommitSha(repoId, repoPath).then(function (info) {
+			let result = {
+				information_for_contributors: [
+					'This file has been converted from https://github.com/' + repoId + '/blob/master/' + repoPath,
+					'If you want to provide a fix or improvement, please create a pull request against the original repository.',
+					'Once accepted there, we are happy to receive an update request.'
+				]
+			};
+
+			if (info) {
+				result.version = 'https://github.com/' + repoId + '/commit/' + info.commitSha;
+			}
+
+			let keys = ['name', 'scopeName', 'comment', 'injections', 'patterns', 'repository'];
+			for (let key of keys) {
+				if (grammar.hasOwnProperty(key)) {
+					result[key] = grammar[key];
+				}
+			}
+
+			try {
+				fs.writeFileSync(dest, JSON.stringify(result, null, '\t').replace(/\n/g, '\r\n'));
+				let cgmanifestRead = JSON.parse(fs.readFileSync('./cgmanifest.json').toString());
+				let promises = new Array();
+				const currentCommitDate = info.commitDate.substr(0, 10);
+
+				// Add commit sha to cgmanifest.
+				if (currentCommitDate > commitDate) {
+					let packageJsonPath = 'https://raw.githubusercontent.com/' + repoId + `/${info.commitSha}/`;
+					if (packageJsonPathOverride) {
+						packageJsonPath += packageJsonPathOverride;
+					}
+					packageJsonPath += 'package.json';
+					for (let i = 0; i < cgmanifestRead.registrations.length; i++) {
+						if (cgmanifestRead.registrations[i].component.git.repositoryUrl.substr(cgmanifestRead.registrations[i].component.git.repositoryUrl.length - repoId.length, repoId.length) === repoId) {
+							cgmanifestRead.registrations[i].component.git.commitHash = info.commitSha;
+								commitDate = currentCommitDate;
+								promises.push(download(packageJsonPath).then(function (packageJson) {
+									if (packageJson) {
+										try {
+											cgmanifestRead.registrations[i].version = JSON.parse(packageJson).version;
+										} catch (e) {
+											console.log('Cannot get version. File does not exist at ' + packageJsonPath);
+										}
+									}
+								}));
+							break;
+						}
+					}
+				}
+				Promise.all(promises).then(function (allResult) {
+					fs.writeFileSync('./cgmanifest.json', JSON.stringify(cgmanifestRead, null, '\t').replace(/\n/g, '\r\n'));
+				});
+				if (info) {
+					console.log('Updated ' + path.basename(dest) + ' to ' + repoId + '@' + info.commitSha.substr(0, 7) + ' (' + currentCommitDate + ')');
+				} else {
+					console.log('Updated ' + path.basename(dest));
+				}
+			} catch (e) {
+				return Promise.reject(e);
+			}
+		});
+
+	}, console.error).catch(e => {
+		console.error(e);
+		process.exit(1);
+	});
+};
+
+if (path.basename(process.argv[1]) === 'update-grammar.js') {
+	for (var i = 3; i < process.argv.length; i += 2) {
+		exports.update(process.argv[2], process.argv[i], process.argv[i + 1]);
+	}
+}

lib/vscode/build/npm/update-localization-extension.js

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+let i18n = require("../lib/i18n");
+
+let fs = require("fs");
+let path = require("path");
+
+let gulp = require('gulp');
+let vfs = require("vinyl-fs");
+let rimraf = require('rimraf');
+let minimist = require('minimist');
+
+function update(options) {
+	let idOrPath = options._;
+	if (!idOrPath) {
+		throw new Error('Argument must be the location of the localization extension.');
+	}
+	let transifex = options.transifex;
+	let location = options.location;
+	if (transifex === true && location !== undefined) {
+		throw new Error('Either --transifex or --location can be specified, but not both.');
+	}
+	if (!transifex && !location) {
+		transifex = true;
+	}
+	if (location !== undefined && !fs.existsSync(location)) {
+		throw new Error(`${location} doesn't exist.`);
+	}
+	let locExtFolder = idOrPath;
+	if (/^\w{2}(-\w+)?$/.test(idOrPath)) {
+		locExtFolder = path.join('..', 'vscode-loc', 'i18n', `vscode-language-pack-${idOrPath}`);
+	}
+	let locExtStat = fs.statSync(locExtFolder);
+	if (!locExtStat || !locExtStat.isDirectory) {
+		throw new Error('No directory found at ' + idOrPath);
+	}
+	let packageJSON = JSON.parse(fs.readFileSync(path.join(locExtFolder, 'package.json')).toString());
+	let contributes = packageJSON['contributes'];
+	if (!contributes) {
+		throw new Error('The extension must define a "localizations" contribution in the "package.json"');
+	}
+	let localizations = contributes['localizations'];
+	if (!localizations) {
+		throw new Error('The extension must define a "localizations" contribution of type array in the "package.json"');
+	}
+
+	localizations.forEach(function (localization) {
+		if (!localization.languageId || !localization.languageName || !localization.localizedLanguageName) {
+			throw new Error('Each localization contribution must define "languageId", "languageName" and "localizedLanguageName" properties.');
+		}
+		let server = localization.server || 'www.transifex.com';
+		let userName = localization.userName || 'api';
+		let apiToken = process.env.TRANSIFEX_API_TOKEN;
+		let languageId = localization.transifexId || localization.languageId;
+		let translationDataFolder = path.join(locExtFolder, 'translations');
+		if (languageId === "zh-cn") {
+			languageId = "zh-hans";
+		}
+		if (languageId === "zh-tw") {
+			languageId = "zh-hant";
+		}
+		if (fs.existsSync(translationDataFolder) && fs.existsSync(path.join(translationDataFolder, 'main.i18n.json'))) {
+			console.log('Clearing  \'' + translationDataFolder + '\'...');
+			rimraf.sync(translationDataFolder);
+		}
+
+		if (transifex) {
+			console.log(`Downloading translations for ${languageId} to '${translationDataFolder}' ...`);
+			let translationPaths = [];
+			i18n.pullI18nPackFiles(server, userName, apiToken, { id: languageId }, translationPaths)
+				.on('error', (error) => {
+					console.log(`Error occurred while importing translations:`);
+					translationPaths = undefined;
+					if (Array.isArray(error)) {
+						error.forEach(console.log);
+					} else if (error) {
+						console.log(error);
+					} else {
+						console.log('Unknown error');
+					}
+				})
+				.pipe(vfs.dest(translationDataFolder))
+				.on('end', function () {
+					if (translationPaths !== undefined) {
+						localization.translations = [];
+						for (let tp of translationPaths) {
+							localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`});
+						}
+						fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t'));
+					}
+				});
+		} else {
+			console.log(`Importing translations for ${languageId} form '${location}' to '${translationDataFolder}' ...`);
+			let translationPaths = [];
+			gulp.src(path.join(location, languageId, '**', '*.xlf'))
+				.pipe(i18n.prepareI18nPackFiles(i18n.externalExtensionsWithTranslations, translationPaths, languageId === 'ps'))
+				.on('error', (error) => {
+					console.log(`Error occurred while importing translations:`);
+					translationPaths = undefined;
+					if (Array.isArray(error)) {
+						error.forEach(console.log);
+					} else if (error) {
+						console.log(error);
+					} else {
+						console.log('Unknown error');
+					}
+				})
+				.pipe(vfs.dest(translationDataFolder))
+				.on('end', function () {
+					if (translationPaths !== undefined) {
+						localization.translations = [];
+						for (let tp of translationPaths) {
+							localization.translations.push({ id: tp.id, path: `./translations/${tp.resourceName}`});
+						}
+						fs.writeFileSync(path.join(locExtFolder, 'package.json'), JSON.stringify(packageJSON, null, '\t'));
+					}
+				});
+		}
+	});
+}
+if (path.basename(process.argv[1]) === 'update-localization-extension.js') {
+	var options = minimist(process.argv.slice(2), {
+		boolean: 'transifex',
+		string: 'location'
+	});
+	update(options);
+}

lib/vscode/build/npm/update-theme.js

@@ -0,0 +1,82 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var plist = require('fast-plist');
+
+var mappings = {
+	"background": ["editor.background"],
+	"foreground": ["editor.foreground"],
+	"hoverHighlight": ["editor.hoverHighlightBackground"],
+	"linkForeground": ["editorLink.foreground"],
+	"selection": ["editor.selectionBackground"],
+	"inactiveSelection": ["editor.inactiveSelectionBackground"],
+	"selectionHighlightColor": ["editor.selectionHighlightBackground"],
+	"wordHighlight": ["editor.wordHighlightBackground"],
+	"wordHighlightStrong": ["editor.wordHighlightStrongBackground"],
+	"findMatchHighlight": ["editor.findMatchHighlightBackground", "peekViewResult.matchHighlightBackground"],
+	"currentFindMatchHighlight": ["editor.findMatchBackground"],
+	"findRangeHighlight": ["editor.findRangeHighlightBackground"],
+	"referenceHighlight": ["peekViewEditor.matchHighlightBackground"],
+	"lineHighlight": ["editor.lineHighlightBackground"],
+	"rangeHighlight": ["editor.rangeHighlightBackground"],
+	"caret": ["editorCursor.foreground"],
+	"invisibles": ["editorWhitespace.foreground"],
+	"guide": ["editorIndentGuide.background"],
+	"ansiBlack": ["terminal.ansiBlack"], "ansiRed": ["terminal.ansiRed"], "ansiGreen": ["terminal.ansiGreen"], "ansiYellow": ["terminal.ansiYellow"],
+	"ansiBlue": ["terminal.ansiBlue"], "ansiMagenta": ["terminal.ansiMagenta"], "ansiCyan": ["terminal.ansiCyan"], "ansiWhite": ["terminal.ansiWhite"],
+	"ansiBrightBlack": ["terminal.ansiBrightBlack"], "ansiBrightRed": ["terminal.ansiBrightRed"], "ansiBrightGreen": ["terminal.ansiBrightGreen"],
+	"ansiBrightYellow": ["terminal.ansiBrightYellow"], "ansiBrightBlue": ["terminal.ansiBrightBlue"], "ansiBrightMagenta": ["terminal.ansiBrightMagenta"],
+	"ansiBrightCyan": ["terminal.ansiBrightCyan"], "ansiBrightWhite": ["terminal.ansiBrightWhite"]
+};
+
+exports.update = function (srcName, destName) {
+	try {
+		console.log('reading ', srcName);
+		let result = {};
+		let plistContent = fs.readFileSync(srcName).toString();
+		let theme = plist.parse(plistContent);
+		let settings = theme.settings;
+		if (Array.isArray(settings)) {
+			let colorMap = {};
+			for (let entry of settings) {
+				let scope = entry.scope;
+				if (scope) {
+					let parts = scope.split(',').map(p => p.trim());
+					if (parts.length > 1) {
+						entry.scope = parts;
+					}
+				} else {
+					var entrySettings = entry.settings;
+					for (let entry in entrySettings) {
+						let mapping = mappings[entry];
+						if (mapping) {
+							for (let newKey of mapping) {
+								colorMap[newKey] = entrySettings[entry];
+							}
+							if (entry !== 'foreground' && entry !== 'background') {
+								delete entrySettings[entry];
+							}
+						}
+					}
+
+				}
+			}
+			result.name = theme.name;
+			result.tokenColors = settings;
+			result.colors = colorMap;
+		}
+		fs.writeFileSync(destName, JSON.stringify(result, null, '\t'));
+	} catch (e) {
+		console.log(e);
+	}
+};
+
+if (path.basename(process.argv[1]) === 'update-theme.js') {
+	exports.update(process.argv[2], process.argv[3]);
+}