Merge commit 'be3e8236086165e5e45a5a10783823874b3f3ebd' as 'lib/vscode'

lib/vscode/extensions/vscode-colorize-tests/.gitignore

@@ -0,0 +1,2 @@
+out
+node_modules

lib/vscode/extensions/vscode-colorize-tests/.vscode/launch.json

@@ -0,0 +1,17 @@
+// A launch configuration that compiles the extension and then opens it inside a new window
+{
+	"version": "0.1.0",
+	"configurations": [
+		{
+			"name": "Launch Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": ["${workspaceFolder}/../../", "${workspaceFolder}/test", "--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out" ],
+			"stopOnEntry": false,
+			"sourceMaps": true,
+			"outDir": "${workspaceFolder}/out",
+			"preLaunchTask": "npm"
+		}
+	]
+}

lib/vscode/extensions/vscode-colorize-tests/.vscode/tasks.json

@@ -0,0 +1,11 @@
+{
+	"version": "2.0.0",
+	"command": "npm",
+	"type": "shell",
+	"presentation": {
+		"reveal": "silent"
+	},
+	"args": ["run", "compile"],
+	"isBackground": true,
+	"problemMatcher": "$tsc-watch"
+}

lib/vscode/extensions/vscode-colorize-tests/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "vscode-colorize-tests",
+  "description": "Colorize tests for VS Code",
+  "version": "0.0.1",
+  "publisher": "vscode",
+  "license": "MIT",
+  "private": true,
+  "activationEvents": [
+    "onLanguage:json"
+  ],
+  "main": "./out/colorizerTestMain",
+  "enableProposedApi": true,
+  "engines": {
+    "vscode": "*"
+  },
+  "scripts": {
+    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-tests ./tsconfig.json"
+  },
+  "dependencies": {
+    "jsonc-parser": "2.2.1"
+  },
+  "devDependencies": {
+    "@types/node": "^12.11.7",
+    "mocha-junit-reporter": "^1.17.0",
+    "mocha-multi-reporters": "^1.1.7",
+    "vscode": "1.1.5"
+  },
+  "contributes": {
+    "semanticTokenTypes": [
+      {
+        "id": "testToken",
+        "description": "A test token"
+      }
+    ],
+    "semanticTokenModifiers": [
+      {
+        "id": "testModifier",
+        "description": "A test modifier"
+      }
+    ],
+    "semanticTokenScopes": [
+      {
+        "scopes": {
+          "testToken": [
+            "entity.name.function.special"
+          ]
+        }
+      }
+    ],
+    "productIconThemes": [
+      {
+        "id": "Test Product Icons",
+        "label": "The Test Product Icon Theme",
+        "path": "./producticons/test-product-icon-theme.json",
+        "_watch": true
+      }
+    ]
+  }
+}

lib/vscode/extensions/vscode-colorize-tests/producticons/mit_license.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) <2013> <Elegant Themes, Inc.>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

lib/vscode/extensions/vscode-colorize-tests/producticons/test-product-icon-theme.json

@@ -0,0 +1,43 @@
+{
+	// ElegantIcons from https://www.elegantthemes.com/icons/elegant_font.zip
+	"fonts": [
+		{
+			"id": "elegant",
+			"src": [
+				{
+					"path": "./ElegantIcons.woff",
+					"format": "woff"
+				}
+			],
+			"weight": "normal",
+			"style": "normal",
+		}
+	],
+	"iconDefinitions": {
+		"chevron-down": {
+			"fontCharacter": "\\43",
+		},
+		"chevron-right": {
+			"fontCharacter": "\\45"
+		},
+		"error": {
+			"fontCharacter": "\\e062"
+		},
+		"warning": {
+			"fontCharacter": "\\e063"
+		},
+		"settings-gear": {
+			"fontCharacter": "\\e030"
+		},
+		"files": {
+			"fontCharacter": "\\e056"
+		},
+		"extensions": {
+			"fontCharacter": "\\e015"
+		},
+		"debug-alt-2": {
+			"fontCharacter": "\\e072"
+		}
+
+	}
+}

lib/vscode/extensions/vscode-colorize-tests/src/colorizer.test.ts

@@ -0,0 +1,76 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import 'mocha';
+import * as assert from 'assert';
+import { commands, Uri } from 'vscode';
+import { join, basename, normalize, dirname } from 'path';
+import * as fs from 'fs';
+
+function assertUnchangedTokens(testFixurePath: string, done: any) {
+	let fileName = basename(testFixurePath);
+
+	return commands.executeCommand('_workbench.captureSyntaxTokens', Uri.file(testFixurePath)).then(data => {
+		try {
+			let resultsFolderPath = join(dirname(dirname(testFixurePath)), 'colorize-results');
+			if (!fs.existsSync(resultsFolderPath)) {
+				fs.mkdirSync(resultsFolderPath);
+			}
+			let resultPath = join(resultsFolderPath, fileName.replace('.', '_') + '.json');
+			if (fs.existsSync(resultPath)) {
+				let previousData = JSON.parse(fs.readFileSync(resultPath).toString());
+				try {
+					assert.deepEqual(data, previousData);
+				} catch (e) {
+					fs.writeFileSync(resultPath, JSON.stringify(data, null, '\t'), { flag: 'w' });
+					if (Array.isArray(data) && Array.isArray(previousData) && data.length === previousData.length) {
+						for (let i= 0; i < data.length; i++) {
+							let d = data[i];
+							let p = previousData[i];
+							if (d.c !== p.c || hasThemeChange(d.r, p.r)) {
+								throw e;
+							}
+						}
+						// different but no tokenization ot color change: no failure
+					} else {
+						throw e;
+					}
+				}
+			} else {
+				fs.writeFileSync(resultPath, JSON.stringify(data, null, '\t'));
+			}
+			done();
+		} catch (e) {
+			done(e);
+		}
+	}, done);
+}
+
+function hasThemeChange(d: any, p: any) : boolean {
+	let keys = Object.keys(d);
+	for (let key of keys) {
+		if (d[key] !== p[key]) {
+			return true;
+		}
+	}
+	return false;
+}
+
+suite('colorization', () => {
+	let extensionsFolder = normalize(join(__dirname, '../../'));
+	let extensions = fs.readdirSync(extensionsFolder);
+	extensions.forEach(extension => {
+		let extensionColorizeFixturePath = join(extensionsFolder, extension, 'test', 'colorize-fixtures');
+		if (fs.existsSync(extensionColorizeFixturePath)) {
+			let fixturesFiles = fs.readdirSync(extensionColorizeFixturePath);
+			fixturesFiles.forEach(fixturesFile => {
+				// define a test for each fixture
+				test(extension + '-' + fixturesFile, function (done) {
+					assertUnchangedTokens(join(extensionColorizeFixturePath, fixturesFile), done);
+				});
+			});
+		}
+	});
+});

lib/vscode/extensions/vscode-colorize-tests/src/colorizerTestMain.ts

@@ -0,0 +1,73 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import * as jsoncParser from 'jsonc-parser';
+
+export function activate(context: vscode.ExtensionContext): any {
+
+	const tokenTypes = ['type', 'struct', 'class', 'interface', 'enum', 'parameterType', 'function', 'variable', 'testToken'];
+	const tokenModifiers = ['static', 'abstract', 'deprecated', 'declaration', 'documentation', 'member', 'async', 'testModifier'];
+
+	const legend = new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
+
+	const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test');
+
+	const documentSemanticHighlightProvider: vscode.DocumentSemanticTokensProvider = {
+		provideDocumentSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
+			const builder = new vscode.SemanticTokensBuilder();
+
+			function addToken(value: string, startLine: number, startCharacter: number, length: number) {
+				const [type, ...modifiers] = value.split('.');
+
+				const selectedModifiers = [];
+
+				let tokenType = legend.tokenTypes.indexOf(type);
+				if (tokenType === -1) {
+					if (type === 'notInLegend') {
+						tokenType = tokenTypes.length + 2;
+					} else {
+						return;
+					}
+				}
+
+				let tokenModifiers = 0;
+				for (const modifier of modifiers) {
+					const index = legend.tokenModifiers.indexOf(modifier);
+					if (index !== -1) {
+						tokenModifiers = tokenModifiers | 1 << index;
+						selectedModifiers.push(modifier);
+					} else if (modifier === 'notInLegend') {
+						tokenModifiers = tokenModifiers | 1 << (legend.tokenModifiers.length + 2);
+						selectedModifiers.push(modifier);
+					}
+				}
+				builder.push(startLine, startCharacter, length, tokenType, tokenModifiers);
+
+				outputChannel.appendLine(`line: ${startLine}, character: ${startCharacter}, length ${length}, ${type} (${tokenType}), ${selectedModifiers} ${tokenModifiers.toString(2)}`);
+			}
+
+			outputChannel.appendLine('---');
+
+			const visitor: jsoncParser.JSONVisitor = {
+				onObjectProperty: (property: string, _offset: number, _length: number, startLine: number, startCharacter: number) => {
+					addToken(property, startLine, startCharacter, property.length + 2);
+				},
+				onLiteralValue: (value: any, _offset: number, length: number, startLine: number, startCharacter: number) => {
+					if (typeof value === 'string') {
+						addToken(value, startLine, startCharacter, length);
+					}
+				}
+			};
+			jsoncParser.visit(document.getText(), visitor);
+
+			return builder.build();
+		}
+	};
+
+
+	context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, documentSemanticHighlightProvider, legend));
+
+}

lib/vscode/extensions/vscode-colorize-tests/src/index.ts

@@ -0,0 +1,30 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+const path = require('path');
+const testRunner = require('vscode/lib/testrunner');
+
+const suite = 'Integration Colorize Tests';
+
+const options: any = {
+	ui: 'tdd',
+	useColors: (!process.env.BUILD_ARTIFACTSTAGINGDIRECTORY && process.platform !== 'win32'),
+	timeout: 60000
+};
+
+if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
+	options.reporter = 'mocha-multi-reporters';
+	options.reporterOptions = {
+		reporterEnabled: 'spec, mocha-junit-reporter',
+		mochaJunitReporterReporterOptions: {
+			testsuitesTitle: `${suite} ${process.platform}`,
+			mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
+		}
+	};
+}
+
+testRunner.configure(options);
+
+export = testRunner;

lib/vscode/extensions/vscode-colorize-tests/src/typings/ref.d.ts

@@ -0,0 +1,9 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/// <reference path="../../../../src/vs/vscode.d.ts" />
+/// <reference path="../../../../src/vs/vscode.proposed.d.ts" />
+/// <reference types='@types/node'/>
+

lib/vscode/extensions/vscode-colorize-tests/test/semantic-test/.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+    "editor.tokenColorCustomizationsExperimental": {
+        "class": "#00b0b0",
+        "interface": "#845faf",
+        "function": "#ff00ff",
+        "*.declaration": {
+            "fontStyle": "underline"
+        },
+        "*.declaration.member": {
+            "fontStyle": "italic bold",
+        }
+    }
+}

lib/vscode/extensions/vscode-colorize-tests/test/semantic-test/semantic-test.json

@@ -0,0 +1,9 @@
+[
+	"class", "function.member.declaration",
+			"parameterType.declaration",  "type", "parameterType.declaration", "type",
+				"variable.declaration", "parameterNames",
+		"function.member.declaration",
+	"interface.declaration",
+		"function.member.declaration", "function.notInLegend"
+
+]

lib/vscode/extensions/vscode-colorize-tests/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"extends": "../shared.tsconfig.json",
+	"compilerOptions": {
+		"outDir": "./out"
+	},
+	"include": [
+		"src/**/*"
+	]
+}