Merge commit 'be3e8236086165e5e45a5a10783823874b3f3ebd' as 'lib/vscode'

lib/vscode/build/azure-pipelines/common/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+*.js

lib/vscode/build/azure-pipelines/common/createAsset.ts

@@ -0,0 +1,133 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as fs from 'fs';
+import { Readable } from 'stream';
+import * as crypto from 'crypto';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import { CosmosClient } from '@azure/cosmos';
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl?: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+if (process.argv.length !== 6) {
+	console.error('Usage: node createAsset.js PLATFORM TYPE NAME FILE');
+	process.exit(-1);
+}
+
+function hashStream(hashName: string, stream: Readable): Promise<string> {
+	return new Promise<string>((c, e) => {
+		const shasum = crypto.createHash(hashName);
+
+		stream
+			.on('data', shasum.update.bind(shasum))
+			.on('error', e)
+			.on('close', () => c(shasum.digest('hex')));
+	});
+}
+
+async function doesAssetExist(blobService: azure.BlobService, quality: string, blobName: string): Promise<boolean | undefined> {
+	const existsResult = await new Promise<azure.BlobService.BlobResult>((c, e) => blobService.doesBlobExist(quality, blobName, (err, r) => err ? e(err) : c(r)));
+	return existsResult.exists;
+}
+
+async function uploadBlob(blobService: azure.BlobService, quality: string, blobName: string, filePath: string, fileName: string): Promise<void> {
+	const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+		contentSettings: {
+			contentType: mime.lookup(filePath),
+			contentDisposition: `attachment; filename="${fileName}"`,
+			cacheControl: 'max-age=31536000, public'
+		}
+	};
+
+	await new Promise<void>((c, e) => blobService.createBlockBlobFromLocalFile(quality, blobName, filePath, blobOptions, err => err ? e(err) : c()));
+}
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+async function main(): Promise<void> {
+	const [, , platform, type, fileName, filePath] = process.argv;
+	const quality = getEnv('VSCODE_QUALITY');
+	const commit = getEnv('BUILD_SOURCEVERSION');
+
+	console.log('Creating asset...');
+
+	const stat = await new Promise<fs.Stats>((c, e) => fs.stat(filePath, (err, stat) => err ? e(err) : c(stat)));
+	const size = stat.size;
+
+	console.log('Size:', size);
+
+	const stream = fs.createReadStream(filePath);
+	const [sha1hash, sha256hash] = await Promise.all([hashStream('sha1', stream), hashStream('sha256', stream)]);
+
+	console.log('SHA1:', sha1hash);
+	console.log('SHA256:', sha256hash);
+
+	const blobName = commit + '/' + fileName;
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const blobExists = await doesAssetExist(blobService, quality, blobName);
+
+	if (blobExists) {
+		console.log(`Blob ${quality}, ${blobName} already exists, not publishing again.`);
+		return;
+	}
+
+	console.log('Uploading blobs to Azure storage...');
+
+	await uploadBlob(blobService, quality, blobName, filePath, fileName);
+
+	console.log('Blobs successfully uploaded.');
+
+	const asset: Asset = {
+		platform,
+		type,
+		url: `${process.env['AZURE_CDN_URL']}/${quality}/${blobName}`,
+		hash: sha1hash,
+		sha256hash,
+		size
+	};
+
+	// Remove this if we ever need to rollback fast updates for windows
+	if (/win32/.test(platform)) {
+		asset.supportsFastUpdate = true;
+	}
+
+	console.log('Asset:', JSON.stringify(asset, null, '  '));
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('createAsset').execute('', [commit, asset, true]);
+}
+
+main().then(() => {
+	console.log('Asset successfully created');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

lib/vscode/build/azure-pipelines/common/createBuild.ts

@@ -0,0 +1,60 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { CosmosClient } from '@azure/cosmos';
+
+if (process.argv.length !== 3) {
+	console.error('Usage: node createBuild.js VERSION');
+	process.exit(-1);
+}
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+async function main(): Promise<void> {
+	const [, , _version] = process.argv;
+	const quality = getEnv('VSCODE_QUALITY');
+	const commit = getEnv('BUILD_SOURCEVERSION');
+	const queuedBy = getEnv('BUILD_QUEUEDBY');
+	const sourceBranch = getEnv('BUILD_SOURCEBRANCH');
+	const version = _version + (quality === 'stable' ? '' : `-${quality}`);
+
+	console.log('Creating build...');
+	console.log('Quality:', quality);
+	console.log('Version:', version);
+	console.log('Commit:', commit);
+
+	const build = {
+		id: commit,
+		timestamp: (new Date()).getTime(),
+		version,
+		isReleased: false,
+		sourceBranch,
+		queuedBy,
+		assets: [],
+		updates: {}
+	};
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('createBuild').execute('', [{ ...build, _partitionKey: '' }]);
+}
+
+main().then(() => {
+	console.log('Build successfully created');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

lib/vscode/build/azure-pipelines/common/extract-telemetry.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+cd $BUILD_STAGINGDIRECTORY
+mkdir extraction
+cd extraction
+git clone --depth 1 https://github.com/microsoft/vscode-extension-telemetry.git
+git clone --depth 1 https://github.com/microsoft/vscode-chrome-debug-core.git
+git clone --depth 1 https://github.com/microsoft/vscode-node-debug2.git
+git clone --depth 1 https://github.com/microsoft/vscode-node-debug.git
+git clone --depth 1 https://github.com/microsoft/vscode-html-languageservice.git
+git clone --depth 1 https://github.com/microsoft/vscode-json-languageservice.git
+node $BUILD_SOURCESDIRECTORY/build/node_modules/.bin/vscode-telemetry-extractor --sourceDir $BUILD_SOURCESDIRECTORY --excludedDir $BUILD_SOURCESDIRECTORY/extensions --outputDir . --applyEndpoints
+node $BUILD_SOURCESDIRECTORY/build/node_modules/.bin/vscode-telemetry-extractor --config $BUILD_SOURCESDIRECTORY/build/azure-pipelines/common/telemetry-config.json -o .
+mkdir -p $BUILD_SOURCESDIRECTORY/.build/telemetry
+mv declarations-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-core.json
+mv config-resolved.json $BUILD_SOURCESDIRECTORY/.build/telemetry/telemetry-extensions.json
+cd ..
+rm -rf extraction

lib/vscode/build/azure-pipelines/common/publish-webview.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+REPO="$(pwd)"
+
+# Publish webview contents
+PACKAGEJSON="$REPO/package.json"
+VERSION=$(node -p "require(\"$PACKAGEJSON\").version")
+
+node build/azure-pipelines/common/publish-webview.js "$REPO/src/vs/workbench/contrib/webview/browser/pre/"

lib/vscode/build/azure-pipelines/common/publish-webview.ts

@@ -0,0 +1,87 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import * as minimist from 'minimist';
+import { basename, join } from 'path';
+
+const fileNames = [
+	'fake.html',
+	'host.js',
+	'index.html',
+	'main.js',
+	'service-worker.js'
+];
+
+async function assertContainer(blobService: azure.BlobService, container: string): Promise<void> {
+	await new Promise<void>((c, e) => blobService.createContainerIfNotExists(container, { publicAccessLevel: 'blob' }, err => err ? e(err) : c()));
+}
+
+async function doesBlobExist(blobService: azure.BlobService, container: string, blobName: string): Promise<boolean | undefined> {
+	const existsResult = await new Promise<azure.BlobService.BlobResult>((c, e) => blobService.doesBlobExist(container, blobName, (err, r) => err ? e(err) : c(r)));
+	return existsResult.exists;
+}
+
+async function uploadBlob(blobService: azure.BlobService, container: string, blobName: string, file: string): Promise<void> {
+	const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+		contentSettings: {
+			contentType: mime.lookup(file),
+			cacheControl: 'max-age=31536000, public'
+		}
+	};
+
+	await new Promise<void>((c, e) => blobService.createBlockBlobFromLocalFile(container, blobName, file, blobOptions, err => err ? e(err) : c()));
+}
+
+async function publish(commit: string, files: readonly string[]): Promise<void> {
+
+	console.log('Publishing...');
+	console.log('Commit:', commit);
+	const storageAccount = process.env['AZURE_WEBVIEW_STORAGE_ACCOUNT']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_WEBVIEW_STORAGE_ACCESS_KEY']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	await assertContainer(blobService, commit);
+
+	for (const file of files) {
+		const blobName = basename(file);
+		const blobExists = await doesBlobExist(blobService, commit, blobName);
+		if (blobExists) {
+			console.log(`Blob ${commit}, ${blobName} already exists, not publishing again.`);
+			continue;
+		}
+		console.log('Uploading blob to Azure storage...');
+		await uploadBlob(blobService, commit, blobName, file);
+	}
+
+	console.log('Blobs successfully uploaded.');
+}
+
+function main(): void {
+	const commit = process.env['BUILD_SOURCEVERSION'];
+
+	if (!commit) {
+		console.warn('Skipping publish due to missing BUILD_SOURCEVERSION');
+		return;
+	}
+
+	const opts = minimist(process.argv.slice(2));
+	const [directory] = opts._;
+
+	const files = fileNames.map(fileName => join(directory, fileName));
+
+	publish(commit, files).catch(err => {
+		console.error(err);
+		process.exit(1);
+	});
+}
+
+if (process.argv.length < 3) {
+	console.error('Usage: node publish.js <directory>');
+	process.exit(-1);
+}
+main();

lib/vscode/build/azure-pipelines/common/releaseBuild.ts

@@ -0,0 +1,70 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import { CosmosClient } from '@azure/cosmos';
+
+function getEnv(name: string): string {
+	const result = process.env[name];
+
+	if (typeof result === 'undefined') {
+		throw new Error('Missing env: ' + name);
+	}
+
+	return result;
+}
+
+interface Config {
+	id: string;
+	frozen: boolean;
+}
+
+function createDefaultConfig(quality: string): Config {
+	return {
+		id: quality,
+		frozen: false
+	};
+}
+
+async function getConfig(client: CosmosClient, quality: string): Promise<Config> {
+	const query = `SELECT TOP 1 * FROM c WHERE c.id = "${quality}"`;
+
+	const res = await client.database('builds').container('config').items.query(query).fetchAll();
+
+	if (res.resources.length === 0) {
+		return createDefaultConfig(quality);
+	}
+
+	return res.resources[0] as Config;
+}
+
+async function main(): Promise<void> {
+	const commit = getEnv('BUILD_SOURCEVERSION');
+	const quality = getEnv('VSCODE_QUALITY');
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const config = await getConfig(client, quality);
+
+	console.log('Quality config:', config);
+
+	if (config.frozen) {
+		console.log(`Skipping release because quality ${quality} is frozen.`);
+		return;
+	}
+
+	console.log(`Releasing build ${commit}...`);
+
+	const scripts = client.database('builds').container(quality).scripts;
+	await scripts.storedProcedure('releaseBuild').execute('', [commit]);
+}
+
+main().then(() => {
+	console.log('Build successfully released');
+	process.exit(0);
+}, err => {
+	console.error(err);
+	process.exit(1);
+});

lib/vscode/build/azure-pipelines/common/sync-mooncake.ts

@@ -0,0 +1,130 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+import * as url from 'url';
+import * as azure from 'azure-storage';
+import * as mime from 'mime';
+import { CosmosClient } from '@azure/cosmos';
+
+function log(...args: any[]) {
+	console.log(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+function error(...args: any[]) {
+	console.error(...[`[${new Date().toISOString()}]`, ...args]);
+}
+
+if (process.argv.length < 3) {
+	error('Usage: node sync-mooncake.js <quality>');
+	process.exit(-1);
+}
+
+interface Build {
+	assets: Asset[];
+}
+
+interface Asset {
+	platform: string;
+	type: string;
+	url: string;
+	mooncakeUrl: string;
+	hash: string;
+	sha256hash: string;
+	size: number;
+	supportsFastUpdate?: boolean;
+}
+
+async function sync(commit: string, quality: string): Promise<void> {
+	log(`Synchronizing Mooncake assets for ${quality}, ${commit}...`);
+
+	const client = new CosmosClient({ endpoint: process.env['AZURE_DOCUMENTDB_ENDPOINT']!, key: process.env['AZURE_DOCUMENTDB_MASTERKEY'] });
+	const container = client.database('builds').container(quality);
+
+	const query = `SELECT TOP 1 * FROM c WHERE c.id = "${commit}"`;
+	const res = await container.items.query<Build>(query, {}).fetchAll();
+
+	if (res.resources.length !== 1) {
+		throw new Error(`No builds found for ${commit}`);
+	}
+
+	const build = res.resources[0];
+
+	log(`Found build for ${commit}, with ${build.assets.length} assets`);
+
+	const storageAccount = process.env['AZURE_STORAGE_ACCOUNT_2']!;
+
+	const blobService = azure.createBlobService(storageAccount, process.env['AZURE_STORAGE_ACCESS_KEY_2']!)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	const mooncakeBlobService = azure.createBlobService(storageAccount, process.env['MOONCAKE_STORAGE_ACCESS_KEY']!, `${storageAccount}.blob.core.chinacloudapi.cn`)
+		.withFilter(new azure.ExponentialRetryPolicyFilter(20));
+
+	// mooncake is fussy and far away, this is needed!
+	blobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+	mooncakeBlobService.defaultClientRequestTimeoutInMs = 10 * 60 * 1000;
+
+	for (const asset of build.assets) {
+		try {
+			const blobPath = url.parse(asset.url).path;
+
+			if (!blobPath) {
+				throw new Error(`Failed to parse URL: ${asset.url}`);
+			}
+
+			const blobName = blobPath.replace(/^\/\w+\//, '');
+
+			log(`Found ${blobName}`);
+
+			if (asset.mooncakeUrl) {
+				log(`  Already in Mooncake ✔️`);
+				continue;
+			}
+
+			const readStream = blobService.createReadStream(quality, blobName, undefined!);
+			const blobOptions: azure.BlobService.CreateBlockBlobRequestOptions = {
+				contentSettings: {
+					contentType: mime.lookup(blobPath),
+					cacheControl: 'max-age=31536000, public'
+				}
+			};
+
+			const writeStream = mooncakeBlobService.createWriteStreamToBlockBlob(quality, blobName, blobOptions, undefined);
+
+			log(`  Uploading to Mooncake...`);
+			await new Promise((c, e) => readStream.pipe(writeStream).on('finish', c).on('error', e));
+
+			log(`  Updating build in DB...`);
+			const mooncakeUrl = `${process.env['MOONCAKE_CDN_URL']}${blobPath}`;
+			await container.scripts.storedProcedure('setAssetMooncakeUrl')
+				.execute('', [commit, asset.platform, asset.type, mooncakeUrl]);
+
+			log(`  Done ✔️`);
+		} catch (err) {
+			error(err);
+		}
+	}
+
+	log(`All done ✔️`);
+}
+
+function main(): void {
+	const commit = process.env['BUILD_SOURCEVERSION'];
+
+	if (!commit) {
+		error('Skipping publish due to missing BUILD_SOURCEVERSION');
+		return;
+	}
+
+	const quality = process.argv[2];
+
+	sync(commit, quality).catch(err => {
+		error(err);
+		process.exit(1);
+	});
+}
+
+main();

lib/vscode/build/azure-pipelines/common/telemetry-config.json

@@ -0,0 +1,72 @@
+[
+	{
+		"eventPrefix": "typescript-language-features/",
+		"sourceDirs": [
+			"../../s/extensions/typescript-language-features"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "git/",
+		"sourceDirs": [
+			"../../s/extensions/git"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "extension-telemetry/",
+		"sourceDirs": [
+			"vscode-extension-telemetry"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "vscode-markdown/",
+		"sourceDirs": [
+			"../../s/extensions/markdown-language-features"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "html-language-features/",
+		"sourceDirs": [
+			"../../s/extensions/html-language-features",
+			"vscode-html-languageservice"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "json-language-features/",
+		"sourceDirs": [
+			"../../s/extensions/json-language-features",
+			"vscode-json-languageservice"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true
+	},
+	{
+		"eventPrefix": "ms-vscode.node2/",
+		"sourceDirs": [
+			"vscode-chrome-debug-core",
+			"vscode-node-debug2"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true,
+		"patchDebugEvents": true
+	},
+	{
+		"eventPrefix": "ms-vscode.node/",
+		"sourceDirs": [
+			"vscode-chrome-debug-core",
+			"vscode-node-debug"
+		],
+		"excludedDirs": [],
+		"applyEndpoints": true,
+		"patchDebugEvents": true
+	}
+]