fix: add missing package.json and source maps (#5040)

* Use --exclude to skip node_modules

Instead of copying and then deleting them.  This will also catch some
node_modules directories that were missed.

* Remove per-extension dependency install

Code packages all the dependencies using webpack for each extension so
there are no dependencies to install.

* Include source maps

I also moved this to its own patch because it feels sufficiently
standalone.

Fixes #5026.

* Refresh language patch

The base is slightly different so it needed to be refreshed.

* Add missing package.json

This was caused by switching to Code's package step which does not
include the package.json.

Fixes #5019.

* Include keytar

It seems this actually is used now.

ci/build/build-release.sh

@@ -78,11 +78,25 @@ EOF
 bundle_vscode() {
   mkdir -p "$VSCODE_OUT_PATH"
 
-  # - Some extensions have a .gitignore which excludes their built source from
-  #   the npm package so exclude any .gitignore files.
-  # - Exclude Node as we will add it ourselves for the standalone and will not
+  local rsync_opts=()
+  if [[ ${DEBUG-} = 1 ]]; then
+    rsync_opts+=(-vh)
+  fi
+
+  # Some extensions have a .gitignore which excludes their built source from the
+  # npm package so exclude any .gitignore files.
+  rsync_opts+=(--exclude .gitignore)
+
+  # Exclude Node as we will add it ourselves for the standalone and will not
   # need it for the npm package.
-  rsync -avh --exclude .gitignore --exclude /node ./lib/vscode-reh-web-*/ "$VSCODE_OUT_PATH"
+  rsync_opts+=(--exclude /node)
+
+  # Exclude Node modules.
+  if [[ $KEEP_MODULES = 0 ]]; then
+    rsync_opts+=(--exclude node_modules)
+  fi
+
+  rsync "${rsync_opts[@]}" ./lib/vscode-reh-web-*/ "$VSCODE_OUT_PATH"
 
   # Add the commit, date, our name, links, and enable telemetry. This just makes
   # telemetry available; telemetry can still be disabled by flag or setting.
@@ -122,19 +136,17 @@ EOF
   ) > "$VSCODE_OUT_PATH/product.json"
 
   # Use the package.json for the web/remote server.  It does not have the right
-  # version though so pull that from the main package.json.  Also remove keytar
-  # since the web does not rely on it and that removes the dependency on
-  # libsecret.
-  jq --slurp '.[0] * {version: .[1].version} | del(.dependencies.keytar)' \
+  # version though so pull that from the main package.json.
+  jq --slurp '.[0] * {version: .[1].version}' \
     "$VSCODE_SRC_PATH/remote/package.json" \
     "$VSCODE_SRC_PATH/package.json" > "$VSCODE_OUT_PATH/package.json"
 
   rsync "$VSCODE_SRC_PATH/remote/yarn.lock" "$VSCODE_OUT_PATH/yarn.lock"
 
-  if [ "$KEEP_MODULES" = 0 ]; then
-    rm -Rf "$VSCODE_OUT_PATH/extensions/node_modules"
-    rm -Rf "$VSCODE_OUT_PATH/node_modules"
-  fi
+  # Include global extension dependencies as well.
+  rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions/package.json"
+  rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions/yarn.lock"
+  rsync "$VSCODE_SRC_PATH/extensions/postinstall.js" "$VSCODE_OUT_PATH/extensions/postinstall.js"
 
   pushd "$VSCODE_OUT_PATH"
   symlink_asar

ci/build/npm-postinstall.sh

@@ -98,14 +98,6 @@ vscode_yarn() {
 
   cd extensions
   yarn --production --frozen-lockfile
-
-  for ext in */; do
-    ext="${ext%/}"
-    echo "extensions/$ext: installing dependencies"
-    cd "$ext"
-    yarn --production --frozen-lockfile
-    cd "$OLDPWD"
-  done
 }
 
 main "$@"

patches/display-language.diff

@@ -79,7 +79,7 @@ Index: code-server/lib/vscode/src/vs/code/browser/workbench/workbench.html
 +		} catch (error) { /* Probably fine. */ }
  		Object.keys(self.webPackagePaths).map(function (key, index) {
  			self.webPackagePaths[key] = new URL(
- 				`{{VS_BASE}}/static/remote/web/node_modules/${key}/${self.webPackagePaths[key]}`,
+ 				`{{VS_BASE}}/static/node_modules/${key}/${self.webPackagePaths[key]}`,
 @@ -52,7 +76,8 @@
  					return value;
  				}

patches/integration.diff

@@ -7,7 +7,6 @@ Prepare Code for integration with code-server
 3. Add the code-server version to the help dialog.
 4. Add ready events for use in an iframe.
 5. Add our icons.
-6. Remove sourcemap host since we cannot upload ours there.
 
 Index: code-server/lib/vscode/src/vs/server/node/server.main.ts
 ===================================================================
@@ -254,16 +253,3 @@ Index: code-server/lib/vscode/src/vs/code/browser/workbench/workbench.html
  		<link rel="manifest" href="/manifest.json" crossorigin="use-credentials" />
  		<link data-name="vs/workbench/workbench.web.main" rel="stylesheet" href="./static/out/vs/workbench/workbench.web.main.css">
  
-Index: code-server/lib/vscode/build/gulpfile.reh.js
-===================================================================
---- code-server.orig/lib/vscode/build/gulpfile.reh.js
-+++ code-server/lib/vscode/build/gulpfile.reh.js
-@@ -365,7 +365,7 @@ function packageTask(type, platform, arc
- 	const minifyTask = task.define(`minify-vscode-${type}`, task.series(
- 		optimizeTask,
- 		util.rimraf(`out-vscode-${type}-min`),
--		common.minifyTask(`out-vscode-${type}`, `https://ticino.blob.core.windows.net/sourcemaps/${commit}/core`)
-+		common.minifyTask(`out-vscode-${type}`, ``)
- 	));
- 	gulp.task(minifyTask);
- 

patches/series

@@ -18,3 +18,4 @@ local-storage.diff
 service-worker.diff
 last-opened.diff
 connection-type.diff
+sourcemaps.diff

patches/sourcemaps.diff

@@ -0,0 +1,43 @@
+Make sourcemaps self-hosted
+
+Normally source maps get removed as part of the build process so prevent that
+from happening.  Also avoid using the windows.net host since obviously we can
+not host our source maps there and want them to be self-hosted even if we could.
+
+To test try debugging/browsing the source of a build in a browser.
+
+Index: code-server/lib/vscode/build/gulpfile.reh.js
+===================================================================
+--- code-server.orig/lib/vscode/build/gulpfile.reh.js
++++ code-server/lib/vscode/build/gulpfile.reh.js
+@@ -195,8 +195,7 @@ function packageTask(type, platform, arc
+ 
+ 		const src = gulp.src(sourceFolderName + '/**', { base: '.' })
+ 			.pipe(rename(function (path) { path.dirname = path.dirname.replace(new RegExp('^' + sourceFolderName), 'out'); }))
+-			.pipe(util.setExecutableBit(['**/*.sh']))
+-			.pipe(filter(['**', '!**/*.js.map']));
++			.pipe(util.setExecutableBit(['**/*.sh']));
+ 
+ 		const workspaceExtensionPoints = ['debuggers', 'jsonValidation'];
+ 		const isUIExtension = (manifest) => {
+@@ -235,9 +234,9 @@ function packageTask(type, platform, arc
+ 			.map(name => `.build/extensions/${name}/**`);
+ 
+ 		const extensions = gulp.src(extensionPaths, { base: '.build', dot: true });
+-		const extensionsCommonDependencies = gulp.src('.build/extensions/node_modules/**', { base: '.build', dot: true });
+-		const sources = es.merge(src, extensions, extensionsCommonDependencies)
++		const extensionsCommonDependencies = gulp.src('.build/extensions/node_modules/**', { base: '.build', dot: true })
+ 			.pipe(filter(['**', '!**/*.js.map'], { dot: true }));
++		const sources = es.merge(src, extensions, extensionsCommonDependencies);
+ 
+ 		let version = packageJson.version;
+ 		const quality = product.quality;
+@@ -363,7 +362,7 @@ function packageTask(type, platform, arc
+ 	const minifyTask = task.define(`minify-vscode-${type}`, task.series(
+ 		optimizeTask,
+ 		util.rimraf(`out-vscode-${type}-min`),
+-		common.minifyTask(`out-vscode-${type}`, `https://ticino.blob.core.windows.net/sourcemaps/${commit}/core`)
++		common.minifyTask(`out-vscode-${type}`, '')
+ 	));
+ 	gulp.task(minifyTask);
+