From 5e86e40fab68205c5e9a0c4f6d6e869de9423510 Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Sun, 10 Apr 2022 22:36:12 -0700 Subject: [PATCH] extensions: pull from github in OSS build (#146863) * extensions: pull from github in OSS build * fixup! address pr comment --- build/azure-pipelines/product-compile.yml | 2 + build/gulpfile.extensions.js | 3 +- build/gulpfile.vscode.web.js | 2 +- build/lib/builtInExtensions.js | 8 ++-- build/lib/builtInExtensions.ts | 9 ++-- build/lib/extensions.js | 53 ++++++++++++++++++--- build/lib/extensions.ts | 57 +++++++++++++++++++++-- build/package.json | 3 +- build/yarn.lock | 37 ++++++++++++--- product.json | 4 +- 10 files changed, 149 insertions(+), 29 deletions(-) diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index 7722c3e069c9c..2b109f3cc5cd1 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -103,6 +103,8 @@ steps: - script: | set -e yarn npm-run-all -lp core-ci extensions-ci hygiene eslint valid-layers-check + env: + GITHUB_TOKEN: "$(github-distro-mixin-password)" displayName: Compile & Hygiene - script: | diff --git a/build/gulpfile.extensions.js b/build/gulpfile.extensions.js index cdc02cba74b4c..2b89c012e9f0c 100644 --- a/build/gulpfile.extensions.js +++ b/build/gulpfile.extensions.js @@ -20,6 +20,7 @@ const root = path.dirname(__dirname); const commit = util.getVersion(root); const plumber = require('gulp-plumber'); const ext = require('./lib/extensions'); +const product = require('../product.json'); const extensionsPath = path.join(path.dirname(__dirname), 'extensions'); @@ -221,7 +222,7 @@ const cleanExtensionsBuildTask = task.define('clean-extensions-build', util.rimr const compileExtensionsBuildTask = task.define('compile-extensions-build', task.series( cleanExtensionsBuildTask, task.define('bundle-extensions-build', () => ext.packageLocalExtensionsStream(false).pipe(gulp.dest('.build'))), - task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false).pipe(gulp.dest('.build'))), + task.define('bundle-marketplace-extensions-build', () => ext.packageMarketplaceExtensionsStream(false, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build'))), )); gulp.task(compileExtensionsBuildTask); diff --git a/build/gulpfile.vscode.web.js b/build/gulpfile.vscode.web.js index 8cc0e41d79419..ea53c1b8c6900 100644 --- a/build/gulpfile.vscode.web.js +++ b/build/gulpfile.vscode.web.js @@ -229,7 +229,7 @@ function packageTask(sourceFolderName, destinationFolderName) { const compileWebExtensionsBuildTask = task.define('compile-web-extensions-build', task.series( task.define('clean-web-extensions-build', util.rimraf('.build/web/extensions')), task.define('bundle-web-extensions-build', () => extensions.packageLocalExtensionsStream(true).pipe(gulp.dest('.build/web'))), - task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true).pipe(gulp.dest('.build/web'))), + task.define('bundle-marketplace-web-extensions-build', () => extensions.packageMarketplaceExtensionsStream(true, product.extensionsGallery?.serviceUrl).pipe(gulp.dest('.build/web'))), task.define('bundle-web-extension-media-build', () => extensions.buildExtensionMedia(false, '.build/web/extensions')), )); gulp.task(compileWebExtensionsBuildTask); diff --git a/build/lib/builtInExtensions.js b/build/lib/builtInExtensions.js index d851a6ee5f9a3..221e5ba7516c8 100644 --- a/build/lib/builtInExtensions.js +++ b/build/lib/builtInExtensions.js @@ -45,15 +45,17 @@ function isUpToDate(extension) { } } function syncMarketplaceExtension(extension) { + const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; + const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]'); if (isUpToDate(extension)) { - log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); + log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); return es.readArray([]); } rimraf.sync(getExtensionPath(extension)); - return ext.fromMarketplace(extension.name, extension.version, extension.metadata) + return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) .pipe(vfs.dest('.build/builtInExtensions')) - .on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎'))); + .on('end', () => log(source, extension.name, ansiColors.green('✔︎'))); } function syncExtension(extension, controlState) { if (extension.platforms) { diff --git a/build/lib/builtInExtensions.ts b/build/lib/builtInExtensions.ts index b41477161c944..f5a03d1304ebf 100644 --- a/build/lib/builtInExtensions.ts +++ b/build/lib/builtInExtensions.ts @@ -69,17 +69,20 @@ function isUpToDate(extension: IExtensionDefinition): boolean { } function syncMarketplaceExtension(extension: IExtensionDefinition): Stream { + const galleryServiceUrl = productjson.extensionsGallery?.serviceUrl; + const source = ansiColors.blue(galleryServiceUrl ? '[marketplace]' : '[github]'); + if (isUpToDate(extension)) { - log(ansiColors.blue('[marketplace]'), `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); + log(source, `${extension.name}@${extension.version}`, ansiColors.green('✔︎')); return es.readArray([]); } rimraf.sync(getExtensionPath(extension)); - return ext.fromMarketplace(extension.name, extension.version, extension.metadata) + return (galleryServiceUrl ? ext.fromMarketplace(galleryServiceUrl, extension) : ext.fromGithub(extension)) .pipe(rename(p => p.dirname = `${extension.name}/${p.dirname}`)) .pipe(vfs.dest('.build/builtInExtensions')) - .on('end', () => log(ansiColors.blue('[marketplace]'), extension.name, ansiColors.green('✔︎'))); + .on('end', () => log(source, extension.name, ansiColors.green('✔︎'))); } function syncExtension(extension: IExtensionDefinition, controlState: 'disabled' | 'marketplace'): Stream { diff --git a/build/lib/extensions.js b/build/lib/extensions.js index 99ebc6dee2b88..50fc4dd945b95 100644 --- a/build/lib/extensions.js +++ b/build/lib/extensions.js @@ -4,13 +4,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ Object.defineProperty(exports, "__esModule", { value: true }); -exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromMarketplace = void 0; +exports.buildExtensionMedia = exports.webpackExtensions = exports.translatePackageJSON = exports.scanBuiltinExtensions = exports.packageMarketplaceExtensionsStream = exports.packageLocalExtensionsStream = exports.fromGithub = exports.fromMarketplace = void 0; const es = require("event-stream"); const fs = require("fs"); const cp = require("child_process"); const glob = require("glob"); const gulp = require("gulp"); const path = require("path"); +const through2 = require("through2"); +const got_1 = require("got"); const File = require("vinyl"); const stats_1 = require("./stats"); const util2 = require("./util"); @@ -169,16 +171,17 @@ function fromLocalNormal(extensionPath) { .catch(err => result.emit('error', err)); return result.pipe((0, stats_1.createStatsStream)(path.basename(extensionPath))); } +const userAgent = 'VSCode Build'; const baseHeaders = { 'X-Market-Client-Id': 'VSCode Build', - 'User-Agent': 'VSCode Build', + 'User-Agent': userAgent, 'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2', }; -function fromMarketplace(extensionName, version, metadata) { +function fromMarketplace(serviceUrl, { name: extensionName, version, metadata }) { const remote = require('gulp-remote-retry-src'); const json = require('gulp-json-editor'); const [publisher, name] = extensionName.split('.'); - const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`; + const url = `${serviceUrl}/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`; fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...'); const options = { base: url, @@ -198,6 +201,44 @@ function fromMarketplace(extensionName, version, metadata) { .pipe(packageJsonFilter.restore); } exports.fromMarketplace = fromMarketplace; +const ghApiHeaders = { + Accept: 'application/vnd.github.v3+json', + 'User-Agent': userAgent, +}; +if (process.env.GITHUB_TOKEN) { + ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64'); +} +const ghDownloadHeaders = { + ...ghApiHeaders, + Accept: 'application/octet-stream', +}; +function fromGithub({ name, version, repo, metadata }) { + const remote = require('gulp-remote-retry-src'); + const json = require('gulp-json-editor'); + fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...'); + const packageJsonFilter = filter('package.json', { restore: true }); + return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], { + base: 'https://api.github.com', + requestOptions: { headers: ghApiHeaders } + }).pipe(through2.obj(function (file, _enc, callback) { + const asset = JSON.parse(file.contents.toString()).assets.find((a) => a.name.endsWith('.vsix')); + if (!asset) { + return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`)); + } + const res = got_1.default.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true }); + file.contents = res.pipe(through2()); + callback(null, file); + })) + .pipe(buffer()) + .pipe(vzip.src()) + .pipe(filter('extension/**')) + .pipe(rename(p => p.dirname = p.dirname.replace(/^extension\/?/, ''))) + .pipe(packageJsonFilter) + .pipe(buffer()) + .pipe(json({ __metadata: metadata })) + .pipe(packageJsonFilter.restore); +} +exports.fromGithub = fromGithub; const excludedExtensions = [ 'vscode-api-tests', 'vscode-colorize-tests', @@ -272,14 +313,14 @@ function packageLocalExtensionsStream(forWeb) { .pipe(util2.setExecutableBit(['**/*.sh']))); } exports.packageLocalExtensionsStream = packageLocalExtensionsStream; -function packageMarketplaceExtensionsStream(forWeb) { +function packageMarketplaceExtensionsStream(forWeb, galleryServiceUrl) { const marketplaceExtensionsDescriptions = [ ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...(forWeb ? webBuiltInExtensions : []) ]; const marketplaceExtensionsStream = minifyExtensionResources(es.merge(...marketplaceExtensionsDescriptions .map(extension => { - const input = fromMarketplace(extension.name, extension.version, extension.metadata) + const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension)) .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); return updateExtensionPackageJSON(input, (data) => { delete data.scripts; diff --git a/build/lib/extensions.ts b/build/lib/extensions.ts index aeea8e6885280..312b830b9033d 100644 --- a/build/lib/extensions.ts +++ b/build/lib/extensions.ts @@ -9,6 +9,8 @@ import * as cp from 'child_process'; import * as glob from 'glob'; import * as gulp from 'gulp'; import * as path from 'path'; +import * as through2 from 'through2'; +import got from 'got'; import { Stream } from 'stream'; import * as File from 'vinyl'; import { createStatsStream } from './stats'; @@ -198,18 +200,19 @@ function fromLocalNormal(extensionPath: string): Stream { return result.pipe(createStatsStream(path.basename(extensionPath))); } +const userAgent = 'VSCode Build'; const baseHeaders = { 'X-Market-Client-Id': 'VSCode Build', - 'User-Agent': 'VSCode Build', + 'User-Agent': userAgent, 'X-Market-User-Id': '291C1CD0-051A-4123-9B4B-30D60EF52EE2', }; -export function fromMarketplace(extensionName: string, version: string, metadata: any): Stream { +export function fromMarketplace(serviceUrl: string, { name: extensionName, version, metadata }: IBuiltInExtension): Stream { const remote = require('gulp-remote-retry-src'); const json = require('gulp-json-editor') as typeof import('gulp-json-editor'); const [publisher, name] = extensionName.split('.'); - const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`; + const url = `${serviceUrl}/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`; fancyLog('Downloading extension:', ansiColors.yellow(`${extensionName}@${version}`), '...'); @@ -232,6 +235,50 @@ export function fromMarketplace(extensionName: string, version: string, metadata .pipe(json({ __metadata: metadata })) .pipe(packageJsonFilter.restore); } + +const ghApiHeaders: Record = { + Accept: 'application/vnd.github.v3+json', + 'User-Agent': userAgent, +}; +if (process.env.GITHUB_TOKEN) { + ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64'); +} +const ghDownloadHeaders = { + ...ghApiHeaders, + Accept: 'application/octet-stream', +}; + +export function fromGithub({ name, version, repo, metadata }: IBuiltInExtension): Stream { + const remote = require('gulp-remote-retry-src'); + const json = require('gulp-json-editor') as typeof import('gulp-json-editor'); + + fancyLog('Downloading extension from GH:', ansiColors.yellow(`${name}@${version}`), '...'); + + const packageJsonFilter = filter('package.json', { restore: true }); + + return remote([`/repos${new URL(repo).pathname}/releases/tags/v${version}`], { + base: 'https://api.github.com', + requestOptions: { headers: ghApiHeaders } + }).pipe(through2.obj(function (file, _enc, callback) { + const asset = JSON.parse(file.contents.toString()).assets.find((a: any) => a.name.endsWith('.vsix')); + if (!asset) { + return callback(new Error(`Could not find vsix in release of ${repo} @ ${version}`)); + } + + const res = got.stream(asset.url, { headers: ghDownloadHeaders, followRedirect: true }); + file.contents = res.pipe(through2()); + callback(null, file); + })) + .pipe(buffer()) + .pipe(vzip.src()) + .pipe(filter('extension/**')) + .pipe(rename(p => p.dirname = p.dirname!.replace(/^extension\/?/, ''))) + .pipe(packageJsonFilter) + .pipe(buffer()) + .pipe(json({ __metadata: metadata })) + .pipe(packageJsonFilter.restore); +} + const excludedExtensions = [ 'vscode-api-tests', 'vscode-colorize-tests', @@ -335,7 +382,7 @@ export function packageLocalExtensionsStream(forWeb: boolean): Stream { ); } -export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream { +export function packageMarketplaceExtensionsStream(forWeb: boolean, galleryServiceUrl?: string): Stream { const marketplaceExtensionsDescriptions = [ ...builtInExtensions.filter(({ name }) => (forWeb ? !marketplaceWebExtensionsExclude.has(name) : true)), ...(forWeb ? webBuiltInExtensions : []) @@ -344,7 +391,7 @@ export function packageMarketplaceExtensionsStream(forWeb: boolean): Stream { es.merge( ...marketplaceExtensionsDescriptions .map(extension => { - const input = fromMarketplace(extension.name, extension.version, extension.metadata) + const input = (galleryServiceUrl ? fromMarketplace(galleryServiceUrl, extension) : fromGithub(extension)) .pipe(rename(p => p.dirname = `extensions/${extension.name}/${p.dirname}`)); return updateExtensionPackageJSON(input, (data: any) => { delete data.scripts; diff --git a/build/package.json b/build/package.json index 6e0915f5b8a91..3403f5c5c137f 100644 --- a/build/package.json +++ b/build/package.json @@ -35,7 +35,7 @@ "@types/request": "^2.47.0", "@types/rimraf": "^2.0.4", "@types/through": "^0.0.29", - "@types/through2": "^2.0.34", + "@types/through2": "^2.0.36", "@types/tmp": "^0.2.1", "@types/underscore": "^1.8.9", "@types/webpack": "^4.41.25", @@ -59,6 +59,7 @@ "mkdirp": "^1.0.4", "p-limit": "^3.1.0", "source-map": "0.6.1", + "through2": "^4.0.2", "tmp": "^0.2.1", "vsce": "^1.100.0", "vscode-universal-bundler": "^0.0.2" diff --git a/build/yarn.lock b/build/yarn.lock index 991283b750282..9b91f75047dc1 100644 --- a/build/yarn.lock +++ b/build/yarn.lock @@ -576,10 +576,10 @@ resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310" integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ== -"@types/through2@^2.0.34": - version "2.0.34" - resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.34.tgz#9c2a259a238dace2a05a2f8e94b786961bc27ac4" - integrity sha512-nhRG8+RuG/L+0fAZBQYaRflXKjTrHOKH8MFTChnf+dNVMxA3wHYYrfj0tztK0W51ABXjGfRCDc0vRkecCOrsow== +"@types/through2@^2.0.36": + version "2.0.36" + resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.36.tgz#35fda0db635827d44c0e08e2c94653e647574a00" + integrity sha512-vuifQksQHJXhV9McpVsXKuhnf3lsoX70PnhcqIAbs9dqLH2NgrGz0DzZPDY3+Yh6eaRqcE1gnCQ6QhBn1/PT5A== dependencies: "@types/node" "*" @@ -1909,7 +1909,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@^2.0.1, inherits@~2.0.3: +inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -2533,6 +2533,15 @@ read@^1.0.7: dependencies: mute-stream "~0.0.4" +readable-stream@3: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readable-stream@^2.3.5: version "2.3.7" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" @@ -2606,7 +2615,7 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-buffer@^5.0.1: +safe-buffer@^5.0.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== @@ -2728,6 +2737,13 @@ stoppable@^1.1.0: resolved "https://registry.yarnpkg.com/stoppable/-/stoppable-1.1.0.tgz#32da568e83ea488b08e4d7ea2c3bcc9d75015d5b" integrity sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw== +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" @@ -2756,6 +2772,13 @@ supports-color@^6.1.0: dependencies: has-flag "^3.0.0" +through2@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" + integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== + dependencies: + readable-stream "3" + through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -2872,7 +2895,7 @@ url-parse-lax@^3.0.0: dependencies: prepend-http "^2.0.0" -util-deprecate@~1.0.1: +util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= diff --git a/product.json b/product.json index ac6b2de18ac95..89c258fea035b 100644 --- a/product.json +++ b/product.json @@ -46,7 +46,7 @@ }, { "name": "ms-vscode.js-debug-companion", - "version": "1.0.16", + "version": "1.0.17", "repo": "https://github.com/microsoft/vscode-js-debug-companion", "metadata": { "id": "99cb0b7f-7354-4278-b8da-6cc79972169d", @@ -76,7 +76,7 @@ }, { "name": "ms-vscode.vscode-js-profile-table", - "version": "1.0.0", + "version": "1.0.1", "repo": "https://github.com/microsoft/vscode-js-profile-visualizer", "metadata": { "id": "7e52b41b-71ad-457b-ab7e-0620f1fc4feb",