ovsx-client: support multiple registries (#12040)

Refactors the `OVSXClient` into a simple interface that can easily be implemented by components such as `OVSXHttpClient` or others. Add a new `OVSXRouterClient` that may route OpenVSX queries to different registries based on an `OVSXRouteConfig` configuration. The path to the configuration file can be specified when starting the backend using the `--ovsx-router-config` CLI param. When this configuration is not specified, we default to the previously configured OpenVSX registry. The goal of these changes was to minimally impact Theia's plugin code base by simply replacing the client interfaces we use to communicate with an OpenVSX registry. This proved more difficult than expected, especially when merging search results: pagination may currently be broken when dealing with multiple registries. Add a sample mock OpenVSX server running as part of Theia's backend. You can find the plugins served by this mock server under `sample-plugins`. This mock is there to allow easier testing of the multi-registry feature.
eclipse-theia · Jun 15, 2023 · 8831c9d · 8831c9d
1 parent 7012fd2
commit 8831c9d
Show file tree

Hide file tree

Showing 62 changed files with 2,781 additions and 629 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,3 +35,4 @@ dependency-check-summary.txt*
 *-trace.json
 .tours
 /performance-result.json
+*.vsix
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -34,7 +34,8 @@
         "--app-project-path=${workspaceFolder}/examples/electron",
         "--remote-debugging-port=9222",
         "--no-app-auto-install",
-        "--plugins=local-dir:../../plugins"
+        "--plugins=local-dir:../../plugins",
+        "--ovsx-router-config=examples/ovsx-router-config.json"
       ],
       "env": {
         "NODE_ENV": "development"
@@ -44,6 +45,7 @@
         "${workspaceFolder}/examples/electron/lib/backend/electron-main.js",
         "${workspaceFolder}/examples/electron/lib/backend/main.js",
         "${workspaceFolder}/examples/electron/lib/**/*.js",
+        "${workspaceFolder}/examples/api-samples/lib/**/*.js",
         "${workspaceFolder}/packages/*/lib/**/*.js",
         "${workspaceFolder}/dev-packages/*/lib/**/*.js"
       ],
@@ -62,7 +64,8 @@
         "--no-cluster",
         "--app-project-path=${workspaceFolder}/examples/browser",
         "--plugins=local-dir:plugins",
-        "--hosted-plugin-inspect=9339"
+        "--hosted-plugin-inspect=9339",
+        "--ovsx-router-config=examples/ovsx-router-config.json"
       ],
       "env": {
         "NODE_ENV": "development"
@@ -71,6 +74,7 @@
       "outFiles": [
         "${workspaceFolder}/examples/browser/src-gen/backend/*.js",
         "${workspaceFolder}/examples/browser/lib/**/*.js",
+        "${workspaceFolder}/examples/api-samples/lib/**/*.js",
         "${workspaceFolder}/packages/*/lib/**/*.js",
         "${workspaceFolder}/dev-packages/*/lib/**/*.js"
       ],

diff --git a/configs/base.tsconfig.json b/configs/base.tsconfig.json
@@ -14,14 +14,14 @@
     "emitDecoratorMetadata": true,
     "downlevelIteration": true,
     "resolveJsonModule": true,
-    "module": "commonjs",
-    "moduleResolution": "node",
-    "target": "ES2017",
+    "module": "CommonJS",
+    "moduleResolution": "Node",
+    "target": "ES2019",
     "jsx": "react",
     "lib": [
-      "ES2017",
+      "ES2019",
       "ES2020.Promise",
-      "dom"
+      "DOM"
     ],
     "sourceMap": true
   }

diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts
@@ -16,22 +16,14 @@
 
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-declare global {
-    interface Array<T> {
-        // Supported since Node >=11.0
-        flat(depth?: number): any
-    }
-}
-
-import { OVSXClient } from '@theia/ovsx-client/lib/ovsx-client';
+import { OVSXApiFilterImpl, OVSXClient } from '@theia/ovsx-client';
 import * as chalk from 'chalk';
 import * as decompress from 'decompress';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 import * as temp from 'temp';
-import { NodeRequestService } from '@theia/request/lib/node-request-service';
 import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
-import { RequestContext } from '@theia/request';
+import { RequestContext, RequestService } from '@theia/request';
 import { RateLimiter } from 'limiter';
 import escapeStringRegexp = require('escape-string-regexp');
 
@@ -59,21 +51,12 @@ export interface DownloadPluginsOptions {
      */
     apiVersion?: string;
 
-    /**
-     * The open-vsx registry API url.
-     */
-    apiUrl?: string;
-
     /**
      * Fetch plugins in parallel
      */
     parallel?: boolean;
 
     rateLimit?: number;
-
-    proxyUrl?: string;
-    proxyAuthorization?: string;
-    strictSsl?: boolean;
 }
 
 interface PluginDownload {
@@ -82,26 +65,17 @@ interface PluginDownload {
     version?: string | undefined
 }
 
-const requestService = new NodeRequestService();
-
-export default async function downloadPlugins(options: DownloadPluginsOptions = {}): Promise<void> {
+export default async function downloadPlugins(ovsxClient: OVSXClient, requestService: RequestService, options: DownloadPluginsOptions = {}): Promise<void> {
     const {
         packed = false,
         ignoreErrors = false,
         apiVersion = DEFAULT_SUPPORTED_API_VERSION,
-        apiUrl = 'https://open-vsx.org/api',
-        parallel = true,
         rateLimit = 15,
-        proxyUrl,
-        proxyAuthorization,
-        strictSsl
+        parallel = true
     } = options;
 
-    requestService.configure({
-        proxyUrl,
-        proxyAuthorization,
-        strictSSL: strictSsl
-    });
+    const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' });
+    const apiFilter = new OVSXApiFilterImpl(apiVersion);
 
     // Collect the list of failures to be appended at the end of the script.
     const failures: string[] = [];
@@ -115,7 +89,7 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
     // Excluded extension ids.
     const excludedIds = new Set<string>(pck.theiaPluginsExcludeIds || []);
 
-    const parallelOrSequence = async (...tasks: Array<() => unknown>) => {
+    const parallelOrSequence = async (tasks: (() => unknown)[]) => {
         if (parallel) {
             await Promise.all(tasks.map(task => task()));
         } else {
@@ -125,13 +99,13 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         }
     };
 
-    const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' });
-
     // Downloader wrapper
-    const downloadPlugin = (plugin: PluginDownload): Promise<void> => downloadPluginAsync(rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
+    const downloadPlugin = async (plugin: PluginDownload): Promise<void> => {
+        await downloadPluginAsync(requestService, rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
+    };
 
     const downloader = async (plugins: PluginDownload[]) => {
-        await parallelOrSequence(...plugins.map(plugin => () => downloadPlugin(plugin)));
+        await parallelOrSequence(plugins.map(plugin => () => downloadPlugin(plugin)));
     };
 
     await fs.mkdir(pluginsDir, { recursive: true });
@@ -146,17 +120,17 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         // This will include both "normal" plugins as well as "extension packs".
         const pluginsToDownload = Object.entries(pck.theiaPlugins)
             .filter((entry: [string, unknown]): entry is [string, string] => typeof entry[1] === 'string')
-            .map(([pluginId, url]) => ({ id: pluginId, downloadUrl: resolveDownloadUrlPlaceholders(url) }));
+            .map(([id, url]) => ({ id, downloadUrl: resolveDownloadUrlPlaceholders(url) }));
         await downloader(pluginsToDownload);
 
-        const handleDependencyList = async (dependencies: Array<string | string[]>) => {
-            const client = new OVSXClient({ apiVersion, apiUrl }, requestService);
+        const handleDependencyList = async (dependencies: (string | string[])[]) => {
             // De-duplicate extension ids to only download each once:
             const ids = new Set<string>(dependencies.flat());
-            await parallelOrSequence(...Array.from(ids, id => async () => {
+            await parallelOrSequence(Array.from(ids, id => async () => {
                 try {
                     await rateLimiter.removeTokens(1);
-                    const extension = await client.getLatestCompatibleExtensionVersion(id);
+                    const { extensions } = await ovsxClient.query({ extensionId: id });
+                    const extension = apiFilter.getLatestCompatibleExtension(extensions);
                     const version = extension?.version;
                     const downloadUrl = extension?.files.download;
                     if (downloadUrl) {
@@ -208,14 +182,15 @@ function resolveDownloadUrlPlaceholders(url: string): string {
 
 /**
  * Downloads a plugin, will make multiple attempts before actually failing.
+ * @param requestService
  * @param failures reference to an array storing all failures.
  * @param plugin plugin short name.
  * @param pluginUrl url to download the plugin at.
  * @param target where to download the plugin in.
  * @param packed whether to decompress or not.
- * @param cachedExtensionPacks the list of cached extension packs already downloaded.
  */
 async function downloadPluginAsync(
+    requestService: RequestService,
     rateLimiter: RateLimiter,
     failures: string[],
     plugin: string,

diff --git a/dev-packages/cli/src/theia.ts b/dev-packages/cli/src/theia.ts
@@ -25,6 +25,8 @@ import checkDependencies from './check-dependencies';
 import downloadPlugins from './download-plugins';
 import runTest from './run-test';
 import { LocalizationManager, extract } from '@theia/localization-manager';
+import { NodeRequestService } from '@theia/request/lib/node-request-service';
+import { ExtensionIdMatchesFilterFactory, OVSXClient, OVSXHttpClient, OVSXRouterClient, RequestContainsFilterFactory } from '@theia/ovsx-client';
 
 const { executablePath } = require('puppeteer');
 
@@ -45,9 +47,7 @@ theiaCli();
 function toStringArray(argv: (string | number)[]): string[];
 function toStringArray(argv?: (string | number)[]): string[] | undefined;
 function toStringArray(argv?: (string | number)[]): string[] | undefined {
-    return argv === undefined
-        ? undefined
-        : argv.map(arg => String(arg));
+    return argv?.map(arg => String(arg));
 }
 
 function rebuildCommand(command: string, target: ApplicationProps.Target): yargs.CommandModule<unknown, {
@@ -314,8 +314,10 @@ async function theiaCli(): Promise<void> {
             apiUrl: string
             parallel: boolean
             proxyUrl?: string
-            proxyAuthentification?: string
+            proxyAuthorization?: string
             strictSsl: boolean
+            rateLimit: number
+            ovsxRouterConfig?: string
         }>({
             command: 'download:plugins',
             describe: 'Download defined external plugins',
@@ -355,17 +357,43 @@ async function theiaCli(): Promise<void> {
                 'proxy-url': {
                     describe: 'Proxy URL'
                 },
-                'proxy-authentification': {
-                    describe: 'Proxy authentification information'
+                'proxy-authorization': {
+                    describe: 'Proxy authorization information'
                 },
                 'strict-ssl': {
                     describe: 'Whether to enable strict SSL mode',
                     boolean: true,
                     default: false
+                },
+                'ovsx-router-config': {
+                    describe: 'JSON configuration file for the OVSX router client',
+                    type: 'string'
                 }
             },
-            handler: async args => {
-                await downloadPlugins(args);
+            handler: async ({ apiUrl, proxyUrl, proxyAuthorization, strictSsl, ovsxRouterConfig, ...options }) => {
+                const requestService = new NodeRequestService();
+                await requestService.configure({
+                    proxyUrl,
+                    proxyAuthorization,
+                    strictSSL: strictSsl
+                });
+                let client: OVSXClient | undefined;
+                if (ovsxRouterConfig) {
+                    const routerConfig = await fs.promises.readFile(ovsxRouterConfig, 'utf8').then(JSON.parse, error => {
+                        console.error(error);
+                    });
+                    if (routerConfig) {
+                        client = await OVSXRouterClient.FromConfig(
+                            routerConfig,
+                            OVSXHttpClient.createClientFactory(requestService),
+                            [RequestContainsFilterFactory, ExtensionIdMatchesFilterFactory]
+                        );
+                    }
+                }
+                if (!client) {
+                    client = new OVSXHttpClient(apiUrl, requestService);
+                }
+                await downloadPlugins(client, requestService, options);
             },
         })
         .command<{

diff --git a/dev-packages/ovsx-client/README.md b/dev-packages/ovsx-client/README.md
@@ -16,6 +16,36 @@ The `@theia/ovsx-client` package is used to interact with `open-vsx` through its
 The package allows clients to fetch extensions and their metadata, search the registry, and
 includes the necessary logic to determine compatibility based on a provided supported API version.
 
+Note that this client only supports a subset of the whole OpenVSX API, only what's relevant to
+clients like Theia applications.
+
+### `OVSXRouterClient`
+
+This class is an `OVSXClient` that can delegate requests to sub-clients based on some configuration (`OVSXRouterConfig`).
+
+```jsonc
+{
+    "registries": {
+        // `[Alias]: URL` pairs to avoid copy pasting URLs down the config
+    },
+    "use": [
+        // List of aliases/URLs to use when no filtering was applied.
+    ],
+    "rules": [
+        {
+            "ifRequestContains": "regex matched against various fields in requests",
+            "ifExtensionIdMatches": "regex matched against the extension id (without version)",
+            "use": [/*
+                List of registries to forward the request to when all the
+                conditions are matched.
+
+                `null` or `[]` means to not forward the request anywhere.
+            */]
+        }
+    ]
+}
+```
+
 ## Additional Information
 
 - [Theia - GitHub](https://github.com/eclipse-theia/theia)

diff --git a/dev-packages/ovsx-client/src/index.ts b/dev-packages/ovsx-client/src/index.ts
@@ -14,5 +14,9 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
 
-export * from './ovsx-client';
+export { OVSXApiFilter, OVSXApiFilterImpl } from './ovsx-api-filter';
+export { OVSXHttpClient } from './ovsx-http-client';
+export { OVSXMockClient } from './ovsx-mock-client';
+export { OVSXRouterClient, OVSXRouterConfig, OVSXRouterFilterFactory as FilterFactory } from './ovsx-router-client';
+export * from './ovsx-router-filters';
 export * from './ovsx-types';