From 05301166a2aa1819429f10a60652e9b7b6eb93c6 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Sat, 18 May 2024 13:11:22 +0800 Subject: [PATCH] feat: add unpkg-white-list to detect sync unpkg files or not (#686) see https://github.com/cnpm/unpkg-white-list ## Summary by CodeRabbit - **New Features** - Introduced a new configuration option `enableSyncUnpkgFilesWhiteList` to enhance package version file synchronization. - **Improvements** - Enhanced logging in package version file operations for better traceability. - Simplified file redirection logic for improved performance and readability. - **Tests** - Added test cases for the new `enableSyncUnpkgFilesWhiteList` configuration to ensure reliability. --- app.ts | 1 + app/core/event/SyncPackageVersionFile.ts | 17 +- app/core/service/PackageVersionFileService.ts | 69 +++++- app/port/config.ts | 4 + .../PackageVersionFileController.ts | 3 - .../package/SavePackageVersionController.ts | 3 +- config/config.default.ts | 1 + .../listFiles.test.ts | 215 +++++++++++++++++- 8 files changed, 303 insertions(+), 10 deletions(-) diff --git a/app.ts b/app.ts index fbacfb46..75014045 100644 --- a/app.ts +++ b/app.ts @@ -2,6 +2,7 @@ import path from 'path'; import { readFile } from 'fs/promises'; import { Application } from 'egg'; import { ChangesStreamService } from './app/core/service/ChangesStreamService'; + declare module 'egg' { interface Application { binaryHTML: string; diff --git a/app/core/event/SyncPackageVersionFile.ts b/app/core/event/SyncPackageVersionFile.ts index 5dc7d20d..83519dad 100644 --- a/app/core/event/SyncPackageVersionFile.ts +++ b/app/core/event/SyncPackageVersionFile.ts @@ -1,7 +1,8 @@ import { Event, Inject } from '@eggjs/tegg'; import { - EggAppConfig, + EggAppConfig, EggLogger, } from 'egg'; +import { ForbiddenError } from 'egg-errors'; import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index'; import { getScopeAndName } from '../../common/PackageUtil'; import { PackageManagerService } from '../service/PackageManagerService'; @@ -11,6 +12,8 @@ class SyncPackageVersionFileEvent { @Inject() protected readonly config: EggAppConfig; @Inject() + protected readonly logger: EggLogger; + @Inject() private readonly packageManagerService: PackageManagerService; @Inject() private readonly packageVersionFileService: PackageVersionFileService; @@ -25,7 +28,17 @@ class SyncPackageVersionFileEvent { const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag( scope, name, version); if (!packageVersion) return; - await this.packageVersionFileService.syncPackageVersionFiles(packageVersion); + try { + await this.packageVersionFileService.syncPackageVersionFiles(packageVersion); + } catch (err) { + if (err instanceof ForbiddenError) { + this.logger.info('[SyncPackageVersionFileEvent.syncPackageVersionFile] ignore sync files, cause: %s', + err.message, + ); + return; + } + throw err; + } } protected async syncPackageReadmeToLatestVersion(fullname: string) { diff --git a/app/core/service/PackageVersionFileService.ts b/app/core/service/PackageVersionFileService.ts index 2d71b067..76eb1413 100644 --- a/app/core/service/PackageVersionFileService.ts +++ b/app/core/service/PackageVersionFileService.ts @@ -7,27 +7,34 @@ import { SingletonProto, Inject, } from '@eggjs/tegg'; +import { ConflictError, ForbiddenError } from 'egg-errors'; +import semver from 'semver'; import { AbstractService } from '../../common/AbstractService'; import { calculateIntegrity, + getFullname, } from '../../common/PackageUtil'; import { createTempDir, mimeLookup } from '../../common/FileUtil'; import { PackageRepository, } from '../../repository/PackageRepository'; import { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository'; +import { PackageVersionRepository } from '../../repository/PackageVersionRepository'; import { DistRepository } from '../../repository/DistRepository'; import { PackageVersionFile } from '../entity/PackageVersionFile'; import { PackageVersion } from '../entity/PackageVersion'; import { Package } from '../entity/Package'; import { PackageManagerService } from './PackageManagerService'; import { CacheAdapter } from '../../common/adapter/CacheAdapter'; -import { ConflictError } from 'egg-errors'; + +const unpkgWhiteListUrl = 'https://github.com/cnpm/unpkg-white-list'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) export class PackageVersionFileService extends AbstractService { + @Inject() + private readonly packageVersionRepository: PackageVersionRepository; @Inject() private readonly packageRepository: PackageRepository; @Inject() @@ -39,6 +46,12 @@ export class PackageVersionFileService extends AbstractService { @Inject() private readonly cacheAdapter: CacheAdapter; + #unpkgWhiteListCurrentVersion: string = ''; + #unpkgWhiteListAllowPackages: Record = {}; + #unpkgWhiteListAllowScopes: string[] = []; + async listPackageVersionFiles(pkgVersion: PackageVersion, directory: string) { await this.#ensurePackageVersionFilesSync(pkgVersion); return await this.packageVersionFileRepository.listPackageVersionFiles(pkgVersion.packageVersionId, directory); @@ -54,16 +67,58 @@ export class PackageVersionFileService extends AbstractService { async #ensurePackageVersionFilesSync(pkgVersion: PackageVersion) { const hasFiles = await this.packageVersionFileRepository.hasPackageVersionFiles(pkgVersion.packageVersionId); if (!hasFiles) { - const lockRes = await this.cacheAdapter.usingLock(`${pkgVersion.packageVersionId}:syncFiles`, 60, async () => { + const lockName = `${pkgVersion.packageVersionId}:syncFiles`; + const lockRes = await this.cacheAdapter.usingLock(lockName, 60, async () => { await this.syncPackageVersionFiles(pkgVersion); }); // lock fail if (!lockRes) { - this.logger.warn('[package:version:syncPackageVersionFiles] check lock fail'); + this.logger.warn('[package:version:syncPackageVersionFiles] check lock:%s fail', lockName); throw new ConflictError('Package version file sync is currently in progress. Please try again later.'); } } + } + + async #updateUnpkgWhiteList() { + if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return; + const whiteListScope = ''; + const whiteListPackageName = 'unpkg-white-list'; + const whiteListPackageVersion = await this.packageVersionRepository.findVersionByTag( + whiteListScope, whiteListPackageName, 'latest'); + if (!whiteListPackageVersion) return; + // same version, skip update for performance + if (this.#unpkgWhiteListCurrentVersion === whiteListPackageVersion) return; + + // update the new version white list + const { manifest } = await this.packageManagerService.showPackageVersionManifest( + whiteListScope, whiteListPackageName, whiteListPackageVersion, false, true); + if (!manifest) return; + this.#unpkgWhiteListCurrentVersion = manifest.version; + this.#unpkgWhiteListAllowPackages = manifest.allowPackages ?? {} as any; + this.#unpkgWhiteListAllowScopes = manifest.allowScopes ?? [] as any; + this.logger.info('[PackageVersionFileService.updateUnpkgWhiteList] version:%s, total %s packages, %s scopes', + whiteListPackageVersion, + Object.keys(this.#unpkgWhiteListAllowPackages).length, + this.#unpkgWhiteListAllowScopes.length, + ); + } + async #checkPackageVersionInUnpkgWhiteList(pkgScope: string, pkgName: string, pkgVersion: string) { + if (!this.config.cnpmcore.enableSyncUnpkgFilesWhiteList) return; + await this.#updateUnpkgWhiteList(); + + // check allow scopes + if (this.#unpkgWhiteListAllowScopes.includes(pkgScope)) return; + + // check allow packages + const fullname = getFullname(pkgScope, pkgName); + const pkgConfig = this.#unpkgWhiteListAllowPackages[fullname]; + if (!pkgConfig) { + throw new ForbiddenError(`"${fullname}" is not allow to unpkg files, see ${unpkgWhiteListUrl}`); + } + if (!pkgConfig.version || !semver.satisfies(pkgVersion, pkgConfig.version)) { + throw new ForbiddenError(`"${fullname}@${pkgVersion}" not satisfies "${pkgConfig.version}" to unpkg files, see ${unpkgWhiteListUrl}`); + } } // ๅŸบไบŽ latest version ๅŒๆญฅ package readme @@ -113,8 +168,16 @@ export class PackageVersionFileService extends AbstractService { async syncPackageVersionFiles(pkgVersion: PackageVersion) { const files: PackageVersionFile[] = []; + // must set enableUnpkg and enableSyncUnpkgFiles = true both + if (!this.config.cnpmcore.enableUnpkg) return files; + if (!this.config.cnpmcore.enableSyncUnpkgFiles) return files; + const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId); if (!pkg) return files; + + // check unpkg white list + await this.#checkPackageVersionInUnpkgWhiteList(pkg.scope, pkg.name, pkgVersion.version); + const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`; const tmpdir = await createTempDir(this.config.dataDir, dirname); const tarFile = `${tmpdir}.tgz`; diff --git a/app/port/config.ts b/app/port/config.ts index 867f2424..2fbd80e1 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -149,6 +149,10 @@ export type CnpmcoreConfig = { * enable sync unpkg files */ enableSyncUnpkgFiles: boolean; + /** + * enable sync unpkg files from the white list, https://github.com/cnpm/unpkg-white-list + */ + enableSyncUnpkgFilesWhiteList: boolean; /** * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag. * in most cases, you should set to false to keep the same behavior as source registry. diff --git a/app/port/controller/PackageVersionFileController.ts b/app/port/controller/PackageVersionFileController.ts index 6e918086..6a65c29b 100644 --- a/app/port/controller/PackageVersionFileController.ts +++ b/app/port/controller/PackageVersionFileController.ts @@ -149,12 +149,9 @@ export class PackageVersionFileController extends AbstractController { if (!file) { const possibleFile = await this.#searchPossibleEntries(packageVersion, path); - if (possibleFile) { const route = `/${fullname}/${versionSpec}/files${possibleFile.path}${hasMeta ? '?meta' : ''}`; - ctx.redirect(route); - return; } diff --git a/app/port/controller/package/SavePackageVersionController.ts b/app/port/controller/package/SavePackageVersionController.ts index b7e43c61..e8214a81 100644 --- a/app/port/controller/package/SavePackageVersionController.ts +++ b/app/port/controller/package/SavePackageVersionController.ts @@ -221,6 +221,7 @@ export class SavePackageVersionController extends AbstractController { const registry = await this.registryManagerService.ensureSelfRegistry(); let packageVersionEntity: PackageVersionEntity | undefined; + const lockName = `${pkg.name}:publish`; const lockRes = await this.cacheAdapter.usingLock(`${pkg.name}:publish`, 60, async () => { packageVersionEntity = await this.packageManagerService.publish({ scope, @@ -240,7 +241,7 @@ export class SavePackageVersionController extends AbstractController { // lock fail if (!lockRes) { - this.logger.warn('[package:version:add] check lock fail'); + this.logger.warn('[package:version:add] check lock:%s fail', lockName); throw new ConflictError('Unable to create the publication lock, please try again later.'); } diff --git a/config/config.default.ts b/config/config.default.ts index 1c090583..6ef39cd3 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -54,6 +54,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { redirectNotFound: true, enableUnpkg: true, enableSyncUnpkgFiles: true, + enableSyncUnpkgFilesWhiteList: false, strictSyncSpecivicVersion: false, enableElasticsearch: !!process.env.CNPMCORE_CONFIG_ENABLE_ES, elasticsearchIndex: 'cnpmcore_packages', diff --git a/test/port/controller/PackageVersionFileController/listFiles.test.ts b/test/port/controller/PackageVersionFileController/listFiles.test.ts index 8c27596e..5d1365d8 100644 --- a/test/port/controller/PackageVersionFileController/listFiles.test.ts +++ b/test/port/controller/PackageVersionFileController/listFiles.test.ts @@ -5,7 +5,6 @@ import { TestUtil } from '../../../../test/TestUtil'; import { PackageVersionFileService } from '../../../../app/core/service/PackageVersionFileService'; import { calculateIntegrity } from '../../../../app/common/PackageUtil'; - describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', () => { let publisher; let adminUser; @@ -354,6 +353,7 @@ describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', assert.equal(called, 1); assert.equal(resList.filter(res => res.status === 409 && res.body.error === '[CONFLICT] Package version file sync is currently in progress. Please try again later.').length, 1); }); + it('should redirect to possible entry', async () => { const tarball = await TestUtil.readFixturesFile('@cnpm/cnpm-test-find-entry-1.0.0.tgz'); const { integrity } = await calculateIntegrity(tarball); @@ -399,5 +399,218 @@ describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', .expect(302) .expect('location', `/${pkg.name}/1.0.0/files/es/json/index.json`); }); + + describe('enableSyncUnpkgFilesWhiteList = true', () => { + it('should 403 package name not in white list', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + + const pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + const res = await app.httpRequest() + .get('/foo/1.0.0/files/index.js') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 403); + assert.equal(res.body.error, '[FORBIDDEN] "foo" is not allow to unpkg files, see https://github.com/cnpm/unpkg-white-list'); + }); + + it('should 403 package version not match', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + + let pkg = await TestUtil.getFullPackage({ + name: 'unpkg-white-list', + version: '0.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + allowPackages: { + foo: { + version: '0.0.0', + }, + }, + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + const res = await app.httpRequest() + .get('/foo/1.0.0/files/index.js') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 403); + assert.equal(res.body.error, '[FORBIDDEN] "foo@1.0.0" not satisfies "0.0.0" to unpkg files, see https://github.com/cnpm/unpkg-white-list'); + }); + + it('should 200 when scope in white list', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + + let pkg = await TestUtil.getFullPackage({ + name: 'unpkg-white-list', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + allowScopes: [ '@cnpm' ], + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + pkg = await TestUtil.getFullPackage({ + name: '@cnpm/foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + const res = await app.httpRequest() + .get('/@cnpm/foo/1.0.0/files/package.json') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 200); + assert(res.body.name); + }); + + it('should 200 when package version in white list', async () => { + mock(app.config.cnpmcore, 'allowPublishNonScopePackage', true); + mock(app.config.cnpmcore, 'enableUnpkg', true); + mock(app.config.cnpmcore, 'enableSyncUnpkgFilesWhiteList', true); + + let pkg = await TestUtil.getFullPackage({ + name: 'unpkg-white-list', + version: '2.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + allowScopes: [ '@cnpm' ], + allowPackages: { + foo: { + version: '*', + }, + }, + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.0', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + + let res = await app.httpRequest() + .get('/foo/1.0.0/files/package.json') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 200); + assert(res.body.name); + + pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.1', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + res = await app.httpRequest() + .get('/foo/1.0.1/files/package.json') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 200); + assert(res.body.name); + + // unpkg-white-list change + pkg = await TestUtil.getFullPackage({ + name: 'unpkg-white-list', + version: '2.0.1', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + allowScopes: [ '@cnpm' ], + allowPackages: { + foo: { + version: '3', + }, + }, + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + pkg = await TestUtil.getFullPackage({ + name: 'foo', + version: '1.0.2', + versionObject: { + description: 'work with utf8mb4 ๐Ÿ’ฉ, ๐Œ† utf8_unicode_ci, foo๐Œ†bar ๐Ÿป', + }, + }); + await app.httpRequest() + .put(`/${pkg.name}`) + .set('authorization', publisher.authorization) + .set('user-agent', publisher.ua) + .send(pkg) + .expect(201); + + res = await app.httpRequest() + .get('/foo/1.0.2/files/package.json') + .expect('content-type', 'application/json; charset=utf-8'); + assert.equal(res.status, 403); + assert.equal(res.body.error, '[FORBIDDEN] "foo@1.0.2" not satisfies "3" to unpkg files, see https://github.com/cnpm/unpkg-white-list'); + }); + }); }); });