fix: should init binary adapter before reuse it (#393)

app/common/FileUtil.ts

@@ -19,7 +19,7 @@ export async function createTempfile(dataDir: string, filename: string) {
 }
 
 export async function downloadToTempfile(httpclient: EggContextHttpClient,
-  dataDir: string, url: string, ignoreDownloadStatuses?: number[], retries = 3) {
+  dataDir: string, url: string, ignoreDownloadStatuses?: number[] | readonly number[], retries = 3) {
   let lastError: any;
   while (retries > 0) {
     try {
@@ -43,7 +43,7 @@ export interface Tempfile {
   timing: HttpClientResponse['res']['timing'];
 }
 async function _downloadToTempfile(httpclient: EggContextHttpClient,
-  dataDir: string, url: string, ignoreDownloadStatuses?: number[]): Promise<Tempfile> {
+  dataDir: string, url: string, ignoreDownloadStatuses?: number[] | readonly number[]): Promise<Tempfile> {
   const tmpfile = await createTempfile(dataDir, url);
   const writeStream = createWriteStream(tmpfile);
   try {

app/common/adapter/binary/AbstractBinary.ts

@@ -9,7 +9,7 @@ export type BinaryItem = {
   url: string;
   size: string | number;
   date: string;
-  ignoreDownloadStatuses?: number[];
+  ignoreDownloadStatuses?: number[] | readonly number[];
 };
 
 export type FetchResult = {
@@ -26,7 +26,8 @@ export abstract class AbstractBinary {
   @Inject()
   protected httpclient: EggHttpClient;
 
-  abstract fetch(dir: string, binaryName?: BinaryName): Promise<FetchResult | undefined>;
+  abstract init(binaryName: BinaryName): Promise<void>;
+  abstract fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined>;
 
   protected async requestXml(url: string) {
     const { status, data, headers } = await this.httpclient.request(url, {

app/common/adapter/binary/ApiBinary.ts

@@ -6,11 +6,15 @@ import { EggAppConfig } from 'egg';
 @SingletonProto()
 @BinaryAdapter(BinaryType.Api)
 export class ApiBinary extends AbstractBinary {
-
   @Inject()
   private readonly config: EggAppConfig;
 
-  async fetch(dir: string, binaryName?: string): Promise<FetchResult | undefined> {
+  async init() {
+    // do nothing
+    return;
+  }
+
+  async fetch(dir: string, binaryName: string): Promise<FetchResult | undefined> {
     const apiUrl = this.config.cnpmcore.syncBinaryFromAPISource || `${this.config.cnpmcore.sourceRegistry}/-/binary`;
     const url = `${apiUrl}/${binaryName}${dir}`;
     const data = await this.requestJSON(url);

app/common/adapter/binary/BucketBinary.ts

@@ -7,6 +7,11 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Bucket)
 export class BucketBinary extends AbstractBinary {
+  async init() {
+    // do nothing
+    return;
+  }
+
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     // /foo/ => foo/
     const binaryConfig = binaries[binaryName];

app/common/adapter/binary/CypressBinary.ts

@@ -5,9 +5,13 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Cypress)
 export class CypressBinary extends AbstractBinary {
-  private dirItems: {
+  private dirItems?: {
     [key: string]: BinaryItem[];
-  };
+  } | null;
+
+  async init() {
+    this.dirItems = undefined;
+  }
 
   async fetch(dir: string): Promise<FetchResult | undefined> {
     if (!this.dirItems) {

app/common/adapter/binary/ElectronBinary.ts

@@ -1,14 +1,14 @@
 import { SingletonProto } from '@eggjs/tegg';
 import { BinaryType } from '../../enum/Binary';
-import binaries from '../../../../config/binaries';
+import binaries, { BinaryName } from '../../../../config/binaries';
 import { BinaryAdapter, BinaryItem, FetchResult } from './AbstractBinary';
 import { GithubBinary } from './GithubBinary';
 
 @SingletonProto()
 @BinaryAdapter(BinaryType.Electron)
 export class ElectronBinary extends GithubBinary {
-  async fetch(dir: string): Promise<FetchResult | undefined> {
-    const releases = await this.initReleases(binaries.electron);
+  async fetch(dir: string, binaryName: BinaryName = 'electron'): Promise<FetchResult | undefined> {
+    const releases = await this.initReleases(binaryName, binaries.electron);
     if (!releases) return;
 
     let items: BinaryItem[] = [];

app/common/adapter/binary/GithubBinary.ts

@@ -6,10 +6,14 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.GitHub)
 export class GithubBinary extends AbstractBinary {
-  private releases?: any[];
+  private releases: Record<string, any[]> = {};
 
-  protected async initReleases(binaryConfig: BinaryTaskConfig) {
-    if (!this.releases) {
+  async init(binaryName: BinaryName) {
+    delete this.releases[binaryName];
+  }
+
+  protected async initReleases(binaryName: BinaryName, binaryConfig: BinaryTaskConfig) {
+    if (!this.releases[binaryName]) {
       // https://docs.github.com/en/rest/reference/releases get three pages
       // https://api.github.com/repos/electron/electron/releases
       // https://api.github.com/repos/electron/electron/releases?per_page=100&page=3
@@ -29,9 +33,9 @@ export class GithubBinary extends AbstractBinary {
         }
         releases = releases.concat(data);
       }
-      this.releases = releases;
+      this.releases[binaryName] = releases;
     }
-    return this.releases;
+    return this.releases[binaryName];
   }
 
   protected formatItems(releaseItem: any, binaryConfig: BinaryTaskConfig) {
@@ -74,7 +78,7 @@ export class GithubBinary extends AbstractBinary {
 
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     const binaryConfig = binaries[binaryName];
-    const releases = await this.initReleases(binaryConfig);
+    const releases = await this.initReleases(binaryName, binaryConfig);
     if (!releases) return;
 
     let items: BinaryItem[] = [];

app/common/adapter/binary/ImageminBinary.ts

@@ -6,6 +6,10 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Imagemin)
 export class ImageminBinary extends AbstractBinary {
+  async init() {
+    // do nothing
+    return;
+  }
 
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     const binaryConfig = binaries[binaryName];

app/common/adapter/binary/NodeBinary.ts

@@ -6,6 +6,11 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Node)
 export class NodeBinary extends AbstractBinary {
+  async init() {
+    // do nothing
+    return;
+  }
+
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
     const binaryConfig = binaries[binaryName];
     const url = `${binaryConfig.distUrl}${dir}`;
@@ -34,6 +39,7 @@ export class NodeBinary extends AbstractBinary {
         url: fileUrl,
         size,
         date,
+        ignoreDownloadStatuses: binaryConfig.options?.ignoreDownloadStatuses,
       });
     }
     return { items, nextParams: null };

app/common/adapter/binary/NodePreGypBinary.ts

@@ -7,6 +7,10 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.NodePreGyp)
 export class NodePreGypBinary extends AbstractBinary {
+  async init() {
+    // do nothing
+    return;
+  }
 
   // https://github.com/mapbox/node-pre-gyp
   async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {

app/common/adapter/binary/PlaywrightBinary.ts

@@ -150,6 +150,10 @@ const DOWNLOAD_PATHS = {
 @BinaryAdapter(BinaryType.Playwright)
 export class PlaywrightBinary extends AbstractBinary {
   private dirItems?: Record<string, BinaryItem[]>;
+  async init() {
+    this.dirItems = undefined;
+  }
+
   async fetch(dir: string): Promise<FetchResult | undefined> {
     if (!this.dirItems) {
       const packageData = await this.requestJSON(PACKAGE_URL);

app/common/adapter/binary/PuppeteerBinary.ts

@@ -5,10 +5,14 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Puppeteer)
 export class PuppeteerBinary extends AbstractBinary {
-  private dirItems: {
+  private dirItems?: {
     [key: string]: BinaryItem[];
   };
 
+  async init() {
+    this.dirItems = undefined;
+  }
+
   async fetch(dir: string): Promise<FetchResult | undefined> {
     if (!this.dirItems) {
       const pkgUrl = 'https://registry.npmjs.com/puppeteer';

app/common/adapter/binary/SqlcipherBinary.ts

@@ -5,6 +5,10 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Sqlcipher)
 export class SqlcipherBinary extends AbstractBinary {
+  async init() {
+    // do nothing
+    return;
+  }
 
   async fetch(dir: string): Promise<FetchResult | undefined> {
     const dirItems: {

app/core/entity/Binary.ts

@@ -10,7 +10,7 @@ interface BinaryData extends EntityData {
   size: number;
   date: string;
   sourceUrl?: string;
-  ignoreDownloadStatuses?: number[];
+  ignoreDownloadStatuses?: number[] | readonly number[];
 }
 
 export class Binary extends Entity {
@@ -22,7 +22,7 @@ export class Binary extends Entity {
   size: number;
   date: string;
   sourceUrl?: string;
-  ignoreDownloadStatuses?: number[];
+  ignoreDownloadStatuses?: number[] | readonly number[];
 
   constructor(data: BinaryData) {
     super(data);

app/core/service/BinarySyncerService.ts

@@ -193,7 +193,8 @@ export class BinarySyncerService extends AbstractService {
           let localFile = '';
           try {
             const { tmpfile, headers, timing } =
-              await downloadToTempfile(this.httpclient, this.config.dataDir, item.sourceUrl!, item.ignoreDownloadStatuses);
+              await downloadToTempfile(
+                this.httpclient, this.config.dataDir, item.sourceUrl!, item.ignoreDownloadStatuses);
             logs.push(`[${isoNow()}][${dir}] 🟢 [${parentIndex}${index}] HTTP content-length: ${headers['content-length']}, timing: ${JSON.stringify(timing)}, ${item.sourceUrl} => ${tmpfile}`);
             localFile = tmpfile;
             const binary = await this.saveBinaryItem(item, tmpfile);
@@ -280,9 +281,13 @@ export class BinarySyncerService extends AbstractService {
     const config = this.config.cnpmcore;
     const binaryConfig = binaries[binaryName];
 
+    let binaryAdapter: AbstractBinary;
     if (config.sourceRegistryIsCNpm) {
-      return await this.eggObjectFactory.getEggObject(AbstractBinary, BinaryType.Api);
+      binaryAdapter = await this.eggObjectFactory.getEggObject(AbstractBinary, BinaryType.Api);
+    } else {
+      binaryAdapter = await this.eggObjectFactory.getEggObject(AbstractBinary, binaryConfig.type);
     }
-    return await this.eggObjectFactory.getEggObject(AbstractBinary, binaryConfig.type);
+    await binaryAdapter.init(binaryName);
+    return binaryAdapter;
   }
 }

config/binaries.ts

@@ -19,6 +19,8 @@ export type BinaryTaskConfig = {
     npmPackageName?: string;
     // custom for NodePreGypBinary
     requiredNapiVersions?: boolean;
+    // ignore download fail response status
+    ignoreDownloadStatuses?: readonly number[],
   },
   disable?: boolean;
 };
@@ -77,6 +79,10 @@ const binaries = {
     ignoreFiles: [
       '/src/Python-1.6.tar.gz',
     ],
+    options: {
+      // https://www.python.org/ftp/python/3.9.14/Python-3.9.14.tar.xz.sig status(403)
+      ignoreDownloadStatuses: [ 403 ],
+    },
   },
   // CypressBinary
   cypress: {
@@ -882,5 +888,4 @@ const BinaryConfigMap: Record<BinaryName, BinaryTaskConfig> = {
   ...binaries,
 };
 
-
 export default BinaryConfigMap;