perf(added improvements for speed and reliability): improvements

These changes fix a couple of performance and reliability problems related to mass probing the cdnjs latest libraries also they bring speed improvements because of the recent changes from the dependencies.
acuciureanu · May 19, 2023 · 5f24821 · 5f24821
1 parent 2f8a270
commit 5f24821
Show file tree

Hide file tree

Showing 5 changed files with 123 additions and 80 deletions.
diff --git a/index.js b/index.js
@@ -57,8 +57,15 @@ program
         let urls = [];
         rl.on('line', (line) => urls.push(line));
         rl.on('close', async () => {
-            concurrency || urls.length ? await check.probeAll(urls, concurrency) : await check.probeAll(urls);
-            process.exit(0);
+            try {
+                concurrency || urls.length ? await check.probeAll(urls, concurrency) : await check.probeAll(urls);
+            } catch (error) {
+                console.error('Error during processing:', error);
+            } finally {
+                process.on('exit', () => {
+                    console.log('Done.');
+                });
+            }
         });
     });
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -16,10 +16,10 @@
     },
     "type": "module",
     "dependencies": {
-        "@supercharge/promise-pool": "^2.3.2",
-        "axios": "^1.2.0",
-        "commander": "^10.0.0",
-        "puppeteer": "^20.2.1",
+        "@supercharge/promise-pool": "^2.4.0",
+        "axios": "^1.4.0",
+        "commander": "^10.0.1",
+        "puppeteer": "^19.8.0",
         "semantic-release": "^21.0.0"
     },
     "devDependencies": {
@@ -38,4 +38,4 @@
             "path": "./node_modules/cz-conventional-changelog"
         }
     }
-}
+}
diff --git a/services/cdnjs.service.js b/services/cdnjs.service.js
@@ -4,8 +4,6 @@ import { load, log } from '../utils/file.utils.js';
 import puppeteer from 'puppeteer';
 import { PromisePool } from '@supercharge/promise-pool';
 
-const browser = await puppeteer.launch({ headless: true });
-
 const client = axios.create({
     baseURL: config.cdnjs.api.url,
     timeout: 5000,
@@ -23,7 +21,7 @@ const getLibraries = async () =>
         )
         .catch((error) => console.log(error));
 
-const probe = async (library) => {
+const probe = async (browser, library) => {
     const page = await browser.newPage();
 
     await page.setJavaScriptEnabled(true);
@@ -44,26 +42,37 @@ const probe = async (library) => {
 };
 
 const probeAll = async (concurrency) => {
+    const browser = await puppeteer.launch({ headless: 'new' });
+
     const libraries = await getLibraries();
-    const { results } = await PromisePool.withConcurrency(concurrency)
-        .for(libraries)
-        .onTaskFinished((library, pool) => {
-            const stats = `[${pool.processedCount()}/${libraries.length} | ${pool.processedPercentage().toFixed(2)}%]`;
-            console.log(`${stats} Processed ${library.latest} ...`);
-        })
-        .process(async (library) => {
-            const result = await probe(library);
-
-            if (result.findings.length > 0) {
-                console.log(JSON.stringify(result, null, 2));
+    try {
+        const { results } = await PromisePool.withConcurrency(concurrency)
+            .for(libraries)
+            .onTaskFinished((library, pool) => {
+                const stats = `[${pool.processedCount()}/${libraries.length} | ${pool.processedPercentage().toFixed(2)}%]`;
+                console.log(`${stats} Processed ${library.latest} ...`);
+            })
+            .process(async (library) => {
+                const result = await probe(browser, library);
+
+                if (result.findings.length > 0) {
+                    console.log(JSON.stringify(result, null, 2));
+                }
+
+                return result;
+            });
+
+        log(results);
+        return results;
+    } finally {
+        const pages = await browser.pages();
+        for (const page of pages) {
+            if (!page.isClosed()) {
+                await page.close();
             }
-
-            return result;
-        });
-
-    log(results);
-    browser.close();
-    return results;
+        }
+        await browser.close();
+    }
 };
 
 export default { probeAll };
diff --git a/services/check.service.js b/services/check.service.js
@@ -4,8 +4,6 @@ import { PromisePool } from '@supercharge/promise-pool';
 
 const payload = load('../sandbox/js/check.payload.js');
 
-const browser = await puppeteer.launch({ headless: true });
-
 const probe = async (pageUrl) => {
     const page = await browser.newPage();
 
@@ -25,14 +23,25 @@ const probe = async (pageUrl) => {
 };
 
 const probeAll = async (urls, concurrency = 10) => {
+    const browser = await puppeteer.launch({ headless: "new" });
+    try {
     const { results } = await PromisePool.for(urls)
         .withConcurrency(concurrency)
         .onTaskFinished((url, pool) => console.log(`[${pool.processedPercentage().toFixed(2)}%] Processed ${url} ...`))
         .process(probe);
 
     log(results);
-    browser.close();
     return results;
+    } finally {
+        const pages = await browser.pages();
+        for (const page of pages) {
+            if (!page.isClosed()) {
+                await page.close();
+            }
+        }
+        await browser.close();
+    }
+    browser.close();
 };
 
 export default { probeAll };