feat(major improvements on pp detection and speed): speeding up the p…

…robing functions
acuciureanu · Feb 9, 2023 · a8edb04 · a8edb04
1 parent a1cceb0
commit a8edb04
Show file tree

Hide file tree

Showing 8 changed files with 64 additions and 33 deletions.
diff --git a/index.js b/index.js
@@ -57,7 +57,7 @@ program
         let urls = [];
         rl.on('line', (line) => urls.push(line));
         rl.on('close', async () => {
-            concurrency ? await check.probeAll(urls, concurrency) : await check.probeAll(urls);
+            concurrency || urls.length ? await check.probeAll(urls, concurrency) : await check.probeAll(urls);
             process.exit(0);
         });
     });

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -19,7 +19,7 @@
         "@supercharge/promise-pool": "^2.3.2",
         "axios": "^1.2.0",
         "commander": "^10.0.0",
-        "puppeteer": "^19.3.0",
+        "puppeteer": "^19.6.3",
         "semantic-release": "^19.0.5"
     },
     "devDependencies": {
@@ -38,4 +38,4 @@
             "path": "./node_modules/cz-conventional-changelog"
         }
     }
-}
+}
diff --git a/results.txt b/results.txt
diff --git a/sandbox/index.html b/sandbox/index.html
@@ -1,5 +1,34 @@
 <!DOCTYPE html>
 
 <html>
-    <script id="target"></script>
-</html>
+<script id="payload">
+    const types = [Object, String, Number, Array, Function, Boolean];
+
+    const prototypesPropertiesReducer = (acc, type) => ({
+        ...acc,
+        [type.name]: Object.getOwnPropertyNames(type.prototype),
+    });
+
+    const prototypePropertyNames = () => types.reduce(prototypesPropertiesReducer, {});
+
+    const probe = () =>
+        Object.keys(prototypePropertyNames()).reduce((acc, key) => {
+            for (let propKey of prototypePropertyNames()[key]) {
+                const payload = `${key}.prototype.${propKey}`;
+                try {
+                    const propValue = eval(payload);
+
+                    if (typeof propValue === 'function' && !propValue.toString().includes('[native code]')) {
+                        // Check if property is user defined and not native
+                        acc.push(payload);
+                    }
+                } catch (e) {
+                    // Nothing to catch now.
+                }
+            }
+            return acc;
+        }, []);
+</script>
+<script id="target" src="LIBRARY"></script>
+
+</html>
diff --git a/sandbox/js/check.payload.js b/sandbox/js/check.payload.js
@@ -5,23 +5,21 @@ const prototypesPropertiesReducer = (acc, type) => ({
     [type.name]: Object.getOwnPropertyNames(type.prototype),
 });
 
-const prototypePropertyNames = types.reduce(prototypesPropertiesReducer, {});
+const prototypePropertyNames = () => types.reduce(prototypesPropertiesReducer, {});
 
 const probe = () =>
-    Object.keys(prototypePropertyNames).reduce((acc, key) => {
-        for (let propKey of prototypePropertyNames[key]) {
+    Object.keys(prototypePropertyNames()).reduce((acc, key) => {
+        for (let propKey of prototypePropertyNames()[key]) {
             const payload = `${key}.prototype.${propKey}`;
             try {
                 const propValue = eval(payload);
                 // Check if property is user defined and not native
-                if (typeof propValue === 'function' && propValue.toString().indexOf('[native code]') === -1) {
+                if (typeof propValue === 'function' && !propValue.toString().includes('[native code]')) {
                     acc.push(payload);
                 }
             } catch (e) {
                 // Nothing to catch now.
             }
         }
         return acc;
-    }, []);
-
-probe();
+    }, []);
diff --git a/services/cdnjs.service.js b/services/cdnjs.service.js
@@ -11,34 +11,23 @@ const client = axios.create({
     timeout: 5000,
 });
 
-const sandboxHtml = load('../sandbox/index.html');
+const createSandbox = (library) => load('../sandbox/index.html').replace('LIBRARY', library);
 
 const getLibraries = async () =>
     client
         .get('/libraries')
-        .then((response) => response.data.results.filter((result) => result.latest !== null))
+        .then((response) => response.data.results.filter((result) => result.latest !== null && result.latest.endsWith(".js")))
         .catch((error) => console.log(error));
 
 const probe = async (library) => {
-    const pageLoadConfig = { waitUntil: 'networkidle0' };
-
     const page = await browser.newPage();
 
-    await page.setJavaScriptEnabled(false);
-    await page.goto(`data:text/html,${sandboxHtml}`, pageLoadConfig);
-
-    const injectedHtml = await page.evaluate((library) => {
-        document.querySelector('#target').setAttribute('src', `${library.latest}`);
-        return document.documentElement.outerHTML;
-    }, library);
-
     await page.setJavaScriptEnabled(true);
+    await page.goto(`data:text/html,${createSandbox(library.latest)}`, { waitUntil: 'networkidle0' });
 
-    await page.goto(`data:text/html,${injectedHtml}`, pageLoadConfig);
+    const results = await page.evaluate(() => probe());
 
-    const results = await page.evaluate('probe()');
-
-    await page.close();
+    await page.close({ runBeforeUnload: true });
 
     return { name: library.name, url: library.latest, findings: results };
 };
@@ -51,7 +40,15 @@ const probeAll = async (concurrency) => {
             const stats = `[${pool.processedCount()}/${libraries.length} | ${pool.processedPercentage().toFixed(2)}%]`;
             console.log(`${stats} Processed ${library.latest} ...`);
         })
-        .process(probe);
+        .process(async (library) => {
+            const result = await probe(library);
+
+            if (result.findings.length > 0) {
+                console.log(JSON.stringify(result, null, 2));
+            }
+
+            return result;
+        });
 
     log(results);
     browser.close();

diff --git a/services/check.service.js b/services/check.service.js
@@ -2,22 +2,29 @@ import { load, log } from '../utils/file.utils.js';
 import puppeteer from 'puppeteer';
 import { PromisePool } from '@supercharge/promise-pool';
 
-const browser = await puppeteer.launch({ headless: true });
-
 const payload = load('../sandbox/js/check.payload.js');
 
+const browser = await puppeteer.launch({ headless: false });
+
 const probe = async (pageUrl) => {
     const page = await browser.newPage();
 
     await page.goto(pageUrl, { waitUntil: 'networkidle0' });
 
-    const results = await page.evaluate(payload);
+    await page.evaluate((payload) => {
+        const script = document.createElement('script');
+        script.innerHTML = payload;
+        document.head.appendChild(script);
+    }, payload);
+
+    const results = await page.evaluate(() => probe());
 
     await page.close();
 
     return { url: pageUrl, findings: results };
 };
 
+
 const probeAll = async (urls, concurrency = 10) => {
     const { results } = await PromisePool.for(urls)
         .withConcurrency(concurrency)