From a74ace8dd1c8a50b5c6163bba7a9e8973dd27d0a Mon Sep 17 00:00:00 2001 From: Faisal Salman Date: Sat, 22 Jun 2024 00:13:13 +0700 Subject: [PATCH] Add Yahoo! Japan to Crawler, Fetcher, & InApp browser --- src/extensions/ua-parser-extensions.js | 100 ++++++++++++------------- src/helpers/ua-parser-helpers.mjs | 2 +- test/specs/browser-crawlers.json | 10 +++ 3 files changed, 61 insertions(+), 51 deletions(-) diff --git a/src/extensions/ua-parser-extensions.js b/src/extensions/ua-parser-extensions.js index 84d391f1..bd0ee405 100644 --- a/src/extensions/ua-parser-extensions.js +++ b/src/extensions/ua-parser-extensions.js @@ -39,43 +39,40 @@ const CLIs = Object.freeze({ const Crawlers = Object.freeze({ browser : [ - // Amazonbot - https://developer.amazon.com/amazonbot - // Applebot - http://apple.com/go/applebot - // Bingbot - http://www.bing.com/bingbot.htm - // DuckDuckBot - http://duckduckgo.com/duckduckbot.html - // FacebookBot - https://developers.facebook.com/docs/sharing/bot/ - // GPTBot - https://platform.openai.com/docs/gptbot - [/((?:amazon|apple|bing|duckduck|facebook|gpt)bot)\/([\w\.]+)/i], - [NAME, VERSION, [TYPE, CRAWLER]], + [ + // Amazonbot - https://developer.amazon.com/amazonbot + // Applebot - http://apple.com/go/applebot + // Bingbot - http://www.bing.com/bingbot.htm + // DuckDuckBot - http://duckduckgo.com/duckduckbot.html + // FacebookBot - https://developers.facebook.com/docs/sharing/bot/ + // GPTBot - https://platform.openai.com/docs/gptbot + /((?:amazon|apple|bing|duckduck|facebook|gpt)bot)\/([\w\.]+)/i, - // Baiduspider https://help.baidu.com/question?prod_id=99&class=0&id=3001 - [/(baiduspider)[-imagevdonsfcpr]{0,6}\/([\w\.]+)/i], - [NAME, VERSION, [TYPE, CRAWLER]], + // Baiduspider https://help.baidu.com/question?prod_id=99&class=0&id=3001 + /(baiduspider)[-imagevdonsfcpr]{0,6}\/([\w\.]+)/i, - // Bytespider - // Yahoo! Slurp - http://help.yahoo.com/help/us/ysearch/slurp - [/((?:bytespider|(?=yahoo! )slurp))/i], - [NAME, [TYPE, CRAWLER]], + // ClaudeBot + /(claude(?:bot|-web))\/([\w\.]+)/i, - // ClaudeBot - [/(claude(?:bot|-web))\/([\w\.]+)/i], - [NAME, VERSION, [TYPE, CRAWLER]], + // Googlebot - http://www.google.com/bot.html + /(google(?:bot|other)(?:-image|-video|-news|-extended)?|(?:storebot-)?google(?:-inspectiontool)?)\/?([\w\.]*)/i, + + // Sogou Spider + /(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i, + + // Yahoo! Japan - https://support.yahoo-net.jp/PccSearch/s/article/H000007955 + /(y!?j-(?:asr|br[uw]|dscv|mmp|vsidx|wsc))\/([\w\.]+)/i, + + // Yandex Bots - https://yandex.com/bots + /(yandex(?:(?:mobile)?(?:accessibility|additional|renderresources|screenshot|sprav)?bot|image(?:s|resizer)|video(?:parser)?|blogs|adnet|favicons|fordomain|market|media|metrika|news|ontodb(?:api)?|pagechecker|partner|rca|tracker|turbo|vertis|webmaster|antivirus))\/([\w\.]+)/i + ], - // Googlebot - http://www.google.com/bot.html - [ - /(google(?:bot|other)(?:-image|-video|-news|-extended)?|(?:storebot-)?google(?:-inspectiontool)?)\/?([\w\.]*)/i - ], - [NAME, VERSION, [TYPE, CRAWLER]], - - // Sogou Spider - [/(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i], [NAME, VERSION, [TYPE, CRAWLER]], - // Yandex Bots - https://yandex.com/bots - [ - /(yandex(?:(?:mobile)?(?:accessibility|additional|renderresources|screenshot|sprav)?bot|image(?:s|resizer)|video(?:parser)?|blogs|adnet|favicons|fordomain|market|media|metrika|news|ontodb(?:api)?|pagechecker|partner|rca|tracker|turbo|vertis|webmaster|antivirus))\/([\w\.]+)/i - ], - [NAME, VERSION, [TYPE, CRAWLER]] + // Bytespider + // Yahoo! Slurp - http://help.yahoo.com/help/us/ysearch/slurp + [/((?:bytespider|(?=yahoo! )slurp))/i], + [NAME, [TYPE, CRAWLER]] ] }); @@ -176,30 +173,29 @@ const Emails = Object.freeze({ const Fetchers = Object.freeze({ browser : [ - // ChatGPT-User - https://platform.openai.com/docs/plugins/bot - // BingPreview / Mastodon / Pinterestbot / Redditbot / Telegrambot / Twitterbot - [/(bingpreview|chatgpt-user|mastodon|(?:discord|linkedin|pinterest|reddit|telegram|twitter)bot)\/([\w\.]+)/i], - [NAME, VERSION, [TYPE, FETCHER]], - - // Google Bots / Snapchat - [/(feedfetcher-google|google-read-aloud|(?=bot; )snapchat)/i], - [NAME, [TYPE, FETCHER]], + [ + // ChatGPT-User - https://platform.openai.com/docs/plugins/bot + // BingPreview / Mastodon / Pinterestbot / Redditbot / Telegrambot / Twitterbot + /(bingpreview|chatgpt-user|mastodon|(?:discord|linkedin|pinterest|reddit|telegram|twitter)bot)\/([\w\.]+)/i, + // Slackbot - https://api.slack.com/robots + /(slack(?:bot)?(?:-imgproxy|-linkexpanding)?) ([\w\.]+)/i, + + // WhatsApp + /(whatsapp)\/([\w\.]+)[\/ ][ianw]/i, - // Slackbot - https://api.slack.com/robots - [/(slack(?:bot)?(?:-imgproxy|-linkexpanding)?) ([\w\.]+)/i], - [NAME, VERSION, [TYPE, FETCHER]], + // Yahoo! Japan + /(y!?j-dlc)\/([\w\.]+)/i, - // WhatsApp - [/(whatsapp)\/([\w\.]+)[\/ ][ianw]/i], - [NAME, VERSION, [TYPE, FETCHER]], - - // Yandex Bots - https://yandex.com/bots - [ + // Yandex Bots - https://yandex.com/bots /(yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, /(yandex(?:sitelinks|userproxy))/i ], - [NAME, VERSION, [TYPE, FETCHER]] + [NAME, VERSION, [TYPE, FETCHER]], + + // Google Bots / Snapchat + [/(feedfetcher-google|google-read-aloud|(?=bot; )snapchat)/i], + [NAME, [TYPE, FETCHER]], ] }); @@ -209,7 +205,11 @@ const Fetchers = Object.freeze({ const InApps = Object.freeze({ browser : [ - [/chatlyio\/([\d\.]+)/i], [VERSION, 'Slack', [TYPE, INAPP]] + // Slack + [/chatlyio\/([\d\.]+)/i], [VERSION, 'Slack', [TYPE, INAPP]], + + // Yahoo! Japan + [/jp\.co\.yahoo\.android\.yjtop\/([\d\.]+)/i], [VERSION, 'Yahoo! Japan', [TYPE, INAPP]] ] }); diff --git a/src/helpers/ua-parser-helpers.mjs b/src/helpers/ua-parser-helpers.mjs index cc3533d2..95d6c0f1 100644 --- a/src/helpers/ua-parser-helpers.mjs +++ b/src/helpers/ua-parser-helpers.mjs @@ -11,7 +11,7 @@ /*jshint esversion: 6 */ -import { CPU, OS, Engine } from '../enums/ua-parser-enums'; +import { CPU, OS, Engine } from '../enums/ua-parser-enums.mjs'; const isAppleSilicon = (res) => res.os.is(OS.MACOS) && res.cpu.is(CPU.ARM); diff --git a/test/specs/browser-crawlers.json b/test/specs/browser-crawlers.json index b72beb28..107ab24b 100644 --- a/test/specs/browser-crawlers.json +++ b/test/specs/browser-crawlers.json @@ -79,6 +79,16 @@ "type" : "crawler" } }, + { + "desc" : "Yahoo! Japan", + "ua" : "Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)", + "expect" : + { + "name" : "Y!J-BRW", + "version" : "1.0", + "type" : "crawler" + } + }, { "desc" : "YandexBot", "ua" : "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",