From 8309f29c8691f1dde45f688b88d683842a00fca7 Mon Sep 17 00:00:00 2001 From: nouchy <33549554+nouchy@users.noreply.github.com> Date: Tue, 28 May 2024 20:17:50 +0200 Subject: [PATCH] Sirdata RTD Module : add eids and post content support and get ready for PBJS 9.0 (#11524) * Add eids and post content suport * multi-character sanitization Modify regular expression to match comments containing newlines. * recursively removing elements to avoid an HTML element injection vulnerability * fix expression vs expected assignment * fix return statement for loop * run completeSanitization instead of sanitizeContent * fix multi-character sanitization * Update sirdataRtdProvider.js * fix missing moduleConfig param * Add filtering eids based on whitelist * fixe default Eids --- modules/sirdataRtdProvider.js | 779 ++++++++++++++----- modules/sirdataRtdProvider.md | 216 +++-- test/spec/modules/sirdataRtdProvider_spec.js | 155 +++- 3 files changed, 827 insertions(+), 323 deletions(-) diff --git a/modules/sirdataRtdProvider.js b/modules/sirdataRtdProvider.js index 29d10a3a071..0ad3b891c40 100644 --- a/modules/sirdataRtdProvider.js +++ b/modules/sirdataRtdProvider.js @@ -7,21 +7,35 @@ * @module modules/sirdataRtdProvider * @requires module:modules/realTimeData */ -import {deepAccess, deepSetValue, isEmpty, logError, logInfo, mergeDeep} from '../src/utils.js'; -import {submodule} from '../src/hook.js'; -import {ajax} from '../src/ajax.js'; -import {findIndex} from '../src/polyfill.js'; -import {getRefererInfo} from '../src/refererDetection.js'; -import {config} from '../src/config.js'; -import {getGlobal} from '../src/prebidGlobal.js'; +import adapterManager from '../src/adapterManager.js'; +import { ajax } from '../src/ajax.js'; +import { + deepAccess, checkCookieSupport, deepSetValue, hasDeviceAccess, inIframe, isEmpty, + logError, logInfo, mergeDeep +} from '../src/utils.js'; +import { findIndex } from '../src/polyfill.js'; +import { getGlobal } from '../src/prebidGlobal.js'; +import { getRefererInfo } from '../src/refererDetection.js'; +import { getStorageManager } from '../src/storageManager.js'; +import { MODULE_TYPE_RTD } from '../src/activities/modules.js'; +import { submodule } from '../src/hook.js'; /** @type {string} */ const MODULE_NAME = 'realTimeData'; const SUBMODULE_NAME = 'SirdataRTDModule'; const ORTB2_NAME = 'sirdata.com'; const LOG_PREFIX = 'Sirdata RTD: '; +const EUIDS_STORAGE_NAME = 'SDDAN'; +// Get the cookie domain from the referer info or fallback to window location hostname +const cookieDomain = getRefererInfo().domain || window.location.hostname; -const partnerIds = { +/** @type {number} */ +const GVLID = 53; + +/** @type {object} */ +const STORAGE = getStorageManager({ moduleType: MODULE_TYPE_RTD, moduleName: SUBMODULE_NAME }); +const bidderAliasRegistry = adapterManager.aliasRegistry || {}; +const biddersId = { // Partner IDs mapping for different SSPs and DSPs 'criteo': 27443, 'openx': 30342, 'pubmatic': 30345, @@ -30,25 +44,9 @@ const partnerIds = { 'yahoossp': 30339, 'rubicon': 27452, 'appnexus': 27446, - 'appnexusAst': 27446, - 'brealtime': 27446, - 'emetriq': 27446, - 'emxdigital': 27446, - 'pagescience': 27446, 'gourmetads': 33394, - 'matomy': 27446, - 'featureforward': 27446, - 'oftmedia': 27446, - 'districtm': 27446, - 'adasta': 27446, - 'beintoo': 27446, - 'gravity': 27446, - 'msq_classic': 27878, - 'msq_max': 27878, - '366_apx': 27878, 'mediasquare': 27878, 'smartadserver': 27440, - 'smart': 27440, 'proxistore': 27484, 'ix': 27248, 'sdRtdForGpt': 27449, @@ -65,57 +63,370 @@ const partnerIds = { 'zeta_global_ssp': 33385, }; -export function getSegmentsAndCategories(reqBidsConfigObj, onDone, moduleConfig, userConsent) { - logInfo(LOG_PREFIX, 'init'); - moduleConfig.params = moduleConfig.params || {}; - - let tcString = (userConsent && userConsent.gdpr && userConsent.gdpr.consentString ? userConsent.gdpr.consentString : ''); - let gdprApplies = (userConsent && userConsent.gdpr && userConsent.gdpr.gdprApplies ? userConsent.gdpr.gdprApplies : ''); - - moduleConfig.params.partnerId = moduleConfig.params.partnerId ? moduleConfig.params.partnerId : 1; - moduleConfig.params.key = moduleConfig.params.key ? moduleConfig.params.key : 1; - moduleConfig.params.actualUrl = moduleConfig.params.actualUrl || null; - - let sirdataDomain; - let sendWithCredentials; - - if (userConsent.coppa || (userConsent.usp && (userConsent.usp[0] === '1' && (userConsent.usp[1] === 'N' || userConsent.usp[2] === 'Y')))) { - // if children or "Do not Sell" management in California, no segments, page categories only whatever TCF signal - sirdataDomain = 'cookieless-data.com'; - sendWithCredentials = false; - gdprApplies = null; - tcString = ''; - } else if (config.getConfig('consentManagement.gdpr')) { - // Default endpoint for Contextual results only is cookieless if gdpr management is set. Needed because the cookie-based endpoint will fail and return error if user is located in Europe and no consent has been given - sirdataDomain = 'cookieless-data.com'; - sendWithCredentials = false; +const eidsProvidersMap = { + 'id5': 'id5-sync.com', + 'id5id': 'id5-sync.com', + 'id5_id': 'id5-sync.com', + 'pubprovided_id': 'pubProvidedId', + 'ppid': 'pubProvidedId', + 'first-id.fr': 'pubProvidedId', + 'sharedid': 'pubcid.org', + 'publishercommonid': 'pubcid.org', + 'pubcid.org': 'pubcid.org', +} + +// params +let params = { + partnerId: 1, + key: 1, + actualUrl: getRefererInfo().stack.pop() || getRefererInfo().page, + cookieAccessGranted: false, + setGptKeyValues: true, + contextualMinRelevancyScore: 30, + preprod: false, + authorizedEids: ['pubProvidedId', 'id5-sync.com', 'pubcid.org'], + avoidPostContent: false, + sirdataDomain: 'cookieless-data.com', + bidders: [] +}; + +/** + * Sets a cookie on the top-level domain + * @param {string} key - The cookie name + * @param {string} value - The cookie value + * @param {string} hostname - The hostname for setting the cookie + * @param {boolean} deleteCookie - The cookie must be deleted + * @returns {boolean} - True if the cookie was successfully set, otherwise false + */ +export function setCookieOnTopDomain(key, value, hostname, deleteCookie) { + const subDomains = hostname.split('.'); + let expTime = new Date(); + expTime.setTime(expTime.getTime() + (deleteCookie ? -1 : 365 * 24 * 60 * 60 * 1000)); // Set expiration time + for (let i = 0; i < subDomains.length; ++i) { + // Try to write the cookie on this subdomain (we want it to be stored only on the TLD+1) + const domain = subDomains.slice(subDomains.length - i - 1).join('.'); + try { + STORAGE.setCookie(key, value, expTime.toUTCString(), 'Lax', '.' + domain); + // Try to read the cookie to check if we wrote it + if (STORAGE.getCookie(key, null) === value) return true; // Check if the cookie was set, and if so top domain was found. If deletion with expire date -1 will parse until complete host + } catch (e) { + logError(LOG_PREFIX, e); + } } + return false; +} - // default global endpoint is cookie-based if no rules falls into cookieless or consent has been given or GDPR doesn't apply +/** + * Retrieves the UID from storage (cookies or local storage) + * @returns {Array|null} - Array of UID objects or null if no UID found + */ +export function getUidFromStorage() { + let cUid = STORAGE.getCookie(EUIDS_STORAGE_NAME, null); + let lsUid = STORAGE.getDataFromLocalStorage(EUIDS_STORAGE_NAME, null); + if (cUid && (!lsUid || cUid !== lsUid)) { + STORAGE.setDataInLocalStorage(EUIDS_STORAGE_NAME, cUid, null); + } else if (lsUid && !cUid) { + setCookieOnTopDomain(EUIDS_STORAGE_NAME, lsUid, cookieDomain, false); + cUid = lsUid; + } + return cUid ? [{ source: 'sddan.com', uids: [{ id: cUid, atype: 1 }] }] : null; +} - if (!sirdataDomain || !gdprApplies || (deepAccess(userConsent, 'gdpr.vendorData.vendor.consents') && userConsent.gdpr.vendorData.vendor.consents[53] && userConsent.gdpr.vendorData.purpose.consents[1] && userConsent.gdpr.vendorData.purpose.consents[4])) { - sirdataDomain = 'sddan.com'; - sendWithCredentials = true; +/** + * Sets the UID in storage (cookies and local storage) + * @param {string} sddanId - The UID to be set + * @returns {boolean} - True if the UID was successfully set, otherwise false + */ +export function setUidInStorage(sddanId) { + if (!sddanId) return false; + sddanId = encodeURI(sddanId.toString()); + setCookieOnTopDomain(EUIDS_STORAGE_NAME, sddanId, cookieDomain, false); + STORAGE.setDataInLocalStorage(EUIDS_STORAGE_NAME, sddanId, null); + return true; +} + +/** + * Merges and cleans objects from two eids arrays based on the 'source' and unique 'id' within the 'uids' array. + * Processes each array to add unique items or merge uids if the source already exists. + * @param {Array} euids1 - The first array to process and merge. + * @param {Array} euids2 - The second array to process and merge. + * @returns {Array} The merged array with unique sources and uid ids. + */ +export function mergeEuidsArrays(euids1, euids2) { + if (isEmpty(euids1)) return euids2; + if (isEmpty(euids2)) return euids1; + const mergedArray = []; + // Helper function to process each array + const processArray = (array) => { + array.forEach(item => { + if (item.uids) { + const foundIndex = findIndex(mergedArray, function (x) { + return x.source === item.source; + }); + if (foundIndex !== -1) { + // Merge uids if the source is found + item.uids.forEach(uid => { + if (!mergedArray[foundIndex].uids.some(u => u.id === uid.id)) { + mergedArray[foundIndex].uids.push(uid); + } + }); + } else { + // Add the entire item if the source does not exist + mergedArray.push({ ...item, uids: [...item.uids] }); + } + } + }); + }; + // Process both euids1 and euids2 + processArray(euids1); + processArray(euids2); + return mergedArray; +} + +/** + * Handles data deletion request by removing stored EU IDs + * @param {Object} moduleConfig - The module configuration + * @returns {boolean} - True if data was deleted successfully + */ +export function onDataDeletionRequest(moduleConfig) { + if (moduleConfig && moduleConfig.params) { + setCookieOnTopDomain(EUIDS_STORAGE_NAME, '', window.location.hostname, true); + STORAGE.removeDataFromLocalStorage(EUIDS_STORAGE_NAME, null); } + return !getUidFromStorage(); +} - let actualUrl = moduleConfig.params.actualUrl || getRefererInfo().stack.pop() || getRefererInfo().page; +/** + * Sends the page content for semantic analysis to Sirdata's server. + * @param {string} postContentToken - The token required to post content. + * @param {string} actualUrl - The actual URL of the current page. + * @returns {boolean} - True if the content was sent successfully + */ +export function postContentForSemanticAnalysis(postContentToken, actualUrl) { + if (!postContentToken || !actualUrl) return false; - const url = 'https://kvt.' + sirdataDomain + '/api/v1/public/p/' + moduleConfig.params.partnerId + '/d/' + moduleConfig.params.key + '/s?callback=&gdpr=' + gdprApplies + '&gdpr_consent=' + tcString + (actualUrl ? '&url=' + encodeURIComponent(actualUrl) : ''); + try { + let content = document.implementation.createHTMLDocument(''); + // Clone the current document content to avoid altering the original page content + content.documentElement.innerHTML = document.documentElement.innerHTML; + // Sanitize the cloned content to remove unnecessary elements and PII + content = sanitizeContent(content); + // Serialize the sanitized content to a string + const payload = new XMLSerializer().serializeToString(content.documentElement); + + if (payload && payload.length > 300 && payload.length < 300000) { + const url = `https://contextual.sirdata.io/api/v1/push/contextual?post_content_token=${postContentToken}&url=${encodeURIComponent(actualUrl)}`; + + // Use the Beacon API if supported to send the payload + if ('sendBeacon' in navigator) { + navigator.sendBeacon(url, payload); + } else { + // Fallback to using AJAX if Beacon API is not supported + ajax(url, {}, payload, { + contentType: 'text/plain', + method: 'POST', + withCredentials: false, // No user-specific data is tied to the request + referrerPolicy: 'unsafe-url', + crossOrigin: true + }); + } + } + } catch (e) { + logError(LOG_PREFIX, e); + return false; + } + return true; +} + +/** + * Executes a callback function when the document is fully loaded. + * @param {function} callback - The function to execute when the document is ready. + */ +export function onDocumentReady(callback) { + if (typeof callback !== 'function') return false; + try { + if (document.readyState && document.readyState !== 'loading') { + callback(); + } else if (typeof document.addEventListener === 'function') { + document.addEventListener('DOMContentLoaded', callback); + } + } catch (e) { + callback(); + } + return true; +} + +/** + * Removes Personally Identifiable Information (PII) from the content + * @param {string} content - The content to be sanitized + * @returns {string} - The sanitized content + */ +export function removePII(content) { + const patterns = [ + /\b(?:\d{4}[ -]?){3}\d{4}\b/g, // Credit card numbers + /\b\d{10,12}\b/g, // US bank account numbers + /\b\d{5}\d{5}\d{11}\d{2}\b/g, // EU bank account numbers + /\b(\d{3}-\d{2}-\d{4}|\d{9}|\d{13}|\d{2} \d{2} \d{2} \d{3} \d{3} \d{3})\b/g, // SSN + /\b[A-Z]{1,2}\d{6,9}\b/g, // Passport numbers + /\b(\d{8,10}|\d{3}-\d{3}-\d{3}-\d{3}|\d{2} \d{2} \d{2} \d{3} \d{3})\b/g, // ID card numbers + /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, // Email addresses + /(\+?\d{1,3}[-.\s]?)?(\(?\d{2,3}\)?[-.\s]?)(\d{2}[-.\s]?){3,4}\d{2}/g // Phone numbers + ]; + patterns.forEach(pattern => { + content = content.replace(pattern, ''); + }); + return content; +} - ajax(url, - { +/** + * Sanitizes the content by removing unnecessary elements and PII + * @param {Object} content - The content to be sanitized + * @returns {Object} - The sanitized content + */ +export function sanitizeContent(content) { + if (content && content.documentElement.innerText && content.documentElement.innerText.length > 500) { + // Reduce size by removing useless content + // Allowed tags + const allowedTags = [ + 'div', 'span', 'a', 'article', 'section', 'p', 'h1', 'h2', 'body', 'b', 'u', 'i', 'big', 'mark', 'ol', 'small', 'strong', 'blockquote', + 'nav', 'menu', 'li', 'ul', 'ins', 'head', 'title', 'main', 'var', 'table', 'caption', 'colgroup', 'col', 'tr', 'td', 'th', + 'summary', 'details', 'dl', 'dt', 'dd' + ]; + + const processElement = (element) => { + Array.from(element.childNodes).reverse().forEach(child => { + if (child.nodeType === Node.ELEMENT_NODE) { + processElement(child); + Array.from(child.attributes).forEach(attr => { + // keeps only id attributes and class attribute if useful for contextualisation + if (attr.name === 'class' && !/^(main|article|product)/.test(attr.value)) { + child.removeAttribute(attr.name); + } else if (attr.name !== 'id') { + child.removeAttribute(attr.name); + } + }); + if (!child.innerHTML.trim() || !allowedTags.includes(child.tagName.toLowerCase())) { // Keeps only allowed Tags (allowedTags) + child.remove(); + } + } + }); + }; + + const removeEmpty = (element) => { // remove empty tags + Array.from(element.childNodes).reverse().forEach(child => { + if (child.nodeType === Node.ELEMENT_NODE) { + removeEmpty(child); + if (!child.innerHTML.trim()) { + child.remove(); + } + } else if (child.nodeType === Node.TEXT_NODE && !child.textContent.trim()) { + child.remove(); + } + }); + }; + + processElement(content.documentElement); + removeEmpty(content.documentElement); + + // Clean any potential PII + content.documentElement.innerHTML = removePII(content.documentElement.innerHTML); + + let htmlContent = content.documentElement.innerHTML; + // Remove HTML comments + // This regex removes HTML comments, including those that might not be properly closed + htmlContent = htmlContent.replace(/|$)/g, ''); + // Remove multiple spaces + htmlContent = htmlContent.replace(/\s+/g, ' '); + // Remove spaces between tags + htmlContent = htmlContent.replace(/>\s+<'); + // Assign the cleaned content + content.documentElement.innerHTML = htmlContent; + } + return content; +} + +/** + * Fetches segments and categories from Sirdata server and processes the response + * @param {Object} reqBidsConfigObj - The bids configuration object + * @param {function} onDone - The callback function to be called upon completion + * @param {Object} moduleConfig - The module Config + * @param {Object} userConsent - The user consent information + */ +export function getSegmentsAndCategories(reqBidsConfigObj, onDone, moduleConfig, userConsent) { + logInfo(LOG_PREFIX, 'get Segments And Categories'); + const adUnits = (reqBidsConfigObj && reqBidsConfigObj.adUnits) || getGlobal().adUnits; + if (!adUnits) { + logInfo(LOG_PREFIX, 'no ad unit, RTD processing is useless'); + onDone(); + return; + } + + const gdprApplies = deepAccess(userConsent, 'gdpr.gdprApplies') ? userConsent.gdpr.gdprApplies : false; + const sirdataSubDomain = params.preprod ? 'kvt-preprod' : 'kvt'; + + let euids; // empty if no right to access device (publisher or user reject) + let privacySignals = ''; + + // Default global endpoint is cookie-based only if no rules falls into cookieless or consent has been given or GDPR doesn't apply + if (hasDeviceAccess() && !userConsent.coppa && (isEmpty(userConsent.usp) || userConsent.usp === -1 || (userConsent.usp[0] === '1' && (userConsent.usp[1] !== 'N' && userConsent.usp[2] !== 'Y'))) && (!gdprApplies || (deepAccess(userConsent, 'gdpr.vendorData.vendor.consents') && userConsent.gdpr.vendorData.vendor.consents[GVLID] && deepAccess(userConsent, 'gdpr.vendorData.purpose.consents') && userConsent.gdpr.vendorData.purpose.consents[1] && (userConsent.gdpr.vendorData.purpose.consents[2] || userConsent.gdpr.vendorData.purpose.consents[3]) && userConsent.gdpr.vendorData.purpose.consents[4])) && (isEmpty(userConsent.gpp) || userConsent.gpp.gppString) && checkCookieSupport()) { + params.sirdataDomain = 'sddan.com'; // cookie based domain + params.cookieAccessGranted = true; // cookies sent in request + + if (gdprApplies && deepAccess(userConsent, 'gdpr.consentString')) { + privacySignals = `&gdpr=${gdprApplies}&gdpr_consent=${userConsent.gdpr.consentString}`; + } else if (!isEmpty(userConsent.usp)) { + privacySignals = `&ccpa_consent=${userConsent.usp.toString()}`; + } else if (deepAccess(userConsent, 'gpp.gppString')) { + const sid = deepAccess(userConsent, 'gpp.applicableSections') ? `&gpp_sid=${userConsent.gpp.applicableSections.join(',')}` : ''; + privacySignals = `&gpp=${userConsent.gpp.gppString}${sid}`; + } + + // Authorized EUIDS from storage and sync global for graph + euids = getUidFromStorage(); // Sirdata Id + + if (!isEmpty(params.authorizedEids) && typeof getGlobal().getUserIds === 'function') { + let filteredEids = {}; + const authorizedEids = params.authorizedEids; + const globalUserIds = getGlobal().getUserIds(); + const globalUserIdsAsEids = getGlobal().getUserIdsAsEids(); + + const hasPubProvidedId = authorizedEids.indexOf('pubProvidedId') !== -1; + + if (hasPubProvidedId && !isEmpty(globalUserIds.pubProvidedId)) { // Publisher allows pubProvidedId + filteredEids = mergeEuidsArrays(filteredEids, globalUserIds.pubProvidedId); + } + + if (!hasPubProvidedId || authorizedEids.length > 1) { // Publisher allows other Id providers + const filteredGlobalEids = globalUserIdsAsEids.filter(entry => authorizedEids.includes(entry.source)); + if (!isEmpty(filteredGlobalEids)) { + filteredEids = mergeEuidsArrays(filteredEids, filteredGlobalEids); + } + } + + if (!isEmpty(filteredEids)) { + euids = mergeEuidsArrays(euids, filteredEids); // merge ids for graph id + } + } + } + + const url = `https://${sirdataSubDomain}.${params.sirdataDomain}/api/v1/public/p/${params.partnerId.toString()}/d/${params.key.toString()}/s?callback=&allowed_post_content=${!params.avoidPostContent}${privacySignals}${params.actualUrl ? `&url=${encodeURIComponent(params.actualUrl)}` : ''}`; + const method = isEmpty(euids) ? 'GET' : 'POST'; + const payload = isEmpty(euids) ? null : JSON.stringify({ external_ids: euids }); + + try { + ajax(url, { success: function (response, req) { if (req.status === 200) { try { const data = JSON.parse(response); if (data && data.segments) { - addSegmentData(reqBidsConfigObj, data, moduleConfig, onDone); + addSegmentData(reqBidsConfigObj, data, adUnits, onDone); } else { onDone(); } } catch (e) { onDone(); - logError('unable to parse Sirdata data' + e); + logError(LOG_PREFIX, 'unable to parse Sirdata data' + e); } } else if (req.status === 204) { onDone(); @@ -123,104 +434,136 @@ export function getSegmentsAndCategories(reqBidsConfigObj, onDone, moduleConfig, }, error: function () { onDone(); - logError('unable to get Sirdata data'); + logError(LOG_PREFIX, 'unable to get Sirdata data'); } - }, - null, - { + }, payload, { contentType: 'text/plain', - method: 'GET', - withCredentials: sendWithCredentials, + method: method, + withCredentials: params.cookieAccessGranted, referrerPolicy: 'unsafe-url', crossOrigin: true }); + } catch (e) { + logError(LOG_PREFIX, e); + } } +/** + * Pushes data to OpenRTB 2.5 fragments for the specified bidder + * @param {Object} ortb2Fragments - The OpenRTB 2.5 fragments + * @param {string} bidder - The bidder name + * @param {Object} data - The data to be pushed + * @param {number} segtaxid - The segment taxonomy ID + * @param {number} cattaxid - The category taxonomy ID + * @returns {boolean} - True if data was pushed successfully + */ export function pushToOrtb2(ortb2Fragments, bidder, data, segtaxid, cattaxid) { try { if (!isEmpty(data.segments)) { if (segtaxid) { setOrtb2Sda(ortb2Fragments, bidder, 'user', data.segments, segtaxid); } else { - setOrtb2(ortb2Fragments, bidder, 'user.ext.data', {sd_rtd: {segments: data.segments}}); + setOrtb2(ortb2Fragments, bidder, 'user.ext.data', { sd_rtd: { segments: data.segments } }); } } if (!isEmpty(data.categories)) { if (cattaxid) { setOrtb2Sda(ortb2Fragments, bidder, 'site', data.categories, cattaxid); } else { - setOrtb2(ortb2Fragments, bidder, 'site.ext.data', {sd_rtd: {categories: data.categories}}); + setOrtb2(ortb2Fragments, bidder, 'site.ext.data', { sd_rtd: { categories: data.categories } }); } } if (!isEmpty(data.categories_score) && !cattaxid) { - setOrtb2(ortb2Fragments, bidder, 'site.ext.data', {sd_rtd: {categories_score: data.categories_score}}); + setOrtb2(ortb2Fragments, bidder, 'site.ext.data', { sd_rtd: { categories_score: data.categories_score } }); } } catch (e) { - logError(e) + logError(LOG_PREFIX, e); } return true; } +/** + * Sets OpenRTB 2.5 Seller Defined Audiences (SDA) data + * @param {Object} ortb2Fragments - The OpenRTB 2.5 fragments + * @param {string} bidder - The bidder name + * @param {string} type - The type of data ('user' or 'site') + * @param {Array} segments - The segments to be set + * @param {number} segtaxValue - The segment taxonomy value + * @returns {boolean} - True if data was set successfully + */ export function setOrtb2Sda(ortb2Fragments, bidder, type, segments, segtaxValue) { try { - let ortb2Data = [{ - name: ORTB2_NAME, - segment: segments.map((segmentId) => ({ id: segmentId })), - }]; - if (segtaxValue) { - ortb2Data[0].ext = { segtax: segtaxValue }; - } - let ortb2Conf = (type === 'site' ? {site: {content: {data: ortb2Data}}} : {user: {data: ortb2Data}}); - if (bidder) { - ortb2Conf = {[bidder]: ortb2Conf}; - } + let ortb2Data = [{ name: ORTB2_NAME, segment: segments.map(segmentId => ({ id: segmentId })) }]; + if (segtaxValue) ortb2Data[0].ext = { segtax: segtaxValue }; + let ortb2Conf = (type === 'site') ? { site: { content: { data: ortb2Data } } } : { user: { data: ortb2Data } }; + if (bidder) ortb2Conf = { [bidder]: ortb2Conf }; mergeDeep(ortb2Fragments, ortb2Conf); } catch (e) { - logError(e) + logError(LOG_PREFIX, e); } return true; } +/** + * Sets OpenRTB 2.5 data at the specified path + * @param {Object} ortb2Fragments - The OpenRTB 2.5 fragments + * @param {string} bidder - The bidder name + * @param {string} path - The path to set the data at + * @param {Object} segments - The segments to be set + * @returns {boolean} - True if data was set successfully + */ export function setOrtb2(ortb2Fragments, bidder, path, segments) { try { - if (isEmpty(segments)) { return false; } + if (isEmpty(segments)) return false; let ortb2Conf = {}; - deepSetValue(ortb2Conf, path, segments || {}); - if (bidder) { - ortb2Conf = {[bidder]: ortb2Conf}; - } + deepSetValue(ortb2Conf, path, segments); + if (bidder) ortb2Conf = { [bidder]: ortb2Conf }; mergeDeep(ortb2Fragments, ortb2Conf); } catch (e) { - logError(e) + logError(LOG_PREFIX, e); } - return true; } +/** + * Loads a custom function for processing ad unit data + * @param {function} todo - The custom function to be executed + * @param {Object} adUnit - The ad unit object + * @param {Object} list - The list of data + * @param {Object} data - The data object + * @param {Object} bid - The bid object + * @returns {boolean} - True if the function was executed successfully + */ export function loadCustomFunction(todo, adUnit, list, data, bid) { try { - if (typeof todo == 'function') { - todo(adUnit, list, data, bid); - } + if (typeof todo === 'function') todo(adUnit, list, data, bid); } catch (e) { - logError(e); + logError(LOG_PREFIX, e); } return true; } +/** + * Gets segments and categories array from the data object + * @param {Object} data - The data object + * @param {number} minScore - The minimum score threshold for contextual relevancy + * @param {string} pid - The partner ID (attributed by Sirdata to bidder) + * @returns {Object} - The segments and categories data + */ export function getSegAndCatsArray(data, minScore, pid) { - let sirdataData = {'segments': [], 'categories': [], 'categories_score': {}}; - minScore = minScore && typeof minScore == 'number' ? minScore : 30; - let cattaxid = data.cattaxid || null; - let segtaxid = data.segtaxid || null; + let sirdataData = { segments: [], categories: [], categories_score: {} }; + minScore = typeof minScore === 'number' ? minScore : 30; + const { cattaxid, segtaxid, segments } = data; + const contextualCategories = data.contextual_categories || {}; + // parses contextual categories try { - if (data && data.contextual_categories) { - for (let catId in data.contextual_categories) { - if (data.contextual_categories.hasOwnProperty(catId) && data.contextual_categories[catId]) { - let value = data.contextual_categories[catId]; - if (value >= minScore && sirdataData.categories.indexOf(catId) === -1) { - if (pid && pid === '27440' && cattaxid) { // Equativ only - sirdataData.categories.push(pid.toString() + 'cc' + catId.toString()); + if (contextualCategories) { + for (let catId in contextualCategories) { + if (contextualCategories.hasOwnProperty(catId) && contextualCategories[catId]) { + let value = contextualCategories[catId]; + if (value >= minScore && !sirdataData.categories.includes(catId)) { + if (pid === '27440' && cattaxid) { // Equativ only + sirdataData.categories.push(`${pid}cc${catId}`); } else { sirdataData.categories.push(catId.toString()); sirdataData.categories_score[catId] = value; @@ -230,15 +573,16 @@ export function getSegAndCatsArray(data, minScore, pid) { } } } catch (e) { - logError(e); + logError(LOG_PREFIX, e); } + // parses user-centric segments (empty if no right to access device/process PII) try { - if (data && data.segments) { - for (let segId in data.segments) { - if (data.segments.hasOwnProperty(segId) && data.segments[segId]) { - let id = data.segments[segId].toString(); - if (pid && pid === '27440' && segtaxid) { // Equativ only - sirdataData.segments.push(pid.toString() + 'us' + id); + if (segments) { + for (let segId in segments) { + if (segments.hasOwnProperty(segId) && segments[segId]) { + let id = segments[segId].toString(); + if (pid === '27440' && segtaxid) { // Equativ only + sirdataData.segments.push(`${pid}us${id}`); } else { sirdataData.segments.push(id); } @@ -246,150 +590,177 @@ export function getSegAndCatsArray(data, minScore, pid) { } } } catch (e) { - logError(e); + logError(LOG_PREFIX, e); } return sirdataData; } -export function applySdaGetSpecificData(data, sirdataData, biddersParamsExist, minScore, reqBids, bid, moduleConfig, indexFound, bidderIndex, adUnit) { - // only share SDA data if whitelisted - if (!biddersParamsExist || indexFound) { +/** + * Applies Seller Defined Audience (SDA) data and specific data for the bidder + * @param {Object} data - The data object + * @param {Object} sirdataData - The Sirdata data object + * @param {boolean} biddersParamsExist - Flag indicating if bidder parameters exist + * @param {Object} reqBids - The request bids object + * @param {Object} bid - The bid object + * @param {number} bidderIndex - The bidder index + * @param {Object} adUnit - The ad unit object + * @param {string} aliasActualBidder - The bidder Alias + * @returns {Object} - The modified Sirdata data + */ +export function applySdaGetSpecificData(data, sirdataData, biddersParamsExist, reqBids, bid, bidderIndex, adUnit, aliasActualBidder) { + // Apply custom function or return Bidder Specific Data if publisher is ok + if (bidderIndex && params.bidders[bidderIndex]?.customFunction && typeof (params.bidders[bidderIndex]?.customFunction) === 'function') { + return loadCustomFunction(params.bidders[bidderIndex].customFunction, adUnit, sirdataData, data, bid); + } + + // Only share Publisher SDA data if whitelisted + if (!biddersParamsExist || bidderIndex) { // SDA Publisher - let sirdataDataForSDA = getSegAndCatsArray(data, minScore, moduleConfig.params.partnerId.toString()); + let sirdataDataForSDA = getSegAndCatsArray(data, params.contextualMinRelevancyScore, params.partnerId.toString()); pushToOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, sirdataDataForSDA, data.segtaxid, data.cattaxid); } - // always share SDA for curation - let curationId = (indexFound && moduleConfig.params.bidders[bidderIndex].hasOwnProperty('curationId') ? moduleConfig.params.bidders[bidderIndex].curationId : (partnerIds[bid.bidder] ? partnerIds[bid.bidder] : null)); - if (curationId) { - // seller defined audience & bidder specific data - if (data.shared_taxonomy && data.shared_taxonomy[curationId]) { - // Get Bidder Specific Data - let curationData = getSegAndCatsArray(data.shared_taxonomy[curationId], minScore, curationId.toString()); - pushToOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, curationData, data.shared_taxonomy[curationId].segtaxid, data.shared_taxonomy[curationId].cattaxid); + // Always share SDA for curation + if (!isEmpty(data.shared_taxonomy)) { + let curationId = (bidderIndex && params.bidders[bidderIndex]?.curationId) || biddersId[aliasActualBidder]; + if (curationId && data.shared_taxonomy[curationId]) { + // Seller defined audience & bidder specific data + let curationData = getSegAndCatsArray(data.shared_taxonomy[curationId], params.contextualMinRelevancyScore, curationId.toString()); + if (!isEmpty(curationData)) { + pushToOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, curationData, data.shared_taxonomy[curationId].segtaxid, data.shared_taxonomy[curationId].cattaxid); + mergeDeep(sirdataData, curationData); + } } } - // Apply custom function or return Bidder Specific Data if publisher is ok - if (!biddersParamsExist || indexFound) { - if (indexFound && moduleConfig.params.bidders[bidderIndex].hasOwnProperty('customFunction')) { - return loadCustomFunction(moduleConfig.params.bidders[bidderIndex].customFunction, adUnit, sirdataData, data, bid); - } else { - return sirdataData; - } - } + return sirdataData; } -export function addSegmentData(reqBids, data, moduleConfig, onDone) { - const adUnits = (reqBids && reqBids.adUnits) || getGlobal().adUnits; - if (!adUnits) { - onDone(); - return; - } - - moduleConfig = moduleConfig || {}; - moduleConfig.params = moduleConfig.params || {}; - const globalMinScore = moduleConfig.params.hasOwnProperty('contextualMinRelevancyScore') ? moduleConfig.params.contextualMinRelevancyScore : 30; - let sirdataData = getSegAndCatsArray(data, globalMinScore, null); +/** + * Adds segment data to the request bids object and processes the data + * @param {Object} reqBids - The request bids object + * @param {Object} data - The data object + * @param {Array} adUnits - The ad units array + * @param {function} onDone - The callback function to be called upon completion + * @returns {Array} - The ad units array + */ +export function addSegmentData(reqBids, data, adUnits, onDone) { + logInfo(LOG_PREFIX, 'Dispatch Segments And Categories'); + const minScore = params.contextualMinRelevancyScore || 30; + let sirdataData = getSegAndCatsArray(data, minScore, ''); - const biddersParamsExist = (!!(moduleConfig.params && moduleConfig.params.bidders && moduleConfig.params.bidders.length > 0)); + const biddersParamsExist = params.bidders.length > 0; // Global ortb2 SDA - if (data.global_taxonomy && !isEmpty(data.global_taxonomy)) { - let globalData = {'segments': [], 'categories': [], 'categories_score': []}; + if (!isEmpty(data.global_taxonomy)) { for (let i in data.global_taxonomy) { + let globalData; if (!isEmpty(data.global_taxonomy[i])) { - globalData = getSegAndCatsArray(data.global_taxonomy[i], globalMinScore, null); - pushToOrtb2(reqBids.ortb2Fragments?.global, null, globalData, data.global_taxonomy[i].segtaxid, data.global_taxonomy[i].cattaxid); + globalData = getSegAndCatsArray(data.global_taxonomy[i], params.contextualMinRelevancyScore, ''); + if (!isEmpty(globalData)) { + pushToOrtb2(reqBids.ortb2Fragments?.global, '', globalData, data.global_taxonomy[i].segtaxid, data.global_taxonomy[i].cattaxid); + } } } } // Google targeting - if (typeof window.googletag !== 'undefined' && (moduleConfig.params.setGptKeyValues || !moduleConfig.params.hasOwnProperty('setGptKeyValues'))) { + if (typeof window.googletag !== 'undefined' && params.setGptKeyValues) { try { - let gptCurationId = (moduleConfig.params.gptCurationId ? moduleConfig.params.gptCurationId : (partnerIds['sdRtdForGpt'] ? partnerIds['sdRtdForGpt'] : null)); - let sirdataMergedList = []; - if (gptCurationId && data.shared_taxonomy && data.shared_taxonomy[gptCurationId]) { - let gamCurationData = getSegAndCatsArray(data.shared_taxonomy[gptCurationId], globalMinScore, null); - sirdataMergedList = sirdataMergedList.concat(gamCurationData.segments).concat(gamCurationData.categories); - } else { - sirdataMergedList = sirdataData.segments.concat(sirdataData.categories); + const gptCurationId = params.gptCurationId || biddersId.sdRtdForGpt; + let sirdataMergedList = [...sirdataData.segments, ...sirdataData.categories]; + + if (gptCurationId && data.shared_taxonomy?.[gptCurationId]) { + const gamCurationData = getSegAndCatsArray(data.shared_taxonomy[gptCurationId], params.contextualMinRelevancyScore, ''); + sirdataMergedList = [...sirdataMergedList, ...gamCurationData.segments, ...gamCurationData.categories]; } - window.googletag.cmd.push(function() { - window.googletag.pubads().getSlots().forEach(function (n) { - if (typeof n.setTargeting !== 'undefined' && sirdataMergedList && sirdataMergedList.length > 0) { - n.setTargeting('sd_rtd', sirdataMergedList); + + window.googletag.cmd.push(() => { + window.googletag.pubads().getSlots().forEach(slot => { + if (typeof slot.setTargeting !== 'undefined' && sirdataMergedList.length > 0) { + slot.setTargeting('sd_rtd', sirdataMergedList); } }); }); } catch (e) { - logError(e); + logError(LOG_PREFIX, e); } } - // Bid targeting level for FPD non-generic biders - let bidderIndex = ''; - let indexFound = false; - adUnits.forEach(adUnit => { - adUnit.hasOwnProperty('bids') && adUnit.bids.forEach(bid => { - bidderIndex = (moduleConfig.params.hasOwnProperty('bidders') ? findIndex(moduleConfig.params.bidders, function (i) { - return i.bidder === bid.bidder; - }) : false); - indexFound = (!!(typeof bidderIndex == 'number' && bidderIndex >= 0)); + return adUnit.bids?.forEach(bid => { + const bidderIndex = findIndex(params.bidders, function (i) { return i.bidder === bid.bidder; }); try { - let minScore = (indexFound && moduleConfig.params.bidders[bidderIndex].hasOwnProperty('contextualMinRelevancyScore') ? moduleConfig.params.bidders[bidderIndex].contextualMinRelevancyScore : globalMinScore); - switch (bid.bidder) { - case 'appnexus': - case 'appnexusAst': - case 'brealtime': - case 'emetriq': - case 'emxdigital': - case 'pagescience': - case 'gourmetads': - case 'matomy': - case 'featureforward': - case 'oftmedia': - case 'districtm': - case 'adasta': - case 'beintoo': - case 'gravity': - case 'msq_classic': - case 'msq_max': - case '366_apx': - sirdataData = applySdaGetSpecificData(data, sirdataData, biddersParamsExist, minScore, reqBids, bid, moduleConfig, indexFound, bidderIndex, adUnit); - if (sirdataData.segments && sirdataData.segments.length > 0) { - setOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, 'user.keywords', 'sd_rtd=' + sirdataData.segments.join(',sd_rtd=')); - } - if (sirdataData.categories && sirdataData.categories.length > 0) { - setOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, 'site.content.keywords', 'sd_rtd=' + sirdataData.categories.join(',sd_rtd=')); - } - break; - - default: - if (!biddersParamsExist || (indexFound && (!moduleConfig.params.bidders[bidderIndex].hasOwnProperty('adUnitCodes') || moduleConfig.params.bidders[bidderIndex].adUnitCodes.indexOf(adUnit.code) !== -1))) { - applySdaGetSpecificData(data, sirdataData, biddersParamsExist, minScore, reqBids, bid, moduleConfig, indexFound, bidderIndex, adUnit); - } + const aliasActualBidder = bidderAliasRegistry[bid.bidder] || bid.bidder; + if (aliasActualBidder === 'appnexus') { + let xandrData = applySdaGetSpecificData(data, sirdataData, biddersParamsExist, reqBids, bid, bidderIndex, adUnit, aliasActualBidder); + // Surprisingly, to date Xandr doesn't support SDA, we need to set specific 'keywords' entries + if (xandrData.segments.length > 0) { + setOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, 'user.keywords', `sd_rtd=${xandrData.segments.join(',sd_rtd=')}`); + } + if (xandrData.categories.length > 0) { + setOrtb2(reqBids.ortb2Fragments?.bidder, bid.bidder, 'site.content.keywords', `sd_rtd=${xandrData.categories.join(',sd_rtd=')}`); + } + } else { + applySdaGetSpecificData(data, sirdataData, biddersParamsExist, reqBids, bid, bidderIndex, adUnit, aliasActualBidder); } } catch (e) { - logError(e); + logError(LOG_PREFIX, e); } - }) + }); }); + + // Trigger onDone onDone(); + + // Postprocessing: should we send async content to categorize content and/or store Sirdata ID (stored in 3PC) within local scope ? + if (params.sirdataDomain === 'sddan.com') { // Means consent has been given for device and content access + if (!isEmpty(data.sddan_id) && params.cookieAccessGranted) { // Device access allowed by publisher and cookies are supported + setUidInStorage(data.sddan_id); // Save Sirdata user ID + } + if (!params.avoidPostContent && params.actualUrl && !inIframe() && !isEmpty(data.post_content_token)) { + onDocumentReady(() => postContentForSemanticAnalysis(data.post_content_token, params.actualUrl)); + } + } return adUnits; } -export function init(config) { - logInfo(LOG_PREFIX, config); +/** + * Initializes the module with the given configuration + * @param {Object} moduleConfig - The module configuration + * @returns {boolean} - True if the initialization was successful + */ +export function init(moduleConfig) { + logInfo(LOG_PREFIX, moduleConfig); + if (typeof (moduleConfig.params) !== 'object' || !moduleConfig.params.key) return false; + if (typeof (moduleConfig.params.authorizedEids) !== 'object' || !Array.isArray(moduleConfig.params.authorizedEids)) { + delete (moduleConfig.params.authorizedEids); // must be array of strings + } else { + // we need it if the publishers uses user Id module name instead of key or source + const resultSet = new Set( + moduleConfig.params.authorizedEids.map(item => { + const formattedItem = item.toLowerCase().replace(/\s+/g, '_'); // Normalize + return eidsProvidersMap[formattedItem] || formattedItem; + }) + ); + moduleConfig.params.authorizedEids = Array.from(resultSet); + } + if (typeof (moduleConfig.params.bidders) !== 'object' || !Array.isArray(moduleConfig.params.bidders)) delete (moduleConfig.params.bidders); // must be array of objects + delete (moduleConfig.params.sirdataDomain); // delete cookieless domain if specified => shouldn't be overridden by publisher + params = Object.assign({}, params, moduleConfig.params); return true; } +/** + * The Sirdata submodule definition + * @type {Object} + */ export const sirdataSubmodule = { name: SUBMODULE_NAME, + gvlid: GVLID, init: init, getBidRequestData: getSegmentsAndCategories }; +// Register the Sirdata submodule with the real time data module submodule(MODULE_NAME, sirdataSubmodule); diff --git a/modules/sirdataRtdProvider.md b/modules/sirdataRtdProvider.md index f67e34db43a..9e712e5fbd6 100644 --- a/modules/sirdataRtdProvider.md +++ b/modules/sirdataRtdProvider.md @@ -1,30 +1,28 @@ -# Sirdata Real-Time Data Submodule +# Sirdata RTD/SDA Module -Module Name: Sirdata Rtd Provider +Module Name: Sirdata Real-time SDA Module Module Type: Rtd Provider Maintainer: bob@sirdata.com # Description -Sirdata provides a disruptive API that allows its partners to leverage its -cutting-edge contextualization technology and its audience segments based on -cookies and consent or without cookies nor consent! +Sirdata provides a disruptive API that allows publishers and SDA compliant SSPs/DSPs to leverage its cutting-edge contextualization technology and its audience segments! -User-based segments and page-level automatic contextual categories will be -attached to bid request objects sent to different SSPs in order to optimize -targeting. +User-based segments and page-level automatic contextual categories will be attached to bid request objects sent to different bidders in order to optimize targeting. -Automatic integration with Google Ad Manager and major bidders like Xandr/Appnexus, -Smartadserver, Index Exchange, Proxistore, Magnite/Rubicon or Triplelift ! +Automatic or custom integration with Google Ad Manager and major bidders like Xandr's, Equativ's, Index Exchange's, Proxistore's, Magnite's or Triplelift's ! -User's country and choice management are included in the module, so it's 100% -compliant with local and regional laws like GDPR and CCPA/CPRA. +User's country and choice management are included in the module, so it's 100% compliant with local and regional laws like GDPR and CCPA/CPRA. ORTB2 compliant and FPD support for Prebid versions < 4.29 -Contact bob@sirdata.com for information. +Fully supports Seller Defined Audience ! Please find the full SDA taxonomy ids list here. -### Publisher Usage +Please contact for more information. + +## Publisher Usage + +### Configure Prebid.js Compile the Sirdata RTD module into your Prebid build: @@ -32,15 +30,39 @@ Compile the Sirdata RTD module into your Prebid build: Add the Sirdata RTD provider to your Prebid config. -Segments ids (user-centric) and category ids (page-centric) will be provided -salted and hashed : you can use them with a dedicated and private matching table. -Should you want to allow a SSP or a partner to curate your media and operate -cross-publishers campaigns with our data, please ask Sirdata (bob@sirdata.com) to -open it for you account. +`actualUrl` MUST be set with actual location of parent page if prebid.js is loaded in an iframe (e.g. hosted). It can be left blank ('') or removed otherwise. + +`partnerId` and `key` should be provided by your partnering SSP or get one and your dedicated taxonomy from Sirdata (). Segments ids (user-centric) and category ids (page-centric) will be provided salted and hashed : you can use them with a dedicated and private matching table. + +Should you want to allow any SSP or a partner to curate your media and operate cross-publishers campaigns with our data, please ask Sirdata () to whitelist him it in your account. + +#### Typical configuration +```javascript +pbjs.setConfig({ + // ... + realTimeData: { + auctionDelay: 1000, + dataProviders: [ + { + name: "SirdataRTDModule", + waitForIt: true, + params: { + partnerId: 1, + key: 1, + } + } + ] + } + // ... +}); ``` -pbjs.setConfig( - ... + +#### Advanced configuration + +```javascript +pbjs.setConfig({ + // ... realTimeData: { auctionDelay: 1000, dataProviders: [ @@ -48,85 +70,114 @@ pbjs.setConfig( name: "SirdataRTDModule", waitForIt: true, params: { - partnerId: 1, + partnerId: 1, key: 1, - setGptKeyValues: true, - contextualMinRelevancyScore: 50, //Min score to filter contextual category globally (0-100 scale) - actualUrl: actual_url, //top location url, for contextual categories - bidders: [{ - bidder: 'appnexus', - adUnitCodes: ['adUnit-1','adUnit-2'], - customFunction: overrideAppnexus, - curationId: '111', - },{ - bidder: 'ix', - sizeLimit: 1200 //specific to Index Exchange, - contextualMinRelevancyScore: 50, //Min score to filter contextual category for curation in the bidder (0-100 scale) - }] + setGptKeyValues: true, + contextualMinRelevancyScore: 50, //Min score to filter contextual category globally (0-100 scale) + actualUrl: '' //top location url, for contextual categories } } ] } - ... -} + // ... +}); ``` ### Parameter Descriptions for the Sirdata Configuration Section -| Name |Type | Description | Notes | -| :------------ | :------------ | :------------ |:------------ | -| name | String | Real time data module name | Mandatory. Always 'SirdataRTDModule' | -| waitForIt | Boolean | Mandatory. Required to ensure that the auction is delayed until prefetch is complete | Optional. Defaults to false but recommended to true | -| params | Object | | Optional | -| params.partnerId | Integer | Partner ID, required to get results and provided by Sirdata. Use 1 for tests and get one running at bob@sirdata.com | Mandatory. Defaults 1. | -| params.key | Integer | Key linked to Partner ID, required to get results and provided by Sirdata. Use 1 for tests and get one running at bob@sirdata.com | Mandatory. Defaults 1. | -| params.setGptKeyValues | Boolean | This parameter Sirdata to set Targeting for GPT/GAM | Optional. Defaults to true. | -| params.contextualMinRelevancyScore | Integer | Min score to keep filter category in the bidders (0-100 scale). Optional. Defaults to 30. | -| params.bidders | Object | Dictionary of bidders you would like to supply Sirdata data for. | Optional. In case no bidder is specified Sirdata will atend to ad data custom and ortb2 to all bidders, adUnits & Globalconfig | +| Name | Type | Description | Notes | +|:-----------------------------------|:--------------------------|:----------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| name | String | Real time data module name | Mandatory. Always 'SirdataRTDModule' | +| waitForIt | Boolean | Required to ensure that the auction is delayed until prefetch is complete | Optional. Default to false but recommended to true | +| params | Object | Settings | Optional | +| params.partnerId | Integer | Partner ID, required to get results and provided by Sirdata. Use 1 for tests and request your Id at | Mandatory. Default 1 | +| params.key | Integer | Key linked to Partner ID, required to get results and provided by Sirdata. Use 1 for tests and request your key at | Mandatory. Default 1 | +| params.setGptKeyValues | Boolean | Sets Targeting for GPT/GAM | Optional. Default to true | +| params.authorizedEids | Array of String | List of authorised Eids for graph. Set [] to prevent all Eids usage | Optional. Default to : ID5, pubProvidedId and sharedId | +| params.avoidPostContent | Boolean | Block contextual data POST from user's device (a crawler is use instead) | Optional. Default to false, and setting it to true results in your content downloaded by Sirdata crawler | +| params.contextualMinRelevancyScore | Integer | Min relevancy score to filter categories sent to the bidders (0-100 scale). | Optional. Defaults to 30. | +| params.bidders | Array of Object | Bidders you want to supply your own data to (works only with your private data bought to Sirdata) | Optional | Bidders can receive common setting : -| Name |Type | Description | Notes | -| :------------ | :------------ | :------------ |:------------ | -| bidder | String | Bidder name | Mandatory if params.bidders are specified | -| adUnitCodes | Array of String | Use if you want to limit data injection to specified adUnits for the bidder | Optional. Default is false and data shared with the bidder isn't filtered | -| customFunction | Function | Use it to override the way data is shared with a bidder | Optional. Default is false | -| curationId | String | Specify the curation ID of the bidder. Provided by Sirdata, request it at bob@sirdata.com | Optional. Default curation ids are specified for main bidders | -| contextualMinRelevancyScore | Integer | Min score to filter contextual categories for curation in the bidder (0-100 scale). Optional. Defaults to 30 or global params.contextualMinRelevancyScore if exits. | -| sizeLimit | Integer | used only for bidder 'ix' to limit the size of the get parameter in Index Exchange ad call | Optional. Default is 1000 | +| Name | Type | Description | Notes | +|:---------------|:----------------|:-----------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------| +| bidder | String | Bidder name | Mandatory if params.bidders are specified | +| adUnitCodes | Array of String | Use if you want to limit data injection to specified adUnits for the bidder | Optional. Default is false and data shared with the bidder isn't filtered | +| customFunction | Function | Use it to override the way data is shared with a bidder | Optional. Default is false | +| curationId | String | Specify the curation ID of the bidder. Provided by Sirdata, request it at | Optional. Default curation ids are specified for main bidders | ### Overriding data sharing function -As indicated above, it is possible to provide your own bid augmentation -functions. This is useful if you know a bid adapter's API supports segment -fields which aren't specifically being added to request objects in the Prebid -bid adapter. -Please see the following example, which provides a function to modify bids for -a bid adapter called ix and overrides the appnexus. +As indicated above, it is possible to provide your own bid augmentation functions. This is useful if you know a bid adapter's API supports segment fields which aren't specifically being added to request objects in the Prebid bid adapter. +Please see the following example, which provides a function to modify bids for a bid adapter called ix and overrides the appnexus. data Object format for usage in this kind of function : + +```json { - "segments":[111111,222222], - "contextual_categories":{"333333":100}, - "shared_taxonomy":{ - "27446":{ //CurationId - "segments":[444444,555555], - "contextual_categories":{"666666":100} - } - } + "segments": [ + 111111, + 222222 + ], + "segtaxid": null, + "cattaxid": null, + "contextual_categories": { + "333333": 100 + }, + "shared_taxonomy": { + "27440": { + "segments": [ + 444444, + 555555 + ], + "segtaxid": 552, + "cattaxid": 553, + "contextual_categories": { + "666666": 100 + } + } + }, + "global_taxonomy": { + "9998": { + "segments": [ + 123, + 234 + ], + "segtaxid": 4, + "cattaxid": 7, + "contextual_categories": { + "345": 100, + "456": 100 + } + }, + "9999": { + "segments": [ + 12345, + 23456 + ], + "segtaxid": 550, + "cattaxid": 551, + "contextual_categories": { + "34567": 100, + "45678": 100 + } + } + } } - ``` + +```javascript function overrideAppnexus (adUnit, segmentsArray, dataObject, bid) { - for (var i = 0; i < segmentsArray.length; i++) { + for (var i = 0; i < segmentsArray.length; i++) { if (segmentsArray[i]) { bid.params.user.segments.push(segmentsArray[i]); } } } -pbjs.setConfig( - ... +pbjs.setConfig({ + // ... realTimeData: { auctionDelay: 1000, dataProviders: [ @@ -134,18 +185,17 @@ pbjs.setConfig( name: "SirdataRTDModule", waitForIt: true, params: { - partnerId: 1, + partnerId: 1, key: 1, - setGptKeyValues: true, - contextualMinRelevancyScore: 50, //Min score to keep contextual category in the bidders (0-100 scale) - actualUrl: actual_url, //top location url, for contextual categories + setGptKeyValues: true, + contextualMinRelevancyScore: 50, //Min score to keep contextual category in the bidders (0-100 scale) + actualUrl: actual_url, //top location url, for contextual categories bidders: [{ bidder: 'appnexus', customFunction: overrideAppnexus, curationId: '111' },{ bidder: 'ix', - sizeLimit: 1200, //specific to Index Exchange customFunction: function(adUnit, segmentsArray, dataObject, bid) { bid.params.contextual.push(dataObject.contextual_categories); }, @@ -153,17 +203,19 @@ pbjs.setConfig( } } ] - } + }, ... -} +}); ``` ### Testing To view an example of available segments returned by Sirdata's backends: -`gulp serve --modules=rtdModule,sirdataRtdProvider,appnexusBidAdapter` +```bash +gulp serve --modules=rtdModule,sirdataRtdProvider,appnexusBidAdapter +``` and then point your browser at: -`http://localhost:9999/integrationExamples/gpt/sirdataRtdProvider_example.html` \ No newline at end of file +[http://localhost:9999/integrationExamples/gpt/sirdataRtdProvider_example.html] diff --git a/test/spec/modules/sirdataRtdProvider_spec.js b/test/spec/modules/sirdataRtdProvider_spec.js index fbb5967bc20..9f6bb30e0b0 100644 --- a/test/spec/modules/sirdataRtdProvider_spec.js +++ b/test/spec/modules/sirdataRtdProvider_spec.js @@ -1,21 +1,123 @@ -import {addSegmentData, getSegmentsAndCategories, sirdataSubmodule, setOrtb2} from 'modules/sirdataRtdProvider.js'; +import { + addSegmentData, + getSegmentsAndCategories, + getUidFromStorage, + loadCustomFunction, + mergeEuidsArrays, + onDataDeletionRequest, + onDocumentReady, + postContentForSemanticAnalysis, + removePII, + sanitizeContent, + setOrtb2, + setUidInStorage, + sirdataSubmodule +} from 'modules/sirdataRtdProvider.js'; +import {expect} from 'chai'; +import {deepSetValue} from 'src/utils.js'; import {server} from 'test/mocks/xhr.js'; const responseHeader = {'Content-Type': 'application/json'}; describe('sirdataRtdProvider', function () { - describe('sirdataSubmodule', function () { + describe('sirdata Submodule init', function () { it('exists', function () { expect(sirdataSubmodule.init).to.be.a('function'); }); it('successfully instantiates', function () { - expect(sirdataSubmodule.init()).to.equal(true); + const moduleConfig = { + params: { + partnerId: 1, + key: 1, + } + }; + expect(sirdataSubmodule.init(moduleConfig)).to.equal(true); }); it('has the correct module name', function () { expect(sirdataSubmodule.name).to.equal('SirdataRTDModule'); }); }); + describe('Sanitize content', function () { + it('removes PII from content', function () { + let doc = document.implementation.createHTMLDocument(''); + let div = doc.createElement('div'); + div.className = 'test'; + div.setAttribute('test', 'test'); + div.textContent = 'My email is test@test.com, My bank account number is 123456789012, my SSN is 123-45-6789, and my credit card number is 1234 5678 9101 1121.Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'; + let div2 = doc.createElement('div'); + let div3 = doc.createElement('div'); + div3.innerText = 'hello'; + div2.appendChild(div3); + div.appendChild(div2); + doc.body.appendChild(div); + const cleanedDom = removePII(doc.documentElement.innerHTML); + const sanitizedDom = sanitizeContent(doc); + expect(cleanedDom).to.equal('
My email is , My bank account number is , my SSN is , and my credit card number is .Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
hello
'); + expect(sanitizedDom.documentElement.innerHTML).to.equal('
My email is , My bank account number is , my SSN is , and my credit card number is .Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
hello
'); + }); + }); + + describe('setUidInStorage', function () { + it('sets Id in Storage', function () { + setUidInStorage('123456789'); + let val = getUidFromStorage(); + expect(val).to.deep.equal([{source: 'sddan.com', uids: [{id: '123456789', atype: 1}]}]); + }); + }); + + describe('mergeEuidsArrays', function () { + it('merges Euids Arrays', function () { + const object1 = [{source: 'sddan.com', uids: [{id: '123456789', atype: 1}]}]; + const object2 = [{source: 'sddan.com', uids: [{id: '987654321', atype: 1}]}]; + const object3 = mergeEuidsArrays(object1, object2); + expect(object3).to.deep.equal([{source: 'sddan.com', uids: [{id: '123456789', atype: 1}, {id: '987654321', atype: 1}]}]); + }); + }); + + describe('onDocumentReady', function () { + it('on Document Ready function execution', function () { + const testString = ''; + const testFunction = function() { return true; }; + let resString; + try { + resString = onDocumentReady(testString); + } catch (e) {} + expect(resString).to.be.false; + let resFunction = onDocumentReady(testFunction); + expect(resFunction).to.be.true; + }); + }); + + describe('postContentForSemanticAnalysis', function () { + it('gets content for analysis', function () { + let res = postContentForSemanticAnalysis('1223456', 'https://www.sirdata.com/'); + let resEmpty = postContentForSemanticAnalysis('1223456', ''); + expect(res).to.be.true; + expect(resEmpty).to.be.false; + }); + }); + + describe('loadCustomFunction', function () { + it('load function', function () { + const res = loadCustomFunction(function(...args) { return true; }, {}, {}, {}, {}); + expect(res).to.be.true; + }); + }); + + describe('onDataDeletionRequest', function () { + it('destroy id', function () { + const moduleConfig = { + params: { + partnerId: 1, + key: 1, + } + }; + const res = onDataDeletionRequest(moduleConfig); + expect(res).to.be.true; + }); + }); + describe('Add Segment Data', function () { it('adds segment data', function () { const firstConfig = { @@ -25,9 +127,12 @@ describe('sirdataRtdProvider', function () { setGptKeyValues: true, gptCurationId: 27449, contextualMinRelevancyScore: 50, + actualUrl: 'https://www.sirdata.com/', + cookieAccessGranted: true, bidders: [] } }; + sirdataSubmodule.init(firstConfig); let adUnits = [ { @@ -68,13 +173,12 @@ describe('sirdataRtdProvider', function () { 'segtaxid': 4, 'cattaxid': 7, 'contextual_categories': {'345': 100, '456': 100} - } + }, + 'sddan_id': '123456789', + 'post_content_token': '987654321' } - }; - - addSegmentData(firstReqBidsConfigObj, firstData, firstConfig, () => { - }); - + } + addSegmentData(firstReqBidsConfigObj, firstData, adUnits, function() { return true; }); expect(firstReqBidsConfigObj.ortb2Fragments.global.user.data[0].ext.segtax).to.equal(4); }); }); @@ -109,6 +213,7 @@ describe('sirdataRtdProvider', function () { }] } }; + sirdataSubmodule.init(config); let reqBidsConfigObj = { adUnits: [{ @@ -197,11 +302,13 @@ describe('sirdataRtdProvider', function () { 'cattaxid': 7, 'contextual_categories': {'345': 100, '456': 100} } - } + }, + 'sddan_id': '123456789', + 'post_content_token': '987654321' }; getSegmentsAndCategories(reqBidsConfigObj, () => { - }, config, {}); + }, {}, {}); let request = server.requests[0]; request.respond(200, responseHeader, JSON.stringify(data)); @@ -228,16 +335,6 @@ describe('sirdataRtdProvider', function () { describe('Set ortb2 for bidder', function () { it('set ortb2 for a givent bidder', function () { - const config = { - params: { - setGptKeyValues: false, - contextualMinRelevancyScore: 50, - bidders: [{ - bidder: 'appnexus', - }] - } - }; - let reqBidsConfigObj = { adUnits: [{ bids: [{ @@ -252,22 +349,6 @@ describe('sirdataRtdProvider', function () { } }; - let data = { - 'segments': [111111, 222222], - 'segtaxid': null, - 'cattaxid': null, - 'contextual_categories': {'333333': 100}, - 'shared_taxonomy': { - '27440': { - 'segments': [444444, 555555], - 'segtaxid': null, - 'cattaxid': null, - 'contextual_categories': {'666666': 100} - } - }, - 'global_taxonomy': {} - }; - window.googletag = window.googletag || {}; window.googletag.cmd = window.googletag.cmd || [];