Add new static network filter option: urltransform

The `urltransform` option allows to redirect a non-blocked network
request to another URL. There are restrictions on its usage:

- require a trusted source -- thus uBO-maintained lists or user
  filters
- the `urltransform` value must start with a `/`

If at least one of these conditions is not fulfilled, the filter
will be invalid and rejected.

The requirement to start with `/` is to enforce that only the path
part of a URL can be modified, thus ensuring the network request
is redirected to the same scheme and authority (as defined at
https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Syntax).

Usage example (redirect requests for CSS resources to a non-existing
resource, for demonstration purpose):

    ||iana.org^$css,urltransform=/notfound.css

Name of this option is inspired from DNR API:
https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/URLTransform

This commit required to bring the concept of "trusted source" to
the static network filtering engine.

platform/mv3/make-scriptlets.js

@@ -88,7 +88,7 @@ export function compile(details) {
     const scriptletToken = details.args[0];
     const resourceEntry = resourceDetails.get(scriptletToken);
     if ( resourceEntry === undefined ) { return; }
-    if ( resourceEntry.requiresTrust && details.isTrusted !== true ) {
+    if ( resourceEntry.requiresTrust && details.trustedSource !== true ) {
         console.log(`Rejecting ${scriptletToken}: source is not trusted`);
         return;
     }

src/js/1p-filters.js

@@ -48,6 +48,7 @@ const cmEditor = new CodeMirror(qs$('#userFilters'), {
     styleActiveLine: {
         nonEmpty: true,
     },
+    trustedSource: true,
 });
 
 uBlockDashboard.patchCodeMirrorEditor(cmEditor);

src/js/asset-viewer.js

@@ -82,6 +82,7 @@ import './codemirror/ubo-static-filtering.js';
         what : 'getAssetContent',
         url: assetKey,
     });
+    cmEditor.setOption('trustedSource', details.trustedSource === true);
     cmEditor.setValue(details && details.content || '');
 
     if ( subscribeElem !== null ) {

src/js/benchmarks.js

@@ -181,7 +181,8 @@ const loadBenchmarkDataset = (( ) => {
         if ( r === 1 ) { blockCount += 1; }
         else if ( r === 2 ) { allowCount += 1; }
         if ( r !== 1 ) {
-            if ( staticNetFilteringEngine.hasQuery(fctxt) ) {
+            staticNetFilteringEngine.transformRequest(fctxt);
+            if ( fctxt.redirectURL !== undefined && staticNetFilteringEngine.hasQuery(fctxt) ) {
                 staticNetFilteringEngine.filterQuery(fctxt, 'removeparam');
             }
             if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) {

src/js/codemirror/ubo-static-filtering.js

@@ -37,12 +37,21 @@ const preparseDirectiveHints = [];
 const originHints = [];
 let hintHelperRegistered = false;
 
+/******************************************************************************/
+
+let trustedSource = false;
+
+CodeMirror.defineOption('trustedSource', false, (cm, state) => {
+    trustedSource = state;
+    self.dispatchEvent(new Event('trustedSource'));
+});
 
 /******************************************************************************/
 
 CodeMirror.defineMode('ubo-static-filtering', function() {
     const astParser = new sfp.AstFilterParser({
         interactive: true,
+        trustedSource,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
     });
     const astWalker = astParser.getWalker();
@@ -205,7 +214,11 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
         return '+';
     };
 
-    return {
+    self.addEventListener('trustedSource', ( ) => {
+        astParser.options.trustedSource = trustedSource;
+    });
+
+   return {
         lineComment: '!',
         token: function(stream) {
             if ( stream.sol() ) {
@@ -977,6 +990,10 @@ CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => {
         }
     };
 
+    self.addEventListener('trustedSource', ( ) => {
+        astParser.options.trustedSource = trustedSource;
+    });
+
     CodeMirror.defineInitHook(cm => {
         cm.on('changes', onChanges);
         cm.on('beforeChange', onBeforeChanges);

src/js/messaging.js

@@ -108,6 +108,7 @@ const onMessage = function(request, sender, callback) {
             dontCache: true,
             needSourceURL: true,
         }).then(result => {
+            result.trustedSource = µb.isTrustedList(result.assetKey);
             callback(result);
         });
         return;

src/js/pagestore.js

@@ -860,7 +860,7 @@ const PageStore = class {
         if ( (fctxt.itype & fctxt.INLINE_ANY) === 0 ) {
             if ( result === 1 ) {
                 this.redirectBlockedRequest(fctxt);
-            } else if ( snfe.hasQuery(fctxt) ) {
+            } else {
                 this.redirectNonBlockedRequest(fctxt);
             }
         }
@@ -922,25 +922,31 @@ const PageStore = class {
     }
 
     redirectBlockedRequest(fctxt) {
-        const directives = staticNetFilteringEngine.redirectRequest(
-            redirectEngine,
-            fctxt
-        );
+        const directives = staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt);
         if ( directives === undefined ) { return; }
         if ( logger.enabled !== true ) { return; }
         fctxt.pushFilters(directives.map(a => a.logData()));
         if ( fctxt.redirectURL === undefined ) { return; }
         fctxt.pushFilter({
             source: 'redirect',
-            raw: redirectEngine.resourceNameRegister
+            raw: directives[directives.length-1].value
         });
     }
 
     redirectNonBlockedRequest(fctxt) {
-        const directives = staticNetFilteringEngine.filterQuery(fctxt);
-        if ( directives === undefined ) { return; }
+        const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt);
+        const pruneDirectives = fctxt.redirectURL === undefined &&
+            staticNetFilteringEngine.hasQuery(fctxt) &&
+            staticNetFilteringEngine.filterQuery(fctxt) ||
+            undefined;
+        if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; }
         if ( logger.enabled !== true ) { return; }
-        fctxt.pushFilters(directives.map(a => a.logData()));
+        if ( transformDirectives !== undefined ) {
+            fctxt.pushFilters(transformDirectives.map(a => a.logData()));
+        }
+        if ( pruneDirectives !== undefined ) {
+            fctxt.pushFilters(pruneDirectives.map(a => a.logData()));
+        }
         if ( fctxt.redirectURL === undefined ) { return; }
         fctxt.pushFilter({
             source: 'redirect',

src/js/redirect-engine.js

@@ -167,7 +167,6 @@ class RedirectEngine {
         this.resources = new Map();
         this.reset();
         this.modifyTime = Date.now();
-        this.resourceNameRegister = '';
     }
 
     reset() {
@@ -183,7 +182,6 @@ class RedirectEngine {
     ) {
         const entry = this.resources.get(this.aliases.get(token) || token);
         if ( entry === undefined ) { return; }
-        this.resourceNameRegister = token;
         return entry.toURL(fctxt, asDataURI);
     }
 

src/js/reverselookup.js

@@ -131,6 +131,7 @@ const fromNetFilter = async function(rawFilter) {
     const writer = new CompiledListWriter();
     const parser = new sfp.AstFilterParser({
         expertMode: true,
+        trustedSource: true,
         maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
     });
@@ -169,6 +170,7 @@ const fromExtendedFilter = async function(details) {
 
     const parser = new sfp.AstFilterParser({
         expertMode: true,
+        trustedSource: true,
         nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
     });
     parser.parse(details.rawFilter);

src/js/scriptlet-filtering.js

@@ -306,7 +306,7 @@ scriptletFilteringEngine.compile = function(parser, writer) {
 
     // Only exception filters are allowed to be global.
     const isException = parser.isException();
-    const normalized = normalizeRawFilter(parser, writer.properties.get('isTrusted'));
+    const normalized = normalizeRawFilter(parser, writer.properties.get('trustedSource'));
 
     // Can fail if there is a mismatch with trust requirement
     if ( normalized === undefined ) { return; }

src/js/static-dnr-filtering.js

@@ -109,8 +109,8 @@ function addExtendedToDNR(context, parser) {
             let details = context.scriptletFilters.get(argsToken);
             if ( details === undefined ) {
                 context.scriptletFilters.set(argsToken, details = { args });
-                if ( context.isTrusted ) {
-                    details.isTrusted = true;
+                if ( context.trustedSource ) {
+                    details.trustedSource = true;
                 }
             }
             if ( not ) {
@@ -299,9 +299,11 @@ function addToDNR(context, list) {
 
         if ( parser.isComment() ) {
             if ( line === `!#trusted on ${context.secret}` ) {
-                context.isTrusted = true;
+                parser.trustedSource = true;
+                context.trustedSource = true;
             } else if ( line === `!#trusted off ${context.secret}` ) {
-                context.isTrusted = false;
+                parser.trustedSource = false;
+                context.trustedSource = false;
             }
             continue;
         }

src/js/static-filtering-parser.js

@@ -188,6 +188,7 @@ export const NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM  = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_SCRIPT       = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_SHIDE        = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_TO           = iota++;
+export const NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_XHR          = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_WEBRTC       = iota++;
 export const NODE_TYPE_NET_OPTION_NAME_WEBSOCKET    = iota++;
@@ -267,6 +268,7 @@ export const nodeTypeFromOptionName = new Map([
     [ 'shide', NODE_TYPE_NET_OPTION_NAME_SHIDE ],
     /* synonym */ [ 'specifichide', NODE_TYPE_NET_OPTION_NAME_SHIDE ],
     [ 'to', NODE_TYPE_NET_OPTION_NAME_TO ],
+    [ 'urltransform', NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM ],
     [ 'xhr', NODE_TYPE_NET_OPTION_NAME_XHR ],
     /* synonym */ [ 'xmlhttprequest', NODE_TYPE_NET_OPTION_NAME_XHR ],
     [ 'webrtc', NODE_TYPE_NET_OPTION_NAME_WEBRTC ],
@@ -1315,6 +1317,7 @@ export class AstFilterParser {
                     break;
                 case NODE_TYPE_NET_OPTION_NAME_REDIRECT:
                 case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
+                case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
                     realBad = isNegated || (isException || hasValue) === false ||
                         modifierType !== 0;
                     if ( realBad ) { break; }
@@ -1374,6 +1377,14 @@ export class AstFilterParser {
                 realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount;
                 break;
             }
+            case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
+                realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount ||
+                    this.options.trustedSource !== true;
+                if ( realBad !== true ) {
+                    const path = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM);
+                    realBad = path.charCodeAt(0) !== 0x2F /* / */;
+                }
+                break;
             case NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
                 realBad = abstractTypeCount || behaviorTypeCount;
                 break;
@@ -2973,6 +2984,7 @@ export const netOptionTokenDescriptors = new Map([
     [ 'shide', { } ],
     /* synonym */ [ 'specifichide', { } ],
     [ 'to', { mustAssign: true } ],
+    [ 'urltransform', { mustAssign: true } ],
     [ 'xhr', { canNegate: true } ],
     /* synonym */ [ 'xmlhttprequest', { canNegate: true } ],
     [ 'webrtc', { } ],

src/js/static-net-filtering.js

@@ -178,18 +178,22 @@ const typeValueToDNRTypeName = [
     'other',
 ];
 
+// Do not change order. Compiled filter lists rely on this order being
+// consistent across sessions.
 const MODIFIER_TYPE_REDIRECT = 1;
 const MODIFIER_TYPE_REDIRECTRULE = 2;
 const MODIFIER_TYPE_REMOVEPARAM = 3;
 const MODIFIER_TYPE_CSP = 4;
 const MODIFIER_TYPE_PERMISSIONS = 5;
+const MODIFIER_TYPE_URLTRANSFORM = 6;
 
 const modifierTypeFromName = new Map([
     [ 'redirect', MODIFIER_TYPE_REDIRECT ],
     [ 'redirect-rule', MODIFIER_TYPE_REDIRECTRULE ],
     [ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ],
     [ 'csp', MODIFIER_TYPE_CSP ],
     [ 'permissions', MODIFIER_TYPE_PERMISSIONS ],
+    [ 'urltransform', MODIFIER_TYPE_URLTRANSFORM ],
 ]);
 
 const modifierNameFromType = new Map([
@@ -198,6 +202,7 @@ const modifierNameFromType = new Map([
     [ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ],
     [ MODIFIER_TYPE_CSP, 'csp' ],
     [ MODIFIER_TYPE_PERMISSIONS, 'permissions' ],
+    [ MODIFIER_TYPE_URLTRANSFORM, 'urltransform' ],
 ]);
 
 //const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111;
@@ -3182,6 +3187,7 @@ class FilterCompiler {
             [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ],
             [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ],
             [ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ],
+            [ sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM, MODIFIER_TYPE_URLTRANSFORM ],
         ]);
         // These top 100 "bad tokens" are collated using the "miss" histogram
         // from tokenHistograms(). The "score" is their occurrence among the
@@ -3484,6 +3490,12 @@ class FilterCompiler {
                 if ( this.toDomainOpt === '' ) { return false; }
                 this.optionUnitBits |= this.TO_BIT;
                 break;
+            case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
+                if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
+                    return false;
+                }
+                this.optionUnitBits |= this.REDIRECT_BIT;
+                break;
             default:
                 break;
         }
@@ -3575,6 +3587,7 @@ class FilterCompiler {
             case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
             case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
             case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
+            case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM:
                 if ( this.processOptionWithValue(parser, type) === false ) {
                     return this.FILTER_INVALID;
                 }
@@ -4521,6 +4534,20 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) {
                 dnrAddRuleError(rule, 'Unsupported modifier exception');
             }
             break;
+        case 'urltransform': {
+            const path = rule.__modifierValue;
+            let priority = rule.priority || 1;
+            if ( rule.__modifierAction !== AllowAction ) {
+                const transform = { path };
+                rule.action.type = 'redirect';
+                rule.action.redirect = { transform };
+                rule.priority = priority + 1;
+            } else {
+                rule.action.type = 'block';
+                rule.priority = priority + 2;
+            }
+            break;
+        }
         default:
             dnrAddRuleError(rule, `Unsupported modifier ${rule.__modifierType}`);
             break;
@@ -5230,18 +5257,27 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) {
     }
     // Redirect to highest-ranked directive
     const directive = directives[highest];
-    if ( (directive.bits & AllowAction) === 0 ) {
-        const { token } = parseRedirectRequestValue(directive);
-        fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token);
-        if ( fctxt.redirectURL === undefined ) { return; }
-    }
+    if ( (directive.bits & AllowAction) !== 0 ) { return directives; }
+    const { token } = parseRedirectRequestValue(directive);
+    fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token);
+    if ( fctxt.redirectURL === undefined ) { return; }
+    return directives;
+};
+
+FilterContainer.prototype.transformRequest = function(fctxt) {
+    const directives = this.matchAndFetchModifiers(fctxt, 'urltransform');
+    if ( directives === undefined ) { return; }
+    const directive = directives[directives.length-1];
+    if ( (directive.bits & AllowAction) !== 0 ) { return directives; }
+    const redirectURL = new URL(fctxt.url);
+    redirectURL.pathname = directive.value;
+    fctxt.redirectURL = redirectURL.href;
     return directives;
 };
 
 function parseRedirectRequestValue(directive) {
     if ( directive.cache === null ) {
-        directive.cache =
-            sfp.parseRedirectValue(directive.value);
+        directive.cache = sfp.parseRedirectValue(directive.value);
     }
     return directive.cache;
 }