feat: script injecting and tools

backend/functions/src/cloud-functions/crawler.ts

@@ -23,7 +23,7 @@ import { JSDomControl } from '../services/jsdom';
 import { FormattedPage, md5Hasher, SnapshotFormatter } from '../services/snapshot-formatter';
 
 export interface ExtraScrappingOptions extends ScrappingOptions {
-    withIframe?: boolean;
+    withIframe?: boolean | 'quoted';
     withShadowDom?: boolean;
     targetSelector?: string | string[];
     removeSelector?: string | string[];
@@ -69,6 +69,10 @@ export class CrawlerHost extends RPCHost {
                 // Potential privacy issue, dont cache if cookies are used
                 return;
             }
+            if (options.injectFrameScripts?.length || options.injectPageScripts?.length) {
+                // Potentially mangeled content, dont cache if scripts are injected
+                return;
+            }
             if (options.locale) {
                 Reflect.set(snapshot, 'locale', options.locale);
             }
@@ -237,7 +241,7 @@ export class CrawlerHost extends RPCHost {
                 throw new SecurityCompromiseError(`Domain ${targetUrl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
             }
         }
-        const crawlOpts = this.configure(crawlerOptions);
+        const crawlOpts = await this.configure(crawlerOptions);
 
 
         if (!ctx.req.accepts('text/plain') && ctx.req.accepts('text/event-stream')) {
@@ -284,7 +288,7 @@ export class CrawlerHost extends RPCHost {
                 const formatted = await this.snapshotFormatter.formatSnapshot(crawlerOptions.respondWith, scrapped, targetUrl, this.urlValidMs);
                 chargeAmount = this.assignChargeAmount(formatted);
 
-                if (crawlerOptions.timeout === undefined) {
+                if (crawlerOptions.isEarlyReturnApplicable()) {
                     return formatted;
                 }
 
@@ -315,7 +319,7 @@ export class CrawlerHost extends RPCHost {
             const formatted = await this.snapshotFormatter.formatSnapshot(crawlerOptions.respondWith, scrapped, targetUrl, this.urlValidMs);
             chargeAmount = this.assignChargeAmount(formatted);
 
-            if (crawlerOptions.timeout === undefined) {
+            if (crawlerOptions.isEarlyReturnApplicable()) {
                 if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
 
                     return assignTransferProtocolMeta(`${formatted.textRepresentation}`,
@@ -557,8 +561,8 @@ export class CrawlerHost extends RPCHost {
 
         let cache;
 
-        const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs;
-        if (cacheTolerance && !crawlOpts?.cookies?.length) {
+        if (!crawlerOpts || crawlerOpts.isCacheQueryApplicable()) {
+            const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs;
             cache = await this.queryCache(urlToCrawl, cacheTolerance);
         }
 
@@ -665,7 +669,7 @@ export class CrawlerHost extends RPCHost {
         }
     }
 
-    configure(opts: CrawlerOptions) {
+    async configure(opts: CrawlerOptions) {
 
         this.threadLocal.set('withGeneratedAlt', opts.withGeneratedAlt);
         this.threadLocal.set('withLinksSummary', opts.withLinksSummary);
@@ -697,6 +701,30 @@ export class CrawlerHost extends RPCHost {
             crawlOpts.extraHeaders['Accept-Language'] = opts.locale;
         }
 
+        if (opts.injectFrameScript?.length) {
+            crawlOpts.injectFrameScripts = (await Promise.all(
+                opts.injectFrameScript.map((x) => {
+                    if (URL.canParse(x)) {
+                        return fetch(x).then((r) => r.text());
+                    }
+
+                    return x;
+                })
+            )).filter(Boolean);
+        }
+
+        if (opts.injectPageScript?.length) {
+            crawlOpts.injectPageScripts = (await Promise.all(
+                opts.injectPageScript.map((x) => {
+                    if (URL.canParse(x)) {
+                        return fetch(x).then((r) => r.text());
+                    }
+
+                    return x;
+                })
+            )).filter(Boolean);
+        }
+
         return crawlOpts;
     }
 

backend/functions/src/cloud-functions/searcher.ts

@@ -140,7 +140,7 @@ export class SearcherHost extends RPCHost {
 
         delete crawlerOptions.html;
 
-        const crawlOpts = this.crawler.configure(crawlerOptions);
+        const crawlOpts = await this.crawler.configure(crawlerOptions);
         const searchQuery = braveSearchExplicitOperators.addTo(q || noSlashPath);
         const r = await this.cachedWebSearch({
             q: searchQuery,
@@ -156,7 +156,7 @@ export class SearcherHost extends RPCHost {
         }
 
         const it = this.fetchSearchResults(crawlerOptions.respondWith, r.web?.results, crawlOpts,
-            { ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs },
+            CrawlerOptions.from({ ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs }),
             count,
         );
 

backend/functions/src/dto/scrapping-options.ts

@@ -192,8 +192,9 @@ export class CrawlerOptions extends AutoCastable {
 
     @Prop({
         default: false,
+        type: [String, Boolean]
     })
-    withIframe!: boolean;
+    withIframe!: boolean | 'quoted';
 
     @Prop({
         default: false,
@@ -211,6 +212,16 @@ export class CrawlerOptions extends AutoCastable {
     @Prop()
     userAgent?: string;
 
+    @Prop({
+        arrayOf: String,
+    })
+    injectPageScript?: string[];
+
+    @Prop({
+        arrayOf: String,
+    })
+    injectFrameScript?: string[];
+
     @Prop({
         validate: (v: number) => v > 0 && v <= 180,
         type: Number,
@@ -293,7 +304,7 @@ export class CrawlerOptions extends AutoCastable {
         }
         const withIframe = ctx?.req.get('x-with-iframe');
         if (withIframe !== undefined) {
-            instance.withIframe = Boolean(withIframe);
+            instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe);
         }
         if (instance.withIframe) {
             instance.timeout ??= null;
@@ -330,6 +341,37 @@ export class CrawlerOptions extends AutoCastable {
 
         return instance;
     }
+
+    isEarlyReturnApplicable() {
+        if (this.timeout !== undefined) {
+            return false;
+        }
+        if (this.waitForSelector?.length) {
+            return false;
+        }
+        if (this.injectFrameScript?.length || this.injectPageScript?.length) {
+            return false;
+        }
+
+        return true;
+    }
+
+    isCacheQueryApplicable() {
+        if (this.noCache) {
+            return false;
+        }
+        if (this.cacheTolerance === 0) {
+            return false;
+        }
+        if (this.setCookies?.length) {
+            return false;
+        }
+        if (this.injectFrameScript?.length || this.injectPageScript?.length) {
+            return false;
+        }
+
+        return true;
+    }
 }
 
 export class CrawlerOptionsHeaderOnly extends CrawlerOptions {
@@ -347,14 +389,14 @@ function filterSelector(s?: string | string[]) {
         return s;
     }
     const sr = Array.isArray(s) ? s : [s];
-    const selectors = sr.filter((i)=> {
+    const selectors = sr.filter((i) => {
         const innerSelectors = i.split(',').map((s) => s.trim());
         const someViolation = innerSelectors.find((x) => x.startsWith('*') || x.startsWith(':') || x.includes('*:'));
         if (someViolation) {
             return false;
         }
         return true;
-    })
+    });
 
     return selectors;
 };

backend/functions/src/services/jsdom.ts

@@ -53,7 +53,13 @@ export class JSDomControl extends AsyncService {
             jsdom.window.document.querySelectorAll('iframe[src],frame[src]').forEach((x) => {
                 const src = x.getAttribute('src');
                 const thisSnapshot = snapshot.childFrames?.find((f) => f.href === src);
-                if (thisSnapshot?.html) {
+                if (options?.withIframe === 'quoted') {
+                    const blockquoteElem = jsdom.window.document.createElement('blockquote');
+                    const preElem = jsdom.window.document.createElement('pre');
+                    preElem.innerHTML = thisSnapshot?.text || '';
+                    blockquoteElem.appendChild(preElem);
+                    x.replaceWith(blockquoteElem);
+                } else if (thisSnapshot?.html) {
                     x.innerHTML = thisSnapshot.html;
                     x.querySelectorAll('script, style').forEach((s) => s.remove());
                     x.querySelectorAll('[src]').forEach((el) => {

backend/functions/src/services/puppeteer.ts

@@ -76,6 +76,8 @@ export interface ScrappingOptions {
     locale?: string;
     referer?: string;
     extraHeaders?: Record<string, string>;
+    injectFrameScripts?: string[];
+    injectPageScripts?: string[];
 }
 
 
@@ -95,9 +97,135 @@ puppeteer.use(puppeteerPageProxy({
     interceptResolutionPriority: 1,
 }));
 
+const SIMULATE_SCROLL = `
+(function () {
+    function createIntersectionObserverEntry(target, isIntersecting, timestamp) {
+        const targetRect = target.getBoundingClientRect();
+        const record = {
+            target,
+            isIntersecting,
+            time: timestamp,
+            // If intersecting, intersectionRect matches boundingClientRect
+            // If not intersecting, intersectionRect is empty (0x0)
+            intersectionRect: isIntersecting
+                ? targetRect
+                : new DOMRectReadOnly(0, 0, 0, 0),
+            // Current bounding client rect of the target
+            boundingClientRect: targetRect,
+            // Intersection ratio is either 0 (not intersecting) or 1 (fully intersecting)
+            intersectionRatio: isIntersecting ? 1 : 0,
+            // Root bounds (viewport in our case)
+            rootBounds: new DOMRectReadOnly(
+                0,
+                0,
+                window.innerWidth,
+                window.innerHeight
+            )
+        };
+        Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);
+        return record;
+    }
+    function cloneIntersectionObserverEntry(entry) {
+        const record = {
+            target: entry.target,
+            isIntersecting: entry.isIntersecting,
+            time: entry.time,
+            intersectionRect: entry.intersectionRect,
+            boundingClientRect: entry.boundingClientRect,
+            intersectionRatio: entry.intersectionRatio,
+            rootBounds: entry.rootBounds
+        };
+        Object.setPrototypeOf(record, window.IntersectionObserverEntry.prototype);
+        return record;
+    }
+    const orig = window.IntersectionObserver;
+    const kCallback = Symbol('callback');
+    const kLastEntryMap = Symbol('lastEntryMap');
+    const liveObservers = new Map();
+    class MangledIntersectionObserver extends orig {
+        constructor(callback, options) {
+            super((entries, observer) => {
+                const lastEntryMap = observer[kLastEntryMap];
+                const lastEntry = entries[entries.length - 1];
+                lastEntryMap.set(lastEntry.target, lastEntry);
+                return callback(entries, observer);
+            }, options);
+            this[kCallback] = callback;
+            this[kLastEntryMap] = new WeakMap();
+            liveObservers.set(this, new Set());
+        }
+        disconnect() {
+            liveObservers.get(this)?.clear();
+            liveObservers.delete(this);
+            return super.disconnect();
+        }
+        observe(target) {
+            const observer = liveObservers.get(this);
+            observer?.add(target);
+            return super.observe(target);
+        }
+        unobserve(target) {
+            const observer = liveObservers.get(this);
+            observer?.delete(target);
+            return super.unobserve(target);
+        }
+    }
+    Object.defineProperty(MangledIntersectionObserver, 'name', { value: 'IntersectionObserver', writable: false });
+    window.IntersectionObserver = MangledIntersectionObserver;
+    function simulateScroll() {
+        for (const [observer, targets] of liveObservers.entries()) {
+            const t0 = performance.now();
+            for (const target of targets) {
+                const entry = createIntersectionObserverEntry(target, true, t0);
+                observer[kCallback]([entry], observer);
+                setTimeout(() => {
+                    const t1 = performance.now();
+                    const lastEntry = observer[kLastEntryMap].get(target);
+                    if (!lastEntry) {
+                        return;
+                    }
+                    const entry2 = { ...cloneIntersectionObserverEntry(lastEntry), time: t1 };
+                    observer[kCallback]([entry2], observer);
+                });
+            }
+        }
+    }
+    window.simulateScroll = simulateScroll;
+})();
+`;
+
+const MUTATION_IDLE_WATCH = `
+(function () {
+    let timeout;
+    const sendMsg = ()=> {
+        document.dispatchEvent(new CustomEvent('mutationIdle'));
+    };
+
+    const cb = () => {
+        if (timeout) {
+            clearTimeout(timeout);
+            timeout = setTimeout(sendMsg, 200);
+        }
+    };
+    const mutationObserver = new MutationObserver(cb);
+
+    document.addEventListener('DOMContentLoaded', () => {
+        mutationObserver.observe(document.documentElement, {
+            childList: true,
+            subtree: true,
+        });
+        timeout = setTimeout(sendMsg, 200);
+    }, { once: true })
+})();
+`;
+
+
 const SCRIPT_TO_INJECT_INTO_FRAME = `
 ${READABILITY_JS}
+${SIMULATE_SCROLL}
+${MUTATION_IDLE_WATCH}
 
+(function(){
 function briefImgs(elem) {
     const imageTags = Array.from((elem || document).querySelectorAll('img[src],img[data-src]'));
 
@@ -271,6 +399,30 @@ function giveSnapshot(stopActiveSnapshot) {
 
     return r;
 }
+function waitForSelector(selectorText) {
+  return new Promise((resolve) => {
+    const existing = document.querySelector(selectorText);
+    if (existing) {
+      resolve(existing);
+      return;
+    }
+    const observer = new MutationObserver(() => {
+      const elem = document.querySelector(selectorText);
+      if (elem) {
+        resolve(document.querySelector(selectorText));
+        observer.disconnect();
+      }
+    });
+    observer.observe(document.documentElement, {
+      childList: true,
+      subtree: true
+    });
+  });
+}
+window.waitForSelector = waitForSelector;
+window.giveSnapshot = giveSnapshot;
+window.briefImgs = briefImgs;
+})();
 `;
 
 @singleton()
@@ -479,26 +631,29 @@ export class PuppeteerControl extends AsyncService {
         });
 
         await page.evaluateOnNewDocument(`
-if (window.self === window.top) {
-    let lastTextLength = 0;
-    const handlePageLoad = () => {
-        if (window.haltSnapshot) {
-            return;
-        }
-        const thisTextLength = (document.body.innerText || '').length;
-        const deltaLength = Math.abs(thisTextLength - lastTextLength);
-        if (10 * deltaLength < lastTextLength) {
-            // Change is not significant
-            return;
-        }
-        const r = giveSnapshot();
-        window.reportSnapshot(r);
-        lastTextLength = thisTextLength;
-    };
-    setInterval(handlePageLoad, 800);
-    document.addEventListener('readystatechange', handlePageLoad);
-    document.addEventListener('load', handlePageLoad);
-}
+(function () {
+    if (window.self === window.top) {
+        let lastTextLength = 0;
+        const handlePageLoad = () => {
+            const thisTextLength = (document.body.innerText || '').length;
+            const deltaLength = Math.abs(thisTextLength - lastTextLength);
+            if (10 * deltaLength < lastTextLength) {
+                // Change is not significant
+                return;
+            }
+            lastTextLength = thisTextLength;
+            if (window.haltSnapshot) {
+                return;
+            }
+            const r = giveSnapshot();
+            window.reportSnapshot(r);
+        };
+        document.addEventListener('readystatechange', handlePageLoad);
+        document.addEventListener('load', handlePageLoad);
+        document.addEventListener('mutationIdle', handlePageLoad);
+    }
+    document.addEventListener('DOMContentLoaded', ()=> window.simulateScroll(), { once: true });
+})();
 `);
 
         this.snMap.set(page, sn);
@@ -550,9 +705,13 @@ if (window.self === window.top) {
         await Promise.race([
             (async () => {
                 const ctx = page.browserContext();
-                await page.close();
-                await ctx.close();
-            })(), delay(5000)
+                try {
+                    await page.close();
+                } finally {
+                    await ctx.close();
+                }
+            })(),
+            delay(5000)
         ]).catch((err) => {
             this.logger.error(`Failed to destroy page ${sn}`, { err: marshalErrorLike(err) });
         });
@@ -601,6 +760,30 @@ if (window.self === window.top) {
                 return req.continue(continueArgs[0], continueArgs[1]);
             });
         }
+        let pageScriptEvaluations: Promise<unknown>[] = [];
+        let frameScriptEvaluations: Promise<unknown>[] = [];
+        if (options?.injectPageScripts?.length) {
+            page.on('framenavigated', (frame) => {
+                if (frame !== page.mainFrame()) {
+                    return;
+                }
+
+                pageScriptEvaluations.push(
+                    Promise.allSettled(options.injectPageScripts!.map((x) => frame.evaluate(x).catch((err) => {
+                        this.logger.warn(`Error in evaluation of page scripts`, { err });
+                    })))
+                );
+            });
+        }
+        if (options?.injectFrameScripts?.length) {
+            page.on('framenavigated', (frame) => {
+                frameScriptEvaluations.push(
+                    Promise.allSettled(options.injectFrameScripts!.map((x) => frame.evaluate(x).catch((err) => {
+                        this.logger.warn(`Error in evaluation of frame scripts`, { err });
+                    })))
+                );
+            });
+        }
         const sn = this.snMap.get(page);
         this.logger.info(`Page ${sn}: Scraping ${url}`, { url });
 
@@ -689,6 +872,7 @@ if (window.self === window.top) {
             goToOptions.referer = options.referer;
         }
 
+        const delayPromise = delay(timeout);
         const gotoPromise = page.goto(url, goToOptions)
             .catch((err) => {
                 if (err instanceof TimeoutError) {
@@ -719,6 +903,8 @@ if (window.self === window.top) {
                         throw stuff;
                     }
                 }
+                await Promise.race([Promise.allSettled([...pageScriptEvaluations, ...frameScriptEvaluations]), delayPromise])
+                    .catch(() => void 0);
                 try {
                     const pSubFrameSnapshots = this.snapshotChildFrames(page);
                     snapshot = await page.evaluate('giveSnapshot(true)') as PageSnapshot;

backend/functions/src/services/snapshot-formatter.ts

@@ -316,7 +316,7 @@ export class SnapshotFormatter extends AsyncService {
             }
         } while (false);
 
-        const cleanText = (contentText || '').trim();
+        const cleanText = contentText?.includes('return') ? contentText.trimEnd() : (contentText || '').trim();
 
         const formatted: FormattedPage = {
             title: (snapshot.parsed?.title || snapshot.title || '').trim(),