fix: jsdom, cache tolerance, screenshot pricing

backend/functions/src/cloud-functions/crawler.ts

@@ -24,6 +24,7 @@ import { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-a
 import { PDFExtractor } from '../services/pdf-extract';
 import { DomainBlockade } from '../db/domain-blockade';
 import { FirebaseRoundTripChecker } from '../shared/services/firebase-roundtrip-checker';
+import { JSDomControl } from '../services/jsdom';
 
 const md5Hasher = new HashManager('md5', 'hex');
 
@@ -74,6 +75,7 @@ export class CrawlerHost extends RPCHost {
     constructor(
         protected globalLogger: Logger,
         protected puppeteerControl: PuppeteerControl,
+        protected jsdomControl: JSDomControl,
         protected altTextService: AltTextService,
         protected pdfExtractor: PDFExtractor,
         protected firebaseObjectStorage: FirebaseStorageBucketControl,
@@ -247,7 +249,7 @@ export class CrawlerHost extends RPCHost {
     }
 
     getGeneralSnapshotMixins(snapshot: PageSnapshot) {
-        const inferred = this.puppeteerControl.inferSnapshot(snapshot);
+        const inferred = this.jsdomControl.inferSnapshot(snapshot);
         const mixin: any = {};
         if (this.threadLocal.get('withImagesSummary')) {
             const imageSummary = {} as { [k: string]: string; };
@@ -296,6 +298,7 @@ export class CrawlerHost extends RPCHost {
 
             return {
                 ...this.getGeneralSnapshotMixins(snapshot),
+                html: snapshot.html,
                 screenshotUrl: snapshot.screenshotUrl,
                 toString() {
                     return this.screenshotUrl;
@@ -353,16 +356,20 @@ export class CrawlerHost extends RPCHost {
                 break;
             }
 
-            let toBeTurnedToMd = snapshot.html;
+            const jsDomElementOfHTML = this.jsdomControl.snippetToElement(snapshot.html, snapshot.href);
+            let toBeTurnedToMd = jsDomElementOfHTML;
             let turnDownService = this.getTurndown({ url: nominalUrl, imgDataUrlToObjectUrl });
             if (mode !== 'markdown' && snapshot.parsed?.content) {
-                const par1 = turnDownService.turndown(snapshot.html);
-                const par2 = turnDownService.turndown(snapshot.parsed.content);
+                const jsDomElementOfParsed = this.jsdomControl.snippetToElement(snapshot.parsed.content, snapshot.href);
+                const par1 = turnDownService.turndown(jsDomElementOfHTML);
+                const par2 = snapshot.parsed.content ? turnDownService.turndown(jsDomElementOfParsed) : '';
 
                 // If Readability did its job
                 if (par2.length >= 0.3 * par1.length) {
                     turnDownService = this.getTurndown({ noRules: true, url: snapshot.href, imgDataUrlToObjectUrl });
-                    toBeTurnedToMd = snapshot.parsed.content;
+                    if (snapshot.parsed.content) {
+                        toBeTurnedToMd = jsDomElementOfParsed;
+                    }
                 }
             }
 
@@ -453,7 +460,7 @@ export class CrawlerHost extends RPCHost {
 
             if (
                 !contentText || (contentText.startsWith('<') && contentText.endsWith('>'))
-                && toBeTurnedToMd !== snapshot.html
+                && toBeTurnedToMd !== jsDomElementOfHTML
             ) {
                 try {
                     contentText = turnDownService.turndown(snapshot.html);
@@ -533,7 +540,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
                     .value();
         }
         if (this.threadLocal.get('withLinksSummary')) {
-            formatted.links = _.invert(this.puppeteerControl.inferSnapshot(snapshot).links || {});
+            formatted.links = _.invert(this.jsdomControl.inferSnapshot(snapshot).links || {});
         }
 
         return formatted as FormattedPage;
@@ -890,19 +897,19 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
                 text: '',
             } as PageSnapshot;
 
-            yield this.puppeteerControl.narrowSnapshot(fakeSnapshot, crawlOpts);
+            yield this.jsdomControl.narrowSnapshot(fakeSnapshot, crawlOpts);
 
             return;
         }
         let cache;
 
-        const cacheTolerance = crawlerOpts?.cacheTolerance || this.cacheValidMs;
+        const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs;
         if (cacheTolerance && !crawlOpts?.cookies?.length) {
             cache = await this.queryCache(urlToCrawl, cacheTolerance);
         }
 
         if (cache?.isFresh && (!crawlOpts?.favorScreenshot || (crawlOpts?.favorScreenshot && cache?.screenshotAvailable))) {
-            yield this.puppeteerControl.narrowSnapshot(cache.snapshot, crawlOpts);
+            yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);
 
             return;
         }
@@ -910,7 +917,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
         try {
             if (crawlOpts?.targetSelector || crawlOpts?.removeSelector || crawlOpts?.withIframe) {
                 for await (const x of this.puppeteerControl.scrap(urlToCrawl, crawlOpts)) {
-                    yield this.puppeteerControl.narrowSnapshot(x, crawlOpts);
+                    yield this.jsdomControl.narrowSnapshot(x, crawlOpts);
                 }
 
                 return;
@@ -920,7 +927,7 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
         } catch (err: any) {
             if (cache && !(err instanceof SecurityCompromiseError)) {
                 this.logger.warn(`Failed to scrap ${urlToCrawl}, but a stale cache is available. Falling back to cache`, { err: marshalErrorLike(err) });
-                yield this.puppeteerControl.narrowSnapshot(cache.snapshot, crawlOpts);
+                yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);
                 return;
             }
             throw err;
@@ -1051,5 +1058,4 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
 
         return this.formatSnapshot(mode, lastSnapshot, url);
     }
-
 }

backend/functions/src/cloud-functions/searcher.ts

@@ -173,7 +173,7 @@ export class SearcherHost extends RPCHost {
         }
 
         const it = this.fetchSearchResults(crawlerOptions.respondWith, r.web?.results, crawlOpts,
-            { ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance || this.pageCacheToleranceMs },
+            { ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs },
             count,
         );
 

backend/functions/src/services/jsdom.ts

@@ -0,0 +1,206 @@
+import { container, singleton } from 'tsyringe';
+import { AsyncService, marshalErrorLike } from 'civkit';
+import { Logger } from '../shared/services/logger';
+import { ExtendedSnapshot, PageSnapshot } from './puppeteer';
+import { JSDOM, VirtualConsole } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+
+const virtualConsole = new VirtualConsole();
+virtualConsole.on('error', () => void 0);
+
+@singleton()
+export class JSDomControl extends AsyncService {
+
+    logger = this.globalLogger.child({ service: this.constructor.name });
+
+    constructor(
+        protected globalLogger: Logger,
+    ) {
+        super(...arguments);
+    }
+
+    override async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+
+    narrowSnapshot(snapshot: PageSnapshot | undefined, options?: {
+        targetSelector?: string | string[];
+        removeSelector?: string | string[];
+        withIframe?: boolean;
+    }): PageSnapshot | undefined {
+        if (snapshot?.parsed && !options?.targetSelector && !options?.removeSelector && !options?.withIframe) {
+            return snapshot;
+        }
+        if (!snapshot?.html) {
+            return snapshot;
+        }
+
+        const jsdom = new JSDOM(snapshot.html, { url: snapshot.href, virtualConsole });
+        const allNodes: Node[] = [];
+        if (options?.withIframe) {
+            jsdom.window.document.querySelectorAll('iframe[src],frame[src]').forEach((x) => {
+                const src = x.getAttribute('src');
+                const thisSnapshot = snapshot.childFrames?.find((f) => f.href === src);
+                if (thisSnapshot?.html) {
+                    x.innerHTML = thisSnapshot.html;
+                    x.querySelectorAll('script, style').forEach((s) => s.remove());
+                    x.querySelectorAll('[src]').forEach((el) => {
+                        el.setAttribute('src', new URL(el.getAttribute('src')!, src!).toString());
+                    });
+                    x.querySelectorAll('[href]').forEach((el) => {
+                        el.setAttribute('href', new URL(el.getAttribute('href')!, src!).toString());
+                    });
+                }
+            });
+        }
+
+        if (Array.isArray(options?.removeSelector)) {
+            for (const rl of options!.removeSelector) {
+                jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove());
+            }
+        } else if (options?.removeSelector) {
+            jsdom.window.document.querySelectorAll(options.removeSelector).forEach((x) => x.remove());
+        }
+
+        if (Array.isArray(options?.targetSelector)) {
+            for (const x of options!.targetSelector.map((x) => jsdom.window.document.querySelectorAll(x))) {
+                x.forEach((el) => {
+                    if (!allNodes.includes(el)) {
+                        allNodes.push(el);
+                    }
+                });
+            }
+        } else if (options?.targetSelector) {
+            jsdom.window.document.querySelectorAll(options.targetSelector).forEach((el) => {
+                if (!allNodes.includes(el)) {
+                    allNodes.push(el);
+                }
+            });
+        } else {
+            allNodes.push(jsdom.window.document);
+        }
+
+        if (!allNodes.length) {
+            return snapshot;
+        }
+        const textChunks: string[] = [];
+        let rootDoc: Document;
+        if (allNodes.length === 1 && allNodes[0].nodeName === '#document') {
+            rootDoc = allNodes[0] as any;
+            if (rootDoc.body.textContent) {
+                textChunks.push(rootDoc.body.textContent);
+            }
+        } else {
+            rootDoc = new JSDOM('', { url: snapshot.href, virtualConsole }).window.document;
+            for (const n of allNodes) {
+                rootDoc.body.appendChild(n);
+                rootDoc.body.appendChild(rootDoc.createTextNode('\n\n'));
+                if (n.textContent) {
+                    textChunks.push(n.textContent);
+                }
+            }
+        }
+
+        let parsed;
+        try {
+            parsed = new Readability(rootDoc.cloneNode(true) as any).parse();
+        } catch (err: any) {
+            this.logger.warn(`Failed to parse selected element`, { err: marshalErrorLike(err) });
+        }
+
+        // No innerText in jsdom
+        // https://github.com/jsdom/jsdom/issues/1245
+        const textContent = textChunks.join('\n\n');
+        const cleanedText = textContent?.split('\n').map((x: any) => x.trimEnd()).join('\n').replace(/\n{3,}/g, '\n\n');
+
+        const imageTags = Array.from(rootDoc.querySelectorAll('img[src],img[data-src]'))
+            .map((x: any) => [x.getAttribute('src'), x.getAttribute('data-src')])
+            .flat()
+            .map((x) => {
+                try {
+                    return new URL(x, snapshot.href).toString();
+                } catch (err) {
+                    return null;
+                }
+            })
+            .filter(Boolean);
+
+        const imageSet = new Set(imageTags);
+
+        const r = {
+            ...snapshot,
+            title: snapshot.title || jsdom.window.document.title,
+            parsed,
+            html: rootDoc.documentElement.outerHTML,
+            text: cleanedText,
+            imgs: snapshot.imgs?.filter((x) => imageSet.has(x.src)) || [],
+        } as PageSnapshot;
+
+        return r;
+    }
+
+    inferSnapshot(snapshot: PageSnapshot): ExtendedSnapshot {
+        const extendedSnapshot = { ...snapshot } as ExtendedSnapshot;
+        try {
+            const jsdom = new JSDOM(snapshot.html, { url: snapshot.href, virtualConsole });
+            const links = Array.from(jsdom.window.document.querySelectorAll('a[href]'))
+                .map((x: any) => [x.getAttribute('href'), x.textContent.replace(/\s+/g, ' ').trim()])
+                .map(([href, text]) => {
+                    if (!text) {
+                        return undefined;
+                    }
+                    try {
+                        const parsed = new URL(href, snapshot.href);
+                        if (parsed.protocol === 'file:' || parsed.protocol === 'javascript:') {
+                            return undefined;
+                        }
+                        return [parsed.toString(), text] as const;
+                    } catch (err) {
+                        return undefined;
+                    }
+                })
+                .filter(Boolean)
+                .reduce((acc, pair) => {
+                    acc[pair![0]] = pair![1];
+                    return acc;
+                }, {} as { [k: string]: string; });
+
+            extendedSnapshot.links = links;
+
+            const imgs = Array.from(jsdom.window.document.querySelectorAll('img[src],img[data-src]'))
+                .map((x: any) => {
+                    let linkPreferredSrc = x.getAttribute('src') || '';
+                    if (linkPreferredSrc.startsWith('data:')) {
+                        const dataSrc = x.getAttribute('data-src') || '';
+                        if (dataSrc && !dataSrc.startsWith('data:')) {
+                            linkPreferredSrc = dataSrc;
+                        }
+                    }
+
+                    return {
+                        src: new URL(linkPreferredSrc, snapshot.href).toString(),
+                        width: parseInt(x.getAttribute('width') || '0'),
+                        height: parseInt(x.getAttribute('height') || '0'),
+                        alt: x.getAttribute('alt') || x.getAttribute('title'),
+                    };
+                });
+
+            extendedSnapshot.imgs = imgs as any;
+        } catch (_err) {
+            void 0;
+        }
+
+        return extendedSnapshot;
+    }
+
+    snippetToElement(snippet?: string, url?: string) {
+        const parsed = new JSDOM(snippet || '', { url, virtualConsole });
+
+        return parsed.window.document.documentElement;
+    }
+}
+
+const jsdomControl = container.resolve(JSDomControl);
+
+export default jsdomControl;

backend/functions/src/services/puppeteer.ts

@@ -3,7 +3,6 @@ import fs from 'fs';
 import { container, singleton } from 'tsyringe';
 import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit';
 import { Logger } from '../shared/services/logger';
-import { JSDOM, VirtualConsole } from 'jsdom';
 
 import type { Browser, CookieParam, Page } from 'puppeteer';
 import puppeteer from 'puppeteer-extra';
@@ -11,16 +10,12 @@ import puppeteer from 'puppeteer-extra';
 import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
 import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
 import { SecurityCompromiseError, ServiceCrashedError } from '../shared/lib/errors';
-import { Readability } from '@mozilla/readability';
 import { TimeoutError } from 'puppeteer';
 const tldExtract = require('tld-extract');
 
 const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
 
 
-const virtualConsole = new VirtualConsole();
-virtualConsole.on('error', () => void 0);
-
 export interface ImgBrief {
     src: string;
     loaded?: boolean;
@@ -685,175 +680,6 @@ document.addEventListener('load', handlePageLoad);
         return r.filter(Boolean);
     }
 
-    narrowSnapshot(snapshot: PageSnapshot | undefined, options?: {
-        targetSelector?: string | string[];
-        removeSelector?: string | string[];
-        withIframe?: boolean;
-    }): PageSnapshot | undefined {
-        if (snapshot?.parsed && !options?.targetSelector && !options?.removeSelector) {
-            return snapshot;
-        }
-        if (!snapshot?.html) {
-            return snapshot;
-        }
-
-        const jsdom = new JSDOM(snapshot.html, { url: snapshot.href, virtualConsole });
-        const allNodes: Node[] = [];
-        if (options?.withIframe) {
-            jsdom.window.document.querySelectorAll('iframe[src]').forEach((x) => {
-                const src = x.getAttribute('src');
-                const thisSnapshot = snapshot.childFrames?.find((f) => f.href === src);
-                if (thisSnapshot?.html) {
-                    x.innerHTML = thisSnapshot.html;
-                    x.querySelectorAll('script, style').forEach((s) => s.remove());
-                    x.querySelectorAll('[src]').forEach((el) => {
-                        el.setAttribute('src', new URL(el.getAttribute('src')!, src!).toString());
-                    });
-                    x.querySelectorAll('[href]').forEach((el) => {
-                        el.setAttribute('href', new URL(el.getAttribute('href')!, src!).toString());
-                    });
-                }
-            });
-        }
-
-        if (Array.isArray(options?.removeSelector)) {
-            for (const rl of options!.removeSelector) {
-                jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove());
-            }
-        } else if (options?.removeSelector) {
-            jsdom.window.document.querySelectorAll(options.removeSelector).forEach((x) => x.remove());
-        }
-
-        if (Array.isArray(options?.targetSelector)) {
-            for (const x of options!.targetSelector.map((x) => jsdom.window.document.querySelectorAll(x))) {
-                x.forEach((el) => {
-                    if (!allNodes.includes(el)) {
-                        allNodes.push(el);
-                    }
-                });
-            }
-        } else if (options?.targetSelector) {
-            jsdom.window.document.querySelectorAll(options.targetSelector).forEach((el) => {
-                if (!allNodes.includes(el)) {
-                    allNodes.push(el);
-                }
-            });
-        } else {
-            allNodes.push(jsdom.window.document);
-        }
-
-        if (!allNodes.length) {
-            return snapshot;
-        }
-        const textChunks: string[] = [];
-        let rootDoc: Document;
-        if (allNodes.length === 1 && allNodes[0].nodeName === '#document') {
-            rootDoc = allNodes[0] as any;
-            if (rootDoc.body.textContent) {
-                textChunks.push(rootDoc.body.textContent);
-            }
-        } else {
-            rootDoc = new JSDOM('', { url: snapshot.href, virtualConsole }).window.document;
-            for (const n of allNodes) {
-                rootDoc.body.appendChild(n);
-                rootDoc.body.appendChild(rootDoc.createTextNode('\n\n'));
-                if (n.textContent) {
-                    textChunks.push(n.textContent);
-                }
-            }
-        }
-
-        let parsed;
-        try {
-            parsed = new Readability(rootDoc.cloneNode(true) as any).parse();
-        } catch (err: any) {
-            this.logger.warn(`Failed to parse selected element`, { err: marshalErrorLike(err) });
-        }
-
-        // No innerText in jsdom
-        // https://github.com/jsdom/jsdom/issues/1245
-        const textContent = textChunks.join('\n\n');
-        const cleanedText = textContent?.split('\n').map((x: any) => x.trimEnd()).join('\n').replace(/\n{3,}/g, '\n\n');
-
-        const imageTags = Array.from(rootDoc.querySelectorAll('img[src],img[data-src]'))
-            .map((x: any) => [x.getAttribute('src'), x.getAttribute('data-src')])
-            .flat()
-            .map((x) => {
-                try {
-                    return new URL(x, snapshot.href).toString();
-                } catch (err) {
-                    return null;
-                }
-            })
-            .filter(Boolean);
-
-        const imageSet = new Set(imageTags);
-
-        const r = {
-            ...snapshot,
-            title: snapshot.title || jsdom.window.document.title,
-            parsed,
-            html: rootDoc.documentElement.outerHTML,
-            text: cleanedText,
-            imgs: snapshot.imgs?.filter((x) => imageSet.has(x.src)) || [],
-        } as PageSnapshot;
-
-        return r;
-    }
-
-    inferSnapshot(snapshot: PageSnapshot): ExtendedSnapshot {
-        const extendedSnapshot = { ...snapshot } as ExtendedSnapshot;
-        try {
-            const jsdom = new JSDOM(snapshot.html, { url: snapshot.href, virtualConsole });
-            const links = Array.from(jsdom.window.document.querySelectorAll('a[href]'))
-                .map((x: any) => [x.getAttribute('href'), x.textContent.replace(/\s+/g, ' ').trim()])
-                .map(([href, text]) => {
-                    if (!text) {
-                        return undefined;
-                    }
-                    try {
-                        const parsed = new URL(href, snapshot.href);
-                        if (parsed.protocol === 'file:' || parsed.protocol === 'javascript:') {
-                            return undefined;
-                        }
-                        return [parsed.toString(), text] as const;
-                    } catch (err) {
-                        return undefined;
-                    }
-                })
-                .filter(Boolean)
-                .reduce((acc, pair) => {
-                    acc[pair![0]] = pair![1];
-                    return acc;
-                }, {} as { [k: string]: string; });
-
-            extendedSnapshot.links = links;
-
-            const imgs = Array.from(jsdom.window.document.querySelectorAll('img[src],img[data-src]'))
-                .map((x: any) => {
-                    let linkPreferredSrc = x.getAttribute('src') || '';
-                    if (linkPreferredSrc.startsWith('data:')) {
-                        const dataSrc = x.getAttribute('data-src') || '';
-                        if (dataSrc && !dataSrc.startsWith('data:')) {
-                            linkPreferredSrc = dataSrc;
-                        }
-                    }
-
-                    return {
-                        src: new URL(linkPreferredSrc, snapshot.href).toString(),
-                        width: parseInt(x.getAttribute('width') || '0'),
-                        height: parseInt(x.getAttribute('height') || '0'),
-                        alt: x.getAttribute('alt') || x.getAttribute('title'),
-                    };
-                });
-
-            extendedSnapshot.imgs = imgs as any;
-        } catch (_err) {
-            void 0;
-        }
-
-        return extendedSnapshot;
-    }
 }
 
 const puppeteerControl = container.resolve(PuppeteerControl);