feat: compound response

backend/functions/src/cloud-functions/crawler.ts

@@ -310,6 +310,13 @@ export class CrawlerHost extends RPCHost {
             return formatted;
         }
 
+        if (crawlerOptions.isRequestingCompoundContentFormat()) {
+            throw new ParamValidationError({
+                path: 'respondWith',
+                message: `You are requesting compound content format, please explicitly accept 'text/event-stream' or 'application/json' in header.`
+            });
+        }
+
         for await (const scrapped of this.cachedScrap(targetUrl, crawlOpts, crawlerOptions)) {
             lastScrapped = scrapped;
             if (crawlerOptions.waitForSelector || ((!scrapped?.parsed?.content || !scrapped?.title?.trim()) && !scrapped?.pdfs?.length)) {
@@ -600,22 +607,22 @@ export class CrawlerHost extends RPCHost {
             return 0;
         }
 
-        const textContent = formatted?.content || formatted?.description || formatted?.text || formatted?.html;
         let amount = 0;
-        do {
-            if (typeof textContent === 'string') {
-                amount = estimateToken(textContent);
-                break;
-            }
-
-            const imageContent = formatted.screenshotUrl || formatted.screenshot;
-
-            if (imageContent) {
-                // OpenAI image token count for 1024x1024 image
-                amount = 765;
-                break;
-            }
-        } while (false);
+        if (formatted.content) {
+            amount += estimateToken(formatted.content);
+        } else if (formatted.description) {
+            amount += estimateToken(formatted.description);
+        }
+        if (formatted.text) {
+            amount += estimateToken(formatted.text);
+        }
+        if (formatted.html) {
+            amount += estimateToken(formatted.html);
+        }
+        if (formatted.screenshotUrl || formatted.screenshot) {
+            // OpenAI image token count for 1024x1024 image
+            amount += 765;
+        }
 
         Object.assign(formatted, { usage: { tokens: amount } });
 
@@ -684,7 +691,7 @@ export class CrawlerHost extends RPCHost {
         const crawlOpts: ExtraScrappingOptions = {
             proxyUrl: opts.proxyUrl,
             cookies: opts.setCookies,
-            favorScreenshot: ['screenshot', 'pageshot'].includes(opts.respondWith),
+            favorScreenshot: ['screenshot', 'pageshot'].some((x) => opts.respondWith.includes(x)),
             removeSelector: opts.removeSelector,
             targetSelector: opts.targetSelector,
             waitForSelector: opts.waitForSelector,

backend/functions/src/dto/scrapping-options.ts

@@ -3,6 +3,16 @@ import type { Request, Response } from 'express';
 import type { CookieParam } from 'puppeteer';
 import { parseString as parseSetCookieString } from 'set-cookie-parser';
 
+export enum CONTENT_FORMAT {
+    CONTENT = 'content',
+    MARKDOWN = 'markdown',
+    HTML = 'html',
+    TEXT = 'text',
+    PAGESHOT = 'pageshot',
+    SCREENSHOT = 'screenshot',
+}
+
+const CONTENT_FORMAT_VALUES = new Set<string>(Object.values(CONTENT_FORMAT));
 
 @Also({
     openapi: {
@@ -35,7 +45,10 @@ import { parseString as parseSetCookieString } from 'set-cookie-parser';
                         `- html\n` +
                         `- text\n` +
                         `- pageshot\n` +
-                        `- screenshot\n`
+                        `- screenshot\n` +
+                        `- content\n` +
+                        `- any combination of the above\n\n` +
+                        `Default: content\n`
                     ,
                     in: 'header',
                     schema: { type: 'string' }
@@ -149,7 +162,8 @@ export class CrawlerOptions extends AutoCastable {
     pdf?: string;
 
     @Prop({
-        default: 'default',
+        default: CONTENT_FORMAT.CONTENT,
+        type: [CONTENT_FORMAT, String]
     })
     respondWith!: string;
 
@@ -372,6 +386,10 @@ export class CrawlerOptions extends AutoCastable {
 
         return true;
     }
+
+    isRequestingCompoundContentFormat() {
+        return !CONTENT_FORMAT_VALUES.has(this.respondWith);
+    }
 }
 
 export class CrawlerOptionsHeaderOnly extends CrawlerOptions {

backend/functions/src/services/snapshot-formatter.ts

@@ -73,7 +73,10 @@ export class SnapshotFormatter extends AsyncService {
         pageshotUrl?: string;
     }, nominalUrl?: URL, urlValidMs = 3600 * 1000 * 4) {
         const t0 = Date.now();
-        if (mode === 'screenshot') {
+        const f = {
+            ...this.getGeneralSnapshotMixins(snapshot),
+        };
+        if (mode.includes('screenshot')) {
             if (snapshot.screenshot && !snapshot.screenshotUrl) {
                 const fid = `instant-screenshots/${randomUUID()}`;
                 await this.firebaseObjectStorage.saveFile(fid, snapshot.screenshot, {
@@ -84,20 +87,13 @@ export class SnapshotFormatter extends AsyncService {
                 snapshot.screenshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);
             }
 
-            const f = {
-                ...this.getGeneralSnapshotMixins(snapshot),
-                // html: snapshot.html,
+            Object.assign(f, {
                 screenshotUrl: snapshot.screenshotUrl,
-            };
-
-            Object.defineProperty(f, 'textRepresentation', { value: `${f.screenshotUrl}\n`, enumerable: false });
-
-            const dt = Date.now() - t0;
-            this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
+            });
 
-            return f as FormattedPage;
+            Object.defineProperty(f, 'textRepresentation', { value: `${f.screenshotUrl}\n`, enumerable: false, configurable: true });
         }
-        if (mode === 'pageshot') {
+        if (mode.includes('pageshot')) {
             if (snapshot.pageshot && !snapshot.pageshotUrl) {
                 const fid = `instant-screenshots/${randomUUID()}`;
                 await this.firebaseObjectStorage.saveFile(fid, snapshot.pageshot, {
@@ -108,31 +104,18 @@ export class SnapshotFormatter extends AsyncService {
                 snapshot.pageshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs);
             }
 
-            const f = {
-                ...this.getGeneralSnapshotMixins(snapshot),
+            Object.assign(f, {
                 html: snapshot.html,
                 pageshotUrl: snapshot.pageshotUrl,
-            } as FormattedPage;
-
-            Object.defineProperty(f, 'textRepresentation', { value: `${f.pageshotUrl}\n`, enumerable: false });
-
-            const dt = Date.now() - t0;
-            this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
-
-            return f;
+            });
+            Object.defineProperty(f, 'textRepresentation', { value: `${f.pageshotUrl}\n`, enumerable: false, configurable: true });
         }
-        if (mode === 'html') {
-            const f = {
-                ...this.getGeneralSnapshotMixins(snapshot),
+        if (mode.includes('html')) {
+            Object.assign(f, {
                 html: snapshot.html,
-            } as FormattedPage;
-
-            Object.defineProperty(f, 'textRepresentation', { value: snapshot.html, enumerable: false });
-
-            const dt = Date.now() - t0;
-            this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
+            });
 
-            return f;
+            Object.defineProperty(f, 'textRepresentation', { value: snapshot.html, enumerable: false, configurable: true });
         }
 
         let pdfMode = false;
@@ -157,19 +140,20 @@ export class SnapshotFormatter extends AsyncService {
             }
         }
 
-        if (mode === 'text') {
-            const f = {
-                ...this.getGeneralSnapshotMixins(snapshot),
+        if (mode.includes('text')) {
+            Object.assign(f, {
                 text: snapshot.text,
-            } as FormattedPage;
-
-            Object.defineProperty(f, 'textRepresentation', { value: snapshot.text, enumerable: false });
+            });
+            Object.defineProperty(f, 'textRepresentation', { value: snapshot.text, enumerable: false, configurable: true });
+        }
 
+        if (!mode.includes('markdown') && !mode.includes('content')) {
             const dt = Date.now() - t0;
             this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
 
             return f;
         }
+
         const imgDataUrlToObjectUrl = !Boolean(this.threadLocal.get('keepImgDataUrl'));
 
         let contentText = '';
@@ -178,7 +162,7 @@ export class SnapshotFormatter extends AsyncService {
         const uid = this.threadLocal.get('uid');
         do {
             if (pdfMode) {
-                contentText = snapshot.parsed?.content || snapshot.text;
+                contentText = (snapshot.parsed?.content || snapshot.text || '').trim();
                 break;
             }
 
@@ -188,14 +172,14 @@ export class SnapshotFormatter extends AsyncService {
                 snapshot.elemCount! > 70_000
             ) {
                 this.logger.warn('Degrading to text to protect the server', { url: snapshot.href });
-                contentText = snapshot.text;
+                contentText = (snapshot.text || '').trimEnd();
                 break;
             }
 
             const jsDomElementOfHTML = this.jsdomControl.snippetToElement(snapshot.html, snapshot.href);
             let toBeTurnedToMd = jsDomElementOfHTML;
             let turnDownService = this.getTurndown({ url: snapshot.rebase || nominalUrl, imgDataUrlToObjectUrl });
-            if (mode !== 'markdown' && snapshot.parsed?.content) {
+            if (!mode.includes('markdown') && snapshot.parsed?.content) {
                 const jsDomElementOfParsed = this.jsdomControl.snippetToElement(snapshot.parsed.content, snapshot.href);
                 const par1 = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML);
                 const par2 = snapshot.parsed.content ? this.jsdomControl.runTurndown(turnDownService, jsDomElementOfParsed) : '';
@@ -300,29 +284,27 @@ export class SnapshotFormatter extends AsyncService {
             ) {
                 toBeTurnedToMd = jsDomElementOfHTML;
                 try {
-                    contentText = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML);
+                    contentText = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML).trim();
                 } catch (err) {
                     this.logger.warn(`Turndown failed to run, retrying without plugins`, { err });
                     const vanillaTurnDownService = this.getTurndown({ url: snapshot.rebase || nominalUrl, imgDataUrlToObjectUrl });
                     try {
-                        contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, jsDomElementOfHTML);
+                        contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, jsDomElementOfHTML).trim();
                     } catch (err2) {
                         this.logger.warn(`Turndown failed to run, giving up`, { err: err2 });
                     }
                 }
             }
             if (this.isPoorlyTransformed(contentText, toBeTurnedToMd)) {
-                contentText = snapshot.text;
+                contentText = (snapshot.text || '').trimEnd();
             }
         } while (false);
 
-        const cleanText = contentText?.includes('return') ? contentText.trimEnd() : (contentText || '').trim();
-
         const formatted: FormattedPage = {
             title: (snapshot.parsed?.title || snapshot.title || '').trim(),
             description: (snapshot.description || '').trim(),
             url: nominalUrl?.toString() || snapshot.href?.trim(),
-            content: cleanText,
+            content: contentText,
             publishedTime: snapshot.parsed?.publishedTime || undefined,
             [Symbol.dispose]: () => { },
         };
@@ -351,8 +333,10 @@ export class SnapshotFormatter extends AsyncService {
             formatted.links = _.invert(this.jsdomControl.inferSnapshot(snapshot).links || {});
         }
 
+        Object.assign(f, formatted);
+
         const textRepresentation = (function (this: typeof formatted) {
-            if (mode === 'markdown') {
+            if (mode.includes('markdown')) {
                 return this.content as string;
             }
 
@@ -395,12 +379,12 @@ ${this.content}
 ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
         }).call(formatted);
 
-        Object.defineProperty(formatted, 'textRepresentation', { value: textRepresentation, enumerable: false });
+        Object.defineProperty(f, 'textRepresentation', { value: textRepresentation, enumerable: false });
 
         const dt = Date.now() - t0;
         this.logger.info(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt });
 
-        return formatted as FormattedPage;
+        return f as FormattedPage;
     }
 
     getGeneralSnapshotMixins(snapshot: PageSnapshot) {

thinapps-shared

@@ -1 +1 @@
-Subproject commit 196d86ad8bc0cb6a9fee3c75df28a6a660c8f17e
+Subproject commit 06cc23b16cafdd17c7e7db996f2167a39cc6d1eb