Include pre-built files for HF deployment

.gitignore

@@ -69,7 +69,7 @@ node_modules/
 toy*.ts
 
 .DS_Store
-build/
+# build/  # Commented out for HF deployment - need pre-built files
 .firebase-emu/
 *.log
 .DS_Store

build/api/crawler.js

@@ -0,0 +1,1170 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a, _b, _c, _d, _e, _f;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlerHost = void 0;
+const tsyringe_1 = require("tsyringe");
+const url_1 = require("url");
+const crypto_1 = require("crypto");
+const lodash_1 = __importDefault(require("lodash"));
+const civ_rpc_1 = require("civkit/civ-rpc");
+const lang_1 = require("civkit/lang");
+const defer_1 = require("civkit/defer");
+const decorators_1 = require("civkit/decorators");
+const fancy_file_1 = require("civkit/fancy-file");
+const crawler_options_1 = require("../dto/crawler-options");
+const crawled_1 = require("../db/crawled");
+const domain_blockade_1 = require("../db/domain-blockade");
+const transform_server_event_stream_1 = require("../lib/transform-server-event-stream");
+const puppeteer_1 = require("../services/puppeteer");
+const jsdom_1 = require("../services/jsdom");
+const snapshot_formatter_1 = require("../services/snapshot-formatter");
+const curl_1 = require("../services/curl");
+const lm_1 = require("../services/lm");
+const misc_1 = require("../utils/misc");
+const cf_browser_rendering_1 = require("../services/cf-browser-rendering");
+const logger_1 = require("../services/logger");
+const rate_limit_1 = require("../shared/services/rate-limit");
+const async_context_1 = require("../services/async-context");
+const registry_1 = require("../services/registry");
+const errors_1 = require("../services/errors");
+const openai_1 = require("../shared/utils/openai");
+const proxy_provider_1 = require("../shared/services/proxy-provider");
+const firebase_storage_bucket_1 = require("../shared/services/firebase-storage-bucket");
+const jina_embeddings_auth_1 = require("../dto/jina-embeddings-auth");
+const robots_text_1 = require("../services/robots-text");
+const temp_file_1 = require("../services/temp-file");
+const misc_2 = require("../services/misc");
+const http_1 = require("civkit/http");
+const geoip_1 = require("../services/geoip");
+const indexProto = {
+    toString: function () {
+        return (0, lodash_1.default)(this)
+            .toPairs()
+            .map(([k, v]) => k ? `[${lodash_1.default.upperFirst(lodash_1.default.lowerCase(k))}] ${v}` : '')
+            .value()
+            .join('\n') + '\n';
+    }
+};
+let CrawlerHost = class CrawlerHost extends civ_rpc_1.RPCHost {
+    constructor(globalLogger, puppeteerControl, curlControl, cfBrowserRendering, proxyProvider, lmControl, jsdomControl, snapshotFormatter, firebaseObjectStorage, rateLimitControl, threadLocal, robotsTxtService, tempFileManager, geoIpService, miscService) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.puppeteerControl = puppeteerControl;
+        this.curlControl = curlControl;
+        this.cfBrowserRendering = cfBrowserRendering;
+        this.proxyProvider = proxyProvider;
+        this.lmControl = lmControl;
+        this.jsdomControl = jsdomControl;
+        this.snapshotFormatter = snapshotFormatter;
+        this.firebaseObjectStorage = firebaseObjectStorage;
+        this.rateLimitControl = rateLimitControl;
+        this.threadLocal = threadLocal;
+        this.robotsTxtService = robotsTxtService;
+        this.tempFileManager = tempFileManager;
+        this.geoIpService = geoIpService;
+        this.miscService = miscService;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.cacheRetentionMs = 1000 * 3600 * 24 * 7;
+        this.cacheValidMs = 1000 * 3600;
+        this.urlValidMs = 1000 * 3600 * 4;
+        this.abuseBlockMs = 1000 * 3600;
+        this.domainProfileRetentionMs = 1000 * 3600 * 24 * 30;
+        this.batchedCaches = [];
+        this.proxyIterMap = new WeakMap();
+        puppeteerControl.on('crawled', async (snapshot, options) => {
+            if (!snapshot.title?.trim() && !snapshot.pdfs?.length) {
+                return;
+            }
+            if (options.cookies?.length || options.private) {
+                // Potential privacy issue, dont cache if cookies are used
+                return;
+            }
+            if (options.injectFrameScripts?.length || options.injectPageScripts?.length || options.viewport) {
+                // Potentially mangeled content, dont cache if scripts are injected
+                return;
+            }
+            if (snapshot.isIntermediate) {
+                return;
+            }
+            if (!snapshot.lastMutationIdle) {
+                // Never reached mutationIdle, presumably too short timeout
+                return;
+            }
+            if (options.locale) {
+                Reflect.set(snapshot, 'locale', options.locale);
+            }
+            const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);
+            if (analyzed.tokens < 200) {
+                // Does not contain enough content
+                if (snapshot.status !== 200) {
+                    return;
+                }
+                if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile')) {
+                    return;
+                }
+            }
+            await this.setToCache(options.url, snapshot);
+        });
+        puppeteerControl.on('abuse', async (abuseEvent) => {
+            this.logger.warn(`Abuse detected on ${abuseEvent.url}, blocking ${abuseEvent.url.hostname}`, { reason: abuseEvent.reason, sn: abuseEvent.sn });
+            await domain_blockade_1.DomainBlockade.save(domain_blockade_1.DomainBlockade.from({
+                domain: abuseEvent.url.hostname.toLowerCase(),
+                triggerReason: `${abuseEvent.reason}`,
+                triggerUrl: abuseEvent.url.toString(),
+                createdAt: new Date(),
+                expireAt: new Date(Date.now() + this.abuseBlockMs),
+            })).catch((err) => {
+                this.logger.warn(`Failed to save domain blockade for ${abuseEvent.url.hostname}`, { err: (0, lang_1.marshalErrorLike)(err) });
+            });
+        });
+        setInterval(() => {
+            const thisBatch = this.batchedCaches;
+            this.batchedCaches = [];
+            if (!thisBatch.length) {
+                return;
+            }
+            const batch = crawled_1.Crawled.DB.batch();
+            for (const x of thisBatch) {
+                batch.set(crawled_1.Crawled.COLLECTION.doc(x._id), x.degradeForFireStore(), { merge: true });
+            }
+            batch.commit()
+                .then(() => {
+                this.logger.debug(`Saved ${thisBatch.length} caches by batch`);
+            })
+                .catch((err) => {
+                this.logger.warn(`Failed to save cache in batch`, { err });
+            });
+        }, 1000 * 10 + Math.round(1000 * Math.random())).unref();
+    }
+    async init() {
+        await this.dependencyReady();
+        if (this.puppeteerControl.effectiveUA) {
+            this.curlControl.impersonateChrome(this.puppeteerControl.effectiveUA);
+        }
+        this.emit('ready');
+    }
+    async getIndex(auth) {
+        const indexObject = Object.create(indexProto);
+        Object.assign(indexObject, {
+            usage1: 'https://r.jina.ai/YOUR_URL',
+            usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',
+            homepage: 'https://jina.ai/reader',
+        });
+        await auth?.solveUID();
+        if (auth && auth.user) {
+            indexObject[''] = undefined;
+            indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`;
+            indexObject.balanceLeft = auth.user.wallet.total_balance;
+        }
+        return indexObject;
+    }
+    async getIndexCtrl(ctx, auth) {
+        const indexObject = await this.getIndex(auth);
+        if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
+            return indexObject;
+        }
+        return (0, civ_rpc_1.assignTransferProtocolMeta)(`${indexObject}`, { contentType: 'text/plain; charset=utf-8', envelope: null });
+    }
+    async crawl(rpcReflect, ctx, auth, crawlerOptionsHeaderOnly, crawlerOptionsParamsAllowed) {
+        const uid = await auth.solveUID();
+        let chargeAmount = 0;
+        const crawlerOptions = ctx.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
+        const tierPolicy = await this.saasAssertTierPolicy(crawlerOptions, auth);
+        // Use koa ctx.URL, a standard URL object to avoid node.js framework prop naming confusion
+        const targetUrl = await this.getTargetUrl((0, misc_1.tryDecodeURIComponent)(`${ctx.URL.pathname}${ctx.URL.search}`), crawlerOptions);
+        if (!targetUrl) {
+            return await this.getIndex(auth);
+        }
+        // Prevent circular crawling
+        this.puppeteerControl.circuitBreakerHosts.add(ctx.hostname.toLowerCase());
+        if (uid) {
+            const user = await auth.assertUser();
+            if (!(user.wallet.total_balance > 0)) {
+                throw new errors_1.InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);
+            }
+            const rateLimitPolicy = auth.getRateLimits('CRAWL') || [
+                parseInt(user.metadata?.speed_level) >= 2 ?
+                    rate_limit_1.RateLimitDesc.from({
+                        occurrence: 5000,
+                        periodSeconds: 60
+                    }) :
+                    rate_limit_1.RateLimitDesc.from({
+                        occurrence: 500,
+                        periodSeconds: 60
+                    })
+            ];
+            const apiRoll = await this.rateLimitControl.simpleRPCUidBasedLimit(rpcReflect, uid, ['CRAWL'], ...rateLimitPolicy);
+            rpcReflect.finally(() => {
+                if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) {
+                    return;
+                }
+                if (chargeAmount) {
+                    auth.reportUsage(chargeAmount, `reader-crawl`).catch((err) => {
+                        this.logger.warn(`Unable to report usage for ${uid}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                    });
+                    apiRoll.chargeAmount = chargeAmount;
+                }
+            });
+        }
+        else if (ctx.ip) {
+            const apiRoll = await this.rateLimitControl.simpleRpcIPBasedLimit(rpcReflect, ctx.ip, ['CRAWL'], [
+                // 20 requests per minute
+                new Date(Date.now() - 60 * 1000), 20
+            ]);
+            rpcReflect.finally(() => {
+                if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) {
+                    return;
+                }
+                apiRoll.chargeAmount = chargeAmount;
+            });
+        }
+        if (!uid) {
+            // Enforce no proxy is allocated for anonymous users due to abuse.
+            crawlerOptions.proxy = 'none';
+            const blockade = (await domain_blockade_1.DomainBlockade.fromFirestoreQuery(domain_blockade_1.DomainBlockade.COLLECTION
+                .where('domain', '==', targetUrl.hostname.toLowerCase())
+                .where('expireAt', '>=', new Date())
+                .limit(1)))[0];
+            if (blockade) {
+                throw new errors_1.SecurityCompromiseError(`Domain ${targetUrl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`);
+            }
+        }
+        const crawlOpts = await this.configure(crawlerOptions);
+        this.logger.info(`Accepting request from ${uid || ctx.ip}`, { opts: crawlerOptions });
+        if (crawlerOptions.robotsTxt) {
+            await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt);
+        }
+        if (rpcReflect.signal.aborted) {
+            return;
+        }
+        if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {
+            const sseStream = new transform_server_event_stream_1.OutputServerEventStream();
+            rpcReflect.return(sseStream);
+            try {
+                for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {
+                    if (!scrapped) {
+                        continue;
+                    }
+                    if (rpcReflect.signal.aborted) {
+                        break;
+                    }
+                    const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);
+                    chargeAmount = this.assignChargeAmount(formatted, tierPolicy);
+                    sseStream.write({
+                        event: 'data',
+                        data: formatted,
+                    });
+                    if (chargeAmount && scrapped.pdfs?.length) {
+                        break;
+                    }
+                }
+            }
+            catch (err) {
+                this.logger.error(`Failed to crawl ${targetUrl}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                sseStream.write({
+                    event: 'error',
+                    data: (0, lang_1.marshalErrorLike)(err),
+                });
+            }
+            sseStream.end();
+            return sseStream;
+        }
+        let lastScrapped;
+        if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
+            try {
+                for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {
+                    lastScrapped = scrapped;
+                    if (rpcReflect.signal.aborted) {
+                        break;
+                    }
+                    if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {
+                        continue;
+                    }
+                    if (!scrapped.title) {
+                        continue;
+                    }
+                    const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);
+                    chargeAmount = this.assignChargeAmount(formatted, tierPolicy);
+                    if (scrapped?.pdfs?.length && !chargeAmount) {
+                        continue;
+                    }
+                    return formatted;
+                }
+            }
+            catch (err) {
+                if (!lastScrapped) {
+                    throw err;
+                }
+            }
+            if (!lastScrapped) {
+                if (crawlOpts.targetSelector) {
+                    throw new civ_rpc_1.AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`);
+                }
+                throw new civ_rpc_1.AssertionFailureError(`No content available for URL ${targetUrl}`);
+            }
+            const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts);
+            chargeAmount = this.assignChargeAmount(formatted, tierPolicy);
+            return formatted;
+        }
+        if (crawlerOptions.isRequestingCompoundContentFormat()) {
+            throw new civ_rpc_1.ParamValidationError({
+                path: 'respondWith',
+                message: `You are requesting compound content format, please explicitly accept 'text/event-stream' or 'application/json' in header.`
+            });
+        }
+        try {
+            for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) {
+                lastScrapped = scrapped;
+                if (rpcReflect.signal.aborted) {
+                    break;
+                }
+                if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) {
+                    continue;
+                }
+                if (!scrapped.title) {
+                    continue;
+                }
+                const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts);
+                chargeAmount = this.assignChargeAmount(formatted, tierPolicy);
+                if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
+                    return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } });
+                }
+                if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {
+                    return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } });
+                }
+                return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null });
+            }
+        }
+        catch (err) {
+            if (!lastScrapped) {
+                throw err;
+            }
+        }
+        if (!lastScrapped) {
+            if (crawlOpts.targetSelector) {
+                throw new civ_rpc_1.AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`);
+            }
+            throw new civ_rpc_1.AssertionFailureError(`No content available for URL ${targetUrl}`);
+        }
+        const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts);
+        chargeAmount = this.assignChargeAmount(formatted, tierPolicy);
+        if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
+            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } });
+        }
+        if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) {
+            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } });
+        }
+        return (0, civ_rpc_1.assignTransferProtocolMeta)(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null });
+    }
+    async getTargetUrl(originPath, crawlerOptions) {
+        let url = '';
+        const targetUrlFromGet = originPath.slice(1);
+        if (crawlerOptions.pdf) {
+            const pdfFile = crawlerOptions.pdf;
+            const identifier = pdfFile instanceof fancy_file_1.FancyFile ? (await pdfFile.sha256Sum) : (0, crypto_1.randomUUID)();
+            url = `blob://pdf/${identifier}`;
+            crawlerOptions.url ??= url;
+        }
+        else if (targetUrlFromGet) {
+            url = targetUrlFromGet.trim();
+        }
+        else if (crawlerOptions.url) {
+            url = crawlerOptions.url.trim();
+        }
+        if (!url) {
+            throw new civ_rpc_1.ParamValidationError({
+                message: 'No URL provided',
+                path: 'url'
+            });
+        }
+        const { url: safeURL, ips } = await this.miscService.assertNormalizedUrl(url);
+        if (this.puppeteerControl.circuitBreakerHosts.has(safeURL.hostname.toLowerCase())) {
+            throw new errors_1.SecurityCompromiseError({
+                message: `Circular hostname: ${safeURL.protocol}`,
+                path: 'url'
+            });
+        }
+        crawlerOptions._hintIps = ips;
+        return safeURL;
+    }
+    getUrlDigest(urlToCrawl) {
+        const normalizedURL = new URL(urlToCrawl);
+        if (!normalizedURL.hash.startsWith('#/')) {
+            normalizedURL.hash = '';
+        }
+        const normalizedUrl = normalizedURL.toString().toLowerCase();
+        const digest = snapshot_formatter_1.md5Hasher.hash(normalizedUrl.toString());
+        return digest;
+    }
+    async *queryCache(urlToCrawl, cacheTolerance) {
+        const digest = this.getUrlDigest(urlToCrawl);
+        const cache = (await (crawled_1.Crawled.fromFirestoreQuery(crawled_1.Crawled.COLLECTION.where('urlPathDigest', '==', digest).orderBy('createdAt', 'desc').limit(1)).catch((err) => {
+            this.logger.warn(`Failed to query cache, unknown issue`, { err });
+            // https://github.com/grpc/grpc-node/issues/2647
+            // https://github.com/googleapis/nodejs-firestore/issues/1023
+            // https://github.com/googleapis/nodejs-firestore/issues/1023
+            return undefined;
+        })))?.[0];
+        yield cache;
+        if (!cache) {
+            return;
+        }
+        const age = Date.now() - cache.createdAt.valueOf();
+        const stale = cache.createdAt.valueOf() < (Date.now() - cacheTolerance);
+        this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for ${urlToCrawl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, {
+            url: urlToCrawl, digest, age, stale, cacheTolerance
+        });
+        let snapshot;
+        let screenshotUrl;
+        let pageshotUrl;
+        const preparations = [
+            this.firebaseObjectStorage.downloadFile(`snapshots/${cache._id}`).then((r) => {
+                snapshot = JSON.parse(r.toString('utf-8'));
+            }),
+            cache.screenshotAvailable ?
+                this.firebaseObjectStorage.signDownloadUrl(`screenshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => {
+                    screenshotUrl = r;
+                }) :
+                Promise.resolve(undefined),
+            cache.pageshotAvailable ?
+                this.firebaseObjectStorage.signDownloadUrl(`pageshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => {
+                    pageshotUrl = r;
+                }) :
+                Promise.resolve(undefined)
+        ];
+        try {
+            await Promise.all(preparations);
+        }
+        catch (_err) {
+            // Swallow cache errors.
+            return undefined;
+        }
+        yield {
+            isFresh: !stale,
+            ...cache,
+            snapshot: {
+                ...snapshot,
+                screenshot: undefined,
+                pageshot: undefined,
+                screenshotUrl,
+                pageshotUrl,
+            }
+        };
+    }
+    async setToCache(urlToCrawl, snapshot) {
+        const digest = this.getUrlDigest(urlToCrawl);
+        this.logger.info(`Caching snapshot of ${urlToCrawl}...`, { url: urlToCrawl, digest, title: snapshot?.title, href: snapshot?.href });
+        const nowDate = new Date();
+        const cache = crawled_1.Crawled.from({
+            _id: (0, crypto_1.randomUUID)(),
+            url: urlToCrawl.toString(),
+            createdAt: nowDate,
+            expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs),
+            htmlSignificantlyModifiedByJs: snapshot.htmlSignificantlyModifiedByJs,
+            urlPathDigest: digest,
+        });
+        const savingOfSnapshot = this.firebaseObjectStorage.saveFile(`snapshots/${cache._id}`, Buffer.from(JSON.stringify({
+            ...snapshot,
+            screenshot: undefined,
+            pageshot: undefined,
+        }), 'utf-8'), {
+            metadata: {
+                contentType: 'application/json',
+            }
+        }).then((r) => {
+            cache.snapshotAvailable = true;
+            return r;
+        });
+        if (snapshot.screenshot) {
+            await this.firebaseObjectStorage.saveFile(`screenshots/${cache._id}`, snapshot.screenshot, {
+                metadata: {
+                    contentType: 'image/png',
+                }
+            });
+            cache.screenshotAvailable = true;
+        }
+        if (snapshot.pageshot) {
+            await this.firebaseObjectStorage.saveFile(`pageshots/${cache._id}`, snapshot.pageshot, {
+                metadata: {
+                    contentType: 'image/png',
+                }
+            });
+            cache.pageshotAvailable = true;
+        }
+        await savingOfSnapshot;
+        this.batchedCaches.push(cache);
+        // const r = await Crawled.save(cache.degradeForFireStore()).catch((err) => {
+        //     this.logger.error(`Failed to save cache for ${urlToCrawl}`, { err: marshalErrorLike(err) });
+        //     return undefined;
+        // });
+        return cache;
+    }
+    async *iterSnapshots(urlToCrawl, crawlOpts, crawlerOpts) {
+        // if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.VLM)) {
+        //     const finalBrowserSnapshot = await this.getFinalSnapshot(urlToCrawl, {
+        //         ...crawlOpts, engine: ENGINE_TYPE.BROWSER
+        //     }, crawlerOpts);
+        //     yield* this.lmControl.geminiFromBrowserSnapshot(finalBrowserSnapshot);
+        //     return;
+        // }
+        if (crawlerOpts?.respondWith.includes(crawler_options_1.CONTENT_FORMAT.READER_LM)) {
+            const finalAutoSnapshot = await this.getFinalSnapshot(urlToCrawl, {
+                ...crawlOpts,
+                engine: crawlOpts?.engine || crawler_options_1.ENGINE_TYPE.AUTO,
+            }, crawler_options_1.CrawlerOptions.from({
+                ...crawlerOpts,
+                respondWith: 'html',
+            }));
+            if (!finalAutoSnapshot?.html) {
+                throw new civ_rpc_1.AssertionFailureError(`Unexpected non HTML content for ReaderLM: ${urlToCrawl}`);
+            }
+            if (crawlerOpts?.instruction || crawlerOpts?.jsonSchema) {
+                const jsonSchema = crawlerOpts.jsonSchema ? JSON.stringify(crawlerOpts.jsonSchema, undefined, 2) : undefined;
+                yield* this.lmControl.readerLMFromSnapshot(crawlerOpts.instruction, jsonSchema, finalAutoSnapshot);
+                return;
+            }
+            try {
+                yield* this.lmControl.readerLMMarkdownFromSnapshot(finalAutoSnapshot);
+            }
+            catch (err) {
+                if (err instanceof http_1.HTTPServiceError && err.status === 429) {
+                    throw new errors_1.ServiceNodeResourceDrainError(`Reader LM is at capacity, please try again later.`);
+                }
+                throw err;
+            }
+            return;
+        }
+        yield* this.cachedScrap(urlToCrawl, crawlOpts, crawlerOpts);
+    }
+    async *cachedScrap(urlToCrawl, crawlOpts, crawlerOpts) {
+        if (crawlerOpts?.html) {
+            const snapshot = {
+                href: urlToCrawl.toString(),
+                html: crawlerOpts.html,
+                title: '',
+                text: '',
+            };
+            yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);
+            return;
+        }
+        if (crawlerOpts?.pdf) {
+            const pdfFile = crawlerOpts.pdf instanceof fancy_file_1.FancyFile ? crawlerOpts.pdf : this.tempFileManager.cacheBuffer(Buffer.from(crawlerOpts.pdf, 'base64'));
+            const pdfLocalPath = (0, url_1.pathToFileURL)((await pdfFile.filePath));
+            const snapshot = {
+                href: urlToCrawl.toString(),
+                html: `<!DOCTYPE html><html><head></head><body style="height: 100%; width: 100%; overflow: hidden; margin:0px; background-color: rgb(82, 86, 89);"><embed style="position:absolute; left: 0; top: 0;" width="100%" height="100%" src="${crawlerOpts.url}"></body></html>`,
+                title: '',
+                text: '',
+                pdfs: [pdfLocalPath.href],
+            };
+            yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);
+            return;
+        }
+        if (crawlOpts?.engine === crawler_options_1.ENGINE_TYPE.CURL ||
+            // deprecated name
+            crawlOpts?.engine === 'direct') {
+            let sideLoaded;
+            try {
+                sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ?
+                    await this.sideLoadWithAllocatedProxy(urlToCrawl, crawlOpts) :
+                    await this.curlControl.sideLoad(urlToCrawl, crawlOpts);
+            }
+            catch (err) {
+                if (err instanceof errors_1.ServiceBadAttemptError) {
+                    throw new civ_rpc_1.AssertionFailureError(err.message);
+                }
+                throw err;
+            }
+            if (!sideLoaded?.file) {
+                throw new civ_rpc_1.AssertionFailureError(`Remote server did not return a body: ${urlToCrawl}`);
+            }
+            const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile(urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName);
+            draftSnapshot.status = sideLoaded.status;
+            draftSnapshot.statusText = sideLoaded.statusText;
+            yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts);
+            return;
+        }
+        if (crawlOpts?.engine === crawler_options_1.ENGINE_TYPE.CF_BROWSER_RENDERING) {
+            const html = await this.cfBrowserRendering.fetchContent(urlToCrawl.href);
+            const snapshot = {
+                href: urlToCrawl.toString(),
+                html,
+                title: '',
+                text: '',
+            };
+            yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts);
+            return;
+        }
+        const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs;
+        const cacheIt = this.queryCache(urlToCrawl, cacheTolerance);
+        let cache = (await cacheIt.next()).value;
+        if (cache?.htmlSignificantlyModifiedByJs === false) {
+            if (crawlerOpts && crawlerOpts.timeout === undefined) {
+                crawlerOpts.respondTiming ??= crawler_options_1.RESPOND_TIMING.HTML;
+            }
+        }
+        if (!crawlerOpts || crawlerOpts.isCacheQueryApplicable()) {
+            cache = (await cacheIt.next()).value;
+        }
+        cacheIt.return(undefined);
+        if (cache?.isFresh &&
+            (!crawlOpts?.favorScreenshot || (crawlOpts?.favorScreenshot && (cache.screenshotAvailable && cache.pageshotAvailable))) &&
+            (lodash_1.default.get(cache.snapshot, 'locale') === crawlOpts?.locale)) {
+            if (cache.snapshot) {
+                cache.snapshot.isFromCache = true;
+            }
+            yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);
+            return;
+        }
+        if (crawlOpts?.engine !== crawler_options_1.ENGINE_TYPE.BROWSER && !this.knownUrlThatSideLoadingWouldCrashTheBrowser(urlToCrawl)) {
+            const sideLoadSnapshotPermitted = crawlerOpts?.browserIsNotRequired() &&
+                [crawler_options_1.RESPOND_TIMING.HTML, crawler_options_1.RESPOND_TIMING.VISIBLE_CONTENT].includes(crawlerOpts.presumedRespondTiming);
+            try {
+                const altOpts = { ...crawlOpts };
+                let sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ?
+                    await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts) :
+                    await this.curlControl.sideLoad(urlToCrawl, altOpts).catch((err) => {
+                        this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: (0, lang_1.marshalErrorLike)(err), href: urlToCrawl.href });
+                        if (err instanceof civ_rpc_1.ApplicationError && !(err instanceof errors_1.ServiceBadAttemptError)) {
+                            return Promise.reject(err);
+                        }
+                        return this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts);
+                    });
+                if (!sideLoaded.file) {
+                    throw new errors_1.ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`);
+                }
+                const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile(urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName).catch((err) => {
+                    if (err instanceof civ_rpc_1.ApplicationError) {
+                        return Promise.reject(new errors_1.ServiceBadAttemptError(err.message));
+                    }
+                    return Promise.reject(err);
+                });
+                draftSnapshot.status = sideLoaded.status;
+                draftSnapshot.statusText = sideLoaded.statusText;
+                if (sideLoaded.status == 200 && !sideLoaded.contentType.startsWith('text/html')) {
+                    yield draftSnapshot;
+                    return;
+                }
+                let analyzed = await this.jsdomControl.analyzeHTMLTextLite(draftSnapshot.html);
+                draftSnapshot.title ??= analyzed.title;
+                draftSnapshot.isIntermediate = true;
+                if (sideLoadSnapshotPermitted) {
+                    yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts);
+                }
+                let fallbackProxyIsUsed = false;
+                if (((!crawlOpts?.allocProxy || crawlOpts.allocProxy !== 'none') && !crawlOpts?.proxyUrl) &&
+                    (analyzed.tokens < 42 || sideLoaded.status !== 200)) {
+                    const proxyLoaded = await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts);
+                    if (!proxyLoaded.file) {
+                        throw new errors_1.ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`);
+                    }
+                    const proxySnapshot = await this.snapshotFormatter.createSnapshotFromFile(urlToCrawl, proxyLoaded.file, proxyLoaded.contentType, proxyLoaded.fileName).catch((err) => {
+                        if (err instanceof civ_rpc_1.ApplicationError) {
+                            return Promise.reject(new errors_1.ServiceBadAttemptError(err.message));
+                        }
+                        return Promise.reject(err);
+                    });
+                    proxySnapshot.status = proxyLoaded.status;
+                    proxySnapshot.statusText = proxyLoaded.statusText;
+                    if (proxyLoaded.status === 200 && crawlerOpts?.browserIsNotRequired()) {
+                    }
+                    analyzed = await this.jsdomControl.analyzeHTMLTextLite(proxySnapshot.html);
+                    if (proxyLoaded.status === 200 || analyzed.tokens >= 200) {
+                        proxySnapshot.isIntermediate = true;
+                        if (sideLoadSnapshotPermitted) {
+                            yield this.jsdomControl.narrowSnapshot(proxySnapshot, crawlOpts);
+                        }
+                        sideLoaded = proxyLoaded;
+                        fallbackProxyIsUsed = true;
+                    }
+                }
+                if (crawlOpts && (sideLoaded.status === 200 || analyzed.tokens >= 200 || crawlOpts.allocProxy)) {
+                    this.logger.info(`Side load seems to work, applying to crawler.`, { url: urlToCrawl.href });
+                    crawlOpts.sideLoad ??= sideLoaded.sideLoadOpts;
+                    if (fallbackProxyIsUsed) {
+                        this.logger.info(`Proxy seems to salvage the page`, { url: urlToCrawl.href });
+                    }
+                }
+            }
+            catch (err) {
+                this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: (0, lang_1.marshalErrorLike)(err), href: urlToCrawl.href });
+                if (err instanceof civ_rpc_1.ApplicationError &&
+                    !(err instanceof errors_1.ServiceBadAttemptError) &&
+                    !(err instanceof civ_rpc_1.DataStreamBrokenError)) {
+                    throw err;
+                }
+            }
+        }
+        else if (crawlOpts?.allocProxy && crawlOpts.allocProxy !== 'none' && !crawlOpts.proxyUrl) {
+            const proxyUrl = await this.proxyProvider.alloc(this.figureOutBestProxyCountry(crawlOpts));
+            crawlOpts.proxyUrl = proxyUrl.href;
+        }
+        try {
+            if (crawlOpts?.targetSelector || crawlOpts?.removeSelector || crawlOpts?.withIframe || crawlOpts?.withShadowDom) {
+                for await (const x of this.puppeteerControl.scrap(urlToCrawl, crawlOpts)) {
+                    yield this.jsdomControl.narrowSnapshot(x, crawlOpts);
+                }
+                return;
+            }
+            yield* this.puppeteerControl.scrap(urlToCrawl, crawlOpts);
+        }
+        catch (err) {
+            if (cache && !(err instanceof errors_1.SecurityCompromiseError)) {
+                this.logger.warn(`Failed to scrap ${urlToCrawl}, but a stale cache is available. Falling back to cache`, { err: (0, lang_1.marshalErrorLike)(err) });
+                yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts);
+                return;
+            }
+            throw err;
+        }
+    }
+    assignChargeAmount(formatted, saasTierPolicy) {
+        if (!formatted) {
+            return 0;
+        }
+        let amount = 0;
+        if (formatted.content) {
+            amount = (0, openai_1.countGPTToken)(formatted.content);
+        }
+        else if (formatted.description) {
+            amount += (0, openai_1.countGPTToken)(formatted.description);
+        }
+        if (formatted.text) {
+            amount += (0, openai_1.countGPTToken)(formatted.text);
+        }
+        if (formatted.html) {
+            amount += (0, openai_1.countGPTToken)(formatted.html);
+        }
+        if (formatted.screenshotUrl || formatted.screenshot) {
+            // OpenAI image token count for 1024x1024 image
+            amount += 765;
+        }
+        if (saasTierPolicy) {
+            amount = this.saasApplyTierPolicy(saasTierPolicy, amount);
+        }
+        Object.assign(formatted, { usage: { tokens: amount } });
+        (0, civ_rpc_1.assignMeta)(formatted, { usage: { tokens: amount } });
+        return amount;
+    }
+    async *scrapMany(urls, options, crawlerOpts) {
+        const iterators = urls.map((url) => this.cachedScrap(url, options, crawlerOpts));
+        const results = iterators.map((_x) => undefined);
+        let nextDeferred = (0, defer_1.Defer)();
+        let concluded = false;
+        const handler = async (it, idx) => {
+            try {
+                for await (const x of it) {
+                    results[idx] = x;
+                    if (x) {
+                        nextDeferred.resolve();
+                        nextDeferred = (0, defer_1.Defer)();
+                    }
+                }
+            }
+            catch (err) {
+                this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: (0, lang_1.marshalErrorLike)(err) });
+            }
+        };
+        Promise.allSettled(iterators.map((it, idx) => handler(it, idx))).finally(() => {
+            concluded = true;
+            nextDeferred.resolve();
+        });
+        yield results;
+        try {
+            while (!concluded) {
+                await nextDeferred.promise;
+                yield results;
+            }
+            yield results;
+        }
+        finally {
+            for (const x of iterators) {
+                x.return();
+            }
+        }
+    }
+    async configure(opts) {
+        this.threadLocal.set('withGeneratedAlt', opts.withGeneratedAlt);
+        this.threadLocal.set('withLinksSummary', opts.withLinksSummary);
+        this.threadLocal.set('withImagesSummary', opts.withImagesSummary);
+        this.threadLocal.set('keepImgDataUrl', opts.keepImgDataUrl);
+        this.threadLocal.set('cacheTolerance', opts.cacheTolerance);
+        this.threadLocal.set('withIframe', opts.withIframe);
+        this.threadLocal.set('withShadowDom', opts.withShadowDom);
+        this.threadLocal.set('userAgent', opts.userAgent);
+        if (opts.timeout) {
+            this.threadLocal.set('timeout', opts.timeout * 1000);
+        }
+        this.threadLocal.set('retainImages', opts.retainImages);
+        this.threadLocal.set('noGfm', opts.noGfm);
+        this.threadLocal.set('DNT', Boolean(opts.doNotTrack));
+        if (opts.markdown) {
+            this.threadLocal.set('turndownOpts', opts.markdown);
+        }
+        const crawlOpts = {
+            proxyUrl: opts.proxyUrl,
+            cookies: opts.setCookies,
+            favorScreenshot: ['screenshot', 'pageshot'].some((x) => opts.respondWith.includes(x)),
+            removeSelector: opts.removeSelector,
+            targetSelector: opts.targetSelector,
+            waitForSelector: opts.waitForSelector,
+            overrideUserAgent: opts.userAgent,
+            timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,
+            withIframe: opts.withIframe,
+            withShadowDom: opts.withShadowDom,
+            locale: opts.locale,
+            referer: opts.referer,
+            viewport: opts.viewport,
+            engine: opts.engine,
+            allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy,
+            proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false,
+            private: Boolean(opts.doNotTrack),
+        };
+        if (crawlOpts.targetSelector?.length) {
+            if (typeof crawlOpts.targetSelector === 'string') {
+                crawlOpts.targetSelector = [crawlOpts.targetSelector];
+            }
+            for (const s of crawlOpts.targetSelector) {
+                for (const e of s.split(',').map((x) => x.trim())) {
+                    if (e.startsWith('*') || e.startsWith(':') || e.includes('*:')) {
+                        throw new civ_rpc_1.ParamValidationError({
+                            message: `Unacceptable selector: '${e}'. We cannot accept match-all selector for performance reasons. Sorry.`,
+                            path: 'targetSelector'
+                        });
+                    }
+                }
+            }
+        }
+        if (opts._hintIps?.length) {
+            const hints = await this.geoIpService.lookupCities(opts._hintIps);
+            const board = {};
+            for (const x of hints) {
+                if (x.country?.code) {
+                    board[x.country.code] = (board[x.country.code] || 0) + 1;
+                }
+            }
+            const hintCountry = lodash_1.default.maxBy(Array.from(Object.entries(board)), 1)?.[0];
+            crawlOpts.countryHint = hintCountry?.toLowerCase();
+        }
+        if (opts.locale) {
+            crawlOpts.extraHeaders ??= {};
+            crawlOpts.extraHeaders['Accept-Language'] = opts.locale;
+        }
+        if (opts.respondWith.includes(crawler_options_1.CONTENT_FORMAT.VLM)) {
+            crawlOpts.favorScreenshot = true;
+        }
+        if (opts.injectFrameScript?.length) {
+            crawlOpts.injectFrameScripts = (await Promise.all(opts.injectFrameScript.map((x) => {
+                if (URL.canParse(x)) {
+                    return fetch(x).then((r) => r.text());
+                }
+                return x;
+            }))).filter(Boolean);
+        }
+        if (opts.injectPageScript?.length) {
+            crawlOpts.injectPageScripts = (await Promise.all(opts.injectPageScript.map((x) => {
+                if (URL.canParse(x)) {
+                    return fetch(x).then((r) => r.text());
+                }
+                return x;
+            }))).filter(Boolean);
+        }
+        return crawlOpts;
+    }
+    async formatSnapshot(crawlerOptions, snapshot, nominalUrl, urlValidMs, scrappingOptions) {
+        const presumedURL = crawlerOptions.base === 'final' ? new URL(snapshot.href) : nominalUrl;
+        const respondWith = crawlerOptions.respondWith;
+        if (respondWith === crawler_options_1.CONTENT_FORMAT.READER_LM || respondWith === crawler_options_1.CONTENT_FORMAT.VLM) {
+            const output = {
+                title: snapshot.title,
+                content: snapshot.parsed?.textContent,
+                url: presumedURL?.href || snapshot.href,
+            };
+            Object.defineProperty(output, 'textRepresentation', {
+                value: snapshot.parsed?.textContent,
+                enumerable: false,
+            });
+            return output;
+        }
+        return this.formatSnapshotWithPDFSideLoad(respondWith, snapshot, presumedURL, urlValidMs, scrappingOptions);
+    }
+    async formatSnapshotWithPDFSideLoad(mode, snapshot, nominalUrl, urlValidMs, scrappingOptions) {
+        const snapshotCopy = lodash_1.default.cloneDeep(snapshot);
+        if (snapshotCopy.pdfs?.length) {
+            const pdfUrl = snapshotCopy.pdfs[0];
+            if (pdfUrl.startsWith('http')) {
+                const sideLoaded = scrappingOptions?.sideLoad?.impersonate[pdfUrl];
+                if (sideLoaded?.status === 200 && sideLoaded.body) {
+                    snapshotCopy.pdfs[0] = (0, url_1.pathToFileURL)(await sideLoaded?.body.filePath).href;
+                    return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs);
+                }
+                const r = await this.curlControl.sideLoad(new URL(pdfUrl), scrappingOptions).catch((err) => {
+                    if (err instanceof errors_1.ServiceBadAttemptError) {
+                        return Promise.reject(new civ_rpc_1.AssertionFailureError(`Failed to load PDF(${pdfUrl}): ${err.message}`));
+                    }
+                    return Promise.reject(err);
+                });
+                if (r.status !== 200) {
+                    throw new civ_rpc_1.AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded status ${r.status}`);
+                }
+                if (!r.contentType.includes('application/pdf')) {
+                    throw new civ_rpc_1.AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded with wrong content type ${r.contentType}`);
+                }
+                if (!r.file) {
+                    throw new civ_rpc_1.AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server did not return a body`);
+                }
+                snapshotCopy.pdfs[0] = (0, url_1.pathToFileURL)(await r.file.filePath).href;
+            }
+        }
+        return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs);
+    }
+    async getFinalSnapshot(url, opts, crawlerOptions) {
+        const it = this.cachedScrap(url, opts, crawlerOptions);
+        let lastSnapshot;
+        let lastError;
+        try {
+            for await (const x of it) {
+                lastSnapshot = x;
+            }
+        }
+        catch (err) {
+            lastError = err;
+        }
+        if (!lastSnapshot && lastError) {
+            throw lastError;
+        }
+        if (!lastSnapshot) {
+            throw new civ_rpc_1.AssertionFailureError(`No content available`);
+        }
+        return lastSnapshot;
+    }
+    async simpleCrawl(mode, url, opts) {
+        const it = this.iterSnapshots(url, { ...opts, minIntervalMs: 500 });
+        let lastSnapshot;
+        let goodEnough = false;
+        try {
+            for await (const x of it) {
+                lastSnapshot = x;
+                if (goodEnough) {
+                    break;
+                }
+                if (lastSnapshot?.parsed?.content) {
+                    // After it's good enough, wait for next snapshot;
+                    goodEnough = true;
+                }
+            }
+        }
+        catch (err) {
+            if (lastSnapshot) {
+                return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs);
+            }
+            throw err;
+        }
+        if (!lastSnapshot) {
+            throw new civ_rpc_1.AssertionFailureError(`No content available`);
+        }
+        return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs);
+    }
+    getDomainProfileUrlDigest(url) {
+        const pathname = url.pathname;
+        const pathVec = pathname.split('/');
+        const parentPath = pathVec.slice(0, -1).join('/');
+        const finalPath = parentPath || pathname;
+        const key = url.origin.toLocaleLowerCase() + finalPath;
+        return {
+            digest: snapshot_formatter_1.md5Hasher.hash(key),
+            path: finalPath,
+        };
+    }
+    async sideLoadWithAllocatedProxy(url, opts) {
+        if (opts?.allocProxy === 'none') {
+            return this.curlControl.sideLoad(url, opts);
+        }
+        let proxy;
+        if (opts) {
+            let it = this.proxyIterMap.get(opts);
+            if (!it) {
+                it = this.proxyProvider.iterAlloc(this.figureOutBestProxyCountry(opts));
+                this.proxyIterMap.set(opts, it);
+            }
+            proxy = (await it.next()).value;
+        }
+        proxy ??= await this.proxyProvider.alloc(this.figureOutBestProxyCountry(opts));
+        this.logger.debug(`Proxy allocated`, { proxy: proxy.href });
+        const r = await this.curlControl.sideLoad(url, {
+            ...opts,
+            proxyUrl: proxy.href,
+        });
+        if (opts && opts.allocProxy) {
+            opts.proxyUrl ??= proxy.href;
+        }
+        return { ...r, proxy };
+    }
+    figureOutBestProxyCountry(opts) {
+        if (!opts) {
+            return 'auto';
+        }
+        let draft;
+        if (opts.allocProxy) {
+            if (this.proxyProvider.supports(opts.allocProxy)) {
+                draft = opts.allocProxy;
+            }
+            else if (opts.allocProxy === 'none') {
+                return 'none';
+            }
+        }
+        if (opts.countryHint) {
+            if (this.proxyProvider.supports(opts.countryHint)) {
+                draft ??= opts.countryHint;
+            }
+        }
+        draft ??= opts.allocProxy || 'auto';
+        return draft;
+    }
+    knownUrlThatSideLoadingWouldCrashTheBrowser(url) {
+        if (url.hostname === 'chromewebstore.google.com') {
+            return true;
+        }
+        return false;
+    }
+    async saasAssertTierPolicy(opts, auth) {
+        let chargeScalar = 1;
+        let minimalCharge = 0;
+        if (opts.withGeneratedAlt) {
+            await auth.assertTier(0, 'Alt text generation');
+            minimalCharge = 765;
+        }
+        if (opts.injectPageScript || opts.injectFrameScript) {
+            await auth.assertTier(0, 'Script injection');
+            minimalCharge = 4_000;
+        }
+        if (opts.withIframe) {
+            await auth.assertTier(0, 'Iframe');
+        }
+        if (opts.engine === crawler_options_1.ENGINE_TYPE.CF_BROWSER_RENDERING) {
+            await auth.assertTier(0, 'Cloudflare browser rendering');
+            minimalCharge = 4_000;
+        }
+        if (opts.respondWith.includes('lm') || opts.engine?.includes('lm')) {
+            await auth.assertTier(0, 'Language model');
+            minimalCharge = 4_000;
+            chargeScalar = 3;
+        }
+        if (opts.proxy && opts.proxy !== 'none') {
+            await auth.assertTier(['auto', 'any'].includes(opts.proxy) ? 0 : 2, 'Proxy allocation');
+            chargeScalar = 5;
+        }
+        return {
+            budget: opts.tokenBudget || 0,
+            chargeScalar,
+            minimalCharge,
+        };
+    }
+    saasApplyTierPolicy(policy, chargeAmount) {
+        const effectiveChargeAmount = policy.chargeScalar * Math.max(chargeAmount, policy.minimalCharge);
+        if (policy.budget && policy.budget < effectiveChargeAmount) {
+            throw new errors_1.BudgetExceededError(`Token budget (${policy.budget}) exceeded, intended charge amount ${effectiveChargeAmount}`);
+        }
+        return effectiveChargeAmount;
+    }
+};
+exports.CrawlerHost = CrawlerHost;
+__decorate([
+    (0, registry_1.Method)({
+        name: 'getIndex',
+        description: 'Index of the service',
+        proto: {
+            http: {
+                action: 'get',
+                path: '/',
+            }
+        },
+        tags: ['misc', 'crawl'],
+        returnType: [String, Object],
+    }),
+    __param(0, (0, registry_1.Ctx)()),
+    __param(1, (0, registry_1.Param)({ required: false })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_c = typeof registry_1.Context !== "undefined" && registry_1.Context) === "function" ? _c : Object, jina_embeddings_auth_1.JinaEmbeddingsAuthDTO]),
+    __metadata("design:returntype", Promise)
+], CrawlerHost.prototype, "getIndexCtrl", null);
+__decorate([
+    (0, registry_1.Method)({
+        name: 'crawlByPostingToIndex',
+        description: 'Crawl any url into markdown',
+        proto: {
+            http: {
+                action: 'POST',
+                path: '/',
+            }
+        },
+        tags: ['crawl'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream],
+    }),
+    (0, registry_1.Method)({
+        description: 'Crawl any url into markdown',
+        proto: {
+            http: {
+                action: ['GET', 'POST'],
+                path: '::url',
+            }
+        },
+        tags: ['crawl'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream, civ_rpc_1.RawString],
+    }),
+    __param(0, (0, registry_1.RPCReflect)()),
+    __param(1, (0, registry_1.Ctx)()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_d = typeof civ_rpc_1.RPCReflection !== "undefined" && civ_rpc_1.RPCReflection) === "function" ? _d : Object, typeof (_e = typeof registry_1.Context !== "undefined" && registry_1.Context) === "function" ? _e : Object, jina_embeddings_auth_1.JinaEmbeddingsAuthDTO,
+        crawler_options_1.CrawlerOptionsHeaderOnly,
+        crawler_options_1.CrawlerOptions]),
+    __metadata("design:returntype", Promise)
+], CrawlerHost.prototype, "crawl", null);
+__decorate([
+    (0, decorators_1.retryWith)((err) => {
+        if (err instanceof errors_1.ServiceBadApproachError) {
+            return false;
+        }
+        if (err instanceof errors_1.ServiceBadAttemptError) {
+            // Keep trying
+            return true;
+        }
+        if (err instanceof civ_rpc_1.ApplicationError) {
+            // Quit with this error
+            return false;
+        }
+        return undefined;
+    }, 3),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_f = typeof URL !== "undefined" && URL) === "function" ? _f : Object, Object]),
+    __metadata("design:returntype", Promise)
+], CrawlerHost.prototype, "sideLoadWithAllocatedProxy", null);
+exports.CrawlerHost = CrawlerHost = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger,
+        puppeteer_1.PuppeteerControl,
+        curl_1.CurlControl,
+        cf_browser_rendering_1.CFBrowserRendering, typeof (_a = typeof proxy_provider_1.ProxyProviderService !== "undefined" && proxy_provider_1.ProxyProviderService) === "function" ? _a : Object, lm_1.LmControl,
+        jsdom_1.JSDomControl,
+        snapshot_formatter_1.SnapshotFormatter,
+        firebase_storage_bucket_1.FirebaseStorageBucketControl, typeof (_b = typeof rate_limit_1.RateLimitControl !== "undefined" && rate_limit_1.RateLimitControl) === "function" ? _b : Object, async_context_1.AsyncLocalContext,
+        robots_text_1.RobotsTxtService,
+        temp_file_1.TempFileManager,
+        geoip_1.GeoIPService,
+        misc_2.MiscService])
+], CrawlerHost);
+//# sourceMappingURL=crawler.js.map

build/api/crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/api/crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,uCAAqC;AACrC,6BAAoC;AACpC,mCAAoC;AACpC,oDAAuB;AAEvB,4CAOwB;AACxB,sCAA+C;AAC/C,wCAAqC;AACrC,kDAA8C;AAC9C,kDAA8C;AAE9C,4DAA+H;AAE/H,2CAAwC;AACxC,2DAAuD;AACvD,wFAA+E;AAE/E,qDAAyF;AACzF,6CAAiD;AACjD,uEAA6F;AAC7F,2CAA+C;AAC/C,uCAA2C;AAC3C,wCAAsD;AACtD,2EAAsE;AAEtE,+CAAkD;AAClD,8DAAgF;AAChF,6DAA8D;AAC9D,mDAA+E;AAC/E,+CAI4B;AAE5B,mDAAwE;AACxE,sEAAyE;AACzE,wFAA0F;AAC1F,sEAAoE;AACpE,yDAA2D;AAC3D,qDAAwD;AACxD,2CAA+C;AAC/C,sCAA+C;AAC/C,6CAAiD;AAcjD,MAAM,UAAU,GAAG;IACf,QAAQ,EAAE;QACN,OAAO,IAAA,gBAAC,EAAC,IAAI,CAAC;aACT,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,gBAAC,CAAC,UAAU,CAAC,gBAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAClE,KAAK,EAAE;aACP,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3B,CAAC;CACJ,CAAC;AAGK,IAAM,WAAW,GAAjB,MAAM,WAAY,SAAQ,iBAAO;IAWpC,YACc,YAA0B,EAC1B,gBAAkC,EAClC,WAAwB,EACxB,kBAAsC,EACtC,aAAmC,EACnC,SAAoB,EACpB,YAA0B,EAC1B,iBAAoC,EACpC,qBAAmD,EACnD,gBAAkC,EAClC,WAA8B,EAC9B,gBAAkC,EAClC,eAAgC,EAChC,YAA0B,EAC1B,WAAwB;QAElC,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QAhBV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,gBAAW,GAAX,WAAW,CAAa;QACxB,uBAAkB,GAAlB,kBAAkB,CAAoB;QACtC,kBAAa,GAAb,aAAa,CAAsB;QACnC,cAAS,GAAT,SAAS,CAAW;QACpB,iBAAY,GAAZ,YAAY,CAAc;QAC1B,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,0BAAqB,GAArB,qBAAqB,CAA8B;QACnD,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,gBAAW,GAAX,WAAW,CAAmB;QAC9B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,oBAAe,GAAf,eAAe,CAAiB;QAChC,iBAAY,GAAZ,YAAY,CAAc;QAC1B,gBAAW,GAAX,WAAW,CAAa;QAzBtC,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,qBAAgB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QACxC,iBAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QAC3B,eAAU,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC;QAC7B,iBAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QAC3B,6BAAwB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC;QAEjD,kBAAa,GAAc,EAAE,CAAC;QA2oC9B,iBAAY,GAAG,IAAI,OAAO,EAAwE,CAAC;QAtnC/F,gBAAgB,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,QAAsB,EAAE,OAA8C,EAAE,EAAE;YAC5G,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBACpD,OAAO;YACX,CAAC;YACD,IAAI,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBAC7C,0DAA0D;gBAC1D,OAAO;YACX,CAAC;YACD,IAAI,OAAO,CAAC,kBAAkB,EAAE,MAAM,IAAI,OAAO,CAAC,iBAAiB,EAAE,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC9F,mEAAmE;gBACnE,OAAO;YACX,CAAC;YACD,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;gBAC1B,OAAO;YACX,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;gBAC7B,2DAA2D;gBAC3D,OAAO;YACX,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACpD,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5E,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACxB,kCAAkC;gBAClC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC1B,OAAO;gBACX,CAAC;gBACD,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC9E,OAAO;gBACX,CAAC;YACL,CAAC;YAED,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,UAAqD,EAAE,EAAE;YACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,GAAG,cAAc,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;YAE/I,MAAM,gCAAc,CAAC,IAAI,CAAC,gCAAc,CAAC,IAAI,CAAC;gBAC1C,MAAM,EAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE;gBAC7C,aAAa,EAAE,GAAG,UAAU,CAAC,MAAM,EAAE;gBACrC,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACrC,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,QAAQ,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC;aACrD,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtH,CAAC,CAAC,CAAC;QAEP,CAAC,CAAC,CAAC;QAEH,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YACrC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,iBAAO,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAEjC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,KAAK,CAAC,GAAG,CAAC,iBAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,mBAAmB,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACvF,CAAC;YAED,KAAK,CAAC,MAAM,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,MAAM,kBAAkB,CAAC,CAAC;YACnE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAC7D,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACpC,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC1E,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAA4B;QACvC,MAAM,WAAW,GAAgD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE;YACvB,MAAM,EAAE,4BAA4B;YACpC,MAAM,EAAE,qCAAqC;YAC7C,QAAQ,EAAE,wBAAwB;SACrC,CAAC,CAAC;QAEH,MAAM,IAAI,EAAE,QAAQ,EAAE,CAAC;QACvB,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACpB,WAAW,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC;YAC5B,WAAW,CAAC,eAAe,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC;YAC9E,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC7D,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAcK,AAAN,KAAK,CAAC,YAAY,CAAQ,GAAY,EAA8B,IAA4B;QAC5F,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9F,OAAO,WAAW,CAAC;QACvB,CAAC;QAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,WAAW,EAAE,EAC9C,EAAE,WAAW,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE,CAC/D,CAAC;IACN,CAAC;IA0BK,AAAN,KAAK,CAAC,KAAK,CACO,UAAyB,EAChC,GAAY,EACnB,IAA2B,EAC3B,wBAAkD,EAClD,2BAA2C;QAE3C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,cAAc,GAAG,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,2BAA2B,CAAC;QACrG,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QAEzE,0FAA0F;QAC1F,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,IAAA,4BAAqB,EAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC;QACzH,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,GAAG,CACzC,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAC7B,CAAC;QAEF,IAAI,GAAG,EAAE,CAAC;YACN,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;YACrC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,iCAAwB,CAAC,gEAAgE,CAAC,CAAC;YACzG,CAAC;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI;gBACnD,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;oBACvC,0BAAa,CAAC,IAAI,CAAC;wBACf,UAAU,EAAE,IAAI;wBAChB,aAAa,EAAE,EAAE;qBACpB,CAAC,CAAC,CAAC;oBACJ,0BAAa,CAAC,IAAI,CAAC;wBACf,UAAU,EAAE,GAAG;wBACf,aAAa,EAAE,EAAE;qBACpB,CAAC;aACT,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,CAC9D,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,EAC1B,GAAG,eAAe,CACrB,CAAC;YAEF,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE;gBACpB,IAAI,cAAc,CAAC,WAAW,IAAI,YAAY,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC;oBAC1E,OAAO;gBACX,CAAC;gBACD,IAAI,YAAY,EAAE,CAAC;oBACf,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACzD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC1F,CAAC,CAAC,CAAC;oBACH,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;gBACxC,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;aAAM,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAC3F;gBACI,yBAAyB;gBACzB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE;aACvC,CACJ,CAAC;YAEF,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE;gBACpB,IAAI,cAAc,CAAC,WAAW,IAAI,YAAY,GAAG,cAAc,CAAC,WAAW,EAAE,CAAC;oBAC1E,OAAO;gBACX,CAAC;gBACD,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;YACxC,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,kEAAkE;YAClE,cAAc,CAAC,KAAK,GAAG,MAAM,CAAC;YAC9B,MAAM,QAAQ,GAAG,CAAC,MAAM,gCAAc,CAAC,kBAAkB,CACrD,gCAAc,CAAC,UAAU;iBACpB,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;iBACvD,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;iBACnC,KAAK,CAAC,CAAC,CAAC,CAChB,CAAC,CAAC,CAAC,CAAC,CAAC;YACN,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,IAAI,gCAAuB,CAAC,UAAU,SAAS,CAAC,QAAQ,kBAAkB,QAAQ,CAAC,QAAQ,IAAI,WAAW,mCAAmC,QAAQ,CAAC,UAAU,IAAI,MAAM,KAAK,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;YACnN,CAAC;QACL,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,GAAG,IAAI,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QACtF,IAAI,cAAc,CAAC,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,SAAS,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC5B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACjE,MAAM,SAAS,GAAG,IAAI,uDAAuB,EAAE,CAAC;YAChD,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAE7B,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;oBACpF,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACZ,SAAS;oBACb,CAAC;oBACD,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC5B,MAAM;oBACV,CAAC;oBAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC7G,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;oBAC9D,SAAS,CAAC,KAAK,CAAC;wBACZ,KAAK,EAAE,MAAM;wBACb,IAAI,EAAE,SAAS;qBAClB,CAAC,CAAC;oBACH,IAAI,YAAY,IAAI,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;wBACxC,MAAM;oBACV,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClF,SAAS,CAAC,KAAK,CAAC;oBACZ,KAAK,EAAE,OAAO;oBACd,IAAI,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC;iBAC9B,CAAC,CAAC;YACP,CAAC;YAED,SAAS,CAAC,GAAG,EAAE,CAAC;YAEhB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,YAAY,CAAC;QACjB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9F,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;oBACpF,YAAY,GAAG,QAAQ,CAAC;oBACxB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC5B,MAAM;oBACV,CAAC;oBACD,IAAI,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,oCAAoC,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC9E,SAAS;oBACb,CAAC;oBACD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;wBAClB,SAAS;oBACb,CAAC;oBAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC7G,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;oBAE9D,IAAI,QAAQ,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;wBAC1C,SAAS;oBACb,CAAC;oBAED,OAAO,SAAS,CAAC;gBACrB,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC;YAED,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;oBAC3B,MAAM,IAAI,+BAAqB,CAAC,gCAAgC,SAAS,yBAAyB,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,CAAC;gBAClN,CAAC;gBACD,MAAM,IAAI,+BAAqB,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YACjH,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YAE9D,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,cAAc,CAAC,iCAAiC,EAAE,EAAE,CAAC;YACrD,MAAM,IAAI,8BAAoB,CAAC;gBAC3B,IAAI,EAAE,aAAa;gBACnB,OAAO,EAAE,2HAA2H;aACvI,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;gBACpF,YAAY,GAAG,QAAQ,CAAC;gBACxB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC5B,MAAM;gBACV,CAAC;gBACD,IAAI,CAAC,QAAQ,IAAI,CAAC,cAAc,CAAC,oCAAoC,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC9E,SAAS;gBACb,CAAC;gBACD,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;oBAClB,SAAS;gBACb,CAAC;gBAED,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;gBAC7G,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;gBAE9D,IAAI,cAAc,CAAC,WAAW,KAAK,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,CAAC;oBACzF,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAC/D,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE,CAChG,CAAC;gBACN,CAAC;gBACD,IAAI,cAAc,CAAC,WAAW,KAAK,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC;oBACrF,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAC/D,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,CAC9F,CAAC;gBACN,CAAC;gBAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YACvI,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,MAAM,GAAG,CAAC;YACd,CAAC;QACL,CAAC;QAED,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC3B,MAAM,IAAI,+BAAqB,CAAC,gCAAgC,SAAS,yBAAyB,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC,CAAC;YAClN,CAAC;YACD,MAAM,IAAI,+BAAqB,CAAC,gCAAgC,SAAS,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjH,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;QAE9D,IAAI,cAAc,CAAC,WAAW,KAAK,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,CAAC;YAEzF,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAC/D,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,EAAE,CAChG,CAAC;QACN,CAAC;QACD,IAAI,cAAc,CAAC,WAAW,KAAK,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC;YAErF,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAC/D,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,CAC9F,CAAC;QACN,CAAC;QAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,SAAS,CAAC,kBAAkB,EAAE,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvI,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,UAAkB,EAAE,cAA8B;QACjE,IAAI,GAAG,GAAW,EAAE,CAAC;QAErB,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7C,IAAI,cAAc,CAAC,GAAG,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC;YACnC,MAAM,UAAU,GAAG,OAAO,YAAY,sBAAS,CAAC,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAU,GAAE,CAAC;YAC3F,GAAG,GAAG,cAAc,UAAU,EAAE,CAAC;YACjC,cAAc,CAAC,GAAG,KAAK,GAAG,CAAC;QAC/B,CAAC;aAAM,IAAI,gBAAgB,EAAE,CAAC;YAC1B,GAAG,GAAG,gBAAgB,CAAC,IAAI,EAAE,CAAC;QAClC,CAAC;aAAM,IAAI,cAAc,CAAC,GAAG,EAAE,CAAC;YAC5B,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACpC,CAAC;QAED,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,IAAI,8BAAoB,CAAC;gBAC3B,OAAO,EAAE,iBAAiB;gBAC1B,IAAI,EAAE,KAAK;aACd,CAAC,CAAC;QACP,CAAC;QAED,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAC9E,IAAI,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,gCAAuB,CAAC;gBAC9B,OAAO,EAAE,sBAAsB,OAAO,CAAC,QAAQ,EAAE;gBACjD,IAAI,EAAE,KAAK;aACd,CAAC,CAAC;QACP,CAAC;QACD,cAAc,CAAC,QAAQ,GAAG,GAAG,CAAC;QAE9B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,YAAY,CAAC,UAAe;QACxB,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,aAAa,CAAC,IAAI,GAAG,EAAE,CAAC;QAC5B,CAAC;QACD,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,8BAAS,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExD,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,CAAC,UAAU,CAAC,UAAe,EAAE,cAAsB;QACrD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAE7C,MAAM,KAAK,GAAG,CACV,MACI,CAAC,iBAAO,CAAC,kBAAkB,CACvB,iBAAO,CAAC,UAAU,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAChG,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACZ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAClE,gDAAgD;YAChD,6DAA6D;YAC7D,6DAA6D;YAE7D,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC,CAAC,CACV,EAAE,CAAC,CAAC,CAAC,CAAC;QAEP,MAAM,KAAK,CAAC;QAEZ,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO;QACX,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,QAAQ,UAAU,wBAAwB,MAAM,KAAK,GAAG,qBAAqB,cAAc,IAAI,EAAE;YAC3J,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,cAAc;SACtD,CAAC,CAAC;QAEH,IAAI,QAAkC,CAAC;QACvC,IAAI,aAAiC,CAAC;QACtC,IAAI,WAA+B,CAAC;QACpC,MAAM,YAAY,GAAG;YACjB,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,aAAa,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;gBACzE,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC/C,CAAC,CAAC;YACF,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBACvB,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,eAAe,KAAK,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC5G,aAAa,GAAG,CAAC,CAAC;gBACtB,CAAC,CAAC,CAAC,CAAC;gBACJ,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;YAC9B,KAAK,CAAC,iBAAiB,CAAC,CAAC;gBACrB,IAAI,CAAC,qBAAqB,CAAC,eAAe,CAAC,aAAa,KAAK,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC1G,WAAW,GAAG,CAAC,CAAC;gBACpB,CAAC,CAAC,CAAC,CAAC;gBACJ,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;SACjC,CAAC;QACF,IAAI,CAAC;YACD,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,IAAI,EAAE,CAAC;YACZ,wBAAwB;YACxB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM;YACF,OAAO,EAAE,CAAC,KAAK;YACf,GAAG,KAAK;YACR,QAAQ,EAAE;gBACN,GAAG,QAAQ;gBACX,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,SAAS;gBACnB,aAAa;gBACb,WAAW;aACsD;SACxE,CAAC;IACN,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,UAAe,EAAE,QAAsB;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAE7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,UAAU,KAAK,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACpI,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAE3B,MAAM,KAAK,GAAG,iBAAO,CAAC,IAAI,CAAC;YACvB,GAAG,EAAE,IAAA,mBAAU,GAAE;YACjB,GAAG,EAAE,UAAU,CAAC,QAAQ,EAAE;YAC1B,SAAS,EAAE,OAAO;YAClB,QAAQ,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAC7D,6BAA6B,EAAE,QAAQ,CAAC,6BAA6B;YACrE,aAAa,EAAE,MAAM;SACxB,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,aAAa,KAAK,CAAC,GAAG,EAAE,EACjF,MAAM,CAAC,IAAI,CACP,IAAI,CAAC,SAAS,CAAC;YACX,GAAG,QAAQ;YACX,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,SAAS;SACtB,CAAC,EACF,OAAO,CACV,EACD;YACI,QAAQ,EAAE;gBACN,WAAW,EAAE,kBAAkB;aAClC;SACJ,CACJ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC/B,OAAO,CAAC,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,eAAe,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,UAAU,EAAE;gBACvF,QAAQ,EAAE;oBACN,WAAW,EAAE,WAAW;iBAC3B;aACJ,CAAC,CAAC;YACH,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC;QACrC,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,aAAa,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE;gBACnF,QAAQ,EAAE;oBACN,WAAW,EAAE,WAAW;iBAC3B;aACJ,CAAC,CAAC;YACH,KAAK,CAAC,iBAAiB,GAAG,IAAI,CAAC;QACnC,CAAC;QACD,MAAM,gBAAgB,CAAC;QACvB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,6EAA6E;QAC7E,mGAAmG;QAEnG,wBAAwB;QACxB,MAAM;QAEN,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,CAAC,aAAa,CAAC,UAAe,EAAE,SAAiC,EAAE,WAA4B;QACjG,+DAA+D;QAC/D,6EAA6E;QAC7E,oDAAoD;QACpD,uBAAuB;QAEvB,6EAA6E;QAE7E,cAAc;QACd,IAAI;QAEJ,IAAI,WAAW,EAAE,WAAW,CAAC,QAAQ,CAAC,gCAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE;gBAC9D,GAAG,SAAS;gBACZ,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,6BAAW,CAAC,IAAI;aAChD,EAAE,gCAAc,CAAC,IAAI,CAAC;gBACnB,GAAG,WAAW;gBACd,WAAW,EAAE,MAAM;aACtB,CAAC,CAAC,CAAC;YAEJ,IAAI,CAAC,iBAAiB,EAAE,IAAI,EAAE,CAAC;gBAC3B,MAAM,IAAI,+BAAqB,CAAC,6CAA6C,UAAU,EAAE,CAAC,CAAC;YAC/F,CAAC;YAED,IAAI,WAAW,EAAE,WAAW,IAAI,WAAW,EAAE,UAAU,EAAE,CAAC;gBACtD,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,UAAU,EAAE,iBAAiB,CAAC,CAAC;gBAEnG,OAAO;YACX,CAAC;YAED,IAAI,CAAC;gBACD,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,4BAA4B,CAAC,iBAAiB,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,GAAG,YAAY,uBAAgB,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACxD,MAAM,IAAI,sCAA6B,CAAC,mDAAmD,CAAC,CAAC;gBACjG,CAAC;gBACD,MAAM,GAAG,CAAC;YACd,CAAC;YAED,OAAO;QACX,CAAC;QAED,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,CAAC,WAAW,CAAC,UAAe,EAAE,SAAiC,EAAE,WAA4B;QAC/F,IAAI,WAAW,EAAE,IAAI,EAAE,CAAC;YACpB,MAAM,QAAQ,GAAG;gBACb,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE;gBAC3B,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,EAAE;aACK,CAAC;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE5D,OAAO;QACX,CAAC;QAED,IAAI,WAAW,EAAE,GAAG,EAAE,CAAC;YACnB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,YAAY,sBAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClJ,MAAM,YAAY,GAAG,IAAA,mBAAa,EAAC,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC7D,MAAM,QAAQ,GAAG;gBACb,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE;gBAC3B,IAAI,EAAE,kOAAkO,WAAW,CAAC,GAAG,kBAAkB;gBACzQ,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC;aACZ,CAAC;YAElB,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAE5D,OAAO;QACX,CAAC;QAED,IACI,SAAS,EAAE,MAAM,KAAK,6BAAW,CAAC,IAAI;YACtC,kBAAkB;YAClB,SAAS,EAAE,MAAM,KAAK,QAAQ,EAChC,CAAC;YACC,IAAI,UAAU,CAAC;YACf,IAAI,CAAC;gBACD,UAAU,GAAG,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC1D,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;oBAC9D,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAE/D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,IAAI,GAAG,YAAY,+BAAsB,EAAE,CAAC;oBACxC,MAAM,IAAI,+BAAqB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACjD,CAAC;gBACD,MAAM,GAAG,CAAC;YACd,CAAC;YACD,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC;gBACpB,MAAM,IAAI,+BAAqB,CAAC,wCAAwC,UAAU,EAAE,CAAC,CAAC;YAC1F,CAAC;YACD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,WAAW,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpJ,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;YACzC,aAAa,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC;YACjD,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;YACjE,OAAO;QACX,CAAC;QACD,IAAI,SAAS,EAAE,MAAM,KAAK,6BAAW,CAAC,oBAAoB,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACzE,MAAM,QAAQ,GAAG;gBACb,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE;gBAC3B,IAAI;gBACJ,KAAK,EAAE,EAAE;gBACT,IAAI,EAAE,EAAE;aACK,CAAC;YAClB,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAC5D,OAAO;QACX,CAAC;QAED,MAAM,cAAc,GAAG,WAAW,EAAE,cAAc,IAAI,IAAI,CAAC,YAAY,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;QAE5D,IAAI,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;QACzC,IAAI,KAAK,EAAE,6BAA6B,KAAK,KAAK,EAAE,CAAC;YACjD,IAAI,WAAW,IAAI,WAAW,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACnD,WAAW,CAAC,aAAa,KAAK,gCAAc,CAAC,IAAI,CAAC;YACtD,CAAC;QACL,CAAC;QAED,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,sBAAsB,EAAE,EAAE,CAAC;YACvD,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;QACzC,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAE1B,IAAI,KAAK,EAAE,OAAO;YACd,CAAC,CAAC,SAAS,EAAE,eAAe,IAAI,CAAC,SAAS,EAAE,eAAe,IAAI,CAAC,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACvH,CAAC,gBAAC,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,SAAS,EAAE,MAAM,CAAC,EACzD,CAAC;YACC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACjB,KAAK,CAAC,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC;YACtC,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;YAElE,OAAO;QACX,CAAC;QAED,IAAI,SAAS,EAAE,MAAM,KAAK,6BAAW,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,2CAA2C,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7G,MAAM,yBAAyB,GAAG,WAAW,EAAE,oBAAoB,EAAE;gBACjE,CAAC,gCAAc,CAAC,IAAI,EAAE,gCAAc,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,qBAAqB,CAAC,CAAC;YACtG,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;gBACjC,IAAI,UAAU,GAAG,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;oBAC9D,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;oBAC5D,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;wBAEpH,IAAI,GAAG,YAAY,0BAAgB,IAAI,CAAC,CAAC,GAAG,YAAY,+BAAsB,CAAC,EAAE,CAAC;4BAC9E,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC/B,CAAC;wBAED,OAAO,IAAI,CAAC,0BAA0B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBAChE,CAAC,CAAC,CAAC;gBACP,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,+BAAsB,CAAC,wCAAwC,UAAU,EAAE,CAAC,CAAC;gBAC3F,CAAC;gBACD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CACrE,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,WAAW,EAAE,UAAU,CAAC,QAAQ,CAC3E,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACZ,IAAI,GAAG,YAAY,0BAAgB,EAAE,CAAC;wBAClC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,+BAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;oBACnE,CAAC;oBACD,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC,CAAC,CAAC;gBACH,aAAa,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;gBACzC,aAAa,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC;gBACjD,IAAI,UAAU,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC9E,MAAM,aAAa,CAAC;oBACpB,OAAO;gBACX,CAAC;gBAED,IAAI,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC/E,aAAa,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,CAAC;gBACvC,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC;gBACpC,IAAI,yBAAyB,EAAE,CAAC;oBAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;gBACrE,CAAC;gBACD,IAAI,mBAAmB,GAAG,KAAK,CAAC;gBAChC,IACI,CAAC,CAAC,CAAC,SAAS,EAAE,UAAU,IAAI,SAAS,CAAC,UAAU,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC;oBACrF,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC,EACrD,CAAC;oBACC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBAC/E,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;wBACpB,MAAM,IAAI,+BAAsB,CAAC,wCAAwC,UAAU,EAAE,CAAC,CAAC;oBAC3F,CAAC;oBACD,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CACrE,UAAU,EAAE,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,QAAQ,CAC9E,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACZ,IAAI,GAAG,YAAY,0BAAgB,EAAE,CAAC;4BAClC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,+BAAsB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;wBACnE,CAAC;wBACD,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC/B,CAAC,CAAC,CAAC;oBACH,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;oBAC1C,aAAa,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;oBAClD,IAAI,WAAW,CAAC,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,oBAAoB,EAAE,EAAE,CAAC;oBACxE,CAAC;oBACD,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBAC3E,IAAI,WAAW,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;wBACvD,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC;wBACpC,IAAI,yBAAyB,EAAE,CAAC;4BAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;wBACrE,CAAC;wBACD,UAAU,GAAG,WAAW,CAAC;wBACzB,mBAAmB,GAAG,IAAI,CAAC;oBAC/B,CAAC;gBACL,CAAC;gBAED,IAAI,SAAS,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC7F,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+CAA+C,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC5F,SAAS,CAAC,QAAQ,KAAK,UAAU,CAAC,YAAY,CAAC;oBAC/C,IAAI,mBAAmB,EAAE,CAAC;wBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;oBAClF,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;gBACpH,IAAI,GAAG,YAAY,0BAAgB;oBAC/B,CAAC,CAAC,GAAG,YAAY,+BAAsB,CAAC;oBACxC,CAAC,CAAC,GAAG,YAAY,+BAAqB,CAAC,EACzC,CAAC;oBACC,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC;QACL,CAAC;aAAM,IAAI,SAAS,EAAE,UAAU,IAAI,SAAS,CAAC,UAAU,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACzF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC,CAAC;YAC3F,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC;QACvC,CAAC;QAED,IAAI,CAAC;YACD,IAAI,SAAS,EAAE,cAAc,IAAI,SAAS,EAAE,cAAc,IAAI,SAAS,EAAE,UAAU,IAAI,SAAS,EAAE,aAAa,EAAE,CAAC;gBAC9G,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBACzD,CAAC;gBAED,OAAO;YACX,CAAC;YAED,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,KAAK,IAAI,CAAC,CAAC,GAAG,YAAY,gCAAuB,CAAC,EAAE,CAAC;gBACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,UAAU,yDAAyD,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACzI,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;gBAClE,OAAO;YACX,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,kBAAkB,CAAC,SAAwB,EAAE,cAA+D;QACxG,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,OAAO,CAAC,CAAC;QACb,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,GAAG,IAAA,sBAAa,EAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,IAAA,sBAAa,EAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,IAAA,sBAAa,EAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,IAAI,IAAA,sBAAa,EAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,SAAS,CAAC,aAAa,IAAI,SAAS,CAAC,UAAU,EAAE,CAAC;YAClD,+CAA+C;YAC/C,MAAM,IAAI,GAAG,CAAC;QAClB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QACxD,IAAA,oBAAU,EAAC,SAAS,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QAErD,OAAO,MAAM,CAAC;IAClB,CAAC;IAGD,KAAK,CAAC,CAAC,SAAS,CAAC,IAAW,EAAE,OAA+B,EAAE,WAA4B;QACvF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;QAEjF,MAAM,OAAO,GAAiC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;QAE/E,IAAI,YAAY,GAAG,IAAA,aAAK,GAAE,CAAC;QAC3B,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,MAAM,OAAO,GAAG,KAAK,EAAE,EAA4C,EAAE,GAAW,EAAE,EAAE;YAChF,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;oBACvB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAEjB,IAAI,CAAC,EAAE,CAAC;wBACJ,YAAY,CAAC,OAAO,EAAE,CAAC;wBACvB,YAAY,GAAG,IAAA,aAAK,GAAE,CAAC;oBAC3B,CAAC;gBAEL,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrF,CAAC;QACL,CAAC,CAAC;QAEF,OAAO,CAAC,UAAU,CACd,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAC/C,CAAC,OAAO,CAAC,GAAG,EAAE;YACX,SAAS,GAAG,IAAI,CAAC;YACjB,YAAY,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC;QAEd,IAAI,CAAC;YACD,OAAO,CAAC,SAAS,EAAE,CAAC;gBAChB,MAAM,YAAY,CAAC,OAAO,CAAC;gBAE3B,MAAM,OAAO,CAAC;YAClB,CAAC;YACD,MAAM,OAAO,CAAC;QAClB,CAAC;gBAAS,CAAC;YACP,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,CAAC,CAAC,MAAM,EAAE,CAAC;YACf,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAoB;QAEhC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAChE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QAC5D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACxD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,SAAS,GAA0B;YACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU;YACxB,eAAe,EAAE,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACrF,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,iBAAiB,EAAE,IAAI,CAAC,SAAS;YACjC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS;YACzD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;YAC5E,cAAc,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;YAC3E,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;SACpC,CAAC;QAEF,IAAI,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC;YACnC,IAAI,OAAO,SAAS,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;gBAC/C,SAAS,CAAC,cAAc,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBACvC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;oBAChD,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC7D,MAAM,IAAI,8BAAoB,CAAC;4BAC3B,OAAO,EAAE,2BAA2B,CAAC,wEAAwE;4BAC7G,IAAI,EAAE,gBAAgB;yBACzB,CAAC,CAAC;oBACP,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClE,MAAM,KAAK,GAA2B,EAAE,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;oBAClB,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC7D,CAAC;YACL,CAAC;YACD,MAAM,WAAW,GAAG,gBAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACvE,SAAS,CAAC,WAAW,GAAG,WAAW,EAAE,WAAW,EAAE,CAAC;QACvD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,SAAS,CAAC,YAAY,KAAK,EAAE,CAAC;YAC9B,SAAS,CAAC,YAAY,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,gCAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,SAAS,CAAC,eAAe,GAAG,IAAI,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,EAAE,MAAM,EAAE,CAAC;YACjC,SAAS,CAAC,kBAAkB,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAC7C,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC7B,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1C,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC,CAAC,CACL,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;YAChC,SAAS,CAAC,iBAAiB,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAC5C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBAC5B,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBAClB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1C,CAAC;gBAED,OAAO,CAAC,CAAC;YACb,CAAC,CAAC,CACL,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAES,KAAK,CAAC,cAAc,CAC1B,cAA8B,EAC9B,QAGC,EACD,UAAgB,EAChB,UAAmB,EACnB,gBAAmC;QAEnC,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAE1F,MAAM,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC;QAC/C,IAAI,WAAW,KAAK,gCAAc,CAAC,SAAS,IAAI,WAAW,KAAK,gCAAc,CAAC,GAAG,EAAE,CAAC;YACjF,MAAM,MAAM,GAAkB;gBAC1B,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW;gBACrC,GAAG,EAAE,WAAW,EAAE,IAAI,IAAI,QAAQ,CAAC,IAAI;aAC1C,CAAC;YAEF,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,oBAAoB,EAAE;gBAChD,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW;gBACnC,UAAU,EAAE,KAAK;aACpB,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,OAAO,IAAI,CAAC,6BAA6B,CAAC,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;IAChH,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,IAAY,EAAE,QAAsB,EAAE,UAAgB,EAAE,UAAmB,EAAE,gBAAmC;QAChJ,MAAM,YAAY,GAAG,gBAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAE3C,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,gBAAgB,EAAE,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;gBACnE,IAAI,UAAU,EAAE,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;oBAChD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAA,mBAAa,EAAC,MAAM,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;oBAC3E,OAAO,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;gBAC7F,CAAC;gBAED,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACvF,IAAI,GAAG,YAAY,+BAAsB,EAAE,CAAC;wBACxC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,+BAAqB,CAAC,sBAAsB,MAAM,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;oBACtG,CAAC;oBAED,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACnB,MAAM,IAAI,+BAAqB,CAAC,sBAAsB,MAAM,8BAA8B,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC1G,CAAC;gBACD,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,+BAAqB,CAAC,sBAAsB,MAAM,+CAA+C,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;gBAChI,CAAC;gBACD,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,IAAI,+BAAqB,CAAC,sBAAsB,MAAM,iCAAiC,CAAC,CAAC;gBACnG,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAA,mBAAa,EAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC;YACrE,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IAC7F,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,GAAQ,EAAE,IAA4B,EAAE,cAA+B;QAC1F,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;QAEvD,IAAI,YAAY,CAAC;QACjB,IAAI,SAAS,CAAC;QACd,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvB,YAAY,GAAG,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,SAAS,GAAG,GAAG,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,YAAY,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,MAAM,IAAI,+BAAqB,CAAC,sBAAsB,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY,EAAE,GAAQ,EAAE,IAA4B;QAClE,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC;QAEpE,IAAI,YAAY,CAAC;QACjB,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC;YACD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;gBACvB,YAAY,GAAG,CAAC,CAAC;gBAEjB,IAAI,UAAU,EAAE,CAAC;oBACb,MAAM;gBACV,CAAC;gBAED,IAAI,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;oBAChC,kDAAkD;oBAClD,UAAU,GAAG,IAAI,CAAC;gBACtB,CAAC;YACL,CAAC;QAEL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,YAAY,EAAE,CAAC;gBACf,OAAO,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAC3F,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;QAED,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,MAAM,IAAI,+BAAqB,CAAC,sBAAsB,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3F,CAAC;IAED,yBAAyB,CAAC,GAAQ;QAC9B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAElD,MAAM,SAAS,GAAG,UAAU,IAAI,QAAQ,CAAC;QAEzC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,iBAAiB,EAAE,GAAG,SAAS,CAAC;QAEvD,OAAO;YACH,MAAM,EAAE,8BAAS,CAAC,IAAI,CAAC,GAAG,CAAC;YAC3B,IAAI,EAAE,SAAS;SAClB,CAAC;IACN,CAAC;IAiBK,AAAN,KAAK,CAAC,0BAA0B,CAAC,GAAQ,EAAE,IAA4B;QACnE,IAAI,IAAI,EAAE,UAAU,KAAK,MAAM,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,KAAK,CAAC;QACV,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,CAAC,EAAE,EAAE,CAAC;gBACN,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACpC,CAAC;YACD,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;QACpC,CAAC;QAED,KAAK,KAAK,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC3C,GAAG,IAAI;YACP,QAAQ,EAAE,KAAK,CAAC,IAAI;SACvB,CAAC,CAAC;QAEH,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC1B,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,IAAI,CAAC;QACjC,CAAC;QAED,OAAO,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC;IAC3B,CAAC;IAES,yBAAyB,CAAC,IAA4B;QAC5D,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,IAAI,KAAK,CAAC;QAEV,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/C,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;YAC5B,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;gBACpC,OAAO,MAAM,CAAC;YAClB,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChD,KAAK,KAAK,IAAI,CAAC,WAAW,CAAC;YAC/B,CAAC;QACL,CAAC;QAED,KAAK,KAAK,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC;QAEpC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,2CAA2C,CAAC,GAAQ;QAChD,IAAI,GAAG,CAAC,QAAQ,KAAK,2BAA2B,EAAE,CAAC;YAC/C,OAAO,IAAI,CAAC;QAChB,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,IAAoB,EAAE,IAA2B;QACxE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;YAChD,aAAa,GAAG,GAAG,CAAC;QACxB,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAClD,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAC7C,aAAa,GAAG,KAAK,CAAC;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,6BAAW,CAAC,oBAAoB,EAAE,CAAC;YACnD,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,8BAA8B,CAAC,CAAC;YACzD,aAAa,GAAG,KAAK,CAAC;QAC1B,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;YAC3C,aAAa,GAAG,KAAK,CAAC;YACtB,YAAY,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YACxF,YAAY,GAAG,CAAC,CAAC;QACrB,CAAC;QAED,OAAO;YACH,MAAM,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC;YAC7B,YAAY;YACZ,aAAa;SAChB,CAAC;IACN,CAAC;IAED,mBAAmB,CAAC,MAA6D,EAAE,YAAoB;QACnG,MAAM,qBAAqB,GAAG,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;QACjG,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YACzD,MAAM,IAAI,4BAAmB,CAAC,iBAAiB,MAAM,CAAC,MAAM,sCAAsC,qBAAqB,EAAE,CAAC,CAAC;QAC/H,CAAC;QAED,OAAO,qBAAqB,CAAC;IACjC,CAAC;CACJ,CAAA;AAlxCY,kCAAW;AAgJd;IAZL,IAAA,iBAAM,EAAC;QACJ,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,sBAAsB;QACnC,KAAK,EAAE;YACH,IAAI,EAAE;gBACF,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,GAAG;aACZ;SACJ;QACD,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;QACvB,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;KAC/B,CAAC;IACkB,WAAA,IAAA,cAAG,GAAE,CAAA;IAAgB,WAAA,IAAA,gBAAK,EAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAA;;yDAApC,kBAAO,oBAAP,kBAAO,gCAAqC,4CAAqB;;+CAU/F;AA0BK;IAvBL,IAAA,iBAAM,EAAC;QACJ,IAAI,EAAE,uBAAuB;QAC7B,WAAW,EAAE,6BAA6B;QAC1C,KAAK,EAAE;YACH,IAAI,EAAE;gBACF,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,GAAG;aACZ;SACJ;QACD,IAAI,EAAE,CAAC,OAAO,CAAC;QACf,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,CAAC;KAChD,CAAC;IACD,IAAA,iBAAM,EAAC;QACJ,WAAW,EAAE,6BAA6B;QAC1C,KAAK,EAAE;YACH,IAAI,EAAE;gBACF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;gBACvB,IAAI,EAAE,OAAO;aAChB;SACJ;QACD,IAAI,EAAE,CAAC,OAAO,CAAC;QACf,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,EAAE,mBAAS,CAAC;KAC3D,CAAC;IAEG,WAAA,IAAA,qBAAU,GAAE,CAAA;IACZ,WAAA,IAAA,cAAG,GAAE,CAAA;;yDADoB,uBAAa,oBAAb,uBAAa,oDAC3B,kBAAO,oBAAP,kBAAO,gCACb,4CAAqB;QACD,0CAAwB;QACrB,gCAAc;;wCA0O9C;AAgwBK;IAdL,IAAA,sBAAS,EAAC,CAAC,GAAG,EAAE,EAAE;QACf,IAAI,GAAG,YAAY,gCAAuB,EAAE,CAAC;YACzC,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,GAAG,YAAY,+BAAsB,EAAE,CAAC;YACxC,cAAc;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,GAAG,YAAY,0BAAgB,EAAE,CAAC;YAClC,uBAAuB;YACvB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC,EAAE,CAAC,CAAC;;yDACiC,GAAG,oBAAH,GAAG;;6DA0BxC;sBA7rCQ,WAAW;IADvB,IAAA,oBAAS,GAAE;qCAaoB,qBAAY;QACR,4BAAgB;QACrB,kBAAW;QACJ,yCAAkB,sBACvB,qCAAoB,oBAApB,qCAAoB,gCACxB,cAAS;QACN,oBAAY;QACP,sCAAiB;QACb,sDAA4B,sBACjC,6BAAgB,oBAAhB,6BAAgB,gCACrB,iCAAiB;QACZ,8BAAgB;QACjB,2BAAe;QAClB,oBAAY;QACb,kBAAW;GA1B7B,WAAW,CAkxCvB"}

build/api/searcher.js

@@ -0,0 +1,771 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a, _b, _c, _d, _e, _f;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SearcherHost = void 0;
+const tsyringe_1 = require("tsyringe");
+const civ_rpc_1 = require("civkit/civ-rpc");
+const lang_1 = require("civkit/lang");
+const hash_1 = require("civkit/hash");
+const lodash_1 = __importDefault(require("lodash"));
+const rate_limit_1 = require("../shared/services/rate-limit");
+const crawler_1 = require("./crawler");
+const crawler_options_1 = require("../dto/crawler-options");
+const snapshot_formatter_1 = require("../services/snapshot-formatter");
+const serper_search_1 = require("../services/serper-search");
+const logger_1 = require("../services/logger");
+const async_context_1 = require("../services/async-context");
+const registry_1 = require("../services/registry");
+const transform_server_event_stream_1 = require("../lib/transform-server-event-stream");
+const jina_embeddings_auth_1 = require("../dto/jina-embeddings-auth");
+const errors_1 = require("../services/errors");
+// import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
+const misc_1 = require("../utils/misc");
+const lru_cache_1 = require("lru-cache");
+const api_roll_1 = require("../shared/db/api-roll");
+const searched_1 = require("../db/searched");
+// import { SerperSearchQueryParams, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
+const internal_1 = require("../services/serp/internal");
+const WORLD_COUNTRY_CODES = Object.keys(WORLD_COUNTRIES).map((x) => x.toLowerCase());
+let SearcherHost = class SearcherHost extends civ_rpc_1.RPCHost {
+    constructor(globalLogger, rateLimitControl, threadLocal, crawler, snapshotFormatter, serperGoogle, serperBing, jinaSerp) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.rateLimitControl = rateLimitControl;
+        this.threadLocal = threadLocal;
+        this.crawler = crawler;
+        this.snapshotFormatter = snapshotFormatter;
+        this.serperGoogle = serperGoogle;
+        this.serperBing = serperBing;
+        this.jinaSerp = jinaSerp;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.cacheRetentionMs = 1000 * 3600 * 24 * 7;
+        this.cacheValidMs = 1000 * 3600;
+        this.pageCacheToleranceMs = 1000 * 3600 * 24;
+        this.reasonableDelayMs = 15_000;
+        this.targetResultCount = 5;
+        this.highFreqKeyCache = new lru_cache_1.LRUCache({
+            max: 256,
+            ttl: 60 * 60 * 1000,
+            updateAgeOnGet: false,
+            updateAgeOnHas: false,
+        });
+        this.batchedCaches = [];
+        setInterval(() => {
+            const thisBatch = this.batchedCaches;
+            this.batchedCaches = [];
+            if (!thisBatch.length) {
+                return;
+            }
+            const batch = searched_1.SERPResult.DB.batch();
+            for (const x of thisBatch) {
+                batch.set(searched_1.SERPResult.COLLECTION.doc(), x.degradeForFireStore());
+            }
+            batch.commit()
+                .then(() => {
+                this.logger.debug(`Saved ${thisBatch.length} caches by batch`);
+            })
+                .catch((err) => {
+                this.logger.warn(`Failed to cache search result in batch`, { err });
+            });
+        }, 1000 * 10 + Math.round(1000 * Math.random())).unref();
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+    async search(rpcReflect, ctx, auth, crawlerOptions, searchExplicitOperators, count, variant, searchEngine, num, gl, hl, location, page, fallback, q) {
+        // We want to make our search API follow SERP schema, so we need to expose 'num' parameter.
+        // Since we used 'count' as 'num' previously, we need to keep 'count' for old users.
+        // Here we combine 'count' and 'num' to 'count' for the rest of the function.
+        count = (num !== undefined ? num : count) ?? 10;
+        const authToken = auth.bearerToken;
+        let highFreqKey;
+        if (authToken && this.highFreqKeyCache.has(authToken)) {
+            highFreqKey = this.highFreqKeyCache.get(authToken);
+            auth.user = highFreqKey.user;
+            auth.uid = highFreqKey.user?.user_id;
+        }
+        const uid = await auth.solveUID();
+        // Return content by default
+        const crawlWithoutContent = crawlerOptions.respondWith.includes('no-content');
+        const withFavicon = Boolean(ctx.get('X-With-Favicons'));
+        this.threadLocal.set('collect-favicon', withFavicon);
+        crawlerOptions.respondTiming ??= crawler_options_1.RESPOND_TIMING.VISIBLE_CONTENT;
+        let chargeAmount = 0;
+        const noSlashPath = decodeURIComponent(ctx.path).slice(1);
+        if (!noSlashPath && !q) {
+            const index = await this.crawler.getIndex(auth);
+            if (!uid) {
+                index.note = 'Authentication is required to use this endpoint. Please provide a valid API key via Authorization header.';
+            }
+            if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
+                return index;
+            }
+            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${index}`, { contentType: 'text/plain', envelope: null });
+        }
+        const user = await auth.assertUser();
+        if (!(user.wallet.total_balance > 0)) {
+            throw new errors_1.InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);
+        }
+        if (highFreqKey?.blockedUntil) {
+            const now = new Date();
+            const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());
+            if (blockedTimeRemaining > 0) {
+                throw rate_limit_1.RateLimitTriggeredError.from({
+                    message: `Per UID rate limit exceeded (async)`,
+                    retryAfter: Math.ceil(blockedTimeRemaining / 1000),
+                });
+            }
+        }
+        const rateLimitPolicy = auth.getRateLimits(rpcReflect.name.toUpperCase()) || [
+            parseInt(user.metadata?.speed_level) >= 2 ?
+                rate_limit_1.RateLimitDesc.from({
+                    occurrence: 1000,
+                    periodSeconds: 60
+                }) :
+                rate_limit_1.RateLimitDesc.from({
+                    occurrence: 100,
+                    periodSeconds: 60
+                })
+        ];
+        const apiRollPromise = this.rateLimitControl.simpleRPCUidBasedLimit(rpcReflect, uid, [rpcReflect.name.toUpperCase()], ...rateLimitPolicy);
+        if (!highFreqKey) {
+            // Normal path
+            await apiRollPromise;
+            if (rateLimitPolicy.some((x) => {
+                const rpm = x.occurrence / (x.periodSeconds / 60);
+                if (rpm >= 400) {
+                    return true;
+                }
+                return false;
+            })) {
+                this.highFreqKeyCache.set(auth.bearerToken, {
+                    user,
+                });
+            }
+        }
+        else {
+            // High freq key path
+            apiRollPromise.then(
+            // Rate limit not triggered, make sure not blocking.
+            () => {
+                delete highFreqKey.blockedUntil;
+            }, 
+            // Rate limit triggered
+            (err) => {
+                if (!(err instanceof rate_limit_1.RateLimitTriggeredError)) {
+                    return;
+                }
+                const now = Date.now();
+                let tgtDate;
+                if (err.retryAfterDate) {
+                    tgtDate = err.retryAfterDate;
+                }
+                else if (err.retryAfter) {
+                    tgtDate = new Date(now + err.retryAfter * 1000);
+                }
+                if (tgtDate) {
+                    const dt = tgtDate.valueOf() - now;
+                    highFreqKey.blockedUntil = tgtDate;
+                    setTimeout(() => {
+                        if (highFreqKey.blockedUntil === tgtDate) {
+                            delete highFreqKey.blockedUntil;
+                        }
+                    }, dt).unref();
+                }
+            }).finally(async () => {
+                // Always asynchronously update user(wallet);
+                const user = await auth.getBrief().catch(() => undefined);
+                if (user) {
+                    highFreqKey.user = user;
+                }
+            });
+        }
+        rpcReflect.finally(async () => {
+            if (chargeAmount) {
+                auth.reportUsage(chargeAmount, `reader-${rpcReflect.name}`).catch((err) => {
+                    this.logger.warn(`Unable to report usage for ${uid}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                });
+                try {
+                    const apiRoll = await apiRollPromise;
+                    apiRoll.chargeAmount = chargeAmount;
+                }
+                catch (err) {
+                    await this.rateLimitControl.record({
+                        uid,
+                        tags: [rpcReflect.name.toUpperCase()],
+                        status: api_roll_1.API_CALL_STATUS.SUCCESS,
+                        chargeAmount,
+                    }).save().catch((err) => {
+                        this.logger.warn(`Failed to save rate limit record`, { err: (0, lang_1.marshalErrorLike)(err) });
+                    });
+                }
+            }
+        });
+        delete crawlerOptions.html;
+        const crawlOpts = await this.crawler.configure(crawlerOptions);
+        const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);
+        let fetchNum = count;
+        if ((page ?? 1) === 1) {
+            fetchNum = count > 10 ? 30 : 20;
+        }
+        let fallbackQuery;
+        let chargeAmountScaler = 1;
+        if (searchEngine === 'bing') {
+            this.threadLocal.set('bing-preferred', true);
+            chargeAmountScaler = 3;
+        }
+        if (variant !== 'web') {
+            chargeAmountScaler = 5;
+        }
+        // Search with fallback logic if enabled
+        const searchParams = {
+            variant,
+            provider: searchEngine,
+            q: searchQuery,
+            num: fetchNum,
+            gl,
+            hl,
+            location,
+            page,
+        };
+        const { results, query: successQuery, tryTimes } = await this.searchWithFallback(searchParams, fallback, crawlerOptions.noCache);
+        chargeAmountScaler *= tryTimes;
+        fallbackQuery = successQuery !== searchQuery ? successQuery : undefined;
+        if (!results.length) {
+            throw new civ_rpc_1.AssertionFailureError(`No search results available for query ${searchQuery}`);
+        }
+        if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) {
+            delete crawlOpts.timeoutMs;
+        }
+        let lastScrapped;
+        const targetResultCount = crawlWithoutContent ? count : count + 2;
+        const trimmedResults = results.filter((x) => Boolean(x.link)).slice(0, targetResultCount).map((x) => this.mapToFinalResults(x));
+        trimmedResults.toString = function () {
+            let r = this.map((x, i) => x ? Reflect.apply(x.toString, x, [i]) : '').join('\n\n').trimEnd() + '\n';
+            if (fallbackQuery) {
+                r = `Fallback query: ${fallbackQuery}\n\n${r}`;
+            }
+            return r;
+        };
+        if (!crawlerOptions.respondWith.includes('no-content') &&
+            ['html', 'text', 'shot', 'markdown', 'content'].some((x) => crawlerOptions.respondWith.includes(x))) {
+            for (const x of trimmedResults) {
+                x.content ??= '';
+            }
+        }
+        const assigningOfGeneralMixins = Promise.allSettled(trimmedResults.map((x) => this.assignGeneralMixin(x)));
+        let it;
+        if (crawlWithoutContent || count === 0) {
+            it = (0, misc_1.toAsyncGenerator)(trimmedResults);
+            await assigningOfGeneralMixins;
+        }
+        else {
+            it = this.fetchSearchResults(crawlerOptions.respondWith, trimmedResults, crawlOpts, crawler_options_1.CrawlerOptions.from({ ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs }), count);
+        }
+        if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) {
+            const sseStream = new transform_server_event_stream_1.OutputServerEventStream();
+            rpcReflect.return(sseStream);
+            try {
+                for await (const scrapped of it) {
+                    if (!scrapped) {
+                        continue;
+                    }
+                    if (rpcReflect.signal.aborted) {
+                        break;
+                    }
+                    chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
+                    lastScrapped = scrapped;
+                    if (fallbackQuery) {
+                        sseStream.write({
+                            event: 'meta',
+                            data: { fallback: fallbackQuery },
+                        });
+                    }
+                    sseStream.write({
+                        event: 'data',
+                        data: scrapped,
+                    });
+                }
+            }
+            catch (err) {
+                this.logger.error(`Failed to collect search result for query ${searchQuery}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                sseStream.write({
+                    event: 'error',
+                    data: (0, lang_1.marshalErrorLike)(err),
+                });
+            }
+            sseStream.end();
+            return sseStream;
+        }
+        let earlyReturn = false;
+        if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
+            let earlyReturnTimer;
+            const setEarlyReturnTimer = () => {
+                if (earlyReturnTimer) {
+                    return;
+                }
+                earlyReturnTimer = setTimeout(async () => {
+                    if (!lastScrapped) {
+                        return;
+                    }
+                    await assigningOfGeneralMixins;
+                    chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
+                    rpcReflect.return(lastScrapped);
+                    earlyReturn = true;
+                }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
+            };
+            for await (const scrapped of it) {
+                lastScrapped = scrapped;
+                if (rpcReflect.signal.aborted || earlyReturn) {
+                    break;
+                }
+                if (lodash_1.default.some(scrapped, (x) => this.pageQualified(x))) {
+                    setEarlyReturnTimer();
+                }
+                if (!this.searchResultsQualified(scrapped, count)) {
+                    continue;
+                }
+                if (earlyReturnTimer) {
+                    clearTimeout(earlyReturnTimer);
+                }
+                await assigningOfGeneralMixins;
+                chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
+                return scrapped;
+            }
+            if (earlyReturnTimer) {
+                clearTimeout(earlyReturnTimer);
+            }
+            if (!lastScrapped) {
+                throw new civ_rpc_1.AssertionFailureError(`No content available for query ${searchQuery}`);
+            }
+            if (!earlyReturn) {
+                await assigningOfGeneralMixins;
+                chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
+            }
+            return lastScrapped;
+        }
+        let earlyReturnTimer;
+        const setEarlyReturnTimer = () => {
+            if (earlyReturnTimer) {
+                return;
+            }
+            earlyReturnTimer = setTimeout(async () => {
+                if (!lastScrapped) {
+                    return;
+                }
+                await assigningOfGeneralMixins;
+                chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
+                rpcReflect.return((0, civ_rpc_1.assignTransferProtocolMeta)(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }));
+                earlyReturn = true;
+            }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs);
+        };
+        for await (const scrapped of it) {
+            lastScrapped = scrapped;
+            if (rpcReflect.signal.aborted || earlyReturn) {
+                break;
+            }
+            if (lodash_1.default.some(scrapped, (x) => this.pageQualified(x))) {
+                setEarlyReturnTimer();
+            }
+            if (!this.searchResultsQualified(scrapped, count)) {
+                continue;
+            }
+            if (earlyReturnTimer) {
+                clearTimeout(earlyReturnTimer);
+            }
+            await assigningOfGeneralMixins;
+            chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery);
+            return (0, civ_rpc_1.assignTransferProtocolMeta)(`${scrapped}`, { contentType: 'text/plain', envelope: null });
+        }
+        if (earlyReturnTimer) {
+            clearTimeout(earlyReturnTimer);
+        }
+        if (!lastScrapped) {
+            throw new civ_rpc_1.AssertionFailureError(`No content available for query ${searchQuery}`);
+        }
+        if (!earlyReturn) {
+            await assigningOfGeneralMixins;
+            chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery);
+        }
+        return (0, civ_rpc_1.assignTransferProtocolMeta)(`${lastScrapped}`, { contentType: 'text/plain', envelope: null });
+    }
+    /**
+     * Search with fallback to progressively shorter queries if no results found
+     * @param params Search parameters
+     * @param useFallback Whether to use the fallback mechanism
+     * @param noCache Whether to bypass cache
+     * @returns Search response and the successful query
+     */
+    async searchWithFallback(params, useFallback = false, noCache = false) {
+        // Try original query first
+        const originalQuery = params.q;
+        const containsRTL = /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(originalQuery);
+        // Extract results based on variant
+        let tryTimes = 1;
+        const results = await this.cachedSearch(params.variant, params, noCache);
+        if (results.length || !useFallback) {
+            return { results, query: params.q, tryTimes };
+        }
+        let queryTerms = originalQuery.split(/\s+/);
+        const lastResort = containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2);
+        this.logger.info(`No results for "${originalQuery}", trying fallback queries`);
+        let terms = [];
+        // fallback n times
+        const n = 4;
+        while (tryTimes < n) {
+            const delta = Math.ceil(queryTerms.length / n) * tryTimes;
+            terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta);
+            const query = terms.join(' ');
+            if (!query) {
+                break;
+            }
+            tryTimes += 1;
+            this.logger.info(`Retrying search with fallback query: "${query}"`);
+            const fallbackParams = { ...params, q: query };
+            const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);
+            if (fallbackResults.length > 0) {
+                return { results: fallbackResults, query: fallbackParams.q, tryTimes };
+            }
+        }
+        if (terms.length > lastResort.length) {
+            const query = lastResort.join(' ');
+            this.logger.info(`Retrying search with fallback query: "${query}"`);
+            const fallbackParams = { ...params, q: query };
+            tryTimes += 1;
+            const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache);
+            if (fallbackResults.length > 0) {
+                return { results: fallbackResults, query, tryTimes };
+            }
+        }
+        return { results, query: originalQuery, tryTimes };
+    }
+    async *fetchSearchResults(mode, searchResults, options, crawlerOptions, count) {
+        if (!searchResults) {
+            return;
+        }
+        const urls = searchResults.map((x) => new URL(x.url));
+        const snapshotMap = new WeakMap();
+        for await (const scrapped of this.crawler.scrapMany(urls, options, crawlerOptions)) {
+            const mapped = scrapped.map((x, i) => {
+                if (!x) {
+                    return {};
+                }
+                if (snapshotMap.has(x)) {
+                    return snapshotMap.get(x);
+                }
+                return this.crawler.formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options).then((r) => {
+                    snapshotMap.set(x, r);
+                    return r;
+                }).catch((err) => {
+                    this.logger.error(`Failed to format snapshot for ${urls[i].href}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                    return {};
+                });
+            });
+            const resultArray = await Promise.all(mapped);
+            for (const [i, v] of resultArray.entries()) {
+                if (v) {
+                    Object.assign(searchResults[i], v);
+                }
+            }
+            yield this.reOrganizeSearchResults(searchResults, count);
+        }
+    }
+    reOrganizeSearchResults(searchResults, count) {
+        const targetResultCount = count || this.targetResultCount;
+        const [qualifiedPages, unqualifiedPages] = lodash_1.default.partition(searchResults, (x) => this.pageQualified(x));
+        const acceptSet = new Set(qualifiedPages);
+        const n = targetResultCount - qualifiedPages.length;
+        for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) {
+            acceptSet.add(x);
+        }
+        const filtered = searchResults.filter((x) => acceptSet.has(x)).slice(0, targetResultCount);
+        const resultArray = filtered;
+        resultArray.toString = searchResults.toString;
+        return resultArray;
+    }
+    assignChargeAmount(formatted, num, scaler, fallbackQuery) {
+        let contentCharge = 0;
+        for (const x of formatted) {
+            const itemAmount = this.crawler.assignChargeAmount(x) || 0;
+            if (!itemAmount) {
+                continue;
+            }
+            contentCharge += itemAmount;
+        }
+        const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler;
+        const final = Math.max(contentCharge, numCharge);
+        if (final === numCharge) {
+            for (const x of formatted) {
+                x.usage = { tokens: Math.ceil(numCharge / formatted.length) };
+            }
+        }
+        const metadata = { usage: { tokens: final } };
+        if (fallbackQuery) {
+            metadata.fallback = fallbackQuery;
+        }
+        (0, civ_rpc_1.assignMeta)(formatted, metadata);
+        return final;
+    }
+    pageQualified(formattedPage) {
+        return formattedPage.title &&
+            formattedPage.content ||
+            formattedPage.screenshotUrl ||
+            formattedPage.pageshotUrl ||
+            formattedPage.text ||
+            formattedPage.html;
+    }
+    searchResultsQualified(results, targetResultCount = this.targetResultCount) {
+        return lodash_1.default.every(results, (x) => this.pageQualified(x)) && results.length >= targetResultCount;
+    }
+    async getFavicon(domain) {
+        const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                return '';
+            }
+            const ab = await response.arrayBuffer();
+            const buffer = Buffer.from(ab);
+            const base64 = buffer.toString('base64');
+            return `data:image/png;base64,${base64}`;
+        }
+        catch (error) {
+            this.logger.warn(`Failed to get favicon base64 string`, { err: (0, lang_1.marshalErrorLike)(error) });
+            return '';
+        }
+    }
+    *iterProviders(preference, variant) {
+        if (preference === 'bing') {
+            yield this.serperBing;
+            yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
+            yield this.serperGoogle;
+            return;
+        }
+        if (preference === 'google') {
+            yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
+            yield this.serperGoogle;
+            yield this.serperGoogle;
+            return;
+        }
+        yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
+        yield this.serperGoogle;
+        yield this.serperGoogle;
+    }
+    async cachedSearch(variant, query, noCache) {
+        const queryDigest = (0, hash_1.objHashMd5B64Of)({ ...query, variant });
+        const provider = query.provider;
+        Reflect.deleteProperty(query, 'provider');
+        let cache;
+        if (!noCache) {
+            cache = (await searched_1.SERPResult.fromFirestoreQuery(searched_1.SERPResult.COLLECTION.where('queryDigest', '==', queryDigest)
+                .orderBy('createdAt', 'desc')
+                .limit(1)))[0];
+            if (cache) {
+                const age = Date.now() - cache.createdAt.valueOf();
+                const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs);
+                this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, {
+                    query, digest: queryDigest, age, stale
+                });
+                if (!stale) {
+                    return cache.response;
+                }
+            }
+        }
+        try {
+            let r;
+            let lastError;
+            outerLoop: for (const client of this.iterProviders(provider, variant)) {
+                const t0 = Date.now();
+                try {
+                    switch (variant) {
+                        case 'images': {
+                            r = await Reflect.apply(client.imageSearch, client, [query]);
+                            break;
+                        }
+                        case 'news': {
+                            r = await Reflect.apply(client.newsSearch, client, [query]);
+                            break;
+                        }
+                        case 'web':
+                        default: {
+                            r = await Reflect.apply(client.webSearch, client, [query]);
+                            break;
+                        }
+                    }
+                    const dt = Date.now() - t0;
+                    this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name });
+                    break outerLoop;
+                }
+                catch (err) {
+                    lastError = err;
+                    const dt = Date.now() - t0;
+                    this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, });
+                }
+            }
+            if (r?.length) {
+                const nowDate = new Date();
+                const record = searched_1.SERPResult.from({
+                    query,
+                    queryDigest,
+                    response: r,
+                    createdAt: nowDate,
+                    expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs)
+                });
+                this.batchedCaches.push(record);
+            }
+            else if (lastError) {
+                throw lastError;
+            }
+            return r;
+        }
+        catch (err) {
+            if (cache) {
+                this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: (0, lang_1.marshalErrorLike)(err) });
+                return cache.response;
+            }
+            throw err;
+        }
+    }
+    mapToFinalResults(input) {
+        const whitelistedProps = [
+            'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks'
+        ];
+        const result = {
+            title: input.title,
+            url: input.link,
+            description: Reflect.get(input, 'snippet'),
+            ...lodash_1.default.pick(input, whitelistedProps),
+        };
+        return result;
+    }
+    async assignGeneralMixin(result) {
+        const collectFavicon = this.threadLocal.get('collect-favicon');
+        if (collectFavicon && result.url) {
+            const url = new URL(result.url);
+            Reflect.set(result, 'favicon', await this.getFavicon(url.origin));
+        }
+        Object.setPrototypeOf(result, searchResultProto);
+    }
+};
+exports.SearcherHost = SearcherHost;
+__decorate([
+    (0, registry_1.Method)({
+        name: 'searchIndex',
+        ext: {
+            http: {
+                action: ['get', 'post'],
+                path: '/search'
+            }
+        },
+        tags: ['search'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream],
+    }),
+    (0, registry_1.Method)({
+        ext: {
+            http: {
+                action: ['get', 'post'],
+                path: '::q'
+            }
+        },
+        tags: ['search'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream, civ_rpc_1.RawString],
+    }),
+    __param(0, (0, registry_1.RPCReflect)()),
+    __param(1, (0, registry_1.Ctx)()),
+    __param(5, (0, registry_1.Param)('count', { validate: (v) => v >= 0 && v <= 20 })),
+    __param(6, (0, registry_1.Param)('type', { type: new Set(['web', 'images', 'news']), default: 'web' })),
+    __param(7, (0, registry_1.Param)('provider', { type: new Set(['google', 'bing']), default: 'google' })),
+    __param(8, (0, registry_1.Param)('num', { validate: (v) => v >= 0 && v <= 20 })),
+    __param(9, (0, registry_1.Param)('gl', { validate: (v) => WORLD_COUNTRY_CODES.includes(v?.toLowerCase()) })),
+    __param(10, (0, registry_1.Param)('hl', { validate: (v) => WORLD_LANGUAGES.some(l => l.code === v) })),
+    __param(11, (0, registry_1.Param)('location')),
+    __param(12, (0, registry_1.Param)('page')),
+    __param(13, (0, registry_1.Param)('fallback', { type: Boolean, default: true })),
+    __param(14, (0, registry_1.Param)('q')),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_d = typeof civ_rpc_1.RPCReflection !== "undefined" && civ_rpc_1.RPCReflection) === "function" ? _d : Object, typeof (_e = typeof registry_1.Context !== "undefined" && registry_1.Context) === "function" ? _e : Object, jina_embeddings_auth_1.JinaEmbeddingsAuthDTO,
+        crawler_options_1.CrawlerOptions, typeof (_f = typeof serper_search_1.GoogleSearchExplicitOperatorsDto !== "undefined" && serper_search_1.GoogleSearchExplicitOperatorsDto) === "function" ? _f : Object, Number, String, String, Number, String, String, String, Number, Boolean, String]),
+    __metadata("design:returntype", Promise)
+], SearcherHost.prototype, "search", null);
+exports.SearcherHost = SearcherHost = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger, typeof (_a = typeof rate_limit_1.RateLimitControl !== "undefined" && rate_limit_1.RateLimitControl) === "function" ? _a : Object, async_context_1.AsyncLocalContext,
+        crawler_1.CrawlerHost,
+        snapshot_formatter_1.SnapshotFormatter, typeof (_b = typeof SerperGoogleSearchService !== "undefined" && SerperGoogleSearchService) === "function" ? _b : Object, typeof (_c = typeof SerperBingSearchService !== "undefined" && SerperBingSearchService) === "function" ? _c : Object, internal_1.InternalJinaSerpService])
+], SearcherHost);
+const dataItems = [
+    { key: 'title', label: 'Title' },
+    { key: 'source', label: 'Source' },
+    { key: 'url', label: 'URL Source' },
+    { key: 'imageUrl', label: 'Image URL' },
+    { key: 'description', label: 'Description' },
+    { key: 'publishedTime', label: 'Published Time' },
+    { key: 'imageWidth', label: 'Image Width' },
+    { key: 'imageHeight', label: 'Image Height' },
+    { key: 'date', label: 'Date' },
+    { key: 'favicon', label: 'Favicon' },
+];
+const searchResultProto = {
+    toString(i) {
+        const chunks = [];
+        for (const item of dataItems) {
+            const v = Reflect.get(this, item.key);
+            if (typeof v !== 'undefined') {
+                if (i === undefined) {
+                    chunks.push(`[${item.label}]: ${v}`);
+                }
+                else {
+                    chunks.push(`[${i + 1}] ${item.label}: ${v}`);
+                }
+            }
+        }
+        if (this.content) {
+            chunks.push(`\n${this.content}`);
+        }
+        if (this.images) {
+            const imageSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Images:`];
+            for (const [k, v] of Object.entries(this.images)) {
+                imageSummaryChunks.push(`- ![${k}](${v})`);
+            }
+            if (imageSummaryChunks.length === 1) {
+                imageSummaryChunks.push('This page does not seem to contain any images.');
+            }
+            chunks.push(imageSummaryChunks.join('\n'));
+        }
+        if (this.links) {
+            const linkSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Links/Buttons:`];
+            if (Array.isArray(this.links)) {
+                for (const [k, v] of this.links) {
+                    linkSummaryChunks.push(`- [${k}](${v})`);
+                }
+            }
+            else {
+                for (const [k, v] of Object.entries(this.links)) {
+                    linkSummaryChunks.push(`- [${k}](${v})`);
+                }
+            }
+            if (linkSummaryChunks.length === 1) {
+                linkSummaryChunks.push('This page does not seem to contain any buttons/links.');
+            }
+            chunks.push(linkSummaryChunks.join('\n'));
+        }
+        return chunks.join('\n');
+    }
+};
+//# sourceMappingURL=searcher.js.map

build/api/searcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"searcher.js","sourceRoot":"","sources":["../../src/api/searcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,uCAAqC;AACrC,4CAEwB;AACxB,sCAA+C;AAC/C,sCAA8C;AAC9C,oDAAuB;AAEvB,8DAAyG;AAEzG,uCAA+D;AAC/D,4DAAwE;AACxE,uEAAuG;AACvG,6DAA6E;AAE7E,+CAAkD;AAClD,6DAA8D;AAC9D,mDAA+E;AAC/E,wFAA+E;AAC/E,sEAAoE;AACpE,+CAA8D;AAE9D,gGAAgG;AAChG,wCAAiD;AAEjD,yCAAqC;AACrC,oDAAwD;AACxD,6CAA4C;AAC5C,iHAAiH;AACjH,wDAAoE;AAGpE,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAa9E,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,iBAAO;IAoBrC,YACc,YAA0B,EAC1B,gBAAkC,EAClC,WAA8B,EAC9B,OAAoB,EACpB,iBAAoC,EACpC,YAAuC,EACvC,UAAmC,EACnC,QAAiC;QAE3C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QATV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,gBAAW,GAAX,WAAW,CAAmB;QAC9B,YAAO,GAAP,OAAO,CAAa;QACpB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,iBAAY,GAAZ,YAAY,CAA2B;QACvC,eAAU,GAAV,UAAU,CAAyB;QACnC,aAAQ,GAAR,QAAQ,CAAyB;QA3B/C,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,qBAAgB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QACxC,iBAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QAC3B,yBAAoB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QAExC,sBAAiB,GAAG,MAAM,CAAC;QAE3B,sBAAiB,GAAG,CAAC,CAAC;QAEtB,qBAAgB,GAAG,IAAI,oBAAQ,CAAyB;YACpD,GAAG,EAAE,GAAG;YACR,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;YACnB,cAAc,EAAE,KAAK;YACrB,cAAc,EAAE,KAAK;SACxB,CAAC,CAAC;QAEH,kBAAa,GAAiB,EAAE,CAAC;QAc7B,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YACrC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,qBAAU,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAEpC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,KAAK,CAAC,GAAG,CAAC,qBAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,KAAK,CAAC,MAAM,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,MAAM,kBAAkB,CAAC,CAAC;YACnE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAC7D,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAuBK,AAAN,KAAK,CAAC,MAAM,CACM,UAAyB,EAChC,GAAY,EACnB,IAA2B,EAC3B,cAA8B,EAC9B,uBAAyD,EAEzD,KAAa,EAEb,OAAkC,EAElC,YAA+B,EAE/B,GAAY,EAC8E,EAAW,EAClB,EAAW,EAC3E,QAAiB,EACrB,IAAa,EACyB,QAAkB,EAC3D,CAAU;QAEtB,2FAA2F;QAC3F,oFAAoF;QACpF,6EAA6E;QAC7E,KAAK,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAEhD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QACnC,IAAI,WAAuC,CAAC;QAC5C,IAAI,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;YACpD,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC;QACzC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,4BAA4B;QAC5B,MAAM,mBAAmB,GAAG,cAAc,CAAC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC9E,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAC;QACrD,cAAc,CAAC,aAAa,KAAK,gCAAc,CAAC,eAAe,CAAC;QAEhE,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;YACrB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,KAAK,CAAC,IAAI,GAAG,2GAA2G,CAAC;YAC7H,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBAE9F,OAAO,KAAK,CAAC;YACjB,CAAC;YAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,KAAK,EAAE,EACxC,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAChD,CAAC;QACN,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,iCAAwB,CAAC,gEAAgE,CAAC,CAAC;QACzG,CAAC;QAED,IAAI,WAAW,EAAE,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,oBAAoB,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAClF,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,oCAAuB,CAAC,IAAI,CAAC;oBAC/B,OAAO,EAAE,qCAAqC;oBAC9C,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;iBACrD,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI;YACzE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvC,0BAAa,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,IAAI;oBAChB,aAAa,EAAE,EAAE;iBACpB,CAAC,CAAC,CAAC;gBACJ,0BAAa,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,GAAG;oBACf,aAAa,EAAE,EAAE;iBACpB,CAAC;SACT,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,CAC/D,UAAU,EAAE,GAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EACjD,GAAG,eAAe,CACrB,CAAC;QAEF,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,cAAc;YACd,MAAM,cAAc,CAAC;YAErB,IAAI,eAAe,CAAC,IAAI,CACpB,CAAC,CAAC,EAAE,EAAE;gBACF,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;gBAClD,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,OAAO,KAAK,CAAC;YACjB,CAAC,CAAC,EACJ,CAAC;gBACC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAY,EAAE;oBACzC,IAAI;iBACP,CAAC,CAAC;YACP,CAAC;QAEL,CAAC;aAAM,CAAC;YACJ,qBAAqB;YACrB,cAAc,CAAC,IAAI;YACf,oDAAoD;YACpD,GAAG,EAAE;gBACD,OAAO,WAAW,CAAC,YAAY,CAAC;YACpC,CAAC;YACD,uBAAuB;YACvB,CAAC,GAAG,EAAE,EAAE;gBACJ,IAAI,CAAC,CAAC,GAAG,YAAY,oCAAuB,CAAC,EAAE,CAAC;oBAC5C,OAAO;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAI,OAAO,CAAC;gBACZ,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC,cAAc,CAAC;gBACjC,CAAC;qBAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBACxB,OAAO,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;gBACpD,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC;oBACnC,WAAW,CAAC,YAAY,GAAG,OAAO,CAAC;oBACnC,UAAU,CAAC,GAAG,EAAE;wBACZ,IAAI,WAAW,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;4BACvC,OAAO,WAAW,CAAC,YAAY,CAAC;wBACpC,CAAC;oBACL,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnB,CAAC;YACL,CAAC,CACJ,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACjB,6CAA6C;gBAC7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC1D,IAAI,IAAI,EAAE,CAAC;oBACP,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC;gBAC5B,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC1B,IAAI,YAAY,EAAE,CAAC;gBACf,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,UAAU,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACtE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1F,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;oBACrC,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;gBAExC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,GAAG;wBACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;wBACrC,MAAM,EAAE,0BAAe,CAAC,OAAO;wBAC/B,YAAY;qBACf,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACzF,CAAC,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,cAAc,CAAC,IAAI,CAAC;QAE3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC;QAEpE,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,QAAQ,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,CAAC;QAED,IAAI,aAAiC,CAAC;QACtC,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAC7C,kBAAkB,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;YACpB,kBAAkB,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,wCAAwC;QACxC,MAAM,YAAY,GAAG;YACjB,OAAO;YACP,QAAQ,EAAE,YAAY;YACtB,CAAC,EAAE,WAAW;YACd,GAAG,EAAE,QAAQ;YACb,EAAE;YACF,EAAE;YACF,QAAQ;YACR,IAAI;SACP,CAAC;QAEF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAC5E,YAAY,EAAE,QAAQ,EAAE,cAAc,CAAC,OAAO,CACjD,CAAC;QACF,kBAAkB,IAAI,QAAQ,CAAC;QAE/B,aAAa,GAAG,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;QAExE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,+BAAqB,CAAC,yCAAyC,WAAW,EAAE,CAAC,CAAC;QAC5F,CAAC;QAED,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,GAAG,MAAM,EAAE,CAAC;YACtD,OAAO,SAAS,CAAC,SAAS,CAAC;QAC/B,CAAC;QAGD,IAAI,YAA+B,CAAC;QACpC,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;QAClE,MAAM,cAAc,GAAU,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QACvI,cAAc,CAAC,QAAQ,GAAG;YACtB,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YACrG,IAAI,aAAa,EAAE,CAAC;gBAChB,CAAC,GAAG,mBAAmB,aAAa,OAAO,CAAC,EAAE,CAAC;YACnD,CAAC;YACD,OAAO,CAAC,CAAC;QACb,CAAC,CAAC;QACF,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC;YAClD,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EACrG,CAAC;YACC,KAAK,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;gBAC7B,CAAC,CAAC,OAAO,KAAK,EAAE,CAAC;YACrB,CAAC;QACL,CAAC;QACD,MAAM,wBAAwB,GAAG,OAAO,CAAC,UAAU,CAC/C,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CACxD,CAAC;QAEF,IAAI,EAAE,CAAC;QAEP,IAAI,mBAAmB,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YACrC,EAAE,GAAG,IAAA,uBAAgB,EAAC,cAAc,CAAC,CAAC;YACtC,MAAM,wBAAwB,CAAC;QACnC,CAAC;aAAM,CAAC;YACJ,EAAE,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,WAAW,EAAE,cAAc,EAAE,SAAS,EAC9E,gCAAc,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,EAAE,cAAc,EAAE,cAAc,CAAC,cAAc,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC,EACtH,KAAK,CACR,CAAC;QACN,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACjE,MAAM,SAAS,GAAG,IAAI,uDAAuB,EAAE,CAAC;YAChD,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC7B,IAAI,CAAC;gBACD,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,EAAE,EAAE,CAAC;oBAC9B,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACZ,SAAS;oBACb,CAAC;oBACD,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBAC5B,MAAM;oBACV,CAAC;oBAED,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;oBAC3F,YAAY,GAAG,QAAQ,CAAC;oBAExB,IAAI,aAAa,EAAE,CAAC;wBAChB,SAAS,CAAC,KAAK,CAAC;4BACZ,KAAK,EAAE,MAAM;4BACb,IAAI,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE;yBACpC,CAAC,CAAC;oBACP,CAAC;oBAED,SAAS,CAAC,KAAK,CAAC;wBACZ,KAAK,EAAE,MAAM;wBACb,IAAI,EAAE,QAAQ;qBACjB,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,WAAW,EAAE,EACxE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CACjC,CAAC;gBACF,SAAS,CAAC,KAAK,CAAC;oBACZ,KAAK,EAAE,OAAO;oBACd,IAAI,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC;iBAC9B,CAAC,CAAC;YACP,CAAC;YAED,SAAS,CAAC,GAAG,EAAE,CAAC;YAEhB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;YAC9F,IAAI,gBAA2D,CAAC;YAChE,MAAM,mBAAmB,GAAG,GAAG,EAAE;gBAC7B,IAAI,gBAAgB,EAAE,CAAC;oBACnB,OAAO;gBACX,CAAC;gBACD,gBAAgB,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;oBACrC,IAAI,CAAC,YAAY,EAAE,CAAC;wBAChB,OAAO;oBACX,CAAC;oBACD,MAAM,wBAAwB,CAAC;oBAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;oBAE/F,UAAU,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;oBAChC,WAAW,GAAG,IAAI,CAAC;gBACvB,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzE,CAAC,CAAC;YAEF,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,EAAE,EAAE,CAAC;gBAC9B,YAAY,GAAG,QAAQ,CAAC;gBACxB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,WAAW,EAAE,CAAC;oBAC3C,MAAM;gBACV,CAAC;gBACD,IAAI,gBAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjD,mBAAmB,EAAE,CAAC;gBAC1B,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;oBAChD,SAAS;gBACb,CAAC;gBACD,IAAI,gBAAgB,EAAE,CAAC;oBACnB,YAAY,CAAC,gBAAgB,CAAC,CAAC;gBACnC,CAAC;gBACD,MAAM,wBAAwB,CAAC;gBAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;gBAE3F,OAAO,QAAQ,CAAC;YACpB,CAAC;YAED,IAAI,gBAAgB,EAAE,CAAC;gBACnB,YAAY,CAAC,gBAAgB,CAAC,CAAC;YACnC,CAAC;YAED,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,MAAM,IAAI,+BAAqB,CAAC,kCAAkC,WAAW,EAAE,CAAC,CAAC;YACrF,CAAC;YAED,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,MAAM,wBAAwB,CAAC;gBAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;YACnG,CAAC;YAED,OAAO,YAAY,CAAC;QACxB,CAAC;QAED,IAAI,gBAA2D,CAAC;QAChE,MAAM,mBAAmB,GAAG,GAAG,EAAE;YAC7B,IAAI,gBAAgB,EAAE,CAAC;gBACnB,OAAO;YACX,CAAC;YACD,gBAAgB,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;gBACrC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAChB,OAAO;gBACX,CAAC;gBACD,MAAM,wBAAwB,CAAC;gBAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;gBAE/F,UAAU,CAAC,MAAM,CAAC,IAAA,oCAA0B,EAAC,GAAG,YAAY,EAAE,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAChH,WAAW,GAAG,IAAI,CAAC;YACvB,CAAC,EAAE,CAAC,CAAC,cAAc,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzE,CAAC,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,EAAE,EAAE,CAAC;YAC9B,YAAY,GAAG,QAAQ,CAAC;YACxB,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,WAAW,EAAE,CAAC;gBAC3C,MAAM;YACV,CAAC;YACD,IAAI,gBAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,mBAAmB,EAAE,CAAC;YAC1B,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC;gBAChD,SAAS;YACb,CAAC;YAED,IAAI,gBAAgB,EAAE,CAAC;gBACnB,YAAY,CAAC,gBAAgB,CAAC,CAAC;YACnC,CAAC;YACD,MAAM,wBAAwB,CAAC;YAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;YAE3F,OAAO,IAAA,oCAA0B,EAAC,GAAG,QAAQ,EAAE,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACpG,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACnB,YAAY,CAAC,gBAAgB,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,MAAM,IAAI,+BAAqB,CAAC,kCAAkC,WAAW,EAAE,CAAC,CAAC;QACrF,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,MAAM,wBAAwB,CAAC;YAC/B,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,KAAK,EAAE,kBAAkB,EAAE,aAAa,CAAC,CAAC;QACnG,CAAC;QAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,YAAY,EAAE,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxG,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CACpB,MAA4F,EAC5F,cAAuB,KAAK,EAC5B,UAAmB,KAAK;QAExB,2BAA2B;QAC3B,MAAM,aAAa,GAAG,MAAM,CAAC,CAAC,CAAC;QAC/B,MAAM,WAAW,GAAG,sIAAsI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAE/K,mCAAmC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACzE,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAClD,CAAC;QAED,IAAI,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC5C,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAElG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,aAAa,4BAA4B,CAAC,CAAC;QAE/E,IAAI,KAAK,GAAa,EAAE,CAAC;QACzB,mBAAmB;QACnB,MAAM,CAAC,GAAG,CAAC,CAAC;QAEZ,OAAO,QAAQ,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;YAC1D,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;YAC/F,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACT,MAAM;YACV,CAAC;YACD,QAAQ,IAAI,CAAC,CAAC;YACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;YACpE,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;YAC/C,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;YACzF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;YACpE,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;YAC/C,QAAQ,IAAI,CAAC,CAAC;YACd,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;YAEzF,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;YACzD,CAAC;QACL,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,CAAC,kBAAkB,CACrB,IAAkF,EAClF,aAA+B,EAC/B,OAA+B,EAC/B,cAA+B,EAC/B,KAAc;QAEd,IAAI,CAAC,aAAa,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAI,CAAC,CAAC,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;QAClC,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC;YACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACjC,IAAI,CAAC,CAAC,EAAE,CAAC;oBACL,OAAO,EAAE,CAAC;gBACd,CAAC;gBACD,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC9B,CAAC;gBACD,OAAO,IAAI,CAAC,OAAO,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;oBAC/F,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAEtB,OAAO,CAAC,CAAC;gBACb,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iCAAiC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAEnG,OAAO,EAAE,CAAC;gBACd,CAAC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;YAEH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAoB,CAAC;YACjE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;gBACzC,IAAI,CAAC,EAAE,CAAC;oBACJ,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBACvC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;IAED,uBAAuB,CAAC,aAA8B,EAAE,KAAc;QAClE,MAAM,iBAAiB,GAAG,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC;QAC1D,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,gBAAC,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpG,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,CAAC;QAE1C,MAAM,CAAC,GAAG,iBAAiB,GAAG,cAAc,CAAC,MAAM,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QAED,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;QAE3F,MAAM,WAAW,GAAG,QAAQ,CAAC;QAE7B,WAAW,CAAC,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC;QAE9C,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,kBAAkB,CAAC,SAA0B,EAAE,GAAW,EAAE,MAAc,EAAE,aAAsB;QAC9F,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAE3D,IAAI,CAAC,UAAU,EAAE,CAAC;gBACd,SAAS;YACb,CAAC;YAED,aAAa,IAAI,UAAU,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC;QAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAEjD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,CAAC,CAAC,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YAClE,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAwB,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnE,IAAI,aAAa,EAAE,CAAC;YAChB,QAAQ,CAAC,QAAQ,GAAG,aAAa,CAAC;QACtC,CAAC;QAED,IAAA,oBAAU,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,aAAa,CAAC,aAA4B;QACtC,OAAO,aAAa,CAAC,KAAK;YACtB,aAAa,CAAC,OAAO;YACrB,aAAa,CAAC,aAAa;YAC3B,aAAa,CAAC,WAAW;YACzB,aAAa,CAAC,IAAI;YAClB,aAAa,CAAC,IAAI,CAAC;IAC3B,CAAC;IAED,sBAAsB,CAAC,OAAwB,EAAE,iBAAiB,GAAG,IAAI,CAAC,iBAAiB;QACvF,OAAO,gBAAC,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;IACjG,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC3B,MAAM,GAAG,GAAG,uDAAuD,MAAM,EAAE,CAAC;QAE5E,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;YACd,CAAC;YACD,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,OAAO,yBAAyB,MAAM,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,CAAC,aAAa,CAAC,UAAmB,EAAE,OAAgB;QAChD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,MAAM,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;YAC5D,MAAM,IAAI,CAAC,YAAY,CAAC;YAExB,OAAO;QACX,CAAC;QAED,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;YAC5D,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,MAAM,IAAI,CAAC,YAAY,CAAC;YAExB,OAAO;QACX,CAAC;QAED,MAAM,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;QAC5D,MAAM,IAAI,CAAC,YAAY,CAAC;QACxB,MAAM,IAAI,CAAC,YAAY,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAAkC,EAAE,KAA0B,EAAE,OAAiB;QAChG,MAAM,WAAW,GAAG,IAAA,sBAAe,EAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAChC,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC1C,IAAI,KAAK,CAAC;QACV,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,KAAK,GAAG,CAAC,MAAM,qBAAU,CAAC,kBAAkB,CACxC,qBAAU,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,WAAW,CAAC;iBACxD,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;iBAC5B,KAAK,CAAC,CAAC,CAAC,CAChB,CAAC,CAAC,CAAC,CAAC,CAAC;YACN,IAAI,KAAK,EAAE,CAAC;gBACR,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACnD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC3E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,sBAAsB,KAAK,CAAC,CAAC,yBAAyB,WAAW,KAAK,GAAG,QAAQ,EAAE;oBAC7I,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,OAAO,KAAK,CAAC,QAAe,CAAC;gBACjC,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACD,IAAI,CAAoB,CAAC;YACzB,IAAI,SAAS,CAAC;YACd,SAAS,EACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACD,QAAQ,OAAO,EAAE,CAAC;wBACd,KAAK,QAAQ,CAAC,CAAC,CAAC;4BACZ,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC7D,MAAM;wBACV,CAAC;wBACD,KAAK,MAAM,CAAC,CAAC,CAAC;4BACV,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC5D,MAAM;wBACV,CAAC;wBACD,KAAK,KAAK,CAAC;wBACX,OAAO,CAAC,CAAC,CAAC;4BACN,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;4BAC3D,MAAM;wBACV,CAAC;oBACL,CAAC;oBACD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;oBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,IAAI,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC5I,MAAM,SAAS,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,SAAS,GAAG,GAAG,CAAC;oBAChB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;oBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,OAAO,iBAAiB,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;gBACzH,CAAC;YACL,CAAC;YAED,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,qBAAU,CAAC,IAAI,CAAC;oBAC3B,KAAK;oBACL,WAAW;oBACX,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,OAAO;oBAClB,QAAQ,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBAChE,CAAC,CAAC;gBAEH,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,SAAS,EAAE,CAAC;gBACnB,MAAM,SAAS,CAAC;YACpB,CAAC;YAED,OAAO,CAAqB,CAAC;QACjC,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4FAA4F,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAE/I,OAAO,KAAK,CAAC,QAAe,CAAC;YACjC,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,iBAAiB,CAAC,KAAqB;QACnC,MAAM,gBAAgB,GAAG;YACrB,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW;SACzE,CAAC;QACF,MAAM,MAAM,GAAG;YACX,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,GAAG,EAAE,KAAK,CAAC,IAAI;YACf,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC;YAC1C,GAAG,gBAAC,CAAC,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC;SACrC,CAAC;QAEF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAqB;QAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE/D,IAAI,cAAc,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACrD,CAAC;CACJ,CAAA;AAjyBY,oCAAY;AAgFf;IArBL,IAAA,iBAAM,EAAC;QACJ,IAAI,EAAE,aAAa;QACnB,GAAG,EAAE;YACD,IAAI,EAAE;gBACF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;gBACvB,IAAI,EAAE,SAAS;aAClB;SACJ;QACD,IAAI,EAAE,CAAC,QAAQ,CAAC;QAChB,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,CAAC;KAChD,CAAC;IACD,IAAA,iBAAM,EAAC;QACJ,GAAG,EAAE;YACD,IAAI,EAAE;gBACF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;gBACvB,IAAI,EAAE,KAAK;aACd;SACJ;QACD,IAAI,EAAE,CAAC,QAAQ,CAAC;QAChB,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,EAAE,mBAAS,CAAC;KAC3D,CAAC;IAEG,WAAA,IAAA,qBAAU,GAAE,CAAA;IACZ,WAAA,IAAA,cAAG,GAAE,CAAA;IAIL,WAAA,IAAA,gBAAK,EAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAE9D,WAAA,IAAA,gBAAK,EAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IAE3E,WAAA,IAAA,gBAAK,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAA;IAE3E,WAAA,IAAA,gBAAK,EAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAE5D,WAAA,IAAA,gBAAK,EAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;IACxF,YAAA,IAAA,gBAAK,EAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAA;IACjF,YAAA,IAAA,gBAAK,EAAC,UAAU,CAAC,CAAA;IACjB,YAAA,IAAA,gBAAK,EAAC,MAAM,CAAC,CAAA;IACb,YAAA,IAAA,gBAAK,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;IACnD,YAAA,IAAA,gBAAK,EAAC,GAAG,CAAC,CAAA;;yDAlBe,uBAAa,oBAAb,uBAAa,oDAC3B,kBAAO,oBAAP,kBAAO,gCACb,4CAAqB;QACX,gCAAc,sBACL,gDAAgC,oBAAhC,gDAAgC;;0CA8Y5D;uBAneQ,YAAY;IADxB,IAAA,oBAAS,GAAE;qCAsBoB,qBAAY,sBACR,6BAAgB,oBAAhB,6BAAgB,gCACrB,iCAAiB;QACrB,qBAAW;QACD,sCAAiB,sBACtB,yBAAyB,oBAAzB,yBAAyB,oDAC3B,uBAAuB,oBAAvB,uBAAuB,gCACzB,kCAAuB;GA5BtC,YAAY,CAiyBxB;AAED,MAAM,SAAS,GAAG;IACd,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE;IAChC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;IAClC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE;IACnC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE;IACvC,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE;IAC5C,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,gBAAgB,EAAE;IACjD,EAAE,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE;IAC3C,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE;IAC7C,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE;IAC9B,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;CACvC,CAAC;AAEF,MAAM,iBAAiB,GAAG;IACtB,QAAQ,CAAsB,CAAU;QACpC,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,OAAO,CAAC,KAAK,WAAW,EAAE,CAAC;gBAC3B,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;oBAClB,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzC,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC;gBAClD,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,kBAAkB,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC;YAC9E,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC/C,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC/C,CAAC;YACD,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClC,kBAAkB,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YAC9E,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,gBAAgB,CAAC,CAAC;YACpF,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9C,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC7C,CAAC;YACL,CAAC;YACD,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,iBAAiB,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACpF,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACJ,CAAC"}

build/api/serp.js

@@ -0,0 +1,513 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a, _b, _c, _d, _e;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SerpHost = void 0;
+const tsyringe_1 = require("tsyringe");
+const civ_rpc_1 = require("civkit/civ-rpc");
+const lang_1 = require("civkit/lang");
+const lodash_1 = __importDefault(require("lodash"));
+const rate_limit_1 = require("../shared/services/rate-limit");
+const logger_1 = require("../services/logger");
+const async_context_1 = require("../services/async-context");
+const registry_1 = require("../services/registry");
+const transform_server_event_stream_1 = require("../lib/transform-server-event-stream");
+const jina_embeddings_auth_1 = require("../dto/jina-embeddings-auth");
+const errors_1 = require("../services/errors");
+const serper_search_1 = require("../shared/3rd-party/serper-search");
+const google_1 = require("../services/serp/google");
+const crawler_options_1 = require("../dto/crawler-options");
+const hash_1 = require("civkit/hash");
+const searched_1 = require("../db/searched");
+const serper_1 = require("../services/serp/serper");
+const lru_cache_1 = require("lru-cache");
+const api_roll_1 = require("../shared/db/api-roll");
+const internal_1 = require("../services/serp/internal");
+const WORLD_COUNTRY_CODES = Object.keys(serper_search_1.WORLD_COUNTRIES).map((x) => x.toLowerCase());
+const indexProto = {
+    toString: function () {
+        return (0, lodash_1.default)(this)
+            .toPairs()
+            .map(([k, v]) => k ? `[${lodash_1.default.upperFirst(lodash_1.default.lowerCase(k))}] ${v}` : '')
+            .value()
+            .join('\n') + '\n';
+    }
+};
+let SerpHost = class SerpHost extends civ_rpc_1.RPCHost {
+    async getIndex(ctx, auth) {
+        const indexObject = Object.create(indexProto);
+        Object.assign(indexObject, {
+            usage1: 'https://r.jina.ai/YOUR_URL',
+            usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',
+            usage3: `${ctx.origin}/?q=YOUR_SEARCH_QUERY`,
+            homepage: 'https://jina.ai/reader',
+        });
+        if (auth && auth.user) {
+            indexObject[''] = undefined;
+            indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`;
+            indexObject.balanceLeft = auth.user.wallet.total_balance;
+        }
+        else {
+            indexObject.note = 'Authentication is required to use this endpoint. Please provide a valid API key via Authorization header.';
+        }
+        return indexObject;
+    }
+    constructor(globalLogger, rateLimitControl, threadLocal, googleSerp, serperGoogle, serperBing, jinaSerp) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.rateLimitControl = rateLimitControl;
+        this.threadLocal = threadLocal;
+        this.googleSerp = googleSerp;
+        this.serperGoogle = serperGoogle;
+        this.serperBing = serperBing;
+        this.jinaSerp = jinaSerp;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.cacheRetentionMs = 1000 * 3600 * 24 * 7;
+        this.cacheValidMs = 1000 * 3600;
+        this.pageCacheToleranceMs = 1000 * 3600 * 24;
+        this.reasonableDelayMs = 15_000;
+        this.targetResultCount = 5;
+        this.highFreqKeyCache = new lru_cache_1.LRUCache({
+            max: 256,
+            ttl: 60 * 60 * 1000,
+            updateAgeOnGet: false,
+            updateAgeOnHas: false,
+        });
+        this.batchedCaches = [];
+        setInterval(() => {
+            const thisBatch = this.batchedCaches;
+            this.batchedCaches = [];
+            if (!thisBatch.length) {
+                return;
+            }
+            const batch = searched_1.SERPResult.DB.batch();
+            for (const x of thisBatch) {
+                batch.set(searched_1.SERPResult.COLLECTION.doc(), x.degradeForFireStore());
+            }
+            batch.commit()
+                .then(() => {
+                this.logger.debug(`Saved ${thisBatch.length} caches by batch`);
+            })
+                .catch((err) => {
+                this.logger.warn(`Failed to cache search result in batch`, { err });
+            });
+        }, 1000 * 10 + Math.round(1000 * Math.random())).unref();
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+    async search(rpcReflect, ctx, crawlerOptions, auth, variant, q, searchEngine, num, gl, _hl, location, page, fallback) {
+        const authToken = auth.bearerToken;
+        let highFreqKey;
+        if (authToken && this.highFreqKeyCache.has(authToken)) {
+            highFreqKey = this.highFreqKeyCache.get(authToken);
+            auth.user = highFreqKey.user;
+            auth.uid = highFreqKey.user?.user_id;
+        }
+        const uid = await auth.solveUID();
+        if (!q) {
+            if (ctx.path === '/') {
+                const indexObject = await this.getIndex(ctx, auth);
+                if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) {
+                    return indexObject;
+                }
+                return (0, civ_rpc_1.assignTransferProtocolMeta)(`${indexObject}`, { contentType: 'text/plain; charset=utf-8', envelope: null });
+            }
+            throw new civ_rpc_1.ParamValidationError({
+                path: 'q',
+                message: `Required but not provided`
+            });
+        }
+        // Return content by default
+        const user = await auth.assertUser();
+        if (!(user.wallet.total_balance > 0)) {
+            throw new errors_1.InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`);
+        }
+        if (highFreqKey?.blockedUntil) {
+            const now = new Date();
+            const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());
+            if (blockedTimeRemaining > 0) {
+                this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`);
+                // throw RateLimitTriggeredError.from({
+                //     message: `Per UID rate limit exceeded (async)`,
+                //     retryAfter: Math.ceil(blockedTimeRemaining / 1000),
+                // });
+            }
+        }
+        const PREMIUM_KEY_LIMIT = 400;
+        const rateLimitPolicy = auth.getRateLimits('SEARCH') || [
+            parseInt(user.metadata?.speed_level) >= 2 ?
+                rate_limit_1.RateLimitDesc.from({
+                    occurrence: PREMIUM_KEY_LIMIT,
+                    periodSeconds: 60
+                }) :
+                rate_limit_1.RateLimitDesc.from({
+                    occurrence: 40,
+                    periodSeconds: 60
+                })
+        ];
+        const apiRollPromise = this.rateLimitControl.simpleRPCUidBasedLimit(rpcReflect, uid, ['SEARCH'], ...rateLimitPolicy);
+        if (!highFreqKey) {
+            // Normal path
+            await apiRollPromise;
+            if (rateLimitPolicy.some((x) => {
+                const rpm = x.occurrence / (x.periodSeconds / 60);
+                if (rpm >= PREMIUM_KEY_LIMIT) {
+                    return true;
+                }
+                return false;
+            })) {
+                this.highFreqKeyCache.set(auth.bearerToken, {
+                    user,
+                });
+            }
+        }
+        else {
+            // High freq key path
+            apiRollPromise.then(
+            // Rate limit not triggered, make sure not blocking.
+            () => {
+                delete highFreqKey.blockedUntil;
+            }, 
+            // Rate limit triggered
+            (err) => {
+                if (!(err instanceof rate_limit_1.RateLimitTriggeredError)) {
+                    return;
+                }
+                const now = Date.now();
+                let tgtDate;
+                if (err.retryAfterDate) {
+                    tgtDate = err.retryAfterDate;
+                }
+                else if (err.retryAfter) {
+                    tgtDate = new Date(now + err.retryAfter * 1000);
+                }
+                if (tgtDate) {
+                    const dt = tgtDate.valueOf() - now;
+                    highFreqKey.blockedUntil = tgtDate;
+                    setTimeout(() => {
+                        if (highFreqKey.blockedUntil === tgtDate) {
+                            delete highFreqKey.blockedUntil;
+                        }
+                    }, dt).unref();
+                }
+            }).finally(async () => {
+                // Always asynchronously update user(wallet);
+                const user = await auth.getBrief().catch(() => undefined);
+                if (user) {
+                    highFreqKey.user = user;
+                }
+            });
+        }
+        let chargeAmount = 0;
+        rpcReflect.finally(async () => {
+            if (chargeAmount) {
+                auth.reportUsage(chargeAmount, `reader-search`).catch((err) => {
+                    this.logger.warn(`Unable to report usage for ${uid}`, { err: (0, lang_1.marshalErrorLike)(err) });
+                });
+                try {
+                    const apiRoll = await apiRollPromise;
+                    apiRoll.chargeAmount = chargeAmount;
+                }
+                catch (err) {
+                    await this.rateLimitControl.record({
+                        uid,
+                        tags: [rpcReflect.name.toUpperCase()],
+                        status: api_roll_1.API_CALL_STATUS.SUCCESS,
+                        chargeAmount,
+                    }).save().catch((err) => {
+                        this.logger.warn(`Failed to save rate limit record`, { err: (0, lang_1.marshalErrorLike)(err) });
+                    });
+                }
+            }
+        });
+        let chargeAmountScaler = 1;
+        if (searchEngine === 'bing') {
+            chargeAmountScaler = 3;
+        }
+        if (variant !== 'web') {
+            chargeAmountScaler = 5;
+        }
+        let realQuery = q;
+        let queryTerms = q.split(/\s+/g).filter((x) => !!x);
+        let results = await this.cachedSearch(variant, {
+            provider: searchEngine,
+            q,
+            num,
+            gl,
+            // hl,
+            location,
+            page,
+        }, crawlerOptions);
+        if (fallback && !results?.length && (!page || page === 1)) {
+            let tryTimes = 1;
+            const containsRTL = /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(q);
+            const lastResort = (containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2)).join(' ');
+            const n = 4;
+            let terms = [];
+            while (tryTimes < n) {
+                const delta = Math.ceil(queryTerms.length / n) * tryTimes;
+                terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta);
+                const query = terms.join(' ');
+                if (!query) {
+                    break;
+                }
+                if (realQuery === query) {
+                    continue;
+                }
+                tryTimes += 1;
+                realQuery = query;
+                this.logger.info(`Retrying search with fallback query: "${realQuery}"`);
+                results = await this.cachedSearch(variant, {
+                    provider: searchEngine,
+                    q: realQuery,
+                    num,
+                    gl,
+                    // hl,
+                    location,
+                }, crawlerOptions);
+                if (results?.length) {
+                    break;
+                }
+            }
+            if (!results?.length && realQuery.length > lastResort.length) {
+                realQuery = lastResort;
+                this.logger.info(`Retrying search with fallback query: "${realQuery}"`);
+                tryTimes += 1;
+                results = await this.cachedSearch(variant, {
+                    provider: searchEngine,
+                    q: realQuery,
+                    num,
+                    gl,
+                    // hl,
+                    location,
+                }, crawlerOptions);
+            }
+            chargeAmountScaler *= tryTimes;
+        }
+        if (!results?.length) {
+            results = [];
+        }
+        const finalResults = results.map((x) => this.mapToFinalResults(x));
+        await Promise.all(finalResults.map((x) => this.assignGeneralMixin(x)));
+        chargeAmount = this.assignChargeAmount(finalResults, chargeAmountScaler);
+        (0, civ_rpc_1.assignMeta)(finalResults, {
+            query: realQuery,
+            fallback: realQuery === q ? undefined : realQuery,
+        });
+        return finalResults;
+    }
+    assignChargeAmount(items, scaler) {
+        const numCharge = Math.ceil(items.length / 10) * 10000 * scaler;
+        (0, civ_rpc_1.assignMeta)(items, { usage: { tokens: numCharge } });
+        return numCharge;
+    }
+    async getFavicon(domain) {
+        const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                return '';
+            }
+            const ab = await response.arrayBuffer();
+            const buffer = Buffer.from(ab);
+            const base64 = buffer.toString('base64');
+            return `data:image/png;base64,${base64}`;
+        }
+        catch (error) {
+            this.logger.warn(`Failed to get favicon base64 string`, { err: (0, lang_1.marshalErrorLike)(error) });
+            return '';
+        }
+    }
+    async configure(opts) {
+        const crawlOpts = {
+            proxyUrl: opts.proxyUrl,
+            cookies: opts.setCookies,
+            overrideUserAgent: opts.userAgent,
+            timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined,
+            locale: opts.locale,
+            referer: opts.referer,
+            viewport: opts.viewport,
+            proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false,
+            allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy,
+        };
+        if (opts.locale) {
+            crawlOpts.extraHeaders ??= {};
+            crawlOpts.extraHeaders['Accept-Language'] = opts.locale;
+        }
+        return crawlOpts;
+    }
+    mapToFinalResults(input) {
+        const whitelistedProps = [
+            'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks'
+        ];
+        const result = {
+            title: input.title,
+            url: input.link,
+            description: Reflect.get(input, 'snippet'),
+            ...lodash_1.default.pick(input, whitelistedProps),
+        };
+        return result;
+    }
+    *iterProviders(preference, variant) {
+        if (preference === 'bing') {
+            yield this.serperBing;
+            yield this.serperGoogle;
+            yield this.googleSerp;
+            return;
+        }
+        if (preference === 'google') {
+            yield this.googleSerp;
+            yield this.googleSerp;
+            yield this.serperGoogle;
+            return;
+        }
+        // yield variant === 'web' ? this.jinaSerp : this.serperGoogle;
+        yield this.serperGoogle;
+        yield this.serperGoogle;
+        yield this.googleSerp;
+    }
+    async cachedSearch(variant, query, opts) {
+        const queryDigest = (0, hash_1.objHashMd5B64Of)({ ...query, variant });
+        const provider = query.provider;
+        Reflect.deleteProperty(query, 'provider');
+        const noCache = opts.noCache;
+        let cache;
+        if (!noCache) {
+            cache = (await searched_1.SERPResult.fromFirestoreQuery(searched_1.SERPResult.COLLECTION.where('queryDigest', '==', queryDigest)
+                .orderBy('createdAt', 'desc')
+                .limit(1)))[0];
+            if (cache) {
+                const age = Date.now() - cache.createdAt.valueOf();
+                const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs);
+                this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, {
+                    query, digest: queryDigest, age, stale
+                });
+                if (!stale) {
+                    return cache.response;
+                }
+            }
+        }
+        const scrappingOptions = await this.configure(opts);
+        try {
+            let r;
+            let lastError;
+            outerLoop: for (const client of this.iterProviders(provider, variant)) {
+                const t0 = Date.now();
+                try {
+                    switch (variant) {
+                        case 'images': {
+                            r = await Reflect.apply(client.imageSearch, client, [query, scrappingOptions]);
+                            break;
+                        }
+                        case 'news': {
+                            r = await Reflect.apply(client.newsSearch, client, [query, scrappingOptions]);
+                            break;
+                        }
+                        case 'web':
+                        default: {
+                            r = await Reflect.apply(client.webSearch, client, [query, scrappingOptions]);
+                            break;
+                        }
+                    }
+                    const dt = Date.now() - t0;
+                    this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name });
+                    break outerLoop;
+                }
+                catch (err) {
+                    lastError = err;
+                    const dt = Date.now() - t0;
+                    this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, });
+                }
+            }
+            if (r?.length) {
+                const nowDate = new Date();
+                const record = searched_1.SERPResult.from({
+                    query,
+                    queryDigest,
+                    response: r,
+                    createdAt: nowDate,
+                    expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs)
+                });
+                this.batchedCaches.push(record);
+            }
+            else if (lastError) {
+                throw lastError;
+            }
+            return r;
+        }
+        catch (err) {
+            if (cache) {
+                this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: (0, lang_1.marshalErrorLike)(err) });
+                return cache.response;
+            }
+            throw err;
+        }
+    }
+    async assignGeneralMixin(result) {
+        const collectFavicon = this.threadLocal.get('collect-favicon');
+        if (collectFavicon && result.link) {
+            const url = new URL(result.link);
+            Reflect.set(result, 'favicon', await this.getFavicon(url.origin));
+        }
+    }
+};
+exports.SerpHost = SerpHost;
+__decorate([
+    (0, registry_1.Method)({
+        name: 'searchIndex',
+        ext: {
+            http: {
+                action: ['get', 'post'],
+                path: '/'
+            }
+        },
+        tags: ['search'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream, civ_rpc_1.RawString],
+    }),
+    (0, registry_1.Method)({
+        ext: {
+            http: {
+                action: ['get', 'post'],
+            }
+        },
+        tags: ['search'],
+        returnType: [String, transform_server_event_stream_1.OutputServerEventStream, civ_rpc_1.RawString],
+    }),
+    __param(0, (0, registry_1.RPCReflect)()),
+    __param(1, (0, registry_1.Ctx)()),
+    __param(4, (0, registry_1.Param)('type', { type: new Set(['web', 'images', 'news']), default: 'web' })),
+    __param(5, (0, registry_1.Param)('q')),
+    __param(6, (0, registry_1.Param)('provider', { type: new Set(['google', 'bing']) })),
+    __param(7, (0, registry_1.Param)('num', { validate: (v) => v >= 0 && v <= 20 })),
+    __param(8, (0, registry_1.Param)('gl', { validate: (v) => WORLD_COUNTRY_CODES.includes(v?.toLowerCase()) })),
+    __param(9, (0, registry_1.Param)('hl', { validate: (v) => serper_search_1.WORLD_LANGUAGES.some(l => l.code === v) })),
+    __param(10, (0, registry_1.Param)('location')),
+    __param(11, (0, registry_1.Param)('page')),
+    __param(12, (0, registry_1.Param)('fallback')),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_d = typeof civ_rpc_1.RPCReflection !== "undefined" && civ_rpc_1.RPCReflection) === "function" ? _d : Object, typeof (_e = typeof registry_1.Context !== "undefined" && registry_1.Context) === "function" ? _e : Object, crawler_options_1.CrawlerOptions,
+        jina_embeddings_auth_1.JinaEmbeddingsAuthDTO, String, String, String, Number, String, String, String, Number, Boolean]),
+    __metadata("design:returntype", Promise)
+], SerpHost.prototype, "search", null);
+exports.SerpHost = SerpHost = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger, typeof (_a = typeof rate_limit_1.RateLimitControl !== "undefined" && rate_limit_1.RateLimitControl) === "function" ? _a : Object, async_context_1.AsyncLocalContext,
+        google_1.GoogleSERP, typeof (_b = typeof serper_1.SerperGoogleSearchService !== "undefined" && serper_1.SerperGoogleSearchService) === "function" ? _b : Object, typeof (_c = typeof serper_1.SerperBingSearchService !== "undefined" && serper_1.SerperBingSearchService) === "function" ? _c : Object, internal_1.InternalJinaSerpService])
+], SerpHost);
+//# sourceMappingURL=serp.js.map

build/api/serp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serp.js","sourceRoot":"","sources":["../../src/api/serp.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,uCAAqC;AACrC,4CAIwB;AACxB,sCAA+C;AAC/C,oDAAuB;AAEvB,8DAAyG;AAEzG,+CAAkD;AAClD,6DAA8D;AAC9D,mDAA+E;AAC/E,wFAA+E;AAC/E,sEAAoE;AACpE,+CAA8D;AAC9D,qEAAqF;AACrF,oDAAqD;AAErD,4DAAwD;AAExD,sCAA8C;AAC9C,6CAA4C;AAC5C,oDAA6F;AAE7F,yCAAqC;AACrC,oDAAwD;AACxD,wDAAoE;AAEpE,MAAM,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,+BAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAOrF,MAAM,UAAU,GAAG;IACf,QAAQ,EAAE;QACN,OAAO,IAAA,gBAAC,EAAC,IAAI,CAAC;aACT,OAAO,EAAE;aACT,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,gBAAC,CAAC,UAAU,CAAC,gBAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAClE,KAAK,EAAE;aACP,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3B,CAAC;CACJ,CAAC;AAGK,IAAM,QAAQ,GAAd,MAAM,QAAS,SAAQ,iBAAO;IAoBjC,KAAK,CAAC,QAAQ,CAAC,GAAY,EAAE,IAA4B;QACrD,MAAM,WAAW,GAAgD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3F,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE;YACvB,MAAM,EAAE,4BAA4B;YACpC,MAAM,EAAE,qCAAqC;YAC7C,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,uBAAuB;YAC5C,QAAQ,EAAE,wBAAwB;SACrC,CAAC,CAAC;QAEH,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACpB,WAAW,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC;YAC5B,WAAW,CAAC,eAAe,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC;YAC9E,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;QAC7D,CAAC;aAAM,CAAC;YACJ,WAAW,CAAC,IAAI,GAAG,2GAA2G,CAAC;QACnI,CAAC;QAED,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,YACc,YAA0B,EAC1B,gBAAkC,EAClC,WAA8B,EAC9B,UAAsB,EACtB,YAAuC,EACvC,UAAmC,EACnC,QAAiC;QAE3C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QARV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,gBAAW,GAAX,WAAW,CAAmB;QAC9B,eAAU,GAAV,UAAU,CAAY;QACtB,iBAAY,GAAZ,YAAY,CAA2B;QACvC,eAAU,GAAV,UAAU,CAAyB;QACnC,aAAQ,GAAR,QAAQ,CAAyB;QA9C/C,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,qBAAgB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;QACxC,iBAAY,GAAG,IAAI,GAAG,IAAI,CAAC;QAC3B,yBAAoB,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QAExC,sBAAiB,GAAG,MAAM,CAAC;QAE3B,sBAAiB,GAAG,CAAC,CAAC;QAEtB,qBAAgB,GAAG,IAAI,oBAAQ,CAAyB;YACpD,GAAG,EAAE,GAAG;YACR,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI;YACnB,cAAc,EAAE,KAAK;YACrB,cAAc,EAAE,KAAK;SACxB,CAAC,CAAC;QAEH,kBAAa,GAAiB,EAAE,CAAC;QAiC7B,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC;YACrC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;YACxB,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO;YACX,CAAC;YACD,MAAM,KAAK,GAAG,qBAAU,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAEpC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,KAAK,CAAC,GAAG,CAAC,qBAAU,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,KAAK,CAAC,MAAM,EAAE;iBACT,IAAI,CAAC,GAAG,EAAE;gBACP,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,SAAS,CAAC,MAAM,kBAAkB,CAAC,CAAC;YACnE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACxE,CAAC,CAAC,CAAC;QACX,CAAC,EAAE,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;IAC7D,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAsBK,AAAN,KAAK,CAAC,MAAM,CACM,UAAyB,EAChC,GAAY,EACnB,cAA8B,EAC9B,IAA2B,EAE3B,OAAkC,EACtB,CAAU,EAEtB,YAAgC,EAEhC,GAAY,EAC8E,EAAW,EAClB,GAAY,EAC5E,QAAiB,EACrB,IAAa,EACT,QAAkB;QAErC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC;QACnC,IAAI,WAAuC,CAAC;QAC5C,IAAI,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;YACpD,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;YAC7B,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC;QACzC,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,EAAE,CAAC;YACL,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC;gBACnB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBACnD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;oBAC9F,OAAO,WAAW,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAA,oCAA0B,EAAC,GAAG,WAAW,EAAE,EAC9C,EAAE,WAAW,EAAE,2BAA2B,EAAE,QAAQ,EAAE,IAAI,EAAE,CAC/D,CAAC;YACN,CAAC;YACD,MAAM,IAAI,8BAAoB,CAAC;gBAC3B,IAAI,EAAE,GAAG;gBACT,OAAO,EAAE,2BAA2B;aACvC,CAAC,CAAC;QACP,CAAC;QACD,4BAA4B;QAC5B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,iCAAwB,CAAC,gEAAgE,CAAC,CAAC;QACzG,CAAC;QAED,IAAI,WAAW,EAAE,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,oBAAoB,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YAClF,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,yCAAyC,CAAC,CAAC;gBAC3F,uCAAuC;gBACvC,sDAAsD;gBACtD,0DAA0D;gBAC1D,MAAM;YACV,CAAC;QACL,CAAC;QAED,MAAM,iBAAiB,GAAG,GAAG,CAAC;QAC9B,MAAM,eAAe,GAAG,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI;YACpD,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;gBACvC,0BAAa,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,iBAAiB;oBAC7B,aAAa,EAAE,EAAE;iBACpB,CAAC,CAAC,CAAC;gBACJ,0BAAa,CAAC,IAAI,CAAC;oBACf,UAAU,EAAE,EAAE;oBACd,aAAa,EAAE,EAAE;iBACpB,CAAC;SACT,CAAC;QAEF,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,sBAAsB,CAC/D,UAAU,EAAE,GAAI,EAAE,CAAC,QAAQ,CAAC,EAC5B,GAAG,eAAe,CACrB,CAAC;QAEF,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,cAAc;YACd,MAAM,cAAc,CAAC;YAErB,IAAI,eAAe,CAAC,IAAI,CACpB,CAAC,CAAC,EAAE,EAAE;gBACF,MAAM,GAAG,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;gBAClD,IAAI,GAAG,IAAI,iBAAiB,EAAE,CAAC;oBAC3B,OAAO,IAAI,CAAC;gBAChB,CAAC;gBAED,OAAO,KAAK,CAAC;YACjB,CAAC,CAAC,EACJ,CAAC;gBACC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAY,EAAE;oBACzC,IAAI;iBACP,CAAC,CAAC;YACP,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,qBAAqB;YACrB,cAAc,CAAC,IAAI;YACf,oDAAoD;YACpD,GAAG,EAAE;gBACD,OAAO,WAAW,CAAC,YAAY,CAAC;YACpC,CAAC;YACD,uBAAuB;YACvB,CAAC,GAAG,EAAE,EAAE;gBACJ,IAAI,CAAC,CAAC,GAAG,YAAY,oCAAuB,CAAC,EAAE,CAAC;oBAC5C,OAAO;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAI,OAAO,CAAC;gBACZ,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC,cAAc,CAAC;gBACjC,CAAC;qBAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBACxB,OAAO,GAAG,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;gBACpD,CAAC;gBAED,IAAI,OAAO,EAAE,CAAC;oBACV,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC;oBACnC,WAAW,CAAC,YAAY,GAAG,OAAO,CAAC;oBACnC,UAAU,CAAC,GAAG,EAAE;wBACZ,IAAI,WAAW,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;4BACvC,OAAO,WAAW,CAAC,YAAY,CAAC;wBACpC,CAAC;oBACL,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnB,CAAC;YACL,CAAC,CACJ,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;gBACjB,6CAA6C;gBAC7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC1D,IAAI,IAAI,EAAE,CAAC;oBACP,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC;gBAC5B,CAAC;YACL,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,UAAU,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE;YAC1B,IAAI,YAAY,EAAE,CAAC;gBACf,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;oBAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC1F,CAAC,CAAC,CAAC;gBACH,IAAI,CAAC;oBACD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;oBACrC,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;gBACxC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;wBAC/B,GAAG;wBACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;wBACrC,MAAM,EAAE,0BAAe,CAAC,OAAO;wBAC/B,YAAY;qBACf,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBACzF,CAAC,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;YAC1B,kBAAkB,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;YACpB,kBAAkB,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEpD,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;YAC3C,QAAQ,EAAE,YAAY;YACtB,CAAC;YACD,GAAG;YACH,EAAE;YACF,MAAM;YACN,QAAQ;YACR,IAAI;SACP,EAAE,cAAc,CAAC,CAAC;QAGnB,IAAI,QAAQ,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;YACxD,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,MAAM,WAAW,GAAG,sIAAsI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnK,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9G,MAAM,CAAC,GAAG,CAAC,CAAC;YACZ,IAAI,KAAK,GAAa,EAAE,CAAC;YACzB,OAAO,QAAQ,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;gBAC1D,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;gBAC/F,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,MAAM;gBACV,CAAC;gBACD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;oBACtB,SAAS;gBACb,CAAC;gBACD,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,SAAS,GAAG,CAAC,CAAC;gBACxE,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBACvC,QAAQ,EAAE,YAAY;oBACtB,CAAC,EAAE,SAAS;oBACZ,GAAG;oBACH,EAAE;oBACF,MAAM;oBACN,QAAQ;iBACX,EAAE,cAAc,CAAC,CAAC;gBACnB,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;oBAClB,MAAM;gBACV,CAAC;YACL,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAC3D,SAAS,GAAG,UAAU,CAAC;gBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yCAAyC,SAAS,GAAG,CAAC,CAAC;gBACxE,QAAQ,IAAI,CAAC,CAAC;gBACd,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE;oBACvC,QAAQ,EAAE,YAAY;oBACtB,CAAC,EAAE,SAAS;oBACZ,GAAG;oBACH,EAAE;oBACF,MAAM;oBACN,QAAQ;iBACX,EAAE,cAAc,CAAC,CAAC;YACvB,CAAC;YAED,kBAAkB,IAAI,QAAQ,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;YACnB,OAAO,GAAG,EAAE,CAAC;QACjB,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC;QAExE,MAAM,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5E,YAAY,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,kBAAkB,CAAC,CAAC;QACzE,IAAA,oBAAU,EAAC,YAAY,EAAE;YACrB,KAAK,EAAE,SAAS;YAChB,QAAQ,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACpD,CAAC,CAAC;QAEH,OAAO,YAAY,CAAC;IACxB,CAAC;IAGD,kBAAkB,CAAC,KAAgB,EAAE,MAAc;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC;QAChE,IAAA,oBAAU,EAAC,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAEpD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAc;QAC3B,MAAM,GAAG,GAAG,uDAAuD,MAAM,EAAE,CAAC;QAE5E,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;YACd,CAAC;YACD,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,OAAO,yBAAyB,MAAM,EAAE,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAoB;QAChC,MAAM,SAAS,GAAqB;YAChC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,UAAU;YACxB,iBAAiB,EAAE,IAAI,CAAC,SAAS;YACjC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS;YACzD,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,cAAc,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;YAC3E,UAAU,EAAE,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK;SAC/E,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,SAAS,CAAC,YAAY,KAAK,EAAE,CAAC;YAC9B,SAAS,CAAC,YAAY,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5D,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,iBAAiB,CAAC,KAAqB;QACnC,MAAM,gBAAgB,GAAG;YACrB,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW;SACzE,CAAC;QACF,MAAM,MAAM,GAAG;YACX,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,GAAG,EAAE,KAAK,CAAC,IAAI;YACf,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC;YAC1C,GAAG,gBAAC,CAAC,IAAI,CAAC,KAAK,EAAE,gBAAgB,CAAC;SACrC,CAAC;QAEF,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,CAAC,aAAa,CAAC,UAAmB,EAAE,OAAgB;QAChD,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,MAAM,IAAI,CAAC,UAAU,CAAC;YAEtB,OAAO;QACX,CAAC;QAED,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,CAAC;YACtB,MAAM,IAAI,CAAC,YAAY,CAAC;YAExB,OAAO;QACX,CAAC;QAED,+DAA+D;QAC/D,MAAM,IAAI,CAAC,YAAY,CAAA;QACvB,MAAM,IAAI,CAAC,YAAY,CAAC;QACxB,MAAM,IAAI,CAAC,UAAU,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAAkC,EAAE,KAA0B,EAAE,IAAoB;QACnG,MAAM,WAAW,GAAG,IAAA,sBAAe,EAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAChC,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,KAAK,CAAC;QACV,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,KAAK,GAAG,CAAC,MAAM,qBAAU,CAAC,kBAAkB,CACxC,qBAAU,CAAC,UAAU,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,EAAE,WAAW,CAAC;iBACxD,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;iBAC5B,KAAK,CAAC,CAAC,CAAC,CAChB,CAAC,CAAC,CAAC,CAAC,CAAC;YACN,IAAI,KAAK,EAAE,CAAC;gBACR,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;gBACnD,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC3E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,WAAW,sBAAsB,KAAK,CAAC,CAAC,yBAAyB,WAAW,KAAK,GAAG,QAAQ,EAAE;oBAC7I,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,KAAK,EAAE,CAAC;oBACT,OAAO,KAAK,CAAC,QAAe,CAAC;gBACjC,CAAC;YACL,CAAC;QACL,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,CAAC;YACD,IAAI,CAAoB,CAAC;YACzB,IAAI,SAAS,CAAC;YACd,SAAS,EACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;gBACzD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,IAAI,CAAC;oBACD,QAAQ,OAAO,EAAE,CAAC;wBACd,KAAK,QAAQ,CAAC,CAAC,CAAC;4BACZ,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC;4BAC/E,MAAM;wBACV,CAAC;wBACD,KAAK,MAAM,CAAC,CAAC,CAAC;4BACV,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC;4BAC9E,MAAM;wBACV,CAAC;wBACD,KAAK,KAAK,CAAC;wBACX,OAAO,CAAC,CAAC,CAAC;4BACN,CAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,KAAK,EAAE,gBAAgB,CAAC,CAAC,CAAC;4BAC7E,MAAM;wBACV,CAAC;oBACL,CAAC;oBACD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;oBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,IAAI,IAAI,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;oBAC5I,MAAM,SAAS,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,SAAS,GAAG,GAAG,CAAC;oBAChB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;oBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,OAAO,iBAAiB,MAAM,CAAC,WAAW,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;gBACzH,CAAC;YACL,CAAC;YAED,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;gBACZ,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,qBAAU,CAAC,IAAI,CAAC;oBAC3B,KAAK;oBACL,WAAW;oBACX,QAAQ,EAAE,CAAC;oBACX,SAAS,EAAE,OAAO;oBAClB,QAAQ,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC;iBAChE,CAAC,CAAC;gBACH,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,SAAS,EAAE,CAAC;gBACnB,MAAM,SAAS,CAAC;YACpB,CAAC;YAED,OAAO,CAAC,CAAC;QACb,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,KAAK,EAAE,CAAC;gBACR,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4FAA4F,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAE/I,OAAO,KAAK,CAAC,QAAe,CAAC;YACjC,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAA+B;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAE/D,IAAI,cAAc,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;CACJ,CAAA;AAxgBY,4BAAQ;AAkGX;IApBL,IAAA,iBAAM,EAAC;QACJ,IAAI,EAAE,aAAa;QACnB,GAAG,EAAE;YACD,IAAI,EAAE;gBACF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;gBACvB,IAAI,EAAE,GAAG;aACZ;SACJ;QACD,IAAI,EAAE,CAAC,QAAQ,CAAC;QAChB,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,EAAE,mBAAS,CAAC;KAC3D,CAAC;IACD,IAAA,iBAAM,EAAC;QACJ,GAAG,EAAE;YACD,IAAI,EAAE;gBACF,MAAM,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;aAC1B;SACJ;QACD,IAAI,EAAE,CAAC,QAAQ,CAAC;QAChB,UAAU,EAAE,CAAC,MAAM,EAAE,uDAAuB,EAAE,mBAAS,CAAC;KAC3D,CAAC;IAEG,WAAA,IAAA,qBAAU,GAAE,CAAA;IACZ,WAAA,IAAA,cAAG,GAAE,CAAA;IAGL,WAAA,IAAA,gBAAK,EAAC,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAA;IAE3E,WAAA,IAAA,gBAAK,EAAC,GAAG,CAAC,CAAA;IACV,WAAA,IAAA,gBAAK,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAA;IAExD,WAAA,IAAA,gBAAK,EAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAE5D,WAAA,IAAA,gBAAK,EAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAA;IACxF,WAAA,IAAA,gBAAK,EAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,+BAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAA;IACjF,YAAA,IAAA,gBAAK,EAAC,UAAU,CAAC,CAAA;IACjB,YAAA,IAAA,gBAAK,EAAC,MAAM,CAAC,CAAA;IACb,YAAA,IAAA,gBAAK,EAAC,UAAU,CAAC,CAAA;;yDAfQ,uBAAa,oBAAb,uBAAa,oDAC3B,kBAAO,oBAAP,kBAAO,gCACH,gCAAc;QACxB,4CAAqB;;sCAgP9B;mBAtVQ,QAAQ;IADpB,IAAA,oBAAS,GAAE;qCA0CoB,qBAAY,sBACR,6BAAgB,oBAAhB,6BAAgB,gCACrB,iCAAiB;QAClB,mBAAU,sBACR,kCAAyB,oBAAzB,kCAAyB,oDAC3B,gCAAuB,oBAAvB,gCAAuB,gCACzB,kCAAuB;GA/CtC,QAAQ,CAwgBpB"}

build/cloud-functions/adaptive-crawler.js

@@ -0,0 +1,510 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var AdaptiveCrawlerHost_1;
+var _a, _b, _c, _d, _e, _f;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AdaptiveCrawlerHost = void 0;
+const civkit_1 = require("civkit");
+const tsyringe_1 = require("tsyringe");
+const shared_1 = require("../shared");
+const jina_embeddings_auth_1 = require("../shared/dto/jina-embeddings-auth");
+const robots_parser_1 = __importDefault(require("robots-parser"));
+const xmldom_1 = require("@xmldom/xmldom");
+const adaptive_crawler_options_1 = require("../dto/adaptive-crawler-options");
+const crawler_options_1 = require("../dto/crawler-options");
+const adaptive_crawl_task_1 = require("../db/adaptive-crawl-task");
+const functions_1 = require("firebase-admin/functions");
+const get_function_url_1 = require("../utils/get-function-url");
+const md5Hasher = new civkit_1.HashManager('md5', 'hex');
+const removeURLHash = (url) => {
+    try {
+        const o = new URL(url);
+        o.hash = '';
+        return o.toString();
+    }
+    catch (e) {
+        return url;
+    }
+};
+let AdaptiveCrawlerHost = class AdaptiveCrawlerHost extends civkit_1.RPCHost {
+    static { AdaptiveCrawlerHost_1 = this; }
+    static { this.__singleCrawlQueueName = 'singleCrawlQueue'; }
+    constructor(globalLogger, firebaseObjectStorage) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.firebaseObjectStorage = firebaseObjectStorage;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        // Actual cache storage (gcp buckets) exists for 7 days, so here we need to select a time < 7 days.
+        this.cacheExpiry = 3 * 1000 * 60 * 60 * 24;
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+    async adaptiveCrawl(rpcReflect, ctx, auth, crawlerOptions, adaptiveCrawlerOptions) {
+        this.logger.debug({
+            adaptiveCrawlerOptions,
+            crawlerOptions,
+        });
+        const uid = await auth.solveUID();
+        const { useSitemap, maxPages } = adaptiveCrawlerOptions;
+        let tmpUrl = ctx.req.url.slice(1)?.trim();
+        if (!tmpUrl) {
+            tmpUrl = crawlerOptions.url?.trim() ?? '';
+        }
+        const targetUrl = new URL(tmpUrl);
+        if (!targetUrl) {
+            const latestUser = uid ? await auth.assertUser() : undefined;
+            if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
+                return this.getIndex(latestUser);
+            }
+            return (0, civkit_1.assignTransferProtocolMeta)(`${this.getIndex(latestUser)}`, { contentType: 'text/plain', envelope: null });
+        }
+        const meta = {
+            targetUrl: targetUrl.toString(),
+            useSitemap,
+            maxPages,
+        };
+        const digest = md5Hasher.hash(JSON.stringify(meta));
+        const shortDigest = Buffer.from(digest, 'hex').toString('base64url');
+        const existing = await adaptive_crawl_task_1.AdaptiveCrawlTask.fromFirestore(shortDigest);
+        if (existing?.createdAt) {
+            if (existing.createdAt.getTime() > Date.now() - this.cacheExpiry) {
+                this.logger.info(`Cache hit for ${shortDigest}, created at ${existing.createdAt.toDateString()}`);
+                return { taskId: shortDigest };
+            }
+            else {
+                this.logger.info(`Cache expired for ${shortDigest}, created at ${existing.createdAt.toDateString()}`);
+            }
+        }
+        await adaptive_crawl_task_1.AdaptiveCrawlTask.COLLECTION.doc(shortDigest).set({
+            _id: shortDigest,
+            status: adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.PENDING,
+            statusText: 'Pending',
+            meta,
+            createdAt: new Date(),
+            urls: [],
+            processed: {},
+            failed: {},
+        });
+        let urls = [];
+        if (useSitemap) {
+            urls = await this.crawlUrlsFromSitemap(targetUrl, maxPages);
+        }
+        if (urls.length > 0) {
+            await adaptive_crawl_task_1.AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
+                status: adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.PROCESSING,
+                statusText: `Processing 0/${urls.length}`,
+                urls,
+            });
+            const promises = [];
+            for (const url of urls) {
+                promises.push((0, functions_1.getFunctions)().taskQueue(AdaptiveCrawlerHost_1.__singleCrawlQueueName).enqueue({
+                    shortDigest, url, token: auth.bearerToken, meta
+                }, {
+                    dispatchDeadlineSeconds: 1800,
+                    uri: await (0, get_function_url_1.getFunctionUrl)(AdaptiveCrawlerHost_1.__singleCrawlQueueName),
+                }));
+            }
+            ;
+            await Promise.all(promises);
+        }
+        else {
+            meta.useSitemap = false;
+            await adaptive_crawl_task_1.AdaptiveCrawlTask.COLLECTION.doc(shortDigest).update({
+                urls: [targetUrl.toString()],
+            });
+            await (0, functions_1.getFunctions)().taskQueue(AdaptiveCrawlerHost_1.__singleCrawlQueueName).enqueue({
+                shortDigest, url: targetUrl.toString(), token: auth.bearerToken, meta
+            }, {
+                dispatchDeadlineSeconds: 1800,
+                uri: await (0, get_function_url_1.getFunctionUrl)(AdaptiveCrawlerHost_1.__singleCrawlQueueName),
+            });
+        }
+        return { taskId: shortDigest };
+    }
+    async adaptiveCrawlStatus(rpcReflect, ctx, auth, taskId, urls = []) {
+        if (!taskId) {
+            throw new civkit_1.ParamValidationError('taskId is required');
+        }
+        const state = await adaptive_crawl_task_1.AdaptiveCrawlTask.fromFirestore(taskId);
+        if (!state) {
+            throw new civkit_1.AssertionFailureError('The task does not exist');
+        }
+        if (state?.createdAt && state.createdAt.getTime() < Date.now() - this.cacheExpiry) {
+            throw new civkit_1.AssertionFailureError('The task has expired');
+        }
+        if (urls.length) {
+            const promises = Object.entries(state?.processed ?? {}).map(async ([url, cachePath]) => {
+                if (urls.includes(url)) {
+                    const raw = await this.firebaseObjectStorage.downloadFile(cachePath);
+                    state.processed[url] = JSON.parse(raw.toString('utf-8'));
+                }
+            });
+            await Promise.all(promises);
+        }
+        return state;
+    }
+    async singleCrawlQueue(shortDigest, url, token, meta) {
+        const error = {
+            reason: ''
+        };
+        const state = await adaptive_crawl_task_1.AdaptiveCrawlTask.fromFirestore(shortDigest);
+        if (state?.status === adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.COMPLETED) {
+            return;
+        }
+        try {
+            url = removeURLHash(url);
+        }
+        catch (e) {
+            error.reason = `Failed to parse url: ${url}`;
+        }
+        this.logger.debug(shortDigest, url, meta);
+        const cachePath = `adaptive-crawl-task/${shortDigest}/${md5Hasher.hash(url)}`;
+        if (!error.reason) {
+            const result = meta.useSitemap
+                ? await this.handleSingleCrawl(shortDigest, url, token, cachePath)
+                : await this.handleSingleCrawlRecursively(shortDigest, url, token, meta, cachePath);
+            if (!result) {
+                return;
+            }
+            error.reason = result.error.reason;
+        }
+        await adaptive_crawl_task_1.AdaptiveCrawlTask.DB.runTransaction(async (transaction) => {
+            const ref = adaptive_crawl_task_1.AdaptiveCrawlTask.COLLECTION.doc(shortDigest);
+            const state = await transaction.get(ref);
+            const data = state.data();
+            if (error.reason) {
+                data.failed[url] = error;
+            }
+            else {
+                data.processed[url] = cachePath;
+            }
+            const status = Object.keys(data.processed).length + Object.keys(data.failed).length >= data.urls.length
+                ? adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.COMPLETED : adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.PROCESSING;
+            const statusText = Object.keys(data.processed).length + Object.keys(data.failed).length >= data.urls.length
+                ? `Completed ${Object.keys(data.processed).length} Succeeded, ${Object.keys(data.failed).length} Failed`
+                : `Processing ${Object.keys(data.processed).length + Object.keys(data.failed).length}/${data.urls.length}`;
+            const payload = {
+                status,
+                statusText,
+                processed: data.processed,
+                failed: data.failed,
+            };
+            if (status === adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.COMPLETED) {
+                payload.finishedAt = new Date();
+                payload.duration = new Date().getTime() - data.createdAt.toDate().getTime();
+            }
+            transaction.update(ref, payload);
+        });
+    }
+    async handleSingleCrawl(shortDigest, url, token, cachePath) {
+        const error = {
+            reason: ''
+        };
+        const response = await fetch('https://r.jina.ai', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${token}`,
+                'Accept': 'application/json',
+            },
+            body: JSON.stringify({ url })
+        });
+        if (!response.ok) {
+            error.reason = `Failed to crawl ${url}, ${response.statusText}`;
+        }
+        else {
+            const json = await response.json();
+            await this.firebaseObjectStorage.saveFile(cachePath, Buffer.from(JSON.stringify(json), 'utf-8'), {
+                metadata: {
+                    contentType: 'application/json',
+                }
+            });
+        }
+        return {
+            error,
+        };
+    }
+    async handleSingleCrawlRecursively(shortDigest, url, token, meta, cachePath) {
+        const error = {
+            reason: ''
+        };
+        const response = await fetch('https://r.jina.ai', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${token}`,
+                'Accept': 'application/json',
+                'X-With-Links-Summary': 'true',
+            },
+            body: JSON.stringify({ url })
+        });
+        if (!response.ok) {
+            error.reason = `Failed to crawl ${url}, ${response.statusText}`;
+        }
+        else {
+            const json = await response.json();
+            await this.firebaseObjectStorage.saveFile(cachePath, Buffer.from(JSON.stringify(json), 'utf-8'), {
+                metadata: {
+                    contentType: 'application/json',
+                }
+            });
+            const title = json.data.title;
+            const description = json.data.description;
+            const links = json.data.links;
+            const relevantUrls = await this.getRelevantUrls(token, { title, description, links });
+            this.logger.debug(`Total urls: ${Object.keys(links).length}, relevant urls: ${relevantUrls.length}`);
+            for (const url of relevantUrls) {
+                let abortContinue = false;
+                let abortBreak = false;
+                await adaptive_crawl_task_1.AdaptiveCrawlTask.DB.runTransaction(async (transaction) => {
+                    const ref = adaptive_crawl_task_1.AdaptiveCrawlTask.COLLECTION.doc(shortDigest);
+                    const state = await transaction.get(ref);
+                    const data = state.data();
+                    if (data.urls.includes(url)) {
+                        this.logger.debug('Recursive CONTINUE', data);
+                        abortContinue = true;
+                        return;
+                    }
+                    const urls = [
+                        ...data.urls,
+                        url
+                    ];
+                    if (urls.length > meta.maxPages || data.status === adaptive_crawl_task_1.AdaptiveCrawlTaskStatus.COMPLETED) {
+                        this.logger.debug('Recursive BREAK', data);
+                        abortBreak = true;
+                        return;
+                    }
+                    transaction.update(ref, { urls });
+                });
+                if (abortContinue) {
+                    continue;
+                }
+                if (abortBreak) {
+                    break;
+                }
+                await (0, functions_1.getFunctions)().taskQueue(AdaptiveCrawlerHost_1.__singleCrawlQueueName).enqueue({
+                    shortDigest, url, token, meta
+                }, {
+                    dispatchDeadlineSeconds: 1800,
+                    uri: await (0, get_function_url_1.getFunctionUrl)(AdaptiveCrawlerHost_1.__singleCrawlQueueName),
+                });
+            }
+            ;
+        }
+        return {
+            error,
+        };
+    }
+    async getRelevantUrls(token, { title, description, links }) {
+        const invalidSuffix = [
+            '.zip',
+            '.docx',
+            '.pptx',
+            '.xlsx',
+        ];
+        const validLinks = Object.entries(links)
+            .map(([title, link]) => link)
+            .filter(link => link.startsWith('http') && !invalidSuffix.some(suffix => link.endsWith(suffix)));
+        let query = '';
+        if (!description) {
+            query += title;
+        }
+        else {
+            query += `TITLE: ${title}; DESCRIPTION: ${description}`;
+        }
+        const data = {
+            model: 'jina-reranker-v2-base-multilingual',
+            query,
+            top_n: 15,
+            documents: validLinks,
+        };
+        const response = await fetch('https://api.jina.ai/v1/rerank', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${token}`
+            },
+            body: JSON.stringify(data)
+        });
+        const json = (await response.json());
+        const highestRelevanceScore = json.results[0]?.relevance_score ?? 0;
+        return json.results.filter(r => r.relevance_score > Math.max(highestRelevanceScore * 0.6, 0.1)).map(r => removeURLHash(r.document.text));
+    }
+    getIndex(user) {
+        // TODO: 需要更新使用方式
+        // const indexObject: Record<string, string | number | undefined> = Object.create(indexProto);
+        // Object.assign(indexObject, {
+        //     usage1: 'https://r.jina.ai/YOUR_URL',
+        //     usage2: 'https://s.jina.ai/YOUR_SEARCH_QUERY',
+        //     homepage: 'https://jina.ai/reader',
+        //     sourceCode: 'https://github.com/jina-ai/reader',
+        // });
+        // if (user) {
+        //     indexObject[''] = undefined;
+        //     indexObject.authenticatedAs = `${user.user_id} (${user.full_name})`;
+        //     indexObject.balanceLeft = user.wallet.total_balance;
+        // }
+        // return indexObject;
+    }
+    async crawlUrlsFromSitemap(url, maxPages) {
+        const sitemapsFromRobotsTxt = await this.getSitemapsFromRobotsTxt(url);
+        const initialSitemaps = [];
+        if (sitemapsFromRobotsTxt === null) {
+            initialSitemaps.push(`${url.origin}/sitemap.xml`);
+        }
+        else {
+            initialSitemaps.push(...sitemapsFromRobotsTxt);
+        }
+        const allUrls = new Set();
+        const processedSitemaps = new Set();
+        const fetchSitemapUrls = async (sitemapUrl) => {
+            sitemapUrl = sitemapUrl.trim();
+            if (processedSitemaps.has(sitemapUrl)) {
+                return;
+            }
+            processedSitemaps.add(sitemapUrl);
+            try {
+                const response = await fetch(sitemapUrl);
+                const sitemapContent = await response.text();
+                const parser = new xmldom_1.DOMParser();
+                const xmlDoc = parser.parseFromString(sitemapContent, 'text/xml');
+                // handle normal sitemap
+                const urlElements = xmlDoc.getElementsByTagName('url');
+                for (let i = 0; i < urlElements.length; i++) {
+                    const locElement = urlElements[i].getElementsByTagName('loc')[0];
+                    if (locElement) {
+                        const loc = locElement.textContent?.trim() || '';
+                        if (loc.startsWith(url.origin) && !loc.endsWith('.xml')) {
+                            allUrls.add(removeURLHash(loc));
+                        }
+                        if (allUrls.size >= maxPages) {
+                            return;
+                        }
+                    }
+                }
+                // handle sitemap index
+                const sitemapElements = xmlDoc.getElementsByTagName('sitemap');
+                for (let i = 0; i < sitemapElements.length; i++) {
+                    const locElement = sitemapElements[i].getElementsByTagName('loc')[0];
+                    if (locElement) {
+                        await fetchSitemapUrls(locElement.textContent?.trim() || '');
+                        if (allUrls.size >= maxPages) {
+                            return;
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                this.logger.error(`Error fetching sitemap ${sitemapUrl}:`, error);
+            }
+        };
+        for (const sitemapUrl of initialSitemaps) {
+            await fetchSitemapUrls(sitemapUrl);
+            if (allUrls.size >= maxPages) {
+                break;
+            }
+        }
+        const urlsToProcess = Array.from(allUrls).slice(0, maxPages);
+        return urlsToProcess;
+    }
+    async getSitemapsFromRobotsTxt(url) {
+        const hostname = url.origin;
+        const robotsUrl = `${hostname}/robots.txt`;
+        const response = await fetch(robotsUrl);
+        if (response.status === 404) {
+            return null;
+        }
+        const robotsTxt = await response.text();
+        if (robotsTxt.length) {
+            const robot = (0, robots_parser_1.default)(robotsUrl, robotsTxt);
+            return robot.getSitemaps();
+        }
+        return null;
+    }
+};
+exports.AdaptiveCrawlerHost = AdaptiveCrawlerHost;
+__decorate([
+    (0, shared_1.CloudHTTPv2)({
+        runtime: {
+            memory: '1GiB',
+            timeoutSeconds: 300,
+            concurrency: 22,
+        },
+        tags: ['Crawler'],
+        httpMethod: ['post', 'get'],
+        returnType: [String],
+    }),
+    __param(0, (0, shared_1.RPCReflect)()),
+    __param(1, (0, shared_1.Ctx)()),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_c = typeof civkit_1.RPCReflection !== "undefined" && civkit_1.RPCReflection) === "function" ? _c : Object, Object, typeof (_d = typeof jina_embeddings_auth_1.JinaEmbeddingsAuthDTO !== "undefined" && jina_embeddings_auth_1.JinaEmbeddingsAuthDTO) === "function" ? _d : Object, crawler_options_1.CrawlerOptions,
+        adaptive_crawler_options_1.AdaptiveCrawlerOptions]),
+    __metadata("design:returntype", Promise)
+], AdaptiveCrawlerHost.prototype, "adaptiveCrawl", null);
+__decorate([
+    (0, shared_1.CloudHTTPv2)({
+        runtime: {
+            memory: '1GiB',
+            timeoutSeconds: 300,
+            concurrency: 22,
+        },
+        tags: ['Crawler'],
+        httpMethod: ['post', 'get'],
+        returnType: adaptive_crawl_task_1.AdaptiveCrawlTask,
+    }),
+    __param(0, (0, shared_1.RPCReflect)()),
+    __param(1, (0, shared_1.Ctx)()),
+    __param(3, (0, shared_1.Param)('taskId')),
+    __param(4, (0, shared_1.Param)('urls')),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [typeof (_e = typeof civkit_1.RPCReflection !== "undefined" && civkit_1.RPCReflection) === "function" ? _e : Object, Object, typeof (_f = typeof jina_embeddings_auth_1.JinaEmbeddingsAuthDTO !== "undefined" && jina_embeddings_auth_1.JinaEmbeddingsAuthDTO) === "function" ? _f : Object, String, Array]),
+    __metadata("design:returntype", Promise)
+], AdaptiveCrawlerHost.prototype, "adaptiveCrawlStatus", null);
+__decorate([
+    (0, shared_1.CloudTaskV2)({
+        name: AdaptiveCrawlerHost.__singleCrawlQueueName,
+        runtime: {
+            cpu: 1,
+            memory: '1GiB',
+            timeoutSeconds: 3600,
+            concurrency: 2,
+            maxInstances: 200,
+            retryConfig: {
+                maxAttempts: 3,
+                minBackoffSeconds: 60,
+            },
+            rateLimits: {
+                maxConcurrentDispatches: 150,
+                maxDispatchesPerSecond: 5,
+            },
+        }
+    }),
+    __param(0, (0, shared_1.Param)('shortDigest')),
+    __param(1, (0, shared_1.Param)('url')),
+    __param(2, (0, shared_1.Param)('token')),
+    __param(3, (0, shared_1.Param)('meta')),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, String, String, Object]),
+    __metadata("design:returntype", Promise)
+], AdaptiveCrawlerHost.prototype, "singleCrawlQueue", null);
+exports.AdaptiveCrawlerHost = AdaptiveCrawlerHost = AdaptiveCrawlerHost_1 = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [typeof (_a = typeof shared_1.Logger !== "undefined" && shared_1.Logger) === "function" ? _a : Object, typeof (_b = typeof shared_1.FirebaseStorageBucketControl !== "undefined" && shared_1.FirebaseStorageBucketControl) === "function" ? _b : Object])
+], AdaptiveCrawlerHost);
+//# sourceMappingURL=adaptive-crawler.js.map

build/cloud-functions/adaptive-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-crawler.js","sourceRoot":"","sources":["../../src/cloud-functions/adaptive-crawler.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,mCAMgB;AAChB,uCAAqC;AACrC,sCAAmH;AAGnH,6EAA2E;AAC3E,kEAAyC;AACzC,2CAA2C;AAE3C,8EAAyE;AACzE,4DAAwD;AAExD,mEAAuF;AACvF,wDAAwD;AACxD,gEAA2D;AAG3D,MAAM,SAAS,GAAG,IAAI,oBAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChD,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,EAAE;IAClC,IAAI,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QACZ,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,GAAG,CAAC;IACf,CAAC;AACL,CAAC,CAAA;AAGM,IAAM,mBAAmB,GAAzB,MAAM,mBAAoB,SAAQ,gBAAO;;aAK5B,2BAAsB,GAAG,kBAAkB,AAArB,CAAsB;IAE5D,YACc,YAAoB,EACpB,qBAAmD;QAE7D,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QAHV,iBAAY,GAAZ,YAAY,CAAQ;QACpB,0BAAqB,GAArB,qBAAqB,CAA8B;QARjE,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,mGAAmG;QACnG,gBAAW,GAAG,CAAC,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IAStC,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAYK,AAAN,KAAK,CAAC,aAAa,CACD,UAAyB,EAChC,GAGN,EACD,IAA2B,EAC3B,cAA8B,EAC9B,sBAA8C;QAE9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACd,sBAAsB;YACtB,cAAc;SACjB,CAAC,CAAC;QAGH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,sBAAsB,CAAC;QAExD,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9C,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAElC,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBAC1G,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC;YAED,OAAO,IAAA,mCAA0B,EAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,EAC5D,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAChD,CAAC;QACN,CAAC;QAED,MAAM,IAAI,GAAG;YACT,SAAS,EAAE,SAAS,CAAC,QAAQ,EAAE;YAC/B,UAAU;YACV,QAAQ;SACX,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,MAAM,uCAAiB,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAEpE,IAAI,QAAQ,EAAE,SAAS,EAAE,CAAC;YACtB,IAAI,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,WAAW,gBAAgB,QAAQ,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAClG,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,WAAW,gBAAgB,QAAQ,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;YAC1G,CAAC;QACL,CAAC;QAED,MAAM,uCAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC;YACpD,GAAG,EAAE,WAAW;YAChB,MAAM,EAAE,6CAAuB,CAAC,OAAO;YACvC,UAAU,EAAE,SAAS;YACrB,IAAI;YACJ,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,EAAE;YACb,MAAM,EAAE,EAAE;SACb,CAAC,CAAC;QAEH,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,IAAI,UAAU,EAAE,CAAC;YACb,IAAI,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,uCAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;gBACvD,MAAM,EAAE,6CAAuB,CAAC,UAAU;gBAC1C,UAAU,EAAE,gBAAgB,IAAI,CAAC,MAAM,EAAE;gBACzC,IAAI;aACP,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,EAAE,CAAC;YACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,QAAQ,CAAC,IAAI,CAAC,IAAA,wBAAY,GAAE,CAAC,SAAS,CAAC,qBAAmB,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAAC;oBACvF,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI;iBAClD,EAAE;oBACC,uBAAuB,EAAE,IAAI;oBAC7B,GAAG,EAAE,MAAM,IAAA,iCAAc,EAAC,qBAAmB,CAAC,sBAAsB,CAAC;iBACxE,CAAC,CAAC,CAAC;YACR,CAAC;YAAA,CAAC;YAEF,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YAExB,MAAM,uCAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;gBACvD,IAAI,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;aAC/B,CAAC,CAAC;YAEH,MAAM,IAAA,wBAAY,GAAE,CAAC,SAAS,CAAC,qBAAmB,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAAC;gBAC/E,WAAW,EAAE,GAAG,EAAE,SAAS,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI;aACxE,EAAE;gBACC,uBAAuB,EAAE,IAAI;gBAC7B,GAAG,EAAE,MAAM,IAAA,iCAAc,EAAC,qBAAmB,CAAC,sBAAsB,CAAC;aACxE,CAAC,CAAA;QACN,CAAC;QAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACnC,CAAC;IAYK,AAAN,KAAK,CAAC,mBAAmB,CACP,UAAyB,EAChC,GAGN,EACD,IAA2B,EACV,MAAc,EAChB,OAAiB,EAAE;QAElC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,6BAAoB,CAAC,oBAAoB,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,uCAAiB,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;QAE5D,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,8BAAqB,CAAC,yBAAyB,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,KAAK,EAAE,SAAS,IAAI,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YAChF,MAAM,IAAI,8BAAqB,CAAC,sBAAsB,CAAC,CAAC;QAC5D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,EAAE,EAAE;gBACnF,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBACrB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;oBACrE,KAAM,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAGD,OAAO,KAAK,CAAC;IACjB,CAAC;IAoBK,AAAN,KAAK,CAAC,gBAAgB,CACI,WAAmB,EAC3B,GAAW,EACT,KAAa,EACd,IAA+B;QAE9C,MAAM,KAAK,GAAG;YACV,MAAM,EAAE,EAAE;SACb,CAAC;QAEF,MAAM,KAAK,GAAG,MAAM,uCAAiB,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,KAAK,EAAE,MAAM,KAAK,6CAAuB,CAAC,SAAS,EAAE,CAAC;YACtD,OAAO;QACX,CAAC;QAED,IAAI,CAAC;YACD,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAM,CAAC,EAAE,CAAC;YACR,KAAK,CAAC,MAAM,GAAG,wBAAwB,GAAG,EAAE,CAAC;QACjD,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,uBAAuB,WAAW,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAE9E,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU;gBAC1B,CAAC,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC;gBAClE,CAAC,CAAC,MAAM,IAAI,CAAC,4BAA4B,CAAC,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;YAExF,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACvC,CAAC;QAED,MAAM,uCAAiB,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;YAC5D,MAAM,GAAG,GAAG,uCAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAC1D,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAkD,CAAC;YAE1E,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACpC,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;gBACnG,CAAC,CAAC,6CAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,6CAAuB,CAAC,UAAU,CAAC;YAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;gBACvG,CAAC,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,eAAe,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,SAAS;gBACxG,CAAC,CAAC,cAAc,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAE/G,MAAM,OAAO,GAA+B;gBACxC,MAAM;gBACN,UAAU;gBACV,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,IAAI,CAAC,MAAM;aACtB,CAAC;YAEF,IAAI,MAAM,KAAK,6CAAuB,CAAC,SAAS,EAAE,CAAC;gBAC/C,OAAO,CAAC,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChC,OAAO,CAAC,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;YAChF,CAAC;YAED,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,WAAmB,EAAE,GAAW,EAAE,KAAa,EAAE,SAAiB;QACtF,MAAM,KAAK,GAAG;YACV,MAAM,EAAE,EAAE;SACb,CAAA;QAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,mBAAmB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,KAAK,EAAE;gBAClC,QAAQ,EAAE,kBAAkB;aAC/B;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;SAChC,CAAC,CAAA;QAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,KAAK,CAAC,MAAM,GAAG,mBAAmB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEnC,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,SAAS,EAC/C,MAAM,CAAC,IAAI,CACP,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACpB,OAAO,CACV,EACD;gBACI,QAAQ,EAAE;oBACN,WAAW,EAAE,kBAAkB;iBAClC;aACJ,CACJ,CAAA;QACL,CAAC;QAED,OAAO;YACH,KAAK;SACR,CAAA;IACL,CAAC;IAED,KAAK,CAAC,4BAA4B,CAC9B,WAAmB,EAAE,GAAW,EAAE,KAAa,EAAE,IAA+B,EAAE,SAAiB;QAEnG,MAAM,KAAK,GAAG;YACV,MAAM,EAAE,EAAE;SACb,CAAA;QACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,mBAAmB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,KAAK,EAAE;gBAClC,QAAQ,EAAE,kBAAkB;gBAC5B,sBAAsB,EAAE,MAAM;aACjC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,KAAK,CAAC,MAAM,GAAG,mBAAmB,GAAG,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,SAAS,EAC/C,MAAM,CAAC,IAAI,CACP,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACpB,OAAO,CACV,EACD;gBACI,QAAQ,EAAE;oBACN,WAAW,EAAE,kBAAkB;iBAClC;aACJ,CACJ,CAAA;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;YAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAA+B,CAAC;YAExD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YACtF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,oBAAoB,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;YAErG,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;gBAC7B,IAAI,aAAa,GAAG,KAAK,CAAC;gBAC1B,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,MAAM,uCAAiB,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;oBAC5D,MAAM,GAAG,GAAG,uCAAiB,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;oBAC1D,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAkD,CAAC;oBAE1E,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAC;wBAC9C,aAAa,GAAG,IAAI,CAAC;wBACrB,OAAO;oBACX,CAAC;oBAED,MAAM,IAAI,GAAG;wBACT,GAAG,IAAI,CAAC,IAAI;wBACZ,GAAG;qBACN,CAAC;oBAEF,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,6CAAuB,CAAC,SAAS,EAAE,CAAC;wBACnF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;wBAC3C,UAAU,GAAG,IAAI,CAAC;wBAClB,OAAO;oBACX,CAAC;oBAED,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;gBACtC,CAAC,CAAC,CAAC;gBAEH,IAAI,aAAa,EAAE,CAAC;oBAChB,SAAS;gBACb,CAAC;gBACD,IAAI,UAAU,EAAE,CAAC;oBACb,MAAM;gBACV,CAAC;gBAED,MAAM,IAAA,wBAAY,GAAE,CAAC,SAAS,CAAC,qBAAmB,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAAC;oBAC/E,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI;iBAChC,EAAE;oBACC,uBAAuB,EAAE,IAAI;oBAC7B,GAAG,EAAE,MAAM,IAAA,iCAAc,EAAC,qBAAmB,CAAC,sBAAsB,CAAC;iBACxE,CAAC,CAAC;YACP,CAAC;YAAA,CAAC;QACN,CAAC;QAED,OAAO;YACH,KAAK;SACR,CAAA;IACL,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAa,EAAE,EACjC,KAAK,EAAE,WAAW,EAAE,KAAK,EAK5B;QACG,MAAM,aAAa,GAAG;YAClB,MAAM;YACN,OAAO;YACP,OAAO;YACP,OAAO;SACV,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;aAC5B,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAErG,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,KAAK,IAAI,KAAK,CAAC;QACnB,CAAC;aAAO,CAAC;YACL,KAAK,IAAI,UAAU,KAAK,kBAAkB,WAAW,EAAE,CAAC;QAC5D,CAAC;QAED,MAAM,IAAI,GAAG;YACT,KAAK,EAAE,oCAAoC;YAC3C,KAAK;YACL,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,UAAU;SACxB,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,+BAA+B,EAAE;YAC1D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,cAAc,EAAE,kBAAkB;gBAClC,eAAe,EAAE,UAAU,KAAK,EAAE;aACrC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC7B,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAQlC,CAAC;QAEF,MAAM,qBAAqB,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,eAAe,IAAI,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7I,CAAC;IAED,QAAQ,CAAC,IAAiC;QACtC,iBAAiB;QACjB,8FAA8F;QAE9F,+BAA+B;QAC/B,4CAA4C;QAC5C,qDAAqD;QACrD,0CAA0C;QAC1C,uDAAuD;QACvD,MAAM;QAEN,cAAc;QACd,mCAAmC;QACnC,2EAA2E;QAC3E,2DAA2D;QAC3D,IAAI;QAEJ,sBAAsB;IAC1B,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,GAAQ,EAAE,QAAgB;QACjD,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC;QAEvE,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,IAAI,qBAAqB,KAAK,IAAI,EAAE,CAAC;YACjC,eAAe,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,cAAc,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACJ,eAAe,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAC;QACnD,CAAC;QAGD,MAAM,OAAO,GAAgB,IAAI,GAAG,EAAE,CAAC;QACvC,MAAM,iBAAiB,GAAgB,IAAI,GAAG,EAAE,CAAC;QAEjD,MAAM,gBAAgB,GAAG,KAAK,EAAE,UAAkB,EAAE,EAAE;YAClD,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;YAE/B,IAAI,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,OAAO;YACX,CAAC;YAED,iBAAiB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAElC,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzC,MAAM,cAAc,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,IAAI,kBAAS,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;gBAElE,wBAAwB;gBACxB,MAAM,WAAW,GAAG,MAAM,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACjE,IAAI,UAAU,EAAE,CAAC;wBACb,MAAM,GAAG,GAAG,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;wBACjD,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;4BACtD,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;wBACpC,CAAC;wBACD,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;4BAC3B,OAAO;wBACX,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,uBAAuB;gBACvB,MAAM,eAAe,GAAG,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;gBAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9C,MAAM,UAAU,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrE,IAAI,UAAU,EAAE,CAAC;wBACb,MAAM,gBAAgB,CAAC,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC7D,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;4BAC3B,OAAO;wBACX,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC;QACL,CAAC,CAAC;QAEF,KAAK,MAAM,UAAU,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;YACnC,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM;YACV,CAAC;QACL,CAAC;QAED,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAE7D,OAAO,aAAa,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,GAAQ;QACnC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC;QAC5B,MAAM,SAAS,GAAG,GAAG,QAAQ,aAAa,CAAC;QAC3C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,IAAA,uBAAY,EAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;;AAjjBQ,kDAAmB;AA8BtB;IAVL,IAAA,oBAAW,EAAC;QACT,OAAO,EAAE;YACL,MAAM,EAAE,MAAM;YACd,cAAc,EAAE,GAAG;YACnB,WAAW,EAAE,EAAE;SAClB;QACD,IAAI,EAAE,CAAC,SAAS,CAAC;QACjB,UAAU,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;QAC3B,UAAU,EAAE,CAAC,MAAM,CAAC;KACvB,CAAC;IAEG,WAAA,IAAA,mBAAU,GAAE,CAAA;IACZ,WAAA,IAAA,YAAG,GAAE,CAAA;;yDADoB,sBAAa,oBAAb,sBAAa,4DAKjC,4CAAqB,oBAArB,4CAAqB,gCACX,gCAAc;QACN,iDAAsB;;wDAiGjD;AAYK;IAVL,IAAA,oBAAW,EAAC;QACT,OAAO,EAAE;YACL,MAAM,EAAE,MAAM;YACd,cAAc,EAAE,GAAG;YACnB,WAAW,EAAE,EAAE;SAClB;QACD,IAAI,EAAE,CAAC,SAAS,CAAC;QACjB,UAAU,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;QAC3B,UAAU,EAAE,uCAAiB;KAChC,CAAC;IAEG,WAAA,IAAA,mBAAU,GAAE,CAAA;IACZ,WAAA,IAAA,YAAG,GAAE,CAAA;IAKL,WAAA,IAAA,cAAK,EAAC,QAAQ,CAAC,CAAA;IACf,WAAA,IAAA,cAAK,EAAC,MAAM,CAAC,CAAA;;yDAPY,sBAAa,oBAAb,sBAAa,4DAKjC,4CAAqB,oBAArB,4CAAqB;;8DA+B9B;AAoBK;IAlBL,IAAA,oBAAW,EAAC;QACT,IAAI,EAAE,mBAAmB,CAAC,sBAAsB;QAChD,OAAO,EAAE;YACL,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,MAAM;YACd,cAAc,EAAE,IAAI;YACpB,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE;gBACT,WAAW,EAAE,CAAC;gBACd,iBAAiB,EAAE,EAAE;aACxB;YACD,UAAU,EAAE;gBACR,uBAAuB,EAAE,GAAG;gBAC5B,sBAAsB,EAAE,CAAC;aAC5B;SACJ;KACJ,CAAC;IAEG,WAAA,IAAA,cAAK,EAAC,aAAa,CAAC,CAAA;IACpB,WAAA,IAAA,cAAK,EAAC,KAAK,CAAC,CAAA;IACZ,WAAA,IAAA,cAAK,EAAC,OAAO,CAAC,CAAA;IACd,WAAA,IAAA,cAAK,EAAC,MAAM,CAAC,CAAA;;;;2DA+DjB;8BA/QQ,mBAAmB;IAD/B,IAAA,oBAAS,GAAE;yDASoB,eAAM,oBAAN,eAAM,oDACG,qCAA4B,oBAA5B,qCAA4B;GATxD,mBAAmB,CAkjB/B"}

build/cloud-functions/data-crunching.js

@@ -0,0 +1,260 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a, _b, _c;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DataCrunchingHost = void 0;
+const civkit_1 = require("civkit");
+const tsyringe_1 = require("tsyringe");
+const shared_1 = require("../shared");
+const lodash_1 = __importDefault(require("lodash"));
+const crawler_1 = require("../api/crawler");
+const crawled_1 = require("../db/crawled");
+const dayjs_1 = __importDefault(require("dayjs"));
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const zlib_1 = require("zlib");
+const functions_1 = require("firebase-admin/functions");
+const snapshot_formatter_1 = require("../services/snapshot-formatter");
+const get_function_url_1 = require("../utils/get-function-url");
+dayjs_1.default.extend(require('dayjs/plugin/utc'));
+let DataCrunchingHost = class DataCrunchingHost extends civkit_1.RPCHost {
+    constructor(globalLogger, crawler, snapshotFormatter, tempFileManager, firebaseObjectStorage) {
+        super(...lodash_1.default.without(arguments, crawler));
+        this.globalLogger = globalLogger;
+        this.crawler = crawler;
+        this.snapshotFormatter = snapshotFormatter;
+        this.tempFileManager = tempFileManager;
+        this.firebaseObjectStorage = firebaseObjectStorage;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.pageCacheCrunchingPrefix = 'crunched-pages';
+        this.pageCacheCrunchingBatchSize = 5000;
+        this.pageCacheCrunchingTMinus = 6 * 24 * 60 * 60 * 1000;
+        this.rev = 7;
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+    // @CloudTaskV2({
+    //     runtime: {
+    //         cpu: 2,
+    //         memory: '4GiB',
+    //         timeoutSeconds: 3600,
+    //         concurrency: 2,
+    //         maxInstances: 200,
+    //         retryConfig: {
+    //             maxAttempts: 3,
+    //             minBackoffSeconds: 60,
+    //         },
+    //         rateLimits: {
+    //             maxConcurrentDispatches: 150,
+    //             maxDispatchesPerSecond: 2,
+    //         },
+    //     },
+    //     tags: ['DataCrunching'],
+    // })
+    async crunchPageCacheWorker(date, offset) {
+        this.logger.info(`Crunching page cache @${date}+${offset}...`);
+        for await (const { fileName, records } of this.iterPageCacheRecords(date, offset)) {
+            this.logger.info(`Crunching ${fileName}...`);
+            const fileOnDrive = await this.crunchCacheRecords(records);
+            const fstream = (0, fs_1.createReadStream)(fileOnDrive.path);
+            const gzipStream = (0, zlib_1.createGzip)();
+            fstream.pipe(gzipStream, { end: true });
+            await this.firebaseObjectStorage.bucket.file(fileName).save(gzipStream, {
+                contentType: 'application/jsonl+gzip',
+            });
+        }
+        this.logger.info(`Crunching page cache @${date}+${offset} done.`);
+        return true;
+    }
+    // @CloudScheduleV2('2 0 * * *', {
+    //     name: 'crunchPageCacheEveryday',
+    //     runtime: {
+    //         cpu: 2,
+    //         memory: '4GiB',
+    //         timeoutSeconds: 1800,
+    //         timeZone: 'UTC',
+    //         retryCount: 3,
+    //         minBackoffSeconds: 60,
+    //     },
+    //     tags: ['DataCrunching'],
+    // })
+    async dispatchPageCacheCrunching() {
+        for await (const { fileName, date, offset } of this.iterPageCacheChunks()) {
+            this.logger.info(`Dispatching ${fileName}...`);
+            // sse.write({ data: `Dispatching ${fileName}...` });
+            await (0, functions_1.getFunctions)().taskQueue('crunchPageCacheWorker').enqueue({ date, offset }, {
+                dispatchDeadlineSeconds: 1800,
+                uri: await (0, get_function_url_1.getFunctionUrl)('crunchPageCacheWorker'),
+            });
+        }
+        return true;
+    }
+    // @CloudHTTPv2({
+    //     runtime: {
+    //         cpu: 2,
+    //         memory: '4GiB',
+    //         timeoutSeconds: 3600,
+    //         concurrency: 2,
+    //         maxInstances: 200,
+    //     },
+    //     tags: ['DataCrunching'],
+    // })
+    // async dispatchPageCacheCrunching(
+    //     @RPCReflect() rpcReflect: RPCReflection
+    // ) {
+    //     const sse = new OutputServerEventStream({ highWaterMark: 4096 });
+    //     rpcReflect.return(sse);
+    //     rpcReflect.catch((err) => {
+    //         sse.end({ data: `Error: ${err.message}` });
+    //     });
+    //     for await (const { fileName, date, offset } of this.iterPageCacheChunks()) {
+    //         this.logger.info(`Dispatching ${fileName}...`);
+    //         sse.write({ data: `Dispatching ${fileName}...` });
+    //         await getFunctions().taskQueue('crunchPageCacheWorker').enqueue({ date, offset }, {
+    //             dispatchDeadlineSeconds: 1800,
+    //             uri: await getFunctionUrl('crunchPageCacheWorker'),
+    //         });
+    //     }
+    //     sse.end({ data: 'done' });
+    //     return true;
+    // }
+    async *iterPageCacheRecords(date, inputOffset) {
+        const startOfToday = (0, dayjs_1.default)().utc().startOf('day');
+        const startingPoint = (0, dayjs_1.default)().utc().subtract(this.pageCacheCrunchingTMinus, 'ms').startOf('day');
+        let theDay = startingPoint;
+        if (date) {
+            theDay = (0, dayjs_1.default)(date).utc().startOf('day');
+        }
+        let counter = 0;
+        if (inputOffset) {
+            counter = parseInt(inputOffset, 10);
+        }
+        while (theDay.isBefore(startOfToday)) {
+            const fileName = `${this.pageCacheCrunchingPrefix}/r${this.rev}/${theDay.format('YYYY-MM-DD')}/${counter}.jsonl.gz`;
+            const offset = counter;
+            counter += this.pageCacheCrunchingBatchSize;
+            const fileExists = (await this.firebaseObjectStorage.bucket.file(fileName).exists())[0];
+            if (fileExists) {
+                continue;
+            }
+            const records = await crawled_1.Crawled.fromFirestoreQuery(crawled_1.Crawled.COLLECTION
+                .where('createdAt', '>=', theDay.toDate())
+                .where('createdAt', '<', theDay.add(1, 'day').toDate())
+                .orderBy('createdAt', 'asc')
+                .offset(offset)
+                .limit(this.pageCacheCrunchingBatchSize));
+            this.logger.info(`Found ${records.length} records for ${theDay.format('YYYY-MM-DD')} at offset ${offset}`, { fileName, counter });
+            if (!records.length) {
+                if (date) {
+                    break;
+                }
+                theDay = theDay.add(1, 'day');
+                counter = 0;
+                continue;
+            }
+            yield { fileName, records };
+            if (offset) {
+                break;
+            }
+        }
+    }
+    async *iterPageCacheChunks() {
+        const startOfToday = (0, dayjs_1.default)().utc().startOf('day');
+        const startingPoint = (0, dayjs_1.default)().utc().subtract(this.pageCacheCrunchingTMinus, 'ms').startOf('day');
+        let theDay = startingPoint;
+        let counter = 0;
+        while (theDay.isBefore(startOfToday)) {
+            const fileName = `${this.pageCacheCrunchingPrefix}/r${this.rev}/${theDay.format('YYYY-MM-DD')}/${counter}.jsonl.gz`;
+            const offset = counter;
+            counter += this.pageCacheCrunchingBatchSize;
+            const fileExists = (await this.firebaseObjectStorage.bucket.file(fileName).exists())[0];
+            if (fileExists) {
+                continue;
+            }
+            const nRecords = (await crawled_1.Crawled.COLLECTION
+                .where('createdAt', '>=', theDay.toDate())
+                .where('createdAt', '<', theDay.add(1, 'day').toDate())
+                .orderBy('createdAt', 'asc')
+                .offset(offset)
+                .limit(this.pageCacheCrunchingBatchSize)
+                .count().get()).data().count;
+            this.logger.info(`Found ${nRecords} records for ${theDay.format('YYYY-MM-DD')} at offset ${offset}`, { fileName, counter });
+            if (nRecords < this.pageCacheCrunchingBatchSize) {
+                theDay = theDay.add(1, 'day');
+                counter = 0;
+            }
+            if (nRecords) {
+                yield { fileName, date: theDay.toISOString(), offset };
+            }
+        }
+    }
+    async crunchCacheRecords(records) {
+        const throttle = new civkit_1.PromiseThrottle(30);
+        const localFilePath = this.tempFileManager.alloc();
+        let nextDrainDeferred = (0, civkit_1.Defer)();
+        nextDrainDeferred.resolve();
+        for (const record of records) {
+            await throttle.acquire();
+            this.firebaseObjectStorage.downloadFile(`snapshots/${record._id}`)
+                .then(async (snapshotTxt) => {
+                try {
+                    const snapshot = JSON.parse(snapshotTxt.toString('utf-8'));
+                    let formatted = await this.snapshotFormatter.formatSnapshot('default', snapshot);
+                    if (!formatted.content) {
+                        formatted = await this.snapshotFormatter.formatSnapshot('markdown', snapshot);
+                    }
+                    await nextDrainDeferred.promise;
+                    await (0, promises_1.appendFile)(localFilePath, JSON.stringify({
+                        url: snapshot.href,
+                        title: snapshot.title || '',
+                        html: snapshot.html || '',
+                        text: snapshot.text || '',
+                        content: formatted.content || '',
+                    }) + '\n', { encoding: 'utf-8' });
+                }
+                catch (err) {
+                    this.logger.warn(`Failed to parse snapshot for ${record._id}`, { err });
+                }
+            })
+                .finally(() => {
+                throttle.release();
+            });
+        }
+        await throttle.nextDrain();
+        const ro = {
+            path: localFilePath
+        };
+        this.tempFileManager.bindPathTo(ro, localFilePath);
+        return ro;
+    }
+};
+exports.DataCrunchingHost = DataCrunchingHost;
+__decorate([
+    __param(0, (0, shared_1.Param)('date')),
+    __param(1, (0, shared_1.Param)('offset', { default: 0 })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String, Number]),
+    __metadata("design:returntype", Promise)
+], DataCrunchingHost.prototype, "crunchPageCacheWorker", null);
+exports.DataCrunchingHost = DataCrunchingHost = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [typeof (_a = typeof shared_1.Logger !== "undefined" && shared_1.Logger) === "function" ? _a : Object, crawler_1.CrawlerHost,
+        snapshot_formatter_1.SnapshotFormatter, typeof (_b = typeof shared_1.TempFileManager !== "undefined" && shared_1.TempFileManager) === "function" ? _b : Object, typeof (_c = typeof shared_1.FirebaseStorageBucketControl !== "undefined" && shared_1.FirebaseStorageBucketControl) === "function" ? _c : Object])
+], DataCrunchingHost);
+//# sourceMappingURL=data-crunching.js.map

build/cloud-functions/data-crunching.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-crunching.js","sourceRoot":"","sources":["../../src/cloud-functions/data-crunching.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,mCAIgB;AAChB,uCAAqC;AACrC,sCAGmB;AACnB,oDAAuB;AACvB,4CAA6C;AAE7C,2CAAwC;AACxC,kDAA0B;AAC1B,2BAAsC;AACtC,0CAAyC;AACzC,+BAAkC;AAClC,wDAAwD;AACxD,uEAAmE;AACnE,gEAA2D;AAE3D,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAGnC,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,gBAAO;IAQ1C,YACc,YAAoB,EAEpB,OAAoB,EACpB,iBAAoC,EACpC,eAAgC,EAChC,qBAAmD;QAE7D,KAAK,CAAC,GAAG,gBAAC,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;QAP9B,iBAAY,GAAZ,YAAY,CAAQ;QAEpB,YAAO,GAAP,OAAO,CAAa;QACpB,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,oBAAe,GAAf,eAAe,CAAiB;QAChC,0BAAqB,GAArB,qBAAqB,CAA8B;QAbjE,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,6BAAwB,GAAG,gBAAgB,CAAC;QAC5C,gCAA2B,GAAG,IAAI,CAAC;QACnC,6BAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACnD,QAAG,GAAG,CAAC,CAAC;IAWR,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,iBAAiB;IACjB,iBAAiB;IACjB,kBAAkB;IAClB,0BAA0B;IAC1B,gCAAgC;IAChC,0BAA0B;IAC1B,6BAA6B;IAC7B,yBAAyB;IACzB,8BAA8B;IAC9B,qCAAqC;IACrC,aAAa;IACb,wBAAwB;IACxB,4CAA4C;IAC5C,yCAAyC;IACzC,aAAa;IACb,SAAS;IACT,+BAA+B;IAC/B,KAAK;IACC,AAAN,KAAK,CAAC,qBAAqB,CACR,IAAY,EACM,MAAc;QAE/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,MAAM,KAAK,CAAC,CAAC;QAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;YAChF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,QAAQ,KAAK,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAA,qBAAgB,EAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,IAAA,iBAAU,GAAE,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;YACxC,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE;gBACpE,WAAW,EAAE,wBAAwB;aACxC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,IAAI,IAAI,MAAM,QAAQ,CAAC,CAAC;QAElE,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,kCAAkC;IAClC,uCAAuC;IACvC,iBAAiB;IACjB,kBAAkB;IAClB,0BAA0B;IAC1B,gCAAgC;IAChC,2BAA2B;IAC3B,yBAAyB;IACzB,iCAAiC;IACjC,SAAS;IACT,+BAA+B;IAC/B,KAAK;IACL,KAAK,CAAC,0BAA0B;QAC5B,IAAI,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,QAAQ,KAAK,CAAC,CAAC;YAC/C,qDAAqD;YAErD,MAAM,IAAA,wBAAY,GAAE,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;gBAC9E,uBAAuB,EAAE,IAAI;gBAC7B,GAAG,EAAE,MAAM,IAAA,iCAAc,EAAC,uBAAuB,CAAC;aACrD,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,iBAAiB;IACjB,iBAAiB;IACjB,kBAAkB;IAClB,0BAA0B;IAC1B,gCAAgC;IAChC,0BAA0B;IAC1B,6BAA6B;IAC7B,SAAS;IACT,+BAA+B;IAC/B,KAAK;IACL,oCAAoC;IACpC,8CAA8C;IAC9C,MAAM;IACN,wEAAwE;IACxE,8BAA8B;IAC9B,kCAAkC;IAClC,sDAAsD;IACtD,UAAU;IACV,mFAAmF;IACnF,0DAA0D;IAC1D,6DAA6D;IAE7D,8FAA8F;IAC9F,6CAA6C;IAC7C,kEAAkE;IAClE,cAAc;IACd,QAAQ;IAER,iCAAiC;IAEjC,mBAAmB;IACnB,IAAI;IAEJ,KAAK,CAAA,CAAE,oBAAoB,CAAC,IAAa,EAAE,WAA6B;QACpE,MAAM,YAAY,GAAG,IAAA,eAAK,GAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,IAAA,eAAK,GAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjG,IAAI,MAAM,GAAG,aAAa,CAAC;QAE3B,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,GAAG,IAAA,eAAK,EAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,WAAW,EAAE,CAAC;YACd,OAAO,GAAG,QAAQ,CAAC,WAAqB,EAAE,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,wBAAwB,KAAK,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,WAAW,CAAC;YACpH,MAAM,MAAM,GAAG,OAAO,CAAC;YACvB,OAAO,IAAI,IAAI,CAAC,2BAA2B,CAAC;YAC5C,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,IAAI,UAAU,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,iBAAO,CAAC,kBAAkB,CAAC,iBAAO,CAAC,UAAU;iBAC9D,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;iBACzC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;iBACtD,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;iBAC3B,MAAM,CAAC,MAAM,CAAC;iBACd,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAC3C,CAAC;YAEF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,cAAc,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAElI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBAClB,IAAI,IAAI,EAAE,CAAC;oBACP,MAAM;gBACV,CAAC;gBACD,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC9B,OAAO,GAAG,CAAC,CAAC;gBACZ,SAAS;YACb,CAAC;YAED,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;YAE5B,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM;YACV,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK,CAAA,CAAE,mBAAmB;QACtB,MAAM,YAAY,GAAG,IAAA,eAAK,GAAE,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,IAAA,eAAK,GAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjG,IAAI,MAAM,GAAG,aAAa,CAAC;QAE3B,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,OAAO,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,GAAG,IAAI,CAAC,wBAAwB,KAAK,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,WAAW,CAAC;YACpH,MAAM,MAAM,GAAG,OAAO,CAAC;YACvB,OAAO,IAAI,IAAI,CAAC,2BAA2B,CAAC;YAC5C,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACxF,IAAI,UAAU,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,MAAM,QAAQ,GAAG,CAAC,MAAM,iBAAO,CAAC,UAAU;iBACrC,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;iBACzC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;iBACtD,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;iBAC3B,MAAM,CAAC,MAAM,CAAC;iBACd,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC;iBACvC,KAAK,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAEjC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,gBAAgB,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,cAAc,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5H,IAAI,QAAQ,GAAG,IAAI,CAAC,2BAA2B,EAAE,CAAC;gBAC9C,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC9B,OAAO,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,IAAI,QAAQ,EAAE,CAAC;gBACX,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,CAAC;YAC3D,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,OAAkB;QACvC,MAAM,QAAQ,GAAG,IAAI,wBAAe,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACnD,IAAI,iBAAiB,GAAG,IAAA,cAAK,GAAE,CAAC;QAChC,iBAAiB,CAAC,OAAO,EAAE,CAAC;QAE5B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,qBAAqB,CAAC,YAAY,CAAC,aAAa,MAAM,CAAC,GAAG,EAAE,CAAC;iBAC7D,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE;gBACxB,IAAI,CAAC;oBACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;oBAE3D,IAAI,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;oBACjF,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;wBACrB,SAAS,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,cAAc,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;oBAClF,CAAC;oBAED,MAAM,iBAAiB,CAAC,OAAO,CAAC;oBAChC,MAAM,IAAA,qBAAU,EAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC;wBAC3C,GAAG,EAAE,QAAQ,CAAC,IAAI;wBAClB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,EAAE;wBAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;wBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI,IAAI,EAAE;wBACzB,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,EAAE;qBACnC,CAAC,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;gBAEtC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,MAAM,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5E,CAAC;YACL,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACV,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC,CAAC,CAAC;QACX,CAAC;QAED,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;QAG3B,MAAM,EAAE,GAAG;YACP,IAAI,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;QAEnD,OAAO,EAAE,CAAC;IACd,CAAC;CACJ,CAAA;AA/PY,8CAAiB;AA2CpB;IACD,WAAA,IAAA,cAAK,EAAC,MAAM,CAAC,CAAA;IACb,WAAA,IAAA,cAAK,EAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAA;;;;8DAiBnC;4BA9DQ,iBAAiB;IAD7B,IAAA,oBAAS,GAAE;yDAUoB,eAAM,oBAAN,eAAM,gCAEX,qBAAW;QACD,sCAAiB,sBACnB,wBAAe,oBAAf,wBAAe,oDACT,qCAA4B,oBAA5B,qCAA4B;GAdxD,iBAAiB,CA+P7B"}

build/db/adaptive-crawl-task.js

@@ -0,0 +1,93 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var AdaptiveCrawlTask_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AdaptiveCrawlTask = exports.AdaptiveCrawlTaskStatus = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+var AdaptiveCrawlTaskStatus;
+(function (AdaptiveCrawlTaskStatus) {
+    AdaptiveCrawlTaskStatus["PENDING"] = "pending";
+    AdaptiveCrawlTaskStatus["PROCESSING"] = "processing";
+    AdaptiveCrawlTaskStatus["COMPLETED"] = "completed";
+    AdaptiveCrawlTaskStatus["FAILED"] = "failed";
+})(AdaptiveCrawlTaskStatus || (exports.AdaptiveCrawlTaskStatus = AdaptiveCrawlTaskStatus = {}));
+let AdaptiveCrawlTask = class AdaptiveCrawlTask extends firestore_1.FirestoreRecord {
+    static { AdaptiveCrawlTask_1 = this; }
+    static { this.collectionName = 'adaptiveCrawlTasks'; }
+    static { this.patchedFields = [
+        'meta',
+    ]; }
+    static from(input) {
+        for (const field of this.patchedFields) {
+            if (typeof input[field] === 'string') {
+                input[field] = (0, civkit_1.parseJSONText)(input[field]);
+            }
+        }
+        return super.from(input);
+    }
+    degradeForFireStore() {
+        const copy = { ...this };
+        for (const field of this.constructor.patchedFields) {
+            if (typeof copy[field] === 'object') {
+                copy[field] = JSON.stringify(copy[field]);
+            }
+        }
+        return copy;
+    }
+};
+exports.AdaptiveCrawlTask = AdaptiveCrawlTask;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], AdaptiveCrawlTask.prototype, "status", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], AdaptiveCrawlTask.prototype, "statusText", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], AdaptiveCrawlTask.prototype, "meta", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Array)
+], AdaptiveCrawlTask.prototype, "urls", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], AdaptiveCrawlTask.prototype, "processed", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], AdaptiveCrawlTask.prototype, "failed", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], AdaptiveCrawlTask.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], AdaptiveCrawlTask.prototype, "finishedAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Number)
+], AdaptiveCrawlTask.prototype, "duration", void 0);
+exports.AdaptiveCrawlTask = AdaptiveCrawlTask = AdaptiveCrawlTask_1 = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], AdaptiveCrawlTask);
+//# sourceMappingURL=adaptive-crawl-task.js.map

build/db/adaptive-crawl-task.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-crawl-task.js","sourceRoot":"","sources":["../../src/db/adaptive-crawl-task.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAmD;AACnD,uDAA0D;AAG1D,IAAY,uBAKX;AALD,WAAY,uBAAuB;IAC/B,8CAAmB,CAAA;IACnB,oDAAyB,CAAA;IACzB,kDAAuB,CAAA;IACvB,4CAAiB,CAAA;AACrB,CAAC,EALW,uBAAuB,uCAAvB,uBAAuB,QAKlC;AAKM,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,2BAAe;;aAClC,mBAAc,GAAG,oBAAoB,AAAvB,CAAwB;aA2C/C,kBAAa,GAAG;QACnB,MAAM;KACT,AAFmB,CAElB;IAEF,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACnC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAa,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAsB,CAAC;IAClD,CAAC;IAEQ,mBAAmB;QACxB,MAAM,IAAI,GAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAK,IAAI,CAAC,WAAwC,CAAC,aAAa,EAAE,CAAC;YAC/E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAQ,CAAC;YACrD,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;;AApEQ,8CAAiB;AAQ1B;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;iDAC+B;AAKjC;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;qDACkB;AAGpB;IADC,IAAA,aAAI,GAAE;;+CAKL;AAGF;IADC,IAAA,aAAI,GAAE;;+CACS;AAGhB;IADC,IAAA,aAAI,GAAE;;oDAGL;AAGF;IADC,IAAA,aAAI,GAAE;;iDAGL;AAGF;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;oDAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACM,IAAI;qDAAC;AAGlB;IADC,IAAA,aAAI,GAAE;;mDACW;4BA1CT,iBAAiB;IAH7B,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,iBAAiB,CAuE7B"}

build/db/crawled.js

@@ -0,0 +1,86 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var Crawled_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Crawled = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+let Crawled = class Crawled extends firestore_1.FirestoreRecord {
+    static { Crawled_1 = this; }
+    static { this.collectionName = 'crawled'; }
+    static { this.patchedFields = [
+        'snapshot'
+    ]; }
+    static from(input) {
+        for (const field of this.patchedFields) {
+            if (typeof input[field] === 'string') {
+                input[field] = (0, civkit_1.parseJSONText)(input[field]);
+            }
+        }
+        return super.from(input);
+    }
+    degradeForFireStore() {
+        const copy = { ...this };
+        for (const field of this.constructor.patchedFields) {
+            if (typeof copy[field] === 'object') {
+                copy[field] = JSON.stringify(copy[field]);
+            }
+        }
+        return copy;
+    }
+};
+exports.Crawled = Crawled;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], Crawled.prototype, "url", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], Crawled.prototype, "urlPathDigest", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Crawled.prototype, "htmlSignificantlyModifiedByJs", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], Crawled.prototype, "snapshot", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Crawled.prototype, "screenshotAvailable", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Crawled.prototype, "pageshotAvailable", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Crawled.prototype, "snapshotAvailable", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], Crawled.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], Crawled.prototype, "expireAt", void 0);
+exports.Crawled = Crawled = Crawled_1 = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], Crawled);
+//# sourceMappingURL=crawled.js.map

build/db/crawled.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawled.js","sourceRoot":"","sources":["../../src/db/crawled.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAmD;AACnD,uDAA0D;AAOnD,IAAM,OAAO,GAAb,MAAM,OAAQ,SAAQ,2BAAe;;aACxB,mBAAc,GAAG,SAAS,AAAZ,CAAa;aAmCpC,kBAAa,GAAG;QACnB,UAAU;KACb,AAFmB,CAElB;IAEF,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACnC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAa,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAY,CAAC;IACxC,CAAC;IAEQ,mBAAmB;QACxB,MAAM,IAAI,GAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAK,IAAI,CAAC,WAA8B,CAAC,aAAa,EAAE,CAAC;YACrE,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAQ,CAAC;YACrD,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;;AA5DQ,0BAAO;AAQhB;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;oCACW;AAKb;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;8CACqB;AAGvB;IADC,IAAA,aAAI,GAAE;;8DACiC;AAGxC;IADC,IAAA,aAAI,GAAE;;yCAC2D;AAGlE;IADC,IAAA,aAAI,GAAE;;oDACuB;AAG9B;IADC,IAAA,aAAI,GAAE;;kDACqB;AAG5B;IADC,IAAA,aAAI,GAAE;;kDACqB;AAG5B;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;0CAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;yCAAC;kBAlCP,OAAO;IAHnB,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,OAAO,CA+DnB"}

build/db/domain-blockade.js

@@ -0,0 +1,46 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DomainBlockade = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+let DomainBlockade = class DomainBlockade extends firestore_1.FirestoreRecord {
+    static { this.collectionName = 'domainBlockades'; }
+};
+exports.DomainBlockade = DomainBlockade;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], DomainBlockade.prototype, "domain", void 0);
+__decorate([
+    (0, civkit_1.Prop)({ required: true }),
+    __metadata("design:type", String)
+], DomainBlockade.prototype, "triggerReason", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], DomainBlockade.prototype, "triggerUrl", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], DomainBlockade.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], DomainBlockade.prototype, "expireAt", void 0);
+exports.DomainBlockade = DomainBlockade = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], DomainBlockade);
+//# sourceMappingURL=domain-blockade.js.map

build/db/domain-blockade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"domain-blockade.js","sourceRoot":"","sources":["../../src/db/domain-blockade.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAoC;AACpC,uDAA0D;AAKnD,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,2BAAe;aAC/B,mBAAc,GAAG,iBAAiB,AAApB,CAAqB;;AAD1C,wCAAc;AAQvB;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;8CACc;AAGhB;IADC,IAAA,aAAI,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;;qDACF;AAGvB;IADC,IAAA,aAAI,GAAE;;kDACa;AAGpB;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;iDAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;gDAAC;yBApBP,cAAc;IAH1B,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,cAAc,CAuB1B"}

build/db/domain-profile.js

@@ -0,0 +1,47 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DomainProfile = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+const crawler_options_1 = require("../dto/crawler-options");
+let DomainProfile = class DomainProfile extends firestore_1.FirestoreRecord {
+    static { this.collectionName = 'domainProfiles'; }
+};
+exports.DomainProfile = DomainProfile;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], DomainProfile.prototype, "path", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], DomainProfile.prototype, "triggerUrl", void 0);
+__decorate([
+    (0, civkit_1.Prop)({ required: true, type: crawler_options_1.ENGINE_TYPE }),
+    __metadata("design:type", String)
+], DomainProfile.prototype, "engine", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], DomainProfile.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], DomainProfile.prototype, "expireAt", void 0);
+exports.DomainProfile = DomainProfile = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], DomainProfile);
+//# sourceMappingURL=domain-profile.js.map

build/db/domain-profile.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"domain-profile.js","sourceRoot":"","sources":["../../src/db/domain-profile.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAoC;AACpC,uDAA0D;AAC1D,4DAAqD;AAK9C,IAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,2BAAe;aAC9B,mBAAc,GAAG,gBAAgB,AAAnB,CAAoB;;AADzC,sCAAa;AAQtB;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;2CACY;AAGd;IADC,IAAA,aAAI,GAAE;;iDACa;AAGpB;IADC,IAAA,aAAI,EAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,6BAAW,EAAE,CAAC;;6CAC5B;AAGhB;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;gDAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;+CAAC;wBApBP,aAAa;IAHzB,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,aAAa,CAuBzB"}

build/db/img-alt.js

@@ -0,0 +1,60 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImgAlt = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+let ImgAlt = class ImgAlt extends firestore_1.FirestoreRecord {
+    static { this.collectionName = 'imgAlts'; }
+};
+exports.ImgAlt = ImgAlt;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], ImgAlt.prototype, "src", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], ImgAlt.prototype, "urlDigest", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Number)
+], ImgAlt.prototype, "width", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Number)
+], ImgAlt.prototype, "height", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], ImgAlt.prototype, "generatedAlt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], ImgAlt.prototype, "originalAlt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], ImgAlt.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], ImgAlt.prototype, "expireAt", void 0);
+exports.ImgAlt = ImgAlt = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], ImgAlt);
+//# sourceMappingURL=img-alt.js.map

build/db/img-alt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"img-alt.js","sourceRoot":"","sources":["../../src/db/img-alt.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAoC;AACpC,uDAA0D;AAMnD,IAAM,MAAM,GAAZ,MAAM,MAAO,SAAQ,2BAAe;aACvB,mBAAc,GAAG,SAAS,AAAZ,CAAa;;AADlC,wBAAM;AAQf;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;mCACW;AAKb;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;yCACiB;AAGnB;IADC,IAAA,aAAI,GAAE;;qCACQ;AAGf;IADC,IAAA,aAAI,GAAE;;sCACS;AAGhB;IADC,IAAA,aAAI,GAAE;;4CACe;AAGtB;IADC,IAAA,aAAI,GAAE;;2CACc;AAGrB;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;yCAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;wCAAC;iBA/BP,MAAM;IAHlB,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,MAAM,CAkClB"}

build/db/pdf.js

@@ -0,0 +1,78 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var PDFContent_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PDFContent = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+let PDFContent = class PDFContent extends firestore_1.FirestoreRecord {
+    static { PDFContent_1 = this; }
+    static { this.collectionName = 'pdfs'; }
+    static { this.patchedFields = [
+        'meta'
+    ]; }
+    static from(input) {
+        for (const field of this.patchedFields) {
+            if (typeof input[field] === 'string') {
+                input[field] = (0, civkit_1.parseJSONText)(input[field]);
+            }
+        }
+        return super.from(input);
+    }
+    degradeForFireStore() {
+        const copy = { ...this };
+        for (const field of this.constructor.patchedFields) {
+            if (typeof copy[field] === 'object') {
+                copy[field] = JSON.stringify(copy[field]);
+            }
+        }
+        return copy;
+    }
+};
+exports.PDFContent = PDFContent;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], PDFContent.prototype, "src", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], PDFContent.prototype, "urlDigest", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], PDFContent.prototype, "meta", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], PDFContent.prototype, "text", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", String)
+], PDFContent.prototype, "content", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], PDFContent.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], PDFContent.prototype, "expireAt", void 0);
+exports.PDFContent = PDFContent = PDFContent_1 = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], PDFContent);
+//# sourceMappingURL=pdf.js.map

build/db/pdf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf.js","sourceRoot":"","sources":["../../src/db/pdf.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAmD;AACnD,uDAA0D;AAMnD,IAAM,UAAU,GAAhB,MAAM,UAAW,SAAQ,2BAAe;;aAC3B,mBAAc,GAAG,MAAM,AAAT,CAAU;aA6BjC,kBAAa,GAAG;QACnB,MAAM;KACT,AAFmB,CAElB;IAEF,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACnC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAa,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAe,CAAC;IAC3C,CAAC;IAEQ,mBAAmB;QACxB,MAAM,IAAI,GAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAK,IAAI,CAAC,WAAiC,CAAC,aAAa,EAAE,CAAC;YACxE,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAQ,CAAC;YACrD,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;;AAtDQ,gCAAU;AAQnB;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;uCACW;AAKb;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;6CACiB;AAGnB;IADC,IAAA,aAAI,GAAE;;wCACsB;AAG7B;IADC,IAAA,aAAI,GAAE;;wCACO;AAGd;IADC,IAAA,aAAI,GAAE;;2CACU;AAGjB;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;6CAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;4CAAC;qBA5BP,UAAU;IAHtB,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,UAAU,CAyDtB"}

build/db/searched.js

@@ -0,0 +1,79 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var SearchResult_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SERPResult = exports.SerperSearchResult = exports.SearchResult = void 0;
+const civkit_1 = require("civkit");
+const firestore_1 = require("../shared/lib/firestore");
+let SearchResult = class SearchResult extends firestore_1.FirestoreRecord {
+    static { SearchResult_1 = this; }
+    static { this.collectionName = 'searchResults'; }
+    static { this.patchedFields = [
+        'query',
+        'response',
+    ]; }
+    static from(input) {
+        for (const field of this.patchedFields) {
+            if (typeof input[field] === 'string') {
+                input[field] = (0, civkit_1.parseJSONText)(input[field]);
+            }
+        }
+        return super.from(input);
+    }
+    degradeForFireStore() {
+        const copy = { ...this };
+        for (const field of this.constructor.patchedFields) {
+            if (typeof copy[field] === 'object') {
+                copy[field] = JSON.stringify(copy[field]);
+            }
+        }
+        return copy;
+    }
+};
+exports.SearchResult = SearchResult;
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", Object)
+], SearchResult.prototype, "query", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        required: true
+    }),
+    __metadata("design:type", String)
+], SearchResult.prototype, "queryDigest", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Object)
+], SearchResult.prototype, "response", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], SearchResult.prototype, "createdAt", void 0);
+__decorate([
+    (0, civkit_1.Prop)(),
+    __metadata("design:type", Date)
+], SearchResult.prototype, "expireAt", void 0);
+exports.SearchResult = SearchResult = SearchResult_1 = __decorate([
+    (0, civkit_1.Also)({
+        dictOf: Object
+    })
+], SearchResult);
+class SerperSearchResult extends SearchResult {
+    static { this.collectionName = 'serperSearchResults'; }
+}
+exports.SerperSearchResult = SerperSearchResult;
+class SERPResult extends SearchResult {
+    static { this.collectionName = 'SERPResults'; }
+}
+exports.SERPResult = SERPResult;
+//# sourceMappingURL=searched.js.map

build/db/searched.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"searched.js","sourceRoot":"","sources":["../../src/db/searched.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAAmD;AACnD,uDAA0D;AAMnD,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,2BAAe;;aAC7B,mBAAc,GAAG,eAAe,AAAlB,CAAmB;aAyB1C,kBAAa,GAAG;QACnB,OAAO;QACP,UAAU;KACb,AAHmB,CAGlB;IAEF,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACnC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAA,sBAAa,EAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/C,CAAC;QACL,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAiB,CAAC;IAC7C,CAAC;IAEQ,mBAAmB;QACxB,MAAM,IAAI,GAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;QAE9B,KAAK,MAAM,KAAK,IAAK,IAAI,CAAC,WAAmC,CAAC,aAAa,EAAE,CAAC;YAC1E,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAQ,CAAC;YACrD,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;;AAnDQ,oCAAY;AAQrB;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;2CACU;AAKZ;IAHC,IAAA,aAAI,EAAC;QACF,QAAQ,EAAE,IAAI;KACjB,CAAC;;iDACmB;AAGrB;IADC,IAAA,aAAI,GAAE;;8CACQ;AAGf;IADC,IAAA,aAAI,GAAE;8BACK,IAAI;+CAAC;AAGjB;IADC,IAAA,aAAI,GAAE;8BACI,IAAI;8CAAC;uBAtBP,YAAY;IAHxB,IAAA,aAAI,EAAC;QACF,MAAM,EAAE,MAAM;KACjB,CAAC;GACW,YAAY,CAoDxB;AAED,MAAa,kBAAmB,SAAQ,YAAY;aAChC,mBAAc,GAAG,qBAAqB,CAAC;;AAD3D,gDAEC;AAED,MAAa,UAAW,SAAQ,YAAY;aACxB,mBAAc,GAAG,aAAa,CAAC;;AADnD,gCAEC"}

build/dto/adaptive-crawler-options.js

@@ -0,0 +1,70 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AdaptiveCrawlerOptions = void 0;
+const civkit_1 = require("civkit");
+let AdaptiveCrawlerOptions = class AdaptiveCrawlerOptions extends civkit_1.AutoCastable {
+    static from(input) {
+        const instance = super.from(input);
+        const ctx = Reflect.get(input, civkit_1.RPC_CALL_ENVIRONMENT);
+        let maxPages = parseInt(ctx?.req.get('x-max-pages') || '');
+        if (!isNaN(maxPages) && maxPages > 0) {
+            instance.maxPages = maxPages <= 100 ? maxPages : 100;
+        }
+        const useSitemap = ctx?.req.get('x-use-sitemap');
+        if (useSitemap !== undefined) {
+            instance.useSitemap = Boolean(useSitemap);
+        }
+        return instance;
+    }
+};
+exports.AdaptiveCrawlerOptions = AdaptiveCrawlerOptions;
+__decorate([
+    (0, civkit_1.Prop)({
+        default: true,
+        desc: 'Use sitemap to crawl the website.',
+    }),
+    __metadata("design:type", Boolean)
+], AdaptiveCrawlerOptions.prototype, "useSitemap", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        default: 10,
+        desc: 'Max number of pages to crawl.',
+        validate: (v) => v >= 1 && v <= 100,
+    }),
+    __metadata("design:type", Number)
+], AdaptiveCrawlerOptions.prototype, "maxPages", void 0);
+exports.AdaptiveCrawlerOptions = AdaptiveCrawlerOptions = __decorate([
+    (0, civkit_1.Also)({
+        openapi: {
+            operation: {
+                parameters: {
+                    'X-Use-Sitemap': {
+                        description: 'Use sitemap to crawl the website.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Max-Depth': {
+                        description: 'Max deep level to crawl.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Max-Pages': {
+                        description: 'Max number of pages to crawl.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                }
+            }
+        }
+    })
+], AdaptiveCrawlerOptions);
+//# sourceMappingURL=adaptive-crawler-options.js.map

build/dto/adaptive-crawler-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-crawler-options.js","sourceRoot":"","sources":["../../src/dto/adaptive-crawler-options.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAwE;AA2BjE,IAAM,sBAAsB,GAA5B,MAAM,sBAAuB,SAAQ,qBAAY;IAcpD,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAA2B,CAAC;QAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,6BAAoB,CAGtC,CAAC;QAEd,IAAI,QAAQ,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,QAAQ,GAAG,QAAQ,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACjD,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ,CAAA;AAjCY,wDAAsB;AAK/B;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,IAAI;QACb,IAAI,EAAE,mCAAmC;KAC5C,CAAC;;0DACmB;AAOrB;IALC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,+BAA+B;QACrC,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG;KAC9C,CAAC;;wDACgB;iCAZT,sBAAsB;IAvBlC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE;YACL,SAAS,EAAE;gBACP,UAAU,EAAE;oBACR,eAAe,EAAE;wBACb,WAAW,EAAE,mCAAmC;wBAChD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,aAAa,EAAE;wBACX,WAAW,EAAE,0BAA0B;wBACvC,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,aAAa,EAAE;wBACX,WAAW,EAAE,+BAA+B;wBAC5C,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;iBACJ;aACJ;SACJ;KACJ,CAAC;GACW,sBAAsB,CAiClC"}

build/dto/crawler-options.js

@@ -0,0 +1,753 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlerOptionsHeaderOnly = exports.CrawlerOptions = exports.BASE_URL_MODES = exports.IMAGE_RETENTION_MODES = exports.RESPOND_TIMING = exports.ENGINE_TYPE = exports.CONTENT_FORMAT = void 0;
+const civ_rpc_1 = require("civkit/civ-rpc");
+const fancy_file_1 = require("civkit/fancy-file");
+const set_cookie_parser_1 = require("set-cookie-parser");
+const turndown_tweakable_options_1 = require("./turndown-tweakable-options");
+var CONTENT_FORMAT;
+(function (CONTENT_FORMAT) {
+    CONTENT_FORMAT["CONTENT"] = "content";
+    CONTENT_FORMAT["MARKDOWN"] = "markdown";
+    CONTENT_FORMAT["HTML"] = "html";
+    CONTENT_FORMAT["TEXT"] = "text";
+    CONTENT_FORMAT["PAGESHOT"] = "pageshot";
+    CONTENT_FORMAT["SCREENSHOT"] = "screenshot";
+    CONTENT_FORMAT["VLM"] = "vlm";
+    CONTENT_FORMAT["READER_LM"] = "readerlm-v2";
+})(CONTENT_FORMAT || (exports.CONTENT_FORMAT = CONTENT_FORMAT = {}));
+var ENGINE_TYPE;
+(function (ENGINE_TYPE) {
+    ENGINE_TYPE["AUTO"] = "auto";
+    ENGINE_TYPE["BROWSER"] = "browser";
+    ENGINE_TYPE["CURL"] = "curl";
+    ENGINE_TYPE["CF_BROWSER_RENDERING"] = "cf-browser-rendering";
+})(ENGINE_TYPE || (exports.ENGINE_TYPE = ENGINE_TYPE = {}));
+var RESPOND_TIMING;
+(function (RESPOND_TIMING) {
+    RESPOND_TIMING["HTML"] = "html";
+    RESPOND_TIMING["VISIBLE_CONTENT"] = "visible-content";
+    RESPOND_TIMING["MUTATION_IDLE"] = "mutation-idle";
+    RESPOND_TIMING["RESOURCE_IDLE"] = "resource-idle";
+    RESPOND_TIMING["MEDIA_IDLE"] = "media-idle";
+    RESPOND_TIMING["NETWORK_IDLE"] = "network-idle";
+})(RESPOND_TIMING || (exports.RESPOND_TIMING = RESPOND_TIMING = {}));
+const CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT));
+exports.IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'];
+const IMAGE_RETENTION_MODE_VALUES = new Set(exports.IMAGE_RETENTION_MODES);
+exports.BASE_URL_MODES = ['initial', 'final'];
+const BASE_URL_MODE_VALUES = new Set(exports.BASE_URL_MODES);
+class Viewport extends civ_rpc_1.AutoCastable {
+}
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: 1024
+    }),
+    __metadata("design:type", Number)
+], Viewport.prototype, "width", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: 1024
+    }),
+    __metadata("design:type", Number)
+], Viewport.prototype, "height", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Number)
+], Viewport.prototype, "deviceScaleFactor", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Viewport.prototype, "isMobile", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Viewport.prototype, "isLandscape", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Boolean)
+], Viewport.prototype, "hasTouch", void 0);
+let CrawlerOptions = class CrawlerOptions extends civ_rpc_1.AutoCastable {
+    static from(input) {
+        const instance = super.from(input);
+        const ctx = Reflect.get(input, civ_rpc_1.RPC_CALL_ENVIRONMENT);
+        const customMode = ctx?.get('x-respond-with') || ctx?.get('x-return-format');
+        if (customMode) {
+            instance.respondWith = customMode;
+        }
+        if (instance.respondWith) {
+            instance.respondWith = instance.respondWith.toLowerCase();
+        }
+        if (instance.respondWith?.includes('lm')) {
+            if (instance.respondWith.includes('content') || instance.respondWith.includes('markdown')) {
+                throw new civ_rpc_1.ParamValidationError({
+                    path: 'respondWith',
+                    message: `LM formats conflicts with content/markdown.`,
+                });
+            }
+        }
+        const locale = ctx?.get('x-locale');
+        if (locale) {
+            instance.locale = locale;
+        }
+        const referer = ctx?.get('x-referer');
+        if (referer) {
+            instance.referer = referer;
+        }
+        const withGeneratedAlt = ctx?.get('x-with-generated-alt');
+        if (withGeneratedAlt) {
+            instance.withGeneratedAlt = Boolean(withGeneratedAlt);
+        }
+        const withLinksSummary = ctx?.get('x-with-links-summary');
+        if (withLinksSummary) {
+            if (withLinksSummary === 'all') {
+                instance.withLinksSummary = withLinksSummary;
+            }
+            else {
+                instance.withLinksSummary = Boolean(withLinksSummary);
+            }
+        }
+        const withImagesSummary = ctx?.get('x-with-images-summary');
+        if (withImagesSummary) {
+            instance.withImagesSummary = Boolean(withImagesSummary);
+        }
+        const retainImages = ctx?.get('x-retain-images');
+        if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) {
+            instance.retainImages = retainImages;
+        }
+        if (instance.withGeneratedAlt) {
+            instance.retainImages = 'all_p';
+        }
+        const noCache = ctx?.get('x-no-cache');
+        if (noCache) {
+            instance.noCache = Boolean(noCache);
+        }
+        if (instance.noCache && instance.cacheTolerance === undefined) {
+            instance.cacheTolerance = 0;
+        }
+        let cacheTolerance = parseInt(ctx?.get('x-cache-tolerance') || '');
+        if (!isNaN(cacheTolerance)) {
+            instance.cacheTolerance = cacheTolerance;
+        }
+        const noGfm = ctx?.get('x-no-gfm');
+        if (noGfm) {
+            instance.noGfm = noGfm === 'table' ? noGfm : Boolean(noGfm);
+        }
+        let timeoutSeconds = parseInt(ctx?.get('x-timeout') || '');
+        if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) {
+            instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180;
+        }
+        else if (ctx?.get('x-timeout')) {
+            instance.timeout = null;
+        }
+        const removeSelector = ctx?.get('x-remove-selector')?.split(', ').filter(Boolean);
+        instance.removeSelector ??= removeSelector?.length ? removeSelector : undefined;
+        const targetSelector = ctx?.get('x-target-selector')?.split(', ').filter(Boolean);
+        instance.targetSelector ??= targetSelector?.length ? targetSelector : undefined;
+        const waitForSelector = ctx?.get('x-wait-for-selector')?.split(', ').filter(Boolean);
+        instance.waitForSelector ??= (waitForSelector?.length ? waitForSelector : undefined) || instance.targetSelector;
+        const overrideUserAgent = ctx?.get('x-user-agent') || undefined;
+        instance.userAgent ??= overrideUserAgent;
+        const engine = ctx?.get('x-engine');
+        if (engine) {
+            instance.engine = engine;
+        }
+        if (instance.engine) {
+            instance.engine = instance.engine.toLowerCase();
+        }
+        if (instance.engine === 'vlm') {
+            instance.engine = ENGINE_TYPE.BROWSER;
+            instance.respondWith = CONTENT_FORMAT.VLM;
+        }
+        else if (instance.engine === 'readerlm-v2') {
+            instance.engine = ENGINE_TYPE.AUTO;
+            instance.respondWith = CONTENT_FORMAT.READER_LM;
+        }
+        const keepImgDataUrl = ctx?.get('x-keep-img-data-url');
+        if (keepImgDataUrl) {
+            instance.keepImgDataUrl = Boolean(keepImgDataUrl);
+        }
+        const withIframe = ctx?.get('x-with-iframe');
+        if (withIframe) {
+            instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe);
+        }
+        if (instance.withIframe) {
+            instance.timeout ??= null;
+        }
+        const withShadowDom = ctx?.get('x-with-shadow-dom');
+        if (withShadowDom) {
+            instance.withShadowDom = Boolean(withShadowDom);
+        }
+        if (instance.withShadowDom) {
+            instance.timeout ??= null;
+        }
+        const cookies = [];
+        const setCookieHeaders = (ctx?.get('x-set-cookie')?.split(', ') || instance.setCookies).filter(Boolean);
+        if (Array.isArray(setCookieHeaders)) {
+            for (const setCookie of setCookieHeaders) {
+                cookies.push({
+                    ...(0, set_cookie_parser_1.parseString)(setCookie, { decodeValues: true }),
+                });
+            }
+        }
+        else if (setCookieHeaders && typeof setCookieHeaders === 'string') {
+            cookies.push({
+                ...(0, set_cookie_parser_1.parseString)(setCookieHeaders, { decodeValues: true }),
+            });
+        }
+        instance.setCookies = cookies;
+        const proxyUrl = ctx?.get('x-proxy-url');
+        instance.proxyUrl ??= proxyUrl || undefined;
+        const proxy = ctx?.get('x-proxy');
+        instance.proxy ??= proxy || undefined;
+        const robotsTxt = ctx?.get('x-robots-txt');
+        instance.robotsTxt ??= robotsTxt || undefined;
+        const tokenBudget = ctx?.get('x-token-budget');
+        instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;
+        const baseMode = ctx?.get('x-base');
+        if (baseMode) {
+            instance.base = baseMode;
+        }
+        const dnt = ctx?.get('dnt');
+        instance.doNotTrack ??= (parseInt(dnt || '') || null);
+        const respondTiming = ctx?.get('x-respond-timing');
+        if (respondTiming) {
+            instance.respondTiming ??= respondTiming;
+        }
+        if (instance.cacheTolerance) {
+            instance.cacheTolerance = instance.cacheTolerance * 1000;
+        }
+        if (ctx) {
+            instance.markdown ??= turndown_tweakable_options_1.TurnDownTweakableOptions.fromCtx(ctx);
+        }
+        return instance;
+    }
+    get presumedRespondTiming() {
+        if (this.respondTiming) {
+            return this.respondTiming;
+        }
+        if (this.timeout && this.timeout >= 20) {
+            return RESPOND_TIMING.NETWORK_IDLE;
+        }
+        if (this.respondWith.includes('shot') || this.respondWith.includes('vlm')) {
+            return RESPOND_TIMING.MEDIA_IDLE;
+        }
+        return RESPOND_TIMING.RESOURCE_IDLE;
+    }
+    isSnapshotAcceptableForEarlyResponse(snapshot) {
+        if (this.waitForSelector?.length) {
+            return false;
+        }
+        const presumedTiming = this.presumedRespondTiming;
+        if (presumedTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) {
+            const now = Date.now();
+            if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) {
+                return true;
+            }
+        }
+        if ((this.respondWith.includes('vlm') || this.respondWith.includes('pageshot')) && !snapshot.pageshot) {
+            return false;
+        }
+        if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) {
+            return false;
+        }
+        if (presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) {
+            const now = Date.now();
+            if ((snapshot.lastContentResourceLoaded + 500) < now) {
+                return true;
+            }
+        }
+        if (this.injectFrameScript?.length || this.injectPageScript?.length) {
+            return false;
+        }
+        if (presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && snapshot.parsed?.content) {
+            return true;
+        }
+        if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) {
+            return true;
+        }
+        if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) {
+            return false;
+        }
+        if (presumedTiming === RESPOND_TIMING.MUTATION_IDLE && snapshot.lastMutationIdle) {
+            return true;
+        }
+        if (this.respondWith.includes('lm')) {
+            return false;
+        }
+        if (this.withIframe) {
+            return false;
+        }
+        return !snapshot.isIntermediate;
+    }
+    isCacheQueryApplicable() {
+        if (this.noCache) {
+            return false;
+        }
+        if (this.cacheTolerance === 0) {
+            return false;
+        }
+        if (this.setCookies?.length) {
+            return false;
+        }
+        if (this.injectFrameScript?.length || this.injectPageScript?.length) {
+            return false;
+        }
+        if (this.viewport) {
+            return false;
+        }
+        return true;
+    }
+    isRequestingCompoundContentFormat() {
+        return !CONTENT_FORMAT_VALUES.has(this.respondWith);
+    }
+    browserIsNotRequired() {
+        if (this.respondTiming && ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(this.respondTiming)) {
+            return false;
+        }
+        if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) {
+            return false;
+        }
+        if (this.injectFrameScript?.length || this.injectPageScript?.length) {
+            return false;
+        }
+        if (this.waitForSelector?.length) {
+            return false;
+        }
+        if (this.withIframe || this.withShadowDom) {
+            return false;
+        }
+        if (this.viewport) {
+            return false;
+        }
+        if (this.pdf) {
+            return false;
+        }
+        if (this.html) {
+            return false;
+        }
+        return true;
+    }
+};
+exports.CrawlerOptions = CrawlerOptions;
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "url", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "html", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        type: BASE_URL_MODE_VALUES,
+        default: 'initial',
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "base", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Base64 encoded PDF.',
+        type: [fancy_file_1.FancyFile, String]
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "pdf", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: CONTENT_FORMAT.CONTENT,
+        type: [CONTENT_FORMAT, String]
+    }),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "respondWith", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Boolean)
+], CrawlerOptions.prototype, "withGeneratedAlt", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({ default: 'all', type: IMAGE_RETENTION_MODE_VALUES }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "retainImages", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "withLinksSummary", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Boolean)
+], CrawlerOptions.prototype, "withImagesSummary", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Boolean)
+], CrawlerOptions.prototype, "noCache", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "noGfm", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Number)
+], CrawlerOptions.prototype, "cacheTolerance", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({ arrayOf: String }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "targetSelector", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({ arrayOf: String }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "waitForSelector", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({ arrayOf: String }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "removeSelector", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Boolean)
+], CrawlerOptions.prototype, "keepImgDataUrl", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+        type: [String, Boolean]
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "withIframe", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        default: false,
+    }),
+    __metadata("design:type", Boolean)
+], CrawlerOptions.prototype, "withShadowDom", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        arrayOf: String,
+    }),
+    __metadata("design:type", Array)
+], CrawlerOptions.prototype, "setCookies", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "proxyUrl", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "proxy", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "userAgent", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "engine", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        arrayOf: String,
+    }),
+    __metadata("design:type", Array)
+], CrawlerOptions.prototype, "injectPageScript", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        arrayOf: String,
+    }),
+    __metadata("design:type", Array)
+], CrawlerOptions.prototype, "injectFrameScript", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        validate: (v) => v > 0 && v <= 180,
+        type: Number,
+        nullable: true,
+    }),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "timeout", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "locale", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "referer", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Number)
+], CrawlerOptions.prototype, "tokenBudget", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Viewport)
+], CrawlerOptions.prototype, "viewport", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "instruction", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "jsonSchema", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "robotsTxt", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", Object)
+], CrawlerOptions.prototype, "doNotTrack", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)(),
+    __metadata("design:type", turndown_tweakable_options_1.TurnDownTweakableOptions)
+], CrawlerOptions.prototype, "markdown", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        type: RESPOND_TIMING,
+    }),
+    __metadata("design:type", String)
+], CrawlerOptions.prototype, "respondTiming", void 0);
+exports.CrawlerOptions = CrawlerOptions = __decorate([
+    (0, civ_rpc_1.Also)({
+        openapi: {
+            operation: {
+                parameters: {
+                    'Accept': {
+                        description: `Specifies your preference for the response format.\n\n` +
+                            `Supported formats: \n` +
+                            `- text/event-stream\n` +
+                            `- application/json or text/json\n` +
+                            `- text/plain`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Cache-Tolerance': {
+                        description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-No-Cache': {
+                        description: `Ignores internal cache if this header is specified with a value.\n\nEquivalent to X-Cache-Tolerance: 0`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Respond-With': {
+                        description: `Specifies the (non-default) form of the crawled data you prefer.\n\n` +
+                            `Supported formats: \n` +
+                            `- markdown\n` +
+                            `- html\n` +
+                            `- text\n` +
+                            `- pageshot\n` +
+                            `- screenshot\n` +
+                            `- content\n` +
+                            `- any combination of the above\n` +
+                            `- readerlm-v2\n` +
+                            `- vlm\n\n` +
+                            `Default: content\n`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Wait-For-Selector': {
+                        description: `Specifies a CSS selector to wait for the appearance of such an element before returning.\n\n` +
+                            'Example: `X-Wait-For-Selector: .content-block`\n',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Target-Selector': {
+                        description: `Specifies a CSS selector for return target instead of the full html.\n\n` +
+                            'Implies `X-Wait-For-Selector: (same selector)`',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Remove-Selector': {
+                        description: `Specifies a CSS selector to remove elements from the full html.\n\n` +
+                            'Example `X-Remove-Selector: nav`',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Keep-Img-Data-Url': {
+                        description: `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\n\n` +
+                            'Example `X-Keep-Img-Data-Url: true`',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Proxy-Url': {
+                        description: `Specifies your custom proxy if you prefer to use one.\n\n` +
+                            `Supported protocols: \n` +
+                            `- http\n` +
+                            `- https\n` +
+                            `- socks4\n` +
+                            `- socks5\n\n` +
+                            `For authentication, https://user:pass@host:port`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Proxy': {
+                        description: `Use a proxy server provided by us.\n\nOptionally specify two-letter country code.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Robots-Txt': {
+                        description: `Load and conform to the respective robot.txt on the target origin.\n\nOptionally specify a bot UA to check against.\n\n`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'DNT': {
+                        description: `When set to 1, prevent the result of this request to be cached in the system.\n\n`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Set-Cookie': {
+                        description: `Sets cookie(s) to the headless browser for your request. \n\n` +
+                            `Syntax is the same with standard Set-Cookie`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-With-Generated-Alt': {
+                        description: `Enable automatic alt-text generating for images without an meaningful alt-text.\n\n` +
+                            `Note: Does not work when \`X-Respond-With\` is specified`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-With-Images-Summary': {
+                        description: `Enable dedicated summary section for images on the page.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-With-links-Summary': {
+                        description: `Enable dedicated summary section for hyper links on the page.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Retain-Images': {
+                        description: `Image retention modes.\n\n` +
+                            `Supported modes: \n` +
+                            `- all: all images\n` +
+                            `- none: no images\n` +
+                            `- alt: only alt text\n` +
+                            `- all_p: all images and with generated alt text\n` +
+                            `- alt_p: only alt text and with generated alt\n\n`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-With-Iframe': {
+                        description: `Enable filling iframe contents into main. (violates standards)`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-With-Shadow-Dom': {
+                        description: `Enable filling shadow dom contents into main. (violates standards)`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-User-Agent': {
+                        description: `Override User-Agent.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Timeout': {
+                        description: `Specify timeout in seconds. Max 180.`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Locale': {
+                        description: 'Specify browser locale for the page.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Referer': {
+                        description: 'Specify referer for the page.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Token-Budget': {
+                        description: 'Specify a budget in tokens.\n\nIf the resulting token cost exceeds the budget, the request is rejected.',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Respond-Timing': {
+                        description: `Explicitly specify the respond timing. One of the following:\n\n` +
+                            `- html: directly return unrendered HTML\n` +
+                            `- visible-content: return immediately when any content becomes available\n` +
+                            `- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` +
+                            `- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\n` +
+                            `- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\n` +
+                            `- network-idle: wait for full load of webpage, also known as networkidle0.\n\n`,
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Engine': {
+                        description: 'Specify the engine to use for crawling.\n\nSupported: browser, direct, cf-browser-rendering',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Base': {
+                        description: 'Select base modes of relative URLs.\n\nSupported: initial, final',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Heading-Style': {
+                        description: 'Heading style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: setext, atx',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Hr': {
+                        description: 'Hr text of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Bullet-List-Marker': {
+                        description: 'Bullet list marker of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: -, +, *',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Em-Delimiter': {
+                        description: 'Em delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: _, *',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Strong-Delimiter': {
+                        description: 'Strong delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: **, __',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Link-Style': {
+                        description: 'Link style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: inlined, referenced, discarded',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                    'X-Md-Link-Reference-Style': {
+                        description: 'Link reference style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: full, collapsed, shortcut, discarded',
+                        in: 'header',
+                        schema: { type: 'string' }
+                    },
+                }
+            }
+        }
+    })
+], CrawlerOptions);
+class CrawlerOptionsHeaderOnly extends CrawlerOptions {
+    static from(input) {
+        const instance = super.from({
+            [civ_rpc_1.RPC_CALL_ENVIRONMENT]: Reflect.get(input, civ_rpc_1.RPC_CALL_ENVIRONMENT),
+        });
+        return instance;
+    }
+}
+exports.CrawlerOptionsHeaderOnly = CrawlerOptionsHeaderOnly;
+//# sourceMappingURL=crawler-options.js.map

build/dto/crawler-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler-options.js","sourceRoot":"","sources":["../../src/dto/crawler-options.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,4CAAsG;AACtG,kDAA8C;AAC9C,yDAAgF;AAEhF,6EAAwE;AAGxE,IAAY,cASX;AATD,WAAY,cAAc;IACtB,qCAAmB,CAAA;IACnB,uCAAqB,CAAA;IACrB,+BAAa,CAAA;IACb,+BAAa,CAAA;IACb,uCAAqB,CAAA;IACrB,2CAAyB,CAAA;IACzB,6BAAW,CAAA;IACX,2CAAyB,CAAA;AAC7B,CAAC,EATW,cAAc,8BAAd,cAAc,QASzB;AAED,IAAY,WAKX;AALD,WAAY,WAAW;IACnB,4BAAa,CAAA;IACb,kCAAmB,CAAA;IACnB,4BAAa,CAAA;IACb,4DAA6C,CAAA;AACjD,CAAC,EALW,WAAW,2BAAX,WAAW,QAKtB;AAED,IAAY,cAOX;AAPD,WAAY,cAAc;IACtB,+BAAa,CAAA;IACb,qDAAmC,CAAA;IACnC,iDAA+B,CAAA;IAC/B,iDAA+B,CAAA;IAC/B,2CAAyB,CAAA;IACzB,+CAA6B,CAAA;AACjC,CAAC,EAPW,cAAc,8BAAd,cAAc,QAOzB;AAED,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;AAEhE,QAAA,qBAAqB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAU,CAAC;AACvF,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAS,6BAAqB,CAAC,CAAC;AAC9D,QAAA,cAAc,GAAG,CAAC,SAAS,EAAE,OAAO,CAAU,CAAC;AAC5D,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAS,sBAAc,CAAC,CAAC;AAE7D,MAAM,QAAS,SAAQ,sBAAY;CAiBlC;AAbG;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,IAAI;KAChB,CAAC;;uCACa;AAIf;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,IAAI;KAChB,CAAC;;wCACc;AAEhB;IADC,IAAA,cAAI,GAAE;;mDACoB;AAE3B;IADC,IAAA,cAAI,GAAE;;0CACY;AAEnB;IADC,IAAA,cAAI,GAAE;;6CACe;AAEtB;IADC,IAAA,cAAI,GAAE;;0CACY;AAkOhB,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,sBAAY;IAsJ5C,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAmB,CAAC;QACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,8BAAoB,CAAwB,CAAC;QAE5E,MAAM,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,gBAAgB,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;QAC7E,IAAI,UAAU,EAAE,CAAC;YACb,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAC;QACtC,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACvB,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QAC9D,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,IAAI,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxF,MAAM,IAAI,8BAAoB,CAAC;oBAC3B,IAAI,EAAE,aAAa;oBACnB,OAAO,EAAE,6CAA6C;iBACzD,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,MAAM,EAAE,CAAC;YACT,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;QAC/B,CAAC;QAED,MAAM,gBAAgB,GAAG,GAAG,EAAE,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACnB,QAAQ,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC1D,CAAC;QACD,MAAM,gBAAgB,GAAG,GAAG,EAAE,GAAG,CAAC,sBAAsB,CAAC,CAAC;QAC1D,IAAI,gBAAgB,EAAE,CAAC;YACnB,IAAI,gBAAgB,KAAK,KAAK,EAAE,CAAC;gBAC7B,QAAQ,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;YACjD,CAAC;iBAAM,CAAC;gBACJ,QAAQ,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;QACD,MAAM,iBAAiB,GAAG,GAAG,EAAE,GAAG,CAAC,uBAAuB,CAAC,CAAC;QAC5D,IAAI,iBAAiB,EAAE,CAAC;YACpB,QAAQ,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,YAAY,GAAG,GAAG,EAAE,GAAG,CAAC,iBAAiB,CAAC,CAAC;QACjD,IAAI,YAAY,IAAI,2BAA2B,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YAChE,QAAQ,CAAC,YAAY,GAAG,YAAmB,CAAC;QAChD,CAAC;QACD,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAC5B,QAAQ,CAAC,YAAY,GAAG,OAAO,CAAC;QACpC,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE,CAAC;YACV,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YAC5D,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,cAAc,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YACzB,QAAQ,CAAC,cAAc,GAAG,cAAc,CAAC;QAC7C,CAAC;QAED,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,KAAK,EAAE,CAAC;YACR,QAAQ,CAAC,KAAK,GAAG,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,cAAc,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YAC/C,QAAQ,CAAC,OAAO,GAAG,cAAc,IAAI,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC;QACpE,CAAC;aAAM,IAAI,GAAG,EAAE,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QAC5B,CAAC;QAED,MAAM,cAAc,GAAG,GAAG,EAAE,GAAG,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClF,QAAQ,CAAC,cAAc,KAAK,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QAChF,MAAM,cAAc,GAAG,GAAG,EAAE,GAAG,CAAC,mBAAmB,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClF,QAAQ,CAAC,cAAc,KAAK,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC;QAChF,MAAM,eAAe,GAAG,GAAG,EAAE,GAAG,CAAC,qBAAqB,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrF,QAAQ,CAAC,eAAe,KAAK,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,cAAc,CAAC;QAChH,MAAM,iBAAiB,GAAG,GAAG,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,SAAS,CAAC;QAChE,QAAQ,CAAC,SAAS,KAAK,iBAAiB,CAAC;QAEzC,MAAM,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,MAAM,EAAE,CAAC;YACT,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC;QAC7B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAClB,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QACpD,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC5B,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YACtC,QAAQ,CAAC,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC;QAC9C,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;YAC3C,QAAQ,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;YACnC,QAAQ,CAAC,WAAW,GAAG,cAAc,CAAC,SAAS,CAAC;QACpD,CAAC;QAED,MAAM,cAAc,GAAG,GAAG,EAAE,GAAG,CAAC,qBAAqB,CAAC,CAAC;QACvD,IAAI,cAAc,EAAE,CAAC;YACjB,QAAQ,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;QACtD,CAAC;QACD,MAAM,UAAU,GAAG,GAAG,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7C,IAAI,UAAU,EAAE,CAAC;YACb,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACjG,CAAC;QACD,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACtB,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC;QAC9B,CAAC;QACD,MAAM,aAAa,GAAG,GAAG,EAAE,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACpD,IAAI,aAAa,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzB,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC;QAC9B,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,gBAAgB,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,cAAc,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,IAAK,QAAQ,CAAC,UAA8B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7H,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAClC,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;gBACvC,OAAO,CAAC,IAAI,CAAC;oBACT,GAAG,IAAA,+BAAoB,EAAC,SAAS,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;iBAC7D,CAAC,CAAC;YACP,CAAC;QACL,CAAC;aAAM,IAAI,gBAAgB,IAAI,OAAO,gBAAgB,KAAK,QAAQ,EAAE,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC;gBACT,GAAG,IAAA,+BAAoB,EAAC,gBAAgB,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aACpE,CAAC,CAAC;QACP,CAAC;QACD,QAAQ,CAAC,UAAU,GAAG,OAAO,CAAC;QAE9B,MAAM,QAAQ,GAAG,GAAG,EAAE,GAAG,CAAC,aAAa,CAAC,CAAC;QACzC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,SAAS,CAAC;QAC5C,MAAM,KAAK,GAAG,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,QAAQ,CAAC,KAAK,KAAK,KAAK,IAAI,SAAS,CAAC;QACtC,MAAM,SAAS,GAAG,GAAG,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC;QAC3C,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC;QAE9C,MAAM,WAAW,GAAG,GAAG,EAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC/C,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,IAAI,SAAS,CAAC;QAElE,MAAM,QAAQ,GAAG,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,GAAG,QAAe,CAAC;QACpC,CAAC;QAED,MAAM,GAAG,GAAG,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;QAC5B,QAAQ,CAAC,UAAU,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;QAEtD,MAAM,aAAa,GAAG,GAAG,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;QACnD,IAAI,aAAa,EAAE,CAAC;YAChB,QAAQ,CAAC,aAAa,KAAK,aAA+B,CAAC;QAC/D,CAAC;QAED,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC1B,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7D,CAAC;QAED,IAAI,GAAG,EAAE,CAAC;YACN,QAAQ,CAAC,QAAQ,KAAK,qDAAwB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChE,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,IAAI,qBAAqB;QACrB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,aAAa,CAAC;QAC9B,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YACrC,OAAO,cAAc,CAAC,YAAY,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxE,OAAO,cAAc,CAAC,UAAU,CAAC;QACrC,CAAC;QAED,OAAO,cAAc,CAAC,aAAa,CAAC;IACxC,CAAC;IAED,oCAAoC,CAAC,QAAsB;QACvD,IAAI,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,cAAc,GAAG,IAAI,CAAC,qBAAqB,CAAC;QAClD,IAAI,cAAc,KAAK,cAAc,CAAC,UAAU,IAAI,QAAQ,CAAC,uBAAuB,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAChH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,uBAAuB,EAAE,QAAQ,CAAC,yBAAyB,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC;gBACpG,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACpG,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxG,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,cAAc,KAAK,cAAc,CAAC,aAAa,IAAI,QAAQ,CAAC,yBAAyB,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACrH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,IAAI,CAAC,QAAQ,CAAC,yBAAyB,GAAG,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC;gBACnD,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,cAAc,KAAK,cAAc,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAChF,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,cAAc,KAAK,cAAc,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC1D,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,cAAc,KAAK,cAAc,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,cAAc,KAAK,cAAc,CAAC,aAAa,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YAC/E,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAClC,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;IACpC,CAAC;IAED,sBAAsB;QAClB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;YAC1B,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,iCAAiC;QAC7B,OAAO,CAAC,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IAED,oBAAoB;QAChB,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5G,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7G,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,EAAE,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAC;YAClE,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAC/B,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACxC,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ,CAAA;AAjbY,wCAAc;AAGvB;IADC,IAAA,cAAI,GAAE;;2CACM;AAGb;IADC,IAAA,cAAI,GAAE;;4CACO;AAMd;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,oBAAoB;QAC1B,OAAO,EAAE,SAAS;KACrB,CAAC;;4CACmC;AAMrC;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,qBAAqB;QAC3B,IAAI,EAAE,CAAC,sBAAS,EAAE,MAAM,CAAC;KAC5B,CAAC;;2CACuB;AAMzB;IAJC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,cAAc,CAAC,OAAO;QAC/B,IAAI,EAAE,CAAC,cAAc,EAAE,MAAM,CAAC;KACjC,CAAC;;mDACmB;AAKrB;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;wDACyB;AAG3B;IADC,IAAA,cAAI,EAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC;;oDACR;AAKpD;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;wDACkC;AAKpC;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;yDAC0B;AAK5B;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;+CACgB;AAKlB;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;6CACuB;AAGzB;IADC,IAAA,cAAI,GAAE;;sDACiB;AAGxB;IADC,IAAA,cAAI,EAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;;sDACS;AAGnC;IADC,IAAA,cAAI,EAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;;uDACU;AAGpC;IADC,IAAA,cAAI,EAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;;sDACS;AAKnC;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;sDACuB;AAMzB;IAJC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC;KAC1B,CAAC;;kDAC8B;AAKhC;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,KAAK;KACjB,CAAC;;qDACsB;AAKxB;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,MAAM;KAClB,CAAC;;kDACoB;AAGtB;IADC,IAAA,cAAI,GAAE;;gDACW;AAGlB;IADC,IAAA,cAAI,GAAE;;6CACQ;AAGf;IADC,IAAA,cAAI,GAAE;;iDACY;AAGnB;IADC,IAAA,cAAI,GAAE;;8CACS;AAKhB;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,MAAM;KAClB,CAAC;;wDAC0B;AAK5B;IAHC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE,MAAM;KAClB,CAAC;;yDAC2B;AAO7B;IALC,IAAA,cAAI,EAAC;QACF,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG;QAC1C,IAAI,EAAE,MAAM;QACZ,QAAQ,EAAE,IAAI;KACjB,CAAC;;+CACsB;AAGxB;IADC,IAAA,cAAI,GAAE;;8CACS;AAGhB;IADC,IAAA,cAAI,GAAE;;+CACU;AAGjB;IADC,IAAA,cAAI,GAAE;;mDACc;AAGrB;IADC,IAAA,cAAI,GAAE;8BACI,QAAQ;gDAAC;AAGpB;IADC,IAAA,cAAI,GAAE;;mDACc;AAGrB;IADC,IAAA,cAAI,GAAE;;kDACa;AAGpB;IADC,IAAA,cAAI,GAAE;;iDACY;AAGnB;IADC,IAAA,cAAI,GAAE;;kDACoB;AAG3B;IADC,IAAA,cAAI,GAAE;8BACI,qDAAwB;gDAAC;AAKpC;IAHC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,cAAc;KACvB,CAAC;;qDAC6B;yBAlJtB,cAAc;IA/N1B,IAAA,cAAI,EAAC;QACF,OAAO,EAAE;YACL,SAAS,EAAE;gBACP,UAAU,EAAE;oBACR,QAAQ,EAAE;wBACN,WAAW,EAAE,wDAAwD;4BACjE,uBAAuB;4BACvB,uBAAuB;4BACvB,mCAAmC;4BACnC,cAAc;wBAElB,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,mBAAmB,EAAE;wBACjB,WAAW,EAAE,sFAAsF;wBACnG,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,YAAY,EAAE;wBACV,WAAW,EAAE,wGAAwG;wBACrH,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,gBAAgB,EAAE;wBACd,WAAW,EAAE,sEAAsE;4BAC/E,uBAAuB;4BACvB,cAAc;4BACd,UAAU;4BACV,UAAU;4BACV,cAAc;4BACd,gBAAgB;4BAChB,aAAa;4BACb,kCAAkC;4BAClC,iBAAiB;4BACjB,WAAW;4BACX,oBAAoB;wBAExB,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,qBAAqB,EAAE;wBACnB,WAAW,EAAE,8FAA8F;4BACvG,kDAAkD;wBAEtD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,mBAAmB,EAAE;wBACjB,WAAW,EAAE,0EAA0E;4BACnF,gDAAgD;wBAEpD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,mBAAmB,EAAE;wBACjB,WAAW,EAAE,qEAAqE;4BAC9E,kCAAkC;wBAEtC,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,qBAAqB,EAAE;wBACnB,WAAW,EAAE,sHAAsH;4BAC/H,qCAAqC;wBAEzC,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,aAAa,EAAE;wBACX,WAAW,EAAE,2DAA2D;4BACpE,yBAAyB;4BACzB,UAAU;4BACV,WAAW;4BACX,YAAY;4BACZ,cAAc;4BACd,iDAAiD;wBACrD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,SAAS,EAAE;wBACP,WAAW,EAAE,mFAAmF;wBAChG,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,cAAc,EAAE;wBACZ,WAAW,EAAE,yHAAyH;wBACtI,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,KAAK,EAAE;wBACH,WAAW,EAAE,mFAAmF;wBAChG,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,cAAc,EAAE;wBACZ,WAAW,EAAE,+DAA+D;4BACxE,6CAA6C;wBACjD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,sBAAsB,EAAE;wBACpB,WAAW,EAAE,qFAAqF;4BAC9F,0DAA0D;wBAC9D,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,uBAAuB,EAAE;wBACrB,WAAW,EAAE,0DAA0D;wBACvE,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,sBAAsB,EAAE;wBACpB,WAAW,EAAE,+DAA+D;wBAC5E,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,iBAAiB,EAAE;wBACf,WAAW,EAAE,4BAA4B;4BACrC,qBAAqB;4BACrB,qBAAqB;4BACrB,qBAAqB;4BACrB,wBAAwB;4BACxB,mDAAmD;4BACnD,mDAAmD;wBACvD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,eAAe,EAAE;wBACb,WAAW,EAAE,gEAAgE;wBAC7E,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,mBAAmB,EAAE;wBACjB,WAAW,EAAE,oEAAoE;wBACjF,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,cAAc,EAAE;wBACZ,WAAW,EAAE,sBAAsB;wBACnC,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,WAAW,EAAE;wBACT,WAAW,EAAE,sCAAsC;wBACnD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,UAAU,EAAE;wBACR,WAAW,EAAE,sCAAsC;wBACnD,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,WAAW,EAAE;wBACT,WAAW,EAAE,+BAA+B;wBAC5C,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,gBAAgB,EAAE;wBACd,WAAW,EAAE,yGAAyG;wBACtH,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,kBAAkB,EAAE;wBAChB,WAAW,EAAE,kEAAkE;4BAC3E,2CAA2C;4BAC3C,4EAA4E;4BAC5E,4FAA4F;4BAC5F,4HAA4H;4BAC5H,4GAA4G;4BAC5G,gFAAgF;wBACpF,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,UAAU,EAAE;wBACR,WAAW,EAAE,6FAA6F;wBAC1G,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,QAAQ,EAAE;wBACN,WAAW,EAAE,kEAAkE;wBAC/E,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,oBAAoB,EAAE;wBAClB,WAAW,EAAE,0LAA0L;wBACvM,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,SAAS,EAAE;wBACP,WAAW,EAAE,0JAA0J;wBACvK,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,yBAAyB,EAAE;wBACvB,WAAW,EAAE,2LAA2L;wBACxM,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,mBAAmB,EAAE;wBACjB,WAAW,EAAE,kLAAkL;wBAC/L,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,uBAAuB,EAAE;wBACrB,WAAW,EAAE,wLAAwL;wBACrM,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,iBAAiB,EAAE;wBACf,WAAW,EAAE,0MAA0M;wBACvN,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;oBACD,2BAA2B,EAAE;wBACzB,WAAW,EAAE,0NAA0N;wBACvO,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC7B;iBACJ;aACJ;SACJ;KACJ,CAAC;GACW,cAAc,CAib1B;AAED,MAAa,wBAAyB,SAAQ,cAAc;IACxD,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC;YACxB,CAAC,8BAAoB,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,8BAAoB,CAAC;SACnE,CAA6B,CAAC;QAE/B,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AARD,4DAQC"}

build/dto/jina-embeddings-auth.js

@@ -0,0 +1,237 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JinaEmbeddingsAuthDTO = void 0;
+const civ_rpc_1 = require("civkit/civ-rpc");
+const escape_1 = require("civkit/escape");
+const lang_1 = require("civkit/lang");
+const logger_1 = __importDefault(require("../services/logger"));
+const registry_1 = require("../services/registry");
+const async_context_1 = require("../services/async-context");
+const secrets_1 = __importDefault(require("../shared/services/secrets"));
+const jina_embeddings_1 = require("../shared/3rd-party/jina-embeddings");
+const jina_embeddings_token_account_1 = require("../shared/db/jina-embeddings-token-account");
+const errors_1 = require("../services/errors");
+const authDtoLogger = logger_1.default.child({ service: 'JinaAuthDTO' });
+const THE_VERY_SAME_JINA_EMBEDDINGS_CLIENT = new jina_embeddings_1.JinaEmbeddingsDashboardHTTP(secrets_1.default.JINA_EMBEDDINGS_DASHBOARD_API_KEY);
+let JinaEmbeddingsAuthDTO = class JinaEmbeddingsAuthDTO extends civ_rpc_1.AutoCastable {
+    constructor() {
+        super(...arguments);
+        this.jinaEmbeddingsDashboard = THE_VERY_SAME_JINA_EMBEDDINGS_CLIENT;
+    }
+    static from(input) {
+        const instance = super.from(input);
+        const ctx = input[civ_rpc_1.RPC_CALL_ENVIRONMENT];
+        if (ctx) {
+            const authorization = ctx.get('authorization');
+            if (authorization) {
+                const authToken = authorization.split(' ')[1] || authorization;
+                instance.bearerToken = authToken;
+            }
+        }
+        if (!instance.bearerToken && input._token) {
+            instance.bearerToken = input._token;
+        }
+        return instance;
+    }
+    async getBrief(ignoreCache) {
+        if (!this.bearerToken) {
+            throw new civ_rpc_1.AuthenticationRequiredError({
+                message: 'Jina API key is required to authenticate. Please get one from https://jina.ai'
+            });
+        }
+        let firestoreDegradation = false;
+        let account;
+        try {
+            account = await jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.fromFirestore(this.bearerToken);
+        }
+        catch (err) {
+            // FireStore would not accept any string as input and may throw if not happy with it
+            firestoreDegradation = true;
+            logger_1.default.warn(`Firestore issue`, { err });
+        }
+        const age = account?.lastSyncedAt ? Date.now() - account.lastSyncedAt.valueOf() : Infinity;
+        const jitter = Math.ceil(Math.random() * 30 * 1000);
+        if (account && !ignoreCache) {
+            if ((age < (180_000 - jitter)) && (account.wallet?.total_balance > 0)) {
+                this.user = account;
+                this.uid = this.user?.user_id;
+                return account;
+            }
+        }
+        if (firestoreDegradation) {
+            logger_1.default.debug(`Using remote UC cached user`);
+            let r;
+            try {
+                r = await this.jinaEmbeddingsDashboard.authorization(this.bearerToken);
+            }
+            catch (err) {
+                if (err?.status === 401) {
+                    throw new civ_rpc_1.AuthenticationFailedError({
+                        message: 'Invalid API key, please get a new one from https://jina.ai'
+                    });
+                }
+                logger_1.default.warn(`Failed load remote cached user: ${err}`, { err });
+                throw new civ_rpc_1.DownstreamServiceError(`Failed to authenticate: ${err}`);
+            }
+            const brief = r?.data;
+            const draftAccount = jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.from({
+                ...account, ...brief, _id: this.bearerToken,
+                lastSyncedAt: new Date()
+            });
+            this.user = draftAccount;
+            this.uid = this.user?.user_id;
+            return draftAccount;
+        }
+        try {
+            // TODO: go back using validateToken after performance issue fixed
+            const r = ((account?.wallet?.total_balance || 0) > 0) ?
+                await this.jinaEmbeddingsDashboard.authorization(this.bearerToken) :
+                await this.jinaEmbeddingsDashboard.validateToken(this.bearerToken);
+            const brief = r.data;
+            const draftAccount = jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.from({
+                ...account, ...brief, _id: this.bearerToken,
+                lastSyncedAt: new Date()
+            });
+            await jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.save(draftAccount.degradeForFireStore(), undefined, { merge: true });
+            this.user = draftAccount;
+            this.uid = this.user?.user_id;
+            return draftAccount;
+        }
+        catch (err) {
+            authDtoLogger.warn(`Failed to get user brief: ${err}`, { err: (0, lang_1.marshalErrorLike)(err) });
+            if (err?.status === 401) {
+                throw new civ_rpc_1.AuthenticationFailedError({
+                    message: 'Invalid API key, please get a new one from https://jina.ai'
+                });
+            }
+            if (account) {
+                this.user = account;
+                this.uid = this.user?.user_id;
+                return account;
+            }
+            throw new civ_rpc_1.DownstreamServiceError(`Failed to authenticate: ${err}`);
+        }
+    }
+    async reportUsage(tokenCount, mdl, endpoint = '/encode') {
+        const user = await this.assertUser();
+        const uid = user.user_id;
+        user.wallet.total_balance -= tokenCount;
+        return this.jinaEmbeddingsDashboard.reportUsage(this.bearerToken, {
+            model_name: mdl,
+            api_endpoint: endpoint,
+            consumer: {
+                id: uid,
+                user_id: uid,
+            },
+            usage: {
+                total_tokens: tokenCount
+            },
+            labels: {
+                model_name: mdl
+            }
+        }).then((r) => {
+            jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.COLLECTION.doc(this.bearerToken)
+                .update({ 'wallet.total_balance': jina_embeddings_token_account_1.JinaEmbeddingsTokenAccount.OPS.increment(-tokenCount) })
+                .catch((err) => {
+                authDtoLogger.warn(`Failed to update cache for ${uid}: ${err}`, { err: (0, lang_1.marshalErrorLike)(err) });
+            });
+            return r;
+        }).catch((err) => {
+            user.wallet.total_balance += tokenCount;
+            authDtoLogger.warn(`Failed to report usage for ${uid}: ${err}`, { err: (0, lang_1.marshalErrorLike)(err) });
+        });
+    }
+    async solveUID() {
+        if (this.uid) {
+            this.ctxMgr.set('uid', this.uid);
+            return this.uid;
+        }
+        if (this.bearerToken) {
+            await this.getBrief();
+            this.ctxMgr.set('uid', this.uid);
+            return this.uid;
+        }
+        return undefined;
+    }
+    async assertUID() {
+        const uid = await this.solveUID();
+        if (!uid) {
+            throw new civ_rpc_1.AuthenticationRequiredError('Authentication failed');
+        }
+        return uid;
+    }
+    async assertUser() {
+        if (this.user) {
+            return this.user;
+        }
+        await this.getBrief();
+        return this.user;
+    }
+    async assertTier(n, feature) {
+        let user;
+        try {
+            user = await this.assertUser();
+        }
+        catch (err) {
+            if (err instanceof civ_rpc_1.AuthenticationRequiredError) {
+                throw new civ_rpc_1.AuthenticationRequiredError({
+                    message: `Authentication is required to use this feature${feature ? ` (${feature})` : ''}. Please provide a valid API key.`
+                });
+            }
+            throw err;
+        }
+        const tier = parseInt(user.metadata?.speed_level);
+        if (isNaN(tier) || tier < n) {
+            throw new errors_1.TierFeatureConstraintError({
+                message: `Your current plan does not support this feature${feature ? ` (${feature})` : ''}. Please upgrade your plan.`
+            });
+        }
+        return true;
+    }
+    getRateLimits(...tags) {
+        const descs = tags.map((x) => this.user?.customRateLimits?.[x] || []).flat().filter((x) => x.isEffective());
+        if (descs.length) {
+            return descs;
+        }
+        return undefined;
+    }
+};
+exports.JinaEmbeddingsAuthDTO = JinaEmbeddingsAuthDTO;
+__decorate([
+    (0, registry_1.InjectProperty)(async_context_1.AsyncLocalContext),
+    __metadata("design:type", async_context_1.AsyncLocalContext)
+], JinaEmbeddingsAuthDTO.prototype, "ctxMgr", void 0);
+exports.JinaEmbeddingsAuthDTO = JinaEmbeddingsAuthDTO = __decorate([
+    (0, civ_rpc_1.Also)({
+        openapi: {
+            operation: {
+                parameters: {
+                    'Authorization': {
+                        description: (0, escape_1.htmlEscape) `Jina Token for authentication.\n\n` +
+                            (0, escape_1.htmlEscape) `- Member of <JinaEmbeddingsAuthDTO>\n\n` +
+                            `- Authorization: Bearer {YOUR_JINA_TOKEN}`,
+                        in: 'header',
+                        schema: {
+                            anyOf: [
+                                { type: 'string', format: 'token' }
+                            ]
+                        }
+                    }
+                }
+            }
+        }
+    })
+], JinaEmbeddingsAuthDTO);
+//# sourceMappingURL=jina-embeddings-auth.js.map

build/dto/jina-embeddings-auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jina-embeddings-auth.js","sourceRoot":"","sources":["../../src/dto/jina-embeddings-auth.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AACA,4CAKwB;AACxB,0CAA2C;AAC3C,sCAA+C;AAI/C,gEAAwC;AACxC,mDAAsD;AACtD,6DAA8D;AAE9D,yEAAmD;AACnD,yEAAkF;AAClF,8FAAwF;AACxF,+CAAgE;AAEhE,MAAM,aAAa,GAAG,gBAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;AAG/D,MAAM,oCAAoC,GAAG,IAAI,6CAA2B,CAAC,iBAAS,CAAC,iCAAiC,CAAC,CAAC;AAsBnH,IAAM,qBAAqB,GAA3B,MAAM,qBAAsB,SAAQ,sBAAY;IAAhD;;QAQH,4BAAuB,GAAG,oCAAoC,CAAC;IA0NnE,CAAC;IAxNG,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAA0B,CAAC;QAE5D,MAAM,GAAG,GAAG,KAAK,CAAC,8BAAoB,CAAY,CAAC;QAEnD,IAAI,GAAG,EAAE,CAAC;YACN,MAAM,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;YAE/C,IAAI,aAAa,EAAE,CAAC;gBAChB,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC;gBAC/D,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAC;YACrC,CAAC;QAEL,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACxC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC;QACxC,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,WAA8B;QACzC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACpB,MAAM,IAAI,qCAA2B,CAAC;gBAClC,OAAO,EAAE,+EAA+E;aAC3F,CAAC,CAAC;QACP,CAAC;QAED,IAAI,oBAAoB,GAAG,KAAK,CAAC;QACjC,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,0DAA0B,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,oFAAoF;YACpF,oBAAoB,GAAG,IAAI,CAAC;YAC5B,gBAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC5C,CAAC;QAGD,MAAM,GAAG,GAAG,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAEpD,IAAI,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;YAC1B,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;gBACpE,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;gBACpB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;gBAE9B,OAAO,OAAO,CAAC;YACnB,CAAC;QACL,CAAC;QAED,IAAI,oBAAoB,EAAE,CAAC;YACvB,gBAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC5C,IAAI,CAAC,CAAC;YACN,IAAI,CAAC;gBACD,CAAC,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC3E,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,GAAG,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;oBACtB,MAAM,IAAI,mCAAyB,CAAC;wBAChC,OAAO,EAAE,4DAA4D;qBACxE,CAAC,CAAC;gBACP,CAAC;gBACD,gBAAM,CAAC,IAAI,CAAC,mCAAmC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC/D,MAAM,IAAI,gCAAsB,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,MAAM,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC;YACtB,MAAM,YAAY,GAAG,0DAA0B,CAAC,IAAI,CAAC;gBACjD,GAAG,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW;gBAC3C,YAAY,EAAE,IAAI,IAAI,EAAE;aAC3B,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;YACzB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;YAE9B,OAAO,YAAY,CAAC;QACxB,CAAC;QAED,IAAI,CAAC;YACD,kEAAkE;YAClE,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;gBACpE,MAAM,IAAI,CAAC,uBAAuB,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACvE,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC;YACrB,MAAM,YAAY,GAAG,0DAA0B,CAAC,IAAI,CAAC;gBACjD,GAAG,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW;gBAC3C,YAAY,EAAE,IAAI,IAAI,EAAE;aAC3B,CAAC,CAAC;YACH,MAAM,0DAA0B,CAAC,IAAI,CAAC,YAAY,CAAC,mBAAmB,EAAE,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAEtG,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;YACzB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;YAE9B,OAAO,YAAY,CAAC;QACxB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,aAAa,CAAC,IAAI,CAAC,6BAA6B,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAEvF,IAAI,GAAG,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;gBACtB,MAAM,IAAI,mCAAyB,CAAC;oBAChC,OAAO,EAAE,4DAA4D;iBACxE,CAAC,CAAC;YACP,CAAC;YAED,IAAI,OAAO,EAAE,CAAC;gBACV,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;gBACpB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC;gBAE9B,OAAO,OAAO,CAAC;YACnB,CAAC;YAGD,MAAM,IAAI,gCAAsB,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;QACvE,CAAC;IACL,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAAkB,EAAE,GAAW,EAAE,WAAmB,SAAS;QAC3E,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;QAExC,OAAO,IAAI,CAAC,uBAAuB,CAAC,WAAW,CAAC,IAAI,CAAC,WAAY,EAAE;YAC/D,UAAU,EAAE,GAAG;YACf,YAAY,EAAE,QAAQ;YACtB,QAAQ,EAAE;gBACN,EAAE,EAAE,GAAG;gBACP,OAAO,EAAE,GAAG;aACf;YACD,KAAK,EAAE;gBACH,YAAY,EAAE,UAAU;aAC3B;YACD,MAAM,EAAE;gBACJ,UAAU,EAAE,GAAG;aAClB;SACJ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;YACV,0DAA0B,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAY,CAAC;iBACvD,MAAM,CAAC,EAAE,sBAAsB,EAAE,0DAA0B,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;iBACzF,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,aAAa,CAAC,IAAI,CAAC,8BAA8B,GAAG,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACpG,CAAC,CAAC,CAAC;YAEP,OAAO,CAAC,CAAC;QACb,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,UAAU,CAAC;YACxC,aAAa,CAAC,IAAI,CAAC,8BAA8B,GAAG,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,uBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAEjC,OAAO,IAAI,CAAC,GAAG,CAAC;QACpB,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAEjC,OAAO,IAAI,CAAC,GAAG,CAAC;QACpB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,SAAS;QACX,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAElC,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,MAAM,IAAI,qCAA2B,CAAC,uBAAuB,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,IAAI,CAAC;QACrB,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEtB,OAAO,IAAI,CAAC,IAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,CAAS,EAAE,OAAgB;QACxC,IAAI,IAAI,CAAC;QACT,IAAI,CAAC;YACD,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACnC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,qCAA2B,EAAE,CAAC;gBAC7C,MAAM,IAAI,qCAA2B,CAAC;oBAClC,OAAO,EAAE,iDAAiD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,mCAAmC;iBAC9H,CAAC,CAAC;YACP,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAClD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,mCAA0B,CAAC;gBACjC,OAAO,EAAE,kDAAkD,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,6BAA6B;aACzH,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,aAAa,CAAC,GAAG,IAAc;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAE5G,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACf,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ,CAAA;AAlOY,sDAAqB;AAM9B;IADC,IAAA,yBAAc,EAAC,iCAAiB,CAAC;8BACzB,iCAAiB;qDAAC;gCANlB,qBAAqB;IApBjC,IAAA,cAAI,EAAC;QACF,OAAO,EAAE;YACL,SAAS,EAAE;gBACP,UAAU,EAAE;oBACR,eAAe,EAAE;wBACb,WAAW,EAAE,IAAA,mBAAU,EAAA,oCAAoC;4BACvD,IAAA,mBAAU,EAAA,yCAAyC;4BACnD,2CAA2C;wBAE/C,EAAE,EAAE,QAAQ;wBACZ,MAAM,EAAE;4BACJ,KAAK,EAAE;gCACH,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE;6BACtC;yBACJ;qBACJ;iBACJ;aACJ;SACJ;KACJ,CAAC;GACW,qBAAqB,CAkOjC"}

build/dto/turndown-tweakable-options.js

@@ -0,0 +1,81 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TurnDownTweakableOptions = void 0;
+const civ_rpc_1 = require("civkit/civ-rpc");
+const lodash_1 = __importDefault(require("lodash"));
+class TurnDownTweakableOptions extends civ_rpc_1.AutoCastable {
+    static fromCtx(ctx, prefix = 'x-md-') {
+        const draft = {};
+        for (const [k, v] of Object.entries(ctx.headers)) {
+            if (k.startsWith(prefix)) {
+                const prop = k.slice(prefix.length);
+                const sk = lodash_1.default.camelCase(prop);
+                draft[sk] = v;
+            }
+        }
+        return this.from(draft);
+    }
+}
+exports.TurnDownTweakableOptions = TurnDownTweakableOptions;
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > headingStyle',
+        type: new Set(['setext', 'atx']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "headingStyle", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > hr',
+        validate: (v) => v.length > 0 && v.length <= 128
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "hr", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > bulletListMarker',
+        type: new Set(['-', '+', '*']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "bulletListMarker", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > emDelimiter',
+        type: new Set(['_', '*']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "emDelimiter", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > strongDelimiter',
+        type: new Set(['__', '**']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "strongDelimiter", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > linkStyle',
+        type: new Set(['inlined', 'referenced', 'discarded']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "linkStyle", void 0);
+__decorate([
+    (0, civ_rpc_1.Prop)({
+        desc: 'Turndown options > linkReferenceStyle',
+        type: new Set(['full', 'collapsed', 'shortcut', 'discarded']),
+    }),
+    __metadata("design:type", String)
+], TurnDownTweakableOptions.prototype, "linkReferenceStyle", void 0);
+//# sourceMappingURL=turndown-tweakable-options.js.map

build/dto/turndown-tweakable-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"turndown-tweakable-options.js","sourceRoot":"","sources":["../../src/dto/turndown-tweakable-options.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,4CAAoD;AAEpD,oDAAuB;AAGvB,MAAa,wBAAyB,SAAQ,sBAAY;IA2CtD,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,GAAE,OAAO;QACxC,MAAM,KAAK,GAA2B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBACvB,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,EAAE,GAAG,gBAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC7B,KAAK,CAAC,EAAE,CAAC,GAAG,CAAW,CAAC;YAC5B,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;CACJ;AAvDD,4DAuDC;AAlDG;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,iCAAiC;QACvC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;KACnC,CAAC;;8DAC8B;AAMhC;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,uBAAuB;QAC7B,QAAQ,EAAE,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;KAC3D,CAAC;;oDACU;AAMZ;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,qCAAqC;QAC3C,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;KACjC,CAAC;;kEACiC;AAMnC;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,gCAAgC;QACtC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;KAC5B,CAAC;;6DACsB;AAMxB;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,oCAAoC;QAC1C,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;KAC9B,CAAC;;iEAC4B;AAM9B;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,8BAA8B;QACpC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;KACxD,CAAC;;2DACiD;AAMnD;IAJC,IAAA,cAAI,EAAC;QACF,IAAI,EAAE,uCAAuC;QAC7C,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;KAChE,CAAC;;oEACmE"}

build/lib/transform-server-event-stream.js

@@ -0,0 +1,151 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OutputServerEventStream = exports.InputServerEventStream = void 0;
+const civkit_1 = require("civkit");
+const stream_1 = require("stream");
+class InputServerEventStream extends stream_1.Transform {
+    constructor(options) {
+        super({
+            ...options,
+            readableObjectMode: true
+        });
+        this.cache = [];
+    }
+    decodeRoutine() {
+        if (!this.cache.length) {
+            return;
+        }
+        const vecs = this.cache.join('').split(/\r?\n\r?\n/);
+        this.cache.length = 0;
+        const lastVec = vecs.pop();
+        if (lastVec) {
+            this.cache.push(lastVec);
+        }
+        for (const x of vecs) {
+            const lines = x.split(/\r?\n/);
+            const event = {};
+            for (const l of lines) {
+                const columnPos = l.indexOf(':');
+                if (columnPos <= 0) {
+                    continue;
+                }
+                const key = l.substring(0, columnPos);
+                const rawValue = l.substring(columnPos + 1);
+                const value = rawValue.startsWith(' ') ? rawValue.slice(1) : rawValue;
+                if (key === 'data') {
+                    if (event.data) {
+                        event.data += value || '\n';
+                    }
+                    else if (event.data === '') {
+                        event.data += '\n';
+                        event.data += value || '\n';
+                    }
+                    else {
+                        event.data = value;
+                    }
+                }
+                else if (key === 'retry') {
+                    event.retry = parseInt(value, 10);
+                }
+                else {
+                    Reflect.set(event, key, value);
+                }
+            }
+            if (event.data) {
+                const parsed = (0, civkit_1.parseJSONText)(event.data);
+                if (parsed && typeof parsed === 'object') {
+                    event.data = parsed;
+                }
+            }
+            if (Object.keys(event).length) {
+                this.push(event);
+            }
+        }
+    }
+    _transform(chunk, encoding, callback) {
+        if (chunk === null) {
+            this.push(null);
+        }
+        this.cache.push(chunk.toString());
+        this.decodeRoutine();
+        callback();
+    }
+    _final(callback) {
+        this.decodeRoutine();
+        callback();
+    }
+}
+exports.InputServerEventStream = InputServerEventStream;
+let OutputServerEventStream = class OutputServerEventStream extends stream_1.Transform {
+    constructor(options) {
+        super({
+            ...options, writableObjectMode: true, encoding: 'utf-8'
+        });
+        this.n = 0;
+    }
+    encodeRoutine(chunk) {
+        if (typeof chunk === 'object') {
+            const lines = [];
+            if (chunk.event) {
+                lines.push(`event: ${chunk.event}`);
+            }
+            if (chunk.data) {
+                if (typeof chunk.data === 'string') {
+                    for (const x of chunk.data.split(/\r?\n/)) {
+                        lines.push(`data: ${x}`);
+                    }
+                }
+                else {
+                    lines.push(`data: ${JSON.stringify(chunk.data)}`);
+                }
+            }
+            if (chunk.id) {
+                lines.push(`id: ${chunk.id}`);
+            }
+            if (chunk.retry) {
+                lines.push(`retry: ${chunk.retry}`);
+            }
+            if (!lines.length) {
+                lines.push(`data: ${JSON.stringify(chunk)}`);
+            }
+            this.push(lines.join('\n'));
+            this.push('\n\n');
+            this.n++;
+            return;
+        }
+        else if (typeof chunk === 'string') {
+            const lines = [];
+            for (const x of chunk.split(/\r?\n/)) {
+                lines.push(`data: ${x}`);
+            }
+            this.push(lines.join('\n'));
+            this.push('\n\n');
+            this.n++;
+        }
+    }
+    _transform(chunk, encoding, callback) {
+        if (chunk === null) {
+            this.push(null);
+        }
+        this.encodeRoutine(chunk);
+        callback();
+    }
+};
+exports.OutputServerEventStream = OutputServerEventStream;
+exports.OutputServerEventStream = OutputServerEventStream = __decorate([
+    (0, civkit_1.TPM)({
+        contentType: 'text/event-stream',
+    }),
+    __metadata("design:paramtypes", [typeof (_a = typeof stream_1.TransformOptions !== "undefined" && stream_1.TransformOptions) === "function" ? _a : Object])
+], OutputServerEventStream);
+//# sourceMappingURL=transform-server-event-stream.js.map

build/lib/transform-server-event-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform-server-event-stream.js","sourceRoot":"","sources":["../../src/lib/transform-server-event-stream.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAA4C;AAC5C,mCAAwE;AAExE,MAAa,sBAAuB,SAAQ,kBAAS;IAGjD,YAAY,OAA0B;QAClC,KAAK,CAAC;YACF,GAAG,OAAO;YACV,kBAAkB,EAAE,IAAI;SAC3B,CAAC,CAAC;QANP,UAAK,GAAa,EAAE,CAAC;IAOrB,CAAC;IAED,aAAa;QACT,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACrB,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,IAAI,OAAO,EAAE,CAAC;YACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnB,MAAM,KAAK,GAAa,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEzC,MAAM,KAAK,GAKP,EAAE,CAAC;YAEP,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;gBACpB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;oBACjB,SAAS;gBACb,CAAC;gBACD,MAAM,GAAG,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;gBACtE,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;oBACjB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;wBACb,KAAK,CAAC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC;oBAChC,CAAC;yBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;wBAC3B,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC;wBACnB,KAAK,CAAC,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC;oBAChC,CAAC;yBAAM,CAAC;wBACJ,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC;oBACvB,CAAC;gBACL,CAAC;qBAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;oBACzB,KAAK,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtC,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;gBACnC,CAAC;YACL,CAAC;YAED,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,MAAM,MAAM,GAAG,IAAA,sBAAa,EAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;oBACvC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC;gBACxB,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;IACL,CAAC;IAEQ,UAAU,CAAC,KAAU,EAAE,QAAwB,EAAE,QAA2B;QACjF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,QAAQ,EAAE,CAAC;IACf,CAAC;IAEQ,MAAM,CAAC,QAAoD;QAChE,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,QAAQ,EAAE,CAAC;IACf,CAAC;CACJ;AApFD,wDAoFC;AAKM,IAAM,uBAAuB,GAA7B,MAAM,uBAAwB,SAAQ,kBAAS;IAGlD,YAAY,OAA0B;QAClC,KAAK,CAAC;YACF,GAAG,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO;SAC1D,CAAC,CAAC;QALP,MAAC,GAAW,CAAC,CAAC;IAMd,CAAC;IAED,aAAa,CAAC,KAKJ;QACN,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAa,EAAE,CAAC;YAE3B,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACjC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;wBACxC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtD,CAAC;YACL,CAAC;YACD,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC;gBACX,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACd,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACjD,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,EAAE,CAAC;YAET,OAAO;QACX,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAC7B,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClB,IAAI,CAAC,CAAC,EAAE,CAAC;QACb,CAAC;IACL,CAAC;IAEQ,UAAU,CAAC,KAAU,EAAE,QAAwB,EAAE,QAA2B;QACjF,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAE1B,QAAQ,EAAE,CAAC;IACf,CAAC;CACJ,CAAA;AAjEY,0DAAuB;kCAAvB,uBAAuB;IAHnC,IAAA,YAAG,EAAC;QACD,WAAW,EAAE,mBAAmB;KACnC,CAAC;yDAIwB,yBAAgB,oBAAhB,yBAAgB;GAH7B,uBAAuB,CAiEnC"}

build/services/alt-text.js

@@ -0,0 +1,131 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AltTextService = void 0;
+const civkit_1 = require("civkit");
+const tsyringe_1 = require("tsyringe");
+const logger_1 = require("./logger");
+const canvas_1 = require("./canvas");
+const common_iminterrogate_1 = require("../shared/services/common-iminterrogate");
+const img_alt_1 = require("../db/img-alt");
+const async_context_1 = require("./async-context");
+const md5Hasher = new civkit_1.HashManager('md5', 'hex');
+let AltTextService = class AltTextService extends civkit_1.AsyncService {
+    constructor(globalLogger, imageInterrogator, canvasService, asyncLocalContext) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.imageInterrogator = imageInterrogator;
+        this.canvasService = canvasService;
+        this.asyncLocalContext = asyncLocalContext;
+        this.altsToIgnore = 'image,img,photo,picture,pic,alt,figure,fig'.split(',');
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+    }
+    async caption(url) {
+        try {
+            const img = await this.canvasService.loadImage(url);
+            const contentTypeHint = Reflect.get(img, 'contentType');
+            if (Math.min(img.naturalHeight, img.naturalWidth) <= 1) {
+                return `A ${img.naturalWidth}x${img.naturalHeight} image, likely be a tacker probe`;
+            }
+            if (Math.min(img.naturalHeight, img.naturalWidth) < 64) {
+                return `A ${img.naturalWidth}x${img.naturalHeight} small image, likely a logo, icon or avatar`;
+            }
+            const resized = this.canvasService.fitImageToSquareBox(img, 1024);
+            const exported = await this.canvasService.canvasToBuffer(resized, 'image/png');
+            const svgHint = contentTypeHint.includes('svg') ? `Beware this image is a SVG rendered on a gray background, the gray background is not part of the image.\n\n` : '';
+            const svgSystemHint = contentTypeHint.includes('svg') ? ` Sometimes the system renders SVG on a gray background. When this happens, you must not include the gray background in the description.` : '';
+            const r = await this.imageInterrogator.interrogate('vertex-gemini-2.0-flash', {
+                image: exported,
+                prompt: `${svgHint}Give a concise image caption descriptive sentence in third person. Start directly with the description.`,
+                system: `You are BLIP2, an image caption model. You will generate Alt Text (in web pages) for any image for a11y purposes. You must not start with "This image is sth...", instead, start direly with "sth..."${svgSystemHint}`,
+            });
+            return r.replaceAll(/[\n\"]|(\.\s*$)/g, '').trim();
+        }
+        catch (err) {
+            throw new civkit_1.AssertionFailureError({ message: `Could not generate alt text for url ${url}`, cause: err });
+        }
+    }
+    async getAltText(imgBrief) {
+        if (!imgBrief.src) {
+            return undefined;
+        }
+        if (imgBrief.alt && !this.altsToIgnore.includes(imgBrief.alt.trim().toLowerCase())) {
+            return imgBrief.alt;
+        }
+        const digest = md5Hasher.hash(imgBrief.src);
+        const shortDigest = Buffer.from(digest, 'hex').toString('base64url');
+        let dims = [];
+        do {
+            if (imgBrief.loaded) {
+                if (imgBrief.naturalWidth && imgBrief.naturalHeight) {
+                    if (Math.min(imgBrief.naturalWidth, imgBrief.naturalHeight) < 64) {
+                        dims = [imgBrief.naturalWidth, imgBrief.naturalHeight];
+                        break;
+                    }
+                }
+            }
+            if (imgBrief.width && imgBrief.height) {
+                if (Math.min(imgBrief.width, imgBrief.height) < 64) {
+                    dims = [imgBrief.width, imgBrief.height];
+                    break;
+                }
+            }
+        } while (false);
+        if (Math.min(...dims) <= 1) {
+            return `A ${dims[0]}x${dims[1]} image, likely be a tacker probe`;
+        }
+        if (Math.min(...dims) < 64) {
+            return `A ${dims[0]}x${dims[1]} small image, likely a logo, icon or avatar`;
+        }
+        const existing = await img_alt_1.ImgAlt.fromFirestore(shortDigest);
+        if (existing) {
+            return existing.generatedAlt || existing.originalAlt || '';
+        }
+        let generatedCaption = '';
+        try {
+            generatedCaption = await this.caption(imgBrief.src);
+        }
+        catch (err) {
+            this.logger.warn(`Unable to generate alt text for ${imgBrief.src}`, { err });
+        }
+        if (this.asyncLocalContext.ctx.DNT) {
+            // Don't cache alt text if DNT is set
+            return generatedCaption;
+        }
+        // Don't try again until the next day
+        const expireMixin = generatedCaption ? {} : { expireAt: new Date(Date.now() + 1000 * 3600 * 24) };
+        await img_alt_1.ImgAlt.COLLECTION.doc(shortDigest).set({
+            _id: shortDigest,
+            src: imgBrief.src || '',
+            width: imgBrief.naturalWidth || 0,
+            height: imgBrief.naturalHeight || 0,
+            urlDigest: digest,
+            originalAlt: imgBrief.alt || '',
+            generatedAlt: generatedCaption || '',
+            createdAt: new Date(),
+            ...expireMixin
+        }, { merge: true });
+        return generatedCaption;
+    }
+};
+exports.AltTextService = AltTextService;
+exports.AltTextService = AltTextService = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger, typeof (_a = typeof common_iminterrogate_1.ImageInterrogationManager !== "undefined" && common_iminterrogate_1.ImageInterrogationManager) === "function" ? _a : Object, canvas_1.CanvasService,
+        async_context_1.AsyncLocalContext])
+], AltTextService);
+;
+//# sourceMappingURL=alt-text.js.map

build/services/alt-text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"alt-text.js","sourceRoot":"","sources":["../../src/services/alt-text.ts"],"names":[],"mappings":";;;;;;;;;;;;;AAAA,mCAA0E;AAC1E,uCAAqC;AACrC,qCAAwC;AACxC,qCAAyC;AACzC,kFAAoF;AAEpF,2CAAuC;AACvC,mDAAoD;AAEpD,MAAM,SAAS,GAAG,IAAI,oBAAW,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAGzC,IAAM,cAAc,GAApB,MAAM,cAAe,SAAQ,qBAAY;IAK5C,YACc,YAA0B,EAC1B,iBAA4C,EAC5C,aAA4B,EAC5B,iBAAoC;QAE9C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QALV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,sBAAiB,GAAjB,iBAAiB,CAA2B;QAC5C,kBAAa,GAAb,aAAa,CAAe;QAC5B,sBAAiB,GAAjB,iBAAiB,CAAmB;QAPlD,iBAAY,GAAG,4CAA4C,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvE,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IASrE,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACrB,IAAI,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACpD,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YACxD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrD,OAAO,KAAK,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,aAAa,kCAAkC,CAAC;YACxF,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,CAAC;gBACrD,OAAO,KAAK,GAAG,CAAC,YAAY,IAAI,GAAG,CAAC,aAAa,6CAA6C,CAAC;YACnG,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YAClE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAE/E,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,6GAA6G,CAAC,CAAC,CAAC,EAAE,CAAC;YACrK,MAAM,aAAa,GAAG,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yIAAyI,CAAC,CAAC,CAAC,EAAE,CAAC;YAEvM,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,yBAAyB,EAAE;gBAC1E,KAAK,EAAE,QAAQ;gBACf,MAAM,EAAE,GAAG,OAAO,yGAAyG;gBAC3H,MAAM,EAAE,wMAAwM,aAAa,EAAE;aAClO,CAAC,CAAC;YAEH,OAAO,CAAC,CAAC,UAAU,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,8BAAqB,CAAC,EAAE,OAAO,EAAE,uCAAuC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3G,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,QAAkB;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAChB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,QAAQ,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACjF,OAAO,QAAQ,CAAC,GAAG,CAAC;QACxB,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QACrE,IAAI,IAAI,GAAa,EAAE,CAAC;QACxB,GAAG,CAAC;YACA,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,IAAI,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;oBAClD,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,aAAa,CAAC,GAAG,EAAE,EAAE,CAAC;wBAC/D,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC;wBACvD,MAAM;oBACV,CAAC;gBACL,CAAC;YACL,CAAC;YAED,IAAI,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACpC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,CAAC;oBACjD,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACzC,MAAM;gBACV,CAAC;YACL,CAAC;QAEL,CAAC,QAAQ,KAAK,EAAE;QAEhB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,kCAAkC,CAAC;QACrE,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YACzB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,6CAA6C,CAAC;QAChF,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,gBAAM,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAEzD,IAAI,QAAQ,EAAE,CAAC;YACX,OAAO,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC;QAC/D,CAAC;QAED,IAAI,gBAAgB,GAAG,EAAE,CAAC;QAE1B,IAAI,CAAC;YACD,gBAAgB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,QAAQ,CAAC,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACjC,qCAAqC;YACrC,OAAO,gBAAgB,CAAC;QAC5B,CAAC;QAED,qCAAqC;QACrC,MAAM,WAAW,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,EAAE,CAAC;QAElG,MAAM,gBAAM,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAG,CACxC;YACI,GAAG,EAAE,WAAW;YAChB,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE;YACvB,KAAK,EAAE,QAAQ,CAAC,YAAY,IAAI,CAAC;YACjC,MAAM,EAAE,QAAQ,CAAC,aAAa,IAAI,CAAC;YACnC,SAAS,EAAE,MAAM;YACjB,WAAW,EAAE,QAAQ,CAAC,GAAG,IAAI,EAAE;YAC/B,YAAY,EAAE,gBAAgB,IAAI,EAAE;YACpC,SAAS,EAAE,IAAI,IAAI,EAAE;YACrB,GAAG,WAAW;SACjB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CACrB,CAAC;QAEF,OAAO,gBAAgB,CAAC;IAC5B,CAAC;CACJ,CAAA;AAzHY,wCAAc;yBAAd,cAAc;IAD1B,IAAA,oBAAS,GAAE;qCAOoB,qBAAY,sBACP,gDAAyB,oBAAzB,gDAAyB,gCAC7B,sBAAa;QACT,iCAAiB;GATzC,cAAc,CAyH1B;AAAA,CAAC"}

build/services/async-context.js

@@ -0,0 +1,21 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AsyncLocalContext = void 0;
+const async_context_1 = require("civkit/async-context");
+const tsyringe_1 = require("tsyringe");
+let AsyncLocalContext = class AsyncLocalContext extends async_context_1.GlobalAsyncContext {
+};
+exports.AsyncLocalContext = AsyncLocalContext;
+exports.AsyncLocalContext = AsyncLocalContext = __decorate([
+    (0, tsyringe_1.singleton)()
+], AsyncLocalContext);
+const instance = tsyringe_1.container.resolve(AsyncLocalContext);
+Reflect.set(process, 'asyncLocalContext', instance);
+exports.default = instance;
+//# sourceMappingURL=async-context.js.map

build/services/async-context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"async-context.js","sourceRoot":"","sources":["../../src/services/async-context.ts"],"names":[],"mappings":";;;;;;;;;AAAA,wDAA0D;AAC1D,uCAAgD;AAGzC,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,kCAAkB;CAAI,CAAA;AAAhD,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,oBAAS,GAAE;GACC,iBAAiB,CAA+B;AAE7D,MAAM,QAAQ,GAAG,oBAAS,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;AACtD,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,mBAAmB,EAAE,QAAQ,CAAC,CAAC;AAEpD,kBAAe,QAAQ,CAAC"}

build/services/blackhole-detector.js

@@ -0,0 +1,80 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BlackHoleDetector = void 0;
+const tsyringe_1 = require("tsyringe");
+const async_service_1 = require("civkit/async-service");
+const logger_1 = require("./logger");
+const timeout_1 = require("civkit/timeout");
+let BlackHoleDetector = class BlackHoleDetector extends async_service_1.AsyncService {
+    constructor(globalLogger) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.maxDelay = 1000 * 30;
+        this.concurrentRequests = 0;
+        this.strikes = 0;
+        if (process.env.NODE_ENV?.startsWith('prod')) {
+            setInterval(() => {
+                this.routine();
+            }, 1000 * 30).unref();
+        }
+    }
+    async init() {
+        await this.dependencyReady();
+        this.logger.debug('BlackHoleDetector started');
+        this.emit('ready');
+    }
+    async routine() {
+        // We give routine a 3s grace period for potentially paused CPU to spin up and process some requests
+        await (0, timeout_1.delay)(3000);
+        const now = Date.now();
+        const lastWorked = this.lastWorkedTs;
+        if (!lastWorked) {
+            return;
+        }
+        const dt = (now - lastWorked);
+        if (this.concurrentRequests > 1 &&
+            this.lastIncomingRequestTs && lastWorked &&
+            this.lastIncomingRequestTs >= lastWorked &&
+            (dt > (this.maxDelay * (this.strikes + 1)))) {
+            this.logger.warn(`BlackHole detected, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);
+            this.strikes += 1;
+        }
+        if (this.strikes >= 3) {
+            this.logger.error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);
+            process.nextTick(() => {
+                this.emit('error', new Error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`));
+                // process.exit(1);
+            });
+        }
+    }
+    incomingRequest() {
+        this.lastIncomingRequestTs = Date.now();
+        this.lastWorkedTs ??= Date.now();
+        this.concurrentRequests++;
+    }
+    doneWithRequest() {
+        this.concurrentRequests--;
+        this.lastDoneRequestTs = Date.now();
+    }
+    itWorked() {
+        this.lastWorkedTs = Date.now();
+        this.strikes = 0;
+    }
+};
+exports.BlackHoleDetector = BlackHoleDetector;
+exports.BlackHoleDetector = BlackHoleDetector = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger])
+], BlackHoleDetector);
+;
+//# sourceMappingURL=blackhole-detector.js.map

build/services/blackhole-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"blackhole-detector.js","sourceRoot":"","sources":["../../src/services/blackhole-detector.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAqC;AACrC,wDAAoD;AACpD,qCAAwC;AACxC,4CAAuC;AAIhC,IAAM,iBAAiB,GAAvB,MAAM,iBAAkB,SAAQ,4BAAY;IAY/C,YAAsB,YAA0B;QAC5C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QADF,iBAAY,GAAZ,YAAY,CAAc;QAVhD,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAKrE,aAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QACrB,uBAAkB,GAAG,CAAC,CAAC;QAEvB,YAAO,GAAG,CAAC,CAAC;QAKR,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,WAAW,CAAC,GAAG,EAAE;gBACb,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,CAAC,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,OAAO;QACT,oGAAoG;QACpG,MAAM,IAAA,eAAK,EAAC,IAAI,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC;QACrC,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC3B,IAAI,CAAC,qBAAqB,IAAI,UAAU;YACxC,IAAI,CAAC,qBAAqB,IAAI,UAAU;YACxC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAC7C,CAAC;YACC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,8BAA8B,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAClI,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QACtB,CAAC;QAED,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,IAAI,CAAC,OAAO,0BAA0B,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,8BAA8B,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC;YAC/J,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;gBAClB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,OAAO,0BAA0B,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,8BAA8B,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC;gBAC3K,mBAAmB;YACvB,CAAC,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,eAAe;QACX,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC9B,CAAC;IACD,eAAe;QACX,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACxC,CAAC;IAED,QAAQ;QACJ,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IACrB,CAAC;CAEJ,CAAA;AAtEY,8CAAiB;4BAAjB,iBAAiB;IAD7B,IAAA,oBAAS,GAAE;qCAa4B,qBAAY;GAZvC,iBAAiB,CAsE7B;AAAA,CAAC"}

build/services/brave-search.js

@@ -0,0 +1,186 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BraveSearchExplicitOperatorsDto = exports.BraveSearchService = void 0;
+const civkit_1 = require("civkit");
+const tsyringe_1 = require("tsyringe");
+const logger_1 = require("./logger");
+const secrets_1 = require("../shared/services/secrets");
+const brave_search_1 = require("../shared/3rd-party/brave-search");
+const geoip_1 = require("./geoip");
+const async_context_1 = require("./async-context");
+const blackhole_detector_1 = require("./blackhole-detector");
+let BraveSearchService = class BraveSearchService extends civkit_1.AsyncService {
+    constructor(globalLogger, secretExposer, geoipControl, threadLocal, blackHoleDetector) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.secretExposer = secretExposer;
+        this.geoipControl = geoipControl;
+        this.threadLocal = threadLocal;
+        this.blackHoleDetector = blackHoleDetector;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+    }
+    async init() {
+        await this.dependencyReady();
+        this.emit('ready');
+        this.braveSearchHTTP = new brave_search_1.BraveSearchHTTP(this.secretExposer.BRAVE_SEARCH_API_KEY);
+    }
+    async webSearch(query) {
+        const ip = this.threadLocal.get('ip');
+        const extraHeaders = {};
+        if (ip) {
+            const geoip = await this.geoipControl.lookupCity(ip, geoip_1.GEOIP_SUPPORTED_LANGUAGES.EN);
+            if (geoip?.city) {
+                extraHeaders['X-Loc-City'] = encodeURIComponent(geoip.city);
+            }
+            if (geoip?.country) {
+                extraHeaders['X-Loc-Country'] = geoip.country.code;
+            }
+            if (geoip?.timezone) {
+                extraHeaders['X-Loc-Timezone'] = geoip.timezone;
+            }
+            if (geoip?.coordinates) {
+                extraHeaders['X-Loc-Lat'] = `${geoip.coordinates[0]}`;
+                extraHeaders['X-Loc-Long'] = `${geoip.coordinates[1]}`;
+            }
+            if (geoip?.subdivisions?.length) {
+                extraHeaders['X-Loc-State'] = encodeURIComponent(`${geoip.subdivisions[0].code}`);
+                extraHeaders['X-Loc-State-Name'] = encodeURIComponent(`${geoip.subdivisions[0].name}`);
+            }
+        }
+        if (this.threadLocal.get('userAgent')) {
+            extraHeaders['User-Agent'] = this.threadLocal.get('userAgent');
+        }
+        const encoded = { ...query };
+        if (encoded.q) {
+            encoded.q = (Buffer.from(encoded.q).toString('ascii') === encoded.q) ? encoded.q : encodeURIComponent(encoded.q);
+        }
+        let maxTries = 11;
+        while (maxTries--) {
+            try {
+                const r = await this.braveSearchHTTP.webSearch(encoded, { headers: extraHeaders });
+                this.blackHoleDetector.itWorked();
+                return r.parsed;
+            }
+            catch (err) {
+                this.logger.error(`Web search failed: ${err?.message}`, { err: (0, civkit_1.marshalErrorLike)(err) });
+                if (err?.status === 429) {
+                    await (0, civkit_1.delay)(500 + 1000 * Math.random());
+                    continue;
+                }
+                throw new civkit_1.DownstreamServiceFailureError({ message: `Search failed` });
+            }
+        }
+        throw new civkit_1.DownstreamServiceFailureError({ message: `Search failed` });
+    }
+};
+exports.BraveSearchService = BraveSearchService;
+exports.BraveSearchService = BraveSearchService = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger,
+        secrets_1.SecretExposer,
+        geoip_1.GeoIPService,
+        async_context_1.AsyncLocalContext,
+        blackhole_detector_1.BlackHoleDetector])
+], BraveSearchService);
+class BraveSearchExplicitOperatorsDto extends civkit_1.AutoCastable {
+    addTo(searchTerm) {
+        const chunks = [];
+        for (const [key, value] of Object.entries(this)) {
+            if (value) {
+                const values = Array.isArray(value) ? value : [value];
+                const textValue = values.map((v) => `${key}:${v}`).join(' OR ');
+                if (textValue) {
+                    chunks.push(textValue);
+                }
+            }
+        }
+        const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks;
+        if (opPart.length) {
+            return [searchTerm, opPart].join(' ');
+        }
+        return searchTerm;
+    }
+    static from(input) {
+        const instance = super.from(input);
+        const ctx = Reflect.get(input, civkit_1.RPC_CALL_ENVIRONMENT);
+        const params = ['ext', 'filetype', 'inbody', 'intitle', 'inpage', 'lang', 'loc', 'site'];
+        for (const p of params) {
+            const customValue = ctx?.req.get(`x-${p}`) || ctx?.req.get(`${p}`);
+            if (!customValue) {
+                continue;
+            }
+            const filtered = customValue.split(', ').filter(Boolean);
+            if (filtered.length) {
+                Reflect.set(instance, p, filtered);
+            }
+        }
+        return instance;
+    }
+}
+exports.BraveSearchExplicitOperatorsDto = BraveSearchExplicitOperatorsDto;
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "ext", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "filetype", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages containing the specified term in the body of the page. Example: to find information about the Nvidia GeForce GTX 1080 Ti, making sure the page contains the keywords “founders edition” in the body, type “nvidia 1080 ti inbody:“founders edition””.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "inbody", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "intitle", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns webpages containing the specified term either in the title or in the body of the page. Example: to find pages about the 2024 Oscars containing the keywords “best costume design” in the page, type “oscars 2024 inpage:“best costume design””.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "inpage", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "lang", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "loc", void 0);
+__decorate([
+    (0, civkit_1.Prop)({
+        arrayOf: String,
+        desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`
+    }),
+    __metadata("design:type", Object)
+], BraveSearchExplicitOperatorsDto.prototype, "site", void 0);
+//# sourceMappingURL=brave-search.js.map

build/services/brave-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"brave-search.js","sourceRoot":"","sources":["../../src/services/brave-search.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,mCAAwI;AACxI,uCAAqC;AACrC,qCAAwC;AACxC,wDAA2D;AAC3D,mEAAyF;AACzF,mCAAkE;AAClE,mDAAoD;AAGpD,6DAAyD;AAGlD,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,qBAAY;IAMhD,YACc,YAA0B,EAC1B,aAA4B,EAC5B,YAA0B,EAC1B,WAA8B,EAC9B,iBAAoC;QAE9C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QANV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAC5B,iBAAY,GAAZ,YAAY,CAAc;QAC1B,gBAAW,GAAX,WAAW,CAAmB;QAC9B,sBAAiB,GAAjB,iBAAiB,CAAmB;QATlD,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IAYrE,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEnB,IAAI,CAAC,eAAe,GAAG,IAAI,8BAAe,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAA2B;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,YAAY,GAAmC,EAAE,CAAC;QACxD,IAAI,EAAE,EAAE,CAAC;YACL,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,EAAE,iCAAyB,CAAC,EAAE,CAAC,CAAC;YAEnF,IAAI,KAAK,EAAE,IAAI,EAAE,CAAC;gBACd,YAAY,CAAC,YAAY,CAAC,GAAG,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,KAAK,EAAE,OAAO,EAAE,CAAC;gBACjB,YAAY,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YACvD,CAAC;YACD,IAAI,KAAK,EAAE,QAAQ,EAAE,CAAC;gBAClB,YAAY,CAAC,gBAAgB,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC;YACpD,CAAC;YACD,IAAI,KAAK,EAAE,WAAW,EAAE,CAAC;gBACrB,YAAY,CAAC,WAAW,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtD,YAAY,CAAC,YAAY,CAAC,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,CAAC;YACD,IAAI,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;gBAC9B,YAAY,CAAC,aAAa,CAAC,GAAG,kBAAkB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAClF,YAAY,CAAC,kBAAkB,CAAC,GAAG,kBAAkB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACpC,YAAY,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACnE,CAAC;QAED,MAAM,OAAO,GAAG,EAAE,GAAG,KAAK,EAAE,CAAC;QAC7B,IAAI,OAAO,CAAC,CAAC,EAAE,CAAC;YACZ,OAAO,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrH,CAAC;QAED,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,OAAO,QAAQ,EAAE,EAAE,CAAC;YAChB,IAAI,CAAC;gBACD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,YAAsC,EAAE,CAAC,CAAC;gBAC7G,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;gBAElC,OAAO,CAAC,CAAC,MAAM,CAAC;YACpB,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,IAAA,yBAAgB,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACxF,IAAI,GAAG,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;oBACtB,MAAM,IAAA,cAAK,EAAC,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;oBACxC,SAAS;gBACb,CAAC;gBAED,MAAM,IAAI,sCAA6B,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;QAED,MAAM,IAAI,sCAA6B,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;IAC1E,CAAC;CAEJ,CAAA;AA9EY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,oBAAS,GAAE;qCAQoB,qBAAY;QACX,uBAAa;QACd,oBAAY;QACb,iCAAiB;QACX,sCAAiB;GAXzC,kBAAkB,CA8E9B;AAGD,MAAa,+BAAgC,SAAQ,qBAAY;IAiD7D,KAAK,CAAC,UAAkB;QACpB,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,KAAK,EAAE,CAAC;gBACR,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACtD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChE,IAAI,SAAS,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAEtF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,MAAM,CAAU,IAAI,CAAC,KAAU;QAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAoC,CAAC;QACtE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,6BAAoB,CAGtC,CAAC;QAEd,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QAEzF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACrB,MAAM,WAAW,GAAG,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACf,SAAS;YACb,CAAC;YAED,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;YACvC,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AA5FD,0EA4FC;AAvFG;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,sJAAsJ;KAC/J,CAAC;;4DACsB;AAMxB;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,yNAAyN;KAClO,CAAC;;iEAC2B;AAM7B;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,yQAAyQ;KAClR,CAAC;;+DACyB;AAM3B;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,8MAA8M;KACvN,CAAC;;gEAC0B;AAM5B;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,yPAAyP;KAClQ,CAAC;;+DACyB;AAM3B;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,sMAAsM;KAC/M,CAAC;;6DACuB;AAMzB;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,sMAAsM;KAC/M,CAAC;;4DACsB;AAMxB;IAJC,IAAA,aAAI,EAAC;QACF,OAAO,EAAE,MAAM;QACf,IAAI,EAAE,wJAAwJ;KACjK,CAAC;;6DACuB"}

build/services/canvas.js

@@ -0,0 +1,183 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CanvasService = void 0;
+const tsyringe_1 = require("tsyringe");
+const civkit_1 = require("civkit");
+const promises_1 = require("fs/promises");
+const logger_1 = require("./logger");
+const temp_file_1 = require("./temp-file");
+const worker_threads_1 = require("worker_threads");
+const path_1 = __importDefault(require("path"));
+const threaded_1 = require("./threaded");
+const downloadFile = async (uri) => {
+    const resp = await fetch(uri);
+    if (!(resp.ok && resp.body)) {
+        throw new Error(`Unexpected response ${resp.statusText}`);
+    }
+    const contentLength = parseInt(resp.headers.get('content-length') || '0');
+    if (contentLength > 1024 * 1024 * 100) {
+        throw new Error('File too large');
+    }
+    const buff = await resp.arrayBuffer();
+    return { buff, contentType: resp.headers.get('content-type') };
+};
+let CanvasService = class CanvasService extends civkit_1.AsyncService {
+    constructor(temp, globalLogger) {
+        super(...arguments);
+        this.temp = temp;
+        this.globalLogger = globalLogger;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+    }
+    async init() {
+        await this.dependencyReady();
+        if (!worker_threads_1.isMainThread) {
+            const { createSvg2png, initialize } = require('svg2png-wasm');
+            const wasmBuff = await (0, promises_1.readFile)(path_1.default.resolve(path_1.default.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm'));
+            const fontBuff = await (0, promises_1.readFile)(path_1.default.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'));
+            await initialize(wasmBuff);
+            this.svg2png = createSvg2png({
+                fonts: [Uint8Array.from(fontBuff)],
+                defaultFontFamily: {
+                    serifFamily: 'Source Han Sans SC',
+                    sansSerifFamily: 'Source Han Sans SC',
+                    cursiveFamily: 'Source Han Sans SC',
+                    fantasyFamily: 'Source Han Sans SC',
+                    monospaceFamily: 'Source Han Sans SC',
+                }
+            });
+        }
+        this.canvas = require('@napi-rs/canvas');
+        this.emit('ready');
+    }
+    async renderSvgToPng(svgContent) {
+        return this.svg2png(svgContent, { backgroundColor: '#D3D3D3' });
+    }
+    async _loadImage(input) {
+        let buff;
+        let contentType;
+        do {
+            if (typeof input === 'string') {
+                if (input.startsWith('data:')) {
+                    const firstComma = input.indexOf(',');
+                    const header = input.slice(0, firstComma);
+                    const data = input.slice(firstComma + 1);
+                    const encoding = header.split(';')[1];
+                    contentType = header.split(';')[0].split(':')[1];
+                    if (encoding?.startsWith('base64')) {
+                        buff = Buffer.from(data, 'base64');
+                    }
+                    else {
+                        buff = Buffer.from(decodeURIComponent(data), 'utf-8');
+                    }
+                    break;
+                }
+                if (input.startsWith('http')) {
+                    const r = await downloadFile(input);
+                    buff = Buffer.from(r.buff);
+                    contentType = r.contentType;
+                    break;
+                }
+            }
+            if (Buffer.isBuffer(input)) {
+                buff = input;
+                const mime = await (0, civkit_1.mimeOf)(buff);
+                contentType = `${mime.mediaType}/${mime.subType}`;
+                break;
+            }
+            throw new civkit_1.ParamValidationError('Invalid input');
+        } while (false);
+        if (!buff) {
+            throw new civkit_1.ParamValidationError('Invalid input');
+        }
+        if (contentType?.includes('svg')) {
+            buff = await this.renderSvgToPng(buff.toString('utf-8'));
+        }
+        const img = await this.canvas.loadImage(buff);
+        Reflect.set(img, 'contentType', contentType);
+        return img;
+    }
+    async loadImage(uri) {
+        const t0 = Date.now();
+        try {
+            const theImage = await this._loadImage(uri);
+            const t1 = Date.now();
+            this.logger.debug(`Image loaded in ${t1 - t0}ms`);
+            return theImage;
+        }
+        catch (err) {
+            if (err?.message?.includes('Unsupported image type') || err?.message?.includes('unsupported')) {
+                this.logger.warn(`Failed to load image ${uri.slice(0, 128)}`, { err });
+                throw new civkit_1.SubmittedDataMalformedError(`Unknown image format for ${uri.slice(0, 128)}`);
+            }
+            throw err;
+        }
+    }
+    fitImageToSquareBox(image, size = 1024) {
+        // this.logger.debug(`Fitting image(${ image.width }x${ image.height }) to ${ size } box`);
+        // const t0 = Date.now();
+        if (image.width <= size && image.height <= size) {
+            if (image instanceof this.canvas.Canvas) {
+                return image;
+            }
+            const canvasInstance = this.canvas.createCanvas(image.width, image.height);
+            const ctx = canvasInstance.getContext('2d');
+            ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, canvasInstance.width, canvasInstance.height);
+            // this.logger.debug(`No need to resize, copied to canvas in ${ Date.now() - t0 } ms`);
+            return canvasInstance;
+        }
+        const aspectRatio = image.width / image.height;
+        const resizedWidth = Math.round(aspectRatio > 1 ? size : size * aspectRatio);
+        const resizedHeight = Math.round(aspectRatio > 1 ? size / aspectRatio : size);
+        const canvasInstance = this.canvas.createCanvas(resizedWidth, resizedHeight);
+        const ctx = canvasInstance.getContext('2d');
+        ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, resizedWidth, resizedHeight);
+        // this.logger.debug(`Resized to ${ resizedWidth }x${ resizedHeight } in ${ Date.now() - t0 } ms`);
+        return canvasInstance;
+    }
+    corpImage(image, x, y, w, h) {
+        // this.logger.debug(`Cropping image(${ image.width }x${ image.height }) to ${ w }x${ h } at ${ x },${ y } `);
+        // const t0 = Date.now();
+        const canvasInstance = this.canvas.createCanvas(w, h);
+        const ctx = canvasInstance.getContext('2d');
+        ctx.drawImage(image, x, y, w, h, 0, 0, w, h);
+        // this.logger.debug(`Crop complete in ${ Date.now() - t0 } ms`);
+        return canvasInstance;
+    }
+    canvasToDataUrl(canvas, mimeType) {
+        // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`);
+        // const t0 = Date.now();
+        return canvas.toDataURLAsync((mimeType || 'image/png'));
+    }
+    async canvasToBuffer(canvas, mimeType) {
+        // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`);
+        // const t0 = Date.now();
+        return canvas.toBuffer((mimeType || 'image/png'));
+    }
+};
+exports.CanvasService = CanvasService;
+__decorate([
+    (0, threaded_1.Threaded)(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [String]),
+    __metadata("design:returntype", Promise)
+], CanvasService.prototype, "renderSvgToPng", null);
+exports.CanvasService = CanvasService = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [temp_file_1.TempFileManager,
+        logger_1.GlobalLogger])
+], CanvasService);
+const instance = tsyringe_1.container.resolve(CanvasService);
+exports.default = instance;
+//# sourceMappingURL=canvas.js.map

build/services/canvas.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"canvas.js","sourceRoot":"","sources":["../../src/services/canvas.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,uCAAgD;AAChD,mCAAqH;AACrH,0CAAuC;AAKvC,qCAAwC;AACxC,2CAA8C;AAE9C,mDAA8C;AAE9C,gDAAwB;AACxB,yCAAsC;AAEtC,MAAM,YAAY,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;IACvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,CAAC;IAC1E,IAAI,aAAa,GAAG,IAAI,GAAG,IAAI,GAAG,GAAG,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAEtC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;AACnE,CAAC,CAAC;AAGK,IAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,qBAAY;IAM3C,YACc,IAAqB,EACrB,YAA0B;QAEpC,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QAHV,SAAI,GAAJ,IAAI,CAAiB;QACrB,iBAAY,GAAZ,YAAY,CAAc;QANxC,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IASrE,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,IAAI,CAAC,6BAAY,EAAE,CAAC;YAChB,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAI,CAAC,OAAO,CAAC,cAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC,CAAC;YACxH,MAAM,QAAQ,GAAG,MAAM,IAAA,mBAAQ,EAAC,cAAI,CAAC,OAAO,CAAC,SAAS,EAAE,4CAA4C,CAAC,CAAC,CAAC;YACvG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,aAAa,CAAC;gBACzB,KAAK,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAClC,iBAAiB,EAAE;oBACf,WAAW,EAAE,oBAAoB;oBACjC,eAAe,EAAE,oBAAoB;oBACrC,aAAa,EAAE,oBAAoB;oBACnC,aAAa,EAAE,oBAAoB;oBACnC,eAAe,EAAE,oBAAoB;iBACxC;aACJ,CAAC,CAAC;QACP,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAGK,AAAN,KAAK,CAAC,cAAc,CAAC,UAAkB;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,CAAC;IACpE,CAAC;IAES,KAAK,CAAC,UAAU,CAAC,KAAsB;QAC7C,IAAI,IAAI,CAAC;QACT,IAAI,WAAW,CAAC;QAChB,GAAG,CAAC;YACA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC5B,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;oBACtC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;oBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;oBACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACtC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBACjD,IAAI,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACjC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;oBACvC,CAAC;yBAAM,CAAC;wBACJ,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;oBAC1D,CAAC;oBACD,MAAM;gBACV,CAAC;gBACD,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;oBACpC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;oBAC3B,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;oBAC5B,MAAM;gBACV,CAAC;YACL,CAAC;YACD,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,IAAI,GAAG,KAAK,CAAC;gBACb,MAAM,IAAI,GAAG,MAAM,IAAA,eAAM,EAAC,IAAI,CAAC,CAAC;gBAChC,WAAW,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAClD,MAAM;YACV,CAAC;YACD,MAAM,IAAI,6BAAoB,CAAC,eAAe,CAAC,CAAC;QACpD,CAAC,QAAQ,KAAK,EAAE;QAEhB,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,MAAM,IAAI,6BAAoB,CAAC,eAAe,CAAC,CAAC;QACpD,CAAC;QAED,IAAI,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;QAE7C,OAAO,GAAG,CAAC;IACf,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAoB;QAChC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAElD,OAAO,QAAQ,CAAC;QACpB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,wBAAwB,CAAC,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5F,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;gBACvE,MAAM,IAAI,oCAA2B,CAAC,4BAA4B,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3F,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED,mBAAmB,CAAC,KAAmC,EAAE,OAAe,IAAI;QACxE,2FAA2F;QAC3F,yBAAyB;QACzB,IAAI,KAAK,CAAC,KAAK,IAAI,IAAI,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YAC9C,IAAI,KAAK,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACtC,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3E,MAAM,GAAG,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAC5C,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;YACzG,uFAAuF;YAEvF,OAAO,cAAc,CAAC;QAC1B,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC;QAC7E,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE9E,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC5C,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;QACzF,mGAAmG;QAEnG,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,SAAS,CAAC,KAAmC,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS,EAAE,CAAS;QACrF,8GAA8G;QAC9G,yBAAyB;QACzB,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,cAAc,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC5C,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7C,iEAAiE;QAEjE,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,eAAe,CAAC,MAAqB,EAAE,QAAqC;QACxE,+EAA+E;QAC/E,yBAAyB;QACzB,OAAO,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAgB,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAqB,EAAE,QAAqC;QAC7E,+EAA+E;QAC/E,yBAAyB;QACzB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAgB,CAAC,CAAC;IACrE,CAAC;CAEJ,CAAA;AA7JY,sCAAa;AAqChB;IADL,IAAA,mBAAQ,GAAE;;;;mDAGV;wBAvCQ,aAAa;IADzB,IAAA,oBAAS,GAAE;qCAQY,2BAAe;QACP,qBAAY;GAR/B,aAAa,CA6JzB;AAED,MAAM,QAAQ,GAAG,oBAAS,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;AAClD,kBAAe,QAAQ,CAAC"}

build/services/cf-browser-rendering.js

@@ -0,0 +1,58 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CFBrowserRendering = void 0;
+const tsyringe_1 = require("tsyringe");
+const async_service_1 = require("civkit/async-service");
+const secrets_1 = require("../shared/services/secrets");
+const logger_1 = require("./logger");
+const cloud_flare_1 = require("../shared/3rd-party/cloud-flare");
+const http_1 = require("civkit/http");
+const errors_1 = require("./errors");
+let CFBrowserRendering = class CFBrowserRendering extends async_service_1.AsyncService {
+    constructor(globalLogger, secretExposer) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.secretExposer = secretExposer;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+    }
+    async init() {
+        await this.dependencyReady();
+        const [account, key] = this.secretExposer.CLOUD_FLARE_API_KEY?.split(':');
+        this.client = new cloud_flare_1.CloudFlareHTTP(account, key);
+        this.emit('ready');
+    }
+    async fetchContent(url) {
+        try {
+            const r = await this.client.fetchBrowserRenderedHTML({ url });
+            return r.parsed.result;
+        }
+        catch (err) {
+            if (err instanceof http_1.HTTPServiceError) {
+                if (err.status === 429) {
+                    // Rate limit exceeded, return empty result
+                    this.logger.warn('Cloudflare browser rendering rate limit exceeded', { url });
+                    throw new errors_1.ServiceNodeResourceDrainError(`Cloudflare browser rendering (our account) is at capacity, please try again later or switch to another engine.`);
+                }
+            }
+            throw err;
+        }
+    }
+};
+exports.CFBrowserRendering = CFBrowserRendering;
+exports.CFBrowserRendering = CFBrowserRendering = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger,
+        secrets_1.SecretExposer])
+], CFBrowserRendering);
+const instance = tsyringe_1.container.resolve(CFBrowserRendering);
+exports.default = instance;
+//# sourceMappingURL=cf-browser-rendering.js.map

build/services/cf-browser-rendering.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cf-browser-rendering.js","sourceRoot":"","sources":["../../src/services/cf-browser-rendering.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,uCAAgD;AAChD,wDAAoD;AACpD,wDAA2D;AAC3D,qCAAwC;AACxC,iEAAiE;AACjE,sCAA+C;AAC/C,qCAAyD;AAGlD,IAAM,kBAAkB,GAAxB,MAAM,kBAAmB,SAAQ,4BAAY;IAKhD,YACc,YAA0B,EAC1B,aAA4B;QAEtC,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QAHV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,kBAAa,GAAb,aAAa,CAAe;QAL1C,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;IAQrE,CAAC;IAGQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,mBAAmB,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM,GAAG,IAAI,4BAAc,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QAE/C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,GAAW;QAC1B,IAAI,CAAC;YACD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,wBAAwB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAE9D,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,GAAG,YAAY,uBAAgB,EAAE,CAAC;gBAClC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACrB,2CAA2C;oBAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kDAAkD,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE9E,MAAM,IAAI,sCAA6B,CAAC,gHAAgH,CAAE,CAAC;gBAC/J,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;CAEJ,CAAA;AAxCY,gDAAkB;6BAAlB,kBAAkB;IAD9B,IAAA,oBAAS,GAAE;qCAOoB,qBAAY;QACX,uBAAa;GAPjC,kBAAkB,CAwC9B;AAED,MAAM,QAAQ,GAAG,oBAAS,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;AAEvD,kBAAe,QAAQ,CAAC"}

build/services/curl.js

@@ -0,0 +1,431 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CurlControl = void 0;
+const async_service_1 = require("civkit/async-service");
+const tsyringe_1 = require("tsyringe");
+const node_libcurl_1 = require("node-libcurl");
+const set_cookie_parser_1 = require("set-cookie-parser");
+const logger_1 = require("./logger");
+const civkit_1 = require("civkit");
+const errors_1 = require("./errors");
+const temp_file_1 = require("../services/temp-file");
+const zlib_1 = require("zlib");
+const simple_zstd_1 = require("simple-zstd");
+const lodash_1 = __importDefault(require("lodash"));
+const async_context_1 = require("./async-context");
+const blackhole_detector_1 = require("./blackhole-detector");
+let CurlControl = class CurlControl extends async_service_1.AsyncService {
+    constructor(globalLogger, tempFileManager, asyncLocalContext, blackHoleDetector) {
+        super(...arguments);
+        this.globalLogger = globalLogger;
+        this.tempFileManager = tempFileManager;
+        this.asyncLocalContext = asyncLocalContext;
+        this.blackHoleDetector = blackHoleDetector;
+        this.logger = this.globalLogger.child({ service: this.constructor.name });
+        this.chromeVersion = `132`;
+        this.safariVersion = `537.36`;
+        this.platform = `Linux`;
+        this.ua = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/${this.safariVersion} (KHTML, like Gecko) Chrome/${this.chromeVersion}.0.0.0 Safari/${this.safariVersion}`;
+        this.lifeCycleTrack = new WeakMap();
+    }
+    async init() {
+        await this.dependencyReady();
+        if (process.platform === 'darwin') {
+            this.platform = `macOS`;
+        }
+        else if (process.platform === 'win32') {
+            this.platform = `Windows`;
+        }
+        this.emit('ready');
+    }
+    impersonateChrome(ua) {
+        this.chromeVersion = ua.match(/Chrome\/(\d+)/)[1];
+        this.safariVersion = ua.match(/AppleWebKit\/([\d\.]+)/)[1];
+        this.ua = ua;
+    }
+    curlImpersonateHeader(curl, headers) {
+        let uaPlatform = this.platform;
+        if (this.ua.includes('Windows')) {
+            uaPlatform = 'Windows';
+        }
+        else if (this.ua.includes('Android')) {
+            uaPlatform = 'Android';
+        }
+        else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) {
+            uaPlatform = 'iOS';
+        }
+        else if (this.ua.includes('CrOS')) {
+            uaPlatform = 'Chrome OS';
+        }
+        else if (this.ua.includes('Macintosh')) {
+            uaPlatform = 'macOS';
+        }
+        const mixinHeaders = {
+            'Sec-Ch-Ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`,
+            'Sec-Ch-Ua-Mobile': '?0',
+            'Sec-Ch-Ua-Platform': `"${uaPlatform}"`,
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': this.ua,
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-User': '?1',
+            'Sec-Fetch-Dest': 'document',
+            'Accept-Encoding': 'gzip, deflate, br, zstd',
+            'Accept-Language': 'en-US,en;q=0.9',
+        };
+        const headersCopy = { ...headers };
+        for (const k of Object.keys(mixinHeaders)) {
+            const lowerK = k.toLowerCase();
+            if (headersCopy[lowerK]) {
+                mixinHeaders[k] = headersCopy[lowerK];
+                delete headersCopy[lowerK];
+            }
+        }
+        Object.assign(mixinHeaders, headersCopy);
+        curl.setOpt(node_libcurl_1.Curl.option.HTTPHEADER, Object.entries(mixinHeaders).flatMap(([k, v]) => {
+            if (Array.isArray(v) && v.length) {
+                return v.map((v2) => `${k}: ${v2}`);
+            }
+            return [`${k}: ${v}`];
+        }));
+        return curl;
+    }
+    urlToFile1Shot(urlToCrawl, crawlOpts) {
+        return new Promise((resolve, reject) => {
+            let contentType = '';
+            const curl = new node_libcurl_1.Curl();
+            curl.enable(node_libcurl_1.CurlFeature.StreamResponse);
+            curl.setOpt('URL', urlToCrawl.toString());
+            curl.setOpt(node_libcurl_1.Curl.option.FOLLOWLOCATION, false);
+            curl.setOpt(node_libcurl_1.Curl.option.SSL_VERIFYPEER, false);
+            curl.setOpt(node_libcurl_1.Curl.option.TIMEOUT_MS, crawlOpts?.timeoutMs || 30_000);
+            curl.setOpt(node_libcurl_1.Curl.option.CONNECTTIMEOUT_MS, 3_000);
+            curl.setOpt(node_libcurl_1.Curl.option.LOW_SPEED_LIMIT, 32768);
+            curl.setOpt(node_libcurl_1.Curl.option.LOW_SPEED_TIME, 5_000);
+            if (crawlOpts?.method) {
+                curl.setOpt(node_libcurl_1.Curl.option.CUSTOMREQUEST, crawlOpts.method.toUpperCase());
+            }
+            if (crawlOpts?.body) {
+                curl.setOpt(node_libcurl_1.Curl.option.POSTFIELDS, crawlOpts.body.toString());
+            }
+            const headersToSet = { ...crawlOpts?.extraHeaders };
+            if (crawlOpts?.cookies?.length) {
+                const cookieKv = {};
+                for (const cookie of crawlOpts.cookies) {
+                    cookieKv[cookie.name] = cookie.value;
+                }
+                for (const cookie of crawlOpts.cookies) {
+                    if (cookie.maxAge && cookie.maxAge < 0) {
+                        delete cookieKv[cookie.name];
+                        continue;
+                    }
+                    if (cookie.expires && cookie.expires < new Date()) {
+                        delete cookieKv[cookie.name];
+                        continue;
+                    }
+                    if (cookie.secure && urlToCrawl.protocol !== 'https:') {
+                        delete cookieKv[cookie.name];
+                        continue;
+                    }
+                    if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) {
+                        delete cookieKv[cookie.name];
+                        continue;
+                    }
+                    if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) {
+                        delete cookieKv[cookie.name];
+                        continue;
+                    }
+                }
+                const cookieChunks = Object.entries(cookieKv).map(([k, v]) => `${k}=${encodeURIComponent(v)}`);
+                headersToSet.cookie ??= cookieChunks.join('; ');
+            }
+            if (crawlOpts?.referer) {
+                headersToSet.referer ??= crawlOpts.referer;
+            }
+            if (crawlOpts?.overrideUserAgent) {
+                headersToSet['user-agent'] ??= crawlOpts.overrideUserAgent;
+            }
+            this.curlImpersonateHeader(curl, headersToSet);
+            if (crawlOpts?.proxyUrl) {
+                const proxyUrlCopy = new URL(crawlOpts.proxyUrl);
+                curl.setOpt(node_libcurl_1.Curl.option.PROXY, proxyUrlCopy.href);
+            }
+            let curlStream;
+            curl.on('error', (err, errCode) => {
+                curl.close();
+                this.logger.warn(`Curl ${urlToCrawl.origin}: ${err}`, { err, urlToCrawl });
+                const err2 = this.digestCurlCode(errCode, err.message) ||
+                    new civkit_1.AssertionFailureError(`Failed to access ${urlToCrawl.origin}: ${err.message}`);
+                err2.cause ??= err;
+                if (curlStream) {
+                    // For some reason, manually emitting error event is required for curlStream.
+                    curlStream.emit('error', err2);
+                    curlStream.destroy(err2);
+                }
+                reject(err2);
+            });
+            curl.setOpt(node_libcurl_1.Curl.option.MAXFILESIZE, 4 * 1024 * 1024 * 1024); // 4GB
+            let status = -1;
+            let statusText;
+            let contentEncoding = '';
+            curl.once('end', () => {
+                if (curlStream) {
+                    curlStream.once('end', () => curl.close());
+                    return;
+                }
+                curl.close();
+            });
+            curl.on('stream', (stream, statusCode, headers) => {
+                this.logger.debug(`CURL: [${statusCode}] ${urlToCrawl.origin}`, { statusCode });
+                status = statusCode;
+                curlStream = stream;
+                for (const headerSet of headers) {
+                    for (const [k, v] of Object.entries(headerSet)) {
+                        if (k.trim().endsWith(':')) {
+                            Reflect.set(headerSet, k.slice(0, k.indexOf(':')), v || '');
+                            Reflect.deleteProperty(headerSet, k);
+                            continue;
+                        }
+                        if (v === undefined) {
+                            Reflect.set(headerSet, k, '');
+                            continue;
+                        }
+                        if (k.toLowerCase() === 'content-type' && typeof v === 'string') {
+                            contentType = v.toLowerCase();
+                        }
+                    }
+                }
+                const lastResHeaders = headers[headers.length - 1];
+                statusText = lastResHeaders.result?.reason;
+                for (const [k, v] of Object.entries(lastResHeaders)) {
+                    const kl = k.toLowerCase();
+                    if (kl === 'content-type') {
+                        contentType = (v || '').toLowerCase();
+                    }
+                    if (kl === 'content-encoding') {
+                        contentEncoding = (v || '').toLowerCase();
+                    }
+                    if (contentType && contentEncoding) {
+                        break;
+                    }
+                }
+                if ([301, 302, 303, 307, 308].includes(statusCode)) {
+                    if (stream) {
+                        stream.resume();
+                    }
+                    resolve({
+                        statusCode: status,
+                        statusText,
+                        data: undefined,
+                        headers: headers,
+                    });
+                    return;
+                }
+                if (!stream) {
+                    resolve({
+                        statusCode: status,
+                        statusText,
+                        data: undefined,
+                        headers: headers,
+                    });
+                    return;
+                }
+                switch (contentEncoding) {
+                    case 'gzip': {
+                        const decompressed = (0, zlib_1.createGunzip)();
+                        stream.pipe(decompressed);
+                        stream.once('error', (err) => {
+                            decompressed.destroy(err);
+                        });
+                        stream = decompressed;
+                        break;
+                    }
+                    case 'deflate': {
+                        const decompressed = (0, zlib_1.createInflate)();
+                        stream.pipe(decompressed);
+                        stream.once('error', (err) => {
+                            decompressed.destroy(err);
+                        });
+                        stream = decompressed;
+                        break;
+                    }
+                    case 'br': {
+                        const decompressed = (0, zlib_1.createBrotliDecompress)();
+                        stream.pipe(decompressed);
+                        stream.once('error', (err) => {
+                            decompressed.destroy(err);
+                        });
+                        stream = decompressed;
+                        break;
+                    }
+                    case 'zstd': {
+                        const decompressed = (0, simple_zstd_1.ZSTDDecompress)();
+                        stream.pipe(decompressed);
+                        stream.once('error', (err) => {
+                            decompressed.destroy(err);
+                        });
+                        stream = decompressed;
+                        break;
+                    }
+                    default: {
+                        break;
+                    }
+                }
+                const fpath = this.tempFileManager.alloc();
+                const fancyFile = civkit_1.FancyFile.auto(stream, fpath);
+                this.tempFileManager.bindPathTo(fancyFile, fpath);
+                resolve({
+                    statusCode: status,
+                    statusText,
+                    data: fancyFile,
+                    headers: headers,
+                });
+            });
+            curl.perform();
+        });
+    }
+    async urlToFile(urlToCrawl, crawlOpts) {
+        let leftRedirection = 6;
+        let cookieRedirects = 0;
+        let opts = { ...crawlOpts };
+        let nextHopUrl = urlToCrawl;
+        const fakeHeaderInfos = [];
+        do {
+            const r = await this.urlToFile1Shot(nextHopUrl, opts);
+            if ([301, 302, 303, 307, 308].includes(r.statusCode)) {
+                fakeHeaderInfos.push(...r.headers);
+                const headers = r.headers[r.headers.length - 1];
+                const location = headers.Location || headers.location;
+                const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie'];
+                if (setCookieHeader) {
+                    const cookieAssignments = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader];
+                    const parsed = cookieAssignments.filter(Boolean).map((x) => (0, set_cookie_parser_1.parseString)(x, { decodeValues: true }));
+                    if (parsed.length) {
+                        opts.cookies = [...(opts.cookies || []), ...parsed];
+                    }
+                    if (!location) {
+                        cookieRedirects += 1;
+                    }
+                }
+                if (!location && !setCookieHeader) {
+                    // Follow curl behavior
+                    return {
+                        statusCode: r.statusCode,
+                        data: r.data,
+                        headers: fakeHeaderInfos.concat(r.headers),
+                    };
+                }
+                if (!location && cookieRedirects > 1) {
+                    throw new errors_1.ServiceBadApproachError(`Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`);
+                }
+                nextHopUrl = new URL(location || '', nextHopUrl);
+                leftRedirection -= 1;
+                continue;
+            }
+            return {
+                statusCode: r.statusCode,
+                statusText: r.statusText,
+                data: r.data,
+                headers: fakeHeaderInfos.concat(r.headers),
+            };
+        } while (leftRedirection > 0);
+        throw new errors_1.ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Too many redirections.`);
+    }
+    async sideLoad(targetUrl, crawlOpts) {
+        const curlResult = await this.urlToFile(targetUrl, crawlOpts);
+        this.blackHoleDetector.itWorked();
+        let finalURL = targetUrl;
+        const sideLoadOpts = {
+            impersonate: {},
+            proxyOrigin: {},
+        };
+        for (const headers of curlResult.headers) {
+            sideLoadOpts.impersonate[finalURL.href] = {
+                status: headers.result?.code || -1,
+                headers: lodash_1.default.omit(headers, 'result'),
+                contentType: headers['Content-Type'] || headers['content-type'],
+            };
+            if (crawlOpts?.proxyUrl) {
+                sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl;
+            }
+            if (headers.result?.code && [301, 302, 307, 308].includes(headers.result.code)) {
+                const location = headers.Location || headers.location;
+                if (location) {
+                    finalURL = new URL(location, finalURL);
+                }
+            }
+        }
+        const lastHeaders = curlResult.headers[curlResult.headers.length - 1];
+        const contentType = (lastHeaders['Content-Type'] || lastHeaders['content-type'])?.toLowerCase() || (await curlResult.data?.mimeType) || 'application/octet-stream';
+        const contentDisposition = lastHeaders['Content-Disposition'] || lastHeaders['content-disposition'];
+        const fileName = contentDisposition?.match(/filename="([^"]+)"/i)?.[1] || finalURL.pathname.split('/').pop();
+        if (sideLoadOpts.impersonate[finalURL.href] && (await curlResult.data?.size)) {
+            sideLoadOpts.impersonate[finalURL.href].body = curlResult.data;
+        }
+        // This should keep the file from being garbage collected and deleted until this asyncContext/request is done.
+        this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data);
+        return {
+            finalURL,
+            sideLoadOpts,
+            chain: curlResult.headers,
+            status: curlResult.statusCode,
+            statusText: curlResult.statusText,
+            headers: lastHeaders,
+            contentType,
+            contentDisposition,
+            fileName,
+            file: curlResult.data
+        };
+    }
+    digestCurlCode(code, msg) {
+        switch (code) {
+            // 400 User errors
+            case node_libcurl_1.CurlCode.CURLE_COULDNT_RESOLVE_HOST: {
+                return new civkit_1.AssertionFailureError(msg);
+            }
+            // Maybe retry but dont retry with curl again
+            case node_libcurl_1.CurlCode.CURLE_OPERATION_TIMEDOUT:
+            case node_libcurl_1.CurlCode.CURLE_UNSUPPORTED_PROTOCOL:
+            case node_libcurl_1.CurlCode.CURLE_PEER_FAILED_VERIFICATION: {
+                return new errors_1.ServiceBadApproachError(msg);
+            }
+            // Retryable errors
+            case node_libcurl_1.CurlCode.CURLE_REMOTE_ACCESS_DENIED:
+            case node_libcurl_1.CurlCode.CURLE_SEND_ERROR:
+            case node_libcurl_1.CurlCode.CURLE_RECV_ERROR:
+            case node_libcurl_1.CurlCode.CURLE_GOT_NOTHING:
+            case node_libcurl_1.CurlCode.CURLE_SSL_CONNECT_ERROR:
+            case node_libcurl_1.CurlCode.CURLE_QUIC_CONNECT_ERROR:
+            case node_libcurl_1.CurlCode.CURLE_COULDNT_RESOLVE_PROXY:
+            case node_libcurl_1.CurlCode.CURLE_COULDNT_CONNECT:
+            case node_libcurl_1.CurlCode.CURLE_PARTIAL_FILE: {
+                return new errors_1.ServiceBadAttemptError(msg);
+            }
+            default: {
+                return undefined;
+            }
+        }
+    }
+};
+exports.CurlControl = CurlControl;
+exports.CurlControl = CurlControl = __decorate([
+    (0, tsyringe_1.singleton)(),
+    __metadata("design:paramtypes", [logger_1.GlobalLogger,
+        temp_file_1.TempFileManager,
+        async_context_1.AsyncLocalContext,
+        blackhole_detector_1.BlackHoleDetector])
+], CurlControl);
+//# sourceMappingURL=curl.js.map

build/services/curl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"curl.js","sourceRoot":"","sources":["../../src/services/curl.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,wDAAoD;AACpD,uCAAqC;AAErC,+CAAuE;AACvE,yDAAwE;AAGxE,qCAAwC;AACxC,mCAA0D;AAC1D,qCAA2E;AAC3E,qDAAwD;AACxD,+BAA2E;AAC3E,6CAA6C;AAC7C,oDAAuB;AAEvB,mDAAoD;AACpD,6DAAyD;AAQlD,IAAM,WAAW,GAAjB,MAAM,WAAY,SAAQ,4BAAY;IAWzC,YACc,YAA0B,EAC1B,eAAgC,EAChC,iBAAoC,EACpC,iBAAoC;QAE9C,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC;QALV,iBAAY,GAAZ,YAAY,CAAc;QAC1B,oBAAe,GAAf,eAAe,CAAiB;QAChC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,sBAAiB,GAAjB,iBAAiB,CAAmB;QAblD,WAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QAErE,kBAAa,GAAW,KAAK,CAAC;QAC9B,kBAAa,GAAW,QAAQ,CAAC;QACjC,aAAQ,GAAW,OAAO,CAAC;QAC3B,OAAE,GAAW,yDAAyD,IAAI,CAAC,aAAa,+BAA+B,IAAI,CAAC,aAAa,iBAAiB,IAAI,CAAC,aAAa,EAAE,CAAC;QAE/K,mBAAc,GAAG,IAAI,OAAO,EAAE,CAAC;IAS/B,CAAC;IAEQ,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QAC5B,CAAC;aAAM,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;QAC9B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,iBAAiB,CAAC,EAAU;QACxB,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAE,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,KAAK,CAAC,wBAAwB,CAAE,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;IAED,qBAAqB,CAAC,IAAU,EAAE,OAAgB;QAC9C,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,UAAU,GAAG,SAAS,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,UAAU,GAAG,SAAS,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5F,UAAU,GAAG,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,UAAU,GAAG,WAAW,CAAC;QAC7B,CAAC;aAAM,IAAI,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACvC,UAAU,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,MAAM,YAAY,GAA2B;YACzC,WAAW,EAAE,qCAAqC,IAAI,CAAC,aAAa,yBAAyB,IAAI,CAAC,aAAa,GAAG;YAClH,kBAAkB,EAAE,IAAI;YACxB,oBAAoB,EAAE,IAAI,UAAU,GAAG;YACvC,2BAA2B,EAAE,GAAG;YAChC,YAAY,EAAE,IAAI,CAAC,EAAE;YACrB,QAAQ,EAAE,yIAAyI;YACnJ,gBAAgB,EAAE,MAAM;YACxB,gBAAgB,EAAE,UAAU;YAC5B,gBAAgB,EAAE,IAAI;YACtB,gBAAgB,EAAE,UAAU;YAC5B,iBAAiB,EAAE,yBAAyB;YAC5C,iBAAiB,EAAE,gBAAgB;SACtC,CAAC;QACF,MAAM,WAAW,GAAuC,EAAE,GAAG,OAAO,EAAE,CAAC;QACvE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtB,YAAY,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;gBACtC,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;YAC/B,CAAC;QACL,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAEzC,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;YAChF,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC/B,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC,CAAC;QAEJ,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,cAAc,CAAC,UAAe,EAAE,SAAgC;QAC5D,OAAO,IAAI,OAAO,CAKf,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnB,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,IAAI,mBAAI,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,0BAAW,CAAC,cAAc,CAAC,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,CAAC,CAAC;YACpE,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAC/C,IAAI,SAAS,EAAE,MAAM,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,aAAa,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;YAC3E,CAAC;YACD,IAAI,SAAS,EAAE,IAAI,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,YAAY,GAAG,EAAE,GAAG,SAAS,EAAE,YAAY,EAAE,CAAC;YACpD,IAAI,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAA2B,EAAE,CAAC;gBAC5C,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;oBACrC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;gBACzC,CAAC;gBACD,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;oBACrC,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrC,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC7B,SAAS;oBACb,CAAC;oBACD,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;wBAChD,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC7B,SAAS;oBACb,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;wBACpD,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC7B,SAAS;oBACb,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;wBAChE,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC7B,SAAS;oBACb,CAAC;oBACD,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;wBAC9D,OAAO,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;wBAC7B,SAAS;oBACb,CAAC;gBACL,CAAC;gBACD,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC/F,YAAY,CAAC,MAAM,KAAK,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpD,CAAC;YACD,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC;gBACrB,YAAY,CAAC,OAAO,KAAK,SAAS,CAAC,OAAO,CAAC;YAC/C,CAAC;YACD,IAAI,SAAS,EAAE,iBAAiB,EAAE,CAAC;gBAC/B,YAAY,CAAC,YAAY,CAAC,KAAK,SAAS,CAAC,iBAAiB,CAAC;YAC/D,CAAC;YAED,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YAE/C,IAAI,SAAS,EAAE,QAAQ,EAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBACjD,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YACtD,CAAC;YAED,IAAI,UAAgC,CAAC;YACrC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;gBAC9B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,UAAU,CAAC,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC;oBAClD,IAAI,8BAAqB,CAAC,oBAAoB,UAAU,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACvF,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC;gBACnB,IAAI,UAAU,EAAE,CAAC;oBACb,6EAA6E;oBAC7E,UAAU,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBAC/B,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAC7B,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,mBAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM;YACpE,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC;YAChB,IAAI,UAA4B,CAAC;YACjC,IAAI,eAAe,GAAG,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE;gBAClB,IAAI,UAAU,EAAE,CAAC;oBACb,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;oBAC3C,OAAO;gBACX,CAAC;gBACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;gBAC9C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,UAAU,KAAK,UAAU,CAAC,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;gBAChF,MAAM,GAAG,UAAU,CAAC;gBACpB,UAAU,GAAG,MAAM,CAAC;gBACpB,KAAK,MAAM,SAAS,IAAK,OAAwB,EAAE,CAAC;oBAChD,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC7C,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;4BACzB,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;4BAC5D,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;4BACrC,SAAS;wBACb,CAAC;wBACD,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;4BAClB,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;4BAC9B,SAAS;wBACb,CAAC;wBACD,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,cAAc,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;4BAC9D,WAAW,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;wBAClC,CAAC;oBACL,CAAC;gBACL,CAAC;gBACD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACnD,UAAU,GAAI,cAA6B,CAAC,MAAM,EAAE,MAAM,CAAC;gBAC3D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;oBAClD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC3B,IAAI,EAAE,KAAK,cAAc,EAAE,CAAC;wBACxB,WAAW,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC1C,CAAC;oBACD,IAAI,EAAE,KAAK,kBAAkB,EAAE,CAAC;wBAC5B,eAAe,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC9C,CAAC;oBACD,IAAI,WAAW,IAAI,eAAe,EAAE,CAAC;wBACjC,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBACjD,IAAI,MAAM,EAAE,CAAC;wBACT,MAAM,CAAC,MAAM,EAAE,CAAC;oBACpB,CAAC;oBACD,OAAO,CAAC;wBACJ,UAAU,EAAE,MAAM;wBAClB,UAAU;wBACV,IAAI,EAAE,SAAS;wBACf,OAAO,EAAE,OAAuB;qBACnC,CAAC,CAAC;oBACH,OAAO;gBACX,CAAC;gBAED,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,OAAO,CAAC;wBACJ,UAAU,EAAE,MAAM;wBAClB,UAAU;wBACV,IAAI,EAAE,SAAS;wBACf,OAAO,EAAE,OAAuB;qBACnC,CAAC,CAAC;oBACH,OAAO;gBACX,CAAC;gBAED,QAAQ,eAAe,EAAE,CAAC;oBACtB,KAAK,MAAM,CAAC,CAAC,CAAC;wBACV,MAAM,YAAY,GAAG,IAAA,mBAAY,GAAE,CAAC;wBACpC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;4BACzB,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC9B,CAAC,CAAC,CAAC;wBACH,MAAM,GAAG,YAAY,CAAC;wBACtB,MAAM;oBACV,CAAC;oBACD,KAAK,SAAS,CAAC,CAAC,CAAC;wBACb,MAAM,YAAY,GAAG,IAAA,oBAAa,GAAE,CAAC;wBACrC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;4BACzB,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC9B,CAAC,CAAC,CAAC;wBACH,MAAM,GAAG,YAAY,CAAC;wBACtB,MAAM;oBACV,CAAC;oBACD,KAAK,IAAI,CAAC,CAAC,CAAC;wBACR,MAAM,YAAY,GAAG,IAAA,6BAAsB,GAAE,CAAC;wBAC9C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;4BACzB,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC9B,CAAC,CAAC,CAAC;wBACH,MAAM,GAAG,YAAY,CAAC;wBACtB,MAAM;oBACV,CAAC;oBACD,KAAK,MAAM,CAAC,CAAC,CAAC;wBACV,MAAM,YAAY,GAAG,IAAA,4BAAc,GAAE,CAAC;wBACtC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;4BACzB,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC9B,CAAC,CAAC,CAAC;wBACH,MAAM,GAAG,YAAY,CAAC;wBACtB,MAAM;oBACV,CAAC;oBACD,OAAO,CAAC,CAAC,CAAC;wBACN,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;gBAC3C,MAAM,SAAS,GAAG,kBAAS,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;gBAChD,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBAClD,OAAO,CAAC;oBACJ,UAAU,EAAE,MAAM;oBAClB,UAAU;oBACV,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,OAAuB;iBACnC,CAAC,CAAC;YACP,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,UAAe,EAAE,SAAgC;QAC7D,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,eAAe,GAAG,CAAC,CAAC;QACxB,IAAI,IAAI,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,UAAU,CAAC;QAC5B,MAAM,eAAe,GAAiB,EAAE,CAAC;QACzC,GAAG,CAAC;YACA,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;YAEtD,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;gBACnD,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAChD,MAAM,QAAQ,GAAuB,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;gBAE1E,MAAM,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;gBACvE,IAAI,eAAe,EAAE,CAAC;oBAClB,MAAM,iBAAiB,GAAG,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;oBAC/F,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,+BAAoB,EAAC,CAAC,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC7G,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,IAAI,CAAC,OAAO,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC;oBACxD,CAAC;oBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACZ,eAAe,IAAI,CAAC,CAAC;oBACzB,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,EAAE,CAAC;oBAChC,uBAAuB;oBACvB,OAAO;wBACH,UAAU,EAAE,CAAC,CAAC,UAAU;wBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;wBACZ,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;qBAC7C,CAAC;gBACN,CAAC;gBACD,IAAI,CAAC,QAAQ,IAAI,eAAe,GAAG,CAAC,EAAE,CAAC;oBACnC,MAAM,IAAI,gCAAuB,CAAC,oBAAoB,UAAU,2DAA2D,CAAC,CAAC;gBACjI,CAAC;gBAED,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;gBACjD,eAAe,IAAI,CAAC,CAAC;gBACrB,SAAS;YACb,CAAC;YAED,OAAO;gBACH,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,UAAU,EAAE,CAAC,CAAC,UAAU;gBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;aAC7C,CAAC;QACN,CAAC,QAAQ,eAAe,GAAG,CAAC,EAAE;QAE9B,MAAM,IAAI,+BAAsB,CAAC,oBAAoB,UAAU,0BAA0B,CAAC,CAAC;IAC/F,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAc,EAAE,SAAgC;QAC3D,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC9D,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC;QAClC,IAAI,QAAQ,GAAG,SAAS,CAAC;QACzB,MAAM,YAAY,GAAqC;YACnD,WAAW,EAAE,EAAE;YACf,WAAW,EAAE,EAAE;SAClB,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvC,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;gBACtC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,CAAC;gBAClC,OAAO,EAAE,gBAAC,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;gBAClC,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC;aAClE,CAAC;YACF,IAAI,SAAS,EAAE,QAAQ,EAAE,CAAC;gBACtB,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC;YACnE,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC;gBACtD,IAAI,QAAQ,EAAE,CAAC;oBACX,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC3C,CAAC;YACL,CAAC;QACL,CAAC;QACD,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtE,MAAM,WAAW,GAAG,CAAC,WAAW,CAAC,cAAc,CAAC,IAAI,WAAW,CAAC,cAAc,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,0BAA0B,CAAC;QACnK,MAAM,kBAAkB,GAAG,WAAW,CAAC,qBAAqB,CAAC,IAAI,WAAW,CAAC,qBAAqB,CAAC,CAAC;QACpG,MAAM,QAAQ,GAAG,kBAAkB,EAAE,KAAK,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;QAE7G,IAAI,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAC3E,YAAY,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;QACnE,CAAC;QAED,8GAA8G;QAC9G,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;QAErE,OAAO;YACH,QAAQ;YACR,YAAY;YACZ,KAAK,EAAE,UAAU,CAAC,OAAO;YACzB,MAAM,EAAE,UAAU,CAAC,UAAU;YAC7B,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,OAAO,EAAE,WAAW;YACpB,WAAW;YACX,kBAAkB;YAClB,QAAQ;YACR,IAAI,EAAE,UAAU,CAAC,IAAI;SACxB,CAAC;IACN,CAAC;IAED,cAAc,CAAC,IAAc,EAAE,GAAW;QACtC,QAAQ,IAAI,EAAE,CAAC;YACX,kBAAkB;YAClB,KAAK,uBAAQ,CAAC,0BAA0B,CAAC,CAAC,CAAC;gBACvC,OAAO,IAAI,8BAAqB,CAAC,GAAG,CAAC,CAAC;YAC1C,CAAC;YAED,6CAA6C;YAC7C,KAAK,uBAAQ,CAAC,wBAAwB,CAAC;YACvC,KAAK,uBAAQ,CAAC,0BAA0B,CAAC;YACzC,KAAK,uBAAQ,CAAC,8BAA8B,CAAC,CAAC,CAAC;gBAC3C,OAAO,IAAI,gCAAuB,CAAC,GAAG,CAAC,CAAC;YAC5C,CAAC;YAED,mBAAmB;YACnB,KAAK,uBAAQ,CAAC,0BAA0B,CAAC;YACzC,KAAK,uBAAQ,CAAC,gBAAgB,CAAC;YAC/B,KAAK,uBAAQ,CAAC,gBAAgB,CAAC;YAC/B,KAAK,uBAAQ,CAAC,iBAAiB,CAAC;YAChC,KAAK,uBAAQ,CAAC,uBAAuB,CAAC;YACtC,KAAK,uBAAQ,CAAC,wBAAwB,CAAC;YACvC,KAAK,uBAAQ,CAAC,2BAA2B,CAAC;YAC1C,KAAK,uBAAQ,CAAC,qBAAqB,CAAC;YACpC,KAAK,uBAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;gBAC/B,OAAO,IAAI,+BAAsB,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC;YAED,OAAO,CAAC,CAAC,CAAC;gBACN,OAAO,SAAS,CAAC;YACrB,CAAC;QACL,CAAC;IACL,CAAC;CACJ,CAAA;AA/aY,kCAAW;sBAAX,WAAW;IADvB,IAAA,oBAAS,GAAE;qCAaoB,qBAAY;QACT,2BAAe;QACb,iCAAiB;QACjB,sCAAiB;GAfzC,WAAW,CA+avB"}

build/services/errors.js

@@ -0,0 +1,101 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BatchSizeTooLargeError = exports.SecurityCompromiseError = exports.HarmfulContentError = exports.BudgetExceededError = exports.LockConflictError = exports.InsufficientBalanceError = exports.TierFeatureConstraintError = exports.InsufficientCreditsError = exports.EmailUnverifiedError = exports.ServiceBadApproachError = exports.ServiceBadAttemptError = exports.ServiceNodeResourceDrainError = exports.ServiceCrashedError = exports.ServiceDisabledError = void 0;
+const civ_rpc_1 = require("civkit/civ-rpc");
+const dayjs_1 = __importDefault(require("dayjs"));
+const utc_1 = __importDefault(require("dayjs/plugin/utc"));
+dayjs_1.default.extend(utc_1.default);
+let ServiceDisabledError = class ServiceDisabledError extends civ_rpc_1.ApplicationError {
+};
+exports.ServiceDisabledError = ServiceDisabledError;
+exports.ServiceDisabledError = ServiceDisabledError = __decorate([
+    (0, civ_rpc_1.StatusCode)(50301)
+], ServiceDisabledError);
+let ServiceCrashedError = class ServiceCrashedError extends civ_rpc_1.ApplicationError {
+};
+exports.ServiceCrashedError = ServiceCrashedError;
+exports.ServiceCrashedError = ServiceCrashedError = __decorate([
+    (0, civ_rpc_1.StatusCode)(50302)
+], ServiceCrashedError);
+let ServiceNodeResourceDrainError = class ServiceNodeResourceDrainError extends civ_rpc_1.ApplicationError {
+};
+exports.ServiceNodeResourceDrainError = ServiceNodeResourceDrainError;
+exports.ServiceNodeResourceDrainError = ServiceNodeResourceDrainError = __decorate([
+    (0, civ_rpc_1.StatusCode)(50303)
+], ServiceNodeResourceDrainError);
+let ServiceBadAttemptError = class ServiceBadAttemptError extends civ_rpc_1.ApplicationError {
+};
+exports.ServiceBadAttemptError = ServiceBadAttemptError;
+exports.ServiceBadAttemptError = ServiceBadAttemptError = __decorate([
+    (0, civ_rpc_1.StatusCode)(50304)
+], ServiceBadAttemptError);
+let ServiceBadApproachError = class ServiceBadApproachError extends ServiceBadAttemptError {
+};
+exports.ServiceBadApproachError = ServiceBadApproachError;
+exports.ServiceBadApproachError = ServiceBadApproachError = __decorate([
+    (0, civ_rpc_1.StatusCode)(50305)
+], ServiceBadApproachError);
+let EmailUnverifiedError = class EmailUnverifiedError extends civ_rpc_1.ApplicationError {
+};
+exports.EmailUnverifiedError = EmailUnverifiedError;
+exports.EmailUnverifiedError = EmailUnverifiedError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40104)
+], EmailUnverifiedError);
+let InsufficientCreditsError = class InsufficientCreditsError extends civ_rpc_1.ApplicationError {
+};
+exports.InsufficientCreditsError = InsufficientCreditsError;
+exports.InsufficientCreditsError = InsufficientCreditsError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40201)
+], InsufficientCreditsError);
+let TierFeatureConstraintError = class TierFeatureConstraintError extends civ_rpc_1.ApplicationError {
+};
+exports.TierFeatureConstraintError = TierFeatureConstraintError;
+exports.TierFeatureConstraintError = TierFeatureConstraintError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40202)
+], TierFeatureConstraintError);
+let InsufficientBalanceError = class InsufficientBalanceError extends civ_rpc_1.ApplicationError {
+};
+exports.InsufficientBalanceError = InsufficientBalanceError;
+exports.InsufficientBalanceError = InsufficientBalanceError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40203)
+], InsufficientBalanceError);
+let LockConflictError = class LockConflictError extends civ_rpc_1.ApplicationError {
+};
+exports.LockConflictError = LockConflictError;
+exports.LockConflictError = LockConflictError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40903)
+], LockConflictError);
+let BudgetExceededError = class BudgetExceededError extends civ_rpc_1.ApplicationError {
+};
+exports.BudgetExceededError = BudgetExceededError;
+exports.BudgetExceededError = BudgetExceededError = __decorate([
+    (0, civ_rpc_1.StatusCode)(40904)
+], BudgetExceededError);
+let HarmfulContentError = class HarmfulContentError extends civ_rpc_1.ApplicationError {
+};
+exports.HarmfulContentError = HarmfulContentError;
+exports.HarmfulContentError = HarmfulContentError = __decorate([
+    (0, civ_rpc_1.StatusCode)(45101)
+], HarmfulContentError);
+let SecurityCompromiseError = class SecurityCompromiseError extends civ_rpc_1.ApplicationError {
+};
+exports.SecurityCompromiseError = SecurityCompromiseError;
+exports.SecurityCompromiseError = SecurityCompromiseError = __decorate([
+    (0, civ_rpc_1.StatusCode)(45102)
+], SecurityCompromiseError);
+let BatchSizeTooLargeError = class BatchSizeTooLargeError extends civ_rpc_1.ApplicationError {
+};
+exports.BatchSizeTooLargeError = BatchSizeTooLargeError;
+exports.BatchSizeTooLargeError = BatchSizeTooLargeError = __decorate([
+    (0, civ_rpc_1.StatusCode)(41201)
+], BatchSizeTooLargeError);
+//# sourceMappingURL=errors.js.map