fix: saas issus

src/api/searcher-serper.ts

@@ -6,7 +6,7 @@ import { marshalErrorLike } from 'civkit/lang';
 import { objHashMd5B64Of } from 'civkit/hash';
 import _ from 'lodash';
 
-import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit';
+import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';
 
 import { CrawlerHost, ExtraScrappingOptions } from './crawler';
 import { SerperSearchResult } from '../db/searched';
@@ -19,8 +19,16 @@ import { AsyncLocalContext } from '../services/async-context';
 import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
 import { OutputServerEventStream } from '../lib/transform-server-event-stream';
 import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
-import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors';
-import { SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperSearchResponse, SerperWebSearchResponse, WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
+import { InsufficientBalanceError } from '../services/errors';
+import {
+    SerperImageSearchResponse,
+    SerperNewsSearchResponse,
+    SerperSearchQueryParams,
+    SerperSearchResponse,
+    SerperWebSearchResponse,
+    WORLD_COUNTRIES,
+    WORLD_LANGUAGES
+} from '../shared/3rd-party/serper-search';
 import { toAsyncGenerator } from '../utils/misc';
 import type { JinaEmbeddingsTokenAccount } from '../shared/db/jina-embeddings-token-account';
 import { LRUCache } from 'lru-cache';
@@ -218,10 +226,10 @@ export class SearcherHost extends RPCHost {
                     }
                     const now = Date.now();
                     let tgtDate;
-                    if (err.retryAfter) {
-                        tgtDate = new Date(now + err.retryAfter * 1000);
-                    } else if (err.retryAfterDate) {
+                    if (err.retryAfterDate) {
                         tgtDate = err.retryAfterDate;
+                    } else if (err.retryAfter) {
+                        tgtDate = new Date(now + err.retryAfter * 1000);
                     }
 
                     if (tgtDate) {

src/api/serp.ts

@@ -7,14 +7,14 @@ import {
 import { marshalErrorLike } from 'civkit/lang';
 import _ from 'lodash';
 
-import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit';
+import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit';
 
 import { GlobalLogger } from '../services/logger';
 import { AsyncLocalContext } from '../services/async-context';
 import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
 import { OutputServerEventStream } from '../lib/transform-server-event-stream';
 import { JinaEmbeddingsAuthDTO } from '../dto/jina-embeddings-auth';
-import { InsufficientBalanceError, RateLimitTriggeredError } from '../services/errors';
+import { InsufficientBalanceError } from '../services/errors';
 import { WORLD_COUNTRIES, WORLD_LANGUAGES } from '../shared/3rd-party/serper-search';
 import { GoogleSERP } from '../services/serp/google';
 import { WebSearchEntry } from '../services/serp/compat';
@@ -172,10 +172,11 @@ export class SerpHost extends RPCHost {
             const now = new Date();
             const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf());
             if (blockedTimeRemaining > 0) {
-                throw RateLimitTriggeredError.from({
-                    message: `Per UID rate limit exceeded (async)`,
-                    retryAfter: Math.ceil(blockedTimeRemaining / 1000),
-                });
+                this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`);
+                // throw RateLimitTriggeredError.from({
+                //     message: `Per UID rate limit exceeded (async)`,
+                //     retryAfter: Math.ceil(blockedTimeRemaining / 1000),
+                // });
             }
         }
 
@@ -229,10 +230,10 @@ export class SerpHost extends RPCHost {
                     }
                     const now = Date.now();
                     let tgtDate;
-                    if (err.retryAfter) {
-                        tgtDate = new Date(now + err.retryAfter * 1000);
-                    } else if (err.retryAfterDate) {
+                    if (err.retryAfterDate) {
                         tgtDate = err.retryAfterDate;
+                    } else if (err.retryAfter) {
+                        tgtDate = new Date(now + err.retryAfter * 1000);
                     }
 
                     if (tgtDate) {

src/services/curl.ts

@@ -61,10 +61,23 @@ export class CurlControl extends AsyncService {
     }
 
     curlImpersonateHeader(curl: Curl, headers?: object) {
+        let uaPlatform = this.platform;
+        if (this.ua.includes('Windows')) {
+            uaPlatform = 'Windows';
+        } else if (this.ua.includes('Android')) {
+            uaPlatform = 'Android';
+        } else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) {
+            uaPlatform = 'iOS';
+        } else if (this.ua.includes('CrOS')) {
+            uaPlatform = 'Chrome OS';
+        } else if (this.ua.includes('Macintosh')) {
+            uaPlatform = 'macOS';
+        }
+
         const mixinHeaders: Record<string, string> = {
-            'sch-ch-ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`,
-            'sec-ch-ua-mobile': '?0',
-            'sec-ch-ua-platform': this.platform,
+            'Sec-Ch-Ua': `Not A(Brand";v="8", "Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}"`,
+            'Sec-Ch-Ua-Mobile': '?0',
+            'Sec-Ch-Ua-Platform': `"${uaPlatform}"`,
             'Upgrade-Insecure-Requests': '1',
             'User-Agent': this.ua,
             'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',

src/services/errors.ts

@@ -1,4 +1,4 @@
-import { ApplicationError, Prop, RPC_TRANSFER_PROTOCOL_META_SYMBOL, StatusCode } from 'civkit/civ-rpc';
+import { ApplicationError, StatusCode } from 'civkit/civ-rpc';
 import _ from 'lodash';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
@@ -46,31 +46,3 @@ export class SecurityCompromiseError extends ApplicationError { }
 
 @StatusCode(41201)
 export class BatchSizeTooLargeError extends ApplicationError { }
-
-
-@StatusCode(42903)
-export class RateLimitTriggeredError extends ApplicationError {
-
-    @Prop({
-        desc: 'Retry after seconds',
-    })
-    retryAfter?: number;
-
-    @Prop({
-        desc: 'Retry after date',
-    })
-    retryAfterDate?: Date;
-
-    protected override get [RPC_TRANSFER_PROTOCOL_META_SYMBOL]() {
-        const retryAfter = this.retryAfter || this.retryAfterDate;
-        if (!retryAfter) {
-            return super[RPC_TRANSFER_PROTOCOL_META_SYMBOL];
-        }
-
-        return _.merge(_.cloneDeep(super[RPC_TRANSFER_PROTOCOL_META_SYMBOL]), {
-            headers: {
-                'Retry-After': `${retryAfter instanceof Date ? dayjs(retryAfter).utc().format('ddd, DD MMM YYYY HH:mm:ss [GMT]') : retryAfter}`,
-            }
-        });
-    }
-}

src/services/serper-search.ts

@@ -56,6 +56,7 @@ export class SerperSearchService extends AsyncService {
         let maxTries = 3;
 
         while (maxTries--) {
+            const t0 = Date.now();
             try {
                 this.logger.debug(`Doing external search`, query);
                 let r;
@@ -101,11 +102,14 @@ export class SerperSearchService extends AsyncService {
                         break;
                     }
                 }
+                const dt = Date.now() - t0;
                 this.blackHoleDetector.itWorked();
+                this.logger.debug(`External search took ${dt}ms`, { searchDt: dt, variant });
 
                 return r.parsed;
             } catch (err: any) {
-                this.logger.error(`${variant} search failed: ${err?.message}`, { err: marshalErrorLike(err) });
+                const dt = Date.now() - t0;
+                this.logger.error(`${variant} search failed: ${err?.message}`, { searchDt: dt, err: marshalErrorLike(err) });
                 if (err?.status === 429) {
                     await delay(500 + 1000 * Math.random());
                     continue;

thinapps-shared

@@ -1 +1 @@
-Subproject commit f30577e73eca151f9d62198ae3b60d0172befaed
+Subproject commit 580ea72e0eddaa115b85dabf29de41d079ecd2d0