Spaces:
Build error
Build error
fix: base parameter
Browse files
backend/functions/src/cloud-functions/crawler.ts
CHANGED
|
@@ -832,7 +832,7 @@ export class CrawlerHost extends RPCHost {
|
|
| 832 |
nominalUrl?: URL,
|
| 833 |
urlValidMs?: number
|
| 834 |
) {
|
| 835 |
-
const presumedURL = crawlerOptions.base === '
|
| 836 |
|
| 837 |
const respondWith = crawlerOptions.respondWith;
|
| 838 |
if (respondWith === CONTENT_FORMAT.READER_LM || respondWith === CONTENT_FORMAT.VLM) {
|
|
|
|
| 832 |
nominalUrl?: URL,
|
| 833 |
urlValidMs?: number
|
| 834 |
) {
|
| 835 |
+
const presumedURL = crawlerOptions.base === 'final' ? new URL(snapshot.href) : nominalUrl;
|
| 836 |
|
| 837 |
const respondWith = crawlerOptions.respondWith;
|
| 838 |
if (respondWith === CONTENT_FORMAT.READER_LM || respondWith === CONTENT_FORMAT.VLM) {
|
backend/functions/src/dto/scrapping-options.ts
CHANGED
|
@@ -25,7 +25,7 @@ const CONTENT_FORMAT_VALUES = new Set<string>(Object.values(CONTENT_FORMAT));
|
|
| 25 |
|
| 26 |
export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const;
|
| 27 |
const IMAGE_RETENTION_MODE_VALUES = new Set<string>(IMAGE_RETENTION_MODES);
|
| 28 |
-
export const BASE_URL_MODES = ['initial', '
|
| 29 |
const BASE_URL_MODE_VALUES = new Set<string>(BASE_URL_MODES);
|
| 30 |
|
| 31 |
class Viewport extends AutoCastable {
|
|
@@ -199,7 +199,7 @@ class Viewport extends AutoCastable {
|
|
| 199 |
schema: { type: 'string' }
|
| 200 |
},
|
| 201 |
'X-Base': {
|
| 202 |
-
description: 'Select base modes of relative URLs.\n\nSupported: initial,
|
| 203 |
in: 'header',
|
| 204 |
schema: { type: 'string' }
|
| 205 |
},
|
|
@@ -490,7 +490,9 @@ export class CrawlerOptions extends AutoCastable {
|
|
| 490 |
instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;
|
| 491 |
|
| 492 |
const baseMode = ctx?.req.get('x-base') || undefined;
|
| 493 |
-
|
|
|
|
|
|
|
| 494 |
|
| 495 |
if (instance.cacheTolerance) {
|
| 496 |
instance.cacheTolerance = instance.cacheTolerance * 1000;
|
|
|
|
| 25 |
|
| 26 |
export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const;
|
| 27 |
const IMAGE_RETENTION_MODE_VALUES = new Set<string>(IMAGE_RETENTION_MODES);
|
| 28 |
+
export const BASE_URL_MODES = ['initial', 'final'] as const;
|
| 29 |
const BASE_URL_MODE_VALUES = new Set<string>(BASE_URL_MODES);
|
| 30 |
|
| 31 |
class Viewport extends AutoCastable {
|
|
|
|
| 199 |
schema: { type: 'string' }
|
| 200 |
},
|
| 201 |
'X-Base': {
|
| 202 |
+
description: 'Select base modes of relative URLs.\n\nSupported: initial, final',
|
| 203 |
in: 'header',
|
| 204 |
schema: { type: 'string' }
|
| 205 |
},
|
|
|
|
| 490 |
instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;
|
| 491 |
|
| 492 |
const baseMode = ctx?.req.get('x-base') || undefined;
|
| 493 |
+
if (baseMode) {
|
| 494 |
+
instance.base = baseMode as any;
|
| 495 |
+
}
|
| 496 |
|
| 497 |
if (instance.cacheTolerance) {
|
| 498 |
instance.cacheTolerance = instance.cacheTolerance * 1000;
|