Spaces:
Build error
Build error
fix: stop early return when timeout is explicitly defined
Browse files
backend/functions/src/cloud-functions/crawler.ts
CHANGED
|
@@ -710,7 +710,9 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
|
|
| 710 |
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
|
| 711 |
chargeAmount = this.getChargeAmount(formatted);
|
| 712 |
|
| 713 |
-
|
|
|
|
|
|
|
| 714 |
}
|
| 715 |
|
| 716 |
if (!lastScrapped) {
|
|
@@ -731,14 +733,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
|
|
| 731 |
|
| 732 |
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
|
| 733 |
chargeAmount = this.getChargeAmount(formatted);
|
| 734 |
-
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
|
| 735 |
|
| 736 |
-
|
| 737 |
-
|
| 738 |
-
);
|
| 739 |
-
}
|
| 740 |
|
| 741 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 742 |
}
|
| 743 |
|
| 744 |
if (!lastScrapped) {
|
|
|
|
| 710 |
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
|
| 711 |
chargeAmount = this.getChargeAmount(formatted);
|
| 712 |
|
| 713 |
+
if (crawlerOptions.timeout !== null) {
|
| 714 |
+
return formatted;
|
| 715 |
+
}
|
| 716 |
}
|
| 717 |
|
| 718 |
if (!lastScrapped) {
|
|
|
|
| 733 |
|
| 734 |
const formatted = await this.formatSnapshot(crawlerOptions.respondWith, scrapped, urlToCrawl);
|
| 735 |
chargeAmount = this.getChargeAmount(formatted);
|
|
|
|
| 736 |
|
| 737 |
+
if (crawlerOptions.timeout !== null) {
|
| 738 |
+
if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) {
|
|
|
|
|
|
|
| 739 |
|
| 740 |
+
return assignTransferProtocolMeta(`${formatted}`,
|
| 741 |
+
{ code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } }
|
| 742 |
+
);
|
| 743 |
+
}
|
| 744 |
+
|
| 745 |
+
return assignTransferProtocolMeta(`${formatted}`, { contentType: 'text/plain', envelope: null });
|
| 746 |
+
}
|
| 747 |
}
|
| 748 |
|
| 749 |
if (!lastScrapped) {
|
backend/functions/src/dto/scrapping-options.ts
CHANGED
|
@@ -171,8 +171,10 @@ export class CrawlerOptions extends AutoCastable {
|
|
| 171 |
|
| 172 |
@Prop({
|
| 173 |
validate: (v: number) => v > 0 && v <= 180,
|
|
|
|
|
|
|
| 174 |
})
|
| 175 |
-
timeout?: number;
|
| 176 |
|
| 177 |
static override from(input: any) {
|
| 178 |
const instance = super.from(input) as CrawlerOptions;
|
|
@@ -213,6 +215,8 @@ export class CrawlerOptions extends AutoCastable {
|
|
| 213 |
let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || '');
|
| 214 |
if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) {
|
| 215 |
instance.timeout = timeoutSeconds;
|
|
|
|
|
|
|
| 216 |
}
|
| 217 |
|
| 218 |
const removeSelector = ctx?.req.get('x-remove-selector')?.split(', ');
|
|
|
|
| 171 |
|
| 172 |
@Prop({
|
| 173 |
validate: (v: number) => v > 0 && v <= 180,
|
| 174 |
+
type: Number,
|
| 175 |
+
nullable: true,
|
| 176 |
})
|
| 177 |
+
timeout?: number | null;
|
| 178 |
|
| 179 |
static override from(input: any) {
|
| 180 |
const instance = super.from(input) as CrawlerOptions;
|
|
|
|
| 215 |
let timeoutSeconds = parseInt(ctx?.req.get('x-timeout') || '');
|
| 216 |
if (!isNaN(timeoutSeconds) && timeoutSeconds > 0 && timeoutSeconds <= 180) {
|
| 217 |
instance.timeout = timeoutSeconds;
|
| 218 |
+
} else if (ctx?.req.get('x-timeout')) {
|
| 219 |
+
instance.timeout = null;
|
| 220 |
}
|
| 221 |
|
| 222 |
const removeSelector = ctx?.req.get('x-remove-selector')?.split(', ');
|
thinapps-shared
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
Subproject commit
|
|
|
|
| 1 |
+
Subproject commit b7fe87fca31e7fbc22db00391b95f3a32dcad997
|