Spaces:
Build error
Build error
fix: caching condition
Browse files- src/api/crawler.ts +6 -1
src/api/crawler.ts
CHANGED
|
@@ -112,7 +112,12 @@ export class CrawlerHost extends RPCHost {
|
|
| 112 |
const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);
|
| 113 |
if (analyzed.tokens < 200) {
|
| 114 |
// Does not contain enough content
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
}
|
| 117 |
|
| 118 |
await this.setToCache(options.url, snapshot);
|
|
|
|
| 112 |
const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);
|
| 113 |
if (analyzed.tokens < 200) {
|
| 114 |
// Does not contain enough content
|
| 115 |
+
if (snapshot.status !== 200) {
|
| 116 |
+
return;
|
| 117 |
+
}
|
| 118 |
+
if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile')) {
|
| 119 |
+
return;
|
| 120 |
+
}
|
| 121 |
}
|
| 122 |
|
| 123 |
await this.setToCache(options.url, snapshot);
|