nomagick commited on
Commit
f0560c6
·
unverified ·
1 Parent(s): 0da71ca

fix: caching condition

Browse files
Files changed (1) hide show
  1. src/api/crawler.ts +6 -1
src/api/crawler.ts CHANGED
@@ -112,7 +112,12 @@ export class CrawlerHost extends RPCHost {
112
  const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);
113
  if (analyzed.tokens < 200) {
114
  // Does not contain enough content
115
- return;
 
 
 
 
 
116
  }
117
 
118
  await this.setToCache(options.url, snapshot);
 
112
  const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html);
113
  if (analyzed.tokens < 200) {
114
  // Does not contain enough content
115
+ if (snapshot.status !== 200) {
116
+ return;
117
+ }
118
+ if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile')) {
119
+ return;
120
+ }
121
  }
122
 
123
  await this.setToCache(options.url, snapshot);