Spaces:
Build error
Build error
fix: potential fix for firestore grpc connection reset
Browse files- Dockerfile +1 -1
- src/api/crawler.ts +13 -1
Dockerfile
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
# syntax=docker/dockerfile:1
|
| 2 |
FROM lwthiker/curl-impersonate:0.6-chrome-slim-bullseye
|
| 3 |
|
| 4 |
-
FROM node:
|
| 5 |
|
| 6 |
RUN apt-get update \
|
| 7 |
&& apt-get install -y wget gnupg \
|
|
|
|
| 1 |
# syntax=docker/dockerfile:1
|
| 2 |
FROM lwthiker/curl-impersonate:0.6-chrome-slim-bullseye
|
| 3 |
|
| 4 |
+
FROM node:22
|
| 5 |
|
| 6 |
RUN apt-get update \
|
| 7 |
&& apt-get install -y wget gnupg \
|
src/api/crawler.ts
CHANGED
|
@@ -519,7 +519,19 @@ export class CrawlerHost extends RPCHost {
|
|
| 519 |
async queryCache(urlToCrawl: URL, cacheTolerance: number) {
|
| 520 |
const digest = this.getUrlDigest(urlToCrawl);
|
| 521 |
|
| 522 |
-
const cache = (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 523 |
|
| 524 |
if (!cache) {
|
| 525 |
return undefined;
|
|
|
|
| 519 |
async queryCache(urlToCrawl: URL, cacheTolerance: number) {
|
| 520 |
const digest = this.getUrlDigest(urlToCrawl);
|
| 521 |
|
| 522 |
+
const cache = (
|
| 523 |
+
await
|
| 524 |
+
(Crawled.fromFirestoreQuery(
|
| 525 |
+
Crawled.COLLECTION.where('urlPathDigest', '==', digest).orderBy('createdAt', 'desc').limit(1)
|
| 526 |
+
).catch((err) => {
|
| 527 |
+
this.logger.warn(`Failed to query cache, unknown issue`, { err });
|
| 528 |
+
// https://github.com/grpc/grpc-node/issues/2647
|
| 529 |
+
// https://github.com/googleapis/nodejs-firestore/issues/1023
|
| 530 |
+
// https://github.com/googleapis/nodejs-firestore/issues/1023
|
| 531 |
+
|
| 532 |
+
return undefined;
|
| 533 |
+
}))
|
| 534 |
+
)?.[0];
|
| 535 |
|
| 536 |
if (!cache) {
|
| 537 |
return undefined;
|