saas: use compression

package-lock.json

@@ -26,6 +26,7 @@
         "htmlparser2": "^9.0.0",
         "jose": "^5.1.0",
         "koa": "^2.16.0",
+        "koa-compress": "^5.1.1",
         "langdetect": "^0.2.1",
         "linkedom": "^0.18.4",
         "lru-cache": "^11.0.2",
@@ -54,6 +55,7 @@
         "@types/busboy": "^1.5.4",
         "@types/cors": "^2.8.17",
         "@types/koa": "^2.15.0",
+        "@types/koa-compress": "^4.0.6",
         "@types/node": "^20.14.13",
         "@types/set-cookie-parser": "^2.4.7",
         "@types/xmldom": "^0.1.34",
@@ -2508,6 +2510,17 @@
         "@types/koa": "*"
       }
     },
+    "node_modules/@types/koa-compress": {
+      "version": "4.0.6",
+      "resolved": "https://registry.npmjs.org/@types/koa-compress/-/koa-compress-4.0.6.tgz",
+      "integrity": "sha512-jJzlsQTeIXRfQzY7y2Dl/syqOxvEaNJzFTJ8ZGaDFWvqgMmCXplfCGjPfHQgU7xYNKq/mCdN3YWus1YOldfIwg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/koa": "*",
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/lodash": {
       "version": "4.17.0",
       "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.0.tgz",
@@ -4197,6 +4210,18 @@
         "node": ">= 12.0.0"
       }
     },
+    "node_modules/compressible": {
+      "version": "2.0.18",
+      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
+      "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": ">= 1.43.0 < 2"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -8313,6 +8338,55 @@
       "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz",
       "integrity": "sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw=="
     },
+    "node_modules/koa-compress": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/koa-compress/-/koa-compress-5.1.1.tgz",
+      "integrity": "sha512-UgMIN7ZoEP2DuoSQmD6CYvFSLt0NReGlc2qSY4bO4Oq0L56OiD9pDG41Kj/zFmVY/A3Wvmn4BqKcfq5H30LGIg==",
+      "license": "MIT",
+      "dependencies": {
+        "bytes": "^3.1.2",
+        "compressible": "^2.0.18",
+        "http-errors": "^1.8.1",
+        "koa-is-json": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 12"
+      }
+    },
+    "node_modules/koa-compress/node_modules/depd": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
+      "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/koa-compress/node_modules/http-errors": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",
+      "integrity": "sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==",
+      "license": "MIT",
+      "dependencies": {
+        "depd": "~1.1.2",
+        "inherits": "2.0.4",
+        "setprototypeof": "1.2.0",
+        "statuses": ">= 1.5.0 < 2",
+        "toidentifier": "1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/koa-compress/node_modules/statuses": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
+      "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/koa-convert": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/koa-convert/-/koa-convert-2.0.0.tgz",
@@ -8325,6 +8399,12 @@
         "node": ">= 10"
       }
     },
+    "node_modules/koa-is-json": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/koa-is-json/-/koa-is-json-1.0.0.tgz",
+      "integrity": "sha512-+97CtHAlWDx0ndt0J8y3P12EWLwTLMXIfMnYDev3wOTwH/RpBGMlfn4bDXlMEg1u73K6XRE9BbUp+5ZAYoRYWw==",
+      "license": "MIT"
+    },
     "node_modules/koa/node_modules/http-errors": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.1.tgz",

package.json

@@ -35,6 +35,7 @@
     "htmlparser2": "^9.0.0",
     "jose": "^5.1.0",
     "koa": "^2.16.0",
+    "koa-compress": "^5.1.1",
     "langdetect": "^0.2.1",
     "linkedom": "^0.18.4",
     "lru-cache": "^11.0.2",
@@ -63,6 +64,7 @@
     "@types/busboy": "^1.5.4",
     "@types/cors": "^2.8.17",
     "@types/koa": "^2.15.0",
+    "@types/koa-compress": "^4.0.6",
     "@types/node": "^20.14.13",
     "@types/set-cookie-parser": "^2.4.7",
     "@types/xmldom": "^0.1.34",

src/stand-alone/crawl.ts

@@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded';
 import { GlobalLogger } from '../services/logger';
 import { AsyncLocalContext } from '../services/async-context';
 import finalizer, { Finalizer } from '../services/finalizer';
+import koaCompress from 'koa-compress';
 
 @singleton()
 export class CrawlStandAloneServer extends KoaServer {
@@ -98,6 +99,23 @@ export class CrawlStandAloneServer extends KoaServer {
     }
 
     registerRoutes(): void {
+        this.koaApp.use(koaCompress({
+            filter(type) {
+                if (type.startsWith('text/')) {
+                    return true;
+                }
+
+                if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
+                    return true;
+                }
+
+                if (type.includes('application/x-ndjson')) {
+                    return true;
+                }
+
+                return false;
+            }
+        }));
         this.koaApp.use(this.makeAssetsServingController());
         this.koaApp.use(this.registry.makeShimController());
     }

src/stand-alone/search.ts

@@ -18,6 +18,7 @@ import { ThreadedServiceRegistry } from '../services/threaded';
 import { GlobalLogger } from '../services/logger';
 import { AsyncLocalContext } from '../services/async-context';
 import finalizer, { Finalizer } from '../services/finalizer';
+import koaCompress from 'koa-compress';
 
 @singleton()
 export class SearchStandAloneServer extends KoaServer {
@@ -106,6 +107,23 @@ export class SearchStandAloneServer extends KoaServer {
     }
 
     registerRoutes(): void {
+        this.koaApp.use(koaCompress({
+            filter(type) {
+                if (type.startsWith('text/')) {
+                    return true;
+                }
+
+                if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
+                    return true;
+                }
+
+                if (type.includes('application/x-ndjson')) {
+                    return true;
+                }
+
+                return false;
+            }
+        }));
         this.koaApp.use(this.makeAssetsServingController());
         this.koaApp.use(this.registry.makeShimController());
     }

src/stand-alone/serp.ts

@@ -18,6 +18,7 @@ import { GlobalLogger } from '../services/logger';
 import { AsyncLocalContext } from '../services/async-context';
 import finalizer, { Finalizer } from '../services/finalizer';
 import { SerpHost } from '../api/serp';
+import koaCompress from 'koa-compress';
 
 @singleton()
 export class SERPStandAloneServer extends KoaServer {
@@ -106,6 +107,23 @@ export class SERPStandAloneServer extends KoaServer {
     }
 
     registerRoutes(): void {
+        this.koaApp.use(koaCompress({
+            filter(type) {
+                if (type.startsWith('text/')) {
+                    return true;
+                }
+
+                if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) {
+                    return true;
+                }
+
+                if (type.includes('application/x-ndjson')) {
+                    return true;
+                }
+
+                return false;
+            }
+        }));
         this.koaApp.use(this.makeAssetsServingController());
         this.koaApp.use(this.registry.makeShimController());
     }