build/services/blackhole-detector.js

"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
    return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BlackHoleDetector = void 0;
const tsyringe_1 = require("tsyringe");
const async_service_1 = require("civkit/async-service");
const logger_1 = require("./logger");
const timeout_1 = require("civkit/timeout");
let BlackHoleDetector = class BlackHoleDetector extends async_service_1.AsyncService {
    constructor(globalLogger) {
        super(...arguments);
        this.globalLogger = globalLogger;
        this.logger = this.globalLogger.child({ service: this.constructor.name });
        this.maxDelay = 1000 * 30;
        this.concurrentRequests = 0;
        this.strikes = 0;
        if (process.env.NODE_ENV?.startsWith('prod')) {
            setInterval(() => {
                this.routine();
            }, 1000 * 30).unref();
        }
    }
    async init() {
        await this.dependencyReady();
        this.logger.debug('BlackHoleDetector started');
        this.emit('ready');
    }
    async routine() {
        // We give routine a 3s grace period for potentially paused CPU to spin up and process some requests
        await (0, timeout_1.delay)(3000);
        const now = Date.now();
        const lastWorked = this.lastWorkedTs;
        if (!lastWorked) {
            return;
        }
        const dt = (now - lastWorked);
        if (this.concurrentRequests > 1 &&
            this.lastIncomingRequestTs && lastWorked &&
            this.lastIncomingRequestTs >= lastWorked &&
            (dt > (this.maxDelay * (this.strikes + 1)))) {
            this.logger.warn(`BlackHole detected, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);
            this.strikes += 1;
        }
        if (this.strikes >= 3) {
            this.logger.error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`);
            process.nextTick(() => {
                this.emit('error', new Error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`));
                // process.exit(1);
            });
        }
    }
    incomingRequest() {
        this.lastIncomingRequestTs = Date.now();
        this.lastWorkedTs ??= Date.now();
        this.concurrentRequests++;
    }
    doneWithRequest() {
        this.concurrentRequests--;
        this.lastDoneRequestTs = Date.now();
    }
    itWorked() {
        this.lastWorkedTs = Date.now();
        this.strikes = 0;
    }
};
exports.BlackHoleDetector = BlackHoleDetector;
exports.BlackHoleDetector = BlackHoleDetector = __decorate([
    (0, tsyringe_1.singleton)(),
    __metadata("design:paramtypes", [logger_1.GlobalLogger])
], BlackHoleDetector);
;
//# sourceMappingURL=blackhole-detector.js.map