| const { v4: uuidv4 } = require('uuid') |
| const crypto = require('crypto') |
| const ProxyHelper = require('../utils/proxyHelper') |
| const axios = require('axios') |
| const redis = require('../models/redis') |
| const config = require('../../config/config') |
| const logger = require('../utils/logger') |
| const { maskToken } = require('../utils/tokenMask') |
| const { |
| logRefreshStart, |
| logRefreshSuccess, |
| logRefreshError, |
| logTokenUsage, |
| logRefreshSkipped |
| } = require('../utils/tokenRefreshLogger') |
| const tokenRefreshService = require('./tokenRefreshService') |
| const LRUCache = require('../utils/lruCache') |
| const { formatDateWithTimezone, getISOStringWithTimezone } = require('../utils/dateHelper') |
|
|
| class ClaudeAccountService { |
| constructor() { |
| this.claudeApiUrl = 'https://console.anthropic.com/v1/oauth/token' |
| this.claudeOauthClientId = '9d1c250a-e61b-44d9-88ed-5944d1962f5e' |
| let maxWarnings = parseInt(process.env.CLAUDE_5H_WARNING_MAX_NOTIFICATIONS || '', 10) |
|
|
| if (Number.isNaN(maxWarnings) && config.claude?.fiveHourWarning) { |
| maxWarnings = parseInt(config.claude.fiveHourWarning.maxNotificationsPerWindow, 10) |
| } |
|
|
| if (Number.isNaN(maxWarnings) || maxWarnings < 1) { |
| maxWarnings = 1 |
| } |
|
|
| this.maxFiveHourWarningsPerWindow = Math.min(maxWarnings, 10) |
|
|
| |
| this.ENCRYPTION_ALGORITHM = 'aes-256-cbc' |
| this.ENCRYPTION_SALT = 'salt' |
|
|
| |
| |
| this._encryptionKeyCache = null |
|
|
| |
| this._decryptCache = new LRUCache(500) |
|
|
| |
| setInterval( |
| () => { |
| this._decryptCache.cleanup() |
| logger.info('🧹 Claude decrypt cache cleanup completed', this._decryptCache.getStats()) |
| }, |
| 10 * 60 * 1000 |
| ) |
| } |
|
|
| |
| async createAccount(options = {}) { |
| const { |
| name = 'Unnamed Account', |
| description = '', |
| email = '', |
| password = '', |
| refreshToken = '', |
| claudeAiOauth = null, |
| proxy = null, |
| isActive = true, |
| accountType = 'shared', |
| platform = 'claude', |
| priority = 50, |
| schedulable = true, |
| subscriptionInfo = null, |
| autoStopOnWarning = false, |
| useUnifiedUserAgent = false, |
| useUnifiedClientId = false, |
| unifiedClientId = '', |
| expiresAt = null |
| } = options |
|
|
| const accountId = uuidv4() |
|
|
| let accountData |
|
|
| if (claudeAiOauth) { |
| |
| accountData = { |
| id: accountId, |
| name, |
| description, |
| email: this._encryptSensitiveData(email), |
| password: this._encryptSensitiveData(password), |
| claudeAiOauth: this._encryptSensitiveData(JSON.stringify(claudeAiOauth)), |
| accessToken: this._encryptSensitiveData(claudeAiOauth.accessToken), |
| refreshToken: this._encryptSensitiveData(claudeAiOauth.refreshToken), |
| expiresAt: claudeAiOauth.expiresAt.toString(), |
| scopes: claudeAiOauth.scopes.join(' '), |
| proxy: proxy ? JSON.stringify(proxy) : '', |
| isActive: isActive.toString(), |
| accountType, |
| platform, |
| priority: priority.toString(), |
| createdAt: new Date().toISOString(), |
| lastUsedAt: '', |
| lastRefreshAt: '', |
| status: 'active', |
| errorMessage: '', |
| schedulable: schedulable.toString(), |
| autoStopOnWarning: autoStopOnWarning.toString(), |
| useUnifiedUserAgent: useUnifiedUserAgent.toString(), |
| useUnifiedClientId: useUnifiedClientId.toString(), |
| unifiedClientId: unifiedClientId || '', |
| |
| subscriptionInfo: subscriptionInfo |
| ? JSON.stringify(subscriptionInfo) |
| : claudeAiOauth.subscriptionInfo |
| ? JSON.stringify(claudeAiOauth.subscriptionInfo) |
| : '', |
| |
| subscriptionExpiresAt: expiresAt || '' |
| } |
| } else { |
| |
| accountData = { |
| id: accountId, |
| name, |
| description, |
| email: this._encryptSensitiveData(email), |
| password: this._encryptSensitiveData(password), |
| refreshToken: this._encryptSensitiveData(refreshToken), |
| accessToken: '', |
| expiresAt: '', |
| scopes: '', |
| proxy: proxy ? JSON.stringify(proxy) : '', |
| isActive: isActive.toString(), |
| accountType, |
| platform, |
| priority: priority.toString(), |
| createdAt: new Date().toISOString(), |
| lastUsedAt: '', |
| lastRefreshAt: '', |
| status: 'created', |
| errorMessage: '', |
| schedulable: schedulable.toString(), |
| autoStopOnWarning: autoStopOnWarning.toString(), |
| useUnifiedUserAgent: useUnifiedUserAgent.toString(), |
| |
| subscriptionInfo: subscriptionInfo ? JSON.stringify(subscriptionInfo) : '', |
| |
| subscriptionExpiresAt: expiresAt || '' |
| } |
| } |
|
|
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| logger.success(`🏢 Created Claude account: ${name} (${accountId})`) |
|
|
| |
| if (claudeAiOauth && claudeAiOauth.accessToken) { |
| |
| const hasProfileScope = claudeAiOauth.scopes && claudeAiOauth.scopes.includes('user:profile') |
|
|
| if (hasProfileScope) { |
| try { |
| const agent = this._createProxyAgent(proxy) |
| await this.fetchAndUpdateAccountProfile(accountId, claudeAiOauth.accessToken, agent) |
| logger.info(`📊 Successfully fetched profile info for new account: ${name}`) |
| } catch (profileError) { |
| logger.warn(`⚠️ Failed to fetch profile info for new account: ${profileError.message}`) |
| } |
| } else { |
| logger.info(`⏩ Skipping profile fetch for account ${name} (no user:profile scope)`) |
| } |
| } |
|
|
| return { |
| id: accountId, |
| name, |
| description, |
| email, |
| isActive, |
| proxy, |
| accountType, |
| platform, |
| priority, |
| status: accountData.status, |
| createdAt: accountData.createdAt, |
| expiresAt: accountData.expiresAt, |
| subscriptionExpiresAt: |
| accountData.subscriptionExpiresAt && accountData.subscriptionExpiresAt !== '' |
| ? accountData.subscriptionExpiresAt |
| : null, |
| scopes: claudeAiOauth ? claudeAiOauth.scopes : [], |
| autoStopOnWarning, |
| useUnifiedUserAgent, |
| useUnifiedClientId, |
| unifiedClientId |
| } |
| } |
|
|
| |
| async refreshAccountToken(accountId) { |
| let lockAcquired = false |
|
|
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
|
|
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| const refreshToken = this._decryptSensitiveData(accountData.refreshToken) |
|
|
| if (!refreshToken) { |
| throw new Error('No refresh token available - manual token update required') |
| } |
|
|
| |
| lockAcquired = await tokenRefreshService.acquireRefreshLock(accountId, 'claude') |
|
|
| if (!lockAcquired) { |
| |
| logger.info( |
| `🔒 Token refresh already in progress for account: ${accountData.name} (${accountId})` |
| ) |
| logRefreshSkipped(accountId, accountData.name, 'claude', 'already_locked') |
|
|
| |
| await new Promise((resolve) => setTimeout(resolve, 2000)) |
|
|
| |
| const updatedData = await redis.getClaudeAccount(accountId) |
| if (updatedData && updatedData.accessToken) { |
| const accessToken = this._decryptSensitiveData(updatedData.accessToken) |
| return { |
| success: true, |
| accessToken, |
| expiresAt: updatedData.expiresAt |
| } |
| } |
|
|
| throw new Error('Token refresh in progress by another process') |
| } |
|
|
| |
| logRefreshStart(accountId, accountData.name, 'claude', 'manual_refresh') |
| logger.info(`🔄 Starting token refresh for account: ${accountData.name} (${accountId})`) |
|
|
| |
| const agent = this._createProxyAgent(accountData.proxy) |
|
|
| const axiosConfig = { |
| headers: { |
| 'Content-Type': 'application/json', |
| Accept: 'application/json, text/plain, */*', |
| 'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
| 'Accept-Language': 'en-US,en;q=0.9', |
| Referer: 'https://claude.ai/', |
| Origin: 'https://claude.ai' |
| }, |
| timeout: 30000 |
| } |
|
|
| if (agent) { |
| axiosConfig.httpAgent = agent |
| axiosConfig.httpsAgent = agent |
| axiosConfig.proxy = false |
| } |
|
|
| const response = await axios.post( |
| this.claudeApiUrl, |
| { |
| grant_type: 'refresh_token', |
| refresh_token: refreshToken, |
| client_id: this.claudeOauthClientId |
| }, |
| axiosConfig |
| ) |
|
|
| if (response.status === 200) { |
| |
| logger.authDetail('Token refresh response', response.data) |
|
|
| |
| logger.info('📊 Token refresh response (analyzing for subscription info):', { |
| status: response.status, |
| hasData: !!response.data, |
| dataKeys: response.data ? Object.keys(response.data) : [] |
| }) |
|
|
| const { access_token, refresh_token, expires_in } = response.data |
|
|
| |
| if ( |
| response.data.subscription || |
| response.data.plan || |
| response.data.tier || |
| response.data.account_type |
| ) { |
| const subscriptionInfo = { |
| subscription: response.data.subscription, |
| plan: response.data.plan, |
| tier: response.data.tier, |
| accountType: response.data.account_type, |
| features: response.data.features, |
| limits: response.data.limits |
| } |
| logger.info('🎯 Found subscription info in refresh response:', subscriptionInfo) |
|
|
| |
| accountData.subscriptionInfo = JSON.stringify(subscriptionInfo) |
| } |
|
|
| |
| accountData.accessToken = this._encryptSensitiveData(access_token) |
| accountData.refreshToken = this._encryptSensitiveData(refresh_token) |
| accountData.expiresAt = (Date.now() + expires_in * 1000).toString() |
| accountData.lastRefreshAt = new Date().toISOString() |
| accountData.status = 'active' |
| accountData.errorMessage = '' |
|
|
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| |
| |
| const hasProfileScope = accountData.scopes && accountData.scopes.includes('user:profile') |
|
|
| if (hasProfileScope) { |
| try { |
| await this.fetchAndUpdateAccountProfile(accountId, access_token, agent) |
| } catch (profileError) { |
| logger.warn(`⚠️ Failed to fetch profile info after refresh: ${profileError.message}`) |
| } |
| } else { |
| logger.debug( |
| `⏩ Skipping profile fetch after refresh for account ${accountId} (no user:profile scope)` |
| ) |
| } |
|
|
| |
| logRefreshSuccess(accountId, accountData.name, 'claude', { |
| accessToken: access_token, |
| refreshToken: refresh_token, |
| expiresAt: accountData.expiresAt, |
| scopes: accountData.scopes |
| }) |
|
|
| logger.success( |
| `🔄 Refreshed token for account: ${accountData.name} (${accountId}) - Access Token: ${maskToken(access_token)}` |
| ) |
|
|
| return { |
| success: true, |
| accessToken: access_token, |
| expiresAt: accountData.expiresAt |
| } |
| } else { |
| throw new Error(`Token refresh failed with status: ${response.status}`) |
| } |
| } catch (error) { |
| |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (accountData) { |
| logRefreshError(accountId, accountData.name, 'claude', error) |
| accountData.status = 'error' |
| accountData.errorMessage = error.message |
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name, |
| platform: 'claude-oauth', |
| status: 'error', |
| errorCode: 'CLAUDE_OAUTH_ERROR', |
| reason: `Token refresh failed: ${error.message}` |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send webhook notification:', webhookError) |
| } |
| } |
|
|
| logger.error(`❌ Failed to refresh token for account ${accountId}:`, error) |
|
|
| throw error |
| } finally { |
| |
| if (lockAcquired) { |
| await tokenRefreshService.releaseRefreshLock(accountId, 'claude') |
| } |
| } |
| } |
|
|
| |
| async getAccount(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
|
|
| if (!accountData || Object.keys(accountData).length === 0) { |
| return null |
| } |
|
|
| return accountData |
| } catch (error) { |
| logger.error('❌ Failed to get Claude account:', error) |
| return null |
| } |
| } |
|
|
| |
| async getValidAccessToken(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
|
|
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| if (accountData.isActive !== 'true') { |
| throw new Error('Account is disabled') |
| } |
|
|
| |
| const expiresAt = parseInt(accountData.expiresAt) |
| const now = Date.now() |
| const isExpired = !expiresAt || now >= expiresAt - 60000 |
|
|
| |
| logTokenUsage(accountId, accountData.name, 'claude', accountData.expiresAt, isExpired) |
|
|
| if (isExpired) { |
| logger.info(`🔄 Token expired/expiring for account ${accountId}, attempting refresh...`) |
| try { |
| const refreshResult = await this.refreshAccountToken(accountId) |
| return refreshResult.accessToken |
| } catch (refreshError) { |
| logger.warn(`⚠️ Token refresh failed for account ${accountId}: ${refreshError.message}`) |
| |
| const currentToken = this._decryptSensitiveData(accountData.accessToken) |
| if (currentToken) { |
| logger.info(`🔄 Using current token for account ${accountId} (refresh failed)`) |
| return currentToken |
| } |
| throw refreshError |
| } |
| } |
|
|
| const accessToken = this._decryptSensitiveData(accountData.accessToken) |
|
|
| if (!accessToken) { |
| throw new Error('No access token available') |
| } |
|
|
| |
| accountData.lastUsedAt = new Date().toISOString() |
| await this.updateSessionWindow(accountId, accountData) |
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| return accessToken |
| } catch (error) { |
| logger.error(`❌ Failed to get valid access token for account ${accountId}:`, error) |
| throw error |
| } |
| } |
|
|
| |
| async getAllAccounts() { |
| try { |
| const accounts = await redis.getAllClaudeAccounts() |
|
|
| |
| const processedAccounts = await Promise.all( |
| accounts.map(async (account) => { |
| |
| const rateLimitInfo = await this.getAccountRateLimitInfo(account.id) |
|
|
| |
| const sessionWindowInfo = await this.getSessionWindowInfo(account.id) |
|
|
| |
| const claudeUsage = this.buildClaudeUsageSnapshot(account) |
|
|
| |
| const scopes = account.scopes && account.scopes.trim() ? account.scopes.split(' ') : [] |
| const isOAuth = scopes.includes('user:profile') && scopes.includes('user:inference') |
| const authType = isOAuth ? 'oauth' : 'setup-token' |
|
|
| return { |
| id: account.id, |
| name: account.name, |
| description: account.description, |
| email: account.email ? this._maskEmail(this._decryptSensitiveData(account.email)) : '', |
| isActive: account.isActive === 'true', |
| proxy: account.proxy ? JSON.parse(account.proxy) : null, |
| status: account.status, |
| errorMessage: account.errorMessage, |
| accountType: account.accountType || 'shared', |
| priority: parseInt(account.priority) || 50, |
| platform: account.platform || 'claude', |
| authType, |
| createdAt: account.createdAt, |
| lastUsedAt: account.lastUsedAt, |
| lastRefreshAt: account.lastRefreshAt, |
| expiresAt: account.expiresAt || null, |
| subscriptionExpiresAt: |
| account.subscriptionExpiresAt && account.subscriptionExpiresAt !== '' |
| ? account.subscriptionExpiresAt |
| : null, |
| |
| |
| scopes: account.scopes && account.scopes.trim() ? account.scopes.split(' ') : [], |
| |
| hasRefreshToken: !!account.refreshToken, |
| |
| subscriptionInfo: account.subscriptionInfo |
| ? JSON.parse(account.subscriptionInfo) |
| : null, |
| |
| rateLimitStatus: rateLimitInfo |
| ? { |
| isRateLimited: rateLimitInfo.isRateLimited, |
| rateLimitedAt: rateLimitInfo.rateLimitedAt, |
| minutesRemaining: rateLimitInfo.minutesRemaining |
| } |
| : null, |
| |
| sessionWindow: sessionWindowInfo || { |
| hasActiveWindow: false, |
| windowStart: null, |
| windowEnd: null, |
| progress: 0, |
| remainingTime: null, |
| lastRequestTime: null |
| }, |
| |
| claudeUsage: claudeUsage || null, |
| |
| schedulable: account.schedulable !== 'false', |
| |
| autoStopOnWarning: account.autoStopOnWarning === 'true', |
| |
| fiveHourAutoStopped: account.fiveHourAutoStopped === 'true', |
| fiveHourStoppedAt: account.fiveHourStoppedAt || null, |
| |
| useUnifiedUserAgent: account.useUnifiedUserAgent === 'true', |
| |
| useUnifiedClientId: account.useUnifiedClientId === 'true', |
| unifiedClientId: account.unifiedClientId || '', |
| |
| stoppedReason: account.stoppedReason || null |
| } |
| }) |
| ) |
|
|
| return processedAccounts |
| } catch (error) { |
| logger.error('❌ Failed to get Claude accounts:', error) |
| throw error |
| } |
| } |
|
|
| |
| async getAccountOverview(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
|
|
| if (!accountData || Object.keys(accountData).length === 0) { |
| return null |
| } |
|
|
| const [sessionWindowInfo, rateLimitInfo] = await Promise.all([ |
| this.getSessionWindowInfo(accountId), |
| this.getAccountRateLimitInfo(accountId) |
| ]) |
|
|
| const sessionWindow = sessionWindowInfo || { |
| hasActiveWindow: false, |
| windowStart: null, |
| windowEnd: null, |
| progress: 0, |
| remainingTime: null, |
| lastRequestTime: accountData.lastRequestTime || null, |
| sessionWindowStatus: accountData.sessionWindowStatus || null |
| } |
|
|
| const rateLimitStatus = rateLimitInfo |
| ? { |
| isRateLimited: !!rateLimitInfo.isRateLimited, |
| rateLimitedAt: rateLimitInfo.rateLimitedAt || null, |
| minutesRemaining: rateLimitInfo.minutesRemaining || 0, |
| rateLimitEndAt: rateLimitInfo.rateLimitEndAt || null |
| } |
| : { |
| isRateLimited: false, |
| rateLimitedAt: null, |
| minutesRemaining: 0, |
| rateLimitEndAt: null |
| } |
|
|
| return { |
| id: accountData.id, |
| accountType: accountData.accountType || 'shared', |
| platform: accountData.platform || 'claude', |
| isActive: accountData.isActive === 'true', |
| schedulable: accountData.schedulable !== 'false', |
| sessionWindow, |
| rateLimitStatus |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to build Claude account overview for ${accountId}:`, error) |
| return null |
| } |
| } |
|
|
| |
| async updateAccount(accountId, updates) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
|
|
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| const allowedUpdates = [ |
| 'name', |
| 'description', |
| 'email', |
| 'password', |
| 'refreshToken', |
| 'proxy', |
| 'isActive', |
| 'claudeAiOauth', |
| 'accountType', |
| 'priority', |
| 'schedulable', |
| 'subscriptionInfo', |
| 'autoStopOnWarning', |
| 'useUnifiedUserAgent', |
| 'useUnifiedClientId', |
| 'unifiedClientId', |
| 'subscriptionExpiresAt' |
| ] |
| const updatedData = { ...accountData } |
| let shouldClearAutoStopFields = false |
|
|
| |
| const oldRefreshToken = this._decryptSensitiveData(accountData.refreshToken) |
|
|
| for (const [field, value] of Object.entries(updates)) { |
| if (allowedUpdates.includes(field)) { |
| if (['email', 'password', 'refreshToken'].includes(field)) { |
| updatedData[field] = this._encryptSensitiveData(value) |
| } else if (field === 'proxy') { |
| updatedData[field] = value ? JSON.stringify(value) : '' |
| } else if (field === 'priority') { |
| updatedData[field] = value.toString() |
| } else if (field === 'subscriptionInfo') { |
| |
| updatedData[field] = typeof value === 'string' ? value : JSON.stringify(value) |
| } else if (field === 'subscriptionExpiresAt') { |
| |
| updatedData[field] = value ? value.toString() : '' |
| } else if (field === 'claudeAiOauth') { |
| |
| if (value) { |
| updatedData.claudeAiOauth = this._encryptSensitiveData(JSON.stringify(value)) |
| updatedData.accessToken = this._encryptSensitiveData(value.accessToken) |
| updatedData.refreshToken = this._encryptSensitiveData(value.refreshToken) |
| updatedData.expiresAt = value.expiresAt.toString() |
| updatedData.scopes = value.scopes.join(' ') |
| updatedData.status = 'active' |
| updatedData.errorMessage = '' |
| updatedData.lastRefreshAt = new Date().toISOString() |
| } |
| } else { |
| updatedData[field] = value !== null && value !== undefined ? value.toString() : '' |
| } |
| } |
| } |
|
|
| |
| if (updates.refreshToken && !oldRefreshToken && updates.refreshToken.trim()) { |
| const newExpiresAt = Date.now() + 10 * 60 * 1000 |
| updatedData.expiresAt = newExpiresAt.toString() |
| logger.info( |
| `🔄 New refresh token added for account ${accountId}, setting expiry to 10 minutes` |
| ) |
| } |
|
|
| |
| if (updates.claudeAiOauth && updates.claudeAiOauth.refreshToken && !oldRefreshToken) { |
| |
| const providedExpiry = parseInt(updates.claudeAiOauth.expiresAt) |
| const now = Date.now() |
| const oneHour = 60 * 60 * 1000 |
|
|
| if (providedExpiry - now > oneHour) { |
| const newExpiresAt = now + 10 * 60 * 1000 |
| updatedData.expiresAt = newExpiresAt.toString() |
| logger.info( |
| `🔄 Adjusted expiry time to 10 minutes for account ${accountId} with refresh token` |
| ) |
| } |
| } |
|
|
| updatedData.updatedAt = new Date().toISOString() |
|
|
| |
| if (Object.prototype.hasOwnProperty.call(updates, 'schedulable')) { |
| |
| delete updatedData.rateLimitAutoStopped |
| delete updatedData.fiveHourAutoStopped |
| delete updatedData.fiveHourStoppedAt |
| delete updatedData.tempErrorAutoStopped |
| |
| delete updatedData.autoStoppedAt |
| delete updatedData.stoppedReason |
| shouldClearAutoStopFields = true |
|
|
| await this._clearFiveHourWarningMetadata(accountId, updatedData) |
|
|
| |
| if (updates.schedulable === true || updates.schedulable === 'true') { |
| logger.info(`✅ Manually enabled scheduling for account ${accountId}`) |
| } else { |
| logger.info(`⛔ Manually disabled scheduling for account ${accountId}`) |
| } |
| } |
|
|
| |
| if (updates.isActive === 'false' && accountData.isActive === 'true') { |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: updatedData.name || 'Unknown Account', |
| platform: 'claude-oauth', |
| status: 'disabled', |
| errorCode: 'CLAUDE_OAUTH_MANUALLY_DISABLED', |
| reason: 'Account manually disabled by administrator' |
| }) |
| } catch (webhookError) { |
| logger.error( |
| 'Failed to send webhook notification for manual account disable:', |
| webhookError |
| ) |
| } |
| } |
|
|
| await redis.setClaudeAccount(accountId, updatedData) |
|
|
| if (shouldClearAutoStopFields) { |
| const fieldsToRemove = [ |
| 'rateLimitAutoStopped', |
| 'fiveHourAutoStopped', |
| 'fiveHourStoppedAt', |
| 'tempErrorAutoStopped', |
| 'autoStoppedAt', |
| 'stoppedReason' |
| ] |
| await this._removeAccountFields(accountId, fieldsToRemove, 'manual_schedule_update') |
| } |
|
|
| logger.success(`📝 Updated Claude account: ${accountId}`) |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error('❌ Failed to update Claude account:', error) |
| throw error |
| } |
| } |
|
|
| |
| async deleteAccount(accountId) { |
| try { |
| |
| const accountGroupService = require('./accountGroupService') |
| await accountGroupService.removeAccountFromAllGroups(accountId) |
|
|
| const result = await redis.deleteClaudeAccount(accountId) |
|
|
| if (result === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| logger.success(`🗑️ Deleted Claude account: ${accountId}`) |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error('❌ Failed to delete Claude account:', error) |
| throw error |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| isSubscriptionExpired(account) { |
| if (!account.subscriptionExpiresAt) { |
| return false |
| } |
|
|
| const expiryDate = new Date(account.subscriptionExpiresAt) |
| const now = new Date() |
|
|
| if (expiryDate <= now) { |
| logger.debug( |
| `⏰ Account ${account.name} (${account.id}) expired at ${account.subscriptionExpiresAt}` |
| ) |
| return true |
| } |
|
|
| return false |
| } |
|
|
| |
| async selectAvailableAccount(sessionHash = null, modelName = null) { |
| try { |
| const accounts = await redis.getAllClaudeAccounts() |
|
|
| let activeAccounts = accounts.filter( |
| (account) => |
| account.isActive === 'true' && |
| account.status !== 'error' && |
| account.schedulable !== 'false' && |
| !this.isSubscriptionExpired(account) |
| ) |
|
|
| |
| if (modelName && modelName.toLowerCase().includes('opus')) { |
| activeAccounts = activeAccounts.filter((account) => { |
| |
| if (account.subscriptionInfo) { |
| try { |
| const info = JSON.parse(account.subscriptionInfo) |
| |
| if (info.hasClaudePro === true && info.hasClaudeMax !== true) { |
| return false |
| } |
| if (info.accountType === 'claude_pro' || info.accountType === 'claude_free') { |
| return false |
| } |
| } catch (e) { |
| |
| return true |
| } |
| } |
| |
| return true |
| }) |
|
|
| if (activeAccounts.length === 0) { |
| throw new Error('No Claude accounts available that support Opus model') |
| } |
| } |
|
|
| if (activeAccounts.length === 0) { |
| throw new Error('No active Claude accounts available') |
| } |
|
|
| |
| if (sessionHash) { |
| const mappedAccountId = await redis.getSessionAccountMapping(sessionHash) |
| if (mappedAccountId) { |
| |
| const mappedAccount = activeAccounts.find((acc) => acc.id === mappedAccountId) |
| if (mappedAccount) { |
| |
| await redis.extendSessionAccountMappingTTL(sessionHash) |
| logger.info( |
| `🎯 Using sticky session account: ${mappedAccount.name} (${mappedAccountId}) for session ${sessionHash}` |
| ) |
| return mappedAccountId |
| } else { |
| logger.warn( |
| `⚠️ Mapped account ${mappedAccountId} is no longer available, selecting new account` |
| ) |
| |
| await redis.deleteSessionAccountMapping(sessionHash) |
| } |
| } |
| } |
|
|
| |
| |
| const sortedAccounts = activeAccounts.sort((a, b) => { |
| const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
| const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
| return aLastUsed - bLastUsed |
| }) |
|
|
| const selectedAccountId = sortedAccounts[0].id |
|
|
| |
| if (sessionHash) { |
| |
| const ttlSeconds = (config.session?.stickyTtlHours || 1) * 60 * 60 |
| await redis.setSessionAccountMapping(sessionHash, selectedAccountId, ttlSeconds) |
| logger.info( |
| `🎯 Created new sticky session mapping: ${sortedAccounts[0].name} (${selectedAccountId}) for session ${sessionHash}` |
| ) |
| } |
|
|
| return selectedAccountId |
| } catch (error) { |
| logger.error('❌ Failed to select available account:', error) |
| throw error |
| } |
| } |
|
|
| |
| async selectAccountForApiKey(apiKeyData, sessionHash = null, modelName = null) { |
| try { |
| |
| if (apiKeyData.claudeAccountId) { |
| const boundAccount = await redis.getClaudeAccount(apiKeyData.claudeAccountId) |
| if ( |
| boundAccount && |
| boundAccount.isActive === 'true' && |
| boundAccount.status !== 'error' && |
| boundAccount.schedulable !== 'false' && |
| !this.isSubscriptionExpired(boundAccount) |
| ) { |
| logger.info( |
| `🎯 Using bound dedicated account: ${boundAccount.name} (${apiKeyData.claudeAccountId}) for API key ${apiKeyData.name}` |
| ) |
| return apiKeyData.claudeAccountId |
| } else { |
| logger.warn( |
| `⚠️ Bound account ${apiKeyData.claudeAccountId} is not available, falling back to shared pool` |
| ) |
| } |
| } |
|
|
| |
| const accounts = await redis.getAllClaudeAccounts() |
|
|
| let sharedAccounts = accounts.filter( |
| (account) => |
| account.isActive === 'true' && |
| account.status !== 'error' && |
| account.schedulable !== 'false' && |
| (account.accountType === 'shared' || !account.accountType) && |
| !this.isSubscriptionExpired(account) |
| ) |
|
|
| |
| if (modelName && modelName.toLowerCase().includes('opus')) { |
| sharedAccounts = sharedAccounts.filter((account) => { |
| |
| if (account.subscriptionInfo) { |
| try { |
| const info = JSON.parse(account.subscriptionInfo) |
| |
| if (info.hasClaudePro === true && info.hasClaudeMax !== true) { |
| return false |
| } |
| if (info.accountType === 'claude_pro' || info.accountType === 'claude_free') { |
| return false |
| } |
| } catch (e) { |
| |
| return true |
| } |
| } |
| |
| return true |
| }) |
|
|
| if (sharedAccounts.length === 0) { |
| throw new Error('No shared Claude accounts available that support Opus model') |
| } |
| } |
|
|
| if (sharedAccounts.length === 0) { |
| throw new Error('No active shared Claude accounts available') |
| } |
|
|
| |
| if (sessionHash) { |
| const mappedAccountId = await redis.getSessionAccountMapping(sessionHash) |
| if (mappedAccountId) { |
| |
| const mappedAccount = sharedAccounts.find((acc) => acc.id === mappedAccountId) |
| if (mappedAccount) { |
| |
| const isRateLimited = await this.isAccountRateLimited(mappedAccountId) |
| if (isRateLimited) { |
| logger.warn( |
| `⚠️ Mapped account ${mappedAccountId} is rate limited, selecting new account` |
| ) |
| await redis.deleteSessionAccountMapping(sessionHash) |
| } else { |
| |
| await redis.extendSessionAccountMappingTTL(sessionHash) |
| logger.info( |
| `🎯 Using sticky session shared account: ${mappedAccount.name} (${mappedAccountId}) for session ${sessionHash}` |
| ) |
| return mappedAccountId |
| } |
| } else { |
| logger.warn( |
| `⚠️ Mapped shared account ${mappedAccountId} is no longer available, selecting new account` |
| ) |
| |
| await redis.deleteSessionAccountMapping(sessionHash) |
| } |
| } |
| } |
|
|
| |
| const nonRateLimitedAccounts = [] |
| const rateLimitedAccounts = [] |
|
|
| for (const account of sharedAccounts) { |
| const isRateLimited = await this.isAccountRateLimited(account.id) |
| if (isRateLimited) { |
| const rateLimitInfo = await this.getAccountRateLimitInfo(account.id) |
| account._rateLimitInfo = rateLimitInfo |
| rateLimitedAccounts.push(account) |
| } else { |
| nonRateLimitedAccounts.push(account) |
| } |
| } |
|
|
| |
| let candidateAccounts = nonRateLimitedAccounts |
|
|
| |
| if (candidateAccounts.length === 0) { |
| logger.warn('⚠️ All shared accounts are rate limited, selecting from rate limited pool') |
| candidateAccounts = rateLimitedAccounts.sort((a, b) => { |
| const aRateLimitedAt = new Date(a._rateLimitInfo.rateLimitedAt).getTime() |
| const bRateLimitedAt = new Date(b._rateLimitInfo.rateLimitedAt).getTime() |
| return aRateLimitedAt - bRateLimitedAt |
| }) |
| } else { |
| |
| candidateAccounts = candidateAccounts.sort((a, b) => { |
| const aLastUsed = new Date(a.lastUsedAt || 0).getTime() |
| const bLastUsed = new Date(b.lastUsedAt || 0).getTime() |
| return aLastUsed - bLastUsed |
| }) |
| } |
|
|
| if (candidateAccounts.length === 0) { |
| throw new Error('No available shared Claude accounts') |
| } |
|
|
| const selectedAccountId = candidateAccounts[0].id |
|
|
| |
| if (sessionHash) { |
| |
| const ttlSeconds = (config.session?.stickyTtlHours || 1) * 60 * 60 |
| await redis.setSessionAccountMapping(sessionHash, selectedAccountId, ttlSeconds) |
| logger.info( |
| `🎯 Created new sticky session mapping for shared account: ${candidateAccounts[0].name} (${selectedAccountId}) for session ${sessionHash}` |
| ) |
| } |
|
|
| logger.info( |
| `🎯 Selected shared account: ${candidateAccounts[0].name} (${selectedAccountId}) for API key ${apiKeyData.name}` |
| ) |
| return selectedAccountId |
| } catch (error) { |
| logger.error('❌ Failed to select account for API key:', error) |
| throw error |
| } |
| } |
|
|
| |
| _createProxyAgent(proxyConfig) { |
| const proxyAgent = ProxyHelper.createProxyAgent(proxyConfig) |
| if (proxyAgent) { |
| logger.info( |
| `🌐 Using proxy for Claude request: ${ProxyHelper.getProxyDescription(proxyConfig)}` |
| ) |
| } else if (proxyConfig) { |
| logger.debug('🌐 Failed to create proxy agent for Claude') |
| } else { |
| logger.debug('🌐 No proxy configured for Claude request') |
| } |
| return proxyAgent |
| } |
|
|
| |
| _encryptSensitiveData(data) { |
| if (!data) { |
| return '' |
| } |
|
|
| try { |
| const key = this._generateEncryptionKey() |
| const iv = crypto.randomBytes(16) |
|
|
| const cipher = crypto.createCipheriv(this.ENCRYPTION_ALGORITHM, key, iv) |
| let encrypted = cipher.update(data, 'utf8', 'hex') |
| encrypted += cipher.final('hex') |
|
|
| |
| return `${iv.toString('hex')}:${encrypted}` |
| } catch (error) { |
| logger.error('❌ Encryption error:', error) |
| return data |
| } |
| } |
|
|
| |
| _decryptSensitiveData(encryptedData) { |
| if (!encryptedData) { |
| return '' |
| } |
|
|
| |
| const cacheKey = crypto.createHash('sha256').update(encryptedData).digest('hex') |
| const cached = this._decryptCache.get(cacheKey) |
| if (cached !== undefined) { |
| return cached |
| } |
|
|
| try { |
| let decrypted = '' |
|
|
| |
| if (encryptedData.includes(':')) { |
| |
| const parts = encryptedData.split(':') |
| if (parts.length === 2) { |
| const key = this._generateEncryptionKey() |
| const iv = Buffer.from(parts[0], 'hex') |
| const encrypted = parts[1] |
|
|
| const decipher = crypto.createDecipheriv(this.ENCRYPTION_ALGORITHM, key, iv) |
| decrypted = decipher.update(encrypted, 'hex', 'utf8') |
| decrypted += decipher.final('utf8') |
|
|
| |
| this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) |
|
|
| |
| if ((this._decryptCache.hits + this._decryptCache.misses) % 1000 === 0) { |
| this._decryptCache.printStats() |
| } |
|
|
| return decrypted |
| } |
| } |
|
|
| |
| |
| try { |
| const decipher = crypto.createDecipher('aes-256-cbc', config.security.encryptionKey) |
| decrypted = decipher.update(encryptedData, 'hex', 'utf8') |
| decrypted += decipher.final('utf8') |
|
|
| |
| this._decryptCache.set(cacheKey, decrypted, 5 * 60 * 1000) |
|
|
| return decrypted |
| } catch (oldError) { |
| |
| logger.warn('⚠️ Could not decrypt data, returning as-is:', oldError.message) |
| return encryptedData |
| } |
| } catch (error) { |
| logger.error('❌ Decryption error:', error) |
| return encryptedData |
| } |
| } |
|
|
| |
| _generateEncryptionKey() { |
| |
| |
| |
| if (!this._encryptionKeyCache) { |
| |
| |
| this._encryptionKeyCache = crypto.scryptSync( |
| config.security.encryptionKey, |
| this.ENCRYPTION_SALT, |
| 32 |
| ) |
| logger.info('🔑 Encryption key derived and cached for performance optimization') |
| } |
| return this._encryptionKeyCache |
| } |
|
|
| |
| _maskEmail(email) { |
| if (!email || !email.includes('@')) { |
| return email |
| } |
|
|
| const [username, domain] = email.split('@') |
| const maskedUsername = |
| username.length > 2 |
| ? `${username.slice(0, 2)}***${username.slice(-1)}` |
| : `${username.slice(0, 1)}***` |
|
|
| return `${maskedUsername}@${domain}` |
| } |
|
|
| |
| _toNumberOrNull(value) { |
| if (value === undefined || value === null || value === '') { |
| return null |
| } |
|
|
| const num = Number(value) |
| return Number.isFinite(num) ? num : null |
| } |
|
|
| |
| async cleanupErrorAccounts() { |
| try { |
| const accounts = await redis.getAllClaudeAccounts() |
| let cleanedCount = 0 |
|
|
| for (const account of accounts) { |
| if (account.status === 'error' && account.lastRefreshAt) { |
| const lastRefresh = new Date(account.lastRefreshAt) |
| const now = new Date() |
| const hoursSinceLastRefresh = (now - lastRefresh) / (1000 * 60 * 60) |
|
|
| |
| if (hoursSinceLastRefresh > 24) { |
| account.status = 'created' |
| account.errorMessage = '' |
| await redis.setClaudeAccount(account.id, account) |
| cleanedCount++ |
| } |
| } |
| } |
|
|
| if (cleanedCount > 0) { |
| logger.success(`🧹 Reset ${cleanedCount} error accounts`) |
| } |
|
|
| return cleanedCount |
| } catch (error) { |
| logger.error('❌ Failed to cleanup error accounts:', error) |
| return 0 |
| } |
| } |
|
|
| |
| async markAccountRateLimited(accountId, sessionHash = null, rateLimitResetTimestamp = null) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const updatedAccountData = { ...accountData } |
| updatedAccountData.rateLimitedAt = new Date().toISOString() |
| updatedAccountData.rateLimitStatus = 'limited' |
| |
| updatedAccountData.schedulable = 'false' |
| |
| updatedAccountData.rateLimitAutoStopped = 'true' |
|
|
| |
| if (rateLimitResetTimestamp) { |
| |
| const resetTime = new Date(rateLimitResetTimestamp * 1000) |
| updatedAccountData.rateLimitEndAt = resetTime.toISOString() |
|
|
| |
| const windowStartTime = new Date(resetTime.getTime() - 5 * 60 * 60 * 1000) |
| updatedAccountData.sessionWindowStart = windowStartTime.toISOString() |
| updatedAccountData.sessionWindowEnd = resetTime.toISOString() |
|
|
| const now = new Date() |
| const minutesUntilEnd = Math.ceil((resetTime - now) / (1000 * 60)) |
| logger.warn( |
| `🚫 Account marked as rate limited with accurate reset time: ${accountData.name} (${accountId}) - ${minutesUntilEnd} minutes remaining until ${resetTime.toISOString()}` |
| ) |
| } else { |
| |
| const windowData = await this.updateSessionWindow(accountId, updatedAccountData) |
| Object.assign(updatedAccountData, windowData) |
|
|
| |
| if (updatedAccountData.sessionWindowEnd) { |
| updatedAccountData.rateLimitEndAt = updatedAccountData.sessionWindowEnd |
| const windowEnd = new Date(updatedAccountData.sessionWindowEnd) |
| const now = new Date() |
| const minutesUntilEnd = Math.ceil((windowEnd - now) / (1000 * 60)) |
| logger.warn( |
| `🚫 Account marked as rate limited until estimated session window ends: ${accountData.name} (${accountId}) - ${minutesUntilEnd} minutes remaining` |
| ) |
| } else { |
| |
| const oneHourLater = new Date(Date.now() + 60 * 60 * 1000) |
| updatedAccountData.rateLimitEndAt = oneHourLater.toISOString() |
| logger.warn( |
| `🚫 Account marked as rate limited (1 hour default): ${accountData.name} (${accountId})` |
| ) |
| } |
| } |
|
|
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| |
| if (sessionHash) { |
| await redis.deleteSessionAccountMapping(sessionHash) |
| logger.info(`🗑️ Deleted sticky session mapping for rate limited account: ${accountId}`) |
| } |
|
|
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name || 'Claude Account', |
| platform: 'claude-oauth', |
| status: 'error', |
| errorCode: 'CLAUDE_OAUTH_RATE_LIMITED', |
| reason: `Account rate limited (429 error). ${rateLimitResetTimestamp ? `Reset at: ${formatDateWithTimezone(rateLimitResetTimestamp)}` : 'Estimated reset in 1-5 hours'}`, |
| timestamp: getISOStringWithTimezone(new Date()) |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send rate limit webhook notification:', webhookError) |
| } |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to mark account as rate limited: ${accountId}`, error) |
| throw error |
| } |
| } |
|
|
| |
| async markAccountOpusRateLimited(accountId, rateLimitResetTimestamp = null) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| const updatedAccountData = { ...accountData } |
| const now = new Date() |
| updatedAccountData.opusRateLimitedAt = now.toISOString() |
|
|
| if (rateLimitResetTimestamp) { |
| const resetTime = new Date(rateLimitResetTimestamp * 1000) |
| updatedAccountData.opusRateLimitEndAt = resetTime.toISOString() |
| logger.warn( |
| `🚫 Account ${accountData.name} (${accountId}) reached Opus weekly cap, resets at ${resetTime.toISOString()}` |
| ) |
| } else { |
| |
| logger.warn( |
| `⚠️ Account ${accountData.name} (${accountId}) reported Opus limit without reset timestamp` |
| ) |
| } |
|
|
| await redis.setClaudeAccount(accountId, updatedAccountData) |
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to mark Opus rate limit for account: ${accountId}`, error) |
| throw error |
| } |
| } |
|
|
| |
| async clearAccountOpusRateLimit(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return { success: true } |
| } |
|
|
| const updatedAccountData = { ...accountData } |
| delete updatedAccountData.opusRateLimitedAt |
| delete updatedAccountData.opusRateLimitEndAt |
|
|
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| const redisKey = `claude:account:${accountId}` |
| if (redis.client && typeof redis.client.hdel === 'function') { |
| await redis.client.hdel(redisKey, 'opusRateLimitedAt', 'opusRateLimitEndAt') |
| } |
|
|
| logger.info(`✅ Cleared Opus rate limit state for account ${accountId}`) |
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to clear Opus rate limit for account: ${accountId}`, error) |
| throw error |
| } |
| } |
|
|
| |
| async isAccountOpusRateLimited(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return false |
| } |
|
|
| if (!accountData.opusRateLimitEndAt) { |
| return false |
| } |
|
|
| const resetTime = new Date(accountData.opusRateLimitEndAt) |
| if (Number.isNaN(resetTime.getTime())) { |
| await this.clearAccountOpusRateLimit(accountId) |
| return false |
| } |
|
|
| const now = new Date() |
| if (now >= resetTime) { |
| await this.clearAccountOpusRateLimit(accountId) |
| return false |
| } |
|
|
| return true |
| } catch (error) { |
| logger.error(`❌ Failed to check Opus rate limit status for account: ${accountId}`, error) |
| return false |
| } |
| } |
|
|
| |
| async clearExpiredOpusRateLimit(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return { success: true } |
| } |
|
|
| if (!accountData.opusRateLimitEndAt) { |
| return { success: true } |
| } |
|
|
| const resetTime = new Date(accountData.opusRateLimitEndAt) |
| if (Number.isNaN(resetTime.getTime()) || new Date() >= resetTime) { |
| await this.clearAccountOpusRateLimit(accountId) |
| } |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to clear expired Opus rate limit for account: ${accountId}`, error) |
| throw error |
| } |
| } |
|
|
| |
| async removeAccountRateLimit(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| const accountKey = `claude:account:${accountId}` |
|
|
| |
| const redisKey = `claude:account:${accountId}` |
| await redis.client.hdel(redisKey, 'rateLimitedAt', 'rateLimitStatus', 'rateLimitEndAt') |
| delete accountData.rateLimitedAt |
| delete accountData.rateLimitStatus |
| delete accountData.rateLimitEndAt |
|
|
| const hadAutoStop = accountData.rateLimitAutoStopped === 'true' |
|
|
| |
| if (hadAutoStop && accountData.schedulable === 'false') { |
| accountData.schedulable = 'true' |
| logger.info(`✅ Auto-resuming scheduling for account ${accountId} after rate limit cleared`) |
| logger.info( |
| `📊 Account ${accountId} state after recovery: schedulable=${accountData.schedulable}` |
| ) |
| } else { |
| logger.info( |
| `ℹ️ Account ${accountId} did not need auto-resume: autoStopped=${accountData.rateLimitAutoStopped}, schedulable=${accountData.schedulable}` |
| ) |
| } |
|
|
| if (hadAutoStop) { |
| await redis.client.hdel(redisKey, 'rateLimitAutoStopped') |
| delete accountData.rateLimitAutoStopped |
| } |
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| |
| await redis.client.hdel( |
| accountKey, |
| 'rateLimitedAt', |
| 'rateLimitStatus', |
| 'rateLimitEndAt', |
| 'rateLimitAutoStopped' |
| ) |
|
|
| logger.success(`✅ Rate limit removed for account: ${accountData.name} (${accountId})`) |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to remove rate limit for account: ${accountId}`, error) |
| throw error |
| } |
| } |
|
|
| |
| async isAccountRateLimited(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return false |
| } |
|
|
| const now = new Date() |
|
|
| |
| if ( |
| (accountData.rateLimitStatus === 'limited' && accountData.rateLimitedAt) || |
| (accountData.rateLimitAutoStopped === 'true' && accountData.rateLimitEndAt) |
| ) { |
| |
| if (accountData.rateLimitEndAt) { |
| const rateLimitEndAt = new Date(accountData.rateLimitEndAt) |
|
|
| |
| if (now >= rateLimitEndAt) { |
| await this.removeAccountRateLimit(accountId) |
| return false |
| } |
|
|
| return true |
| } else if (accountData.rateLimitedAt) { |
| |
| const rateLimitedAt = new Date(accountData.rateLimitedAt) |
| const hoursSinceRateLimit = (now - rateLimitedAt) / (1000 * 60 * 60) |
|
|
| |
| if (hoursSinceRateLimit >= 1) { |
| await this.removeAccountRateLimit(accountId) |
| return false |
| } |
|
|
| return true |
| } |
| } |
|
|
| return false |
| } catch (error) { |
| logger.error(`❌ Failed to check rate limit status for account: ${accountId}`, error) |
| return false |
| } |
| } |
|
|
| |
| async getAccountRateLimitInfo(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return null |
| } |
|
|
| if (accountData.rateLimitStatus === 'limited' && accountData.rateLimitedAt) { |
| const rateLimitedAt = new Date(accountData.rateLimitedAt) |
| const now = new Date() |
| const minutesSinceRateLimit = Math.floor((now - rateLimitedAt) / (1000 * 60)) |
|
|
| let minutesRemaining |
| let rateLimitEndAt |
|
|
| |
| if (accountData.rateLimitEndAt) { |
| ;({ rateLimitEndAt } = accountData) |
| const endTime = new Date(accountData.rateLimitEndAt) |
| minutesRemaining = Math.max(0, Math.ceil((endTime - now) / (1000 * 60))) |
| } else { |
| |
| minutesRemaining = Math.max(0, 60 - minutesSinceRateLimit) |
| |
| const endTime = new Date(rateLimitedAt.getTime() + 60 * 60 * 1000) |
| rateLimitEndAt = endTime.toISOString() |
| } |
|
|
| return { |
| isRateLimited: minutesRemaining > 0, |
| rateLimitedAt: accountData.rateLimitedAt, |
| minutesSinceRateLimit, |
| minutesRemaining, |
| rateLimitEndAt |
| } |
| } |
|
|
| return { |
| isRateLimited: false, |
| rateLimitedAt: null, |
| minutesSinceRateLimit: 0, |
| minutesRemaining: 0, |
| rateLimitEndAt: null |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to get rate limit info for account: ${accountId}`, error) |
| return null |
| } |
| } |
|
|
| |
| async updateSessionWindow(accountId, accountData = null) { |
| try { |
| |
| if (!accountData) { |
| accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
| } |
|
|
| const now = new Date() |
| const currentTime = now.getTime() |
|
|
| let shouldClearSessionStatus = false |
| let shouldClearFiveHourFlags = false |
|
|
| |
| if (accountData.sessionWindowStart && accountData.sessionWindowEnd) { |
| const windowEnd = new Date(accountData.sessionWindowEnd).getTime() |
|
|
| |
| if (currentTime < windowEnd) { |
| accountData.lastRequestTime = now.toISOString() |
| return accountData |
| } |
|
|
| |
| const windowStart = new Date(accountData.sessionWindowStart) |
| logger.info( |
| `⏰ Session window expired for account ${accountData.name} (${accountId}): ${windowStart.toISOString()} - ${new Date(windowEnd).toISOString()}` |
| ) |
| } |
|
|
| |
| const windowStart = this._calculateSessionWindowStart(now) |
| const windowEnd = this._calculateSessionWindowEnd(windowStart) |
|
|
| |
| accountData.sessionWindowStart = windowStart.toISOString() |
| accountData.sessionWindowEnd = windowEnd.toISOString() |
| accountData.lastRequestTime = now.toISOString() |
|
|
| |
| if (accountData.sessionWindowStatus) { |
| delete accountData.sessionWindowStatus |
| delete accountData.sessionWindowStatusUpdatedAt |
| await this._clearFiveHourWarningMetadata(accountId, accountData) |
| shouldClearSessionStatus = true |
| } |
|
|
| |
| if (accountData.fiveHourAutoStopped === 'true' && accountData.schedulable === 'false') { |
| logger.info( |
| `✅ Auto-resuming scheduling for account ${accountData.name} (${accountId}) - new session window started` |
| ) |
| accountData.schedulable = 'true' |
| delete accountData.fiveHourAutoStopped |
| delete accountData.fiveHourStoppedAt |
| await this._clearFiveHourWarningMetadata(accountId, accountData) |
| shouldClearFiveHourFlags = true |
|
|
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name || 'Claude Account', |
| platform: 'claude', |
| status: 'resumed', |
| errorCode: 'CLAUDE_5H_LIMIT_RESUMED', |
| reason: '进入新的5小时窗口,已自动恢复调度', |
| timestamp: getISOStringWithTimezone(new Date()) |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send webhook notification:', webhookError) |
| } |
| } |
|
|
| if (shouldClearSessionStatus || shouldClearFiveHourFlags) { |
| const fieldsToRemove = [] |
| if (shouldClearFiveHourFlags) { |
| fieldsToRemove.push('fiveHourAutoStopped', 'fiveHourStoppedAt') |
| } |
| if (shouldClearSessionStatus) { |
| fieldsToRemove.push('sessionWindowStatus', 'sessionWindowStatusUpdatedAt') |
| } |
| await this._removeAccountFields(accountId, fieldsToRemove, 'session_window_refresh') |
| } |
|
|
| logger.info( |
| `🕐 Created new session window for account ${accountData.name} (${accountId}): ${windowStart.toISOString()} - ${windowEnd.toISOString()} (from current time)` |
| ) |
|
|
| return accountData |
| } catch (error) { |
| logger.error(`❌ Failed to update session window for account ${accountId}:`, error) |
| throw error |
| } |
| } |
|
|
| |
| _calculateSessionWindowStart(requestTime) { |
| |
| const windowStart = new Date(requestTime) |
| windowStart.setMinutes(0) |
| windowStart.setSeconds(0) |
| windowStart.setMilliseconds(0) |
|
|
| return windowStart |
| } |
|
|
| |
| _calculateSessionWindowEnd(startTime) { |
| const endTime = new Date(startTime) |
| endTime.setHours(endTime.getHours() + 5) |
| return endTime |
| } |
|
|
| async _clearFiveHourWarningMetadata(accountId, accountData = null) { |
| if (accountData) { |
| delete accountData.fiveHourWarningWindow |
| delete accountData.fiveHourWarningCount |
| delete accountData.fiveHourWarningLastSentAt |
| } |
|
|
| try { |
| if (redis.client && typeof redis.client.hdel === 'function') { |
| await redis.client.hdel( |
| `claude:account:${accountId}`, |
| 'fiveHourWarningWindow', |
| 'fiveHourWarningCount', |
| 'fiveHourWarningLastSentAt' |
| ) |
| } |
| } catch (error) { |
| logger.warn( |
| `⚠️ Failed to clear five-hour warning metadata for account ${accountId}: ${error.message}` |
| ) |
| } |
| } |
|
|
| |
| async getSessionWindowInfo(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| return null |
| } |
|
|
| |
| if (!accountData.sessionWindowStart || !accountData.sessionWindowEnd) { |
| return { |
| hasActiveWindow: false, |
| windowStart: null, |
| windowEnd: null, |
| progress: 0, |
| remainingTime: null, |
| lastRequestTime: accountData.lastRequestTime || null, |
| sessionWindowStatus: accountData.sessionWindowStatus || null |
| } |
| } |
|
|
| const now = new Date() |
| const windowStart = new Date(accountData.sessionWindowStart) |
| const windowEnd = new Date(accountData.sessionWindowEnd) |
| const currentTime = now.getTime() |
|
|
| |
| if (currentTime >= windowEnd.getTime()) { |
| return { |
| hasActiveWindow: false, |
| windowStart: accountData.sessionWindowStart, |
| windowEnd: accountData.sessionWindowEnd, |
| progress: 100, |
| remainingTime: 0, |
| lastRequestTime: accountData.lastRequestTime || null, |
| sessionWindowStatus: accountData.sessionWindowStatus || null |
| } |
| } |
|
|
| |
| const totalDuration = windowEnd.getTime() - windowStart.getTime() |
| const elapsedTime = currentTime - windowStart.getTime() |
| const progress = Math.round((elapsedTime / totalDuration) * 100) |
|
|
| |
| const remainingTime = Math.round((windowEnd.getTime() - currentTime) / (1000 * 60)) |
|
|
| return { |
| hasActiveWindow: true, |
| windowStart: accountData.sessionWindowStart, |
| windowEnd: accountData.sessionWindowEnd, |
| progress, |
| remainingTime, |
| lastRequestTime: accountData.lastRequestTime || null, |
| sessionWindowStatus: accountData.sessionWindowStatus || null |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to get session window info for account ${accountId}:`, error) |
| return null |
| } |
| } |
|
|
| |
| async fetchOAuthUsage(accountId, accessToken = null, agent = null) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| if (!accessToken) { |
| accessToken = await this.getValidAccessToken(accountId) |
| } |
|
|
| |
| if (!agent) { |
| agent = this._createProxyAgent(accountData.proxy) |
| } |
|
|
| logger.debug(`📊 Fetching OAuth usage for account: ${accountData.name} (${accountId})`) |
|
|
| |
| const axiosConfig = { |
| headers: { |
| Authorization: `Bearer ${accessToken}`, |
| 'Content-Type': 'application/json', |
| Accept: 'application/json', |
| 'anthropic-beta': 'oauth-2025-04-20', |
| 'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
| 'Accept-Language': 'en-US,en;q=0.9' |
| }, |
| timeout: 15000 |
| } |
|
|
| if (agent) { |
| axiosConfig.httpAgent = agent |
| axiosConfig.httpsAgent = agent |
| axiosConfig.proxy = false |
| } |
|
|
| const response = await axios.get('https://api.anthropic.com/api/oauth/usage', axiosConfig) |
|
|
| if (response.status === 200 && response.data) { |
| logger.debug('✅ Successfully fetched OAuth usage data:', { |
| accountId, |
| fiveHour: response.data.five_hour?.utilization, |
| sevenDay: response.data.seven_day?.utilization, |
| sevenDayOpus: response.data.seven_day_opus?.utilization |
| }) |
|
|
| return response.data |
| } |
|
|
| logger.warn(`⚠️ Failed to fetch OAuth usage for account ${accountId}: ${response.status}`) |
| return null |
| } catch (error) { |
| |
| if (error.response?.status === 403) { |
| logger.debug( |
| `⚠️ OAuth usage API returned 403 for account ${accountId}. This account likely uses Setup Token instead of OAuth.` |
| ) |
| return null |
| } |
|
|
| |
| logger.error( |
| `❌ Failed to fetch OAuth usage for account ${accountId}:`, |
| error.response?.data || error.message |
| ) |
| return null |
| } |
| } |
|
|
| |
| buildClaudeUsageSnapshot(accountData) { |
| const updatedAt = accountData.claudeUsageUpdatedAt |
|
|
| const fiveHourUtilization = this._toNumberOrNull(accountData.claudeFiveHourUtilization) |
| const fiveHourResetsAt = accountData.claudeFiveHourResetsAt |
| const sevenDayUtilization = this._toNumberOrNull(accountData.claudeSevenDayUtilization) |
| const sevenDayResetsAt = accountData.claudeSevenDayResetsAt |
| const sevenDayOpusUtilization = this._toNumberOrNull(accountData.claudeSevenDayOpusUtilization) |
| const sevenDayOpusResetsAt = accountData.claudeSevenDayOpusResetsAt |
|
|
| const hasFiveHourData = fiveHourUtilization !== null || fiveHourResetsAt |
| const hasSevenDayData = sevenDayUtilization !== null || sevenDayResetsAt |
| const hasSevenDayOpusData = sevenDayOpusUtilization !== null || sevenDayOpusResetsAt |
|
|
| if (!updatedAt && !hasFiveHourData && !hasSevenDayData && !hasSevenDayOpusData) { |
| return null |
| } |
|
|
| const now = Date.now() |
|
|
| return { |
| updatedAt, |
| fiveHour: { |
| utilization: fiveHourUtilization, |
| resetsAt: fiveHourResetsAt, |
| remainingSeconds: fiveHourResetsAt |
| ? Math.max(0, Math.floor((new Date(fiveHourResetsAt).getTime() - now) / 1000)) |
| : null |
| }, |
| sevenDay: { |
| utilization: sevenDayUtilization, |
| resetsAt: sevenDayResetsAt, |
| remainingSeconds: sevenDayResetsAt |
| ? Math.max(0, Math.floor((new Date(sevenDayResetsAt).getTime() - now) / 1000)) |
| : null |
| }, |
| sevenDayOpus: { |
| utilization: sevenDayOpusUtilization, |
| resetsAt: sevenDayOpusResetsAt, |
| remainingSeconds: sevenDayOpusResetsAt |
| ? Math.max(0, Math.floor((new Date(sevenDayOpusResetsAt).getTime() - now) / 1000)) |
| : null |
| } |
| } |
| } |
|
|
| |
| async updateClaudeUsageSnapshot(accountId, usageData) { |
| if (!usageData || typeof usageData !== 'object') { |
| return |
| } |
|
|
| const updates = {} |
|
|
| |
| if (usageData.five_hour) { |
| if (usageData.five_hour.utilization !== undefined) { |
| updates.claudeFiveHourUtilization = String(usageData.five_hour.utilization) |
| } |
| if (usageData.five_hour.resets_at) { |
| updates.claudeFiveHourResetsAt = usageData.five_hour.resets_at |
| } |
| } |
|
|
| |
| if (usageData.seven_day) { |
| if (usageData.seven_day.utilization !== undefined) { |
| updates.claudeSevenDayUtilization = String(usageData.seven_day.utilization) |
| } |
| if (usageData.seven_day.resets_at) { |
| updates.claudeSevenDayResetsAt = usageData.seven_day.resets_at |
| } |
| } |
|
|
| |
| if (usageData.seven_day_opus) { |
| if (usageData.seven_day_opus.utilization !== undefined) { |
| updates.claudeSevenDayOpusUtilization = String(usageData.seven_day_opus.utilization) |
| } |
| if (usageData.seven_day_opus.resets_at) { |
| updates.claudeSevenDayOpusResetsAt = usageData.seven_day_opus.resets_at |
| } |
| } |
|
|
| if (Object.keys(updates).length === 0) { |
| return |
| } |
|
|
| updates.claudeUsageUpdatedAt = new Date().toISOString() |
|
|
| const accountData = await redis.getClaudeAccount(accountId) |
| if (accountData && Object.keys(accountData).length > 0) { |
| Object.assign(accountData, updates) |
| await redis.setClaudeAccount(accountId, accountData) |
| logger.debug( |
| `📊 Updated Claude usage snapshot for account ${accountId}:`, |
| Object.keys(updates) |
| ) |
| } |
| } |
|
|
| |
| async fetchAndUpdateAccountProfile(accountId, accessToken = null, agent = null) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const hasProfileScope = accountData.scopes && accountData.scopes.includes('user:profile') |
| if (!hasProfileScope) { |
| logger.warn( |
| `⚠️ Account ${accountId} does not have user:profile scope, cannot fetch profile` |
| ) |
| throw new Error('Account does not have user:profile permission') |
| } |
|
|
| |
| if (!accessToken) { |
| accessToken = this._decryptSensitiveData(accountData.accessToken) |
| if (!accessToken) { |
| throw new Error('No access token available') |
| } |
| } |
|
|
| |
| if (!agent) { |
| agent = this._createProxyAgent(accountData.proxy) |
| } |
|
|
| logger.info(`📊 Fetching profile info for account: ${accountData.name} (${accountId})`) |
|
|
| |
| const axiosConfig = { |
| headers: { |
| Authorization: `Bearer ${accessToken}`, |
| 'Content-Type': 'application/json', |
| Accept: 'application/json', |
| 'User-Agent': 'claude-cli/1.0.56 (external, cli)', |
| 'Accept-Language': 'en-US,en;q=0.9' |
| }, |
| timeout: 15000 |
| } |
|
|
| if (agent) { |
| axiosConfig.httpAgent = agent |
| axiosConfig.httpsAgent = agent |
| axiosConfig.proxy = false |
| } |
|
|
| const response = await axios.get('https://api.anthropic.com/api/oauth/profile', axiosConfig) |
|
|
| if (response.status === 200 && response.data) { |
| const profileData = response.data |
|
|
| logger.info('✅ Successfully fetched profile data:', { |
| email: profileData.account?.email, |
| hasClaudeMax: profileData.account?.has_claude_max, |
| hasClaudePro: profileData.account?.has_claude_pro, |
| organizationType: profileData.organization?.organization_type |
| }) |
|
|
| |
| const subscriptionInfo = { |
| |
| email: profileData.account?.email, |
| fullName: profileData.account?.full_name, |
| displayName: profileData.account?.display_name, |
| hasClaudeMax: profileData.account?.has_claude_max || false, |
| hasClaudePro: profileData.account?.has_claude_pro || false, |
| accountUuid: profileData.account?.uuid, |
|
|
| |
| organizationName: profileData.organization?.name, |
| organizationUuid: profileData.organization?.uuid, |
| billingType: profileData.organization?.billing_type, |
| rateLimitTier: profileData.organization?.rate_limit_tier, |
| organizationType: profileData.organization?.organization_type, |
|
|
| |
| accountType: |
| profileData.account?.has_claude_max === true |
| ? 'claude_max' |
| : profileData.account?.has_claude_pro === true |
| ? 'claude_pro' |
| : 'free', |
|
|
| |
| profileFetchedAt: new Date().toISOString() |
| } |
|
|
| |
| accountData.subscriptionInfo = JSON.stringify(subscriptionInfo) |
| accountData.profileUpdatedAt = new Date().toISOString() |
|
|
| |
| if (profileData.account?.email) { |
| accountData.email = this._encryptSensitiveData(profileData.account.email) |
| } |
|
|
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| logger.success( |
| `✅ Updated account profile for ${accountData.name} (${accountId}) - Type: ${subscriptionInfo.accountType}` |
| ) |
|
|
| return subscriptionInfo |
| } else { |
| throw new Error(`Failed to fetch profile with status: ${response.status}`) |
| } |
| } catch (error) { |
| if (error.response?.status === 401) { |
| logger.warn(`⚠️ Profile API returned 401 for account ${accountId} - token may be invalid`) |
| } else if (error.response?.status === 403) { |
| logger.warn( |
| `⚠️ Profile API returned 403 for account ${accountId} - insufficient permissions` |
| ) |
| } else { |
| logger.error(`❌ Failed to fetch profile for account ${accountId}:`, error.message) |
| } |
| throw error |
| } |
| } |
|
|
| |
| async updateAllAccountProfiles() { |
| try { |
| logger.info('🔄 Starting batch profile update for all accounts...') |
|
|
| const accounts = await redis.getAllClaudeAccounts() |
| let successCount = 0 |
| let failureCount = 0 |
| const results = [] |
|
|
| for (const account of accounts) { |
| |
| if (account.isActive !== 'true' || account.status === 'error') { |
| logger.info(`⏩ Skipping inactive/error account: ${account.name} (${account.id})`) |
| continue |
| } |
|
|
| |
| const hasProfileScope = account.scopes && account.scopes.includes('user:profile') |
| if (!hasProfileScope) { |
| logger.info( |
| `⏩ Skipping account without user:profile scope: ${account.name} (${account.id})` |
| ) |
| results.push({ |
| accountId: account.id, |
| accountName: account.name, |
| success: false, |
| error: 'No user:profile permission (Setup Token account)' |
| }) |
| continue |
| } |
|
|
| try { |
| |
| const accessToken = await this.getValidAccessToken(account.id) |
| if (accessToken) { |
| const profileInfo = await this.fetchAndUpdateAccountProfile(account.id, accessToken) |
| successCount++ |
| results.push({ |
| accountId: account.id, |
| accountName: account.name, |
| success: true, |
| accountType: profileInfo.accountType |
| }) |
| } |
| } catch (error) { |
| failureCount++ |
| results.push({ |
| accountId: account.id, |
| accountName: account.name, |
| success: false, |
| error: error.message |
| }) |
| logger.warn( |
| `⚠️ Failed to update profile for account ${account.name} (${account.id}): ${error.message}` |
| ) |
| } |
|
|
| |
| await new Promise((resolve) => setTimeout(resolve, 1000)) |
| } |
|
|
| logger.success(`✅ Profile update completed: ${successCount} success, ${failureCount} failed`) |
|
|
| return { |
| totalAccounts: accounts.length, |
| successCount, |
| failureCount, |
| results |
| } |
| } catch (error) { |
| logger.error('❌ Failed to update account profiles:', error) |
| throw error |
| } |
| } |
|
|
| |
| async initializeSessionWindows(forceRecalculate = false) { |
| try { |
| logger.info('🔄 Initializing session windows for all Claude accounts...') |
|
|
| const accounts = await redis.getAllClaudeAccounts() |
| let validWindowCount = 0 |
| let expiredWindowCount = 0 |
| let noWindowCount = 0 |
| const now = new Date() |
|
|
| for (const account of accounts) { |
| |
| if (forceRecalculate && (account.sessionWindowStart || account.sessionWindowEnd)) { |
| logger.info(`🔄 Force recalculating window for account ${account.name} (${account.id})`) |
| delete account.sessionWindowStart |
| delete account.sessionWindowEnd |
| delete account.lastRequestTime |
| await redis.setClaudeAccount(account.id, account) |
| } |
|
|
| |
| if (account.sessionWindowStart && account.sessionWindowEnd) { |
| const windowEnd = new Date(account.sessionWindowEnd) |
| const windowStart = new Date(account.sessionWindowStart) |
| const timeUntilExpires = Math.round((windowEnd.getTime() - now.getTime()) / (1000 * 60)) |
|
|
| if (now.getTime() < windowEnd.getTime()) { |
| |
| validWindowCount++ |
| logger.info( |
| `✅ Account ${account.name} (${account.id}) has valid window: ${windowStart.toISOString()} - ${windowEnd.toISOString()} (${timeUntilExpires} minutes remaining)` |
| ) |
| } else { |
| |
| expiredWindowCount++ |
| logger.warn( |
| `⏰ Account ${account.name} (${account.id}) window expired: ${windowStart.toISOString()} - ${windowEnd.toISOString()}` |
| ) |
|
|
| |
| delete account.sessionWindowStart |
| delete account.sessionWindowEnd |
| delete account.lastRequestTime |
| await redis.setClaudeAccount(account.id, account) |
| } |
| } else { |
| noWindowCount++ |
| logger.info( |
| `📭 Account ${account.name} (${account.id}) has no session window - will create on next request` |
| ) |
| } |
| } |
|
|
| logger.success('✅ Session window initialization completed:') |
| logger.success(` 📊 Total accounts: ${accounts.length}`) |
| logger.success(` ✅ Valid windows: ${validWindowCount}`) |
| logger.success(` ⏰ Expired windows: ${expiredWindowCount}`) |
| logger.success(` 📭 No windows: ${noWindowCount}`) |
|
|
| return { |
| total: accounts.length, |
| validWindows: validWindowCount, |
| expiredWindows: expiredWindowCount, |
| noWindows: noWindowCount |
| } |
| } catch (error) { |
| logger.error('❌ Failed to initialize session windows:', error) |
| return { |
| total: 0, |
| validWindows: 0, |
| expiredWindows: 0, |
| noWindows: 0, |
| error: error.message |
| } |
| } |
| } |
|
|
| |
| async markAccountError(accountId, errorType, sessionHash = null) { |
| const ERROR_CONFIG = { |
| unauthorized: { |
| status: 'unauthorized', |
| errorMessage: 'Account unauthorized (401 errors detected)', |
| timestampField: 'unauthorizedAt', |
| errorCode: 'CLAUDE_OAUTH_UNAUTHORIZED', |
| logMessage: 'unauthorized' |
| }, |
| blocked: { |
| status: 'blocked', |
| errorMessage: 'Account blocked (403 error detected - account may be suspended by Claude)', |
| timestampField: 'blockedAt', |
| errorCode: 'CLAUDE_OAUTH_BLOCKED', |
| logMessage: 'blocked' |
| } |
| } |
|
|
| try { |
| const errorConfig = ERROR_CONFIG[errorType] |
| if (!errorConfig) { |
| throw new Error(`Unsupported error type: ${errorType}`) |
| } |
|
|
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const updatedAccountData = { ...accountData } |
| updatedAccountData.status = errorConfig.status |
| updatedAccountData.schedulable = 'false' |
| updatedAccountData.errorMessage = errorConfig.errorMessage |
| updatedAccountData[errorConfig.timestampField] = new Date().toISOString() |
|
|
| |
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| |
| if (sessionHash) { |
| await redis.client.del(`sticky_session:${sessionHash}`) |
| logger.info(`🗑️ Deleted sticky session mapping for hash: ${sessionHash}`) |
| } |
|
|
| logger.warn( |
| `⚠️ Account ${accountData.name} (${accountId}) marked as ${errorConfig.logMessage} and disabled for scheduling` |
| ) |
|
|
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name, |
| platform: 'claude-oauth', |
| status: errorConfig.status, |
| errorCode: errorConfig.errorCode, |
| reason: errorConfig.errorMessage, |
| timestamp: getISOStringWithTimezone(new Date()) |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send webhook notification:', webhookError) |
| } |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to mark account ${accountId} as ${errorType}:`, error) |
| throw error |
| } |
| } |
|
|
| |
| async markAccountUnauthorized(accountId, sessionHash = null) { |
| return this.markAccountError(accountId, 'unauthorized', sessionHash) |
| } |
|
|
| |
| async markAccountBlocked(accountId, sessionHash = null) { |
| return this.markAccountError(accountId, 'blocked', sessionHash) |
| } |
|
|
| |
| async resetAccountStatus(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const updatedAccountData = { ...accountData } |
|
|
| |
| if (updatedAccountData.accessToken) { |
| updatedAccountData.status = 'active' |
| } else { |
| updatedAccountData.status = 'created' |
| } |
|
|
| |
| updatedAccountData.schedulable = 'true' |
| |
| delete updatedAccountData.rateLimitAutoStopped |
| delete updatedAccountData.fiveHourAutoStopped |
| delete updatedAccountData.fiveHourStoppedAt |
| delete updatedAccountData.tempErrorAutoStopped |
| delete updatedAccountData.fiveHourWarningWindow |
| delete updatedAccountData.fiveHourWarningCount |
| delete updatedAccountData.fiveHourWarningLastSentAt |
| |
| delete updatedAccountData.autoStoppedAt |
| delete updatedAccountData.stoppedReason |
|
|
| |
| delete updatedAccountData.errorMessage |
| delete updatedAccountData.unauthorizedAt |
| delete updatedAccountData.blockedAt |
| delete updatedAccountData.rateLimitedAt |
| delete updatedAccountData.rateLimitStatus |
| delete updatedAccountData.rateLimitEndAt |
| delete updatedAccountData.tempErrorAt |
| delete updatedAccountData.sessionWindowStart |
| delete updatedAccountData.sessionWindowEnd |
|
|
| |
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| |
| const fieldsToDelete = [ |
| 'errorMessage', |
| 'unauthorizedAt', |
| 'blockedAt', |
| 'rateLimitedAt', |
| 'rateLimitStatus', |
| 'rateLimitEndAt', |
| 'tempErrorAt', |
| 'sessionWindowStart', |
| 'sessionWindowEnd', |
| |
| 'rateLimitAutoStopped', |
| 'fiveHourAutoStopped', |
| 'fiveHourStoppedAt', |
| 'fiveHourWarningWindow', |
| 'fiveHourWarningCount', |
| 'fiveHourWarningLastSentAt', |
| 'tempErrorAutoStopped', |
| |
| 'autoStoppedAt', |
| 'stoppedReason' |
| ] |
| await redis.client.hdel(`claude:account:${accountId}`, ...fieldsToDelete) |
|
|
| |
| const errorKey = `claude_account:${accountId}:401_errors` |
| await redis.client.del(errorKey) |
|
|
| |
| const rateLimitKey = `ratelimit:${accountId}` |
| await redis.client.del(rateLimitKey) |
|
|
| |
| const serverErrorKey = `claude_account:${accountId}:5xx_errors` |
| await redis.client.del(serverErrorKey) |
|
|
| logger.info( |
| `✅ Successfully reset all error states for account ${accountData.name} (${accountId})` |
| ) |
|
|
| return { |
| success: true, |
| account: { |
| id: accountId, |
| name: accountData.name, |
| status: updatedAccountData.status, |
| schedulable: updatedAccountData.schedulable === 'true' |
| } |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to reset account status for ${accountId}:`, error) |
| throw error |
| } |
| } |
|
|
| |
| async cleanupTempErrorAccounts() { |
| try { |
| const accounts = await redis.getAllClaudeAccounts() |
| let cleanedCount = 0 |
| const TEMP_ERROR_RECOVERY_MINUTES = 5 |
|
|
| for (const account of accounts) { |
| if (account.status === 'temp_error' && account.tempErrorAt) { |
| const tempErrorAt = new Date(account.tempErrorAt) |
| const now = new Date() |
| const minutesSinceTempError = (now - tempErrorAt) / (1000 * 60) |
|
|
| |
| if (minutesSinceTempError > TEMP_ERROR_RECOVERY_MINUTES) { |
| account.status = 'active' |
| |
| if (account.tempErrorAutoStopped === 'true') { |
| account.schedulable = 'true' |
| delete account.tempErrorAutoStopped |
| } |
| delete account.errorMessage |
| delete account.tempErrorAt |
| await redis.setClaudeAccount(account.id, account) |
|
|
| |
| await redis.client.hdel( |
| `claude:account:${account.id}`, |
| 'errorMessage', |
| 'tempErrorAt', |
| 'tempErrorAutoStopped' |
| ) |
|
|
| |
| await this.clearInternalErrors(account.id) |
| cleanedCount++ |
| logger.success(`🧹 Reset temp_error status for account ${account.name} (${account.id})`) |
| } |
| } |
| } |
|
|
| if (cleanedCount > 0) { |
| logger.success(`🧹 Reset ${cleanedCount} temp_error accounts`) |
| } |
|
|
| return cleanedCount |
| } catch (error) { |
| logger.error('❌ Failed to cleanup temp_error accounts:', error) |
| return 0 |
| } |
| } |
|
|
| |
| async recordServerError(accountId, statusCode) { |
| try { |
| const key = `claude_account:${accountId}:5xx_errors` |
|
|
| |
| await redis.client.incr(key) |
| await redis.client.expire(key, 300) |
|
|
| logger.info(`📝 Recorded ${statusCode} error for account ${accountId}`) |
| } catch (error) { |
| logger.error(`❌ Failed to record ${statusCode} error for account ${accountId}:`, error) |
| } |
| } |
|
|
| |
| async recordInternalError(accountId) { |
| return this.recordServerError(accountId, 500) |
| } |
|
|
| |
| async getServerErrorCount(accountId) { |
| try { |
| const key = `claude_account:${accountId}:5xx_errors` |
|
|
| const count = await redis.client.get(key) |
| return parseInt(count) || 0 |
| } catch (error) { |
| logger.error(`❌ Failed to get 5xx error count for account ${accountId}:`, error) |
| return 0 |
| } |
| } |
|
|
| |
| async getInternalErrorCount(accountId) { |
| return this.getServerErrorCount(accountId) |
| } |
|
|
| |
| async clearInternalErrors(accountId) { |
| try { |
| const key = `claude_account:${accountId}:5xx_errors` |
|
|
| await redis.client.del(key) |
| logger.info(`✅ Cleared 5xx error count for account ${accountId}`) |
| } catch (error) { |
| logger.error(`❌ Failed to clear 5xx errors for account ${accountId}:`, error) |
| } |
| } |
|
|
| |
| async markAccountTempError(accountId, sessionHash = null) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const updatedAccountData = { ...accountData } |
| updatedAccountData.status = 'temp_error' |
| updatedAccountData.schedulable = 'false' |
| updatedAccountData.errorMessage = 'Account temporarily disabled due to consecutive 500 errors' |
| updatedAccountData.tempErrorAt = new Date().toISOString() |
| |
| updatedAccountData.tempErrorAutoStopped = 'true' |
|
|
| |
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| |
| setTimeout( |
| async () => { |
| try { |
| const account = await redis.getClaudeAccount(accountId) |
| if (account && account.status === 'temp_error' && account.tempErrorAt) { |
| |
| const tempErrorAt = new Date(account.tempErrorAt) |
| const now = new Date() |
| const minutesSince = (now - tempErrorAt) / (1000 * 60) |
|
|
| if (minutesSince >= 5) { |
| |
| account.status = 'active' |
| |
| if (account.tempErrorAutoStopped === 'true') { |
| account.schedulable = 'true' |
| delete account.tempErrorAutoStopped |
| } |
| delete account.errorMessage |
| delete account.tempErrorAt |
|
|
| await redis.setClaudeAccount(accountId, account) |
|
|
| |
| await redis.client.hdel( |
| `claude:account:${accountId}`, |
| 'errorMessage', |
| 'tempErrorAt', |
| 'tempErrorAutoStopped' |
| ) |
|
|
| |
| await this.clearInternalErrors(accountId) |
|
|
| logger.success( |
| `✅ Auto-recovered temp_error after 5 minutes: ${account.name} (${accountId})` |
| ) |
| } else { |
| logger.debug( |
| `⏰ Temp error timer triggered but only ${minutesSince.toFixed(1)} minutes passed for ${account.name} (${accountId})` |
| ) |
| } |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to auto-recover temp_error account ${accountId}:`, error) |
| } |
| }, |
| 6 * 60 * 1000 |
| ) |
|
|
| |
| if (sessionHash) { |
| await redis.client.del(`sticky_session:${sessionHash}`) |
| logger.info(`🗑️ Deleted sticky session mapping for hash: ${sessionHash}`) |
| } |
|
|
| logger.warn( |
| `⚠️ Account ${accountData.name} (${accountId}) marked as temp_error and disabled for scheduling` |
| ) |
|
|
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name, |
| platform: 'claude-oauth', |
| status: 'temp_error', |
| errorCode: 'CLAUDE_OAUTH_TEMP_ERROR', |
| reason: 'Account temporarily disabled due to consecutive 500 errors' |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send webhook notification:', webhookError) |
| } |
|
|
| return { success: true } |
| } catch (error) { |
| logger.error(`❌ Failed to mark account ${accountId} as temp_error:`, error) |
| throw error |
| } |
| } |
|
|
| |
| async updateSessionWindowStatus(accountId, status) { |
| try { |
| |
| if (!accountId || !status) { |
| logger.warn( |
| `Invalid parameters for updateSessionWindowStatus: accountId=${accountId}, status=${status}` |
| ) |
| return |
| } |
|
|
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData || Object.keys(accountData).length === 0) { |
| logger.warn(`Account not found: ${accountId}`) |
| return |
| } |
|
|
| |
| const validStatuses = ['allowed', 'allowed_warning', 'rejected'] |
| if (!validStatuses.includes(status)) { |
| logger.warn(`Invalid session window status: ${status} for account ${accountId}`) |
| return |
| } |
|
|
| const now = new Date() |
| const nowIso = now.toISOString() |
|
|
| |
| accountData.sessionWindowStatus = status |
| accountData.sessionWindowStatusUpdatedAt = nowIso |
|
|
| |
| if (status === 'allowed_warning' && accountData.autoStopOnWarning === 'true') { |
| const alreadyAutoStopped = |
| accountData.schedulable === 'false' && accountData.fiveHourAutoStopped === 'true' |
|
|
| if (!alreadyAutoStopped) { |
| const windowIdentifier = |
| accountData.sessionWindowEnd || accountData.sessionWindowStart || 'unknown' |
|
|
| let warningCount = 0 |
| if (accountData.fiveHourWarningWindow === windowIdentifier) { |
| const parsedCount = parseInt(accountData.fiveHourWarningCount || '0', 10) |
| warningCount = Number.isNaN(parsedCount) ? 0 : parsedCount |
| } |
|
|
| const maxWarningsPerWindow = this.maxFiveHourWarningsPerWindow |
|
|
| logger.warn( |
| `⚠️ Account ${accountData.name} (${accountId}) approaching 5h limit, auto-stopping scheduling` |
| ) |
| accountData.schedulable = 'false' |
| |
| accountData.fiveHourAutoStopped = 'true' |
| accountData.fiveHourStoppedAt = nowIso |
| |
| accountData.stoppedReason = '5小时使用量接近限制,已自动停止调度' |
|
|
| const canSendWarning = warningCount < maxWarningsPerWindow |
| let updatedWarningCount = warningCount |
|
|
| accountData.fiveHourWarningWindow = windowIdentifier |
| if (canSendWarning) { |
| updatedWarningCount += 1 |
| accountData.fiveHourWarningLastSentAt = nowIso |
| } |
| accountData.fiveHourWarningCount = updatedWarningCount.toString() |
|
|
| if (canSendWarning) { |
| |
| try { |
| const webhookNotifier = require('../utils/webhookNotifier') |
| await webhookNotifier.sendAccountAnomalyNotification({ |
| accountId, |
| accountName: accountData.name || 'Claude Account', |
| platform: 'claude', |
| status: 'warning', |
| errorCode: 'CLAUDE_5H_LIMIT_WARNING', |
| reason: '5小时使用量接近限制,已自动停止调度', |
| timestamp: getISOStringWithTimezone(now) |
| }) |
| } catch (webhookError) { |
| logger.error('Failed to send webhook notification:', webhookError) |
| } |
| } else { |
| logger.debug( |
| `⚠️ Account ${accountData.name} (${accountId}) reached max ${maxWarningsPerWindow} warning notifications for current 5h window, skipping webhook` |
| ) |
| } |
| } else { |
| logger.debug( |
| `⚠️ Account ${accountData.name} (${accountId}) already auto-stopped for 5h limit, skipping duplicate warning` |
| ) |
| } |
| } |
|
|
| await redis.setClaudeAccount(accountId, accountData) |
|
|
| logger.info( |
| `📊 Updated session window status for account ${accountData.name} (${accountId}): ${status}` |
| ) |
| } catch (error) { |
| logger.error(`❌ Failed to update session window status for account ${accountId}:`, error) |
| } |
| } |
|
|
| |
| async markAccountOverloaded(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData) { |
| throw new Error('Account not found') |
| } |
|
|
| |
| const overloadMinutes = config.overloadHandling?.enabled || 0 |
|
|
| if (overloadMinutes === 0) { |
| logger.info('⏭️ 529 error handling is disabled') |
| return { success: false, error: '529 error handling is disabled' } |
| } |
|
|
| const overloadKey = `account:overload:${accountId}` |
| const ttl = overloadMinutes * 60 |
|
|
| await redis.setex( |
| overloadKey, |
| ttl, |
| JSON.stringify({ |
| accountId, |
| accountName: accountData.name, |
| markedAt: new Date().toISOString(), |
| expiresAt: new Date(Date.now() + ttl * 1000).toISOString() |
| }) |
| ) |
|
|
| logger.warn( |
| `🚫 Account ${accountData.name} (${accountId}) marked as overloaded for ${overloadMinutes} minutes` |
| ) |
|
|
| |
| const updates = { |
| lastOverloadAt: new Date().toISOString(), |
| errorMessage: `529错误 - 过载${overloadMinutes}分钟` |
| } |
|
|
| const updatedAccountData = { ...accountData, ...updates } |
| await redis.setClaudeAccount(accountId, updatedAccountData) |
|
|
| return { success: true, accountName: accountData.name, duration: overloadMinutes } |
| } catch (error) { |
| logger.error(`❌ Failed to mark account as overloaded: ${accountId}`, error) |
| |
| return { success: false, error: error.message } |
| } |
| } |
|
|
| |
| async isAccountOverloaded(accountId) { |
| try { |
| |
| const overloadMinutes = config.overloadHandling?.enabled || 0 |
| if (overloadMinutes === 0) { |
| return false |
| } |
|
|
| const overloadKey = `account:overload:${accountId}` |
| const overloadData = await redis.get(overloadKey) |
|
|
| if (overloadData) { |
| |
| return true |
| } |
|
|
| |
| return false |
| } catch (error) { |
| logger.error(`❌ Failed to check if account is overloaded: ${accountId}`, error) |
| return false |
| } |
| } |
|
|
| |
| async removeAccountOverload(accountId) { |
| try { |
| const accountData = await redis.getClaudeAccount(accountId) |
| if (!accountData) { |
| throw new Error('Account not found') |
| } |
|
|
| const overloadKey = `account:overload:${accountId}` |
| await redis.del(overloadKey) |
|
|
| logger.info(`✅ Account ${accountData.name} (${accountId}) overload status removed`) |
|
|
| |
| if (accountData.errorMessage && accountData.errorMessage.includes('529错误')) { |
| const updatedAccountData = { ...accountData } |
| delete updatedAccountData.errorMessage |
| delete updatedAccountData.lastOverloadAt |
| await redis.setClaudeAccount(accountId, updatedAccountData) |
| } |
| } catch (error) { |
| logger.error(`❌ Failed to remove overload status for account: ${accountId}`, error) |
| |
| } |
| } |
|
|
| |
| |
| |
| |
| |
| async checkAndRecoverFiveHourStoppedAccounts() { |
| const result = { |
| checked: 0, |
| recovered: 0, |
| accounts: [] |
| } |
|
|
| try { |
| const accounts = await this.getAllAccounts() |
| const now = new Date() |
|
|
| for (const account of accounts) { |
| |
| |
| if (account.fiveHourAutoStopped === true && account.schedulable === false) { |
| result.checked++ |
|
|
| |
| const lockKey = `lock:account:${account.id}:recovery` |
| const lockValue = `${Date.now()}_${Math.random()}` |
| const lockTTL = 5000 |
|
|
| try { |
| |
| const lockAcquired = await redis.setAccountLock(lockKey, lockValue, lockTTL) |
| if (!lockAcquired) { |
| logger.debug( |
| `⏭️ Account ${account.name} (${account.id}) is being processed by another instance` |
| ) |
| continue |
| } |
|
|
| |
| const latestAccount = await redis.getClaudeAccount(account.id) |
| if ( |
| !latestAccount || |
| latestAccount.fiveHourAutoStopped !== 'true' || |
| latestAccount.schedulable !== 'false' |
| ) { |
| |
| await redis.releaseAccountLock(lockKey, lockValue) |
| continue |
| } |
|
|
| |
| let shouldRecover = false |
| let newWindowStart = null |
| let newWindowEnd = null |
|
|
| if (latestAccount.sessionWindowEnd) { |
| const windowEnd = new Date(latestAccount.sessionWindowEnd) |
|
|
| |
| if (now.getTime() > windowEnd.getTime() + 60000) { |
| shouldRecover = true |
|
|
| |
| |
| newWindowStart = new Date(windowEnd) |
| newWindowStart.setMilliseconds(newWindowStart.getMilliseconds() + 1) |
| newWindowEnd = new Date(newWindowStart) |
| newWindowEnd.setHours(newWindowEnd.getHours() + 5) |
|
|
| logger.info( |
| `🔄 Account ${latestAccount.name} (${latestAccount.id}) has entered new session window. ` + |
| `Old window: ${latestAccount.sessionWindowStart} - ${latestAccount.sessionWindowEnd}, ` + |
| `New window: ${newWindowStart.toISOString()} - ${newWindowEnd.toISOString()}` |
| ) |
| } |
| } else { |
| |
| if (latestAccount.fiveHourStoppedAt) { |
| const stoppedAt = new Date(latestAccount.fiveHourStoppedAt) |
| const hoursSinceStopped = (now.getTime() - stoppedAt.getTime()) / (1000 * 60 * 60) |
|
|
| |
| if (hoursSinceStopped > 5.017) { |
| |
| shouldRecover = true |
| newWindowStart = this._calculateSessionWindowStart(now) |
| newWindowEnd = this._calculateSessionWindowEnd(newWindowStart) |
|
|
| logger.info( |
| `🔄 Account ${latestAccount.name} (${latestAccount.id}) stopped ${hoursSinceStopped.toFixed(2)} hours ago, recovering` |
| ) |
| } |
| } |
| } |
|
|
| if (shouldRecover) { |
| |
| const updatedAccountData = { ...latestAccount } |
|
|
| |
| updatedAccountData.schedulable = 'true' |
| delete updatedAccountData.fiveHourAutoStopped |
| delete updatedAccountData.fiveHourStoppedAt |
| await this._clearFiveHourWarningMetadata(account.id, updatedAccountData) |
| delete updatedAccountData.stoppedReason |
|
|
| |
| if (newWindowStart && newWindowEnd) { |
| updatedAccountData.sessionWindowStart = newWindowStart.toISOString() |
| updatedAccountData.sessionWindowEnd = newWindowEnd.toISOString() |
|
|
| |
| delete updatedAccountData.sessionWindowStatus |
| delete updatedAccountData.sessionWindowStatusUpdatedAt |
| } |
|
|
| |
| await redis.setClaudeAccount(account.id, updatedAccountData) |
|
|
| const fieldsToRemove = ['fiveHourAutoStopped', 'fiveHourStoppedAt'] |
| if (newWindowStart && newWindowEnd) { |
| fieldsToRemove.push('sessionWindowStatus', 'sessionWindowStatusUpdatedAt') |
| } |
| await this._removeAccountFields(account.id, fieldsToRemove, 'five_hour_recovery_task') |
|
|
| result.recovered++ |
| result.accounts.push({ |
| id: latestAccount.id, |
| name: latestAccount.name, |
| oldWindow: latestAccount.sessionWindowEnd |
| ? { |
| start: latestAccount.sessionWindowStart, |
| end: latestAccount.sessionWindowEnd |
| } |
| : null, |
| newWindow: |
| newWindowStart && newWindowEnd |
| ? { |
| start: newWindowStart.toISOString(), |
| end: newWindowEnd.toISOString() |
| } |
| : null |
| }) |
|
|
| logger.info( |
| `✅ Auto-resumed scheduling for account ${latestAccount.name} (${latestAccount.id}) - 5-hour limit expired` |
| ) |
| } |
|
|
| |
| await redis.releaseAccountLock(lockKey, lockValue) |
| } catch (error) { |
| |
| if (lockKey && lockValue) { |
| try { |
| await redis.releaseAccountLock(lockKey, lockValue) |
| } catch (unlockError) { |
| logger.error(`Failed to release lock for account ${account.id}:`, unlockError) |
| } |
| } |
| logger.error( |
| `❌ Failed to check/recover 5-hour stopped account ${account.name} (${account.id}):`, |
| error |
| ) |
| } |
| } |
| } |
|
|
| if (result.recovered > 0) { |
| logger.info( |
| `🔄 5-hour limit recovery completed: ${result.recovered}/${result.checked} accounts recovered` |
| ) |
| } |
|
|
| return result |
| } catch (error) { |
| logger.error('❌ Failed to check and recover 5-hour stopped accounts:', error) |
| throw error |
| } |
| } |
|
|
| async _removeAccountFields(accountId, fields = [], context = 'general_cleanup') { |
| if (!Array.isArray(fields) || fields.length === 0) { |
| return |
| } |
|
|
| const filteredFields = fields.filter((field) => typeof field === 'string' && field.trim()) |
| if (filteredFields.length === 0) { |
| return |
| } |
|
|
| const accountKey = `claude:account:${accountId}` |
|
|
| try { |
| await redis.client.hdel(accountKey, ...filteredFields) |
| logger.debug( |
| `🧹 已在 ${context} 阶段为账号 ${accountId} 删除字段 [${filteredFields.join(', ')}]` |
| ) |
| } catch (error) { |
| logger.error( |
| `❌ 无法在 ${context} 阶段为账号 ${accountId} 删除字段 [${filteredFields.join(', ')}]:`, |
| error |
| ) |
| } |
| } |
| } |
|
|
| module.exports = new ClaudeAccountService() |
|
|