Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | 3x 3x 3x 26x 26x 26x 26x 26x 42x 42x 19x 23x 23x 21x 20x 1x 21x 2x 23x 19x 19x 19x 19x 17x 17x 17x 6x 6x 6x 5x 5x 5x 1x 4x 3x 5x 5x 5x 17x 17x 17x 17x 17x 17x 9x 17x 17x 17x 17x 17x 6x 6x 4x 4x 8x 8x 8x 8x 6x 6x 6x 6x 5x 5x 5x 1x 1x 1x 1x 1x 26x 3x | /**
* Alert manager — creates alerts, tracks error spikes, and sends FCM notifications.
*
* Usage:
* const alertManager = require('./alertManagerInstance');
* alertManager.createAlert('error_spike', 'critical', 'Error spike on /api/users', '15 errors in 5 min', { route: '/api/users' });
* alertManager.trackError('/api/users');
* alertManager.trackSlowEndpoint('/api/rooms', 5200);
*/
const crypto = require('node:crypto');
const DEFAULT_ALERT_CONFIG = {
errorSpikeThreshold: 10,
errorSpikeWindowMinutes: 5,
slowEndpointThresholdMs: 3000,
cronFailureAlert: true,
crashReportAlert: true,
firestoreQuotaWarningPercent: 80,
serverMemoryWarningPercent: 85,
pm2RestartAlert: true,
fcmRecipientUserIds: [],
};
const CONFIG_CACHE_TTL = 60 * 1000; // 60 seconds
function createAlertManager(db, messaging) {
// In-memory error tracking: route -> timestamp[]
const errorWindows = new Map();
// Deduplication: route -> last alert timestamp
const errorAlertDedup = new Map();
const slowAlertDedup = new Map();
// Config cache
let cachedConfig = null;
let configLoadedAt = 0;
async function loadConfig() {
const now = Date.now();
if (cachedConfig && now - configLoadedAt < CONFIG_CACHE_TTL) {
return cachedConfig;
}
try {
const snap = await db.collection('alertConfig').doc('settings').get();
if (snap.exists) {
cachedConfig = { ...DEFAULT_ALERT_CONFIG, ...snap.data() };
} else {
cachedConfig = { ...DEFAULT_ALERT_CONFIG };
}
configLoadedAt = now;
} catch {
// Firestore unavailable — fall back to cached config or defaults
Eif (!cachedConfig) cachedConfig = { ...DEFAULT_ALERT_CONFIG };
}
return cachedConfig;
}
async function createAlert(type, severity, title, message, context = {}) {
try {
const id = crypto.randomBytes(16).toString('hex');
const alertDoc = {
id,
type,
severity,
title,
message,
context,
createdAt: new Date().toISOString(),
status: 'unresolved',
acknowledgedBy: null,
resolvedBy: null,
resolvedAt: null,
};
await db.collection('alerts').doc(id).set(alertDoc);
// Send FCM notifications to admin users
const config = await loadConfig();
const recipientIds = config.fcmRecipientUserIds || [];
for (const userId of recipientIds) {
try {
const userSnap = await db.collection('users').doc(userId).get();
if (!userSnap.exists) continue;
const userData = userSnap.data();
const tokens = [];
if (Array.isArray(userData.fcmTokens)) {
tokens.push(...userData.fcmTokens);
} else if (typeof userData.fcmToken === 'string' && userData.fcmToken) {
tokens.push(userData.fcmToken);
}
for (const token of tokens) {
try {
await messaging.send({
notification: { title, body: message },
token,
});
} catch {
// Intentionally swallowed — FCM delivery is best-effort, must never disrupt alerting
}
}
} catch {
// Intentionally swallowed — user lookup failure must not prevent other recipients from being notified
}
}
} catch {
// Intentionally swallowed — alert creation must never throw to avoid masking the original error
}
}
async function trackError(route) {
try {
const now = Date.now();
const config = await loadConfig();
const windowMs = (config.errorSpikeWindowMinutes || 5) * 60 * 1000;
const threshold = config.errorSpikeThreshold || 10;
// Add timestamp to rolling window
if (!errorWindows.has(route)) {
errorWindows.set(route, []);
}
const timestamps = errorWindows.get(route);
timestamps.push(now);
// Prune old entries
const cutoff = now - windowMs;
while (timestamps.length > 0 && timestamps[0] < cutoff) {
timestamps.shift();
}
// Check threshold
if (timestamps.length >= threshold) {
// Deduplicate: don't re-alert for same route within window
const lastAlert = errorAlertDedup.get(route) || 0;
if (now - lastAlert > windowMs) {
errorAlertDedup.set(route, now);
await createAlert(
'error_spike',
'critical',
`Error spike on ${route}`,
`${timestamps.length} errors in ${config.errorSpikeWindowMinutes} minutes`,
{ route, errorCount: timestamps.length },
);
}
}
} catch {
// Intentionally swallowed — error tracking must never throw to avoid recursive error loops
}
}
async function trackSlowEndpoint(route, durationMs) {
try {
const config = await loadConfig();
const threshold = config.slowEndpointThresholdMs || 3000;
if (durationMs <= threshold) return;
const now = Date.now();
const DEDUP_WINDOW = 5 * 60 * 1000; // 5 minutes
const lastAlert = slowAlertDedup.get(route) || 0;
if (now - lastAlert <= DEDUP_WINDOW) return;
slowAlertDedup.set(route, now);
await createAlert(
'slow_endpoint',
'warning',
`Slow endpoint: ${route}`,
`Response took ${durationMs}ms (threshold: ${threshold}ms)`,
{ route, durationMs, thresholdMs: threshold },
);
} catch {
// Intentionally swallowed — slow endpoint tracking must never throw to avoid disrupting request flow
}
}
function getConfig() {
return cachedConfig || { ...DEFAULT_ALERT_CONFIG };
}
// Test helpers
function _clearState() {
errorWindows.clear();
errorAlertDedup.clear();
slowAlertDedup.clear();
cachedConfig = null;
configLoadedAt = 0;
}
return {
createAlert,
trackError,
trackSlowEndpoint,
getConfig,
_clearState,
};
}
module.exports = { createAlertManager, DEFAULT_ALERT_CONFIG };
|