Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 | 7x 7x 7x 30x 30x 30x 30x 30x 56x 56x 26x 30x 30x 28x 24x 4x 28x 2x 30x 29x 29x 29x 29x 27x 27x 27x 6x 6x 6x 5x 5x 5x 1x 4x 3x 5x 5x 5x 5x 5x 17x 17x 17x 17x 17x 17x 9x 17x 17x 17x 17x 17x 6x 6x 4x 4x 12x 12x 12x 12x 6x 6x 6x 6x 5x 5x 9x 9x 9x 9x 9x 9x 30x 7x | /**
* Alert manager — creates alerts, tracks error spikes, and sends FCM notifications.
*
* Usage:
* const alertManager = require('./alertManagerInstance');
* alertManager.createAlert('error_spike', 'critical', 'Error spike on /api/users', '15 errors in 5 min', { route: '/api/users' });
* alertManager.trackError('/api/users');
* alertManager.trackSlowEndpoint('/api/rooms', 5200);
*/
const crypto = require('node:crypto');
const DEFAULT_ALERT_CONFIG = {
errorSpikeThreshold: 10,
errorSpikeWindowMinutes: 5,
slowEndpointThresholdMs: 3000,
cronFailureAlert: true,
crashReportAlert: true,
firestoreQuotaWarningPercent: 80,
serverMemoryWarningPercent: 85,
pm2RestartAlert: true,
fcmRecipientUserIds: [],
};
const CONFIG_CACHE_TTL = 60 * 1000; // 60 seconds
function createAlertManager(db, messaging) {
// In-memory error tracking: route -> timestamp[]
const errorWindows = new Map();
// Deduplication: route -> last alert timestamp
const errorAlertDedup = new Map();
const slowAlertDedup = new Map();
// Config cache
let cachedConfig = null;
let configLoadedAt = 0;
async function loadConfig() {
const now = Date.now();
if (cachedConfig && now - configLoadedAt < CONFIG_CACHE_TTL) {
return cachedConfig;
}
try {
const snap = await db.collection('alertConfig').doc('settings').get();
if (snap.exists) {
cachedConfig = { ...DEFAULT_ALERT_CONFIG, ...snap.data() };
} else {
cachedConfig = { ...DEFAULT_ALERT_CONFIG };
}
configLoadedAt = now;
} catch {
// Firestore unavailable — fall back to cached config or defaults
Eif (!cachedConfig) cachedConfig = { ...DEFAULT_ALERT_CONFIG };
}
return cachedConfig;
}
async function createAlert(type, severity, title, message, context = {}) {
try {
const id = crypto.randomBytes(16).toString('hex');
const alertDoc = {
id,
type,
severity,
title,
message,
context,
createdAt: new Date().toISOString(),
status: 'unresolved',
acknowledgedBy: null,
resolvedBy: null,
resolvedAt: null,
};
await db.collection('alerts').doc(id).set(alertDoc);
// Send FCM notifications to admin users
const config = await loadConfig();
const recipientIds = config.fcmRecipientUserIds || [];
for (const userId of recipientIds) {
try {
const userSnap = await db.collection('users').doc(userId).get();
if (!userSnap.exists) continue;
const userData = userSnap.data();
const tokens = [];
if (Array.isArray(userData.fcmTokens)) {
tokens.push(...userData.fcmTokens);
} else if (typeof userData.fcmToken === 'string' && userData.fcmToken) {
tokens.push(userData.fcmToken);
}
// SHY-0496: installation IDs are addressed by the same send() call,
// through `fid` rather than `token`. Gathering only tokens meant an
// operational alert never reached a migrated device.
const fids = Array.isArray(userData.fcmInstallationIds)
? userData.fcmInstallationIds
: [];
for (const fid of fids) {
try {
await messaging.send({
notification: { title, body: message },
fid,
});
} catch {
// Swallowed for the same reason as the token loop below: FCM
// delivery is best-effort and must never disrupt alerting.
}
}
for (const token of tokens) {
try {
await messaging.send({
notification: { title, body: message },
token,
});
} catch {
// Intentionally swallowed — FCM delivery is best-effort, must never disrupt alerting
}
}
} catch {
// Intentionally swallowed — user lookup failure must not prevent other recipients from being notified
}
}
} catch {
// Intentionally swallowed — alert creation must never throw to avoid masking the original error
}
}
async function trackError(route) {
try {
const now = Date.now();
const config = await loadConfig();
const windowMs = (config.errorSpikeWindowMinutes || 5) * 60 * 1000;
const threshold = config.errorSpikeThreshold || 10;
// Add timestamp to rolling window
if (!errorWindows.has(route)) {
errorWindows.set(route, []);
}
const timestamps = errorWindows.get(route);
timestamps.push(now);
// Prune old entries
const cutoff = now - windowMs;
while (timestamps.length > 0 && timestamps[0] < cutoff) {
timestamps.shift();
}
// Check threshold
if (timestamps.length >= threshold) {
// Deduplicate: don't re-alert for same route within window
const lastAlert = errorAlertDedup.get(route) || 0;
if (now - lastAlert > windowMs) {
errorAlertDedup.set(route, now);
await createAlert(
'error_spike',
'critical',
`Error spike on ${route}`,
`${timestamps.length} errors in ${config.errorSpikeWindowMinutes} minutes`,
{ route, errorCount: timestamps.length },
);
}
}
} catch {
// Intentionally swallowed — error tracking must never throw to avoid recursive error loops
}
}
async function trackSlowEndpoint(route, durationMs) {
try {
const config = await loadConfig();
const threshold = config.slowEndpointThresholdMs || 3000;
if (durationMs <= threshold) return;
const now = Date.now();
const DEDUP_WINDOW = 5 * 60 * 1000; // 5 minutes
const lastAlert = slowAlertDedup.get(route) || 0;
if (now - lastAlert <= DEDUP_WINDOW) return;
slowAlertDedup.set(route, now);
await createAlert(
'slow_endpoint',
'warning',
`Slow endpoint: ${route}`,
`Response took ${durationMs}ms (threshold: ${threshold}ms)`,
{ route, durationMs, thresholdMs: threshold },
);
} catch {
// Intentionally swallowed — slow endpoint tracking must never throw to avoid disrupting request flow
}
}
function getConfig() {
return cachedConfig || { ...DEFAULT_ALERT_CONFIG };
}
// Test helpers
function _clearState() {
errorWindows.clear();
errorAlertDedup.clear();
slowAlertDedup.clear();
cachedConfig = null;
configLoadedAt = 0;
}
return {
createAlert,
trackError,
trackSlowEndpoint,
getConfig,
_clearState,
};
}
module.exports = { createAlertManager, DEFAULT_ALERT_CONFIG };
|