Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | 2x 2x 2x 2x 2x 11x 11x 13x 12x 12x 12x 12x 4x 12x 11x 5x 1x 1x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 2x 4x 1x 1x 1x 1x 1x 1x 4x 2x 2x | /**
* Server health metrics check — memory usage + PM2 restart detection.
*
* Originally a 5-min in-process cron; now invoked by the Better Stack
* heartbeat endpoint at GET /api/system/health. The in-flight guard
* below ensures concurrent heartbeat hits don't double-fire the PM2
* check or race on `lastRestartCounts` (which would emit duplicate
* alerts because alertManager has no dedup of its own).
*/
const { execFile } = require('node:child_process');
const os = require('node:os');
const log = require('../utils/log');
// Track last-known restart counts to only alert on NEW restarts. Now
// shared across HTTP-triggered invocations; the in-flight guard below
// keeps the read-then-write race window single-threaded.
const lastRestartCounts = {};
// Concurrent-invocation guard. Multiple Better Stack hits (or a future
// secondary monitor) firing during a slow `pm2 jlist` (10s timeout)
// would otherwise overlap, double-counting restarts and forking
// duplicate child processes. A boolean flag is enough — JavaScript's
// single-threaded model means the read+write in the guard is atomic.
let inFlight = false;
/**
* Pure restart-delta detection (extracted SHY-0120 slice 6, EPIC-0003).
*
* Given a parsed `pm2 jlist` array and the last-known restart counts (MUTATED
* in place to the new counts), return the processes that have NEW restarts
* since the last check. A process is flagged only when its count INCREASED
* from a previously-seen positive baseline (`lastKnown > 0`) — the first
* sighting just records the baseline, so a fresh server start never alerts.
* Processes without a `pm2_env` are skipped (and NOT recorded), matching the
* original inline behaviour exactly. No real collaborator → unit-tested with
* real data arrays; the `execFile` boundary is exercised live in the
* integration test (a real `pm2` restart between runs is not CI-inducible).
*
* @returns {Array<{name: string, newRestarts: number, total: number}>}
*/
function detectPm2Restarts(processes, lastCounts) {
const restarted = [];
for (const proc of processes) {
if (!proc.pm2_env) continue;
const name = proc.name;
const restarts = proc.pm2_env.restart_time || 0;
const lastKnown = lastCounts[name] || 0;
if (restarts > lastKnown && lastKnown > 0) {
restarted.push({ name, newRestarts: restarts - lastKnown, total: restarts });
}
lastCounts[name] = restarts;
}
return restarted;
}
async function serverHealth(alertManager) {
if (inFlight) {
log.debug('server-health', 'check already in flight — skipping concurrent invocation');
return;
}
inFlight = true;
try {
return await runHealthCheck(alertManager);
} finally {
inFlight = false;
}
}
async function runHealthCheck(alertManager) {
// Check memory usage — RSS vs system total (not V8 heap ratio, which is
// misleadingly high because V8 keeps heapTotal close to heapUsed)
const mem = process.memoryUsage();
const systemTotalBytes = os.totalmem();
const rssPercent = (mem.rss / systemTotalBytes) * 100;
const rssMB = Math.round(mem.rss / 1024 / 1024);
const systemTotalMB = Math.round(systemTotalBytes / 1024 / 1024);
const config = alertManager.getConfig();
const memThreshold = config.serverMemoryWarningPercent || 30;
if (rssPercent > memThreshold) {
await alertManager.createAlert(
'high_memory',
'warning',
'High server memory usage',
`RSS at ${rssMB}MB / ${systemTotalMB}MB (${rssPercent.toFixed(1)}%, threshold: ${memThreshold}%)`,
{
rssMB,
systemTotalMB,
rssPercent: Math.round(rssPercent * 10) / 10,
},
);
}
// Check PM2 restart count — only alert on NEW restarts since last check
if (config.pm2RestartAlert) {
try {
await new Promise((resolve) => {
// eslint-disable-next-line sonarjs/no-os-command-from-path -- PATH inherited from managed PM2 service account
execFile('pm2', ['jlist'], { timeout: 10000 }, (err, stdout) => {
Eif (err || !stdout) {
resolve();
return;
}
try {
const processes = JSON.parse(stdout);
for (const { name, newRestarts, total } of detectPm2Restarts(
processes,
lastRestartCounts,
)) {
alertManager
.createAlert(
'pm2_restart',
'warning',
`PM2 process restarted: ${name}`,
`${newRestarts} new restart(s) (total: ${total})`,
{
processName: name,
restartCount: total,
newRestarts,
},
)
.catch((alertErr) =>
log.error('server-health', 'Failed to create PM2 restart alert', {
error: alertErr.message,
}),
);
}
} catch {
// PM2 output parsing failed — logged as warning but non-fatal for health check
log.warn('cron', 'serverHealth: failed to parse PM2 output');
}
resolve();
});
});
} catch (pm2Err) {
// PM2 binary unavailable or exec failed — log and continue health check
log.warn('cron', 'serverHealth: PM2 check failed', { error: pm2Err.message });
}
}
log.debug('cron', 'serverHealth: check completed', {
rssMB,
rssPercent: Math.round(rssPercent * 10) / 10,
});
}
module.exports = serverHealth;
// Exported for unit-test access — the pure restart-delta logic is tested with
// real data arrays (no execFile mock); see tests/cron/serverHealth.unit.test.js.
module.exports.detectPm2Restarts = detectPm2Restarts;
|