From 135e6a7cd31d3f51de0c19637ccc681847158221 Mon Sep 17 00:00:00 2001 From: Steven Roberts Date: Sun, 27 Sep 2026 10:51:22 -0700 Subject: [PATCH] Add gannet daily health report to Server Status and a Gannet Dashboard check_servers.py now parses gannet's daily owlshell/latest.txt report (plus the last 30 dated copies for history) into a top-level gannet_stats field, relayed through the server-status branch like raven_stats. - Server Status table gets a "Daily health" column; gannet's shows its alerts and links to the dashboard. - New "Gannet Health" section on Computing Hardware (alerts + disks). - New Gannet Dashboard page: KPI tiles, alerts, storage with inodes, volume-use and load history charts, memory, RAID, services, failed units, kernel errors, and the raw report. Co-Authored-By: Claude Opus 5.5 --- docs/Computing-Hardware.md | 13 +- docs/Gannet-Dashboard.md | 10 + docs/javascripts/gannet-health.js | 515 +++++++++++++++++++++++++++++ docs/stylesheets/server-status.css | 181 ++++++++++ mkdocs.yml | 2 + scripts/README.md | 34 +- scripts/check_servers.py | 230 +++++++++++++ 7 files changed, 982 insertions(+), 3 deletions(-) create mode 100644 docs/Gannet-Dashboard.md create mode 100644 docs/javascripts/gannet-health.js diff --git a/docs/Computing-Hardware.md b/docs/Computing-Hardware.md index 74300f85..bb3a9b6d 100644 --- a/docs/Computing-Hardware.md +++ b/docs/Computing-Hardware.md @@ -10,23 +10,26 @@ Below is a list of computing resources we have available, as well as some links - + + + +
ServerStatusLast checked
ServerStatusLast checkedDaily health
raven.fish.washington.edu checking… —disk / CPU below
gannet.fish.washington.edu checking… —loading…
klone.hyak.uw.edu checking… —UW-IT status
@@ -51,6 +54,14 @@ Checks run automatically — the timestamp shows when each host was last reached This comes from a [daily snapshot](https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log) (`df` plus CPU load) that raven writes to gannet each morning around 7am, not from the live network check above — so it can be up to a day old, and it is flagged if the snapshot itself goes stale. Drives are listed fullest first. +### Gannet Health + +
+

Loading gannet's latest health report…

+
+ +Gannet writes a [daily health report](https://gannet.fish.washington.edu/v1_web/owlshell/latest.txt) covering disk and inode use, memory, RAID, services, and kernel errors. See the **[Gannet Dashboard](Gannet-Dashboard.md)** for the full report and day-by-day history. + ## Accounts You will need accounts with the following services in order to minimally function in the Roberts Lab: diff --git a/docs/Gannet-Dashboard.md b/docs/Gannet-Dashboard.md new file mode 100644 index 00000000..ae7c653b --- /dev/null +++ b/docs/Gannet-Dashboard.md @@ -0,0 +1,10 @@ +# Gannet Dashboard + +Health of `gannet.fish.washington.edu`, the lab's Synology NAS, from the [daily health report](https://gannet.fish.washington.edu/v1_web/owlshell/latest.txt) it writes each morning. The server status probers pick up each new report within about an hour of it being written, so this page is at most a day old. It is flagged if the report goes stale. For whether gannet is answering right now, see [Server Status](Computing-Hardware.md#server-status). + +
+

Loading gannet's latest health report…

+
+ +!!! note + Volumes at 90% or more are marked critical and 75% or more are marked as warnings. The history charts are built from the dated `gannet_health_YYYY-MM-DD.txt` copies of the report (the last 30 days). diff --git a/docs/javascripts/gannet-health.js b/docs/javascripts/gannet-health.js new file mode 100644 index 00000000..0d9c011f --- /dev/null +++ b/docs/javascripts/gannet-health.js @@ -0,0 +1,515 @@ +/* + * Gannet daily health report, on two pages: + * + * Computing Hardware - the "Daily health" cell in the status table + * ([data-health="gannet"]) and the compact + * "Gannet Health" section ([data-gannet-summary]). + * Gannet Dashboard - the full dashboard ([data-gannet-dashboard]). + * + * The report itself is written once a day on gannet by gannet_health.sh + * (https://gannet.fish.washington.edu/v1_web/owlshell/latest.txt). The + * probers in scripts/ parse it into doc.gannet_stats and relay it through the + * server-status branch, because gannet does not send CORS headers and the + * page cannot fetch the text file directly. See scripts/README.md. + * + * Like raven_stats, the freshest copy wins by its own "generated" timestamp, + * not by which prober ran last. + */ +(function () { + "use strict"; + + // Daily report: 30 hours clears normal day-to-day slip in when the cron + // fires while still catching a stuck job within about a day. + var STALE_MS = 30 * 60 * 60 * 1000; + var SOURCES = ["internal.json", "external.json"]; + var DEFAULT_BASE = + "https://raw.githubusercontent.com/RobertsLab/resources/server-status/status"; + var REPORT_URL = "https://gannet.fish.washington.edu/v1_web/owlshell/latest.txt"; + var SVG_NS = "http://www.w3.org/2000/svg"; + + function fetchSource(base, name) { + return fetch(base + "/" + name + "?t=" + Date.now(), { cache: "no-store" }) + .then(function (res) { + return res.ok ? res.json() : null; + }) + .catch(function () { + return null; + }); + } + + function mergeStats(docs) { + var best = null; + docs.forEach(function (doc) { + if (!doc || !doc.gannet_stats) return; + var at = Date.parse(doc.gannet_stats.generated); + if (isNaN(at)) return; + if (best && best.generatedAt >= at) return; + best = doc.gannet_stats; + best.generatedAt = at; + }); + return best; + } + + function relativeTime(then, now) { + var mins = Math.round((now - then) / 60000); + if (mins < 1) return "just now"; + if (mins < 60) return mins + " min ago"; + var hrs = Math.round(mins / 60); + if (hrs < 24) return hrs + " hr ago"; + var days = Math.round(hrs / 24); + return days === 1 ? "1 day ago" : days + " days ago"; + } + + function el(tag, className, text) { + var node = document.createElement(tag); + if (className) node.className = className; + if (text != null) node.textContent = text; + return node; + } + + function fullnessClass(pct) { + if (pct >= 90) return "ss-disk-critical"; + if (pct >= 75) return "ss-disk-warn"; + return ""; + } + + function alertClass(level) { + if (level === "critical") return "gh-critical"; + if (level === "warning" || level === "warn") return "gh-warning"; + return "gh-info"; + } + + // The worst alert decides the overall state; words carry it, color helps. + function overall(stats) { + var alerts = stats.alerts || []; + if (alerts.some(function (a) { return a.level === "critical"; })) { + return { cls: "ss-down", label: "critical" }; + } + if (alerts.length) return { cls: "gh-dot-warn", label: "warning" }; + return { cls: "ss-up", label: "healthy" }; + } + + function metaText(stats, now) { + var stale = now - stats.generatedAt > STALE_MS; + var text = + "Report generated " + + new Date(stats.generatedAt).toLocaleString() + + " (" + + relativeTime(stats.generatedAt, now) + + ")"; + if (stale) text += " · stale — the report cron on gannet may have stopped"; + return { text: text, stale: stale }; + } + + function barCell(pct) { + var td = el("td", "ss-disk-pct " + (pct == null ? "" : fullnessClass(pct))); + if (pct == null) { + td.textContent = "—"; + return td; + } + var bar = el("span", "ss-bar"); + var fill = el("span", "ss-bar-fill"); + fill.style.width = pct + "%"; + bar.appendChild(fill); + td.appendChild(bar); + td.appendChild(el("span", "ss-bar-label", pct + "%")); + return td; + } + + function table(headers, rows) { + // No class: mkdocs-material only styles table:not([class]). + var t = el("table"); + var thead = el("thead"); + var tr = el("tr"); + headers.forEach(function (h) { tr.appendChild(el("th", null, h)); }); + thead.appendChild(tr); + t.appendChild(thead); + var tbody = el("tbody"); + rows.forEach(function (row) { tbody.appendChild(row); }); + t.appendChild(tbody); + return t; + } + + function row(cells) { + var tr = el("tr"); + cells.forEach(function (c) { + tr.appendChild(c instanceof Node ? c : el("td", null, c == null ? "—" : String(c))); + }); + return tr; + } + + function alertList(stats) { + var alerts = stats.alerts || []; + if (!alerts.length) return el("p", "gh-ok", "✓ No alerts in the latest report."); + var ul = el("ul", "gh-alerts"); + alerts.forEach(function (a) { + var li = el("li", alertClass(a.level)); + li.appendChild(el("strong", null, a.level.toUpperCase())); + li.appendChild(document.createTextNode(" " + a.message)); + ul.appendChild(li); + }); + return ul; + } + + function diskTable(stats, withInodes) { + var disks = (stats.disks || []).slice().sort(function (a, b) { + return b.use_percent - a.use_percent; + }); + var headers = ["Mount", "Size", "Used", "Available", "% Full"]; + if (withInodes) headers.push("Inodes"); + return table( + headers, + disks.map(function (d) { + var mount = el("td", "ss-mount", d.mount); + mount.title = d.filesystem + (d.type ? " (" + d.type + ")" : ""); + var cells = [mount, d.size, d.used, d.available, barCell(d.use_percent)]; + if (withInodes) cells.push(d.inode_percent == null ? "—" : d.inode_percent + "%"); + return row(cells); + }) + ); + } + + // ---- Computing Hardware page ------------------------------------------- + + function paintHealthCell(cell, stats, now) { + cell.innerHTML = ""; + if (!stats) { + cell.textContent = "no report yet"; + return; + } + var state = overall(stats); + var link = el("a"); + link.href = cell.getAttribute("data-dashboard") || "../Gannet-Dashboard/"; + link.appendChild(el("span", "ss-dot " + state.cls)); + var alerts = stats.alerts || []; + link.appendChild( + el( + "span", + "ss-label", + alerts.length + ? alerts.length + (alerts.length === 1 ? " alert" : " alerts") + ": " + alerts[0].message + : "no alerts" + ) + ); + cell.appendChild(link); + cell.title = metaText(stats, now).text; + } + + function paintSummary(root, stats, now) { + root.innerHTML = ""; + if (!stats) { + root.appendChild(el("p", null, "No gannet health report available yet.")); + return; + } + var meta = metaText(stats, now); + var bits = [meta.text]; + if (stats.uptime) bits.push("up " + stats.uptime); + if (stats.load) bits.push("load " + stats.load.join(" / ") + (stats.cpu_cores ? " on " + stats.cpu_cores + " cores" : "")); + root.appendChild(el("p", "ss-raven-meta" + (meta.stale ? " ss-stats-stale" : ""), bits.join(" · "))); + root.appendChild(alertList(stats)); + root.appendChild(diskTable(stats, false)); + } + + // ---- Dashboard --------------------------------------------------------- + + function tile(label, value, sub, cls) { + var t = el("div", "gh-tile " + (cls || "")); + t.appendChild(el("div", "gh-tile-label", label)); + t.appendChild(el("div", "gh-tile-value", value)); + if (sub) t.appendChild(el("div", "gh-tile-sub", sub)); + return t; + } + + function section(root, title) { + var s = el("section", "gh-section"); + s.appendChild(el("h2", null, title)); + root.appendChild(s); + return s; + } + + function svg(tag, attrs) { + var node = document.createElementNS(SVG_NS, tag); + Object.keys(attrs || {}).forEach(function (k) { node.setAttribute(k, attrs[k]); }); + return node; + } + + /* + * Small line chart. series: [{name, cls, points: [{x: index, y, label}]}]. + * yMax fixes the scale (100 for percentages); ref draws a dashed reference + * line (e.g. CPU core count for load). One y-axis only. + */ + function lineChart(dates, series, opts) { + var W = 640, H = 200, L = 40, R = 90, T = 12, B = 28; + var yMax = opts.yMax; + if (yMax == null) { + yMax = 1; + series.forEach(function (s) { + s.points.forEach(function (p) { if (p.y > yMax) yMax = p.y; }); + }); + if (opts.ref != null && opts.ref > yMax) yMax = opts.ref; + yMax = Math.ceil(yMax * 1.15); + } + var n = dates.length; + function x(i) { return n <= 1 ? L + (W - L - R) / 2 : L + (i * (W - L - R)) / (n - 1); } + function y(v) { return T + (H - T - B) * (1 - v / yMax); } + + var chart = svg("svg", { + viewBox: "0 0 " + W + " " + H, + class: "gh-chart", + role: "img", + "aria-label": opts.label + }); + + [0, 0.5, 1].forEach(function (f) { + var v = Math.round(yMax * f); + chart.appendChild(svg("line", { x1: L, x2: W - R, y1: y(v), y2: y(v), class: "gh-grid" })); + var t = svg("text", { x: L - 6, y: y(v) + 4, class: "gh-axis", "text-anchor": "end" }); + t.textContent = v + (opts.unit || ""); + chart.appendChild(t); + }); + + if (opts.ref != null) { + chart.appendChild(svg("line", { x1: L, x2: W - R, y1: y(opts.ref), y2: y(opts.ref), class: "gh-ref" })); + var rt = svg("text", { x: W - R + 6, y: y(opts.ref) + 4, class: "gh-axis" }); + rt.textContent = opts.refLabel; + chart.appendChild(rt); + } + + [0, n - 1].forEach(function (i, k) { + if (k === 1 && n <= 1) return; + var t = svg("text", { x: x(i), y: H - 8, class: "gh-axis", "text-anchor": k ? "end" : "start" }); + if (n <= 1) t.setAttribute("text-anchor", "middle"); + t.textContent = dates[i]; + chart.appendChild(t); + }); + + series.forEach(function (s) { + var pts = s.points; + if (pts.length > 1) { + chart.appendChild( + svg("polyline", { + points: pts.map(function (p) { return x(p.x) + "," + y(p.y); }).join(" "), + class: "gh-line " + s.cls + }) + ); + } + pts.forEach(function (p) { + var dot = svg("circle", { cx: x(p.x), cy: y(p.y), r: 4, class: "gh-point " + s.cls }); + var title = svg("title"); + title.textContent = s.name + " · " + dates[p.x] + ": " + p.label; + dot.appendChild(title); + chart.appendChild(dot); + // Bigger invisible hit target for the tooltip. + var hit = svg("circle", { cx: x(p.x), cy: y(p.y), r: 10, class: "gh-hit" }); + hit.appendChild(title.cloneNode(true)); + chart.appendChild(hit); + }); + // Direct label at the last point so identity is never color alone. + var last = pts[pts.length - 1]; + if (last) { + var lbl = svg("text", { x: x(last.x) + 8, y: y(last.y) + 4, class: "gh-direct" }); + lbl.textContent = s.name + " " + last.label; + chart.appendChild(lbl); + } + }); + return chart; + } + + function historyNote(history) { + return el( + "p", + "gh-note", + history.length < 2 + ? "History fills in as daily reports accumulate (" + history.length + " day so far). Hover a point for its value." + : "Last " + history.length + " daily reports. Hover a point for its value." + ); + } + + function paintDashboard(root, stats, now) { + root.innerHTML = ""; + if (!stats) { + root.appendChild(el("p", null, "No gannet health report available yet.")); + return; + } + + var meta = metaText(stats, now); + var head = el("p", "ss-raven-meta" + (meta.stale ? " ss-stats-stale" : "")); + head.appendChild(document.createTextNode(meta.text + " · ")); + var raw = el("a", null, "raw report"); + raw.href = REPORT_URL; + head.appendChild(raw); + root.appendChild(head); + + // KPI tiles + var tiles = el("div", "gh-tiles"); + var state = overall(stats); + var alerts = stats.alerts || []; + tiles.appendChild( + tile("Overall", state.label, alerts.length + (alerts.length === 1 ? " alert" : " alerts"), + state.label === "healthy" ? "gh-good" : state.label === "critical" ? "gh-critical" : "gh-warning") + ); + var disks = (stats.disks || []).filter(function (d) { return d.mount !== "/"; }); + disks.sort(function (a, b) { return b.use_percent - a.use_percent; }); + disks.forEach(function (d) { + var c = fullnessClass(d.use_percent); + tiles.appendChild( + tile(d.mount, d.use_percent + "% full", d.available + " free of " + d.size, + c === "ss-disk-critical" ? "gh-critical" : c === "ss-disk-warn" ? "gh-warning" : "") + ); + }); + if (stats.load) { + var cores = stats.cpu_cores; + tiles.appendChild( + tile("Load (15 min)", String(stats.load[2]), + cores ? "on " + cores + " cores" : "", + cores && stats.load[2] > cores ? "gh-warning" : "") + ); + } + if (stats.memory && stats.memory.mem) { + var m = stats.memory.mem; + tiles.appendChild(tile("Memory available", m.available || m.free, "of " + m.total)); + } + var raid = stats.raid || []; + if (raid.length) { + var ok = raid.filter(function (a) { return a.healthy; }).length; + tiles.appendChild( + tile("RAID arrays", ok + " / " + raid.length + " healthy", "", ok === raid.length ? "gh-good" : "gh-critical") + ); + } + if (stats.uptime) tiles.appendChild(tile("Uptime", stats.uptime, "")); + root.appendChild(tiles); + + section(root, "Alerts").appendChild(alertList(stats)); + + var storage = section(root, "Storage"); + storage.appendChild(diskTable(stats, true)); + + var history = stats.history || []; + var dates = history.map(function (h) { return h.date; }); + var volumes = ["/volume2", "/volume1"]; + var volCls = ["gh-s1", "gh-s2"]; + var volSeries = volumes.map(function (mount, k) { + var pts = []; + history.forEach(function (h, i) { + if (h.disks && h.disks[mount] != null) pts.push({ x: i, y: h.disks[mount], label: h.disks[mount] + "%" }); + }); + return { name: mount, cls: volCls[k], points: pts }; + }).filter(function (s) { return s.points.length; }); + if (volSeries.length) { + storage.appendChild(el("h3", null, "Volume use over time")); + var legend = el("div", "gh-legend"); + volSeries.forEach(function (s) { + var item = el("span", "gh-legend-item"); + item.appendChild(el("span", "gh-swatch " + s.cls)); + item.appendChild(document.createTextNode(s.name)); + legend.appendChild(item); + }); + storage.appendChild(legend); + storage.appendChild(lineChart(dates, volSeries, { yMax: 100, unit: "%", label: "Percent full per volume by day" })); + storage.appendChild(historyNote(history)); + } + + var sys = section(root, "System"); + var loadPts = []; + history.forEach(function (h, i) { + if (h.load) loadPts.push({ x: i, y: h.load[2], label: String(h.load[2]) }); + }); + if (loadPts.length) { + sys.appendChild(el("h3", null, "15-minute load average over time")); + sys.appendChild( + lineChart(dates, [{ name: "load", cls: "gh-s1", points: loadPts }], { + ref: stats.cpu_cores, + refLabel: stats.cpu_cores ? stats.cpu_cores + " cores" : "", + label: "15-minute load average by day" + }) + ); + sys.appendChild(historyNote(history)); + } + if (stats.memory) { + sys.appendChild(el("h3", null, "Memory")); + var mem = stats.memory; + var memRows = []; + if (mem.mem) memRows.push(row(["RAM", mem.mem.total, mem.mem.used, mem.mem.free, mem.mem.available])); + if (mem.swap) memRows.push(row(["Swap", mem.swap.total, mem.swap.used, mem.swap.free, null])); + sys.appendChild(table(["", "Total", "Used", "Free", "Available"], memRows)); + } + + if (raid.length) { + var rs = section(root, "RAID"); + rs.appendChild( + table( + ["Array", "Level", "State", "Members", "Health"], + raid.map(function (a) { + var h = el("td", a.healthy ? "gh-ok" : "gh-critical", a.healthy ? "✓ all members up" : "✗ degraded"); + return row([a.name, a.level, a.state, a.active + " / " + a.members, h]); + }) + ) + ); + } + + var svc = section(root, "Services"); + svc.appendChild( + table( + ["Service", "State"], + (stats.services || []).map(function (s) { + return row([s.name, el("td", s.state === "active" ? "gh-ok" : "gh-critical", s.state)]); + }) + ) + ); + var failed = stats.failed_units || []; + svc.appendChild(el("h3", null, "Failed systemd units")); + if (failed.length) { + svc.appendChild( + table(["Unit", "Description"], failed.map(function (f) { return row([f.unit, f.description]); })) + ); + } else { + svc.appendChild(el("p", "gh-ok", "✓ None.")); + } + + var logs = section(root, "Kernel errors (last 24 h)"); + var kernel = stats.kernel_errors || []; + if (kernel.length) { + logs.appendChild(el("pre", null, kernel.join("\n"))); + } else { + logs.appendChild(el("p", "gh-ok", "✓ None.")); + } + if (stats.smart && stats.smart.length) { + logs.appendChild(el("h3", null, "SMART drive health")); + logs.appendChild(el("p", null, stats.smart.join(" "))); + } + + if (stats.raw) { + var details = el("details", "gh-raw"); + details.appendChild(el("summary", null, "Full text of the latest report")); + details.appendChild(el("pre", null, stats.raw)); + root.appendChild(details); + } + } + + function render() { + var cells = document.querySelectorAll('[data-health="gannet"]'); + var summary = document.querySelector("[data-gannet-summary]"); + var dashboard = document.querySelector("[data-gannet-dashboard]"); + if (!cells.length && !summary && !dashboard) return; + + var holder = dashboard || summary; + var base = (holder && holder.getAttribute("data-status-base")) || DEFAULT_BASE; + + Promise.all(SOURCES.map(function (n) { return fetchSource(base, n); })).then(function (docs) { + var stats = mergeStats(docs); + var now = Date.now(); + Array.prototype.forEach.call(cells, function (c) { paintHealthCell(c, stats, now); }); + if (summary) paintSummary(summary, stats, now); + if (dashboard) paintDashboard(dashboard, stats, now); + }); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", render); + } else { + render(); + } + if (window.document$ && typeof window.document$.subscribe === "function") { + window.document$.subscribe(render); + } +})(); diff --git a/docs/stylesheets/server-status.css b/docs/stylesheets/server-status.css index 86ccae92..1a7df58f 100644 --- a/docs/stylesheets/server-status.css +++ b/docs/stylesheets/server-status.css @@ -113,3 +113,184 @@ height: 100%; background: var(--ss-bar-color, #5b8def); } + +/* Gannet daily health: status-table cell, summary section, dashboard. + * Status colors reuse the ss-* values above; series colors are the + * validated categorical slots 1 (blue) and 2 (orange), stepped per scheme. + */ + +.gh-dot-warn { + --ss-color: #d99100; +} + +[data-md-color-scheme="slate"] .gh-dot-warn { + --ss-color: #f0b429; +} + +.ss-health a { + color: inherit; +} + +.md-typeset ul.gh-alerts { + list-style: none; + padding-left: 0; + margin-left: 0; +} + +.md-typeset ul.gh-alerts li { + list-style: none; + margin-left: 0; + margin-bottom: 0.3em; + padding: 0.35em 0.7em; + border-left: 4px solid var(--gh-status, #9aa0a6); + background: rgba(128, 128, 128, 0.08); + border-radius: 0 0.2em 0.2em 0; +} + +.gh-critical { + --gh-status: #d64545; +} + +.gh-warning { + --gh-status: #d99100; +} + +.gh-good { + --gh-status: #2e9e4f; +} + +[data-md-color-scheme="slate"] .gh-critical { + --gh-status: #ff6b6b; +} + +[data-md-color-scheme="slate"] .gh-warning { + --gh-status: #f0b429; +} + +[data-md-color-scheme="slate"] .gh-good { + --gh-status: #4ad07d; +} + +td.gh-critical { + font-weight: 600; +} + +.gh-tiles { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(10rem, 1fr)); + gap: 0.6rem; + margin: 1em 0 1.5em; +} + +.gh-tile { + padding: 0.6em 0.8em; + border: 1px solid rgba(128, 128, 128, 0.25); + border-top: 4px solid var(--gh-status, rgba(128, 128, 128, 0.25)); + border-radius: 0.3em; +} + +.gh-tile-label { + font-size: 0.75em; + opacity: 0.75; + font-family: var(--md-code-font-family, monospace); +} + +.gh-tile-value { + font-size: 1.3em; + font-weight: 600; + font-variant-numeric: tabular-nums; +} + +.gh-tile-sub { + font-size: 0.75em; + opacity: 0.75; +} + +.gh-note { + font-size: 0.8em; + opacity: 0.75; +} + +.gh-chart { + width: 100%; + max-width: 44rem; + height: auto; + display: block; + --gh-s1: #2a78d6; + --gh-s2: #eb6834; +} + +[data-md-color-scheme="slate"] .gh-chart, +[data-md-color-scheme="slate"] .gh-legend { + --gh-s1: #3987e5; + --gh-s2: #d95926; +} + +.gh-legend { + --gh-s1: #2a78d6; + --gh-s2: #eb6834; + font-size: 0.8em; +} + +.gh-legend-item { + margin-right: 1em; +} + +.gh-swatch { + display: inline-block; + width: 0.8em; + height: 0.8em; + margin-right: 0.35em; + border-radius: 0.2em; + vertical-align: middle; +} + +.gh-swatch.gh-s1 { background: var(--gh-s1); } +.gh-swatch.gh-s2 { background: var(--gh-s2); } + +.gh-grid { + stroke: currentColor; + stroke-opacity: 0.15; +} + +.gh-ref { + stroke: currentColor; + stroke-opacity: 0.5; + stroke-dasharray: 4 4; +} + +.gh-axis, +.gh-direct { + fill: currentColor; + font-size: 11px; + font-variant-numeric: tabular-nums; +} + +.gh-axis { + fill-opacity: 0.7; +} + +.gh-line { + fill: none; + stroke-width: 2; + stroke-linejoin: round; +} + +.gh-line.gh-s1 { stroke: var(--gh-s1); } +.gh-line.gh-s2 { stroke: var(--gh-s2); } + +.gh-point { + stroke: var(--md-default-bg-color, #fff); + stroke-width: 2; +} + +.gh-point.gh-s1 { fill: var(--gh-s1); } +.gh-point.gh-s2 { fill: var(--gh-s2); } + +.gh-hit { + fill: transparent; +} + +.gh-raw pre { + font-size: 0.75em; +} diff --git a/mkdocs.yml b/mkdocs.yml index f822a9c3..350291c6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -32,6 +32,7 @@ extra_css: extra_javascript: - javascripts/server-status.js + - javascripts/gannet-health.js markdown_extensions: - admonition @@ -85,6 +86,7 @@ nav: - Agentic Coding Tools: 'Agentic-Coding-Tools.md' - Code Snippets: 'code_Snippets.md' - Computing Hardware: 'Computing-Hardware.md' + - Gannet Dashboard: 'Gannet-Dashboard.md' - Lab Software: 'Lab-Software.md' - 'Klone Guides': diff --git a/scripts/README.md b/scripts/README.md index 1470ff30..7c87aa28 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -75,6 +75,36 @@ the table, flagged stale if the snapshot is older than 30 hours. Drives at 90%+ full are marked critical and 75%+ warn, in the text as well as the bar color. +## Gannet's daily health report + +Gannet runs `gannet_health.sh` once a day and writes the result to +, plus a dated +copy, `gannet_health_YYYY-MM-DD.txt`, in the same directory. The report covers +uptime and load, `df` and inode use, memory, RAID (`/proc/mdstat`), services +and failed systemd units, kernel errors, and a summary of alerts. + +Gannet does not send CORS headers, so the page cannot fetch the report itself. +Instead `check_servers.py` fetches and parses it (`fetch_gannet_stats`) under +either profile and writes a top-level `gannet_stats` field. It is kept out of +`hosts` for the same reason as `raven_stats`. The field holds the parsed +sections, the raw text (`raw`), and a `history` list with one small entry per +dated report for the last 30 days (load, per-mount use %, alert count). That +list is built by reading the directory index and fetching each dated file. + +`docs/javascripts/gannet-health.js` merges `gannet_stats` on its own +`generated` timestamp and renders it in three places: + +- the **Daily health** cell for gannet in the status table + (`[data-health="gannet"]`), +- the **Gannet Health** section on Computing Hardware + (`[data-gannet-summary]`: alerts and the disk table), and +- the **Gannet Dashboard** page (`docs/Gannet-Dashboard.md`, + `[data-gannet-dashboard]`), with the full report and history charts. + +A report older than 30 hours is flagged stale. If the report format changes, +`parse_gannet_report` treats every section as optional, so a partial parse +still renders. + The branch is rewritten as a single root commit on every run. At one check every 10 minutes an append-only branch would add roughly 50,000 commits a year to a repo that everyone clones. @@ -139,8 +169,8 @@ and that the lights on the handbook page go green within a few minutes. ## Notes and limits - A green light means the port answered. It says nothing about Slurm health - or whether jobs are running. Raven's disk space and CPU load are covered - separately by the snapshot described above; gannet and klone have no + or whether jobs are running. Raven's disk space and CPU load, and gannet's daily health + report, are covered separately as described above; klone has no equivalent. - `raw.githubusercontent.com` caches for about 5 minutes, so the page can lag the actual check by that much on top of the check interval. diff --git a/scripts/check_servers.py b/scripts/check_servers.py index 8b354f6d..9f0f5080 100755 --- a/scripts/check_servers.py +++ b/scripts/check_servers.py @@ -42,6 +42,25 @@ # check below, which only the internal one can reach. RAVEN_STATS_URL = "https://gannet.fish.washington.edu/v1_web/owlshell/bu-github/ghr.log" +# Gannet writes a daily health report (owlshell/gannet_health.sh) to its own +# public web root: latest.txt plus a dated gannet_health_YYYY-MM-DD.txt copy. +# The dated copies are what the dashboard's history charts are built from. +GANNET_REPORT_DIR = "https://gannet.fish.washington.edu/v1_web/owlshell/" +GANNET_LATEST_URL = GANNET_REPORT_DIR + "latest.txt" +GANNET_HISTORY_DAYS = 30 + +_SECTION = re.compile(r"^==== (.+?) ====$") +_DATED_REPORT = re.compile(r'href="(gannet_health_(\d{4}-\d{2}-\d{2})\.txt)"') +# "/dev/md0 ext4 2.3G 1.7G 535M 76% /" -- df -hT, so there is a type column. +_DF_T_LINE = re.compile(r"^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)%\s+(\S+)$") +_INODE_LINE = re.compile(r"^(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(\S+)$") +_LOAD = re.compile(r"load average:\s*([\d.]+),\s*([\d.]+),\s*([\d.]+)") +_UPTIME = re.compile(r"\bup\s+(.+?),\s+\d+\s+users?,") +_RAID_HEAD = re.compile(r"^(md\d+)\s*:\s*(\S+)\s+(raid\w+)") +_RAID_STATE = re.compile(r"\[(\d+)/(\d+)\]\s+\[([U_]+)\]") +_ALERT = re.compile(r"^\*\s+(\S+)\s+(.*)$") +_TZ_OFFSETS = {"PDT": "-0700", "PST": "-0800", "UTC": "+0000", "GMT": "+0000"} + # Each disk line looks like "/dev/sdd1 7.3T 6.3T 602G 92% /home/shared/...": # filesystem, size, used, available, use%, mount point. _DISK_LINE = re.compile(r"^(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\d+)%\s+(\S+)$") @@ -181,6 +200,211 @@ def fetch_raven_stats(timeout): return {"generated": generated, "cpu_percent": cpu_percent, "disks": disks} +def _fetch_text(url, timeout): + request = urllib.request.Request(url) + request.add_header("User-Agent", "robertslab-handbook-status-check") + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read().decode("utf-8", "replace"), response.headers.get( + "Last-Modified" + ) + + +def _iso_utc(dt): + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _split_sections(body): + """{section title: [lines]} for the '==== Title ====' blocks, plus the + lines before the first one under ''.""" + sections = {"": []} + current = "" + for line in body.splitlines(): + match = _SECTION.match(line.strip()) + if match: + current = match.group(1) + sections[current] = [] + else: + sections[current].append(line.rstrip()) + return sections + + +def _nonblank(lines): + return [line.strip() for line in lines if line.strip()] + + +def parse_gannet_report(body, last_modified=None): + """Parse one gannet_health.sh report into a dict. Returns None if it does + not look like a report at all. + + Every section is optional: the script skips some (SMART needs root) and + may grow new ones, and a partial parse is more useful than none. + """ + sections = _split_sections(body) + if len(sections) < 2: + return None + + generated = None + for line in sections[""]: + if line.startswith("Generated:"): + stamp = line.split(":", 1)[1].strip() + parts = stamp.rsplit(" ", 1) + if len(parts) == 2 and parts[1] in _TZ_OFFSETS: + stamp = parts[0] + " " + _TZ_OFFSETS[parts[1]] + try: + generated = _iso_utc(datetime.strptime(stamp, "%Y-%m-%d %H:%M:%S %z")) + except ValueError: + generated = None + if generated is None and last_modified: + try: + generated = _iso_utc(parsedate_to_datetime(last_modified)) + except (TypeError, ValueError): + pass + + report = {"generated": generated} + + uptime_lines = sections.get("Uptime and load", []) + for line in uptime_lines: + load = _LOAD.search(line) + if load: + report["load"] = [float(x) for x in load.groups()] + up = _UPTIME.search(line) + if up: + report["uptime"] = up.group(1).strip() + if line.startswith("CPU cores:"): + try: + report["cpu_cores"] = int(line.split(":", 1)[1]) + except ValueError: + pass + + inodes = {} + for line in sections.get("Inode usage", []): + match = _INODE_LINE.match(line.strip()) + if match: + inodes[match.group(6)] = int(match.group(5)) + + # Skip loop devices and the Synology @appdata bind mounts: they re-list + # /volume1 under other paths and would just duplicate its row. + disks = [] + for line in sections.get("Disk usage", []): + match = _DF_T_LINE.match(line.strip()) + if not match: + continue + filesystem, fstype, size, used, avail, pct, mount = match.groups() + if filesystem.startswith("/dev/loop") or "/@" in mount: + continue + disks.append( + { + "filesystem": filesystem, + "type": fstype, + "size": size, + "used": used, + "available": avail, + "use_percent": int(pct), + "mount": mount, + "inode_percent": inodes.get(mount), + } + ) + report["disks"] = disks + + memory = {} + for line in sections.get("Memory", []): + fields = line.split() + if fields and fields[0] in ("Mem:", "Swap:") and len(fields) >= 4: + key = fields[0].rstrip(":").lower() + entry = {"total": fields[1], "used": fields[2], "free": fields[3]} + if key == "mem" and len(fields) >= 7: + entry["available"] = fields[6] + memory[key] = entry + report["memory"] = memory + + arrays = [] + for line in sections.get("RAID status", []): + head = _RAID_HEAD.match(line.strip()) + if head: + arrays.append( + {"name": head.group(1), "state": head.group(2), "level": head.group(3)} + ) + continue + state = _RAID_STATE.search(line) + if state and arrays and "members" not in arrays[-1]: + arrays[-1]["members"] = int(state.group(1)) + arrays[-1]["active"] = int(state.group(2)) + arrays[-1]["healthy"] = ( + "_" not in state.group(3) and state.group(1) == state.group(2) + ) + report["raid"] = arrays + + services, failed, in_failed = [], [], False + for line in _nonblank(sections.get("Services", [])): + if line.startswith("Failed units"): + in_failed = True + continue + fields = line.split() + if in_failed: + # "unit.service loaded failed failed Description words..." + failed.append( + {"unit": fields[0], "description": " ".join(fields[4:]) or None} + ) + elif len(fields) == 2: + services.append({"name": fields[0], "state": fields[1]}) + report["services"] = services + report["failed_units"] = failed + + kernel = _nonblank(sections.get("Recent kernel errors (last 24h)", [])) + report["kernel_errors"] = [] if kernel == ["None."] else kernel + report["smart"] = _nonblank(sections.get("SMART drive health", [])) + report["shares"] = _nonblank(sections.get("Share connections", [])) + report["changes"] = _nonblank(sections.get("Change since last report", [])) + + alerts = [] + for line in _nonblank(sections.get("Summary", [])): + match = _ALERT.match(line) + if match: + alerts.append({"level": match.group(1).lower(), "message": match.group(2)}) + report["alerts"] = alerts + return report + + +def _history_point(date, report): + return { + "date": date, + "load": report.get("load"), + "disks": {d["mount"]: d["use_percent"] for d in report.get("disks", [])}, + "alerts": len(report.get("alerts", [])), + } + + +def fetch_gannet_stats(timeout): + """Latest gannet health report, parsed, plus a short daily history built + from the dated copies. Returns None if latest.txt is unavailable, for the + same reason as fetch_raven_stats.""" + try: + body, last_modified = _fetch_text(GANNET_LATEST_URL, timeout) + except (urllib.error.URLError, socket.timeout, OSError): + return None + report = parse_gannet_report(body, last_modified) + if not report: + return None + report["raw"] = body + + history = [] + try: + index, _ = _fetch_text(GANNET_REPORT_DIR, timeout) + dated = sorted(set(_DATED_REPORT.findall(index)), key=lambda x: x[1]) + except (urllib.error.URLError, socket.timeout, OSError): + dated = [] + for name, date in dated[-GANNET_HISTORY_DAYS:]: + try: + text, _ = _fetch_text(GANNET_REPORT_DIR + name, timeout) + except (urllib.error.URLError, socket.timeout, OSError): + continue + parsed = parse_gannet_report(text) + if parsed: + history.append(_history_point(date, parsed)) + report["history"] = history + return report + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--profile", required=True, choices=["internal", "external"]) @@ -209,6 +433,12 @@ def main(): raven_stats = fetch_raven_stats(args.timeout) if raven_stats: document["raven_stats"] = raven_stats + + # Same reasoning as raven_stats: a daily report, merged on its own + # "generated" timestamp by the page, so it stays out of "hosts". + gannet_stats = fetch_gannet_stats(args.timeout) + if gannet_stats: + document["gannet_stats"] = gannet_stats text = json.dumps(document, indent=2, sort_keys=True) if args.out: