mirror of
https://git.valherun.com/valherun/pxe_boot_alma.git
synced 2026-10-14 18:22:03 -04:00
andersenandClaude Fable 5
6d398b826a
- restructure boot/kickstarts/scripts/docs into homelab/ - docs: alma9->10 breaking changes, CIS hardening, kickstart changes - add find_cron_entries.sh helper and codex running memory notes Co-Authored-By: Claude Fable 5 <[email protected]>
139 lines
3.8 KiB
Bash
Executable File
139 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT=/usr/local/valherun/pxe_boot_alma
|
|
WATCH_ROOT=/tmp/probe-stage-watch
|
|
LOG_FILE="$WATCH_ROOT/cron-monitor.log"
|
|
STATE_FILE="$WATCH_ROOT/state.env"
|
|
LATEST_FILE="$WATCH_ROOT/latest-summary.txt"
|
|
TRIAGE_FILE="$WATCH_ROOT/needs-triage.env"
|
|
FAIL_LOG="$WATCH_ROOT/failures.md"
|
|
RESTART_HELPER="$ROOT/tools/restart_probe_watcher.sh"
|
|
TRIAGE_STALE_AFTER_SEC="${TRIAGE_STALE_AFTER_SEC:-600}"
|
|
|
|
mkdir -p "$WATCH_ROOT"
|
|
|
|
timestamp() {
|
|
date '+%F %T'
|
|
}
|
|
|
|
triage_age_seconds() {
|
|
if [[ ! -f "$TRIAGE_FILE" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
local triage_epoch
|
|
triage_epoch="$(stat -c %Y "$TRIAGE_FILE" 2>/dev/null || echo 0)"
|
|
[[ "$triage_epoch" -gt 0 ]] || return 1
|
|
echo "$(( $(date +%s) - triage_epoch ))"
|
|
}
|
|
|
|
clear_obsolete_watcher_triage() {
|
|
[[ -f "$TRIAGE_FILE" ]] || return 0
|
|
|
|
# shellcheck disable=SC1090
|
|
source "$TRIAGE_FILE"
|
|
case "${TRIAGE_OUTCOME:-}" in
|
|
watcher\ missing/stale\ state|watcher\ running/state\ stale)
|
|
rm -f "$TRIAGE_FILE"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
request_monitor_triage() {
|
|
local outcome="$1"
|
|
|
|
if [[ -f "$TRIAGE_FILE" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local latest_failure_stage="unknown"
|
|
local latest_failure_suspect="unknown"
|
|
local latest_failure_attempt="unknown"
|
|
if [[ -f "$FAIL_LOG" ]]; then
|
|
latest_failure_stage="$(tail -n 1 "$FAIL_LOG" | awk -F'|' '{gsub(/^ +| +$/, "", $4); print $4}')"
|
|
latest_failure_attempt="$(tail -n 1 "$FAIL_LOG" | awk -F'|' '{gsub(/^ +| +$/, "", $3); print $3}')"
|
|
latest_failure_suspect="$(tail -n 1 "$FAIL_LOG" | awk -F'|' '{gsub(/^ +| +$/, "", $5); print $5}')"
|
|
fi
|
|
|
|
if [[ -f "$STATE_FILE" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "$STATE_FILE"
|
|
fi
|
|
|
|
cat > "$TRIAGE_FILE" <<EOF
|
|
TRIAGE_REQUESTED_AT='$(timestamp)'
|
|
TRIAGE_STAGE='${CURRENT_STAGE:-$latest_failure_stage}'
|
|
TRIAGE_MODE='${CURRENT_MODE:-unknown}'
|
|
TRIAGE_ATTEMPT='${latest_failure_attempt}'
|
|
TRIAGE_SUSPECT='${CURRENT_SUSPECT:-$latest_failure_suspect}'
|
|
TRIAGE_OUTCOME='$outcome'
|
|
TRIAGE_ATTEMPT_DIR=''
|
|
EOF
|
|
}
|
|
|
|
watcher_ps="$(ps -ef | grep '[p]robe_stage_watch.sh' || true)"
|
|
if [[ -n "$watcher_ps" ]]; then
|
|
watcher_status="running"
|
|
else
|
|
watcher_status="missing"
|
|
if [[ -x "$RESTART_HELPER" ]]; then
|
|
"$RESTART_HELPER" >> "$LOG_FILE" 2>&1 || true
|
|
watcher_ps="$(ps -ef | grep '[p]robe_stage_watch.sh' || true)"
|
|
if [[ -n "$watcher_ps" ]]; then
|
|
watcher_status="running"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
state_age="unknown"
|
|
triage_age="unknown"
|
|
if [[ -f "$STATE_FILE" ]]; then
|
|
now_epoch="$(date +%s)"
|
|
state_epoch="$(stat -c %Y "$STATE_FILE" 2>/dev/null || echo 0)"
|
|
if [[ "$state_epoch" -gt 0 ]]; then
|
|
state_age="$(( now_epoch - state_epoch ))s"
|
|
fi
|
|
fi
|
|
if [[ -f "$TRIAGE_FILE" ]]; then
|
|
triage_age="$(triage_age_seconds || true)"
|
|
[[ -n "${triage_age:-}" ]] || triage_age="unknown"
|
|
fi
|
|
|
|
{
|
|
echo "[$(timestamp)] watcher=$watcher_status state_age=$state_age"
|
|
if [[ -f "$LATEST_FILE" ]]; then
|
|
sed 's/^/ /' "$LATEST_FILE"
|
|
else
|
|
echo " latest-summary=missing"
|
|
fi
|
|
if [[ "$watcher_status" == "missing" ]]; then
|
|
echo " action-needed=watcher-not-running"
|
|
elif [[ "$state_age" != "unknown" ]] && [[ "${state_age%s}" -gt 180 ]]; then
|
|
echo " action-needed=state-stale"
|
|
fi
|
|
if [[ -f "$TRIAGE_FILE" ]]; then
|
|
echo " action-needed=llm-triage"
|
|
sed 's/^/ triage: /' "$TRIAGE_FILE"
|
|
echo " triage_age=$triage_age"
|
|
fi
|
|
} >> "$LOG_FILE"
|
|
|
|
if [[ "$watcher_status" == "missing" ]]; then
|
|
request_monitor_triage "watcher missing/stale state"
|
|
elif [[ "$state_age" != "unknown" ]] && [[ "${state_age%s}" -gt 180 ]]; then
|
|
request_monitor_triage "watcher running/state stale"
|
|
else
|
|
clear_obsolete_watcher_triage
|
|
fi
|
|
|
|
if [[ -f "$TRIAGE_FILE" && -x "$ROOT/tools/llm_failure_router.sh" ]]; then
|
|
"$ROOT/tools/llm_failure_router.sh" >> "$LOG_FILE" 2>&1 || true
|
|
fi
|
|
|
|
if ! ps -ef | grep -q '[p]robe_stage_watch.sh'; then
|
|
if [[ -x "$RESTART_HELPER" ]]; then
|
|
"$RESTART_HELPER" >> "$LOG_FILE" 2>&1 || true
|
|
fi
|
|
fi
|