pxe_boot_alma/tools/restart_probe_watcher.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=/usr/local/valherun/pxe_boot_alma
WATCH_ROOT=/tmp/probe-stage-watch
STATE_FILE="$WATCH_ROOT/state.env"
TRIAGE_FILE="$WATCH_ROOT/needs-triage.env"
RESTART_LOG="$WATCH_ROOT/restart.log"
NOHUP_LOG="$WATCH_ROOT/auto-restart.nohup"
LOCK_FILE="$WATCH_ROOT/probe-stage-watch.lock"
ACTIVE_STATE_MAX_AGE="${ACTIVE_STATE_MAX_AGE:-180}"
TRIAGE_RETRY_AFTER_SEC="${TRIAGE_RETRY_AFTER_SEC:-600}"
LIBVIRT_URI="${LIBVIRT_URI:-qemu:///system}"

timestamp() {
  date '+%F %T'
}

state_age_seconds() {
  if [[ ! -f "$STATE_FILE" ]]; then
    return 1
  fi

  local state_epoch
  state_epoch="$(stat -c %Y "$STATE_FILE" 2>/dev/null || echo 0)"
  [[ "$state_epoch" =~ ^[0-9]+$ ]] || return 1
  [[ "$state_epoch" -gt 0 ]] || return 1
  echo "$(( $(date +%s) - state_epoch ))"
}

state_phase_is_active() {
  case "${1:-unknown}" in
    attempt-start|attempt-prep|install|disk-boot|boot|baseline-capture|snapshot-stage|promote)
      return 0
      ;;
    *)
      return 1
      ;;
  esac
}

triage_age_seconds() {
  if [[ ! -f "$TRIAGE_FILE" ]]; then
    return 1
  fi

  local triage_epoch
  triage_epoch="$(stat -c %Y "$TRIAGE_FILE" 2>/dev/null || echo 0)"
  [[ "$triage_epoch" =~ ^[0-9]+$ ]] || return 1
  [[ "$triage_epoch" -gt 0 ]] || return 1
  echo "$(( $(date +%s) - triage_epoch ))"
}

watcher_process_running() {
  ps -ef | grep -q '[p]robe_stage_watch.sh'
}

payload_source_lines() {
  local attempt_dir="$1"
  local manifest="$attempt_dir/payload-manifest.txt"
  local stage_apply="$attempt_dir/stage-apply.sh"

  if [[ -f "$manifest" ]] && grep -q '^# Source: ' "$manifest"; then
    grep '^# Source: ' "$manifest"
    return 0
  fi

  if [[ -f "$stage_apply" ]] && grep -q '^# Source: ' "$stage_apply"; then
    grep '^# Source: ' "$stage_apply"
    return 0
  fi

  return 1
}

triage_payload_is_stale() {
  local attempt_dir="$1"
  local line source_path payload_sha current_sha

  [[ -d "$attempt_dir" ]] || return 1

  while IFS= read -r line; do
    [[ "$line" =~ ^#\ Source:\ (.+)\ sha256=([0-9a-f]{64})$ ]] || continue
    source_path="${BASH_REMATCH[1]}"
    payload_sha="${BASH_REMATCH[2]}"
    if [[ -f "$source_path" ]]; then
      current_sha="$(sha256sum "$source_path" | awk '{print $1}')"
      if [[ "$current_sha" != "$payload_sha" ]]; then
        return 0
      fi
    fi
  done < <(payload_source_lines "$attempt_dir" || true)

  return 1
}

mkdir -p "$WATCH_ROOT"
exec 9>"$LOCK_FILE"
if ! flock -n 9; then
  echo "[$(timestamp)] watcher already running (lock held)" >> "$RESTART_LOG"
  exit 0
fi
flock -u 9

if watcher_process_running; then
  echo "[$(timestamp)] watcher already running (ps)" >> "$RESTART_LOG"
  exit 0
fi

stage="baseline"
phase="unknown"
mode="unknown"
suspect="baseline"
if [[ -f "$STATE_FILE" ]]; then
  # shellcheck disable=SC1090
  source "$STATE_FILE"
  stage="${CURRENT_STAGE:-baseline}"
  phase="${CURRENT_PHASE:-unknown}"
  mode="${CURRENT_MODE:-unknown}"
  suspect="${CURRENT_SUSPECT:-baseline}"
fi

if [[ "$phase" == "full-pass" ]]; then
  echo "[$(timestamp)] restart skipped phase=$phase" >> "$RESTART_LOG"
  exit 0
fi

if [[ -f "$TRIAGE_FILE" ]]; then
  # shellcheck disable=SC1090
  source "$TRIAGE_FILE"
  triage_age="$(triage_age_seconds || true)"
  if triage_payload_is_stale "${TRIAGE_ATTEMPT_DIR:-}"; then
    echo "[$(timestamp)] clearing stale triage for attempt_dir=${TRIAGE_ATTEMPT_DIR:-unknown}" >> "$RESTART_LOG"
    rm -f "$TRIAGE_FILE"
  elif [[ "$phase" == "snapshot-stale" \
      && "${TRIAGE_STAGE:-}" == "$stage" \
      && "${TRIAGE_MODE:-}" == "$mode" \
      && "${TRIAGE_SUSPECT:-}" == "$suspect" ]]; then
    echo "[$(timestamp)] clearing obsolete triage superseded by state phase=$phase stage=$stage suspect=$suspect" >> "$RESTART_LOG"
    rm -f "$TRIAGE_FILE"
  else
    case "${TRIAGE_OUTCOME:-}" in
      watcher\ missing/stale\ state|watcher\ running/state\ stale)
        rm -f "$TRIAGE_FILE"
        ;;
      snapshot\ stage\ apply\ failed|snapshot\ boot\ failed/no\ ssh)
        # No active watcher (verified above). If the attempt has no source-hash
        # lines, staleness cannot be determined — clear triage so a post-fix
        # repo state can resume. When source lines DO exist, triage_payload_is_stale
        # already cleared the file above on any drift. If the triage has sat for
        # too long with no watcher, force a fresh retry rather than idling forever.
        if ! payload_source_lines "${TRIAGE_ATTEMPT_DIR:-}" &>/dev/null; then
          echo "[$(timestamp)] clearing unverifiable snapshot triage outcome=${TRIAGE_OUTCOME:-unknown} attempt_dir=${TRIAGE_ATTEMPT_DIR:-unknown}" >> "$RESTART_LOG"
          rm -f "$TRIAGE_FILE"
        elif [[ -n "${triage_age:-}" && "$triage_age" -ge "$TRIAGE_RETRY_AFTER_SEC" ]]; then
          echo "[$(timestamp)] clearing aged snapshot triage outcome=${TRIAGE_OUTCOME:-unknown} after ${triage_age}s attempt_dir=${TRIAGE_ATTEMPT_DIR:-unknown}" >> "$RESTART_LOG"
          rm -f "$TRIAGE_FILE"
        else
          echo "[$(timestamp)] restart skipped triage_outcome=${TRIAGE_OUTCOME:-unknown}" >> "$RESTART_LOG"
          exit 0
        fi
        ;;
      *)
        echo "[$(timestamp)] restart skipped triage_outcome=${TRIAGE_OUTCOME:-unknown}" >> "$RESTART_LOG"
        exit 0
        ;;
    esac
  fi
fi

nohup env LIBVIRT_DEFAULT_URI="$LIBVIRT_URI" LIBVIRT_URI="$LIBVIRT_URI" \
  "$ROOT/tools/probe_stage_watch.sh" "$stage" >> "$NOHUP_LOG" 2>&1 &
echo "[$(timestamp)] watcher restarted stage=$stage phase=$phase pid=$!" >> "$RESTART_LOG"