pxe_boot_alma/tools/codex_harness_supervisor.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=/usr/local/valherun/pxe_boot_alma
WATCH_ROOT=/tmp/probe-stage-watch
SUP_ROOT="$WATCH_ROOT/supervisor"
LOCK_FILE="$SUP_ROOT/lock"
RUN_LOG="$SUP_ROOT/runs.log"
PROMPT_FILE="$SUP_ROOT/prompt.txt"
LAST_MSG_FILE="$SUP_ROOT/last-message.txt"
STDOUT_FILE="$SUP_ROOT/codex-stdout.txt"
LATEST_FILE="$WATCH_ROOT/latest-summary.txt"
STATE_FILE="$WATCH_ROOT/state.env"
FAIL_LOG="$WATCH_ROOT/failures.md"
MEMORY_FILE="$ROOT/codex-running-memory.md"
TRIAGE_FILE="$WATCH_ROOT/needs-triage.env"

mkdir -p "$SUP_ROOT"

timestamp() {
  date '+%F %T'
}

latest_attempt_dir() {
  find "$WATCH_ROOT" -maxdepth 1 -type d -name '20*-attempt-*' | sort | tail -n 1
}

payload_source_lines() {
  local attempt_dir="$1"
  local manifest="$attempt_dir/payload-manifest.txt"
  local stage_apply="$attempt_dir/stage-apply.sh"

  if [[ -f "$manifest" ]]; then
    grep '^# Source: ' "$manifest" && return 0
  fi

  if [[ -f "$stage_apply" ]]; then
    grep '^# Source: ' "$stage_apply" && return 0
  fi

  return 1
}

print_payload_drift() {
  local attempt_dir="$1"
  local manifest="$attempt_dir/payload-manifest.txt"
  local stage_apply="$attempt_dir/stage-apply.sh"
  local found=0
  local line_count=0
  local source_mode="missing"

  if [[ -f "$manifest" ]] && grep -q '^# Source: ' "$manifest"; then
    source_mode="manifest"
  elif [[ -f "$stage_apply" ]] && grep -q '^# Source: ' "$stage_apply"; then
    source_mode="stage-apply"
  else
    echo "payload-source-drift: unavailable (no source hashes in payload-manifest.txt or stage-apply.sh)"
    return 0
  fi

  while IFS= read -r line; do
    [[ "$line" =~ ^#\ Source:\ (.+)\ sha256=([0-9a-f]{64})$ ]] || continue
    line_count=$((line_count + 1))
    local source_path="${BASH_REMATCH[1]}"
    local payload_sha="${BASH_REMATCH[2]}"
    local current_sha=""

    if [[ -f "$source_path" ]]; then
      current_sha="$(sha256sum "$source_path" | awk '{print $1}')"
      if [[ "$current_sha" != "$payload_sha" ]]; then
        if [[ "$found" == "0" ]]; then
          echo "payload-source-drift:"
          found=1
        fi
        echo "STALE payload script: $source_path payload_sha=$payload_sha current_sha=$current_sha"
      fi
    fi
  done < <(payload_source_lines "$attempt_dir" || true)

  if [[ "$line_count" == "0" ]]; then
    echo "payload-source-drift: unavailable (source hash parsing failed)"
  elif [[ "$found" == "0" && "$source_mode" == "manifest" ]]; then
    echo "payload-source-drift: none"
  elif [[ "$found" == "0" ]]; then
    echo "payload-source-drift: none (recovered from stage-apply.sh; payload-manifest.txt lacked source hashes)"
  fi
}

print_triage_status() {
  if [[ -f "$TRIAGE_FILE" ]]; then
    cat "$TRIAGE_FILE"
    return
  fi

  if [[ -f "$STATE_FILE" ]]; then
    # shellcheck disable=SC1090
    source "$STATE_FILE"
  fi

  echo "no active triage request"
  echo "watcher_phase='${CURRENT_PHASE:-unknown}'"
  echo "watcher_detail='${CURRENT_DETAIL:-}'"
}

{
  flock -n 9 || exit 0

  attempt_dir="$(latest_attempt_dir || true)"
  {
    echo "# Codex Harness Supervisor"
    echo
    echo "Time: $(timestamp)"
    echo "Repo: $ROOT"
    echo
    echo "Your job:"
    echo "- Inspect the current harness and latest attempt artifacts."
    echo "- If the harness itself is wrong or stale, patch the harness."
    echo "- If the current stage exposed a kickstart issue, patch the repo or harness as needed."
    echo "- Update $MEMORY_FILE with a short new finding if you learn something important."
    echo "- Do not remove scripts permanently from the repo. Temporary staged exclusion is allowed only via harness/stage config."
    echo "- Keep changes small and targeted."
    echo
    echo "Current watcher summary:"
    if [[ -f "$LATEST_FILE" ]]; then
      cat "$LATEST_FILE"
    else
      echo "latest-summary.txt missing"
    fi
    echo
    echo "Current watcher state:"
    if [[ -f "$STATE_FILE" ]]; then
      cat "$STATE_FILE"
    else
      echo "state.env missing"
    fi
    echo
    echo "Triage request:"
    print_triage_status
    echo
    echo "Failure ledger tail:"
    if [[ -f "$FAIL_LOG" ]]; then
      tail -n 40 "$FAIL_LOG"
    else
      echo "failures.md missing"
    fi
    echo
    echo "Running memory tail:"
    if [[ -f "$MEMORY_FILE" ]]; then
      tail -n 80 "$MEMORY_FILE"
    else
      echo "codex-running-memory.md missing"
    fi
    echo
    echo "Latest attempt dir: ${attempt_dir:-missing}"
    if [[ -n "${attempt_dir:-}" ]]; then
      echo
      echo "Latest attempt log:"
      sed -n '1,240p' "$attempt_dir/attempt.log" 2>/dev/null || true
      echo
      echo "Latest attempt files:"
      find "$attempt_dir" -maxdepth 1 -type f | sort
      echo
      if [[ -f "$attempt_dir/boot.log" ]]; then
        echo "boot.log tail:"
        tail -n 200 "$attempt_dir/boot.log"
        echo
      fi
      if [[ -f "$attempt_dir/ks-post.log" ]]; then
        echo "ks-post.log tail:"
        tail -n 200 "$attempt_dir/ks-post.log"
        echo
      fi
      if [[ -f "$attempt_dir/stage-apply.log" ]]; then
        echo "stage-apply.log tail:"
        tail -n 200 "$attempt_dir/stage-apply.log"
        echo
      fi
      if [[ -f "$attempt_dir/stage-apply.stdout" ]]; then
        echo "stage-apply.stdout tail:"
        tail -n 200 "$attempt_dir/stage-apply.stdout"
        echo
      fi
      if [[ -f "$attempt_dir/payload-manifest.txt" ]]; then
        echo "payload-manifest:"
        cat "$attempt_dir/payload-manifest.txt"
        echo
        print_payload_drift "$attempt_dir"
        echo
      fi
    fi
    echo
    echo "Please make any needed repo changes directly, then end with a concise summary."
  } > "$PROMPT_FILE"

  echo "[$(timestamp)] supervisor start" >> "$RUN_LOG"
  if codex exec \
      --dangerously-bypass-approvals-and-sandbox \
      -C "$ROOT" \
      -o "$LAST_MSG_FILE" \
      - < "$PROMPT_FILE" > "$STDOUT_FILE" 2>&1; then
    status="ok"
  else
    status="failed"
  fi

  {
    echo "[$(timestamp)] supervisor status=$status"
    echo "--- last message ---"
    cat "$LAST_MSG_FILE" 2>/dev/null || true
    echo
  } >> "$RUN_LOG"

} 9>"$LOCK_FILE"