From 310f897067719da92c0460e0cee990418a9d2857 Mon Sep 17 00:00:00 2001
From: ETWen <eric441151893@gmail.com>
Date: Thu, 20 Aug 2026 10:47:35 +0800
Subject: [PATCH] feat(bmc): Add bmc_monitor.sh, log build/uptime/reboot-cause
 in A and C

bmc_monitor.sh (V1.1.0): polls the BMC from the SONiC host via
`bmc-manager run`, appending to a rotating log. Runs detached, so no
interactive SSH session to the BMC is needed and the BMC's busybox
toolchain never comes into play. Subcommands start/stop/status/fg/tail/
clear; the polled command list, interval, per-command timeout and log
rotation are configured at the top of the file. START_LOG_MODE defaults
to "new", so one run means one log. Default COMMANDS are `free -m` and
`memtester 500M 1`, i.e. the BMC is memory-stressed alongside the host
soak.

Script A: `show boot` after the time sync to record the image the DUT
booted, and `show uptime` at the end of the traffic baseline.

Script C: `show reboot-cause` and `show uptime` after the traffic
report, so an unplanned reset during the soak is visible in the
closing snapshot rather than only in dmesg.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014RetWKFZFG1ZHcitQAyhwM
---
 src/Script_ABC_Blanton/1_Blanton_Script_A.ttl |   7 +
 src/Script_ABC_Blanton/3_Blanton_Script_C.ttl |   6 +
 .../Blanton_Script/bmc_monitor.sh             | 341 ++++++++++++++++++
 3 files changed, 354 insertions(+)
 create mode 100644 src/Script_ABC_Blanton/Blanton_Script/bmc_monitor.sh

diff --git a/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl b/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl
index 009262f..eabac5b 100644
--- a/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl
+++ b/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl
@@ -49,6 +49,9 @@ gettime timestr
 sprintf2 cmd 'sudo date -s "%s %s"' datestr timestr
 sendln cmd
 
+; ========== Show Build Name ==========
+sendln "show boot"
+
 ; ========== Load Shell Script ==========
 wait prompt_sonic_root
 sendln "source ~/Blanton_Script/blanton_fpga_pcimem.sh"
@@ -112,6 +115,10 @@ sendln "blanton_traffic_linespeed stop"
 wait prompt_sonic_root
 sendln "blanton_traffic_linespeed report"
 
+; ========== Show Power on uptime ==========
+wait prompt_sonic_root
+sendln "show uptime"
+
 
 wait prompt_sonic_root
 messagebox 'Start Script B' 'Script A DONE'
\ No newline at end of file
diff --git a/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl b/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl
index 6ce7df8..016a234 100644
--- a/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl
+++ b/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl
@@ -37,5 +37,11 @@ sendln "blanton_traffic_linespeed stop"
 wait prompt_sonic_root
 sendln "blanton_traffic_linespeed report"
 
+wait prompt_sonic_root
+sendln "show reboot-cause"
+
+; ========== Show Power on uptime ==========
+wait prompt_sonic_root
+sendln "show uptime"
 
 messagebox 'GOOD JOB! Test Case DONE' 'teraterm'
\ No newline at end of file
diff --git a/src/Script_ABC_Blanton/Blanton_Script/bmc_monitor.sh b/src/Script_ABC_Blanton/Blanton_Script/bmc_monitor.sh
new file mode 100644
index 0000000..40af6fd
--- /dev/null
+++ b/src/Script_ABC_Blanton/Blanton_Script/bmc_monitor.sh
@@ -0,0 +1,341 @@
+#!/bin/bash
+###############################################################################
+# bmc_monitor.sh
+#
+# Version : V1.1.0
+# Author  : ETWen
+# Date    : 20260820
+#
+# Purpose : Periodically execute a list of commands on the BMC through
+#           `bmc-manager run "<cmd>"` from the SONiC host side, and append all
+#           output to a rotating log file. Designed to run detached in the
+#           background so no interactive SSH session to the BMC is required.
+#
+# Notes   : - Everything runs on the HOST. Nothing is deployed to the BMC, so
+#             the BMC busybox toolchain (no base64 / no setsid ...) is a
+#             non-issue.
+#           - `bmc-manager run` prints the remote command output on stdout and
+#             its own banner on stderr; the two streams are captured separately
+#             so the log stays clean.
+#
+# Version History
+# V1.0.0     20260820    Initial Version
+# V1.1.0     20260820    `start` clears the previous log by default
+#                        (START_LOG_MODE: new / archive / append)
+###############################################################################
+
+set -u
+
+###############################################################################
+# User Configurable Section  -- normally this is the only part you need to edit
+###############################################################################
+
+# Commands executed on the BMC, one per line, in this order.
+# Add a line to add a command; comment it out to disable it.
+COMMANDS=(
+    "free -m"
+    "memtester 500M 1"
+    #"uptime"
+    #"cat /proc/loadavg"
+    #"cat /proc/meminfo"
+    #"df -h /tmp"
+    #"dmesg | tail -n 30"
+)
+
+# Seconds to wait after a full round (all COMMANDS executed) before the next one
+INTERVAL_SEC=5
+
+# Seconds to wait between two commands inside the same round (0 = no wait)
+INTER_CMD_DELAY_SEC=0
+
+# Per-command timeout in seconds; prevents a hung bmc-manager from freezing the
+# whole loop. Set to 0 to disable.
+CMD_TIMEOUT_SEC=30
+
+# Stop after this many rounds. 0 = run until stopped manually.
+MAX_ROUNDS=0
+
+# Command runner. Final invocation is:
+#     "$RUNNER_BIN" "${RUNNER_ARGS[@]}" "<command string>"
+#   BMC side  : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(run)
+#   BMC by IP : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(--ip 192.168.200.200 run)
+#   Host side : RUNNER_BIN="bash"        ; RUNNER_ARGS=(-c)
+RUNNER_BIN="bmc-manager"
+RUNNER_ARGS=(run)
+
+# stderr handling of the runner (bmc-manager banner lives here)
+#   0 = log stderr only when the command fails (recommended, keeps log clean)
+#   1 = always log stderr
+LOG_STDERR=0
+
+# Logging
+LOG_DIR=""                      # empty = <script directory>/log
+LOG_NAME="bmc_poll.log"
+MAX_LOG_SIZE_MB=100             # rotate above this size; 0 = never rotate
+MAX_LOG_KEEP=5                  # keep .1 .. .N
+
+# What `start` does with the log left behind by the previous run:
+#   new     = begin with an empty log; previous content and rotated copies are
+#             discarded (default -- one run, one log)
+#   archive = rename the previous log to <log>.YYYYmmdd-HHMMSS, then begin empty
+#   append  = keep appending to the existing log
+START_LOG_MODE="new"
+
+###############################################################################
+# Internal
+###############################################################################
+SCRIPT_NAME="$(basename -- "${BASH_SOURCE[0]}")"
+SCRIPT_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/${SCRIPT_NAME}"
+[ -n "${LOG_DIR}" ] || LOG_DIR="$(dirname -- "${SCRIPT_PATH}")/log"
+LOG_FILE="${LOG_DIR}/${LOG_NAME}"
+PID_FILE="${LOG_DIR}/${LOG_NAME%.log}.pid"
+
+LOG_TO_FILE=0
+STOP=0
+SLEEP_PID=""
+
+ts()  { date '+%Y-%m-%d %H:%M:%S'; }
+die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
+
+on_signal() {
+    STOP=1
+    [ -n "${SLEEP_PID}" ] && kill "${SLEEP_PID}" 2>/dev/null
+}
+
+# sleep that can be aborted immediately by SIGTERM/SIGINT
+interruptible_sleep() {
+    local sec="$1"
+    [ "${sec}" -gt 0 ] 2>/dev/null || return 0
+    sleep "${sec}" &
+    SLEEP_PID=$!
+    wait "${SLEEP_PID}" 2>/dev/null
+    SLEEP_PID=""
+}
+
+# echo the pid and return 0 when a live instance is found
+is_running() {
+    local pid
+    [ -f "${PID_FILE}" ] || return 1
+    pid="$(cat "${PID_FILE}" 2>/dev/null)"
+    [[ "${pid}" =~ ^[0-9]+$ ]] || return 1
+    kill -0 "${pid}" 2>/dev/null || return 1
+    grep -qa "${SCRIPT_NAME}" "/proc/${pid}/cmdline" 2>/dev/null || return 1
+    printf '%s' "${pid}"
+    return 0
+}
+
+rotate_log_if_needed() {
+    local max_bytes size i
+    [ "${LOG_TO_FILE}" -eq 1 ] || return 0
+    [ "${MAX_LOG_SIZE_MB}" -gt 0 ] || return 0
+    [ -f "${LOG_FILE}" ] || return 0
+
+    max_bytes=$(( MAX_LOG_SIZE_MB * 1024 * 1024 ))
+    size="$(stat -c %s "${LOG_FILE}" 2>/dev/null || echo 0)"
+    [ "${size}" -ge "${max_bytes}" ] || return 0
+
+    for (( i = MAX_LOG_KEEP - 1; i >= 1; i-- )); do
+        [ -f "${LOG_FILE}.${i}" ] && mv -f "${LOG_FILE}.${i}" "${LOG_FILE}.$(( i + 1 ))"
+    done
+    mv -f "${LOG_FILE}" "${LOG_FILE}.1"
+    exec >>"${LOG_FILE}" 2>&1          # re-bind stdout/stderr to the new file
+    printf '[%s] INFO : log rotated at %s MB\n' "$(ts)" "${MAX_LOG_SIZE_MB}"
+}
+
+# Remove the rotated copies (.1 .. .N) only. Timestamped archives are kept.
+remove_rotated_logs() {
+    local i
+    for (( i = 1; i <= MAX_LOG_KEEP + 1; i++ )); do
+        rm -f "${LOG_FILE}.${i}"
+    done
+}
+
+# Apply START_LOG_MODE just before a background run is launched.
+prepare_log_on_start() {
+    local stamp
+    case "${START_LOG_MODE}" in
+        append)
+            return 0
+            ;;
+        archive)
+            if [ -s "${LOG_FILE}" ]; then
+                stamp="$(date '+%Y%m%d-%H%M%S')"
+                mv -f "${LOG_FILE}" "${LOG_FILE}.${stamp}" \
+                    || die "cannot archive ${LOG_FILE}"
+                printf 'previous log archived: %s.%s\n' "${LOG_FILE}" "${stamp}"
+            fi
+            remove_rotated_logs
+            ;;
+        new)
+            # Truncate rather than unlink: the inode is preserved, so a `tail -f`
+            # that is already attached keeps following the new run.
+            if [ -f "${LOG_FILE}" ]; then
+                : > "${LOG_FILE}" || die "cannot truncate ${LOG_FILE}"
+            fi
+            remove_rotated_logs
+            ;;
+        *)
+            die "invalid START_LOG_MODE: ${START_LOG_MODE}"
+            ;;
+    esac
+}
+
+run_one_command() {
+    local cmd="$1"
+    local out err rc t0 elapsed err_file
+
+    err_file="$(mktemp)"
+    t0=${SECONDS}
+    if [ "${CMD_TIMEOUT_SEC}" -gt 0 ]; then
+        out="$(timeout "${CMD_TIMEOUT_SEC}" \
+               "${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
+    else
+        out="$("${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
+    fi
+    rc=$?
+    elapsed=$(( SECONDS - t0 ))
+    err="$(cat "${err_file}" 2>/dev/null)"
+    rm -f "${err_file}"
+
+    printf '===== [%s] CMD: %s | rc=%d | %ds =====\n' "$(ts)" "${cmd}" "${rc}" "${elapsed}"
+    printf '%s\n' "${out}"
+    if [ "${rc}" -eq 124 ]; then
+        printf '[%s] WARN : timeout after %ds\n' "$(ts)" "${CMD_TIMEOUT_SEC}"
+    fi
+    if [ -n "${err}" ] && { [ "${LOG_STDERR}" -eq 1 ] || [ "${rc}" -ne 0 ]; }; then
+        printf -- '--- stderr ---\n%s\n' "${err}"
+    fi
+    printf '\n'
+}
+
+main_loop() {
+    local round=0 cmd
+
+    [ "${#COMMANDS[@]}" -gt 0 ] || die "COMMANDS is empty"
+    command -v "${RUNNER_BIN}" >/dev/null 2>&1 || die "runner not found: ${RUNNER_BIN}"
+
+    trap on_signal INT TERM
+
+    printf '#############################################################\n'
+    printf '[%s] START: pid=%d interval=%ds timeout=%ds cmds=%d\n' \
+           "$(ts)" "$$" "${INTERVAL_SEC}" "${CMD_TIMEOUT_SEC}" "${#COMMANDS[@]}"
+    printf '#############################################################\n'
+
+    while [ "${STOP}" -eq 0 ]; do
+        round=$(( round + 1 ))
+        printf -- '---------- ROUND %d @ %s ----------\n' "${round}" "$(ts)"
+
+        for cmd in "${COMMANDS[@]}"; do
+            [ "${STOP}" -eq 0 ] || break
+            run_one_command "${cmd}"
+            [ "${INTER_CMD_DELAY_SEC}" -gt 0 ] && interruptible_sleep "${INTER_CMD_DELAY_SEC}"
+        done
+
+        rotate_log_if_needed
+
+        if [ "${MAX_ROUNDS}" -gt 0 ] && [ "${round}" -ge "${MAX_ROUNDS}" ]; then
+            break
+        fi
+        # Fixed gap between rounds. For a fixed cadence instead, replace with:
+        #   interruptible_sleep $(( INTERVAL_SEC - round_elapsed ))
+        [ "${STOP}" -eq 0 ] && interruptible_sleep "${INTERVAL_SEC}"
+    done
+
+    printf '[%s] STOP : pid=%d after %d round(s)\n' "$(ts)" "$$" "${round}"
+}
+
+###############################################################################
+# Sub-commands
+###############################################################################
+do_start() {
+    local pid
+    pid="$(is_running)" && die "already running (pid=${pid})"
+    mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
+    prepare_log_on_start
+
+    if command -v setsid >/dev/null 2>&1; then
+        setsid "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
+    else
+        nohup "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
+        disown 2>/dev/null
+    fi
+
+    sleep 1
+    pid="$(is_running)" || die "start failed, check ${LOG_FILE}"
+    printf 'started (pid=%s)\nlog: %s\n' "${pid}" "${LOG_FILE}"
+}
+
+do_daemon() {
+    mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
+    LOG_TO_FILE=1
+    exec >>"${LOG_FILE}" 2>&1
+    printf '%d\n' "$$" > "${PID_FILE}"
+    trap 'rm -f "${PID_FILE}"' EXIT
+    main_loop
+}
+
+do_stop() {
+    local pid i
+    pid="$(is_running)" || { printf 'not running\n'; rm -f "${PID_FILE}"; return 0; }
+
+    kill -TERM -"${pid}" 2>/dev/null || kill -TERM "${pid}" 2>/dev/null
+    for (( i = 0; i < 20; i++ )); do
+        kill -0 "${pid}" 2>/dev/null || break
+        sleep 0.5
+    done
+    if kill -0 "${pid}" 2>/dev/null; then
+        printf 'SIGTERM ignored, sending SIGKILL\n'
+        kill -KILL -"${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null
+    fi
+    rm -f "${PID_FILE}"
+    printf 'stopped (pid=%s)\n' "${pid}"
+}
+
+do_status() {
+    local pid
+    if pid="$(is_running)"; then
+        printf 'status : running (pid=%s)\n' "${pid}"
+    else
+        printf 'status : stopped\n'
+    fi
+    printf 'log    : %s\n' "${LOG_FILE}"
+    [ -f "${LOG_FILE}" ] && printf 'size   : %s\n' "$(du -h "${LOG_FILE}" | cut -f1)"
+    printf 'cmds   : %d, interval=%ds\n' "${#COMMANDS[@]}" "${INTERVAL_SEC}"
+}
+
+usage() {
+    cat <<EOF
+Usage: ${SCRIPT_NAME} {start|stop|status|fg|tail|clear}
+
+  start    Run detached in the background; the previous log is discarded first
+           (see START_LOG_MODE)
+  stop     Stop the background instance
+  status   Show pid / log path / log size
+  fg       Run in the foreground, output to the terminal (for a quick check)
+  tail     tail -f the log file
+  clear    Remove the log file and all rotated copies (must be stopped first)
+
+Log file : ${LOG_FILE}
+
+To add a command, edit the COMMANDS array at the top of this script.
+EOF
+}
+
+###############################################################################
+# Entry point
+###############################################################################
+case "${1:-}" in
+    start)    do_start ;;
+    stop)     do_stop ;;
+    status)   do_status ;;
+    fg)       LOG_TO_FILE=0; main_loop ;;
+    tail)     tail -n 50 -f "${LOG_FILE}" ;;
+    clear)
+        is_running >/dev/null && die "still running, stop it first"
+        rm -f "${LOG_FILE}" "${LOG_FILE}".[0-9]*
+        printf 'log cleared\n'
+        ;;
+    __daemon) do_daemon ;;
+    -h|--help|"") usage ;;
+    *)        printf '[ERROR] unknown sub-command: %s\n\n' "$1" >&2; usage; exit 1 ;;
+esac