etwen/Blanton_TTL_Script
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 18 Wiki Activity

feat(bmc): Add bmc_monitor.sh, log build/uptime/reboot-cause in A and C

Browse Source 
bmc_monitor.sh (V1.1.0): polls the BMC from the SONiC host via
`bmc-manager run`, appending to a rotating log. Runs detached, so no
interactive SSH session to the BMC is needed and the BMC's busybox
toolchain never comes into play. Subcommands start/stop/status/fg/tail/
clear; the polled command list, interval, per-command timeout and log
rotation are configured at the top of the file. START_LOG_MODE defaults
to "new", so one run means one log. Default COMMANDS are `free -m` and
`memtester 500M 1`, i.e. the BMC is memory-stressed alongside the host
soak.

Script A: `show boot` after the time sync to record the image the DUT
booted, and `show uptime` at the end of the traffic baseline.

Script C: `show reboot-cause` and `show uptime` after the traffic
report, so an unplanned reset during the soak is visible in the
closing snapshot rather than only in dmesg.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014RetWKFZFG1ZHcitQAyhwM
This commit is contained in:
etwen
2026-08-20 10:47:35 +08:00
co-authored by Claude Opus 5
parent fedf697977
commit 310f897067
3 changed files with 354 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Script_ABC_Blanton/1_Blanton_Script_A.ttl
+7
View File
@@ -49,6 +49,9 @@ gettime timestr
sprintf2 cmd 'sudo date -s "%s %s"' datestr timestr
sendln cmd
; ========== Show Build Name ==========
sendln "show boot"
; ========== Load Shell Script ==========
wait prompt_sonic_root
sendln "source ~/Blanton_Script/blanton_fpga_pcimem.sh"
@@ -112,6 +115,10 @@ sendln "blanton_traffic_linespeed stop"
wait prompt_sonic_root
sendln "blanton_traffic_linespeed report"
; ========== Show Power on uptime ==========
wait prompt_sonic_root
sendln "show uptime"
wait prompt_sonic_root
messagebox 'Start Script B' 'Script A DONE'
src/Script_ABC_Blanton/3_Blanton_Script_C.ttl
+6
View File
@@ -37,5 +37,11 @@ sendln "blanton_traffic_linespeed stop"
wait prompt_sonic_root
sendln "blanton_traffic_linespeed report"
wait prompt_sonic_root
sendln "show reboot-cause"
; ========== Show Power on uptime ==========
wait prompt_sonic_root
sendln "show uptime"
messagebox 'GOOD JOB! Test Case DONE' 'teraterm'
src/Script_ABC_Blanton/Blanton_Script/bmc_monitor.sh
+341
View File
@@ -0,0 +1,341 @@
#!/bin/bash
###############################################################################
# bmc_monitor.sh
#
# Version : V1.1.0
# Author : ETWen
# Date : 20260820
#
# Purpose : Periodically execute a list of commands on the BMC through
# `bmc-manager run "<cmd>"` from the SONiC host side, and append all
# output to a rotating log file. Designed to run detached in the
# background so no interactive SSH session to the BMC is required.
#
# Notes : - Everything runs on the HOST. Nothing is deployed to the BMC, so
# the BMC busybox toolchain (no base64 / no setsid ...) is a
# non-issue.
# - `bmc-manager run` prints the remote command output on stdout and
# its own banner on stderr; the two streams are captured separately
# so the log stays clean.
#
# Version History
# V1.0.0 20260820 Initial Version
# V1.1.0 20260820 `start` clears the previous log by default
# (START_LOG_MODE: new / archive / append)
###############################################################################
set -u
###############################################################################
# User Configurable Section -- normally this is the only part you need to edit
###############################################################################
# Commands executed on the BMC, one per line, in this order.
# Add a line to add a command; comment it out to disable it.
COMMANDS=(
"free -m"
"memtester 500M 1"
#"uptime"
#"cat /proc/loadavg"
#"cat /proc/meminfo"
#"df -h /tmp"
#"dmesg | tail -n 30"
)
# Seconds to wait after a full round (all COMMANDS executed) before the next one
INTERVAL_SEC=5
# Seconds to wait between two commands inside the same round (0 = no wait)
INTER_CMD_DELAY_SEC=0
# Per-command timeout in seconds; prevents a hung bmc-manager from freezing the
# whole loop. Set to 0 to disable.
CMD_TIMEOUT_SEC=30
# Stop after this many rounds. 0 = run until stopped manually.
MAX_ROUNDS=0
# Command runner. Final invocation is:
# "$RUNNER_BIN" "${RUNNER_ARGS[@]}" "<command string>"
# BMC side : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(run)
# BMC by IP : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(--ip 192.168.200.200 run)
# Host side : RUNNER_BIN="bash" ; RUNNER_ARGS=(-c)
RUNNER_BIN="bmc-manager"
RUNNER_ARGS=(run)
# stderr handling of the runner (bmc-manager banner lives here)
# 0 = log stderr only when the command fails (recommended, keeps log clean)
# 1 = always log stderr
LOG_STDERR=0
# Logging
LOG_DIR="" # empty = <script directory>/log
LOG_NAME="bmc_poll.log"
MAX_LOG_SIZE_MB=100 # rotate above this size; 0 = never rotate
MAX_LOG_KEEP=5 # keep .1 .. .N
# What `start` does with the log left behind by the previous run:
# new = begin with an empty log; previous content and rotated copies are
# discarded (default -- one run, one log)
# archive = rename the previous log to <log>.YYYYmmdd-HHMMSS, then begin empty
# append = keep appending to the existing log
START_LOG_MODE="new"
###############################################################################
# Internal
###############################################################################
SCRIPT_NAME="$(basename -- "${BASH_SOURCE[0]}")"
SCRIPT_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/${SCRIPT_NAME}"
[ -n "${LOG_DIR}" ] || LOG_DIR="$(dirname -- "${SCRIPT_PATH}")/log"
LOG_FILE="${LOG_DIR}/${LOG_NAME}"
PID_FILE="${LOG_DIR}/${LOG_NAME%.log}.pid"
LOG_TO_FILE=0
STOP=0
SLEEP_PID=""
ts() { date '+%Y-%m-%d %H:%M:%S'; }
die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
on_signal() {
STOP=1
[ -n "${SLEEP_PID}" ] && kill "${SLEEP_PID}" 2>/dev/null
}
# sleep that can be aborted immediately by SIGTERM/SIGINT
interruptible_sleep() {
local sec="$1"
[ "${sec}" -gt 0 ] 2>/dev/null || return 0
sleep "${sec}" &
SLEEP_PID=$!
wait "${SLEEP_PID}" 2>/dev/null
SLEEP_PID=""
}
# echo the pid and return 0 when a live instance is found
is_running() {
local pid
[ -f "${PID_FILE}" ] || return 1
pid="$(cat "${PID_FILE}" 2>/dev/null)"
[[ "${pid}" =~ ^[0-9]+$ ]] || return 1
kill -0 "${pid}" 2>/dev/null || return 1
grep -qa "${SCRIPT_NAME}" "/proc/${pid}/cmdline" 2>/dev/null || return 1
printf '%s' "${pid}"
return 0
}
rotate_log_if_needed() {
local max_bytes size i
[ "${LOG_TO_FILE}" -eq 1 ] || return 0
[ "${MAX_LOG_SIZE_MB}" -gt 0 ] || return 0
[ -f "${LOG_FILE}" ] || return 0
max_bytes=$(( MAX_LOG_SIZE_MB * 1024 * 1024 ))
size="$(stat -c %s "${LOG_FILE}" 2>/dev/null || echo 0)"
[ "${size}" -ge "${max_bytes}" ] || return 0
for (( i = MAX_LOG_KEEP - 1; i >= 1; i-- )); do
[ -f "${LOG_FILE}.${i}" ] && mv -f "${LOG_FILE}.${i}" "${LOG_FILE}.$(( i + 1 ))"
done
mv -f "${LOG_FILE}" "${LOG_FILE}.1"
exec >>"${LOG_FILE}" 2>&1 # re-bind stdout/stderr to the new file
printf '[%s] INFO : log rotated at %s MB\n' "$(ts)" "${MAX_LOG_SIZE_MB}"
}
# Remove the rotated copies (.1 .. .N) only. Timestamped archives are kept.
remove_rotated_logs() {
local i
for (( i = 1; i <= MAX_LOG_KEEP + 1; i++ )); do
rm -f "${LOG_FILE}.${i}"
done
}
# Apply START_LOG_MODE just before a background run is launched.
prepare_log_on_start() {
local stamp
case "${START_LOG_MODE}" in
append)
return 0
;;
archive)
if [ -s "${LOG_FILE}" ]; then
stamp="$(date '+%Y%m%d-%H%M%S')"
mv -f "${LOG_FILE}" "${LOG_FILE}.${stamp}" \
|| die "cannot archive ${LOG_FILE}"
printf 'previous log archived: %s.%s\n' "${LOG_FILE}" "${stamp}"
fi
remove_rotated_logs
;;
new)
# Truncate rather than unlink: the inode is preserved, so a `tail -f`
# that is already attached keeps following the new run.
if [ -f "${LOG_FILE}" ]; then
: > "${LOG_FILE}" || die "cannot truncate ${LOG_FILE}"
fi
remove_rotated_logs
;;
*)
die "invalid START_LOG_MODE: ${START_LOG_MODE}"
;;
esac
}
run_one_command() {
local cmd="$1"
local out err rc t0 elapsed err_file
err_file="$(mktemp)"
t0=${SECONDS}
if [ "${CMD_TIMEOUT_SEC}" -gt 0 ]; then
out="$(timeout "${CMD_TIMEOUT_SEC}" \
"${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
else
out="$("${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
fi
rc=$?
elapsed=$(( SECONDS - t0 ))
err="$(cat "${err_file}" 2>/dev/null)"
rm -f "${err_file}"
printf '===== [%s] CMD: %s | rc=%d | %ds =====\n' "$(ts)" "${cmd}" "${rc}" "${elapsed}"
printf '%s\n' "${out}"
if [ "${rc}" -eq 124 ]; then
printf '[%s] WARN : timeout after %ds\n' "$(ts)" "${CMD_TIMEOUT_SEC}"
fi
if [ -n "${err}" ] && { [ "${LOG_STDERR}" -eq 1 ] || [ "${rc}" -ne 0 ]; }; then
printf -- '--- stderr ---\n%s\n' "${err}"
fi
printf '\n'
}
main_loop() {
local round=0 cmd
[ "${#COMMANDS[@]}" -gt 0 ] || die "COMMANDS is empty"
command -v "${RUNNER_BIN}" >/dev/null 2>&1 || die "runner not found: ${RUNNER_BIN}"
trap on_signal INT TERM
printf '#############################################################\n'
printf '[%s] START: pid=%d interval=%ds timeout=%ds cmds=%d\n' \
"$(ts)" "$$" "${INTERVAL_SEC}" "${CMD_TIMEOUT_SEC}" "${#COMMANDS[@]}"
printf '#############################################################\n'
while [ "${STOP}" -eq 0 ]; do
round=$(( round + 1 ))
printf -- '---------- ROUND %d @ %s ----------\n' "${round}" "$(ts)"
for cmd in "${COMMANDS[@]}"; do
[ "${STOP}" -eq 0 ] || break
run_one_command "${cmd}"
[ "${INTER_CMD_DELAY_SEC}" -gt 0 ] && interruptible_sleep "${INTER_CMD_DELAY_SEC}"
done
rotate_log_if_needed
if [ "${MAX_ROUNDS}" -gt 0 ] && [ "${round}" -ge "${MAX_ROUNDS}" ]; then
break
fi
# Fixed gap between rounds. For a fixed cadence instead, replace with:
# interruptible_sleep $(( INTERVAL_SEC - round_elapsed ))
[ "${STOP}" -eq 0 ] && interruptible_sleep "${INTERVAL_SEC}"
done
printf '[%s] STOP : pid=%d after %d round(s)\n' "$(ts)" "$$" "${round}"
}
###############################################################################
# Sub-commands
###############################################################################
do_start() {
local pid
pid="$(is_running)" && die "already running (pid=${pid})"
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
prepare_log_on_start
if command -v setsid >/dev/null 2>&1; then
setsid "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
else
nohup "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
disown 2>/dev/null
fi
sleep 1
pid="$(is_running)" || die "start failed, check ${LOG_FILE}"
printf 'started (pid=%s)\nlog: %s\n' "${pid}" "${LOG_FILE}"
}
do_daemon() {
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
LOG_TO_FILE=1
exec >>"${LOG_FILE}" 2>&1
printf '%d\n' "$$" > "${PID_FILE}"
trap 'rm -f "${PID_FILE}"' EXIT
main_loop
}
do_stop() {
local pid i
pid="$(is_running)" || { printf 'not running\n'; rm -f "${PID_FILE}"; return 0; }
kill -TERM -"${pid}" 2>/dev/null || kill -TERM "${pid}" 2>/dev/null
for (( i = 0; i < 20; i++ )); do
kill -0 "${pid}" 2>/dev/null || break
sleep 0.5
done
if kill -0 "${pid}" 2>/dev/null; then
printf 'SIGTERM ignored, sending SIGKILL\n'
kill -KILL -"${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null
fi
rm -f "${PID_FILE}"
printf 'stopped (pid=%s)\n' "${pid}"
}
do_status() {
local pid
if pid="$(is_running)"; then
printf 'status : running (pid=%s)\n' "${pid}"
else
printf 'status : stopped\n'
fi
printf 'log : %s\n' "${LOG_FILE}"
[ -f "${LOG_FILE}" ] && printf 'size : %s\n' "$(du -h "${LOG_FILE}" | cut -f1)"
printf 'cmds : %d, interval=%ds\n' "${#COMMANDS[@]}" "${INTERVAL_SEC}"
}
usage() {
cat <<EOF
Usage: ${SCRIPT_NAME} {start|stop|status|fg|tail|clear}
start Run detached in the background; the previous log is discarded first
(see START_LOG_MODE)
stop Stop the background instance
status Show pid / log path / log size
fg Run in the foreground, output to the terminal (for a quick check)
tail tail -f the log file
clear Remove the log file and all rotated copies (must be stopped first)
Log file : ${LOG_FILE}
To add a command, edit the COMMANDS array at the top of this script.
EOF
}
###############################################################################
# Entry point
###############################################################################
case "${1:-}" in
start) do_start ;;
stop) do_stop ;;
status) do_status ;;
fg) LOG_TO_FILE=0; main_loop ;;
tail) tail -n 50 -f "${LOG_FILE}" ;;
clear)
is_running >/dev/null && die "still running, stop it first"
rm -f "${LOG_FILE}" "${LOG_FILE}".[0-9]*
printf 'log cleared\n'
;;
__daemon) do_daemon ;;
-h|--help|"") usage ;;
*) printf '[ERROR] unknown sub-command: %s\n\n' "$1" >&2; usage; exit 1 ;;
esac