feat(bmc): Add bmc_monitor.sh, log build/uptime/reboot-cause in A and C
bmc_monitor.sh (V1.1.0): polls the BMC from the SONiC host via `bmc-manager run`, appending to a rotating log. Runs detached, so no interactive SSH session to the BMC is needed and the BMC's busybox toolchain never comes into play. Subcommands start/stop/status/fg/tail/ clear; the polled command list, interval, per-command timeout and log rotation are configured at the top of the file. START_LOG_MODE defaults to "new", so one run means one log. Default COMMANDS are `free -m` and `memtester 500M 1`, i.e. the BMC is memory-stressed alongside the host soak. Script A: `show boot` after the time sync to record the image the DUT booted, and `show uptime` at the end of the traffic baseline. Script C: `show reboot-cause` and `show uptime` after the traffic report, so an unplanned reset during the soak is visible in the closing snapshot rather than only in dmesg. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014RetWKFZFG1ZHcitQAyhwM
This commit is contained in:
@@ -49,6 +49,9 @@ gettime timestr
|
|||||||
sprintf2 cmd 'sudo date -s "%s %s"' datestr timestr
|
sprintf2 cmd 'sudo date -s "%s %s"' datestr timestr
|
||||||
sendln cmd
|
sendln cmd
|
||||||
|
|
||||||
|
; ========== Show Build Name ==========
|
||||||
|
sendln "show boot"
|
||||||
|
|
||||||
; ========== Load Shell Script ==========
|
; ========== Load Shell Script ==========
|
||||||
wait prompt_sonic_root
|
wait prompt_sonic_root
|
||||||
sendln "source ~/Blanton_Script/blanton_fpga_pcimem.sh"
|
sendln "source ~/Blanton_Script/blanton_fpga_pcimem.sh"
|
||||||
@@ -112,6 +115,10 @@ sendln "blanton_traffic_linespeed stop"
|
|||||||
wait prompt_sonic_root
|
wait prompt_sonic_root
|
||||||
sendln "blanton_traffic_linespeed report"
|
sendln "blanton_traffic_linespeed report"
|
||||||
|
|
||||||
|
; ========== Show Power on uptime ==========
|
||||||
|
wait prompt_sonic_root
|
||||||
|
sendln "show uptime"
|
||||||
|
|
||||||
|
|
||||||
wait prompt_sonic_root
|
wait prompt_sonic_root
|
||||||
messagebox 'Start Script B' 'Script A DONE'
|
messagebox 'Start Script B' 'Script A DONE'
|
||||||
@@ -37,5 +37,11 @@ sendln "blanton_traffic_linespeed stop"
|
|||||||
wait prompt_sonic_root
|
wait prompt_sonic_root
|
||||||
sendln "blanton_traffic_linespeed report"
|
sendln "blanton_traffic_linespeed report"
|
||||||
|
|
||||||
|
wait prompt_sonic_root
|
||||||
|
sendln "show reboot-cause"
|
||||||
|
|
||||||
|
; ========== Show Power on uptime ==========
|
||||||
|
wait prompt_sonic_root
|
||||||
|
sendln "show uptime"
|
||||||
|
|
||||||
messagebox 'GOOD JOB! Test Case DONE' 'teraterm'
|
messagebox 'GOOD JOB! Test Case DONE' 'teraterm'
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
###############################################################################
|
||||||
|
# bmc_monitor.sh
|
||||||
|
#
|
||||||
|
# Version : V1.1.0
|
||||||
|
# Author : ETWen
|
||||||
|
# Date : 20260820
|
||||||
|
#
|
||||||
|
# Purpose : Periodically execute a list of commands on the BMC through
|
||||||
|
# `bmc-manager run "<cmd>"` from the SONiC host side, and append all
|
||||||
|
# output to a rotating log file. Designed to run detached in the
|
||||||
|
# background so no interactive SSH session to the BMC is required.
|
||||||
|
#
|
||||||
|
# Notes : - Everything runs on the HOST. Nothing is deployed to the BMC, so
|
||||||
|
# the BMC busybox toolchain (no base64 / no setsid ...) is a
|
||||||
|
# non-issue.
|
||||||
|
# - `bmc-manager run` prints the remote command output on stdout and
|
||||||
|
# its own banner on stderr; the two streams are captured separately
|
||||||
|
# so the log stays clean.
|
||||||
|
#
|
||||||
|
# Version History
|
||||||
|
# V1.0.0 20260820 Initial Version
|
||||||
|
# V1.1.0 20260820 `start` clears the previous log by default
|
||||||
|
# (START_LOG_MODE: new / archive / append)
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# User Configurable Section -- normally this is the only part you need to edit
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
# Commands executed on the BMC, one per line, in this order.
|
||||||
|
# Add a line to add a command; comment it out to disable it.
|
||||||
|
COMMANDS=(
|
||||||
|
"free -m"
|
||||||
|
"memtester 500M 1"
|
||||||
|
#"uptime"
|
||||||
|
#"cat /proc/loadavg"
|
||||||
|
#"cat /proc/meminfo"
|
||||||
|
#"df -h /tmp"
|
||||||
|
#"dmesg | tail -n 30"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Seconds to wait after a full round (all COMMANDS executed) before the next one
|
||||||
|
INTERVAL_SEC=5
|
||||||
|
|
||||||
|
# Seconds to wait between two commands inside the same round (0 = no wait)
|
||||||
|
INTER_CMD_DELAY_SEC=0
|
||||||
|
|
||||||
|
# Per-command timeout in seconds; prevents a hung bmc-manager from freezing the
|
||||||
|
# whole loop. Set to 0 to disable.
|
||||||
|
CMD_TIMEOUT_SEC=30
|
||||||
|
|
||||||
|
# Stop after this many rounds. 0 = run until stopped manually.
|
||||||
|
MAX_ROUNDS=0
|
||||||
|
|
||||||
|
# Command runner. Final invocation is:
|
||||||
|
# "$RUNNER_BIN" "${RUNNER_ARGS[@]}" "<command string>"
|
||||||
|
# BMC side : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(run)
|
||||||
|
# BMC by IP : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(--ip 192.168.200.200 run)
|
||||||
|
# Host side : RUNNER_BIN="bash" ; RUNNER_ARGS=(-c)
|
||||||
|
RUNNER_BIN="bmc-manager"
|
||||||
|
RUNNER_ARGS=(run)
|
||||||
|
|
||||||
|
# stderr handling of the runner (bmc-manager banner lives here)
|
||||||
|
# 0 = log stderr only when the command fails (recommended, keeps log clean)
|
||||||
|
# 1 = always log stderr
|
||||||
|
LOG_STDERR=0
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
LOG_DIR="" # empty = <script directory>/log
|
||||||
|
LOG_NAME="bmc_poll.log"
|
||||||
|
MAX_LOG_SIZE_MB=100 # rotate above this size; 0 = never rotate
|
||||||
|
MAX_LOG_KEEP=5 # keep .1 .. .N
|
||||||
|
|
||||||
|
# What `start` does with the log left behind by the previous run:
|
||||||
|
# new = begin with an empty log; previous content and rotated copies are
|
||||||
|
# discarded (default -- one run, one log)
|
||||||
|
# archive = rename the previous log to <log>.YYYYmmdd-HHMMSS, then begin empty
|
||||||
|
# append = keep appending to the existing log
|
||||||
|
START_LOG_MODE="new"
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Internal
|
||||||
|
###############################################################################
|
||||||
|
SCRIPT_NAME="$(basename -- "${BASH_SOURCE[0]}")"
|
||||||
|
SCRIPT_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/${SCRIPT_NAME}"
|
||||||
|
[ -n "${LOG_DIR}" ] || LOG_DIR="$(dirname -- "${SCRIPT_PATH}")/log"
|
||||||
|
LOG_FILE="${LOG_DIR}/${LOG_NAME}"
|
||||||
|
PID_FILE="${LOG_DIR}/${LOG_NAME%.log}.pid"
|
||||||
|
|
||||||
|
LOG_TO_FILE=0
|
||||||
|
STOP=0
|
||||||
|
SLEEP_PID=""
|
||||||
|
|
||||||
|
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||||
|
die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
|
||||||
|
|
||||||
|
on_signal() {
|
||||||
|
STOP=1
|
||||||
|
[ -n "${SLEEP_PID}" ] && kill "${SLEEP_PID}" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# sleep that can be aborted immediately by SIGTERM/SIGINT
|
||||||
|
interruptible_sleep() {
|
||||||
|
local sec="$1"
|
||||||
|
[ "${sec}" -gt 0 ] 2>/dev/null || return 0
|
||||||
|
sleep "${sec}" &
|
||||||
|
SLEEP_PID=$!
|
||||||
|
wait "${SLEEP_PID}" 2>/dev/null
|
||||||
|
SLEEP_PID=""
|
||||||
|
}
|
||||||
|
|
||||||
|
# echo the pid and return 0 when a live instance is found
|
||||||
|
is_running() {
|
||||||
|
local pid
|
||||||
|
[ -f "${PID_FILE}" ] || return 1
|
||||||
|
pid="$(cat "${PID_FILE}" 2>/dev/null)"
|
||||||
|
[[ "${pid}" =~ ^[0-9]+$ ]] || return 1
|
||||||
|
kill -0 "${pid}" 2>/dev/null || return 1
|
||||||
|
grep -qa "${SCRIPT_NAME}" "/proc/${pid}/cmdline" 2>/dev/null || return 1
|
||||||
|
printf '%s' "${pid}"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
rotate_log_if_needed() {
|
||||||
|
local max_bytes size i
|
||||||
|
[ "${LOG_TO_FILE}" -eq 1 ] || return 0
|
||||||
|
[ "${MAX_LOG_SIZE_MB}" -gt 0 ] || return 0
|
||||||
|
[ -f "${LOG_FILE}" ] || return 0
|
||||||
|
|
||||||
|
max_bytes=$(( MAX_LOG_SIZE_MB * 1024 * 1024 ))
|
||||||
|
size="$(stat -c %s "${LOG_FILE}" 2>/dev/null || echo 0)"
|
||||||
|
[ "${size}" -ge "${max_bytes}" ] || return 0
|
||||||
|
|
||||||
|
for (( i = MAX_LOG_KEEP - 1; i >= 1; i-- )); do
|
||||||
|
[ -f "${LOG_FILE}.${i}" ] && mv -f "${LOG_FILE}.${i}" "${LOG_FILE}.$(( i + 1 ))"
|
||||||
|
done
|
||||||
|
mv -f "${LOG_FILE}" "${LOG_FILE}.1"
|
||||||
|
exec >>"${LOG_FILE}" 2>&1 # re-bind stdout/stderr to the new file
|
||||||
|
printf '[%s] INFO : log rotated at %s MB\n' "$(ts)" "${MAX_LOG_SIZE_MB}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Remove the rotated copies (.1 .. .N) only. Timestamped archives are kept.
|
||||||
|
remove_rotated_logs() {
|
||||||
|
local i
|
||||||
|
for (( i = 1; i <= MAX_LOG_KEEP + 1; i++ )); do
|
||||||
|
rm -f "${LOG_FILE}.${i}"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Apply START_LOG_MODE just before a background run is launched.
|
||||||
|
prepare_log_on_start() {
|
||||||
|
local stamp
|
||||||
|
case "${START_LOG_MODE}" in
|
||||||
|
append)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
archive)
|
||||||
|
if [ -s "${LOG_FILE}" ]; then
|
||||||
|
stamp="$(date '+%Y%m%d-%H%M%S')"
|
||||||
|
mv -f "${LOG_FILE}" "${LOG_FILE}.${stamp}" \
|
||||||
|
|| die "cannot archive ${LOG_FILE}"
|
||||||
|
printf 'previous log archived: %s.%s\n' "${LOG_FILE}" "${stamp}"
|
||||||
|
fi
|
||||||
|
remove_rotated_logs
|
||||||
|
;;
|
||||||
|
new)
|
||||||
|
# Truncate rather than unlink: the inode is preserved, so a `tail -f`
|
||||||
|
# that is already attached keeps following the new run.
|
||||||
|
if [ -f "${LOG_FILE}" ]; then
|
||||||
|
: > "${LOG_FILE}" || die "cannot truncate ${LOG_FILE}"
|
||||||
|
fi
|
||||||
|
remove_rotated_logs
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
die "invalid START_LOG_MODE: ${START_LOG_MODE}"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
run_one_command() {
|
||||||
|
local cmd="$1"
|
||||||
|
local out err rc t0 elapsed err_file
|
||||||
|
|
||||||
|
err_file="$(mktemp)"
|
||||||
|
t0=${SECONDS}
|
||||||
|
if [ "${CMD_TIMEOUT_SEC}" -gt 0 ]; then
|
||||||
|
out="$(timeout "${CMD_TIMEOUT_SEC}" \
|
||||||
|
"${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
|
||||||
|
else
|
||||||
|
out="$("${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
|
||||||
|
fi
|
||||||
|
rc=$?
|
||||||
|
elapsed=$(( SECONDS - t0 ))
|
||||||
|
err="$(cat "${err_file}" 2>/dev/null)"
|
||||||
|
rm -f "${err_file}"
|
||||||
|
|
||||||
|
printf '===== [%s] CMD: %s | rc=%d | %ds =====\n' "$(ts)" "${cmd}" "${rc}" "${elapsed}"
|
||||||
|
printf '%s\n' "${out}"
|
||||||
|
if [ "${rc}" -eq 124 ]; then
|
||||||
|
printf '[%s] WARN : timeout after %ds\n' "$(ts)" "${CMD_TIMEOUT_SEC}"
|
||||||
|
fi
|
||||||
|
if [ -n "${err}" ] && { [ "${LOG_STDERR}" -eq 1 ] || [ "${rc}" -ne 0 ]; }; then
|
||||||
|
printf -- '--- stderr ---\n%s\n' "${err}"
|
||||||
|
fi
|
||||||
|
printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
main_loop() {
|
||||||
|
local round=0 cmd
|
||||||
|
|
||||||
|
[ "${#COMMANDS[@]}" -gt 0 ] || die "COMMANDS is empty"
|
||||||
|
command -v "${RUNNER_BIN}" >/dev/null 2>&1 || die "runner not found: ${RUNNER_BIN}"
|
||||||
|
|
||||||
|
trap on_signal INT TERM
|
||||||
|
|
||||||
|
printf '#############################################################\n'
|
||||||
|
printf '[%s] START: pid=%d interval=%ds timeout=%ds cmds=%d\n' \
|
||||||
|
"$(ts)" "$$" "${INTERVAL_SEC}" "${CMD_TIMEOUT_SEC}" "${#COMMANDS[@]}"
|
||||||
|
printf '#############################################################\n'
|
||||||
|
|
||||||
|
while [ "${STOP}" -eq 0 ]; do
|
||||||
|
round=$(( round + 1 ))
|
||||||
|
printf -- '---------- ROUND %d @ %s ----------\n' "${round}" "$(ts)"
|
||||||
|
|
||||||
|
for cmd in "${COMMANDS[@]}"; do
|
||||||
|
[ "${STOP}" -eq 0 ] || break
|
||||||
|
run_one_command "${cmd}"
|
||||||
|
[ "${INTER_CMD_DELAY_SEC}" -gt 0 ] && interruptible_sleep "${INTER_CMD_DELAY_SEC}"
|
||||||
|
done
|
||||||
|
|
||||||
|
rotate_log_if_needed
|
||||||
|
|
||||||
|
if [ "${MAX_ROUNDS}" -gt 0 ] && [ "${round}" -ge "${MAX_ROUNDS}" ]; then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
# Fixed gap between rounds. For a fixed cadence instead, replace with:
|
||||||
|
# interruptible_sleep $(( INTERVAL_SEC - round_elapsed ))
|
||||||
|
[ "${STOP}" -eq 0 ] && interruptible_sleep "${INTERVAL_SEC}"
|
||||||
|
done
|
||||||
|
|
||||||
|
printf '[%s] STOP : pid=%d after %d round(s)\n' "$(ts)" "$$" "${round}"
|
||||||
|
}
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Sub-commands
|
||||||
|
###############################################################################
|
||||||
|
do_start() {
|
||||||
|
local pid
|
||||||
|
pid="$(is_running)" && die "already running (pid=${pid})"
|
||||||
|
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
|
||||||
|
prepare_log_on_start
|
||||||
|
|
||||||
|
if command -v setsid >/dev/null 2>&1; then
|
||||||
|
setsid "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
|
||||||
|
else
|
||||||
|
nohup "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
|
||||||
|
disown 2>/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
|
sleep 1
|
||||||
|
pid="$(is_running)" || die "start failed, check ${LOG_FILE}"
|
||||||
|
printf 'started (pid=%s)\nlog: %s\n' "${pid}" "${LOG_FILE}"
|
||||||
|
}
|
||||||
|
|
||||||
|
do_daemon() {
|
||||||
|
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
|
||||||
|
LOG_TO_FILE=1
|
||||||
|
exec >>"${LOG_FILE}" 2>&1
|
||||||
|
printf '%d\n' "$$" > "${PID_FILE}"
|
||||||
|
trap 'rm -f "${PID_FILE}"' EXIT
|
||||||
|
main_loop
|
||||||
|
}
|
||||||
|
|
||||||
|
do_stop() {
|
||||||
|
local pid i
|
||||||
|
pid="$(is_running)" || { printf 'not running\n'; rm -f "${PID_FILE}"; return 0; }
|
||||||
|
|
||||||
|
kill -TERM -"${pid}" 2>/dev/null || kill -TERM "${pid}" 2>/dev/null
|
||||||
|
for (( i = 0; i < 20; i++ )); do
|
||||||
|
kill -0 "${pid}" 2>/dev/null || break
|
||||||
|
sleep 0.5
|
||||||
|
done
|
||||||
|
if kill -0 "${pid}" 2>/dev/null; then
|
||||||
|
printf 'SIGTERM ignored, sending SIGKILL\n'
|
||||||
|
kill -KILL -"${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null
|
||||||
|
fi
|
||||||
|
rm -f "${PID_FILE}"
|
||||||
|
printf 'stopped (pid=%s)\n' "${pid}"
|
||||||
|
}
|
||||||
|
|
||||||
|
do_status() {
|
||||||
|
local pid
|
||||||
|
if pid="$(is_running)"; then
|
||||||
|
printf 'status : running (pid=%s)\n' "${pid}"
|
||||||
|
else
|
||||||
|
printf 'status : stopped\n'
|
||||||
|
fi
|
||||||
|
printf 'log : %s\n' "${LOG_FILE}"
|
||||||
|
[ -f "${LOG_FILE}" ] && printf 'size : %s\n' "$(du -h "${LOG_FILE}" | cut -f1)"
|
||||||
|
printf 'cmds : %d, interval=%ds\n' "${#COMMANDS[@]}" "${INTERVAL_SEC}"
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: ${SCRIPT_NAME} {start|stop|status|fg|tail|clear}
|
||||||
|
|
||||||
|
start Run detached in the background; the previous log is discarded first
|
||||||
|
(see START_LOG_MODE)
|
||||||
|
stop Stop the background instance
|
||||||
|
status Show pid / log path / log size
|
||||||
|
fg Run in the foreground, output to the terminal (for a quick check)
|
||||||
|
tail tail -f the log file
|
||||||
|
clear Remove the log file and all rotated copies (must be stopped first)
|
||||||
|
|
||||||
|
Log file : ${LOG_FILE}
|
||||||
|
|
||||||
|
To add a command, edit the COMMANDS array at the top of this script.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
# Entry point
|
||||||
|
###############################################################################
|
||||||
|
case "${1:-}" in
|
||||||
|
start) do_start ;;
|
||||||
|
stop) do_stop ;;
|
||||||
|
status) do_status ;;
|
||||||
|
fg) LOG_TO_FILE=0; main_loop ;;
|
||||||
|
tail) tail -n 50 -f "${LOG_FILE}" ;;
|
||||||
|
clear)
|
||||||
|
is_running >/dev/null && die "still running, stop it first"
|
||||||
|
rm -f "${LOG_FILE}" "${LOG_FILE}".[0-9]*
|
||||||
|
printf 'log cleared\n'
|
||||||
|
;;
|
||||||
|
__daemon) do_daemon ;;
|
||||||
|
-h|--help|"") usage ;;
|
||||||
|
*) printf '[ERROR] unknown sub-command: %s\n\n' "$1" >&2; usage; exit 1 ;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user