feat(bmc): Split DDR stress out of bmc_monitor into bmc_monitor_ddr.sh
bmc_monitor.sh was doing two unrelated jobs: sampling `free -m` every few seconds, and running `memtester 500M 1` as a stress load. They want different timeouts -- a monitor should give up in 30 s, a memtester pass legitimately runs for minutes -- and mixing them meant either cutting memtester short or letting a hung sample stall the loop. bmc_monitor.sh now polls `free -m` only, keeping CMD_TIMEOUT_SEC=30. bmc_monitor_ddr.sh is the stress half: memtester only, with CMD_TIMEOUT_SEC raised to 3600. The two are otherwise identical. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014RetWKFZFG1ZHcitQAyhwM
This commit is contained in:
Claude Opus 5
@@ -34,7 +34,6 @@ set -u
|
||||
# Add a line to add a command; comment it out to disable it.
|
||||
COMMANDS=(
|
||||
"free -m"
|
||||
"memtester 500M 1"
|
||||
#"uptime"
|
||||
#"cat /proc/loadavg"
|
||||
#"cat /proc/meminfo"
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# bmc_monitor_ddr.sh
|
||||
#
|
||||
# Version : V1.1.0
|
||||
# Author : ETWen
|
||||
# Date : 20260820
|
||||
#
|
||||
# Purpose : Periodically execute a list of commands on the BMC through
|
||||
# `bmc-manager run "<cmd>"` from the SONiC host side, and append all
|
||||
# output to a rotating log file. Designed to run detached in the
|
||||
# background so no interactive SSH session to the BMC is required.
|
||||
#
|
||||
# Notes : - Everything runs on the HOST. Nothing is deployed to the BMC, so
|
||||
# the BMC busybox toolchain (no base64 / no setsid ...) is a
|
||||
# non-issue.
|
||||
# - `bmc-manager run` prints the remote command output on stdout and
|
||||
# its own banner on stderr; the two streams are captured separately
|
||||
# so the log stays clean.
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260820 Initial Version
|
||||
# V1.1.0 20260820 `start` clears the previous log by default
|
||||
# (START_LOG_MODE: new / archive / append)
|
||||
###############################################################################
|
||||
|
||||
set -u
|
||||
|
||||
###############################################################################
|
||||
# User Configurable Section -- normally this is the only part you need to edit
|
||||
###############################################################################
|
||||
|
||||
# Commands executed on the BMC, one per line, in this order.
|
||||
# Add a line to add a command; comment it out to disable it.
|
||||
COMMANDS=(
|
||||
"memtester 500M 1"
|
||||
#"uptime"
|
||||
#"cat /proc/loadavg"
|
||||
#"cat /proc/meminfo"
|
||||
#"df -h /tmp"
|
||||
#"dmesg | tail -n 30"
|
||||
)
|
||||
|
||||
# Seconds to wait after a full round (all COMMANDS executed) before the next one
|
||||
INTERVAL_SEC=5
|
||||
|
||||
# Seconds to wait between two commands inside the same round (0 = no wait)
|
||||
INTER_CMD_DELAY_SEC=0
|
||||
|
||||
# Per-command timeout in seconds; prevents a hung bmc-manager from freezing the
|
||||
# whole loop. Set to 0 to disable.
|
||||
CMD_TIMEOUT_SEC=3600
|
||||
|
||||
# Stop after this many rounds. 0 = run until stopped manually.
|
||||
MAX_ROUNDS=0
|
||||
|
||||
# Command runner. Final invocation is:
|
||||
# "$RUNNER_BIN" "${RUNNER_ARGS[@]}" "<command string>"
|
||||
# BMC side : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(run)
|
||||
# BMC by IP : RUNNER_BIN="bmc-manager" ; RUNNER_ARGS=(--ip 192.168.200.200 run)
|
||||
# Host side : RUNNER_BIN="bash" ; RUNNER_ARGS=(-c)
|
||||
RUNNER_BIN="bmc-manager"
|
||||
RUNNER_ARGS=(run)
|
||||
|
||||
# stderr handling of the runner (bmc-manager banner lives here)
|
||||
# 0 = log stderr only when the command fails (recommended, keeps log clean)
|
||||
# 1 = always log stderr
|
||||
LOG_STDERR=0
|
||||
|
||||
# Logging
|
||||
LOG_DIR="" # empty = <script directory>/log
|
||||
LOG_NAME="bmc_poll.log"
|
||||
MAX_LOG_SIZE_MB=100 # rotate above this size; 0 = never rotate
|
||||
MAX_LOG_KEEP=5 # keep .1 .. .N
|
||||
|
||||
# What `start` does with the log left behind by the previous run:
|
||||
# new = begin with an empty log; previous content and rotated copies are
|
||||
# discarded (default -- one run, one log)
|
||||
# archive = rename the previous log to <log>.YYYYmmdd-HHMMSS, then begin empty
|
||||
# append = keep appending to the existing log
|
||||
START_LOG_MODE="new"
|
||||
|
||||
###############################################################################
|
||||
# Internal
|
||||
###############################################################################
|
||||
SCRIPT_NAME="$(basename -- "${BASH_SOURCE[0]}")"
|
||||
SCRIPT_PATH="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/${SCRIPT_NAME}"
|
||||
[ -n "${LOG_DIR}" ] || LOG_DIR="$(dirname -- "${SCRIPT_PATH}")/log"
|
||||
LOG_FILE="${LOG_DIR}/${LOG_NAME}"
|
||||
PID_FILE="${LOG_DIR}/${LOG_NAME%.log}.pid"
|
||||
|
||||
LOG_TO_FILE=0
|
||||
STOP=0
|
||||
SLEEP_PID=""
|
||||
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
die() { printf '[ERROR] %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
on_signal() {
|
||||
STOP=1
|
||||
[ -n "${SLEEP_PID}" ] && kill "${SLEEP_PID}" 2>/dev/null
|
||||
}
|
||||
|
||||
# sleep that can be aborted immediately by SIGTERM/SIGINT
|
||||
interruptible_sleep() {
|
||||
local sec="$1"
|
||||
[ "${sec}" -gt 0 ] 2>/dev/null || return 0
|
||||
sleep "${sec}" &
|
||||
SLEEP_PID=$!
|
||||
wait "${SLEEP_PID}" 2>/dev/null
|
||||
SLEEP_PID=""
|
||||
}
|
||||
|
||||
# echo the pid and return 0 when a live instance is found
|
||||
is_running() {
|
||||
local pid
|
||||
[ -f "${PID_FILE}" ] || return 1
|
||||
pid="$(cat "${PID_FILE}" 2>/dev/null)"
|
||||
[[ "${pid}" =~ ^[0-9]+$ ]] || return 1
|
||||
kill -0 "${pid}" 2>/dev/null || return 1
|
||||
grep -qa "${SCRIPT_NAME}" "/proc/${pid}/cmdline" 2>/dev/null || return 1
|
||||
printf '%s' "${pid}"
|
||||
return 0
|
||||
}
|
||||
|
||||
rotate_log_if_needed() {
|
||||
local max_bytes size i
|
||||
[ "${LOG_TO_FILE}" -eq 1 ] || return 0
|
||||
[ "${MAX_LOG_SIZE_MB}" -gt 0 ] || return 0
|
||||
[ -f "${LOG_FILE}" ] || return 0
|
||||
|
||||
max_bytes=$(( MAX_LOG_SIZE_MB * 1024 * 1024 ))
|
||||
size="$(stat -c %s "${LOG_FILE}" 2>/dev/null || echo 0)"
|
||||
[ "${size}" -ge "${max_bytes}" ] || return 0
|
||||
|
||||
for (( i = MAX_LOG_KEEP - 1; i >= 1; i-- )); do
|
||||
[ -f "${LOG_FILE}.${i}" ] && mv -f "${LOG_FILE}.${i}" "${LOG_FILE}.$(( i + 1 ))"
|
||||
done
|
||||
mv -f "${LOG_FILE}" "${LOG_FILE}.1"
|
||||
exec >>"${LOG_FILE}" 2>&1 # re-bind stdout/stderr to the new file
|
||||
printf '[%s] INFO : log rotated at %s MB\n' "$(ts)" "${MAX_LOG_SIZE_MB}"
|
||||
}
|
||||
|
||||
# Remove the rotated copies (.1 .. .N) only. Timestamped archives are kept.
|
||||
remove_rotated_logs() {
|
||||
local i
|
||||
for (( i = 1; i <= MAX_LOG_KEEP + 1; i++ )); do
|
||||
rm -f "${LOG_FILE}.${i}"
|
||||
done
|
||||
}
|
||||
|
||||
# Apply START_LOG_MODE just before a background run is launched.
|
||||
prepare_log_on_start() {
|
||||
local stamp
|
||||
case "${START_LOG_MODE}" in
|
||||
append)
|
||||
return 0
|
||||
;;
|
||||
archive)
|
||||
if [ -s "${LOG_FILE}" ]; then
|
||||
stamp="$(date '+%Y%m%d-%H%M%S')"
|
||||
mv -f "${LOG_FILE}" "${LOG_FILE}.${stamp}" \
|
||||
|| die "cannot archive ${LOG_FILE}"
|
||||
printf 'previous log archived: %s.%s\n' "${LOG_FILE}" "${stamp}"
|
||||
fi
|
||||
remove_rotated_logs
|
||||
;;
|
||||
new)
|
||||
# Truncate rather than unlink: the inode is preserved, so a `tail -f`
|
||||
# that is already attached keeps following the new run.
|
||||
if [ -f "${LOG_FILE}" ]; then
|
||||
: > "${LOG_FILE}" || die "cannot truncate ${LOG_FILE}"
|
||||
fi
|
||||
remove_rotated_logs
|
||||
;;
|
||||
*)
|
||||
die "invalid START_LOG_MODE: ${START_LOG_MODE}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
run_one_command() {
|
||||
local cmd="$1"
|
||||
local out err rc t0 elapsed err_file
|
||||
|
||||
err_file="$(mktemp)"
|
||||
t0=${SECONDS}
|
||||
if [ "${CMD_TIMEOUT_SEC}" -gt 0 ]; then
|
||||
out="$(timeout "${CMD_TIMEOUT_SEC}" \
|
||||
"${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
|
||||
else
|
||||
out="$("${RUNNER_BIN}" "${RUNNER_ARGS[@]}" "${cmd}" 2>"${err_file}")"
|
||||
fi
|
||||
rc=$?
|
||||
elapsed=$(( SECONDS - t0 ))
|
||||
err="$(cat "${err_file}" 2>/dev/null)"
|
||||
rm -f "${err_file}"
|
||||
|
||||
printf '===== [%s] CMD: %s | rc=%d | %ds =====\n' "$(ts)" "${cmd}" "${rc}" "${elapsed}"
|
||||
printf '%s\n' "${out}"
|
||||
if [ "${rc}" -eq 124 ]; then
|
||||
printf '[%s] WARN : timeout after %ds\n' "$(ts)" "${CMD_TIMEOUT_SEC}"
|
||||
fi
|
||||
if [ -n "${err}" ] && { [ "${LOG_STDERR}" -eq 1 ] || [ "${rc}" -ne 0 ]; }; then
|
||||
printf -- '--- stderr ---\n%s\n' "${err}"
|
||||
fi
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
main_loop() {
|
||||
local round=0 cmd
|
||||
|
||||
[ "${#COMMANDS[@]}" -gt 0 ] || die "COMMANDS is empty"
|
||||
command -v "${RUNNER_BIN}" >/dev/null 2>&1 || die "runner not found: ${RUNNER_BIN}"
|
||||
|
||||
trap on_signal INT TERM
|
||||
|
||||
printf '#############################################################\n'
|
||||
printf '[%s] START: pid=%d interval=%ds timeout=%ds cmds=%d\n' \
|
||||
"$(ts)" "$$" "${INTERVAL_SEC}" "${CMD_TIMEOUT_SEC}" "${#COMMANDS[@]}"
|
||||
printf '#############################################################\n'
|
||||
|
||||
while [ "${STOP}" -eq 0 ]; do
|
||||
round=$(( round + 1 ))
|
||||
printf -- '---------- ROUND %d @ %s ----------\n' "${round}" "$(ts)"
|
||||
|
||||
for cmd in "${COMMANDS[@]}"; do
|
||||
[ "${STOP}" -eq 0 ] || break
|
||||
run_one_command "${cmd}"
|
||||
[ "${INTER_CMD_DELAY_SEC}" -gt 0 ] && interruptible_sleep "${INTER_CMD_DELAY_SEC}"
|
||||
done
|
||||
|
||||
rotate_log_if_needed
|
||||
|
||||
if [ "${MAX_ROUNDS}" -gt 0 ] && [ "${round}" -ge "${MAX_ROUNDS}" ]; then
|
||||
break
|
||||
fi
|
||||
# Fixed gap between rounds. For a fixed cadence instead, replace with:
|
||||
# interruptible_sleep $(( INTERVAL_SEC - round_elapsed ))
|
||||
[ "${STOP}" -eq 0 ] && interruptible_sleep "${INTERVAL_SEC}"
|
||||
done
|
||||
|
||||
printf '[%s] STOP : pid=%d after %d round(s)\n' "$(ts)" "$$" "${round}"
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Sub-commands
|
||||
###############################################################################
|
||||
do_start() {
|
||||
local pid
|
||||
pid="$(is_running)" && die "already running (pid=${pid})"
|
||||
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
|
||||
prepare_log_on_start
|
||||
|
||||
if command -v setsid >/dev/null 2>&1; then
|
||||
setsid "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
|
||||
else
|
||||
nohup "${SCRIPT_PATH}" __daemon >/dev/null 2>&1 &
|
||||
disown 2>/dev/null
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
pid="$(is_running)" || die "start failed, check ${LOG_FILE}"
|
||||
printf 'started (pid=%s)\nlog: %s\n' "${pid}" "${LOG_FILE}"
|
||||
}
|
||||
|
||||
do_daemon() {
|
||||
mkdir -p "${LOG_DIR}" || die "cannot create ${LOG_DIR}"
|
||||
LOG_TO_FILE=1
|
||||
exec >>"${LOG_FILE}" 2>&1
|
||||
printf '%d\n' "$$" > "${PID_FILE}"
|
||||
trap 'rm -f "${PID_FILE}"' EXIT
|
||||
main_loop
|
||||
}
|
||||
|
||||
do_stop() {
|
||||
local pid i
|
||||
pid="$(is_running)" || { printf 'not running\n'; rm -f "${PID_FILE}"; return 0; }
|
||||
|
||||
kill -TERM -"${pid}" 2>/dev/null || kill -TERM "${pid}" 2>/dev/null
|
||||
for (( i = 0; i < 20; i++ )); do
|
||||
kill -0 "${pid}" 2>/dev/null || break
|
||||
sleep 0.5
|
||||
done
|
||||
if kill -0 "${pid}" 2>/dev/null; then
|
||||
printf 'SIGTERM ignored, sending SIGKILL\n'
|
||||
kill -KILL -"${pid}" 2>/dev/null || kill -KILL "${pid}" 2>/dev/null
|
||||
fi
|
||||
rm -f "${PID_FILE}"
|
||||
printf 'stopped (pid=%s)\n' "${pid}"
|
||||
}
|
||||
|
||||
do_status() {
|
||||
local pid
|
||||
if pid="$(is_running)"; then
|
||||
printf 'status : running (pid=%s)\n' "${pid}"
|
||||
else
|
||||
printf 'status : stopped\n'
|
||||
fi
|
||||
printf 'log : %s\n' "${LOG_FILE}"
|
||||
[ -f "${LOG_FILE}" ] && printf 'size : %s\n' "$(du -h "${LOG_FILE}" | cut -f1)"
|
||||
printf 'cmds : %d, interval=%ds\n' "${#COMMANDS[@]}" "${INTERVAL_SEC}"
|
||||
}
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: ${SCRIPT_NAME} {start|stop|status|fg|tail|clear}
|
||||
|
||||
start Run detached in the background; the previous log is discarded first
|
||||
(see START_LOG_MODE)
|
||||
stop Stop the background instance
|
||||
status Show pid / log path / log size
|
||||
fg Run in the foreground, output to the terminal (for a quick check)
|
||||
tail tail -f the log file
|
||||
clear Remove the log file and all rotated copies (must be stopped first)
|
||||
|
||||
Log file : ${LOG_FILE}
|
||||
|
||||
To add a command, edit the COMMANDS array at the top of this script.
|
||||
EOF
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
# Entry point
|
||||
###############################################################################
|
||||
case "${1:-}" in
|
||||
start) do_start ;;
|
||||
stop) do_stop ;;
|
||||
status) do_status ;;
|
||||
fg) LOG_TO_FILE=0; main_loop ;;
|
||||
tail) tail -n 50 -f "${LOG_FILE}" ;;
|
||||
clear)
|
||||
is_running >/dev/null && die "still running, stop it first"
|
||||
rm -f "${LOG_FILE}" "${LOG_FILE}".[0-9]*
|
||||
printf 'log cleared\n'
|
||||
;;
|
||||
__daemon) do_daemon ;;
|
||||
-h|--help|"") usage ;;
|
||||
*) printf '[ERROR] unknown sub-command: %s\n\n' "$1" >&2; usage; exit 1 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user