diff --git a/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl b/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl index 010019f..53111e9 100644 --- a/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl +++ b/src/Script_ABC_Blanton/1_Blanton_Script_A.ttl @@ -1,7 +1,7 @@ ; ============================================================================= ; Script A for Blanton -; Version : V1.0.5 -; Date : 2026-08-21 +; Version : V1.0.6 +; Date : 2026-08-24 ; Author : ETWen ; ============================================================================= ; Version History: @@ -25,6 +25,18 @@ ; V1.0.5 2026-08-21 config.ttl Add SWB_UNIT0/SWB_UNIT1 ; utils/wait_init.ttl Follow config.ttl SWB_UNIT0/SWB_UNIT1 ; ScriptA/B/C Traffic follows SWB_UNIT: -u 0 / -u 1 / skip +; V1.0.6 2026-08-24 utils/show_dmesg.ttl Edited dmesg grep i2c & clear event +; ScriptA HW Test Session +; ScriptC HW Test Session Finish +; ScriptA bmc-first-enroll, bmc version/status +; ScriptB Stress test DDR, SSD, USB, BMC DDR, BMC USB Stress test +; ScriptC BMC USB Stress test result +; ScriptA DUT Info +; ScriptAC 10G/1G MGMT ping +; ScriptA Edited 100G Port Status +; ScriptA, utils/setup_pmon.ttl Add TPM +; ScriptAC NVME Error Info +; ScriptAC Add PCIe Summary ; ============================================================================= include "config.ttl" @@ -59,21 +71,50 @@ gettime timestr sprintf2 cmd 'sudo date -s "%s %s"' datestr timestr sendln cmd -; ========== Show Build Name ========== +; ========== HW Test Session ========== wait prompt_sonic_root -sendln "show boot" - -; ========== EEPROM Info ========== +sendln "hw-test-session start" wait prompt_sonic_root -sendln "hpe-eeprom-tlv show --bus 4 --addr 0x50" +sendln "hw-test-session log" +wait prompt_sonic_root +sendln "hw-test-session status" ; ========== Wait DUT Init ========== wait prompt_sonic_root include "utils/wait_init.ttl" -; ========== BMC version ========== +; ========== DUT Info ========== wait prompt_sonic_root -sendln "bmc-manager run 'cat /etc/issue'" +sendln "show boot" +wait prompt_sonic_root +sendln "show version" +wait prompt_sonic_root +sendln "fwutil show status" +wait prompt_sonic_root +sendln "show system-memory" +wait prompt_sonic_root +sendln "show service" +wait prompt_sonic_root +sendln "show platform syseeprom" +wait prompt_sonic_root +sendln "show platform ssdhealth" +wait prompt_sonic_root +sendln "qfx5252-tpm-version" +wait prompt_sonic_root +sendln "/usr/sbin/nvme smart-log -H /dev/nvme0" +wait prompt_sonic_root +sendln "/usr/sbin/smartctl -x /dev/nvme0" + +; ========== BMC version ========== +;wait prompt_sonic_root +;sendln "bmc-manager run 'cat /etc/issue'" +wait prompt_sonic_root +sendln "bmc-first-enroll" +wait prompt_sonic_root +sendln "bmc-manager version" +wait prompt_sonic_root +sendln "bmc-manager status" + ; ========== Load Shell Script ========== wait prompt_sonic_root @@ -98,6 +139,9 @@ include "utils/show_pcie_error_reg_DDR.ttl" include "utils/show_pcie_error_reg_I210.ttl" include "utils/show_pcie_error_reg_SSD.ttl" +wait prompt_sonic_root +sendln "/usr/sbin/ras-mc-ctl --summary" + ; ========== Check PCIE tree + PCIE link status + GET PCIE bandwidth Test ========== include "utils/pcie_bus.ttl" @@ -115,12 +159,24 @@ include "utils/show_dmesg.ttl" ;wait prompt_sonic_root ;flushrecv +; ========== 10G/1G MGMT Ping Test ========== +wait prompt_sonic_root +sendln "./Blanton_Script/mgmt_ping_monitor.sh start" +wait prompt_sonic_root +sendln "~/Blanton_Script/mgmt_ping_monitor.sh stop" +wait prompt_sonic_root +sendln "cat ~/Blanton_Script/log/mgmt_ping.log" + + ; ========== 100G Port Status ========== +; Lpmode Disable +wait prompt_sonic_root +sendln "sudo sfputil lpmode off Ethernet513" +wait prompt_sonic_root +sendln "sudo sfputil lpmode off Ethernet514" wait prompt_sonic_root sendln "config interface -n asic0 startup Ethernet513" wait prompt_sonic_root -sendln "config interface -n asic0 shutdown Ethernet513" -wait prompt_sonic_root sendln "config interface -n asic1 startup Ethernet514" wait prompt_sonic_root sendln "config save -y" diff --git a/src/Script_ABC_Blanton/2_Blanton_Script_B.ttl b/src/Script_ABC_Blanton/2_Blanton_Script_B.ttl index 61ff9a3..6081d8c 100644 --- a/src/Script_ABC_Blanton/2_Blanton_Script_B.ttl +++ b/src/Script_ABC_Blanton/2_Blanton_Script_B.ttl @@ -20,45 +20,68 @@ sendln 'chmod +x ~/hammer/tools/amd/mlucas-avx2' wait prompt_sonic_root sendln "~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 > ~/hammer/tools/amd/mlucas_amm_log 2>&1 &" -; ========== BMC Stress Test ========== +; ========== BMC DDR Stress Test ========== +;wait prompt_sonic_root +;sendln "./Blanton_Script/bmc_monitor_ddr.sh start" +;wait prompt_sonic_root +;sendln "./Blanton_Script/bmc_monitor.sh start" wait prompt_sonic_root -sendln "./Blanton_Script/bmc_monitor_ddr.sh start" -wait prompt_sonic_root -sendln "./Blanton_Script/bmc_monitor.sh start" - +sendln "bgctl run bmc-manager run '/usr/bin/memtester 64M 1'" ; ========== DDR MEMORY STRESS TEST ========== wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_mem.py &" -wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_mem.py &" -wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_mem.py &" -wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_mem.py &" -wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_mem.py &" +sendln "bgctl run /usr/sbin/memtester 1G 100" ; ========== SSD read/write ========== wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_ssd.py &" +sendln "bgctl run qfx5252-stress-ssd 1G --runtime 3600 --passes 5 --write --force --log /host/hw-eval/current/qfx5252-stress-ssd.log" ; ========== USB read/write ========== wait prompt_sonic_root -sendln "python ~/hammer/tools/stress_usb.py &" +sendln "bgctl run qfx5252-stress-usb /dev/sda1 256M --runtime 60 --passes 1 --write --log /host/hw-eval/qfx5252-stress-usb.log" ; ========== SHOW Background job ========== wait prompt_sonic_root sendln "jobs" +wait prompt_sonic_root +sendln "bgctl list" + +; ========== BMC USB Test ========== +wait prompt_sonic_root +sendln 'echo "NCMIF=$(ls -d /sys/bus/usb/drivers/cdc_ncm/*/net/* 2>/dev/null | head -n 1 | xargs -r basename)"' +timeout = 15 +waitregex 'NCMIF=[A-Za-z0-9_-]+' +if result = 0 then + messagebox 'cdc_ncm interface not found. Ping test skipped.' 'ERROR' + goto skip_ping +endif +strlen matchstr +cut_len = result - 6 ; 'NCMIF=' = 6 chars +strcopy matchstr 7 cut_len ncm_if ; -> ncm_if = "eth2" / "eth3" + +sprintf2 cmd 'ip -br link show %s' ncm_if +wait prompt_sonic_root +sendln cmd + +wait prompt_sonic_root +sendln 'systemctl stop qfx5252-bmc-usb-net-test.service 2>/dev/null || true' +wait prompt_sonic_root +sendln 'systemctl reset-failed qfx5252-bmc-usb-net-test.service 2>/dev/null || true' +sprintf2 cmd 'systemd-run --unit=qfx5252-bmc-usb-net-test --property=Type=exec /usr/bin/ping -I %s -i 1 -W 2 -w 14400 192.168.200.200' ncm_if +wait prompt_sonic_root +sendln cmd + +:skip_ping +; restore the default (no cap): the traffic init below walks 108 VLANs per +; unit and takes far longer than the 15 s set for the NCM probe above. +timeout = 0 + +; ========== 10G/1G MGMT Ping Test ========== +wait prompt_sonic_root +sendln "./Blanton_Script/mgmt_ping_monitor.sh start" ; ========== 100G Port ========== -; Lpmode Disable -wait prompt_sonic_root -sendln "sudo sfputil lpmode off Ethernet513" -wait prompt_sonic_root -sendln "sudo sfputil lpmode off Ethernet514" - wait prompt_sonic_root sendln "show interfaces status Ethernet513,Ethernet514" diff --git a/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl b/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl index 0b264cf..340167e 100644 --- a/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl +++ b/src/Script_ABC_Blanton/3_Blanton_Script_C.ttl @@ -12,6 +12,21 @@ pause 1 ; ========== Kill Process ========== include "utils/kill_all_process.ttl" +; bgctrl all stop +wait prompt_sonic_root +sendln "bgctl stop --all" +wait prompt_sonic_root +sendln "bgctl stop --all" +wait prompt_sonic_root +sendln "bgctl reset --yes" +; BMC USB Test STOP +wait prompt_sonic_root +sendln "systemctl stop qfx5252-bmc-usb-net-test.service" +; 10G/1G MGMT Test STOP +wait prompt_sonic_root +sendln "~/Blanton_Script/mgmt_ping_monitor.sh stop" + + ; ========== TAKE DATA ========== include "utils/setup_pmon.ttl" include "utils/show_dmesg.ttl" @@ -25,22 +40,33 @@ include "utils/show_pcie_error_reg_DDR.ttl" include "utils/show_pcie_error_reg_I210.ttl" include "utils/show_pcie_error_reg_SSD.ttl" -; ========== BMC TAKE DATA ========== wait prompt_sonic_root -sendln "./Blanton_Script/bmc_monitor_ddr.sh stop" +sendln "/usr/sbin/ras-mc-ctl --summary" + +; ========== BMC DDR TAKE DATA ========== +;wait prompt_sonic_root +;sendln "./Blanton_Script/bmc_monitor_ddr.sh stop" +;wait prompt_sonic_root +;sendln "./Blanton_Script/bmc_monitor.sh stop" +;wait prompt_sonic_root +;sendln "cat ./Blanton_Script/log/bmc_poll.log" +;wait prompt_sonic_root +;sendln "cat ./Blanton_Script/log/bmc_ddr.log" + +; ========== BMC USB Test result ========== wait prompt_sonic_root -sendln "./Blanton_Script/bmc_monitor.sh stop" +sendln "journalctl -u qfx5252-bmc-usb-net-test.service --no-pager" + +; ========== 10G/1G MGMT Test result ========== wait prompt_sonic_root -sendln "cat ./Blanton_Script/log/bmc_poll.log" +sendln "ip -s link show eth0" wait prompt_sonic_root -sendln "cat ./Blanton_Script/log/bmc_ddr.log" +sendln "ip -s link show eth1" ; ========== CHECK Stress results ========== wait prompt_sonic_root sendln "cat ~/hammer/tools/amd/mlucas_amm_log" -wait prompt_sonic_root -sendln "cat ~/hammer/tools/log.stress_ssd" ; ========== CHECK Traffic counters ========== if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then @@ -62,6 +88,13 @@ else endif +; ========== NVME Err Info ========== +wait prompt_sonic_root +sendln "/usr/sbin/nvme smart-log -H /dev/nvme0" +wait prompt_sonic_root +sendln "/usr/sbin/smartctl -x /dev/nvme0" + + wait prompt_sonic_root sendln "show reboot-cause" @@ -69,4 +102,8 @@ sendln "show reboot-cause" wait prompt_sonic_root sendln "show uptime" +; ========== HW Test Session ========== +wait prompt_sonic_root +sendln "hw-test-session finish" + messagebox 'GOOD JOB! Test Case DONE' 'teraterm' \ No newline at end of file diff --git a/src/Script_ABC_Blanton/Blanton_Script/mgmt_ping_monitor.sh b/src/Script_ABC_Blanton/Blanton_Script/mgmt_ping_monitor.sh new file mode 100644 index 0000000..7ed96e3 --- /dev/null +++ b/src/Script_ABC_Blanton/Blanton_Script/mgmt_ping_monitor.sh @@ -0,0 +1,463 @@ +#!/bin/bash +############################################################################### +# mgmt_ping_monitor.sh +# +# Version : V1.0.0 +# Author : ETWen +# Date : 20260824 +# +# Purpose : Exercise both management NICs and ping each one's own target, +# appending everything to a rotating log. One round is +# +# 10G (eth0) : ip link up + address/route replace -> settle +# -> ping N x TARGET_10G -> ip -s link show +# 1G (eth1) : ip link up + address/route replace -> settle +# -> ping N x TARGET_1G -> ip -s link show +# +# Designed to run detached in the background, same sub-commands as +# bmc_monitor.sh: start/stop/status/fg/tail/clear. +# +# Notes : - Both NICs are brought up and LEFT up. Addresses and default +# routes are applied with iproute2 `replace`, which is idempotent: +# re-running a round does not stack duplicate addresses or routes. +# - The default routes carry different metrics (100 / 200), so the +# lower one wins for off-subnet traffic. They do NOT decide which +# NIC a ping leaves by: `ping -I ` binds to the device. +# - !! SAME-SUBNET WARNING !! With both NICs up on one subnet the +# target ARPs for the address and Linux may answer from either +# NIC, so replies can arrive on the wrong one and the result is +# not attributable. ARP_STRICT=1 sets arp_ignore/arp_announce to +# stop that. Leave it on unless the two NICs are on separate +# subnets, as in the iproute2 example this was modelled on. +# - Each leg logs `ip -s link show ` and the TX/RX packet +# delta across the burst, which is the direct evidence of which +# NIC actually carried the traffic. +# - The first pings after bringing a link up are lost to ARP, not to +# the link: a bench capture showed 4x "Destination Host +# Unreachable" on the first burst and 0% loss on the very next +# one. LINK_SETTLE_SEC absorbs that; it is not packet loss. +# +# Version History +# V1.0.0 20260824 Initial Version +# V2.0.0 20260824 Switch from ifconfig to iproute2 (ip link / address +# replace / route replace with per-NIC metric). Both +# NICs now stay up -- no link down between legs. Add +# `ip -s link show` per leg with a TX/RX delta, and +# ARP_STRICT for the same-subnet case. +############################################################################### + +set -u + +############################################################################### +# User Configurable Section -- normally this is the only part you need to edit +############################################################################### + +# --- 10G management port --- +IF_10G="eth0" +IP_10G="192.168.1.99" +TARGET_10G="192.168.1.30" # host this NIC pings +PLEN_10G=24 # prefix length, e.g. 24 = /24 +GW_10G="" # default gateway; empty = do not touch routing +METRIC_10G=100 # lower metric wins for off-subnet traffic + +# --- 1G management port --- +IF_1G="eth1" +IP_1G="192.168.1.101" +TARGET_1G="192.168.1.31" # host this NIC pings +PLEN_1G=24 +GW_1G="" +METRIC_1G=200 + +# --- ping --- +PING_COUNT=10 # echo requests per leg +MAX_LOSS_PCT=0 # loss above this marks the leg FAIL + +# Both NICs stay up. If they share a subnet the target's ARP can be answered +# by either one, so a reply may arrive on the NIC that did not send. Setting +# this applies arp_ignore=1 / arp_announce=2 to both, which keeps each NIC to +# its own address. RAM only -- reverts on reboot. Set 0 for separate subnets. +ARP_STRICT=1 + +# Prefix for the ip commands; empty because the daemon already runs as root. +SUDO="" + +# Seconds to wait after bringing a link up before pinging, so ARP is resolved +# and the PHY has finished autoneg. The bench needed a few seconds on the 1G. +LINK_SETTLE_SEC=5 + +# Seconds between rounds (0 = start the next round immediately) +INTERVAL_SEC=10 + +# Per-ping-burst timeout in seconds; stops a wedged ping from freezing the +# loop. 0 disables. Must exceed PING_COUNT (one packet per second). +PING_TIMEOUT_SEC=60 + +# Stop after this many rounds. 0 = run until stopped manually. +MAX_ROUNDS=0 + +# Logging +LOG_DIR="" # empty =