diff --git a/src/Script_ABC_Blanton/Blanton_Script/blanton_ber.sh b/src/Script_ABC_Blanton/Blanton_Script/blanton_ber.sh index eac1d52..70eaa34 100644 --- a/src/Script_ABC_Blanton/Blanton_Script/blanton_ber.sh +++ b/src/Script_ABC_Blanton/Blanton_Script/blanton_ber.sh @@ -1,8 +1,31 @@ #!/usr/bin/env bash # ============================================================================= -# blanton_ber.sh -- SerDes BER test over every cabled port (PRBS via phydiag) +# blanton_ber.sh -- SerDes BER test, cabled ports PLUS the 100G uplink # ============================================================================= # Version History: +# V1.1.1 2026-09-15 Fix: "the BCM shell rejected 'phydiag ... prbs get'" on +# a run that was in fact working. The rejection check +# grepped the WHOLE output, case-insensitively, for +# ERROR|Invalid|... -- and 'prbs get' prints one line per +# lane, so a lane that had not locked ("PRBS Error!") +# matched. The command was fine; the output was data. +# Two consequences: a good run reported a hard error, and +# the lock-count check that was supposed to judge exactly +# those lanes never ran. Per-lane result lines are now +# excluded from the check, the match is case-sensitive, +# and a real rejection prints the offending line. +# The rejection check had been there since V1.0.0, so +# every earlier run carried the same blind spot. +# V1.1.0 2026-09-15 Adds port 268, the 100G uplink, which the old port list +# deliberately skipped (it jumps 256-259 -> 270-308). +# The per-lane table is unchanged; the summary now breaks +# each unit into "200G ports" and "100G p268" plus a +# TOTAL row, so the uplink cannot hide inside a 432-lane +# average. A group that reports no lane at all counts as +# a failure, not a pass - silence on the port under test +# is not a result. Developed on the bench as +# blanton_ber_test.sh and folded back over this file once +# it was proven, so there is one BER script again. # V1.0.1 2026-08-28 Fix: every lane reported NA and the Port column showed # "NA". bcmcmd returns CRLF, and the BER is the last token # on its line, so the CR stayed glued to the number: it @@ -19,8 +42,24 @@ # ============================================================================= # # This is the WHOLE-SWITCH BER sweep: every cabled port, both lanes, PRBS-31 -# through the SerDes. Not to be confused with port_prbs_monitor.sh, which drives -# the two 100G uplinks only and polls them continuously. +# through the SerDes -- and, unlike blanton_ber.sh, the 100G uplink as well. +# Not to be confused with port_prbs_monitor.sh, which drives the two 100G +# uplinks only and polls them continuously. +# +# The port list is two groups: +# BER_PORTS_200G the cabled 200G ports (432 lanes per unit) +# BER_PORTS_100G the 100G uplink, port 268 = ce0, one per unit +# They are measured in one phydiag pass, but the summary reports them +# separately per unit, plus a TOTAL: +# +# Unit GROUP LANES PASS FAIL NA WORST BER ... +# unit 0 200G ports 432 432 0 0 2.07e-08 PASS +# unit 0 100G p268 4 4 0 0 1.10e-11 PASS +# unit 0 TOTAL 436 436 0 0 2.07e-08 PASS +# unit 1 ... +# +# A 100G row of "NO DATA" means the uplink reported no lane at all, and is +# treated as a failure -- silence on the port under test is not a pass. # # Order of operations (matches tools/BER_Test.txt): # init phydiag prbs set p=3 arm PRBS on every lane @@ -51,7 +90,7 @@ # -u|--unit <0|1|all|0,1> bcmcmd unit(s). Default: $BER_UNITS ("0 1") # -c|--criteria report: PASS threshold (default $BER_MAX) # -i|--interval start: prbsstat Interval (default $BER_INTERVAL) -# -p|--ports override the port list +# -p|--ports override the whole port list (both groups) # -F|--fail-only report: list only the lanes that FAIL # --tsv report: tab-separated (paste into Excel) # --raw report: also print the raw 'prbsstat Ber' output @@ -63,7 +102,8 @@ # # Environment overrides: # BCMCMD_CMD=bcmcmd BER_UNITS="0 1" BER_MAX=1e-6 BER_INTERVAL=30 -# BER_PORTS="1-24,..." BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1 +# BER_PORTS="1-24,..." BER_PORTS_200G=... BER_PORTS_100G=268 +# BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1 # ============================================================================= # === Config ================================================================== @@ -73,7 +113,18 @@ BER_UNITS="${BER_UNITS:-0 1}" # one DUT = two switch boards (0 and 1) # The cabled ports, in bcm logical-port numbering. Same list as # blanton_traffic_linespeed.sh's TL_PS_PORTS -- kept as one string so it can be # pasted straight from/into a manual phydiag command. -BER_PORTS="${BER_PORTS:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}" +# +# Note the gap: the list runs 256-259 then 270-308, so 260..269 are not in it. +# 268 is ce0, the 100G uplink, and it is the reason this fork exists. +BER_PORTS_200G="${BER_PORTS_200G:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}" + +# The 100G uplink. One port per unit -- unit 0 and unit 1 each have their own +# ce0(268), so a two-unit run measures two of them. Comma-separated if more are +# ever added; the summary treats everything in here as the "100G" group. +BER_PORTS_100G="${BER_PORTS_100G:-268}" + +# What actually goes on the phydiag command line. -p|--ports overrides the lot. +BER_PORTS="${BER_PORTS:-${BER_PORTS_200G},${BER_PORTS_100G}}" BER_POLY="${BER_POLY:-p=3}" # passed to 'prbs set' BER_INTERVAL="${BER_INTERVAL:-30}" # passed to 'prbsstat STArt Interval=' @@ -111,6 +162,24 @@ _ber_dbg() { [ "$DEBUG_MODE" = "1" ] && echo -e "\033[90m[DEBG] $*\033[0m" >&2; BER_ERR_PAT="${BER_ERR_PAT:-ERROR|Failed to execute|Invalid|Syntax error|Unavailable|Unknown command}" BER_BENIGN_PAT="${BER_BENIGN_PAT:-polling socket timeout}" +# A per-lane result line -- " 185 : PRBS OK!..." from 'prbs get', or +# " 203[1] : 5.05e-09" from 'prbsstat Ber'. These are MEASUREMENTS, never +# shell diagnostics, and they must be kept out of the rejection check: +# a lane that did not lock prints its own wording (PRBS Error!, and similar), +# which a case-insensitive match on "ERROR" reads as the BCM shell refusing +# the command. That is what turned a perfectly good run into +# [ERR ] unit 1: the BCM shell rejected 'phydiag ... prbs get' +# while init/start/report were in fact working -- and it skipped the +# lock-count check that was supposed to judge those very lanes. +BER_LANE_LINE='^[ \t]*[0-9]+(\[[0-9]+\])?[ \t]*:' + +# _ber_shell_error -- echo the offending line(s), return 0 if any +_ber_shell_error() { + printf '%s\n' "$1" | tr -d '\r' \ + | grep -vE "$BER_LANE_LINE" \ + | grep -E "$BER_ERR_PAT" +} + # _ber_bcm -- wraps in dsh -c 'phydiag ...' # echoes the DUT output on stdout, returns 0 ok / 1 failed _ber_bcm() { @@ -125,11 +194,14 @@ _ber_bcm() { local out rc out=$("$BCMCMD_CMD" -n "$unit" -c "$cmd" &1); rc=$? printf '%s\n' "$out" - if echo "$out" | grep -qiE "$BER_ERR_PAT"; then + local why + why=$(_ber_shell_error "$out") + if [ -n "$why" ]; then _ber_err "unit $unit: the BCM shell rejected 'phydiag ... $*'" + printf '%s\n' "$why" | sed 's/^/ /' >&2 return 1 fi - if [ "$rc" -ne 0 ] && ! echo "$out" | grep -qiE "$BER_BENIGN_PAT"; then + if [ "$rc" -ne 0 ] && ! printf '%s\n' "$out" | grep -qiE "$BER_BENIGN_PAT"; then _ber_err "unit $unit: bcmcmd exited $rc on 'phydiag ... $*'" return 1 fi @@ -243,13 +315,32 @@ _ber_render() { -v FAILONLY="$BER_FAIL_ONLY" \ -v TSV="$BER_TSV" \ -v COLOR="$color" \ - -v SIDE="$side" ' + -v SIDE="$side" \ + -v P100="$BER_PORTS_100G" ' function vc(v) { if (!COLOR) return v if (v == "PASS") return "\033[32m" v "\033[0m" if (v == "FAIL") return "\033[31m" v "\033[0m" return "\033[33m" v "\033[0m" } + # Fold one lane into a group tally. Called once for its own group and + # and once for ALL, so the TOTAL row is never a hand-added sum that can + # drift away from the parts. + function acct(g, verdict, berstr, berval, label) { + GN[g]++ + if (verdict == "PASS") GP[g]++ + else if (verdict == "FAIL") GF[g]++ + else GNA[g]++ + if (berval >= 0 && (!(g in GW) || berval > GW[g])) { + GW[g] = berval; GWS[g] = berstr; GWL[g] = label + } + } + function emit(g, lanes) { + lanes = GN[g] + 0 + printf "SUM %s %d %d %d %d %s %s\n", g, lanes, GP[g] + 0, GF[g] + 0, GNA[g] + 0, + (lanes > 0 && (g in GWS)) ? GWS[g] : "-", + (lanes > 0 && (g in GWL)) ? GWL[g] : "-" > SIDE + } BEGIN { LIM = MAXBER + 0 NC = 4 @@ -258,6 +349,18 @@ _ber_render() { HDR[3] = "BER"; AL[3] = 1 HDR[4] = "RESULT"; AL[4] = 1 worst = -1; worstlane = "-" + + # Expand the 100G list ("268" or "268,270-271") into a lookup, so a lane + # can be put in the right group by its port number alone. + np = split(P100, pp, /[ ,\t]+/) + for (i2 = 1; i2 <= np; i2++) { + if (pp[i2] == "") continue + if (split(pp[i2], rr, "-") == 2) { + for (j2 = rr[1] + 0; j2 <= rr[2] + 0; j2++) IS100[j2] = 1 + } else { + IS100[pp[i2] + 0] = 1 + } + } } # bcmcmd hands back CRLF. On the cd/traffic report the counter is followed by # a delta, so the CR got trimmed with the rest of the line; here the BER is @@ -282,6 +385,8 @@ _ber_render() { seen++ n++ PORT[n] = port; LANE[n] = lane; BERS[n] = ber + grp = ((port + 0) in IS100) ? "P100" : "P200" + GRP[n] = grp # a lane that reported something non-numeric has no usable measurement if (ber ~ /^[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$/) { v = ber + 0 @@ -290,7 +395,10 @@ _ber_render() { if (v > worst) { worst = v; worstlane = port "[" lane "]"; worststr = ber } } else { V[n] = "NA"; nna++ + v = -1 } + acct(grp, V[n], ber, v, port "[" lane "]") + acct("ALL", V[n], ber, v, port "[" lane "]") } END { if (seen == 0) { @@ -327,19 +435,32 @@ _ber_render() { } print "" - printf "# lanes measured : %d\n", n + printf "# lanes measured : %d (200G %d, 100G %d)\n", n, GN["P200"] + 0, GN["P100"] + 0 printf "# %s %d %s %d %s %d (criteria BER < %s)\n", \ vc("PASS"), npass + 0, vc("FAIL"), nfail + 0, vc("NA"), nna + 0, MAXBER if (worst >= 0) printf "# worst lane : %s %s\n", worstlane, worststr - if (SIDE != "" && SIDE != "/dev/null") - printf "SUM %d %d %d %d %s %s\n", n, npass + 0, nfail + 0, nna + 0, \ - (worst >= 0 ? worststr : "-"), worstlane > SIDE + if (GN["P100"] + 0 == 0) + printf "# %s the 100G port list (%s) matched no lane in this capture\n", vc("NA"), P100 + else + printf "# 100G p%s : %s %d %s %d %s %d worst %s %s\n", P100, + vc("PASS"), GP["P100"] + 0, vc("FAIL"), GF["P100"] + 0, + vc("NA"), GNA["P100"] + 0, + ("P100" in GWS) ? GWS["P100"] : "-", + ("P100" in GWL) ? GWL["P100"] : "-" + if (SIDE != "" && SIDE != "/dev/null") { + emit("P200"); emit("P100"); emit("ALL") + } exit (nfail || nna) ? 2 : 0 }' "$log" } -# _ber_summary -- one table for every unit, printed last +# _ber_summary -- one table for every unit, printed last. +# +# Each unit gets three rows: the 200G ports, the 100G uplink on its own, and a +# TOTAL. Unit 0 and unit 1 each have their own ce0(268), so the 100G row is per +# unit too -- a bad uplink on one board is what this fork exists to make +# visible, and averaged into 432 other lanes it would not be. _ber_summary() { local stamp="$1" unit side g r y n worst=0 g=$'\033[32m'; r=$'\033[31m'; y=$'\033[33m'; n=$'\033[0m' @@ -347,27 +468,41 @@ _ber_summary() { echo "" echo "===== summary : all units =====" - printf '%-8s %8s %8s %8s %8s %-12s %-10s %s\n' \ - "Unit" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT" + printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %s\n' \ + "Unit" "GROUP" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT" - local tag lanes pass fail na wber wlane v any=0 + local tag grp lanes pass fail na wber wlane v any=0 label for unit in $BER_UNITS; do - side="$BER_LOG_DIR/berside_unit${unit}_${stamp}" + side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}" [ -r "$side" ] || continue - read -r tag lanes pass fail na wber wlane < "$side" 2>/dev/null - [ "$tag" = "SUM" ] || continue - any=1 - if [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then - v="${r}FAIL${n}"; worst=2 - else - v="${g}PASS${n}" - fi - printf '%-8s %8s %8s %8s %8s %-12s %-10s %b\n' \ - "unit $unit" "$lanes" "$pass" "$fail" "$na" "$wber" "$wlane" "$v" + while read -r tag grp lanes pass fail na wber wlane; do + [ "$tag" = "SUM" ] || continue + any=1 + case "$grp" in + P200) label="200G ports" ;; + P100) label="100G p${BER_PORTS_100G}" ;; + ALL) label="TOTAL" ;; + *) label="$grp" ;; + esac + # A group with no lanes is not a pass. On the 100G row it means the + # uplink never reported -- exactly the case worth flagging. + if [ "${lanes:-0}" -eq 0 ]; then + v="${y}NO DATA${n}" + [ "$grp" = "P100" ] && worst=2 + elif [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then + v="${r}FAIL${n}"; worst=2 + else + v="${g}PASS${n}" + fi + printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %b\n' \ + "unit $unit" "$label" "$lanes" "$pass" "$fail" "$na" \ + "$wber" "$wlane" "$v" + done < "$side" done [ "$any" = "1" ] || echo " (no unit produced a result)" echo "" printf 'criteria: BER < %s\n' "$BER_MAX" + printf '100G group: port(s) %s -- one per unit\n' "$BER_PORTS_100G" echo "" return "$worst" } @@ -391,7 +526,7 @@ blanton_ber_report() { stamp=$(date +%Y%m%d_%H%M%S 2>/dev/null || echo now) for unit in $BER_UNITS; do log="$BER_LOG_DIR/prbsber_unit${unit}_${stamp}.log" - side="$BER_LOG_DIR/berside_unit${unit}_${stamp}" + side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}" : > "$side" if ! _ber_bcm "$unit" "prbsstat Ber" > "$log" 2>&1; then _ber_err "unit $unit: 'prbsstat Ber' failed (see $log)"; worst=1; continue @@ -409,22 +544,24 @@ blanton_ber_report() { _ber_summary "$stamp"; rc=$? [ "$rc" -gt "$worst" ] && worst=$rc - rm -f "$BER_LOG_DIR"/berside_unit*_"${stamp}" + rm -f "$BER_LOG_DIR"/bertside_unit*_"${stamp}" return "$worst" } # === Help ==================================================================== blanton_ber_help() { - echo -e "\033[1mblanton_ber.sh - SerDes BER test over every cabled port (PRBS via phydiag)\033[0m" + echo -e "\033[1mblanton_ber.sh - SerDes BER test: cabled 200G ports + the 100G uplink\033[0m" echo "" echo -e " Units : \033[36m$BER_UNITS\033[0m poly: \033[36m$BER_POLY\033[0m interval: \033[36m${BER_INTERVAL}s\033[0m" echo -e " Criteria : BER < \033[36m$BER_MAX\033[0m" - echo -e " Ports : \033[36m$BER_PORTS\033[0m" + echo -e " 200G : \033[36m$BER_PORTS_200G\033[0m" + echo -e " 100G : \033[36m$BER_PORTS_100G\033[0m (ce0, one per unit -- reported on its own row)" echo "" echo -e "\033[1mSubcommands\033[0m" echo -e " \033[33minit\033[0m prbs set $BER_POLY + prbs get (checks every lane locked)" echo -e " \033[33mstart\033[0m prbsstat STArt Interval=$BER_INTERVAL" echo -e " \033[33mreport\033[0m prbsstat Ber -> per-lane table + PASS/FAIL + summary" + echo -e " summary splits each unit into 200G ports / 100G p$BER_PORTS_100G / TOTAL" echo -e " \033[33mstop\033[0m prbsstat STOp" echo -e " \033[33mclear\033[0m prbs clear (disarms PRBS -- 'init' again before measuring)" echo -e " \033[33mhelp\033[0m" @@ -433,7 +570,7 @@ blanton_ber_help() { echo -e " \033[33m-u|--unit\033[0m <0|1|all> bcmcmd unit(s); 'all' = 0 1 (default: $BER_UNITS)" echo -e " \033[33m-c|--criteria\033[0m PASS threshold (default $BER_MAX)" echo -e " \033[33m-i|--interval\033[0m prbsstat Interval (default $BER_INTERVAL)" - echo -e " \033[33m-p|--ports\033[0m override the port list" + echo -e " \033[33m-p|--ports\033[0m override the whole port list (both groups)" echo -e " \033[33m-F|--fail-only\033[0m report: list only failing lanes" echo -e " \033[33m--tsv\033[0m report: tab-separated" echo -e " \033[33m--raw\033[0m report: also print the raw output"