Compare commits
13
Commits
b84188322d
..
V1.2.3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d3fb4f929 | ||
|
|
0ad76ea8b0 | ||
|
|
4ce4d6c62e | ||
|
|
b911ccd658 | ||
|
|
dd558a7b0f | ||
|
|
690be5db7d | ||
|
|
a5b7c4757b | ||
|
|
f5694345fb | ||
|
|
7a1967bc9a | ||
|
|
496e3ab650 | ||
|
|
29ec4c2239 | ||
|
|
d73b119dc3 | ||
|
|
79a3604ac4 |
+66
-19
@@ -136,6 +136,7 @@ Blanton_TTL_Script/
|
||||
│ ├── 100G_PRBS.txt # 100G PRBS 原始指令(port_prbs_monitor.sh 的來源)
|
||||
│ ├── TR518.txt # 內建封包測試 tr 518 → blanton_tr518.sh
|
||||
│ ├── BER_Test.txt # 全機 PRBS BER 指令 → blanton_ber.sh
|
||||
│ ├── 100G_dsc_cycle.txt # 100G tuning + link 循環 + DSC → bcm_100g_dsc_cycle.sh
|
||||
│ ├── Multiphase_{high3,low3,comboA,comboB,Save}.txt # → blanton_multiphase_margin.sh
|
||||
│ ├── Multiphase_Netname # 38 條軌的 net 名稱(區塊順序對應)
|
||||
│ ├── 100G_Traffic.txt # ce0 VLAN 138 打流 → blanton_traffic_linespeed.sh
|
||||
@@ -161,7 +162,7 @@ Blanton_TTL_Script/
|
||||
│
|
||||
├── 4_Blanton_Script_thermal_safety.ttl # 熱 / 安規:獨立跑,不接 A→B→C
|
||||
├── 5_Blanton_Script_Reboot_Cycle.ttl # 重開機循環:獨立跑,while 1 無止境(未上機驗證)
|
||||
├── config.ttl # 全域設定:testcase 名、EN_Margin / EN_log / EN_timestamp、
|
||||
├── config.ttl # 全域設定:testcase 名、EN_Margin / EN_TX_FIR / EN_log / EN_timestamp、
|
||||
│ # 風扇與 SWB 宣告、SSD/USB/BMC_USB 壓力秒數、三個 prompt 字串
|
||||
├── hammer/ # 🚫 gitignored — mlucas ELF binary(~99 MB),DUT image 自帶,
|
||||
│ # publish.sh 也排除;Script B / Script 4 只負責 chmod +x 後執行
|
||||
@@ -173,6 +174,7 @@ Blanton_TTL_Script/
|
||||
│ ├── pcie_bus.ttl # lspci -tvvv / -vv
|
||||
│ ├── setup_pmon.ttl # show platform summary/fan/temperature/psustatus/voltage/
|
||||
│ │ # current/ssdhealth/leak status/leak channels + TPM(11 項)
|
||||
│ │ # V1.2.3 起 + ddr5_temp.sh + hsc_temp_readback.sh
|
||||
│ ├── show_dmesg.ttl # dmesg -T 合併 error 正則 + i2c 過濾 + dmesg -C 清空
|
||||
│ ├── show_margin_status.ttl # source margin_status_all.sh(掃 9 顆 LTC2980)
|
||||
│ ├── kill_all_process.ttl # kill $(jobs -p)
|
||||
@@ -199,7 +201,13 @@ Blanton_TTL_Script/
|
||||
├── mgmt_ping_monitor.sh # 10G/1G 管理網路 ping 監控(iproute2)
|
||||
│ # eth0→TARGET_10G、eth1→TARGET_1G,兩張卡同時持續 ping
|
||||
├── port_prbs_monitor.sh # 100G uplink PRBS 測試(bcmcmd phy diag)
|
||||
│ # ⚠️ V1.1.9 起 A/B/C 內註解掉的呼叫已整段刪除,只剩手動執行
|
||||
├── blanton_ber.sh # 全機 SerDes BER 掃描(phydiag prbs / prbsstat)
|
||||
│ # V1.2.2 起含 port 268(100G ce0),summary 每 unit 分 200G/100G/TOTAL
|
||||
├── bcm_100g_dsc_cycle.sh # 100G link down/up 循環 + 每輪 DSC 與 retimer TX/RX 讀回(未驗證)
|
||||
├── bcm_dsc_scan.sh # 全機 SerDes DSC dump(phydiag <port> dsc,~445 port × 2 unit)
|
||||
│ # 由 EN_TX_FIR 控制,A 與 C 各掃一次;單次約 10 分鐘
|
||||
├── bcm_dsc_parse.sh # DSC raw log → <prefix>_lane.csv / _pm.csv(純 sh + awk)
|
||||
├── blanton_multiphase_margin.sh # 多相 VRM margin(原生 i2c,38 條軌)
|
||||
├── pwr_brick.sh # PDB / CPB power brick 讀值(VI2C ch4)
|
||||
├── nfc_polling.sh # NFC tag 背景輪詢 + PASS/FAIL 統計
|
||||
@@ -208,9 +216,12 @@ Blanton_TTL_Script/
|
||||
├── TH6_SWB0_power_readback.sh # SWB0 19 條軌 V×I + 總功耗
|
||||
├── TH6_SWB1_power_readback.sh # SWB1 同上
|
||||
│ # ⚠️ 已納入交付,但 A/B/C 的呼叫仍註解掉(bring-up 中)
|
||||
├── ddr5_temp.sh # DDR5 DIMM 溫度(SPD5118 @ i2c-5 0x50/0x51,reg 0x31)
|
||||
├── hsc_temp_readback.sh # LTC4287 HSC 溫度(i2c-28 0x40/0x45,內/外部感溫)
|
||||
├── avt_pmpower.sh # AMD CPU 封裝功耗(AVTCMD pmm get_pmdata;AVT 是 NDA,不隨包)
|
||||
├── usb_target.sh # 偵測 USB 裝置節點/掛載點(排除系統碟)
|
||||
├── bmc_monitor.sh # BMC 監控:free -m + I2C 寫入/讀回圖樣測試
|
||||
├── bmc_monitor_ddr.sh # ⚠️ 已停用:B/C 內呼叫處已註解,DDR 壓力改走 bgctl + bmc-manager
|
||||
├── bmc_monitor_ddr.sh # ⚠️ 已停用:V1.1.9 起 B/C 內的註解區塊已刪除,DDR 壓力走 bgctl + bmc-manager
|
||||
│ # ↑ 三支都要 chmod +x 才能跑(見 Key Constraints)
|
||||
└── LTC2980_Margin_Script/ # 電壓 margin 子專案(另有獨立 repo)
|
||||
├── margin.sh # margin_init/status/set/apply_profile/save
|
||||
@@ -370,10 +381,10 @@ PROFILE_CHANNELS=( "0:high:8" "1:low:8" ... ) # "channel:operation:change_perc
|
||||
|
||||
| 模組 | 進入點 | 功能 |
|
||||
|------|--------|------|
|
||||
| **Script A(Pre-test)** | `1_Blanton_Script_A.ttl` V1.1.8 | root 登入 → `date -s` 對時 → 清 job log → **收掉前一輪殘留**(`hw-test-session finish` → `bgctl reset --yes` → `bgctl stop --all` → `bgctl list` 確認清空)→ `hw-test-session start` → **設定風扇轉速** → **等資料面就緒**(`wait_init.ttl`)→ DUT 清單(boot/version/fwutil/syseeprom/ssdhealth/TPM/NVMe)→ BMC enroll + version/status → source bash 工具 → 清/讀 7 組 PCIe AER + `ras-mc-ctl --summary` → `lspci` → margin → PMON → dmesg → **10G/1G 管理網路 ping** → **NFC 基線**(start → 30s → stop → report)→ 100G uplink 設定與狀態 → **一輪流量基線** → `show uptime` |
|
||||
| **Script B(Stress)** | `2_Blanton_Script_B.ttl` | **V1.1.8 起 `mlucas-avx2 -cpu 0:15` 由 `systemd-run --unit=mlucas-amm`(`Type=exec`、`Restart=always`)啟動,不再是 shell 的 `&` 背景 job** + `bgctl run` 的 `memtester` / SSD / USB 壓力(v1.1.7 起 SSD/USB 的 `--runtime`/`--passes` 讀 `config.ttl` 的 `SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT`;BMC memtester 不帶次數 = 跑到被停) + BMC DDR(`bmc-manager run memtester`)→ BMC USB net test(探測 cdc_ncm 介面,`systemd-run` 掛 `BMC_USB_RUNTIME` 秒的 ping,V1.1.8 前寫死 4 小時)→ 啟動 mgmt ping 監控 → **啟動 NFC 輪詢** → `jobs` / `systemctl is-active mlucas-amm` / `bgctl list` → traffic(依 `SWB_UNIT`)→ `while 1` 每輪 PMON + `bgctl list`/`jobs`/`systemctl is-active mlucas-amm` + `pause 60`(**V1.1.4 起 margin 不在迴圈內**)|
|
||||
| **Script C(Post-test)** | `3_Blanton_Script_C.ttl` | `kill $(jobs -p)` + **`systemctl stop mlucas-amm`**(systemd unit 不是 shell job,`jobs -p` 抓不到)+ `bgctl stop --all` / `reset` + 停 BMC USB unit、mgmt ping 與 **NFC 輪詢** → PMON + dmesg → 7 組 AER + `ras-mc-ctl` → BMC USB `journalctl` **寫進 `log/bmc_usb_net.log`**(不再印在 console) → `ip -s link show eth0/eth1` → `cat` mlucas log(並 `cp` 進 job 目錄一起帶走)→ **NFC `report`** → traffic `stop` + `report` → **BER 掃描**(依 `SWB_UNIT`)→ NVMe 健康 → `show reboot-cause` → **margin 掃描**(`EN_Margin`,V1.1.4 由 Script B 移來)→ `uptime` → **job log 複製到 USB(帶時戳)** → `exit` + wait『Script done』→ `hw-test-session finish` |
|
||||
| **Script 4(Thermal / Safety)** | `4_Blanton_Script_thermal_safety.ttl` | **獨立腳本,不接 A→B→C。** Script A 式的環境建立(登入 / session / 風扇 / `wait_init` / DUT 清單 / BMC / PCIe 清除 / `pcie_bus` / PMON / dmesg)→ **滿載**(`mlucas-avx2 -cpu 0:15`,V1.1.8 起與 Script B 同樣掛成 `mlucas-amm` systemd unit + `blanton_tr518.sh start`)→ `while 1`:PMON + `TH6_SWB0/1_power_readback.sh` + `bgctl list` / `jobs` / `systemctl is-active mlucas-amm`,`pause 60`。**沒有收尾段** —— 中斷後 DUT 仍在 `port cd lb=mac` / `l2 learn off` / `test mode nr=yes`,且 CPU 仍滿載,要手動 `blanton_tr518.sh stop` + `systemctl stop mlucas-amm` |
|
||||
| **Script A(Pre-test)** | `1_Blanton_Script_A.ttl` V1.2.3 | root 登入 → `date -s` 對時 → 清 job log → **收掉前一輪殘留**(`hw-test-session finish` → `bgctl reset --yes` → `bgctl stop --all` → `bgctl list` 確認清空)→ `hw-test-session start` → **設定風扇轉速** → **等資料面就緒**(`wait_init.ttl`)→ DUT 清單(boot/version/fwutil/syseeprom/ssdhealth/TPM/NVMe)→ BMC enroll + version/status → source bash 工具 → 清/讀 7 組 PCIe AER + `ras-mc-ctl --summary` → `lspci` → margin → PMON → dmesg → **10G/1G 管理網路 ping** → **NFC 基線**(start → 30s → stop → report)→ 100G uplink 設定與狀態 → **BER 基線掃描**(依 `SWB_UNIT`,V1.2.1 新增)→ **一輪流量基線** → **SerDes DSC 基線**(`EN_TX_FIR`)→ `show uptime` |
|
||||
| **Script B(Stress)** | `2_Blanton_Script_B.ttl` | **V1.1.8 起 `mlucas-avx2 -cpu 0:15` 由 `systemd-run --unit=mlucas-amm`(`Type=exec`、`Restart=always`)啟動,不再是 shell 的 `&` 背景 job** + `bgctl run` 的 `memtester` / SSD / USB 壓力(v1.1.7 起 SSD/USB 的 `--runtime`/`--passes` 讀 `config.ttl` 的 `SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT`;BMC memtester 不帶次數 = 跑到被停) + BMC DDR(`bmc-manager run memtester`)→ BMC USB net test(探測 cdc_ncm 介面,`systemd-run` 掛 `BMC_USB_RUNTIME` 秒的 ping,V1.1.8 前寫死 4 小時)→ 啟動 mgmt ping 監控 → **啟動 NFC 輪詢** → `jobs` / `systemctl is-active mlucas-amm` / `bgctl list` → traffic(依 `SWB_UNIT`)→ `while 1` 每輪 PMON + `bgctl list`/`jobs`/`systemctl is-active mlucas-amm` + `sleep 60`(**V1.1.4 起 margin 不在迴圈內**;V1.2.0 起等待走 DUT 端 `sleep` 而非 macro 端 `pause`)|
|
||||
| **Script C(Post-test)** | `3_Blanton_Script_C.ttl` | `kill $(jobs -p)` + **`systemctl stop mlucas-amm`**(systemd unit 不是 shell job,`jobs -p` 抓不到)+ `bgctl stop --all` / `reset` + 停 BMC USB unit、mgmt ping 與 **NFC 輪詢** → PMON + dmesg → 7 組 AER + `ras-mc-ctl` → BMC USB `journalctl` **寫進 `log/bmc_usb_net.log`**(不再印在 console) → `ip -s link show eth0/eth1` → `cat` mlucas log(並 `cp` 進 job 目錄一起帶走)→ **NFC `report`** → traffic `stop` + `report` → **SerDes DSC 收尾**(`EN_TX_FIR`)→ **BER 掃描**(依 `SWB_UNIT`)→ NVMe 健康 → `show reboot-cause` → **margin 掃描**(`EN_Margin`,V1.1.4 由 Script B 移來)→ `uptime` → **job log 複製到 USB(帶時戳)** → `exit` + wait『Script done』→ `hw-test-session finish` |
|
||||
| **Script 4(Thermal / Safety)** | `4_Blanton_Script_thermal_safety.ttl` | **獨立腳本,不接 A→B→C。** Script A 式的環境建立(登入 / session / 風扇 / `wait_init` / DUT 清單 / BMC / PCIe 清除 / `pcie_bus` / PMON / dmesg)→ **滿載**(`mlucas-avx2 -cpu 0:15`,V1.1.8 起與 Script B 同樣掛成 `mlucas-amm` systemd unit + `blanton_tr518.sh start`)→ `while 1`:PMON + `TH6_SWB0/1_power_readback.sh` + `avt_pmpower.sh`(CPU 封裝功耗,V1.2.3)+ `bgctl list` / `jobs` / `systemctl is-active mlucas-amm`,`pause 60`。**沒有收尾段** —— 中斷後 DUT 仍在 `port cd lb=mac` / `l2 learn off` / `test mode nr=yes`,且 CPU 仍滿載,要手動 `blanton_tr518.sh stop` + `systemctl stop mlucas-amm` |
|
||||
| **Script 5(Reboot Cycle)** | `5_Blanton_Script_Reboot_Cycle.ttl` | **獨立腳本,不接 A→B→C。** `while 1`:登入 → `sudo -i` → 對時 → 風扇 → **Check Pt1~Pt13**(DUT/BMC info、mgmt ping、NFC、SSD、USB、BMC USB、port status + 15s traffic、BER、NVMe、platform data、dmesg i2c、PCIe error counters)→ `show uptime` → `reboot` → `wait "Booting in blind mode"`。`cnt_cycle` 每輪 +1 印在標頭。**沒有結束條件** —— 要停就關掉 Tera Term 的 macro。**尚未上機驗證** |
|
||||
| **SWB loopback 流量** | `blanton_traffic_linespeed <sub> [-u 0\|1\|all]` | `init`(VLAN 30..137,成對 `cdN`/`cdN+32`)→ `clear` → `start`(`tx 200 length=324 PatternRandom=yes`,v0.7.0 起)→ `stop` → `report`(per-pair TX/RX 交叉比對 PASS/FAIL);另有 `ps`(link/speed)與 `run`(一條龍)。**V0.6.0 起含 100G uplink `ce0`**(VLAN 138):它沒有 loopback 對象,靠 **ingress mirror 打回自己**維持循環,`report` 最後一列就是它,底下再接一份跨 unit summary |
|
||||
| **FPGA 暫存器存取** | `cb_fpga <fn> <addr> [data]`、`fpga_rescan` | pcimem 打 `resource0` + BAR 相對 offset,32-bit word;BDF 於 source 時自動偵測;`pmc/icb/swb0/swb1_fpga` 走 VSPI 窗口 |
|
||||
@@ -386,9 +397,11 @@ PROFILE_CHANNELS=( "0:high:8" "1:low:8" ... ) # "channel:operation:change_perc
|
||||
| **Margin 寫 NVM** | `margin_save` / `margin_save all` / `margin_save blanton` | ⚠️ 永久寫入。`0x15` 送 PMBus **send-byte**(不帶 data byte)。`all` = 依 `settings/` 走 9 顆、跑完還原原本載入的板子;`blanton` = 本平台捷徑,18 顆全走原生 i2c bus(`MARGIN_BLANTON_STORE` 表),不需 `margin_init`。每筆之間 `MARGIN_STORE_SETTLE`(0.5s),工具**不 poll busy bit** |
|
||||
| **管理網路 ping** | `mgmt_ping_monitor.sh {start\|stop\|status\|fg\|tail\|clear\|summary}` | 一輪 = 10G leg + 1G leg,各自 `ip address/route replace` 後 `ping -I <if>` 打自己的對端;兩張卡都保持 up。每 leg 記 `ip -s link show` 與該卡的 TX/RX packet delta,另出一行可 grep 的 `RESULT` |
|
||||
| **BMC 監控** | `bmc_monitor.sh {start\|stop\|status\|fg\|tail\|clear}` | 由 host 端 `bmc-manager run` 週期取樣 `free -m`,另加一組 **I2C 寫入/讀回圖樣測試**(bus 1、addr 0x41、offset 0x00C0,寫 `55AA55AA` 讀回、再寫 `AA55AA55` 讀回)。B 啟動、C 停止並 `cat log/bmc_poll.log` |
|
||||
| **BMC DDR 壓力** | `bmc_monitor_ddr.sh` | ⚠️ **已停用** —— B/C 內呼叫處已註解,改由 `bgctl run bmc-manager run memtester` 取代。檔案保留未刪 |
|
||||
| **100G PRBS** | `port_prbs_monitor.sh {start [both\|a\|b]\|stop\|status\|report\|clear}` | 兩個 uplink 各一個 worker,可同時或單獨跑。setup(`lpmode off` → `prbs set` → `prbsstat STArt`)→ 週期 `prbs get`(判定)+ `prbsstat Ber`(只記錄)→ teardown(`STOp` + `prbs clear`)。輸出含 `PRBS OK!` 才算 PASS。**⚠️ A/B/C 的呼叫目前註解掉** |
|
||||
| **SerDes BER 掃描** | `blanton_ber.sh {init\|clear\|start\|stop\|report} [-u 0\|1\|all]` | 全機每個對接 port 的 PRBS BER:`prbs set p=3` + `prbs get`(確認每條 lane 都鎖上)→ `prbsstat STArt Interval=30` →(等一個 interval)→ `prbsstat Ber` → `STOp` → `prbs clear`。`report` 一列一條 lane,**BER < 1e-6 才 PASS**,最後跨 unit 的 summary 帶最差 lane。⚠️ PRBS 期間 link 顯示 DOWN、不過流量 |
|
||||
| **BMC DDR 壓力** | `bmc_monitor_ddr.sh` | ⚠️ **已停用** —— V1.1.9 起 B/C 內的註解區塊已刪除,改由 `bgctl run bmc-manager run memtester` 取代。檔案保留未刪 |
|
||||
| **SerDes DSC 擷取** | `bcm_dsc_scan.sh [-u 0\|1] [-f <name>] [-F] [-n] [-C]` + `bcm_dsc_parse.sh -i <log> -o <prefix>` | `bcmcmd -n <unit> -c "dsh -c 'phydiag <port> dsc'"` 掃全清單(1 個 100G + 445 個 200G port)× 2 unit,raw log 寫進 `<script_dir>/log` 再 `cp` 到 `/host/hw-eval/jobs`。`-f` 支援 `%u` / `%t`;沒帶 `%u` 時自動插 `_u0`/`_u1`。成功與否**不看 `bcmcmd` 的 exit code**,改比對 `SERDES DISPLAY DIAG DATA END` 數量 + 實際迭代筆數,結尾印 `### DSC_SCAN_DONE rc=<n>`。parse 端輸出 `_lane.csv`(每 lane:TXEQ/RXFFE/DFE/EYE/SNR)與 `_pm.csv`(每 pm_id:溫度/PLL/lane 數/統計)。**由 `EN_TX_FIR` 控制,預設關** |
|
||||
| **100G PRBS** | `port_prbs_monitor.sh {start [both\|a\|b]\|stop\|status\|report\|clear}` | 兩個 uplink 各一個 worker,可同時或單獨跑。setup(`lpmode off` → `prbs set` → `prbsstat STArt`)→ 週期 `prbs get`(判定)+ `prbsstat Ber`(只記錄)→ teardown(`STOp` + `prbs clear`)。輸出含 `PRBS OK!` 才算 PASS。**⚠️ V1.1.9 起 A/B/C 內的註解呼叫已刪除,只能手動跑** |
|
||||
| **SerDes BER 掃描** | `blanton_ber.sh {init\|clear\|start\|stop\|report} [-u 0\|1\|all]` | 全機每個對接 port 的 PRBS BER:`prbs set p=3` + `prbs get`(確認每條 lane 都鎖上)→ `prbsstat STArt Interval=30` →(等一個 interval)→ `prbsstat Ber` → `STOp` → `prbs clear`。`report` 一列一條 lane,**BER < 1e-6 才 PASS**,最後跨 unit 的 summary 帶最差 lane。⚠️ PRBS 期間 link 顯示 DOWN、不過流量。V1.2.1 起 Script A 與 Script C 各跑一次(A 在流量之前、C 在流量報表之後),夾住 soak。**V1.2.2(腳本 v1.1.0)起清單納入 port 268(100G uplink ce0)**,summary 每個 unit 拆成 200G ports / 100G p268 / TOTAL 三列;某組零 lane 算 FAIL |
|
||||
| **100G link 循環 + DSC** | `bcm_100g_dsc_cycle.sh <cycles> [-d] [-T] [-P] [-n]` | tuning(`docker pause gbsyncd*` → `serdes-tx-set sys 0..7` → 每 lane `tx_taps set`)一次,之後每輪:`port 268 en=0` → 輪詢 `bcmcmd ps 268` 等 `!ena` → `en=1` → 等 `up` → 兩個 unit 的 `phydiag 268 dsc` + `serdes-tx-get`/`serdes-rx-get sys 0..7`。log 一次執行一份(`~/dsc_log/`),每輪一段 `### TEST CYCLE n/N`,結尾 `### DSC_CYCLE_DONE rc=<n>`。**全 ASCII 輸出**(DUT console 會把 UTF-8 弄成亂碼)。⚠️ 尚未上機驗證 |
|
||||
| **TR518 負載** | `blanton_tr518.sh {start\|stop} [-u 0\|1\|all]` | `port cd lb=mac` → `l2 learn off` → `test mode nr=yes` → `tr 518 ... testphase=1`,然後 settle `TR_SETTLE_SEC`(120s)再讓人量功耗。`stop` 只送 `port cd lb=none`(照 `TR518.txt`),L2 學習與 test mode **不會還原**,`TR_RESTORE_*=1` 才會 |
|
||||
| **PDB / CPB brick** | `pwr_brick.sh show [-r]` | 5 顆 brick 的 Vin / Iin / Vout / Iout / Temp(SWB0 `0x62`/`0x63`、SWB1 `0x60`/`0x61`、CPB `0x26`)。四顆 ADPM12200 **每個量各自一個 PMBus PAGE**(Vin 9、Iin 10、Vout 2、Iout 14、Temp 18),讀之前一定要先寫 PAGE —— `blanton_pwr_data.sh` 完全沒寫,所以整張 PDB 表都是 NA。換算用手冊 DIRECT 公式 `X=(1/m)(Y×10⁻ᴿ−b)`。每個值重讀到通過三道檢查為止(非 `0xFFFF`、非 `0x0000`、落在合理範圍)—— 這條匯流排**只壞高位元組**,`11.98 V` 會變成 `32.00 V`,全一過濾擋不住 |
|
||||
| **SWB 軌功耗** | `TH6_SWB{0,1}_power_readback.sh` | 讀 `show platform voltage/current` **各一次**,19 條軌 V×I 與總功耗。舊版每軌各讀一次(38 次呼叫、每軌不同時刻),總和是不同瞬間的加總 |
|
||||
@@ -510,6 +523,11 @@ pwr_data
|
||||
`vlan create` 的 `Entry exists`(重跑 init)與 `vlan remove` 的 `not found`(init 前先 stop)
|
||||
當良性情況分開計數。另外 `bcmcmd` 一律以 `</dev/null` 執行 ——
|
||||
否則它會吃掉 while 迴圈的 stdin,導致**只有第一個 VLAN 被設定**(V0.3.1 修的 bug)。
|
||||
**2026-09-07 `bcm_dsc_scan.sh` 又踩了同一個坑**(port 清單走 stdin,445 個 port 只掃了 1 個),
|
||||
而且因為 `### PORT` marker 與 dsc 區塊數量一起變少,原本的比對檢查完全看不出來、`rc=0`。
|
||||
修法:迴圈改讀 fd 3(`done 3< "$PORTFILE"`)+ `bcmcmd ... < /dev/null`,再加一道
|
||||
「實際迭代筆數 == 清單筆數」的獨立檢查。**寫任何餵檔案給 `while` 又在裡面呼叫 `bcmcmd` 的迴圈,
|
||||
都要假設它會讀 stdin;而且完整性檢查不能只比對兩個會一起變少的數字。**
|
||||
11. **DUT 密碼不得進 git**:`1_Blanton_Script_A.ttl` 目前的 `sendln "YourPaSsWoRd"` 是 placeholder,
|
||||
實際帳密必須放 `secret/config_secret.ttl`(gitignored)並由 `config.ttl` include。
|
||||
12. **`.sh` / `.conf` 必須是 LF**:CRLF 的 shell script 在 SONiC 上直接掛
|
||||
@@ -549,8 +567,15 @@ pwr_data
|
||||
⚠️ **`4_Blanton_Script_thermal_safety.ttl` 用同一個 `mlucas-amm` unit,但它沒有收尾段**,
|
||||
所以那行 `systemctl stop mlucas-amm` 必須由測試員手動下 —— 這是 Script 4 目前
|
||||
唯一一個「不下就會延續到下一輪」的負載。
|
||||
18. **`config.ttl` 少一個變數會讓 macro 當場中止**,不是當成 0。V1.1.8 新增 `EN_timestamp`
|
||||
(Script A `logopen` 的第 5 個參數)與 `BMC_USB_RUNTIME`(Script B 的 ping unit)。
|
||||
18. **`pause` 停的是 macro,`sleep` 停的是 DUT。** V1.2.0 起 A/B/C 的等待一律改成
|
||||
`sendln "sleep N"` + `wait prompt_sonic_root`。`pause N` 期間 macro 不理會 DUT,shell 是閒著的,
|
||||
等於用猜的估上一個指令要跑多久 —— DUT 一忙就會提早往下送指令;改成 DUT 端 `sleep` 之後,
|
||||
prompt 回來才繼續,兩邊不可能走散,而且這段等待會帶時戳進 log,事後看得出哪一輪跑久了。
|
||||
⚠️ **Script 4 與 Script 5 尚未轉換,仍在用 `pause`**;B/C 開頭的 `pause 1` 也保留
|
||||
(它在登入前執行,沒有 prompt 可等)。
|
||||
19. **`config.ttl` 少一個變數會讓 macro 當場中止**,不是當成 0。V1.1.8 新增 `EN_timestamp`
|
||||
(Script A `logopen` 的第 5 個參數)與 `BMC_USB_RUNTIME`(Script B 的 ping unit);
|
||||
V1.2.0 再加 `EN_TX_FIR`(A/C 的 DSC 掃描與 C 的四行 `cp`,預設 0)。
|
||||
只更新 `.ttl` 卻沿用舊的 `config.ttl`,Script A 會停在 `logopen` 那一行。
|
||||
另外 **時間戳的「格式」不在 macro 裡** —— 它在 Tera Term 的
|
||||
Setup → Additional settings → Log,存在那台電腦的 `TERATERM.INI`,會跟著 PC 而不是跟著腳本走。
|
||||
@@ -787,7 +812,7 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
- [ ] `CHANGELOG.md`:延續三支腳本頂端的 Version History,集中成一份
|
||||
- [ ] 三支腳本的 header 版號與 `CHANGELOG.md` 對齊,`config.ttl` 加 `script_version` 並寫進 log 開頭
|
||||
- [ ] `docs/TTL_Script_Blanton_Status_*.xlsx` 更新為全 OK,或把剩餘項目移進 Future Extensions
|
||||
- [x] 打 git tag,push `nas` + `gitea`(已至 `V1.1.8`;`publish/` 依 Script A 檔頭版號打包)
|
||||
- [x] 打 git tag,push `nas` + `gitea`(已至 `V1.2.3`;`publish/` 依 Script A 檔頭版號打包)
|
||||
|
||||
**驗收條件:** 從 zip 解壓到一台乾淨的測試 NB,照 README 操作,不看原始碼就能完成一輪 A→B→C
|
||||
並產出 summary;`publish/` 內容與 git tag 一致。
|
||||
@@ -821,14 +846,23 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
Script A 既不往下走也不報錯。可加最大重試次數 + `messagebox` 提示。
|
||||
- **腳本的執行權**:`mgmt_ping_monitor.sh` 與 `bmc_monitor.sh`(以及已停用的 `bmc_monitor_ddr.sh`)需人工 `chmod +x`。
|
||||
可在 Script B 起它之前補一行(比照既有的 `chmod +x ~/hammer/tools/amd/mlucas-avx2`),或在 publish 時處理。
|
||||
- **`bmc_monitor_ddr.sh` 已停用**:B/C 內的呼叫已註解,DDR 壓力改由 `bgctl run bmc-manager run memtester`
|
||||
取代。檔案還留在 repo,待確認不再需要後刪除,或在檔頭標註停用。
|
||||
- **`bmc_monitor_ddr.sh` 已停用**:V1.1.9 起 B/C 內那段註解掉的呼叫已刪除,DDR 壓力由
|
||||
`bgctl run bmc-manager run memtester` 取代。檔案還留在 repo,待確認不再需要後刪除,或在檔頭標註停用。
|
||||
(`bmc_monitor.sh` 已於 2026-08-25 重新啟用,並加上 I2C 寫入/讀回圖樣測試。)
|
||||
- **`rm /host/hw-eval/jobs/*` 清不掉子目錄**:Script A 用的是不帶 `-r` 的 `rm`,但 Script C
|
||||
是 `cp -r` 那個目錄,代表裡面可能有子目錄。清不乾淨的話,上一輪的 job log 會被當成這輪的一起
|
||||
複製到 USB。要確認 `bgctl` 的產出結構,必要時改成 `rm -rf .../jobs/*`。
|
||||
- **100G PRBS 尚未接進 A/B/C**:`port_prbs_monitor.sh` 已隨包交付、可手動執行,但三支腳本裡的呼叫
|
||||
都還註解著(bring-up 中)。要啟用時記得 PRBS 期間 link 會顯示 down,`stop` 之後才會回來。
|
||||
- **100G PRBS 已從 A/B/C 移除**:`port_prbs_monitor.sh` 仍隨包交付、仍可手動執行,但 V1.1.9 起
|
||||
三支腳本裡那些註解掉的呼叫已整段刪除(連同 Script C 的 `bmc_monitor_ddr.sh` 區塊)。
|
||||
要接回去得重寫,不是把註解拿掉。啟用時記得 PRBS 期間 link 會顯示 down,`stop` 之後才會回來。
|
||||
- **DSC 擷取預設是關的**:`EN_TX_FIR = 0`,所以出貨設定不會跑 `bcm_dsc_scan.sh`。
|
||||
開啟後 Script A 與 Script C 各掃一次、整輪多約 20 分鐘;只要單邊資料可以在腳本裡改成 `-u 0`。
|
||||
掃描本身是唯讀(`phydiag ... dsc` 不改 SerDes 設定),但**它與 `blanton_ber.sh` /
|
||||
`blanton_traffic_linespeed` / `blanton_tr518.sh` 一樣都在動同一批 port,不要同時跑**。
|
||||
- **`bcm_100g_dsc_cycle.sh` 尚未上機驗證**:`port en=0/1` 之後 link 多久會動、以及
|
||||
`docker pause gbsyncd*` 期間的影響,都要上機才知道。等待是讀 SDK(`bcmcmd ps <port>`)而不是
|
||||
`show interfaces status`,所以理論上不受 gbsyncd 暫停影響,但沒實測過。
|
||||
結束時(含 Ctrl-C)會自動 `docker unpause`,`RESUME_GBSYNCD=0` 可關掉。
|
||||
- **Script 5(Reboot Cycle)尚未上機驗證**:`5_Blanton_Script_Reboot_Cycle.ttl` 隨 V1.1.7 交付,
|
||||
但還沒在真 DUT 上跑過一輪完整循環。兩個要現場確認的點:`wait "Booting in blind mode"` 是否
|
||||
真的在每一版開機訊息裡出現(沒出現就會永遠等下去,而且沒有 `timeout` 保護);以及每輪重跑
|
||||
@@ -892,9 +926,22 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
- **margin 在 soak 期間不再取樣**(V1.1.4):只剩 Script A 的起始值與 Script C 的結束值。
|
||||
漂移發生在中間就看不到了。若要恢復,比較好的做法是在 B 的迴圈內每 N 輪才掃一次,
|
||||
而不是每輪都掃。
|
||||
- **`blanton_ber.sh` 的 `sleep 30` 與 `BER_INTERVAL` 是兩處要同步改的數字**:Script C 寫死
|
||||
`sendln "sleep 30"`,腳本預設 `BER_INTERVAL=30`。改一邊沒改另一邊,report 會讀到還沒累積完
|
||||
一個 interval 的數字,而且不會有任何警告。
|
||||
- ✅ **`blanton_ber.sh` 不再把 lane 的量測結果當成 shell 錯誤**(v1.1.1,2026-09-15):
|
||||
`prbs get` 每條 lane 一行,沒鎖上的會印 `PRBS Error!`,而錯誤樣式是拿
|
||||
`grep -i 'ERROR|Invalid|...'` 掃整份輸出的 —— 於是正常的 init 會報「the BCM shell rejected」,
|
||||
而且提早 return,**把「哪幾條 lane 沒鎖」這個真正有用的資訊吞掉**。現在 per-lane 行排除在
|
||||
掃描外、改成大小寫敏感、真正的拒絕會把該行印出來。
|
||||
⚠️ **這類「完整性檢查自己看不到失敗」的錯在這個 repo 已經出現三次**
|
||||
(`bcm_dsc_scan.sh` v1.3.0 數 marker、`blanton_ber.sh` v1.1.1、以及 power cycle 報告的
|
||||
ping 解析把 100% 丟包當成「沒資料」)。寫檢查時要問:**失敗長什麼樣,這段程式碼看得到嗎?**
|
||||
- **`blanton_ber.sh` 的 `sleep 30` 與 `BER_INTERVAL` 是兩處要同步改的數字**:Script A(V1.2.1 起)
|
||||
與 Script C 都寫死 `sendln "sleep 30"`,腳本預設 `BER_INTERVAL=30`。改一邊沒改另一邊,report 會
|
||||
讀到還沒累積完一個 interval 的數字,而且不會有任何警告。現在有**三處**要一起改。
|
||||
- **Script A 的 BER `clear` 與第一輪流量之間沒有 settle**:V1.2.1 把 BER 掃描放在流量**之前**,
|
||||
而 PRBS 是把 lane 拉出正常運作、`clear` 才放回去的。Script A 的 `blanton_ber.sh clear` 之後
|
||||
緊接著就是 `blanton_traffic_linespeed ps` / `init`,中間沒有等待;Script C 的 BER 在流量報表
|
||||
之後,沒有這個順序。上機時比對兩份 traffic report —— Script A 有 NA/FAIL 而 Script C 沒有,
|
||||
就是 lane 尚未回穩,在 `clear` 後加一個 `sleep` 即可。**尚未上機驗證。**
|
||||
- **`ce0` 的 ingress mirror 尚未上機驗證**:`mirror port ce0 Mode=Ingress DestPort=ce0` 這條路是
|
||||
推論出來的(鏡像複本不受 egress filter 的 source-port knockout 限制),語法取自 DUT 的
|
||||
`help mirror`,但**還沒證明硬體真的接受 destport == srcport**。驗證方式是隔 10 秒看兩次
|
||||
|
||||
@@ -9,30 +9,38 @@ DUT 端的取數能力由一組 **bash 工具庫** 提供(CB FPGA `pcimem` →
|
||||
|
||||
完整架構見 [ARCHITECTURE.md](ARCHITECTURE.md)。
|
||||
|
||||
**目前狀態(2026-09-06)**:Script A 為 **V1.1.8**(A 集中記錄所有 TTL 的變更)。涵蓋
|
||||
**目前狀態(2026-09-23)**:Script A 為 **V1.2.3**(A 集中記錄所有 TTL 的變更)。涵蓋
|
||||
PCIe AER / DDR EDAC / PMON+TPM / 電壓 margin(9 顆 LTC2980)/ SWB loopback 線速流量 /
|
||||
10G+1G 管理網路 ping / BMC DDR 與 USB / NVMe 健康,壓力層走平台的 `bgctl`。
|
||||
switch unit 數量由 `config.ttl` 的 `SWB_UNIT0` / `SWB_UNIT1` 宣告,風扇轉速由 `FAN_SPEED` 指定
|
||||
(預設 100 = 全速;soak 是熱測試,散熱不該是變因)。
|
||||
測試 job log 收在 DUT 的 `/host/hw-eval/jobs/`,Script C 結束時掛好 USB 再帶時戳複製到 `/mnt/usb/`。
|
||||
100G PRBS(`port_prbs_monitor.sh`)已隨包交付,但 **A/B/C 的呼叫仍註解掉**,還在 bring-up。
|
||||
Script B 的 soak 迴圈每輪印 `bgctl list` + `jobs` + `systemctl is-active mlucas-amm`,並 `pause 60`
|
||||
(V1.0.10 之前完全沒有 pause)。V1.1.8 起 AMD CPU 壓力(`mlucas-avx2`)由 `systemd-run` 掛成
|
||||
100G PRBS(`port_prbs_monitor.sh`)已隨包交付、可手動執行,但 **V1.1.9 起 A/B/C 裡註解掉的呼叫已整段刪除**,
|
||||
要接回去得重寫(`bmc_monitor_ddr.sh` 在 Script C 的註解區塊同時移除)。
|
||||
V1.1.9 新增 **SerDes DSC 擷取**:`bcm_dsc_scan.sh`(DUT 端,`phydiag <port> dsc` 掃全機 ~445 port × 2 unit,
|
||||
約 10 分鐘)由 Script A 與 Script C 各跑一次夾住 soak,`bcm_dsc_parse.sh`(Host 端)把 raw log 轉成
|
||||
`_lane.csv` / `_pm.csv`。**由 `config.ttl` 的 `EN_TX_FIR` 控制,預設 0(關)** —— 開了整輪測試多 ~20 分鐘。
|
||||
Script B 的 soak 迴圈每輪印 `bgctl list` + `jobs` + `systemctl is-active mlucas-amm`,並 `sendln "sleep 60"`
|
||||
(V1.0.10 之前完全沒有間隔;V1.2.0 起 A/B/C 的等待一律改成 DUT 端 `sleep` 而非 macro 端 `pause`)。V1.1.8 起 AMD CPU 壓力(`mlucas-avx2`)由 `systemd-run` 掛成
|
||||
`mlucas-amm` transient unit(`Restart=always`),不再是 shell 的 `&` 背景 job —— 掉線也不會停,
|
||||
Script C 用 `systemctl stop mlucas-amm` 收掉並把 log 複製到隨身碟。
|
||||
V1.1.0 新增全機 SerDes BER 掃描(`blanton_ber.sh`,**Script C 已接**,依 `SWB_UNIT` 分支)、
|
||||
V1.1.0 新增全機 SerDes BER 掃描(`blanton_ber.sh`,依 `SWB_UNIT` 分支);**V1.2.1 起 Script A 也接了一次**,
|
||||
**V1.2.2 起清單納入 port 268(100G uplink,ce0)**,summary 每個 unit 拆成「200G ports / 100G p268 / TOTAL」三列;
|
||||
在第一輪流量之前取基線,與 Script C 流量報表之後的收尾掃描一起夾住 soak(同 DSC 的做法)。另有
|
||||
TR518 負載(`blanton_tr518.sh`,**TTL 尚未接**)、SWB 軌功耗(`TH6_SWB{0,1}_power_readback.sh`),
|
||||
線速流量也納入 100G `ce0`(VLAN 138,靠 ingress mirror 維持循環)。
|
||||
Script A 開頭會先收掉前一輪殘留(`hw-test-session finish` + `bgctl reset/stop --all`),
|
||||
再 `bgctl list` 確認沒有 job 留著佔資源。
|
||||
NFC tag 讀取由 `nfc_polling.sh` 背景輪詢(A 做基線、B 起、C 停並 report)。
|
||||
另有 **`4_Blanton_Script_thermal_safety.ttl`**(熱 / 安規,**獨立跑不接 A→B→C**,滿載後每分鐘
|
||||
取 PMON + 兩片 SWB 的軌功耗;**沒有收尾段**,中斷後要手動 `blanton_tr518.sh stop`)。
|
||||
以及 **`5_Blanton_Script_Reboot_Cycle.ttl`**(重開機循環,**獨立跑不接 A→B→C**:登入 → 風扇 →
|
||||
取 PMON + 兩片 SWB 的軌功耗 + **AMD CPU 封裝功耗**(`avt_pmpower.sh`,V1.2.3);**沒有收尾段**,中斷後要手動 `blanton_tr518.sh stop`)。
|
||||
**V1.2.3 起 PMON 快照(`utils/setup_pmon.ttl`)多了 DDR5 DIMM 溫度(`ddr5_temp.sh`)與 LTC4287 HSC 溫度
|
||||
(`hsc_temp_readback.sh`)**,A/B/C/4/5 全部吃到;HSC 那支每次約 +4 秒。
|
||||
以及 **`5_Blanton_Script_Boot_Cycle.ttl`**(重開機循環,**獨立跑不接 A→B→C**:登入 → 風扇 →
|
||||
Check Pt1~Pt13 → `reboot` → `wait "Booting in blind mode"`,`cnt_cycle` 每輪遞增,`while 1` 無止境,
|
||||
要停就關 Tera Term 的 macro)。**尚未上機驗證**。
|
||||
流程圖:`docs/Blanton_Test_Flow.drawio`。
|
||||
最後一次發布是 **V1.1.5**;SWB 的 I2C 通道與位址已於 2026-08-19 上機驗證。
|
||||
最後一次發布是 **V1.2.3**;SWB 的 I2C 通道與位址已於 2026-08-19 上機驗證。
|
||||
margin 的 `VNOM` 已於 2026-08-30 改為 Blanton 實測中位數(144 條軌中 86 條上調 0.8~2.5%)。
|
||||
交付用 `./publish/publish.sh` 打包。
|
||||
|
||||
@@ -123,6 +131,15 @@ bash ~/Blanton_Script/port_prbs_monitor.sh report # PASS/FAIL 統計(沒
|
||||
bash ~/Blanton_Script/port_prbs_monitor.sh stop # 收尾會跑 prbsstat STOp + prbs clear
|
||||
# ⚠️ PRBS 打起來時 link 會顯示 down,這是正常的 —— 所以 start 刻意不印 port status
|
||||
|
||||
# ── 溫度 / CPU 功耗(V1.2.3,已接進 PMON / Script 4)──
|
||||
~/Blanton_Script/ddr5_temp.sh # DDR5 DIMM 溫度(SPD5118,預設 bus 5, 0x50 0x51)
|
||||
~/Blanton_Script/ddr5_temp.sh 5 0x50 # 指定 bus / 位址
|
||||
~/Blanton_Script/hsc_temp_readback.sh # LTC4287 HSC_MAIN(0x40)/HSC_STBY(0x45) 內外部溫度,bus 28,約 4 秒
|
||||
~/Blanton_Script/avt_pmpower.sh -p ~/AVT_Linux_NDA_2.8.37/AVTCMD # CPU SktPwr / PPT / MemPwr
|
||||
~/Blanton_Script/avt_pmpower.sh -q -p ... # 只印摘要,不倒整張 PM table
|
||||
# ⚠️ AVT 是 NDA,不在交付包裡:DUT 上要先解壓到 ~/AVT_Linux_NDA_2.8.37/,沒有就每輪 exit 2
|
||||
# ⚠️ 數值是 SMU telemetry 不是量測,要準就對照 VR 的 PMBus READ_POUT
|
||||
|
||||
# ── PDB / CPB power brick 讀值 ──────────────────────
|
||||
./Blanton_Script/pwr_brick.sh show # Vin/Iin/Vout/Iout/Temp × 5 顆
|
||||
./Blanton_Script/pwr_brick.sh show -r # 多印 raw word
|
||||
@@ -147,9 +164,38 @@ sleep 30 # 至少等一個 interval 再 rep
|
||||
./Blanton_Script/blanton_ber.sh report # 每 lane 一列,BER < 1e-6 才 PASS,最後 summary
|
||||
./Blanton_Script/blanton_ber.sh stop # prbsstat STOp
|
||||
./Blanton_Script/blanton_ber.sh clear # prbs clear(會解除 PRBS,要重測得再 init)
|
||||
# summary 每個 unit 三列:200G ports / 100G p268 / TOTAL(兩個 unit 各有自己的 ce0(268))
|
||||
# ⚠️ 某一組「一條 lane 都沒回報」算 FAIL 不算 PASS —— 待測的 port 沉默不是通過
|
||||
# -u 0 / -u 1 單邊;-c 1e-9 換門檻;-F 只列失敗;-f <log> 離線重解
|
||||
# ⚠️ 不能跟 blanton_traffic_linespeed / blanton_tr518 同時跑(都會接管同一批 port)
|
||||
|
||||
# ── SerDes DSC 擷取(TX_FIR / RXFFE / DFE / EYE / SNR)──
|
||||
./Blanton_Script/bcm_dsc_scan.sh # unit 0+1 全掃,約 10 分鐘
|
||||
./Blanton_Script/bcm_dsc_scan.sh -u 0 # 只掃一個 unit(時間減半)
|
||||
./Blanton_Script/bcm_dsc_scan.sh -f my.log -F # 自訂檔名 + 覆蓋
|
||||
./Blanton_Script/bcm_dsc_scan.sh -n # dry-run,只列要掃的 port
|
||||
# log: ~/Blanton_Script/log/,掃完自動 cp 到 /host/hw-eval/jobs(-C 可關)
|
||||
# 檔名沒帶 %u 時會自動插 _u0 / _u1,所以兩個 unit 不會互相覆蓋
|
||||
# 結尾固定印 "### DSC_SCAN_DONE rc=<n>";rc=0 才是全部 port 都拿到完整 dump
|
||||
# ⚠️ 由 TTL 的 EN_TX_FIR 控制,預設 0(關)—— A 與 C 各掃一次,開了整輪多 ~20 分鐘
|
||||
|
||||
# ── DSC raw log → CSV(離線解,Host 或 DUT 都可)──────
|
||||
sh ~/Blanton_Script/bcm_dsc_parse.sh -i blanton_dsc_ScriptA_u0.log -o u0
|
||||
# 產出 u0_lane.csv(每 lane 一列)+ u0_pm.csv(每 pm_id 一列)
|
||||
# 純 sh + awk(mawk 相容),欄位靠括號群組順序抓,SDK 改排版不會壞;
|
||||
# 群組數 != 9 的列標成 UNPARSED 印到 stderr,不會安靜解錯
|
||||
|
||||
# ── 100G link down/up 循環 + DSC(TTL 尚未接)────────
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh # 印說明
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 200 # 跑 200 輪(-c 200 亦可)
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 200 -d # 每行指令以灰色 [CMD] 印出來
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 20 -T # 跳過 tuning
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 1 -n # dry-run,不碰硬體也不 sleep
|
||||
# log: ~/dsc_log/100G_dsc_cycle_test_<ts>.log,每輪一段 '### TEST CYCLE n/N'
|
||||
# 每輪收:兩個 unit 的 phydiag dsc + serdes-tx-get / serdes-rx-get sys 0..7
|
||||
# 等 link 是輪詢 'bcmcmd ps 268'(每 -p 秒,上限 -w 秒),不是 show interfaces status
|
||||
# ⚠️ 尚未上機驗證;與 blanton_ber / blanton_traffic_linespeed / blanton_tr518 互斥
|
||||
|
||||
# ── TR518 負載 + 量功耗(TTL 尚未接)─────────────────
|
||||
./Blanton_Script/blanton_tr518.sh start # 兩 unit,跑完 settle 120s
|
||||
./Blanton_Script/TH6_SWB0_power_readback.sh # 19 條軌 V×I + TOTAL POWER
|
||||
@@ -304,9 +350,17 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
`Failed to execute the diagnostic command` 都照樣回 0),所以 `blanton_traffic_linespeed.sh`
|
||||
是**比對輸出**而非看 exit code。另外 `bcmcmd` 一律要 `</dev/null` ——
|
||||
否則它會吃掉 while 迴圈的 stdin,**只有第一個 VLAN 被設定**(V0.3.1 修過的 bug,很難看出來)。
|
||||
**2026-09-07 `bcm_dsc_scan.sh` 又踩了同一個坑**:port 清單走 stdin,第一次 `bcmcmd` 就把剩下的吃光,
|
||||
445 個 port 只掃了 1 個,而且 marker 與 dump 數量一起變少 → 比對通過、exit 0、log 看起來正常。
|
||||
修法是迴圈改讀 fd 3(`done 3< "$PORTFILE"`)+ `bcmcmd ... < /dev/null`,並加一道
|
||||
「實際迭代筆數 == 清單筆數」的檢查。**任何 `while ... done < file` 裡呼叫 `bcmcmd` 都要當它會讀 stdin。**
|
||||
- ⚠️ **`stress_hhmd.py` / `stress_pcie.py` 已於 2026-08-17 由 KC 移除**(Status 表 ScriptB #5/#7)。
|
||||
Script B 現在是 **8 個背景 job**(1 mlucas + 5 mem + ssd + usb),不是 10 個。
|
||||
I2C/PCIe 的持續讀取壓力目前沒有替代品。
|
||||
- ⚠️ **`pause` 是停 macro,`sleep` 是停 DUT**(V1.2.0 起 A/B/C 一律用後者):`pause N` 期間 macro 不理會
|
||||
DUT,shell 是閒著的,等於用猜的估上一個指令跑多久;`sendln "sleep N"` + `wait prompt_sonic_root`
|
||||
把等待放到 DUT 上,prompt 回來才繼續,兩邊不會走散,而且這一段會帶時戳進 log。
|
||||
**Script 4 / Script 5 尚未轉換,仍在用 `pause`**;B/C 開頭那個 `pause 1` 也保留(登入前沒有 prompt 可等)
|
||||
- ⚠️ **soak 迴圈沒有 `pause`**:取樣間隔不是 Status 表寫的 10 分鐘,而是取決於指令執行時間。
|
||||
(`-u 0` 已於 `491e807` 移除,A/B/C 現在都跑兩個 switch unit。)
|
||||
- 🔒 **`.sh` / `.conf` 一定要 LF**。CRLF 的 shell script 在 SONiC 上直接掛(`$'\r': command not found`);
|
||||
@@ -325,6 +379,15 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
- ⚠️ **兩張管理網卡同網段會 ARP flux**:`mgmt_ping_monitor.sh` 讓 eth0/eth1 同時 up,同一個 /24 下
|
||||
對端 ARP 可能由任一張卡回應,回包落在沒送封包的那張。`ARP_STRICT=1` 擋掉;判讀時看 `RESULT` 行的
|
||||
`nic_tx`/`nic_rx`(該卡自己的 packet delta)—— **ping 成功但 `nic_tx` 接近 0 = 封包從另一張卡出去的**
|
||||
- ⚠️ **`prbs get` 印的每一行是資料,不是 shell 的錯誤訊息**:沒鎖上的 lane 會印 `PRBS Error!`,
|
||||
而 `blanton_ber.sh` 從 v1.0.0 就用 `grep -i 'ERROR|Invalid|...'` 掃**整份輸出**判斷「指令被拒絕」——
|
||||
於是**跑得好好的 init 會報 `the BCM shell rejected 'phydiag ... prbs get'`**,而且因為提早 return,
|
||||
那段「數 PRBS OK / 總數」的檢查根本沒跑,**你看到一個假錯誤,卻看不到哪幾條 lane 真的沒鎖**。
|
||||
v1.1.1 已把 per-lane 行排除在錯誤掃描外、改成大小寫敏感,並把真正的拒絕訊息印出來
|
||||
- ⚠️ **BER 的 `clear` 才是把 lane 放回正常運作的動作**:V1.2.1 起 Script A 在第一輪流量**之前**跑 BER,
|
||||
而 `blanton_ber.sh clear` 與緊接著的 `blanton_traffic_linespeed ps` 之間**沒有任何 settle**。
|
||||
Script C 是流量報表之後才跑 BER,沒有這個順序問題。第一次上機時比對兩邊的 traffic report ——
|
||||
**Script A 有 NA/FAIL 而 Script C 沒有,就是 lane 還沒回穩**,在 `clear` 後補一個 `sleep` 即可
|
||||
- ⚠️ **PRBS 跑起來時 `show interfaces status` 會顯示 DOWN**,那是 link 離開正常運作模式,不是故障。
|
||||
`port_prbs_monitor.sh` 的 `start` 因此刻意不印 port status(避免測試員誤判),只在 `prbs clear` 之後的報表裡印
|
||||
- ⚠️ **`ip -s link` 的欄位順序是 `bytes packets errors ...`**,要 packet 數取的是**第 2 欄**不是第 1 欄
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# v1.2.0 — SerDes DSC capture, and the macro now waits on the DUT
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**Full-machine SerDes DSC capture, before and after the soak**
|
||||
* `bcm_dsc_scan.sh` runs `phydiag <port> dsc` through the Broadcom diag shell across the whole port list — one 100G port plus the 200G ranges, about 445 per unit, on both units — and writes the raw log under `Blanton_Script/log/`, then copies it to `/host/hw-eval/jobs/`.
|
||||
* Script A takes a baseline and Script C takes a closing dump, so the pair brackets the stress run and the two are directly comparable.
|
||||
* `-u 0` scans one unit, `-f <name>` names the log (`%u` → `u0`/`u1`, `%t` → timestamp), `-F` overwrites, `-n` dry runs, `-C` skips the copy. A name without `%u` gets `_u0` / `_u1` inserted automatically, so unit 1 can never overwrite unit 0.
|
||||
* Every run ends with `### DSC_SCAN_DONE rc=<n>`. `rc=0` means every port returned a complete dump.
|
||||
* `phydiag ... dsc` is read-only — it does not change any SerDes setting.
|
||||
|
||||
**`bcm_dsc_parse.sh` turns a raw DSC log into two CSVs**
|
||||
* `<prefix>_lane.csv` — one row per lane: TXEQ, RXFFE, DFE, EYE, SNR and the rest.
|
||||
* `<prefix>_pm.csv` — one row per `pm_id`: temperature, PLL, lane count, SNR/EYE statistics.
|
||||
* POSIX `sh` + `awk`, no Python, so it runs on the DUT as happily as on a laptop. Lane rows are read by the order of their bracket groups rather than by fixed column positions, so the SDK re-spacing its output does not break parsing; a row that does not yield 9 groups is reported as `UNPARSED` on stderr instead of being parsed wrongly.
|
||||
|
||||
**`EN_TX_FIR` in `config.ttl` — the DSC scan is off by default**
|
||||
* A full scan is about 10 minutes and both Script A and Script C do one, so the shipped default (`0`) keeps ~20 minutes out of a routine run. Set it to `1` when the SerDes data is what you are after.
|
||||
|
||||
**The macro now waits on the DUT instead of on itself**
|
||||
* Every `pause N` in Scripts A, B and C became `sendln "sleep N"` followed by `wait prompt_sonic_root`.
|
||||
* `pause` stopped the macro for N seconds without touching the DUT — the shell sat idle and the macro was guessing how long the previous command would take. On a loaded DUT the guess ran short and the next command went out before the previous one had finished, putting the whole run a beat ahead of the switch. Now the wait happens on the DUT and the macro resumes when the prompt actually comes back.
|
||||
* It also means the wait appears in the console log with a timestamp, so a round that ran long is visible afterwards.
|
||||
* Timings changed with it: the management-ping sample in Script A is 15 s (was 30), the settle after each traffic `clear` is 5 s (was 15). The traffic run itself stays 15 s and Script B's monitor loop stays 60 s.
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**The DSC scan only ever scanned one port — and reported success**
|
||||
* The scan loop read its port list on stdin, and `bcmcmd` reads stdin too, so the first call swallowed the rest of the file and the loop ended after a single port.
|
||||
* Nothing looked wrong. The log held one port marker and one dsc block, the marker-to-dump comparison passed because both numbers shrank together, the exit code was 0, and a 445-port scan simply finished in seconds.
|
||||
* The loop now reads on file descriptor 3 and `bcmcmd` is given `< /dev/null`, and there is a new check that the number of iterations matches the number of ports in the list — a check that cannot be fooled by the loop ending early.
|
||||
|
||||
**Dead call sites removed from Scripts A, B and C**
|
||||
* The commented-out 100G PRBS calls, a duplicated commented 100G port-status block in Script A, and the stale `bmc_monitor_ddr.sh` block in Script C have been deleted. They had been dead since bring-up.
|
||||
* `port_prbs_monitor.sh` still ships and still runs by hand — but re-enabling it means writing the calls again, not un-commenting them.
|
||||
|
||||
## 🔧 Changes
|
||||
|
||||
* **Script C no longer prints the BMC poll log or the mlucas log to the console.** Both files are still copied to `/host/hw-eval/jobs/` and out to the USB stick with everything else; only the master log got shorter.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.0.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
**After copying to the DUT:**
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh # bcm_dsc_scan.sh is invoked as ./Blanton_Script/... and needs it
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Replace `config.ttl` along with the macros.** `EN_TX_FIR` is new, and Tera Term aborts on an undefined variable rather than treating it as 0.
|
||||
* **`bcm_dsc_scan.sh` needs the execute bit.** Script A and Script C invoke it as `./Blanton_Script/bcm_dsc_scan.sh`. Without `chmod +x` it fails with `Permission denied`, the macro walks straight past it, and Script C's four `cp` lines then each report a missing file.
|
||||
* **The DSC scan shares the ports with `blanton_ber.sh`, `blanton_traffic_linespeed` and `blanton_tr518.sh`** — do not run it alongside any of them.
|
||||
* **Script 4 and Script 5 still use `pause`.** Only A, B and C were converted.
|
||||
* **`--runtime` is the job's overall limit, not a per-pass one.** `--passes` does not extend it; for a 24 hour soak set `RUNTIME` to `86400`.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **Script 5 (reboot cycle) is still unverified on hardware**, has no exit condition, and its `wait "Booting in blind mode"` has no `timeout` guard.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **The BMC USB service log only exists on the USB stick.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
* **Still unverified from v1.1.0**: the `ce0` ingress mirror, the 120 s settle in `blanton_tr518.sh`, and `test mode nr=no` on stop.
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
* [Blanton_Test_Flow.drawio](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/docs/Blanton_Test_Flow.drawio) — the scripts side by side
|
||||
|
||||
**Full changelog:** [V1.1.8...V1.2.0](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.1.8...V1.2.0)
|
||||
@@ -0,0 +1,49 @@
|
||||
# v1.2.1 — A BER baseline, so the SerDes numbers have something to be compared against
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**Script A now takes a SerDes BER sweep before the first traffic run**
|
||||
* `blanton_ber.sh init → start → sleep 30 → report → stop → clear`, branched on `SWB_UNIT0` / `SWB_UNIT1` exactly like the traffic block beside it.
|
||||
* Script C already ran the closing sweep after its traffic report. With this one at the front, the pair brackets the soak — the same shape as the DSC scans added in v1.2.0, and the same reason: a BER figure on its own says very little, a *change* in it says a lot.
|
||||
* The `sleep 30` matches `blanton_ber.sh`'s `BER_INTERVAL`. Reporting before one full interval has accumulated reads a window that has not finished, and nothing warns you about it.
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**The new block's comment described the wrong position**
|
||||
* It was copied out of Script C, which runs BER *after* the traffic report, and the comment came with it and still said "after". In Script A the block runs *before* traffic. The comment now says what the code does and why the ordering is what it is.
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Watch Script A's first traffic report on the first run.** PRBS holds the lanes out of normal operation and `blanton_ber.sh clear` is what releases them — and in Script A nothing settles between that `clear` and the `blanton_traffic_linespeed ps` / `init` that follows. Script C does not have this ordering, so the two reports are a built-in comparison: **if Script A shows NA or FAIL pairs that Script C does not, the lanes had not come back yet**, and a `sleep` after `clear` is the fix. This has not been run on hardware yet.
|
||||
* **`sleep 30` and `BER_INTERVAL` now live in three places** — Script A, Script C, and `blanton_ber.sh` itself. Changing one without the others silently reports a partial interval.
|
||||
* **BER, `blanton_traffic_linespeed` and `blanton_tr518.sh` cannot overlap.** They all take over the same ports. Script A now touches those ports twice before the soak even starts.
|
||||
* **`EN_TX_FIR` is still 0 by default**, so the DSC scans do not run unless you turn them on; a full scan is ~10 min in each of Script A and Script C.
|
||||
* **Replace `config.ttl` along with the macros.** Tera Term aborts the macro on an undefined variable rather than treating it as 0.
|
||||
* **`bcm_dsc_scan.sh` needs the execute bit**: `chmod +x ~/Blanton_Script/*.sh` after copying.
|
||||
* **`--runtime` is the job's overall limit, not a per-pass one.** `--passes` does not extend it.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **Script 5 (reboot cycle) has no exit condition.** A 199-cycle / 60-hour run has now been analysed: PCIe AER stayed at 0, all 218 traffic pairs and all 864 lanes passed every cycle, and the BMC never reset. Two things it did surface — `bmc-first-enroll` does not exist on that BMC build, and NFC returned FAIL on every cycle.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **The BMC USB service log only exists on the USB stick.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
* **Still unverified from v1.1.0**: the `ce0` ingress mirror, the 120 s settle in `blanton_tr518.sh`, and `test mode nr=no` on stop.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.1.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
|
||||
**Full changelog:** [V1.2.0...V1.2.1](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.2.0...V1.2.1)
|
||||
@@ -0,0 +1,63 @@
|
||||
# v1.2.2 — The 100G uplink joins the BER sweep, and a boot cycle that can pull the power
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**The BER sweep now measures port 268, the 100G uplink**
|
||||
* The port list ran `256-259` straight to `270-308`, so `ce0(268)` — the 100G uplink — was never in it. It is now its own group alongside the 200G ports.
|
||||
* The summary gives **each unit three rows**: `200G ports`, `100G p268`, and `TOTAL`. Unit 0 and unit 1 each have their own `ce0(268)`, and a marginal uplink lane averaged into 432 others is invisible; on its own row it is not.
|
||||
* A group that reports no lane at all now counts as a **failure**, not a pass. Silence on the port under test is the easiest way to ship a green report that measured nothing.
|
||||
|
||||
**Script 5 is now a boot cycle, warm or cold**
|
||||
* `5_Blanton_Script_Reboot_Cycle.ttl` → `5_Blanton_Script_Boot_Cycle.ttl`, with `EN_Cold_Warm_boot` in `config.ttl` choosing: `1` = `reboot` (warm), `2` = a cold cycle by writing the CB FPGA, `0` = neither.
|
||||
* The log file and the per-round banner are named after the kind of boot: `Blanton_Reboot_Cycle_<ts>.log` / `Reboot Cycle : N`, or `Blanton_Power_Cycle_<ts>.log` / `Power Cycle : N`.
|
||||
* Both stress logs are removed before each run. `qfx5252-stress-*` appends to its `--log` and the macro `cat`s the whole thing, so every cycle was printing back everything the earlier cycles had written — 289 lines at cycle 1 and 36,555 by cycle 199. At roughly 6.2 ms a line on a 115200 console that accounted for **+212 s of the +214 s the cycle grew** over a 199-cycle run. The DUT was never getting slower.
|
||||
|
||||
**`bcm_100g_dsc_cycle.sh` — 100G link down/up cycling with SerDes capture**
|
||||
* Tuning once, then N cycles of `port 268 en=0` → wait → `en=1` → wait → `phydiag 268 dsc` on both units → retimer `serdes-tx-get` / `serdes-rx-get` for `sys 0..7`. One log per run under `~/dsc_log/`, split by `### TEST CYCLE n/N`.
|
||||
* Waiting is a **poll of `bcmcmd ps 268`**, not `show interfaces status`. `port en=` changes the SDK, and the SONiC table only follows once gbsyncd has synced it — and this test pauses gbsyncd.
|
||||
* `-d` echoes every command as a grey `[CMD]` line before running it; `-n` is a dry run that touches nothing; the cycle count can be a bare argument (`./bcm_100g_dsc_cycle.sh 200`).
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**`blanton_ber.sh init` reported a hard error on runs that were working**
|
||||
* `[ERR ] unit 1: the BCM shell rejected 'phydiag ... prbs get'` — except the command was fine. `prbs get` prints one line per lane, a lane that has not locked says `PRBS Error!`, and the rejection check grepped the **whole** output case-insensitively for `ERROR|Invalid|...`.
|
||||
* Worse than the false alarm: the function returned early, so the lock count that was meant to judge those very lanes never ran. The operator saw an error they could not act on, and did **not** see which lanes had failed to lock. That check has been there since v1.0.0.
|
||||
* Per-lane result lines are now excluded, the match is case-sensitive, and a genuine rejection prints the offending line.
|
||||
|
||||
## 🔧 Changes
|
||||
|
||||
* **Script A takes a BER baseline before its first traffic run** (v1.2.1), so together with Script C's closing sweep the pair brackets the soak.
|
||||
* **`EN_TX_FIR`** gates the SerDes DSC scans in Script A and Script C. Default `0`: a full scan is ~10 min and both scripts do one.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.2.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Replace `config.ttl` with the macros.** `EN_Cold_Warm_boot` is new, and Tera Term aborts on an undefined variable rather than treating it as 0.
|
||||
* **Do not run the boot cycle with `EN_Cold_Warm_boot = 0`.** Nothing reboots, the `wait "Booting in blind mode"` underneath has no timeout, and since v1.2.2 there is no `logopen` either — so it hangs *and* leaves no log showing where. `0` is the value for the A → B → C flow, where that macro is not used.
|
||||
* **The cold path writes a raw FPGA register.** `0x480` is `ICB_CTRL_REG`; the register map carries no bit definitions for it and `0xD0000` is not documented anywhere in this repo. The BDF `0000:05:00.0` is hard-coded while `blanton_fpga_pcimem.sh` auto-detects it because it moves per unit — **check the BDF on every board before running the cold path**.
|
||||
* **Anything that parses these logs must accept both banners**, `Reboot Cycle : N` and `Power Cycle : N`. A parser that greps only the first finds zero cycles in a power-cycle log and reports nothing without complaining.
|
||||
* **`bcm_100g_dsc_cycle.sh` has not been run on hardware.** It pauses `gbsyncd0/1` for tuning and unpauses them on exit, Ctrl-C included; `-P` leaves them alone.
|
||||
* **`sleep 30` and `BER_INTERVAL` now live in three places** — Script A, Script C and `blanton_ber.sh`.
|
||||
* **BER, the DSC cycle, `blanton_traffic_linespeed` and `blanton_tr518.sh` cannot overlap.** They all take the same ports.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
|
||||
**Full changelog:** [V1.2.1...V1.2.2](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.2.1...V1.2.2)
|
||||
@@ -0,0 +1,53 @@
|
||||
# v1.2.3 — DIMM and hot-swap temperatures in every snapshot, and CPU power next to the switch rails
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**Every PMON snapshot now reads DDR5 DIMM and hot-swap controller temperatures**
|
||||
* `utils/setup_pmon.ttl` runs `ddr5_temp.sh` and `hsc_temp_readback.sh` right after `show platform temperature`. Because `setup_pmon` is shared, this lands everywhere a snapshot is taken: Script A's baseline, each Script B soak round, Script C, the Script 4 loop and every Script 5 check point.
|
||||
* `ddr5_temp.sh` reads the SPD5118 hub on each DIMM: i2c bus 5, `0x50` / `0x51`, register `0x31` (`bit[12:2]`, sign bit 12, 0.25 °C/LSB). A read that fails prints `read fail` — never a number it did not read. Bus and addresses can be passed: `ddr5_temp.sh [bus] [addr ...]`.
|
||||
* `hsc_temp_readback.sh` reads both LTC4287 hot-swap controllers on i2c bus 28 — `HSC_MAIN @ 0x40` (PU11 internal / PQ1 external) and `HSC_STBY @ 0x45` (PU46 internal / PQ53 external). The source is selected through `0xF3`, then `READ_TEMPERATURE` `0x8D` is read in Kelvin and converted.
|
||||
|
||||
**Script 4 logs AMD CPU package power every round**
|
||||
* `avt_pmpower.sh` runs `AVTCMD -module pmm "get_pmdata()"` and summarizes `SktPwr`, `PPT Fast` / `PPT Slow` against their limits, and `MemPwr`.
|
||||
* It runs after the two `TH6_SWB` power readbacks, so each sample now puts the switch rails and the CPU package side by side.
|
||||
* `-q` prints the summary only. Exit `2` = AVTCMD not found, `3` = AVTCMD failed, `4` = the PM table parsed to nothing (format changed).
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**`hsc_temp_readback.sh` was saved with CRLF line endings**
|
||||
* Copied to the DUT as-is, bash on SONiC fails on the first line (`$'\r': command not found`). The shipped file is LF.
|
||||
* Its comments said the delay between I2C transactions was 100 ms; `DELAY=0.5` is what actually runs. Comments corrected and a Version History header added.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.3.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/AVT_Linux_NDA_2.8.37/AVTCMD # Script 4 only - must already be there
|
||||
```
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Redeploy `Blanton_Script/`.** If an earlier copy of `hsc_temp_readback.sh` reached the DUT, it is CRLF and will not run.
|
||||
* **`chmod +x` after deploying.** `setup_pmon` calls both new scripts by path. Without the exec bit each call is `Permission denied` and the snapshot carries on without those rows — nothing stops, nothing warns.
|
||||
* **Each PMON snapshot is ~4 s longer.** `hsc_temp_readback.sh` sleeps 0.5 s twice for each of its four readings. It adds up in Script 5, which snapshots at every check point.
|
||||
* **AVT is not in this package (NDA).** Unpack it on the DUT at exactly `~/AVT_Linux_NDA_2.8.37/` before Script 4. Missing, the call exits `2` every round and the loop runs on without a CPU power reading.
|
||||
* **Script 4 logs the full PM table once a minute**, because the call is made without `-q`.
|
||||
* **CPU power is SMU telemetry, not a measurement.** Its absolute accuracy is whatever the BIOS SVI calibration gives — cross-check against the VR's PMBus `READ_POUT` when the number matters.
|
||||
* **None of the three new scripts has been checked against the Script A/B/C flow on hardware yet.**
|
||||
* **Do not run the boot cycle with `EN_Cold_Warm_boot = 0`**, and check the hard-coded BDF on every board before the cold path (see v1.2.2).
|
||||
* **BER, the DSC cycle, `blanton_traffic_linespeed` and `blanton_tr518.sh` cannot overlap.** They all take the same ports.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`FAN_SPEED` is 100.**
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
|
||||
**Full changelog:** [V1.2.2...V1.2.3](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.2.2...V1.2.3)
|
||||
@@ -1,7 +1,7 @@
|
||||
; =============================================================================
|
||||
; Script A for Blanton
|
||||
; Version : V1.1.9
|
||||
; Date : 2026-09-07
|
||||
; Version : V1.2.3
|
||||
; Date : 2026-09-23
|
||||
; Author : ETWen
|
||||
; =============================================================================
|
||||
; Version History:
|
||||
@@ -484,6 +484,262 @@
|
||||
; bmc_monitor.sh and friends in the chmod +x line after
|
||||
; deployment. Without it both scans fail with Permission
|
||||
; denied and the macro walks straight past it
|
||||
; V1.2.0 2026-09-07
|
||||
; Blanton_Script/bcm_dsc_scan.sh v1.3.0 bcmcmd was eating the port list
|
||||
; The scan loop read the port file on stdin, and bcmcmd
|
||||
; reads stdin too, so the first call swallowed the rest of
|
||||
; the file and the loop ended after ONE port
|
||||
; Nothing looked wrong: the log held one PORT marker and
|
||||
; one dsc block, markers == dumps, exit code 0. A 445 port
|
||||
; scan finished in seconds and reported success
|
||||
; Fixed by reading the loop on fd 3 (done 3< "$PORTFILE")
|
||||
; and giving bcmcmd < /dev/null
|
||||
; Added a check that the iteration count equals the port
|
||||
; list count. The marker / dump comparison cannot catch a
|
||||
; loop that ends early, because both numbers shrink
|
||||
; together; the Saved: line now reports dumps out of TOTAL,
|
||||
; not out of markers
|
||||
; !! this is the same bcmcmd trap
|
||||
; blanton_traffic_linespeed.sh hit at v0.3.1, where only
|
||||
; the first VLAN was configured. Every while ... done <
|
||||
; file loop that calls bcmcmd needs < /dev/null - assume
|
||||
; it will read stdin unless proven otherwise
|
||||
; ScriptA / ScriptB / ScriptC Wait on the DUT with 'sleep', not on the macro with 'pause'
|
||||
; pause N stops the macro for N seconds without touching
|
||||
; the DUT: the shell sits idle and the macro is guessing
|
||||
; how long the last command took. sendln "sleep N" + wait
|
||||
; prompt_sonic_root moves the wait onto the DUT, so the
|
||||
; macro resumes when the prompt actually comes back and the
|
||||
; two cannot drift apart
|
||||
; It also lands in the console log with a timestamp, so a
|
||||
; round that ran long is visible afterwards instead of
|
||||
; invisible
|
||||
; Durations changed with it: ScriptA mgmt ping 30 -> 15 s;
|
||||
; the settle after every traffic 'clear' 15 -> 5 s; the
|
||||
; traffic run itself stays 15 s; ScriptB's monitor loop
|
||||
; stays 60 s
|
||||
; !! Script4 and Script5 were NOT converted and still use
|
||||
; pause. The 'pause 1' at the top of ScriptB / ScriptC
|
||||
; also stays - it only spaces the log header and runs
|
||||
; before there is a prompt to wait for
|
||||
; config.ttl Add EN_TX_FIR to gate the SerDes DSC scan
|
||||
; EN_TX_FIR = 1 enables bcm_dsc_scan.sh in ScriptA and
|
||||
; ScriptC, and the four cp lines that carry
|
||||
; blanton_dsc_Script{A,C}_u{0,1}.log to /host/hw-eval/jobs.
|
||||
; Default is 0
|
||||
; A full scan is ~10 min and both scripts do one, so
|
||||
; leaving it off keeps ~20 min out of a routine run. Turn
|
||||
; it on when the SerDes data is what you are after
|
||||
; ScriptC Stop printing the BMC poll and mlucas logs to the console
|
||||
; cat bmc_poll.log and cat mlucas_amm_log are commented
|
||||
; out. Both files are still copied to /host/hw-eval/jobs
|
||||
; and out to the USB stick with everything else, so no data
|
||||
; is lost - the master log is just shorter
|
||||
; V1.2.1 2026-09-08
|
||||
; ScriptA Add a BER baseline sweep before the traffic run
|
||||
; blanton_ber.sh init / start / sleep 30 / report / stop /
|
||||
; clear, branched on SWB_UNIT0 and SWB_UNIT1 the same way
|
||||
; the traffic block is. ScriptC already took the closing
|
||||
; sweep after its traffic report, so the two now bracket
|
||||
; the soak the way the DSC scans do
|
||||
; sleep 30 is not arbitrary: it is blanton_ber.sh's
|
||||
; BER_INTERVAL. 'report' before one full interval has
|
||||
; accumulated reads a window that has not finished
|
||||
; The block was copied from ScriptC, which runs it AFTER
|
||||
; traffic, and its comment said so. In ScriptA it runs
|
||||
; BEFORE traffic, so the comment was rewritten to match
|
||||
; where the code actually is
|
||||
; !! nothing settles between 'blanton_ber.sh clear' and
|
||||
; 'blanton_traffic_linespeed ps' that follows it. PRBS
|
||||
; holds the lanes out of normal operation and 'clear' is
|
||||
; what releases them, so the first traffic run of
|
||||
; ScriptA now starts on lanes that have just been
|
||||
; released. ScriptC does not have this ordering. Watch
|
||||
; ScriptA's first traffic report for NA or FAIL pairs
|
||||
; that ScriptC's does not show - if they appear, a sleep
|
||||
; after 'clear' is the fix
|
||||
; !! ScriptA now runs BER twice as far as the switch is
|
||||
; concerned: once here and once inside ScriptC. Neither
|
||||
; can overlap blanton_traffic_linespeed or
|
||||
; blanton_tr518.sh - they all take over the same ports
|
||||
; V1.2.2 2026-09-15
|
||||
; 5_.._Boot_Cycle.ttl Renamed from 5_Blanton_Script_Reboot_Cycle.ttl
|
||||
; It does warm and cold now, so 'Reboot' was too narrow.
|
||||
; Same macro otherwise - git records it as a rename
|
||||
; 5_.._Boot_Cycle.ttl Warm or cold boot, chosen by EN_Cold_Warm_boot
|
||||
; 1 = 'reboot' (warm, what the macro always did). 2 = cold,
|
||||
; by writing the CB FPGA: pcimem
|
||||
; /sys/bus/pci/devices/0000:05:00.0/resource0 0x480 w
|
||||
; 0xD0000. 0 = neither
|
||||
; !! EN_Cold_Warm_boot = 0 hangs the macro. Neither branch
|
||||
; runs, so nothing reboots, and the 'wait "Booting in
|
||||
; blind mode"' underneath has no timeout - cycle 1
|
||||
; finishes and the macro then waits for a boot that will
|
||||
; never come. 0 is the value for the A/B/C flow, where
|
||||
; this macro is not used at all; do not run ScriptBoot
|
||||
; with it
|
||||
; !! 0x480 is ICB_CTRL_REG in
|
||||
; docs/Blantons_FPGA_Registers.md, and that table
|
||||
; carries no bit definitions for it. 0xD0000 is not
|
||||
; written down anywhere in this repo either. Whoever
|
||||
; knows what those bits do should put it in the register
|
||||
; doc before someone changes the value
|
||||
; !! the BDF 0000:05:00.0 is hard-coded here, while
|
||||
; blanton_fpga_pcimem.sh auto-detects the CB FPGA
|
||||
; precisely because it moves per unit (see the V1.0.x
|
||||
; notes and Key Constraints #4). On a DUT where the FPGA
|
||||
; is not at 05:00.0 this becomes a raw 32-bit write to
|
||||
; an unknown register on some other device. Verify the
|
||||
; BDF on every unit before running the cold path
|
||||
; !! NOT verified on hardware. The warm path has 199 cycles
|
||||
; behind it; the cold path has none
|
||||
; 5_.._Boot_Cycle.ttl Truncate the two stress logs before each run
|
||||
; rm -rf /host/hw-eval/jobs/qfx5252-stress-ssd.log and the
|
||||
; usb one, added ahead of each qfx5252-stress-* call
|
||||
; qfx5252-stress-* APPENDS to its --log file and the macro
|
||||
; cats the whole thing afterwards, so every cycle printed
|
||||
; back everything the ones before it had written
|
||||
; Measured on the 199 cycle log from 2026-09-04: cycle 1
|
||||
; printed 289 lines, cycle 199 printed 36,555. At ~6.2 ms a
|
||||
; line - which is what a 115200 baud console costs - that
|
||||
; is +212 s of the +214 s the cycle grew over the run. The
|
||||
; stress itself is fixed at --runtime 30, and the other
|
||||
; eleven segments between check points drifted by less than
|
||||
; 1.5 s each. The DUT was never getting slower; the test
|
||||
; was reading its own history aloud once per cycle
|
||||
; 5_.._Boot_Cycle.ttl Name the log file and the banner after the kind of boot
|
||||
; EN_Cold_Warm_boot = 1 gives
|
||||
; Logs\Blanton_Reboot_Cycle_<ts>.log and a 'Reboot Cycle :
|
||||
; N' banner; = 2 gives Blanton_Power_Cycle_<ts>.log and
|
||||
; 'Power Cycle : N'. Three sites now branch on the flag:
|
||||
; the log name, the banner and the boot itself
|
||||
; !! with 0 there is no longer a logopen either, so the run
|
||||
; that already hangs at wait "Booting in blind mode" now
|
||||
; also leaves no log to show where it stopped. EN_log =
|
||||
; 1 no longer guarantees a log in this macro -
|
||||
; EN_Cold_Warm_boot has to be 1 or 2 as well
|
||||
; !! anything that parses these logs has to accept BOTH
|
||||
; banners. The cycle banner is the record separator the
|
||||
; check-point and per-segment timing reports are built
|
||||
; on, so a parser that only greps 'Reboot Cycle :' finds
|
||||
; zero cycles in a power-cycle log and silently reports
|
||||
; nothing at all
|
||||
; The logopen timestamp flag is still a literal 1 here
|
||||
; rather than EN_timestamp the way ScriptA does it. Leaving
|
||||
; it on is the right call - the per-segment timing analysis
|
||||
; is derived from those timestamps and there is nothing
|
||||
; else to recover them from - but it does mean this macro
|
||||
; ignores EN_timestamp = 0
|
||||
; config.ttl Add EN_Cold_Warm_boot
|
||||
; 0 = normal test (A/B/C), 1 = reboot cycle, 2 = power
|
||||
; cycle. Default 0 so that opening the shared config cannot
|
||||
; reboot anyone's DUT by accident
|
||||
; Blanton_Script/blanton_ber.sh v1.1.0 Add port 268, the 100G uplink
|
||||
; The port list jumps 256-259 -> 270-308, so 268 (ce0) was
|
||||
; never measured. It is now its own group: BER_PORTS_200G
|
||||
; and BER_PORTS_100G, joined for the phydiag command but
|
||||
; reported apart
|
||||
; The summary gives each unit three rows - 200G ports /
|
||||
; 100G p268 / TOTAL - because unit 0 and unit 1 each have
|
||||
; their own ce0(268). A bad uplink lane averaged into 432
|
||||
; others is invisible; on its own row it is not
|
||||
; TOTAL is not a hand-added sum: every lane is folded into
|
||||
; its own group and into ALL separately, so the parts
|
||||
; cannot drift from the whole
|
||||
; !! a group that reports no lane at all is a FAILURE, not
|
||||
; a pass. Silence on the port under test is the easiest
|
||||
; way to ship a green report that measured nothing
|
||||
; Blanton_Script/blanton_ber.sh v1.1.1 Stop reading lane results as shell errors
|
||||
; init kept answering "the BCM shell rejected 'phydiag ...
|
||||
; prbs get'" on runs that were working. The check grepped
|
||||
; the WHOLE output case-insensitively for
|
||||
; ERROR|Invalid|..., and 'prbs get' prints one line per
|
||||
; lane - a lane that had not locked says PRBS Error!, which
|
||||
; matched
|
||||
; So a good run reported a hard error, and worse, the
|
||||
; function returned before the lock-count check that was
|
||||
; supposed to judge those very lanes. The operator saw a
|
||||
; failure and did NOT see which lanes were unlocked
|
||||
; Per-lane lines are now excluded from the check, the match
|
||||
; is case-sensitive, and a genuine rejection prints the
|
||||
; offending line so it can be read
|
||||
; !! this was in the file from v1.0.0. Any earlier init
|
||||
; that printed that error was probably fine, and any
|
||||
; earlier init that passed said nothing about lanes that
|
||||
; had failed to lock
|
||||
; Blanton_Script/bcm_100g_dsc_cycle.sh Created v1.2.0 - 100G link down/up cycle with DSC capture
|
||||
; Tuning once (gbsyncd pause, serdes-tx-set sys 0..7,
|
||||
; tx_taps per lane), then N cycles of: port en=0, wait,
|
||||
; port en=1, wait, phydiag dsc on both units, serdes-tx-get
|
||||
; and serdes-rx-get sys 0..7. One log per run under
|
||||
; ~/dsc_log, split by '### TEST CYCLE n/N'
|
||||
; Waiting polls 'bcmcmd ps <port>' every -p seconds up to
|
||||
; -w, and reads the state from the row whose first field
|
||||
; ends in (<port>). Asking the SDK matters here: 'port en='
|
||||
; changes the SDK, while 'show interfaces status' only
|
||||
; follows once gbsyncd has synced it - and this test pauses
|
||||
; gbsyncd
|
||||
; -d prints every command as [CMD] in grey before running
|
||||
; it, CMD_DELAY paces them, -n is a dry run that touches
|
||||
; nothing and sleeps not at all
|
||||
; !! all output is ASCII English on purpose. The DUT serial
|
||||
; console mangles UTF-8: a Chinese error message came
|
||||
; back as "ERROR: -c ? 1", which is worse than no
|
||||
; message
|
||||
; !! NOT verified on hardware
|
||||
; tools/100G_dsc_cycle.txt Created - the hand-typed command list behind that script
|
||||
; Same role as BER_Test.txt and TR518.txt: the bench
|
||||
; version of the commands, kept so the .sh can be checked
|
||||
; against what a person would actually type. Change one and
|
||||
; change the other
|
||||
; V1.2.3 2026-09-23 utils/setup_pmon.ttl Add DDR5 DIMM temperature + hot-swap controller temperature
|
||||
; ~/Blanton_Script/ddr5_temp.sh, then hsc_temp_readback.sh,
|
||||
; right after 'show platform temperature'
|
||||
; setup_pmon is shared, so this lands in every PMON snapshot:
|
||||
; ScriptA baseline, every ScriptB soak round, ScriptC, the
|
||||
; Script 4 loop and each Script 5 check point
|
||||
; !! hsc_temp_readback.sh sleeps 0.5 s twice per reading and
|
||||
; takes four readings - ~4 s added to every PMON snapshot
|
||||
; !! both are called by path, not through bash, so they need
|
||||
; the exec bit on the DUT: chmod +x ~/Blanton_Script/*.sh
|
||||
; after deploying. Without it each call is Permission
|
||||
; denied and the snapshot just goes on without the rows
|
||||
; Blanton_Script/ddr5_temp.sh Created v1.0.1 - DDR5 DIMM temperature from the SPD5118 hub
|
||||
; i2cget bus 5, DIMM 0x50 / 0x51, register 0x31 (MR49/MR50)
|
||||
; bit[12:2], sign bit 12, 0.25 C/LSB. Bus and addresses can
|
||||
; be given as arguments: ddr5_temp.sh [bus] [addr ...]
|
||||
; A failed read prints 'read fail' and moves on to the next
|
||||
; DIMM - it never prints a number it did not read
|
||||
; Blanton_Script/hsc_temp_readback.sh Created - LTC4287 hot-swap controller temperatures
|
||||
; Native i2c bus 28. HSC_MAIN @ 0x40 (PU11 internal / PQ1
|
||||
; external) and HSC_STBY @ 0x45 (PU46 internal / PQ53
|
||||
; external). Source picked by writing 0xF3 (0xEB internal,
|
||||
; 0xEF external), then READ_TEMPERATURE 0x8D, Kelvin -> C
|
||||
; v1.0.1 Add the header + Version History, fix the delay
|
||||
; comments (they said 100 ms; DELAY is 0.5 s)
|
||||
; !! v1.0.0 was saved with CRLF line endings. On SONiC that
|
||||
; fails outright ($'\r': command not found) - now LF
|
||||
; Blanton_Script/avt_pmpower.sh Created v1.0.0 - AMD CPU package power from the AVT PM table
|
||||
; AVTCMD -module pmm "get_pmdata()", then a summary of
|
||||
; SktPwr, PPT Fast / Slow against their limits, and MemPwr
|
||||
; -p <AVTCMD path>, -q for the summary without the raw table
|
||||
; Exit 2 = AVTCMD not found, 3 = AVTCMD failed, 4 = the table
|
||||
; parsed to nothing (format changed)
|
||||
; !! SMU telemetry, not a measurement - its absolute accuracy
|
||||
; is whatever the BIOS SVI calibration gives. Cross-check
|
||||
; against the VR's PMBus READ_POUT when the number matters
|
||||
; 4_Blanton_Script_thermal_safety.ttl Add CPU package power to every loop round
|
||||
; avt_pmpower.sh -p ~/AVT_Linux_NDA_2.8.37/AVTCMD, after the
|
||||
; two TH6_SWB power readbacks - so each round now has the
|
||||
; switch rails and the CPU package side by side
|
||||
; !! AVT is NOT in Blanton_Script and does not travel with
|
||||
; this package (NDA). It has to be unpacked on the DUT at
|
||||
; exactly ~/AVT_Linux_NDA_2.8.37/ beforehand. If it is not
|
||||
; there the script exits 2 every round and the loop carries
|
||||
; on without a CPU power reading
|
||||
; !! without -q the full PM table is printed every round, and
|
||||
; Script 4 loops once a minute - the log grows by one full
|
||||
; table per minute
|
||||
; =============================================================================
|
||||
include "config.ttl"
|
||||
|
||||
@@ -649,7 +905,8 @@ include "utils/show_dmesg.ttl"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh start"
|
||||
wait prompt_sonic_root
|
||||
pause 30
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/mgmt_ping_monitor.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ~/Blanton_Script/log/mgmt_ping.log"
|
||||
@@ -692,6 +949,56 @@ sendln "config feature state lldp disabled"
|
||||
wait prompt_sonic_root
|
||||
sendln "config save -y"
|
||||
|
||||
; ========== BER ==========
|
||||
; Baseline sweep, taken BEFORE the traffic run - ScriptC takes the closing
|
||||
; one after its traffic report, so the pair brackets the soak.
|
||||
; PRBS takes the SerDes out of normal operation; 'clear' is what puts the
|
||||
; lanes back, so it has to run before the traffic block below.
|
||||
; sleep 30 matches blanton_ber.sh BER_INTERVAL - reporting earlier reads an
|
||||
; interval that has not finished accumulating.
|
||||
if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear"
|
||||
elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear -u 0"
|
||||
elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear -u 1"
|
||||
else
|
||||
|
||||
endif
|
||||
|
||||
if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed ps"
|
||||
@@ -700,12 +1007,14 @@ if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report"
|
||||
@@ -717,12 +1026,14 @@ elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report -u 0"
|
||||
@@ -734,12 +1045,14 @@ elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report -u 1"
|
||||
@@ -747,9 +1060,11 @@ else
|
||||
|
||||
endif
|
||||
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptA.log -F"
|
||||
if EN_TX_FIR = 1 then
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptA.log -F"
|
||||
endif
|
||||
|
||||
; ========== Show Power on uptime ==========
|
||||
wait prompt_sonic_root
|
||||
|
||||
@@ -109,7 +109,8 @@ timeout = 0
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh start"
|
||||
wait prompt_sonic_root
|
||||
pause 3
|
||||
sendln "sleep 3"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh status"
|
||||
|
||||
; ========== NFC Test ==========
|
||||
@@ -139,7 +140,8 @@ if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start"
|
||||
@@ -151,7 +153,8 @@ elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 0"
|
||||
@@ -163,7 +166,8 @@ elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 1"
|
||||
@@ -173,7 +177,7 @@ endif
|
||||
|
||||
|
||||
|
||||
; ========== Monitor loop: platform data + background jobs (pause 60s per round) ==========
|
||||
; ========== Monitor loop: platform data + background jobs (sleep 60s per round) ==========
|
||||
while 1
|
||||
include "utils/setup_pmon.ttl"
|
||||
wait prompt_sonic_root
|
||||
@@ -182,5 +186,6 @@ while 1
|
||||
sendln "jobs"
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl is-active mlucas-amm"
|
||||
pause 60
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 60"
|
||||
endwhile
|
||||
|
||||
@@ -53,8 +53,8 @@ sendln "/usr/sbin/ras-mc-ctl --summary"
|
||||
; ========== BMC Monitor TAKE DATA ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bmc_monitor.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ./Blanton_Script/log/bmc_poll.log"
|
||||
; wait prompt_sonic_root
|
||||
; sendln "cat ./Blanton_Script/log/bmc_poll.log"
|
||||
|
||||
; ========== BMC USB Test result ==========
|
||||
wait prompt_sonic_root
|
||||
@@ -64,8 +64,8 @@ sendln "journalctl -u qfx5252-bmc-usb-net-test.service --no-pager -o short-iso >
|
||||
|
||||
|
||||
; ========== CHECK Stress results ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ~/hammer/tools/amd/mlucas_amm_log"
|
||||
; wait prompt_sonic_root
|
||||
; sendln "cat ~/hammer/tools/amd/mlucas_amm_log"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
@@ -150,10 +150,11 @@ else
|
||||
|
||||
endif
|
||||
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptC.log -F"
|
||||
|
||||
if EN_TX_FIR = 1 then
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptC.log -F"
|
||||
endif
|
||||
|
||||
; ========== NVME Err Info ==========
|
||||
wait prompt_sonic_root
|
||||
@@ -199,14 +200,17 @@ wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/mgmt_ping.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/nfc_poll.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u1.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u1.log /host/hw-eval/jobs/"
|
||||
|
||||
if EN_TX_FIR = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u1.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u1.log /host/hw-eval/jobs/"
|
||||
endif
|
||||
|
||||
wait prompt_sonic_root
|
||||
getdate ts_date "%Y%m%d"
|
||||
|
||||
@@ -152,6 +152,8 @@ while 1
|
||||
sendln "~/Blanton_Script/TH6_SWB0_power_readback.sh"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/TH6_SWB1_power_readback.sh"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/avt_pmpower.sh -p ~/AVT_Linux_NDA_2.8.37/AVTCMD"
|
||||
|
||||
; Background jobs Check
|
||||
wait prompt_sonic_root
|
||||
|
||||
+31
-7
@@ -8,8 +8,14 @@ gettime str_starttime "%Y%m%d-%H%M%S"
|
||||
getdir mdir
|
||||
|
||||
if EN_log = 1 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Reboot_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Reboot_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Power_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
else
|
||||
endif
|
||||
endif
|
||||
|
||||
cnt_cycle = 0
|
||||
@@ -26,9 +32,16 @@ while 1
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 10"
|
||||
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mReboot Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mReboot Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mPower Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
else
|
||||
endif
|
||||
|
||||
; ========== Pre Setting ==========
|
||||
; Make log directory
|
||||
@@ -172,6 +185,8 @@ while 1
|
||||
sprintf2 strCKPt5 "echo -e '\033[36mCheck Pt5 : SSD Test\033[0m'"
|
||||
sendln strCKPt5
|
||||
|
||||
wait prompt_sonic_root
|
||||
sendln "rm -rf /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
sendln "qfx5252-stress-ssd 1G --runtime 30 --passes 2 --write --force --log /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
@@ -189,6 +204,8 @@ while 1
|
||||
waitregex '<<DEV=([^>]*)>>'
|
||||
usb_dev = groupmatchstr1
|
||||
|
||||
wait prompt_sonic_root
|
||||
sendln "rm -rf /host/hw-eval/jobs/qfx5252-stress-usb.log"
|
||||
wait prompt_sonic_root
|
||||
sprintf2 cmd "qfx5252-stress-usb %s 256M --runtime 30 --passes 2 --write --log /host/hw-eval/jobs/qfx5252-stress-usb.log" usb_dev
|
||||
sendln cmd
|
||||
@@ -412,8 +429,15 @@ while 1
|
||||
sendln "show uptime"
|
||||
|
||||
; ========== Show Power on uptime ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "reboot"
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "reboot"
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
wait prompt_sonic_root
|
||||
sendln "pcimem /sys/bus/pci/devices/0000:05:00.0/resource0 0x480 w 0xD0000"
|
||||
else
|
||||
endif
|
||||
|
||||
|
||||
wait "Booting in blind mode"
|
||||
endwhile
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/bin/bash
|
||||
#===============================================================================
|
||||
# Script : avt_pmpower.sh
|
||||
# Purpose : Dump AMD AVT PM table (get_pmdata) and summarize package power.
|
||||
#
|
||||
# Author : ETWen
|
||||
# Date : 20260923
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260923 Initial Version
|
||||
#
|
||||
# Usage : ./avt_pmpower.sh [-p <AVTCMD path>] [-q] [-h]
|
||||
# -p Path to AVTCMD (default: $AVTCMD_PATH or ./AVTCMD)
|
||||
# -q Quiet: summary only, suppress raw PM table
|
||||
# -h Help
|
||||
#
|
||||
# Exit : 0 ok / 1 usage / 2 AVTCMD not found / 3 AVTCMD failed
|
||||
# 4 parse failed (no power fields found)
|
||||
#
|
||||
# Note : Values come from SMU telemetry, not direct measurement. Absolute
|
||||
# accuracy depends on BIOS SVI telemetry calibration. Cross-check
|
||||
# against VR PMBus READ_POUT when accuracy matters.
|
||||
#===============================================================================
|
||||
|
||||
set -u
|
||||
set -o pipefail
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Defaults
|
||||
#------------------------------------------------------------------------------
|
||||
AVTCMD_BIN="${AVTCMD_PATH:-./AVTCMD}"
|
||||
AVT_MODULE="pmm"
|
||||
AVT_CMD="get_pmdata()"
|
||||
QUIET=0
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# usage - print help
|
||||
#------------------------------------------------------------------------------
|
||||
usage() {
|
||||
sed -n '/^# Usage/,/^# Exit/p' "$0" | sed 's/^# \{0,1\}//'
|
||||
}
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
#------------------------------------------------------------------------------
|
||||
while getopts ":p:qh" opt; do
|
||||
case "$opt" in
|
||||
p) AVTCMD_BIN="$OPTARG" ;;
|
||||
q) QUIET=1 ;;
|
||||
h) usage; exit 0 ;;
|
||||
:) echo "ERROR: -$OPTARG requires an argument" >&2; usage >&2; exit 1 ;;
|
||||
\?) echo "ERROR: unknown option -$OPTARG" >&2; usage >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Pre-flight check
|
||||
#------------------------------------------------------------------------------
|
||||
if [ ! -x "$AVTCMD_BIN" ]; then
|
||||
echo "ERROR: AVTCMD not found or not executable: $AVTCMD_BIN" >&2
|
||||
echo " Use -p <path> or export AVTCMD_PATH=<path>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "WARN : not running as root, AVTCMD may fail to access SMU." >&2
|
||||
fi
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Collect PM table
|
||||
#------------------------------------------------------------------------------
|
||||
RAW="$("$AVTCMD_BIN" -module "$AVT_MODULE" "$AVT_CMD" 2>&1)"
|
||||
RC=$?
|
||||
if [ $RC -ne 0 ]; then
|
||||
echo "ERROR: AVTCMD exited with code $RC" >&2
|
||||
echo "$RAW" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ "$QUIET" -eq 0 ]; then
|
||||
printf '%s\n' "$RAW"
|
||||
fi
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Parse & summarize
|
||||
# - split on whitespace and comma, keep only "key:number" tokens
|
||||
# - bracketed tokens ([Pkg:0], [MISC]) are skipped by the regex anchor
|
||||
#------------------------------------------------------------------------------
|
||||
printf '%s\n' "$RAW" | awk '
|
||||
{
|
||||
n = split($0, a, /[ ,]+/)
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (a[i] ~ /^[A-Za-z0-9_()]+:[-0-9.]+$/) {
|
||||
split(a[i], kv, ":")
|
||||
m[kv[1]] = kv[2]
|
||||
found++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# pct - usage against limit, guard divide-by-zero
|
||||
function pct(v, l) {
|
||||
return (l > 0) ? sprintf("%5.1f%%", 100 * v / l) : " n/a"
|
||||
}
|
||||
|
||||
# has - field present in PM table
|
||||
function has(k) {
|
||||
return (k in m)
|
||||
}
|
||||
|
||||
END {
|
||||
if (found == 0) {
|
||||
print "ERROR: no PM fields parsed, PM table format may have changed." > "/dev/stderr"
|
||||
exit 4
|
||||
}
|
||||
|
||||
print "===== Package Power (live) ====="
|
||||
|
||||
if (has("SktPwr(W)"))
|
||||
printf "%-9s: %7.2f W\n", "SktPwr", m["SktPwr(W)"]
|
||||
|
||||
if (has("PPT_VALUE_FAST"))
|
||||
printf "%-9s: %7.2f / %3d W %s\n", "PPT Fast",
|
||||
m["PPT_VALUE_FAST"], m["PPT_LIMIT_FAST"],
|
||||
pct(m["PPT_VALUE_FAST"], m["PPT_LIMIT_FAST"])
|
||||
|
||||
if (has("PPT_VALUE_SLOW"))
|
||||
printf "%-9s: %7.2f / %3d W %s (%ss avg)\n", "PPT Slow",
|
||||
m["PPT_VALUE_SLOW"], m["PPT_LIMIT_SLOW"],
|
||||
pct(m["PPT_VALUE_SLOW"], m["PPT_LIMIT_SLOW"]),
|
||||
(has("SlowPPTTimeConstant") ? m["SlowPPTTimeConstant"] : "?")
|
||||
|
||||
if (has("MemPwr(W)"))
|
||||
printf "%-9s: %7.2f W\n", "MemPwr", m["MemPwr(W)"]
|
||||
}
|
||||
'
|
||||
exit $?
|
||||
@@ -0,0 +1,608 @@
|
||||
#!/bin/sh
|
||||
# =============================================================================
|
||||
# bcm_100g_dsc_cycle.sh
|
||||
#
|
||||
# Author : ETWen
|
||||
# Date : 20260909
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260909 Initial Version
|
||||
# V1.1.0 20260909 All runtime output is ASCII English - the DUT serial
|
||||
# console mangles UTF-8 (a Chinese error message came
|
||||
# back as "ERROR: -c ? 1"). Cycle count may be given as
|
||||
# a bare argument. Added -d to echo every command, and
|
||||
# CMD_DELAY to pace them.
|
||||
# V1.2.0 20260909 Wait on "bcmcmd ps <port>" instead of
|
||||
# "show interfaces status". The state comes from the
|
||||
# SDK, which is what "port en=" actually changes, so it
|
||||
# no longer depends on gbsyncd syncing into the DB.
|
||||
#
|
||||
# Description:
|
||||
# Link down/up cycling on the 100G uplink. After every re-link it collects a
|
||||
# SerDes DSC dump and reads the retimer back, so you can see whether the
|
||||
# tuned TX taps settle to the same place each time the link comes up.
|
||||
#
|
||||
# One run has two parts:
|
||||
# 1. Tuning -- once. pause gbsyncd -> serdes-tx-set sys 0..7
|
||||
# -> tx_taps set on four lanes of each unit
|
||||
# 2. Cycle -- repeated N times:
|
||||
# port en=0 -> wait until "ps <port>" reads !ena on both units
|
||||
# port en=1 -> wait until "ps <port>" reads up on both units
|
||||
# phydiag <port> dsc on each unit
|
||||
# serdes-tx-get / serdes-rx-get sys 0..7
|
||||
# all of it appended to one log, under a per-cycle heading
|
||||
#
|
||||
# Command source: tools/100G_dsc_cycle.txt (the hand-typed bench version).
|
||||
# Change that file and this one together.
|
||||
#
|
||||
# Deployment:
|
||||
# Script : /root/Blanton_Script/bcm_100g_dsc_cycle.sh
|
||||
# Log : ~/dsc_log/100G_dsc_cycle_test_<YYYYmmdd_HHMMSS>.log
|
||||
# One file per run, every cycle in it, split by "### TEST CYCLE n/N".
|
||||
#
|
||||
# Paths come from $HOME. Run as root (after sudo -i) and ~ is /root.
|
||||
#
|
||||
# Usage:
|
||||
# ./bcm_100g_dsc_cycle.sh # print this help
|
||||
# ./bcm_100g_dsc_cycle.sh 200 # 200 cycles
|
||||
# ./bcm_100g_dsc_cycle.sh -c 200 # same thing
|
||||
# ./bcm_100g_dsc_cycle.sh -c 200 -d # echo every command as it is issued
|
||||
# ./bcm_100g_dsc_cycle.sh -c 10 -T # skip tuning, cycle only
|
||||
# ./bcm_100g_dsc_cycle.sh -c 1 -n # dry run, print commands, touch nothing
|
||||
#
|
||||
# Exit code:
|
||||
# 0 = every cycle linked down and up as expected, and all data was collected
|
||||
# 1 = a cycle timed out waiting for link, or data was missing
|
||||
# 2 = bad argument / path
|
||||
# 3 = a required tool is missing (bcmcmd / show / blanton_barchetta_tool)
|
||||
#
|
||||
# The last line is always "### DSC_CYCLE_DONE rc=<n>", for a TTL wait to
|
||||
# key on.
|
||||
#
|
||||
# Design notes:
|
||||
# - POSIX sh. No bash, no Python.
|
||||
# - All runtime output is plain ASCII. The DUT console is not UTF-8 clean:
|
||||
# Chinese comes back as "M-^A"-style garbage, and an error you cannot read
|
||||
# is worse than no error at all. Comments are English for the same reason -
|
||||
# whoever debugs this on the bench is reading it through that console.
|
||||
# - Link state comes from "bcmcmd -n <unit> -c 'ps <port>'". Its header is
|
||||
# two stacked lines, so a column index cannot be derived from it the way
|
||||
# it can from a SONiC table. Instead the data row is found by its first
|
||||
# field ending in "(<port>)" - e.g. ce0(268) - and the state is the field
|
||||
# after it:
|
||||
# ce0(268) !ena 4 100G FD ... <- port disabled
|
||||
# ce0(268) up 4 100G FD ... <- link up
|
||||
# "down" is therefore tested as "not up", which covers !ena as well as an
|
||||
# enabled port whose link has not come back.
|
||||
# - Asking the SDK rather than "show interfaces status" matters here: "port
|
||||
# en=" changes the SDK, and the SONiC table only follows once gbsyncd and
|
||||
# orchagent have synced it into the DB. Since this test pauses gbsyncd,
|
||||
# the SONiC table was the wrong thing to wait on.
|
||||
# - bcmcmd reads stdin, and inside a while loop it will swallow the loop's
|
||||
# input. blanton_traffic_linespeed.sh hit this at v0.3.1 (only the first
|
||||
# VLAN was configured) and bcm_dsc_scan.sh hit it again at v1.3.0 (only
|
||||
# the first port was scanned). Every external command here gets
|
||||
# < /dev/null.
|
||||
# - bcmcmd exits 0 even when the diag shell rejects a command, so the DSC is
|
||||
# verified by counting "SERDES DISPLAY DIAG DATA END", and the retimer read
|
||||
# by counting sections that actually have output under them - not by exit
|
||||
# code, and not by counting headings this script prints itself.
|
||||
#
|
||||
# Cautions:
|
||||
# - !! While gbsyncd is paused the gearbox state stops being synced into the
|
||||
# DB. The wait reads the SDK directly ("ps <port>") so it is not affected,
|
||||
# but anything you check by hand with "show interfaces status" during the
|
||||
# run may be stale. Set PAUSE_GBSYNCD=0 (or -P) if that gets in the way.
|
||||
# - !! On exit, including Ctrl-C, both gbsyncd containers are unpaused again
|
||||
# so nothing is left paused. Set RESUME_GBSYNCD=0 to keep them paused.
|
||||
# - !! phydiag dsc is read-only, but port en=0/1 and tx_taps set do change
|
||||
# the hardware. Do not run this at the same time as blanton_ber.sh,
|
||||
# blanton_traffic_linespeed or blanton_tr518.sh - they take the same ports.
|
||||
# - !! NOT verified on hardware yet.
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Knobs - change these for a different DUT or port, not the flow below
|
||||
# ----------------------------------------------------------------------------
|
||||
PORT=268 # bcmcmd port number, not a port name
|
||||
UNITS="0 1" # units to drive
|
||||
SYS_LANES="0 1 2 3 4 5 6 7" # blanton_barchetta_tool sys lanes, tuning
|
||||
SYS_TX="pre=-10 main=87 post=0" # TX setting applied to each of them
|
||||
|
||||
# tx_taps, one line per lane: "<unit> <lane> <main> <pre1> <post1>"
|
||||
TX_TAPS="
|
||||
0 0 31 4 13
|
||||
0 1 31 4 13
|
||||
0 2 27 4 13
|
||||
0 3 29 4 13
|
||||
1 0 29 4 13
|
||||
1 1 30 4 13
|
||||
1 2 31 4 13
|
||||
1 3 30 4 13
|
||||
"
|
||||
|
||||
TX_LANES="0 1 2 3 4 5 6 7" # retimer TX lanes read back each cycle
|
||||
RX_LANES="0 1 2 3 4 5 6 7" # retimer RX lanes read back each cycle
|
||||
|
||||
SUDO="" # set to "sudo" if not running as root
|
||||
PAUSE_GBSYNCD=1 # pause gbsyncd0/1 before tuning
|
||||
RESUME_GBSYNCD=1 # unpause again on exit, Ctrl-C included
|
||||
GBSYNCD="gbsyncd0 gbsyncd1"
|
||||
|
||||
LINK_WAIT=90 # give up waiting for a link state after this
|
||||
POLL=2 # seconds between "ps <port>" reads while waiting
|
||||
SETTLE=2 # quiet time after link up, before collecting
|
||||
CMD_DELAY=0.2 # pause after every command issued to the DUT
|
||||
|
||||
OUTDIR="${HOME:-/root}/dsc_log"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
CYCLES=0
|
||||
DRYRUN=0
|
||||
DOTUNE=1
|
||||
DEBUG=0
|
||||
RC=0
|
||||
|
||||
usage() {
|
||||
cat >&2 << 'USAGE'
|
||||
Usage: bcm_100g_dsc_cycle.sh <cycles> [options]
|
||||
bcm_100g_dsc_cycle.sh -c <cycles> [options]
|
||||
|
||||
-c <n> how many cycles to run. May also be given as a bare argument.
|
||||
-o <dir> log directory (default: ~/dsc_log)
|
||||
-w <sec> how long to wait for a link state (default: 90)
|
||||
-p <sec> how often to re-read the port state (default: 2)
|
||||
-s <sec> quiet time after link up, before collecting (default: 2)
|
||||
-d debug: echo every command as [CMD] before running it
|
||||
-T skip tuning, go straight to the cycles
|
||||
-P leave gbsyncd alone (no pause, no unpause)
|
||||
-n dry run: print the commands, touch nothing
|
||||
-h this help
|
||||
|
||||
One log per run:
|
||||
~/dsc_log/100G_dsc_cycle_test_<YYYYmmdd_HHMMSS>.log
|
||||
|
||||
Each cycle is one "### TEST CYCLE n/N" section holding, in order: the "ps"
|
||||
output showing the port down, the same showing it up, phydiag dsc for both
|
||||
units, and the retimer serdes-tx-get / serdes-rx-get for sys 0..7.
|
||||
|
||||
Waiting for a port state is a poll, not a single read: "bcmcmd ps <port>" on
|
||||
every unit, every -p seconds, until both agree or -w seconds have passed.
|
||||
|
||||
Examples:
|
||||
./bcm_100g_dsc_cycle.sh 200
|
||||
./bcm_100g_dsc_cycle.sh 200 -d
|
||||
./bcm_100g_dsc_cycle.sh -c 20 -T -w 120
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ $# -eq 0 ] && usage
|
||||
|
||||
# getopts stops at the first argument that is not an option, so "200 -d" would
|
||||
# leave -d unread. Parse, take the bare count, then parse whatever followed it.
|
||||
# getopts inside a function reads the function's own arguments; OPTIND is
|
||||
# global, so it has to be reset on each entry.
|
||||
PARSED=0
|
||||
parse_opts() {
|
||||
OPTIND=1
|
||||
while getopts "c:o:w:p:s:dTPnh" opt; do
|
||||
case "$opt" in
|
||||
c) CYCLES="$OPTARG" ;;
|
||||
o) OUTDIR="$OPTARG" ;;
|
||||
w) LINK_WAIT="$OPTARG" ;;
|
||||
p) POLL="$OPTARG" ;;
|
||||
s) SETTLE="$OPTARG" ;;
|
||||
d) DEBUG=1 ;;
|
||||
T) DOTUNE=0 ;;
|
||||
P) PAUSE_GBSYNCD=0; RESUME_GBSYNCD=0 ;;
|
||||
n) DRYRUN=1 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
PARSED=$((OPTIND - 1))
|
||||
}
|
||||
|
||||
parse_opts "$@"
|
||||
shift "$PARSED"
|
||||
|
||||
# Allow the count as a bare argument: "... 200" == "... -c 200". That is what
|
||||
# most people type first, and answering it with an error helps nobody.
|
||||
if [ "$CYCLES" = "0" ] && [ $# -ge 1 ]; then
|
||||
case "$1" in
|
||||
-*) : ;; # an option; leave it for the second pass
|
||||
*) CYCLES="$1"; shift ;; # anything else is meant to be the count,
|
||||
esac # and the check below says so if it is not
|
||||
fi
|
||||
|
||||
# Anything after the count is options again: "200 -d -T"
|
||||
if [ $# -gt 0 ]; then
|
||||
parse_opts "$@"
|
||||
shift "$PARSED"
|
||||
fi
|
||||
|
||||
if [ $# -gt 0 ]; then
|
||||
echo "ERROR: unexpected argument '$1'" >&2
|
||||
echo " run with no argument to see the usage" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
case "$CYCLES" in
|
||||
''|*[!0-9]*)
|
||||
echo "ERROR: cycle count must be a positive integer (got: '$CYCLES')" >&2
|
||||
echo " try: $0 200 or $0 -c 200" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
if [ "$CYCLES" -lt 1 ]; then
|
||||
echo "ERROR: cycle count must be at least 1 (got: $CYCLES)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Colour, only when stdout is a terminal, so a redirected run stays clean
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
|
||||
C_INFO=$(printf '\033[1;34m') # blue
|
||||
C_OK=$(printf '\033[1;32m') # green
|
||||
C_WARN=$(printf '\033[1;33m') # yellow
|
||||
C_ERR=$(printf '\033[1;31m') # red
|
||||
C_CMD=$(printf '\033[90m') # dark grey
|
||||
C_OFF=$(printf '\033[0m')
|
||||
else
|
||||
C_INFO=''; C_OK=''; C_WARN=''; C_ERR=''; C_CMD=''; C_OFF=''
|
||||
fi
|
||||
|
||||
# A dry run touches nothing, so it must not sleep either - otherwise
|
||||
# "-c 200 -n" sits there for as long as the real thing would.
|
||||
naptime() { [ "$DRYRUN" = "1" ] && return 0; [ "${1:-0}" = "0" ] && return 0; sleep "$1"; }
|
||||
|
||||
info() { printf '%s[info]%s %s\n' "$C_INFO" "$C_OFF" "$*"; }
|
||||
ok() { printf '%s[ OK ]%s %s\n' "$C_OK" "$C_OFF" "$*"; }
|
||||
warn() { printf '%s[warn]%s %s\n' "$C_WARN" "$C_OFF" "$*" >&2; }
|
||||
err() { printf '%s[ERR ]%s %s\n' "$C_ERR" "$C_OFF" "$*" >&2; }
|
||||
|
||||
# Both -d and -n print commands. During collection the whole block is
|
||||
# redirected into a temp file, so those lines would end up in the file with
|
||||
# nothing on screen. Bind fd 9 to the terminal up front and print there.
|
||||
if { [ "$DEBUG" = "1" ] || [ "$DRYRUN" = "1" ]; } && ( true > /dev/tty ) 2>/dev/null; then
|
||||
exec 9> /dev/tty
|
||||
else
|
||||
exec 9>&1
|
||||
fi
|
||||
# ':' is a special built-in in dash (the DUT's /bin/sh); a failed redirection
|
||||
# on one kills the whole shell with no message. Hence (true > /dev/tty) in a
|
||||
# subshell above - if that cannot open, only the subshell dies.
|
||||
|
||||
# Print a command the way you would type it: quote any argument that needs it,
|
||||
# so the line can be copied back to the prompt as is.
|
||||
# NOTE: sh has no local variables - every name assigned here is global. All
|
||||
# helper variables are therefore prefixed. _show_cmd used to assign "tag",
|
||||
# which silently overwrote the caller's cycle label and turned every later
|
||||
# message into "[info] [CMD] : Wait Port Link Down".
|
||||
_show_cmd() {
|
||||
_sc_tag="$1"; _sc_col="$2"; shift 2
|
||||
{
|
||||
printf '%s%s%s' "$_sc_col" "$_sc_tag" "$C_OFF"
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
''|*[!A-Za-z0-9_./=:,+-]*) printf " '%s'" "$a" ;;
|
||||
*) printf ' %s' "$a" ;;
|
||||
esac
|
||||
done
|
||||
printf '\n'
|
||||
} >&9
|
||||
}
|
||||
|
||||
# Every external command goes through here: never reads stdin, echoed when
|
||||
# asked, and paced by CMD_DELAY so the DUT is not hammered.
|
||||
run() {
|
||||
[ "$DEBUG" = "1" ] && [ "$DRYRUN" = "0" ] && _show_cmd '[CMD]' "$C_CMD" "$@"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" "$@"
|
||||
return 0
|
||||
fi
|
||||
"$@" < /dev/null
|
||||
_cmd_rc=$?
|
||||
naptime "$CMD_DELAY"
|
||||
return $_cmd_rc
|
||||
}
|
||||
|
||||
# Same, but discards output. Do not write "> /dev/null" at the call site or
|
||||
# the [CMD] line goes with it.
|
||||
runq() {
|
||||
[ "$DEBUG" = "1" ] && [ "$DRYRUN" = "0" ] && _show_cmd '[CMD]' "$C_CMD" "$@"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" "$@"
|
||||
return 0
|
||||
fi
|
||||
"$@" < /dev/null > /dev/null 2>&1
|
||||
_cmd_rc=$?
|
||||
naptime "$CMD_DELAY"
|
||||
return $_cmd_rc
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Preflight
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DRYRUN" = "0" ]; then
|
||||
command -v bcmcmd > /dev/null 2>&1 || { err "not found: bcmcmd"; exit 3; }
|
||||
# -T only skips tuning; the per-cycle retimer read still needs this tool
|
||||
if [ ! -x /usr/sbin/blanton_barchetta_tool ]; then
|
||||
err "not found: /usr/sbin/blanton_barchetta_tool (tuning and the per-cycle retimer read both use it)"
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
|
||||
# busybox sleep may not take a fraction
|
||||
if [ "$DRYRUN" = "0" ] && [ "$CMD_DELAY" != "0" ] && ! sleep "$CMD_DELAY" 2>/dev/null; then
|
||||
warn "sleep '$CMD_DELAY' not supported, falling back to 1"
|
||||
CMD_DELAY=1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTDIR" 2>/dev/null || { err "cannot create outdir: $OUTDIR"; exit 2; }
|
||||
[ -w "$OUTDIR" ] || { err "outdir not writable: $OUTDIR"; exit 2; }
|
||||
|
||||
TS=$(date '+%Y%m%d_%H%M%S')
|
||||
LOG="${OUTDIR}/100G_dsc_cycle_test_${TS}.log"
|
||||
: > "$LOG" || { err "cannot write $LOG"; exit 2; }
|
||||
|
||||
STATUS_TMP=$(mktemp) || { err "mktemp failed"; exit 2; }
|
||||
CYCLE_TMP=$(mktemp) || { err "mktemp failed"; exit 2; }
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# gbsyncd must go back the way it was found, or the container stays paused
|
||||
# ----------------------------------------------------------------------------
|
||||
PAUSED=0
|
||||
resume_gbsyncd() {
|
||||
[ "$PAUSED" = "1" ] || return 0
|
||||
[ "$RESUME_GBSYNCD" = "1" ] || return 0
|
||||
PAUSED=0
|
||||
for _rg_c in $GBSYNCD; do
|
||||
if runq $SUDO docker unpause "$_rg_c"; then
|
||||
info "gbsyncd resumed: $_rg_c"
|
||||
else
|
||||
warn "docker unpause $_rg_c failed - check it by hand"
|
||||
fi
|
||||
done
|
||||
}
|
||||
on_exit() { rm -f "$STATUS_TMP" "$CYCLE_TMP"; resume_gbsyncd; }
|
||||
on_int() {
|
||||
echo ""
|
||||
warn "interrupted by user"
|
||||
resume_gbsyncd
|
||||
printf '\n### DSC_CYCLE_DONE rc=130 (INTERRUPTED)\n' >> "$LOG"
|
||||
printf '### DSC_CYCLE_DONE rc=130 (INTERRUPTED)\n'
|
||||
rm -f "$STATUS_TMP" "$CYCLE_TMP"
|
||||
exit 130
|
||||
}
|
||||
trap on_exit EXIT
|
||||
trap on_int INT TERM
|
||||
|
||||
tee_log() { tee -a "$LOG"; }
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Link state. Find which field "Oper" is from the header row, then read that
|
||||
# field on every data row. Returns 0 only when every interface matches.
|
||||
# The table itself is left in $STATUS_TMP for the caller to print.
|
||||
# ----------------------------------------------------------------------------
|
||||
# $1 = "up" or "down". Runs "ps <port>" on every unit, leaves the combined
|
||||
# output in $STATUS_TMP for the caller to print, and returns 0 only when every
|
||||
# unit agrees.
|
||||
link_is() {
|
||||
_li_want="$1"
|
||||
: > "$STATUS_TMP"
|
||||
for _li_u in $UNITS; do
|
||||
printf '\n[unit %s] bcmcmd -n %s -c "ps %s"\n' "$_li_u" "$_li_u" "$PORT" >> "$STATUS_TMP"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" bcmcmd -n "$_li_u" -c "ps $PORT"
|
||||
printf ' ce0(%s) %s 4 100G FD (dry-run)\n' \
|
||||
"$PORT" "$([ "$_li_want" = up ] && echo 'up ' || echo '!ena')" >> "$STATUS_TMP"
|
||||
continue
|
||||
fi
|
||||
[ "$DEBUG" = "1" ] && _show_cmd '[CMD]' "$C_CMD" bcmcmd -n "$_li_u" -c "ps $PORT"
|
||||
bcmcmd -n "$_li_u" -c "ps $PORT" < /dev/null >> "$STATUS_TMP" 2>&1
|
||||
naptime "$CMD_DELAY"
|
||||
done
|
||||
|
||||
# The data row is the one whose first field ends in "(<port>)"; the state
|
||||
# is the field right after it. "down" is "not up", so !ena counts.
|
||||
awk -v port="$PORT" -v want="$_li_want" -v units="$(echo "$UNITS" | wc -w)" '
|
||||
$1 ~ ("\\(" port "\\)$") {
|
||||
n++
|
||||
if (want == "up") { if ($2 != "up") bad++ }
|
||||
else { if ($2 == "up") bad++ }
|
||||
}
|
||||
END { exit (n >= units && bad == 0) ? 0 : 1 }
|
||||
' "$STATUS_TMP"
|
||||
}
|
||||
|
||||
# $1 = wanted state (Down/up), $2 = label for the messages
|
||||
wait_link() {
|
||||
_wl_want="$1"; _wl_tag="$2"
|
||||
_wl_waited=0
|
||||
while :; do
|
||||
if link_is "$_wl_want"; then
|
||||
{
|
||||
printf '\n--- port %s after %ss ---\n' "$_wl_want" "$_wl_waited"
|
||||
cat "$STATUS_TMP"
|
||||
} | tee_log
|
||||
ok "$_wl_tag : port $_wl_want after ${_wl_waited}s"
|
||||
return 0
|
||||
fi
|
||||
[ "$_wl_waited" -ge "$LINK_WAIT" ] && break
|
||||
naptime "$POLL"
|
||||
_wl_waited=$((_wl_waited + POLL))
|
||||
done
|
||||
{
|
||||
printf '\n--- TIMEOUT waiting for port %s (%ss) ---\n' "$_wl_want" "$LINK_WAIT"
|
||||
cat "$STATUS_TMP"
|
||||
} | tee_log
|
||||
err "$_wl_tag : TIMEOUT waiting for port $_wl_want (${LINK_WAIT}s)"
|
||||
return 1
|
||||
}
|
||||
|
||||
port_en() {
|
||||
for _pe_u in $UNITS; do
|
||||
runq bcmcmd -n "$_pe_u" -c "port ${PORT} en=$1"
|
||||
done
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Log header
|
||||
# ----------------------------------------------------------------------------
|
||||
{
|
||||
printf '=====================================================================\n'
|
||||
printf '100G DSC cycle test\n'
|
||||
printf ' script : bcm_100g_dsc_cycle.sh V1.2.0\n'
|
||||
printf ' started : %s\n' "$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
printf ' cycles : %s\n' "$CYCLES"
|
||||
printf ' port : %s units : %s\n' "$PORT" "$UNITS"
|
||||
printf ' tuning : %s\n' "$([ "$DOTUNE" = 1 ] && echo yes || echo 'skipped (-T)')"
|
||||
printf ' tx lanes : %s\n' "$TX_LANES"
|
||||
printf ' rx lanes : %s\n' "$RX_LANES"
|
||||
printf ' link wait: %ss poll %ss settle %ss cmd delay %ss\n' \
|
||||
"$LINK_WAIT" "$POLL" "$SETTLE" "$CMD_DELAY"
|
||||
printf '=====================================================================\n'
|
||||
} >> "$LOG"
|
||||
|
||||
info "log : $LOG"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 1. Tuning, once
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DOTUNE" = "1" ]; then
|
||||
info "Tuning"
|
||||
printf '\n### TUNING %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
if [ "$PAUSE_GBSYNCD" = "1" ]; then
|
||||
for c in $GBSYNCD; do
|
||||
if runq $SUDO docker pause "$c"; then
|
||||
PAUSED=1
|
||||
info "gbsyncd paused: $c"
|
||||
else
|
||||
warn "docker pause $c failed (container not running?)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0 2>&1 | tee_log
|
||||
|
||||
for l in $SYS_LANES; do
|
||||
# shellcheck disable=SC2086
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys "$l" $SYS_TX 2>&1 | tee_log
|
||||
done
|
||||
|
||||
echo "$TX_TAPS" | while read -r u lane main pre1 post1; do
|
||||
[ -n "${u:-}" ] || continue
|
||||
run bcmcmd -n "$u" -c \
|
||||
" dsh -c 'phydiag ${PORT} tx_taps set lane=${lane} main=${main} pre1=${pre1} post1=${post1}' " \
|
||||
2>&1 | tee_log
|
||||
done
|
||||
|
||||
ok "Tuning done"
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 2. Test cycles
|
||||
# ----------------------------------------------------------------------------
|
||||
n=1
|
||||
while [ "$n" -le "$CYCLES" ]; do
|
||||
tag="Test Cycle $n / $CYCLES"
|
||||
|
||||
printf '\n\n### TEST CYCLE %s/%s %s\n' "$n" "$CYCLES" \
|
||||
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
# ---- Disable Port ------------------------------------------------------
|
||||
info "$tag : Disable Port"
|
||||
port_en 0
|
||||
|
||||
info "$tag : Wait Port Link Down"
|
||||
wait_link down "$tag" || RC=1
|
||||
|
||||
# ---- Enable Port -------------------------------------------------------
|
||||
info "$tag : Enable Port"
|
||||
port_en 1
|
||||
|
||||
info "$tag : Wait Link up"
|
||||
if wait_link up "$tag"; then
|
||||
naptime "$SETTLE"
|
||||
else
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# ---- Collect -----------------------------------------------------------
|
||||
# Written to a per-cycle temp file first, then appended. Appending to the
|
||||
# log and then re-reading it to check would mean re-reading an ever larger
|
||||
# file every round - O(n^2) over 200 cycles.
|
||||
info "$tag : Collect Data"
|
||||
: > "$CYCLE_TMP"
|
||||
for u in $UNITS; do
|
||||
{
|
||||
printf '\n--- phydiag %s dsc : unit %s ---\n' "$PORT" "$u"
|
||||
run bcmcmd -n "$u" -c "dsh -c 'phydiag ${PORT} dsc'" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
info "$tag : Read Retimer TX"
|
||||
for l in $TX_LANES; do
|
||||
{
|
||||
printf '\n--- serdes-tx-get sys %s ---\n' "$l"
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys "$l" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
info "$tag : Read Retimer RX"
|
||||
for l in $RX_LANES; do
|
||||
{
|
||||
printf '\n--- serdes-rx-get sys %s ---\n' "$l"
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys "$l" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
cat "$CYCLE_TMP" >> "$LOG"
|
||||
|
||||
if [ "$DRYRUN" = "0" ]; then
|
||||
# bcmcmd exits 0 even when it failed, so count the end marker instead
|
||||
got=$(grep -c 'SERDES DISPLAY DIAG DATA END' "$CYCLE_TMP" 2>/dev/null || true)
|
||||
want=$(echo "$UNITS" | wc -w)
|
||||
if [ "${got:-0}" -lt "$want" ]; then
|
||||
err "$tag : got only ${got:-0}/${want} DSC dumps"
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# The section headings are printed by this script, so counting them
|
||||
# only proves the loop ran. Count sections that have output under them.
|
||||
for d in tx rx; do
|
||||
case "$d" in
|
||||
tx) wantl="$TX_LANES" ;;
|
||||
rx) wantl="$RX_LANES" ;;
|
||||
esac
|
||||
got_l=$(awk -v pat="--- serdes-${d}-get sys " '
|
||||
index($0, pat) == 1 { if (insec && body) ok++; insec = 1; body = 0; next }
|
||||
/^--- / || /^### / { if (insec && body) ok++; insec = 0; next }
|
||||
insec && NF { body = 1 }
|
||||
END { if (insec && body) ok++; print ok + 0 }
|
||||
' "$CYCLE_TMP")
|
||||
want_l=$(echo "$wantl" | wc -w)
|
||||
if [ "${got_l:-0}" -lt "$want_l" ]; then
|
||||
err "$tag : retimer $(echo "$d" | tr 'a-z' 'A-Z') returned data for only ${got_l:-0}/${want_l} lanes"
|
||||
RC=1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
n=$((n + 1))
|
||||
done
|
||||
|
||||
resume_gbsyncd
|
||||
|
||||
printf '\n### DSC_CYCLE_DONE rc=%s\n' "$RC" >> "$LOG"
|
||||
if [ "$RC" = "0" ]; then
|
||||
ok "all $CYCLES cycles done -> $LOG"
|
||||
else
|
||||
err "some cycles did not complete as expected - see $LOG"
|
||||
fi
|
||||
printf '### DSC_CYCLE_DONE rc=%s\n' "$RC"
|
||||
exit "$RC"
|
||||
@@ -12,6 +12,9 @@
|
||||
# 避免灌爆 Tera Term logopen 的 log
|
||||
# V1.2.0 20260906 新增 -f 自訂 log 檔名(支援 %u / %t 佔位符)與
|
||||
# -F 強制覆蓋;預設不覆蓋既有檔案,自動加序號
|
||||
# V1.3.0 20260907 修正 bcmcmd 讀取 stdin 導致掃描迴圈在第一筆之後
|
||||
# 提前結束的問題(迴圈改用 fd 3,bcmcmd 導向
|
||||
# /dev/null);新增實際迭代筆數與清單筆數的檢查
|
||||
#
|
||||
# Description:
|
||||
# 在 Blanton SONiC DUT 上批次執行 Broadcom diag shell 的
|
||||
@@ -278,16 +281,18 @@ for U in $UNITS; do
|
||||
: > "$LOG" || { echo "ERROR: cannot write $LOG" >&2; exit 2; }
|
||||
LOGS="$LOGS $LOG"
|
||||
|
||||
printf '### SCAN_BEGIN unit=%s time=%s total=%s script=bcm_dsc_scan_V1.1.0\n' \
|
||||
printf '### SCAN_BEGIN unit=%s time=%s total=%s script=bcm_dsc_scan_V1.3.0\n' \
|
||||
"$U" "$(date '+%Y-%m-%d %H:%M:%S')" "$TOTAL" >> "$LOG"
|
||||
|
||||
echo "[unit $U] scanning $TOTAL ports -> $LOG" >&2
|
||||
|
||||
SEQ=0
|
||||
while IFS=, read -r P SPD; do
|
||||
while IFS=, read -r P SPD <&3; do
|
||||
SEQ=$((SEQ + 1))
|
||||
printf '### PORT unit=%s port=%s speed=%s seq=%s/%s\n' "$U" "$P" "$SPD" "$SEQ" "$TOTAL" >> "$LOG"
|
||||
bcmcmd -n "$U" -c "dsh -c 'phydiag $P dsc'" >> "$LOG" 2>&1 || RC=1
|
||||
# bcmcmd 會讀取 stdin,若不導成 /dev/null 會把 PORTFILE 剩餘內容吸走,
|
||||
# 導致迴圈在第一筆之後就讀到 EOF 而提前結束。
|
||||
bcmcmd -n "$U" -c "dsh -c 'phydiag $P dsc'" >> "$LOG" 2>&1 < /dev/null || RC=1
|
||||
|
||||
if [ "$TTY" -eq 1 ]; then
|
||||
printf '\r[unit %s] %s/%s port=%-5s' "$U" "$SEQ" "$TOTAL" "$P" >&2
|
||||
@@ -296,19 +301,26 @@ for U in $UNITS; do
|
||||
fi
|
||||
|
||||
sleep "$DELAY"
|
||||
done < "$PORTFILE"
|
||||
done 3< "$PORTFILE"
|
||||
|
||||
[ "$TTY" -eq 1 ] && printf '\n' >&2
|
||||
printf '### SCAN_END unit=%s time=%s\n' "$U" "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
# 概略檢查:每筆 PORT marker 應對應一個 dsc 區塊
|
||||
# 檢查 1:實際迭代筆數是否等於 port 清單筆數。
|
||||
# 迴圈若因 stdin 被子程序吸走而提前結束,只靠下面的 marker/dump 比對抓不到。
|
||||
if [ "$SEQ" -ne "$TOTAL" ]; then
|
||||
echo "ERROR: unit $U 只跑了 $SEQ/$TOTAL 筆,迴圈提前結束" >&2
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# 檢查 2:每筆 PORT marker 應對應一個 dsc 區塊
|
||||
MARK=$(grep -c '^### PORT ' "$LOG" || true)
|
||||
DUMP=$(grep -c 'SERDES DISPLAY DIAG DATA END' "$LOG" || true)
|
||||
if [ "$MARK" -ne "$DUMP" ]; then
|
||||
echo "WARN: unit $U markers=$MARK dsc_blocks=$DUMP (有 port 未取得完整 dump)" >&2
|
||||
RC=1
|
||||
fi
|
||||
echo "Saved: $LOG (${DUMP}/${MARK} dumps)" >&2
|
||||
echo "Saved: $LOG (${DUMP}/${TOTAL} dumps)" >&2
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
@@ -1,8 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# blanton_ber.sh -- SerDes BER test over every cabled port (PRBS via phydiag)
|
||||
# blanton_ber.sh -- SerDes BER test, cabled ports PLUS the 100G uplink
|
||||
# =============================================================================
|
||||
# Version History:
|
||||
# V1.1.1 2026-09-15 Fix: "the BCM shell rejected 'phydiag ... prbs get'" on
|
||||
# a run that was in fact working. The rejection check
|
||||
# grepped the WHOLE output, case-insensitively, for
|
||||
# ERROR|Invalid|... -- and 'prbs get' prints one line per
|
||||
# lane, so a lane that had not locked ("PRBS Error!")
|
||||
# matched. The command was fine; the output was data.
|
||||
# Two consequences: a good run reported a hard error, and
|
||||
# the lock-count check that was supposed to judge exactly
|
||||
# those lanes never ran. Per-lane result lines are now
|
||||
# excluded from the check, the match is case-sensitive,
|
||||
# and a real rejection prints the offending line.
|
||||
# The rejection check had been there since V1.0.0, so
|
||||
# every earlier run carried the same blind spot.
|
||||
# V1.1.0 2026-09-15 Adds port 268, the 100G uplink, which the old port list
|
||||
# deliberately skipped (it jumps 256-259 -> 270-308).
|
||||
# The per-lane table is unchanged; the summary now breaks
|
||||
# each unit into "200G ports" and "100G p268" plus a
|
||||
# TOTAL row, so the uplink cannot hide inside a 432-lane
|
||||
# average. A group that reports no lane at all counts as
|
||||
# a failure, not a pass - silence on the port under test
|
||||
# is not a result. Developed on the bench as
|
||||
# blanton_ber_test.sh and folded back over this file once
|
||||
# it was proven, so there is one BER script again.
|
||||
# V1.0.1 2026-08-28 Fix: every lane reported NA and the Port column showed
|
||||
# "NA". bcmcmd returns CRLF, and the BER is the last token
|
||||
# on its line, so the CR stayed glued to the number: it
|
||||
@@ -19,8 +42,24 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# This is the WHOLE-SWITCH BER sweep: every cabled port, both lanes, PRBS-31
|
||||
# through the SerDes. Not to be confused with port_prbs_monitor.sh, which drives
|
||||
# the two 100G uplinks only and polls them continuously.
|
||||
# through the SerDes -- and, unlike blanton_ber.sh, the 100G uplink as well.
|
||||
# Not to be confused with port_prbs_monitor.sh, which drives the two 100G
|
||||
# uplinks only and polls them continuously.
|
||||
#
|
||||
# The port list is two groups:
|
||||
# BER_PORTS_200G the cabled 200G ports (432 lanes per unit)
|
||||
# BER_PORTS_100G the 100G uplink, port 268 = ce0, one per unit
|
||||
# They are measured in one phydiag pass, but the summary reports them
|
||||
# separately per unit, plus a TOTAL:
|
||||
#
|
||||
# Unit GROUP LANES PASS FAIL NA WORST BER ...
|
||||
# unit 0 200G ports 432 432 0 0 2.07e-08 PASS
|
||||
# unit 0 100G p268 4 4 0 0 1.10e-11 PASS
|
||||
# unit 0 TOTAL 436 436 0 0 2.07e-08 PASS
|
||||
# unit 1 ...
|
||||
#
|
||||
# A 100G row of "NO DATA" means the uplink reported no lane at all, and is
|
||||
# treated as a failure -- silence on the port under test is not a pass.
|
||||
#
|
||||
# Order of operations (matches tools/BER_Test.txt):
|
||||
# init phydiag <ports> prbs set p=3 arm PRBS on every lane
|
||||
@@ -51,7 +90,7 @@
|
||||
# -u|--unit <0|1|all|0,1> bcmcmd unit(s). Default: $BER_UNITS ("0 1")
|
||||
# -c|--criteria <ber> report: PASS threshold (default $BER_MAX)
|
||||
# -i|--interval <sec> start: prbsstat Interval (default $BER_INTERVAL)
|
||||
# -p|--ports <list> override the port list
|
||||
# -p|--ports <list> override the whole port list (both groups)
|
||||
# -F|--fail-only report: list only the lanes that FAIL
|
||||
# --tsv report: tab-separated (paste into Excel)
|
||||
# --raw report: also print the raw 'prbsstat Ber' output
|
||||
@@ -63,7 +102,8 @@
|
||||
#
|
||||
# Environment overrides:
|
||||
# BCMCMD_CMD=bcmcmd BER_UNITS="0 1" BER_MAX=1e-6 BER_INTERVAL=30
|
||||
# BER_PORTS="1-24,..." BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1
|
||||
# BER_PORTS="1-24,..." BER_PORTS_200G=... BER_PORTS_100G=268
|
||||
# BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1
|
||||
# =============================================================================
|
||||
|
||||
# === Config ==================================================================
|
||||
@@ -73,7 +113,18 @@ BER_UNITS="${BER_UNITS:-0 1}" # one DUT = two switch boards (0 and 1)
|
||||
# The cabled ports, in bcm logical-port numbering. Same list as
|
||||
# blanton_traffic_linespeed.sh's TL_PS_PORTS -- kept as one string so it can be
|
||||
# pasted straight from/into a manual phydiag command.
|
||||
BER_PORTS="${BER_PORTS:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}"
|
||||
#
|
||||
# Note the gap: the list runs 256-259 then 270-308, so 260..269 are not in it.
|
||||
# 268 is ce0, the 100G uplink, and it is the reason this fork exists.
|
||||
BER_PORTS_200G="${BER_PORTS_200G:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}"
|
||||
|
||||
# The 100G uplink. One port per unit -- unit 0 and unit 1 each have their own
|
||||
# ce0(268), so a two-unit run measures two of them. Comma-separated if more are
|
||||
# ever added; the summary treats everything in here as the "100G" group.
|
||||
BER_PORTS_100G="${BER_PORTS_100G:-268}"
|
||||
|
||||
# What actually goes on the phydiag command line. -p|--ports overrides the lot.
|
||||
BER_PORTS="${BER_PORTS:-${BER_PORTS_200G},${BER_PORTS_100G}}"
|
||||
|
||||
BER_POLY="${BER_POLY:-p=3}" # passed to 'prbs set'
|
||||
BER_INTERVAL="${BER_INTERVAL:-30}" # passed to 'prbsstat STArt Interval='
|
||||
@@ -111,6 +162,24 @@ _ber_dbg() { [ "$DEBUG_MODE" = "1" ] && echo -e "\033[90m[DEBG] $*\033[0m" >&2;
|
||||
BER_ERR_PAT="${BER_ERR_PAT:-ERROR|Failed to execute|Invalid|Syntax error|Unavailable|Unknown command}"
|
||||
BER_BENIGN_PAT="${BER_BENIGN_PAT:-polling socket timeout}"
|
||||
|
||||
# A per-lane result line -- " 185 : PRBS OK!..." from 'prbs get', or
|
||||
# " 203[1] : 5.05e-09" from 'prbsstat Ber'. These are MEASUREMENTS, never
|
||||
# shell diagnostics, and they must be kept out of the rejection check:
|
||||
# a lane that did not lock prints its own wording (PRBS Error!, and similar),
|
||||
# which a case-insensitive match on "ERROR" reads as the BCM shell refusing
|
||||
# the command. That is what turned a perfectly good run into
|
||||
# [ERR ] unit 1: the BCM shell rejected 'phydiag ... prbs get'
|
||||
# while init/start/report were in fact working -- and it skipped the
|
||||
# lock-count check that was supposed to judge those very lanes.
|
||||
BER_LANE_LINE='^[ \t]*[0-9]+(\[[0-9]+\])?[ \t]*:'
|
||||
|
||||
# _ber_shell_error <output> -- echo the offending line(s), return 0 if any
|
||||
_ber_shell_error() {
|
||||
printf '%s\n' "$1" | tr -d '\r' \
|
||||
| grep -vE "$BER_LANE_LINE" \
|
||||
| grep -E "$BER_ERR_PAT"
|
||||
}
|
||||
|
||||
# _ber_bcm <unit> <phydiag args...> -- wraps in dsh -c 'phydiag <ports> ...'
|
||||
# echoes the DUT output on stdout, returns 0 ok / 1 failed
|
||||
_ber_bcm() {
|
||||
@@ -125,11 +194,14 @@ _ber_bcm() {
|
||||
local out rc
|
||||
out=$("$BCMCMD_CMD" -n "$unit" -c "$cmd" </dev/null 2>&1); rc=$?
|
||||
printf '%s\n' "$out"
|
||||
if echo "$out" | grep -qiE "$BER_ERR_PAT"; then
|
||||
local why
|
||||
why=$(_ber_shell_error "$out")
|
||||
if [ -n "$why" ]; then
|
||||
_ber_err "unit $unit: the BCM shell rejected 'phydiag ... $*'"
|
||||
printf '%s\n' "$why" | sed 's/^/ /' >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$rc" -ne 0 ] && ! echo "$out" | grep -qiE "$BER_BENIGN_PAT"; then
|
||||
if [ "$rc" -ne 0 ] && ! printf '%s\n' "$out" | grep -qiE "$BER_BENIGN_PAT"; then
|
||||
_ber_err "unit $unit: bcmcmd exited $rc on 'phydiag ... $*'"
|
||||
return 1
|
||||
fi
|
||||
@@ -243,13 +315,32 @@ _ber_render() {
|
||||
-v FAILONLY="$BER_FAIL_ONLY" \
|
||||
-v TSV="$BER_TSV" \
|
||||
-v COLOR="$color" \
|
||||
-v SIDE="$side" '
|
||||
-v SIDE="$side" \
|
||||
-v P100="$BER_PORTS_100G" '
|
||||
function vc(v) {
|
||||
if (!COLOR) return v
|
||||
if (v == "PASS") return "\033[32m" v "\033[0m"
|
||||
if (v == "FAIL") return "\033[31m" v "\033[0m"
|
||||
return "\033[33m" v "\033[0m"
|
||||
}
|
||||
# Fold one lane into a group tally. Called once for its own group and
|
||||
# and once for ALL, so the TOTAL row is never a hand-added sum that can
|
||||
# drift away from the parts.
|
||||
function acct(g, verdict, berstr, berval, label) {
|
||||
GN[g]++
|
||||
if (verdict == "PASS") GP[g]++
|
||||
else if (verdict == "FAIL") GF[g]++
|
||||
else GNA[g]++
|
||||
if (berval >= 0 && (!(g in GW) || berval > GW[g])) {
|
||||
GW[g] = berval; GWS[g] = berstr; GWL[g] = label
|
||||
}
|
||||
}
|
||||
function emit(g, lanes) {
|
||||
lanes = GN[g] + 0
|
||||
printf "SUM %s %d %d %d %d %s %s\n", g, lanes, GP[g] + 0, GF[g] + 0, GNA[g] + 0,
|
||||
(lanes > 0 && (g in GWS)) ? GWS[g] : "-",
|
||||
(lanes > 0 && (g in GWL)) ? GWL[g] : "-" > SIDE
|
||||
}
|
||||
BEGIN {
|
||||
LIM = MAXBER + 0
|
||||
NC = 4
|
||||
@@ -258,6 +349,18 @@ _ber_render() {
|
||||
HDR[3] = "BER"; AL[3] = 1
|
||||
HDR[4] = "RESULT"; AL[4] = 1
|
||||
worst = -1; worstlane = "-"
|
||||
|
||||
# Expand the 100G list ("268" or "268,270-271") into a lookup, so a lane
|
||||
# can be put in the right group by its port number alone.
|
||||
np = split(P100, pp, /[ ,\t]+/)
|
||||
for (i2 = 1; i2 <= np; i2++) {
|
||||
if (pp[i2] == "") continue
|
||||
if (split(pp[i2], rr, "-") == 2) {
|
||||
for (j2 = rr[1] + 0; j2 <= rr[2] + 0; j2++) IS100[j2] = 1
|
||||
} else {
|
||||
IS100[pp[i2] + 0] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
# bcmcmd hands back CRLF. On the cd/traffic report the counter is followed by
|
||||
# a delta, so the CR got trimmed with the rest of the line; here the BER is
|
||||
@@ -282,6 +385,8 @@ _ber_render() {
|
||||
seen++
|
||||
n++
|
||||
PORT[n] = port; LANE[n] = lane; BERS[n] = ber
|
||||
grp = ((port + 0) in IS100) ? "P100" : "P200"
|
||||
GRP[n] = grp
|
||||
# a lane that reported something non-numeric has no usable measurement
|
||||
if (ber ~ /^[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$/) {
|
||||
v = ber + 0
|
||||
@@ -290,7 +395,10 @@ _ber_render() {
|
||||
if (v > worst) { worst = v; worstlane = port "[" lane "]"; worststr = ber }
|
||||
} else {
|
||||
V[n] = "NA"; nna++
|
||||
v = -1
|
||||
}
|
||||
acct(grp, V[n], ber, v, port "[" lane "]")
|
||||
acct("ALL", V[n], ber, v, port "[" lane "]")
|
||||
}
|
||||
END {
|
||||
if (seen == 0) {
|
||||
@@ -327,19 +435,32 @@ _ber_render() {
|
||||
}
|
||||
|
||||
print ""
|
||||
printf "# lanes measured : %d\n", n
|
||||
printf "# lanes measured : %d (200G %d, 100G %d)\n", n, GN["P200"] + 0, GN["P100"] + 0
|
||||
printf "# %s %d %s %d %s %d (criteria BER < %s)\n", \
|
||||
vc("PASS"), npass + 0, vc("FAIL"), nfail + 0, vc("NA"), nna + 0, MAXBER
|
||||
if (worst >= 0)
|
||||
printf "# worst lane : %s %s\n", worstlane, worststr
|
||||
if (SIDE != "" && SIDE != "/dev/null")
|
||||
printf "SUM %d %d %d %d %s %s\n", n, npass + 0, nfail + 0, nna + 0, \
|
||||
(worst >= 0 ? worststr : "-"), worstlane > SIDE
|
||||
if (GN["P100"] + 0 == 0)
|
||||
printf "# %s the 100G port list (%s) matched no lane in this capture\n", vc("NA"), P100
|
||||
else
|
||||
printf "# 100G p%s : %s %d %s %d %s %d worst %s %s\n", P100,
|
||||
vc("PASS"), GP["P100"] + 0, vc("FAIL"), GF["P100"] + 0,
|
||||
vc("NA"), GNA["P100"] + 0,
|
||||
("P100" in GWS) ? GWS["P100"] : "-",
|
||||
("P100" in GWL) ? GWL["P100"] : "-"
|
||||
if (SIDE != "" && SIDE != "/dev/null") {
|
||||
emit("P200"); emit("P100"); emit("ALL")
|
||||
}
|
||||
exit (nfail || nna) ? 2 : 0
|
||||
}' "$log"
|
||||
}
|
||||
|
||||
# _ber_summary <stamp> -- one table for every unit, printed last
|
||||
# _ber_summary <stamp> -- one table for every unit, printed last.
|
||||
#
|
||||
# Each unit gets three rows: the 200G ports, the 100G uplink on its own, and a
|
||||
# TOTAL. Unit 0 and unit 1 each have their own ce0(268), so the 100G row is per
|
||||
# unit too -- a bad uplink on one board is what this fork exists to make
|
||||
# visible, and averaged into 432 other lanes it would not be.
|
||||
_ber_summary() {
|
||||
local stamp="$1" unit side g r y n worst=0
|
||||
g=$'\033[32m'; r=$'\033[31m'; y=$'\033[33m'; n=$'\033[0m'
|
||||
@@ -347,27 +468,41 @@ _ber_summary() {
|
||||
|
||||
echo ""
|
||||
echo "===== summary : all units ====="
|
||||
printf '%-8s %8s %8s %8s %8s %-12s %-10s %s\n' \
|
||||
"Unit" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT"
|
||||
printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %s\n' \
|
||||
"Unit" "GROUP" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT"
|
||||
|
||||
local tag lanes pass fail na wber wlane v any=0
|
||||
local tag grp lanes pass fail na wber wlane v any=0 label
|
||||
for unit in $BER_UNITS; do
|
||||
side="$BER_LOG_DIR/berside_unit${unit}_${stamp}"
|
||||
side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}"
|
||||
[ -r "$side" ] || continue
|
||||
read -r tag lanes pass fail na wber wlane < "$side" 2>/dev/null
|
||||
[ "$tag" = "SUM" ] || continue
|
||||
any=1
|
||||
if [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then
|
||||
v="${r}FAIL${n}"; worst=2
|
||||
else
|
||||
v="${g}PASS${n}"
|
||||
fi
|
||||
printf '%-8s %8s %8s %8s %8s %-12s %-10s %b\n' \
|
||||
"unit $unit" "$lanes" "$pass" "$fail" "$na" "$wber" "$wlane" "$v"
|
||||
while read -r tag grp lanes pass fail na wber wlane; do
|
||||
[ "$tag" = "SUM" ] || continue
|
||||
any=1
|
||||
case "$grp" in
|
||||
P200) label="200G ports" ;;
|
||||
P100) label="100G p${BER_PORTS_100G}" ;;
|
||||
ALL) label="TOTAL" ;;
|
||||
*) label="$grp" ;;
|
||||
esac
|
||||
# A group with no lanes is not a pass. On the 100G row it means the
|
||||
# uplink never reported -- exactly the case worth flagging.
|
||||
if [ "${lanes:-0}" -eq 0 ]; then
|
||||
v="${y}NO DATA${n}"
|
||||
[ "$grp" = "P100" ] && worst=2
|
||||
elif [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then
|
||||
v="${r}FAIL${n}"; worst=2
|
||||
else
|
||||
v="${g}PASS${n}"
|
||||
fi
|
||||
printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %b\n' \
|
||||
"unit $unit" "$label" "$lanes" "$pass" "$fail" "$na" \
|
||||
"$wber" "$wlane" "$v"
|
||||
done < "$side"
|
||||
done
|
||||
[ "$any" = "1" ] || echo " (no unit produced a result)"
|
||||
echo ""
|
||||
printf 'criteria: BER < %s\n' "$BER_MAX"
|
||||
printf '100G group: port(s) %s -- one per unit\n' "$BER_PORTS_100G"
|
||||
echo ""
|
||||
return "$worst"
|
||||
}
|
||||
@@ -391,7 +526,7 @@ blanton_ber_report() {
|
||||
stamp=$(date +%Y%m%d_%H%M%S 2>/dev/null || echo now)
|
||||
for unit in $BER_UNITS; do
|
||||
log="$BER_LOG_DIR/prbsber_unit${unit}_${stamp}.log"
|
||||
side="$BER_LOG_DIR/berside_unit${unit}_${stamp}"
|
||||
side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}"
|
||||
: > "$side"
|
||||
if ! _ber_bcm "$unit" "prbsstat Ber" > "$log" 2>&1; then
|
||||
_ber_err "unit $unit: 'prbsstat Ber' failed (see $log)"; worst=1; continue
|
||||
@@ -409,22 +544,24 @@ blanton_ber_report() {
|
||||
|
||||
_ber_summary "$stamp"; rc=$?
|
||||
[ "$rc" -gt "$worst" ] && worst=$rc
|
||||
rm -f "$BER_LOG_DIR"/berside_unit*_"${stamp}"
|
||||
rm -f "$BER_LOG_DIR"/bertside_unit*_"${stamp}"
|
||||
return "$worst"
|
||||
}
|
||||
|
||||
# === Help ====================================================================
|
||||
blanton_ber_help() {
|
||||
echo -e "\033[1mblanton_ber.sh - SerDes BER test over every cabled port (PRBS via phydiag)\033[0m"
|
||||
echo -e "\033[1mblanton_ber.sh - SerDes BER test: cabled 200G ports + the 100G uplink\033[0m"
|
||||
echo ""
|
||||
echo -e " Units : \033[36m$BER_UNITS\033[0m poly: \033[36m$BER_POLY\033[0m interval: \033[36m${BER_INTERVAL}s\033[0m"
|
||||
echo -e " Criteria : BER < \033[36m$BER_MAX\033[0m"
|
||||
echo -e " Ports : \033[36m$BER_PORTS\033[0m"
|
||||
echo -e " 200G : \033[36m$BER_PORTS_200G\033[0m"
|
||||
echo -e " 100G : \033[36m$BER_PORTS_100G\033[0m (ce0, one per unit -- reported on its own row)"
|
||||
echo ""
|
||||
echo -e "\033[1mSubcommands\033[0m"
|
||||
echo -e " \033[33minit\033[0m prbs set $BER_POLY + prbs get (checks every lane locked)"
|
||||
echo -e " \033[33mstart\033[0m prbsstat STArt Interval=$BER_INTERVAL"
|
||||
echo -e " \033[33mreport\033[0m prbsstat Ber -> per-lane table + PASS/FAIL + summary"
|
||||
echo -e " summary splits each unit into 200G ports / 100G p$BER_PORTS_100G / TOTAL"
|
||||
echo -e " \033[33mstop\033[0m prbsstat STOp"
|
||||
echo -e " \033[33mclear\033[0m prbs clear (disarms PRBS -- 'init' again before measuring)"
|
||||
echo -e " \033[33mhelp\033[0m"
|
||||
@@ -433,7 +570,7 @@ blanton_ber_help() {
|
||||
echo -e " \033[33m-u|--unit\033[0m <0|1|all> bcmcmd unit(s); 'all' = 0 1 (default: $BER_UNITS)"
|
||||
echo -e " \033[33m-c|--criteria\033[0m <ber> PASS threshold (default $BER_MAX)"
|
||||
echo -e " \033[33m-i|--interval\033[0m <sec> prbsstat Interval (default $BER_INTERVAL)"
|
||||
echo -e " \033[33m-p|--ports\033[0m <list> override the port list"
|
||||
echo -e " \033[33m-p|--ports\033[0m <list> override the whole port list (both groups)"
|
||||
echo -e " \033[33m-F|--fail-only\033[0m report: list only failing lanes"
|
||||
echo -e " \033[33m--tsv\033[0m report: tab-separated"
|
||||
echo -e " \033[33m--raw\033[0m report: also print the raw output"
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
#==============================================================================
|
||||
# Author : ETWen
|
||||
# Date : 20260923
|
||||
# Purpose : Read DDR5 DIMM temperature from SPD5118 hub (MR49/MR50 @ 0x31)
|
||||
# Usage : ./ddr5_temp_V1.0.1.sh [bus] [addr ...]
|
||||
# default: bus=5, addr=0x50 0x51
|
||||
#
|
||||
# Version History:
|
||||
# V1.0.0 20260923 Initial Version
|
||||
# V1.0.1 20260923 Output label "DDR temp"
|
||||
#==============================================================================
|
||||
|
||||
BUS=${1:-5}
|
||||
shift 2>/dev/null
|
||||
ADDRS=("$@")
|
||||
[[ ${#ADDRS[@]} -eq 0 ]] && ADDRS=(0x50 0x51)
|
||||
|
||||
REG_TEMP=0x31
|
||||
|
||||
for ADDR in "${ADDRS[@]}"; do
|
||||
raw=$(i2cget -y "$BUS" "$ADDR" "$REG_TEMP" w 2>&1)
|
||||
if [[ $? -ne 0 || ! $raw =~ ^0x[0-9a-fA-F]+$ ]]; then
|
||||
echo "[bus $BUS $ADDR] read fail: $raw"
|
||||
continue
|
||||
fi
|
||||
|
||||
# bit[12:2] = temperature, bit12 = sign, 0.25 C/LSB
|
||||
v=$(( (raw >> 2) & 0x7FF ))
|
||||
(( v & 0x400 )) && v=$(( v - 0x800 ))
|
||||
|
||||
awk -v b="$BUS" -v a="$ADDR" -v r="$raw" -v v="$v" \
|
||||
'BEGIN { printf "[bus %s %s] DDR temp: %.2f C (raw=%s)\n", b, a, v * 0.25, r }'
|
||||
done
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
#==============================================================================
|
||||
# Author : ETWen
|
||||
# Date : 20260923
|
||||
# Purpose : Read LTC4287 hot-swap controller temperatures (HSC_MAIN / HSC_STBY)
|
||||
# Select source via 0xF3 (0xEB internal / 0xEF external), then read
|
||||
# READ_TEMPERATURE 0x8D (Kelvin) and convert to Celsius
|
||||
# Usage : ./hsc_temp_readback.sh
|
||||
# bus = BUS (28), HSC_MAIN @ 0x40, HSC_STBY @ 0x45
|
||||
#
|
||||
# Version History:
|
||||
# V1.0.0 20260922 Initial Version
|
||||
# V1.0.1 20260923 Add header; fix delay comments (DELAY is 0.5 s, not 100 ms);
|
||||
# CRLF -> LF (CRLF breaks bash on SONiC)
|
||||
#==============================================================================
|
||||
|
||||
BUS=28
|
||||
DELAY=0.5 # seconds between I2C transactions; ~4 s per run (4 readings x 2)
|
||||
|
||||
read_temp() {
|
||||
local addr="$1"
|
||||
local name="$2"
|
||||
local sel="$3"
|
||||
|
||||
# Select temperature source
|
||||
i2ctransfer -f -y "$BUS" w3@"$addr" 0xF3 "$sel" 0x00 > /dev/null
|
||||
|
||||
# DELAY between I2C transactions
|
||||
sleep "$DELAY"
|
||||
|
||||
# Read temperature register 0x8D
|
||||
raw=$(i2cget -f -y "$BUS" "$addr" 0x8D w 2>/dev/null)
|
||||
|
||||
if [ -z "$raw" ]; then
|
||||
printf "%-32s : READ ERROR\n" "$name"
|
||||
sleep "$DELAY"
|
||||
return
|
||||
fi
|
||||
|
||||
# Remove 0x prefix and convert hexadecimal to decimal
|
||||
raw_dec=$((16#${raw#0x}))
|
||||
|
||||
# LTC4287 temperature is in Kelvin
|
||||
temp_c=$(awk "BEGIN {printf \"%.2f\", $raw_dec - 273.15}")
|
||||
|
||||
printf "%-32s : %8s °C (raw %s)\n" \
|
||||
"$name" "$temp_c" "$raw"
|
||||
|
||||
# DELAY before next I2C transaction
|
||||
sleep "$DELAY"
|
||||
}
|
||||
|
||||
echo "=========================================================="
|
||||
echo " LTC4287 Temperature Readback"
|
||||
echo "=========================================================="
|
||||
|
||||
# HSC_MAIN - LTC4287 @ 0x40
|
||||
read_temp 0x40 "HSC_MAIN Internal (PU11)" 0xEB
|
||||
read_temp 0x40 "HSC_MAIN External (PQ1)" 0xEF
|
||||
|
||||
# HSC_STBY - LTC4287 @ 0x45
|
||||
read_temp 0x45 "HSC_STBY Internal (PU46)" 0xEB
|
||||
read_temp 0x45 "HSC_STBY External (PQ53)" 0xEF
|
||||
|
||||
echo "=========================================================="
|
||||
@@ -5,6 +5,8 @@ project_name = "Blanton"
|
||||
strTestcase="ENV" ;ENV,EMC,Margin...etc
|
||||
|
||||
EN_Margin = 0 ; 1: Enable Margin Test, 0: Disable Margin Test
|
||||
EN_TX_FIR = 0 ; 1: Enable bcm_dsc_scan.sh collect port TX_FIR, 0: Disable (saves ~20 min: ScriptA and ScriptC each scan for ~10 min)
|
||||
|
||||
EN_log = 1 ; 1: Enable log, 0: Disable log
|
||||
EN_timestamp = 1 ; 1: Enable timestamp in log, 0: Disable timestamp in log
|
||||
|
||||
@@ -32,4 +34,13 @@ BMC_USB_RUNTIME = 172800 ; Unit seconds, 172800 = 2 days
|
||||
;================================================================
|
||||
prompt_login = "sonic login:"
|
||||
prompt_sonic = "admin@sonic:~$"
|
||||
prompt_sonic_root = "root@sonic:~#"
|
||||
prompt_sonic_root = "root@sonic:~#"
|
||||
|
||||
|
||||
;================================================================
|
||||
; Reboot or Power Cycle
|
||||
;================================================================
|
||||
; 0: Normal Test
|
||||
; 1: Enable Reboot Cycle
|
||||
; 2: Enable Power Cycle
|
||||
EN_Cold_Warm_boot = 0
|
||||
@@ -5,6 +5,10 @@ sendln "show platform fan"
|
||||
wait prompt_sonic_root
|
||||
sendln "show platform temperature"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/ddr5_temp.sh"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/hsc_temp_readback.sh"
|
||||
wait prompt_sonic_root
|
||||
sendln "show platform psustatus"
|
||||
wait prompt_sonic_root
|
||||
sendln "show platform voltage"
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
cd /usr/sbin
|
||||
sudo docker pause gbsyncd1
|
||||
sudo docker pause gbsyncd0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0
|
||||
|
||||
//Tuning
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 0 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 1 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 2 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 3 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 4 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 5 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 6 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 7 pre=-10 main=87 post=0
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=0 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=1 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=2 main=27 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=3 main=29 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=0 main=29 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=1 main=30 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=2 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=3 main=30 pre1=4 post1=13' "
|
||||
|
||||
以下Test Cycle 目標200次
|
||||
//Disable Port
|
||||
bcmcmd -n 0 -c "port 268 en=0"
|
||||
bcmcmd -n 1 -c "port 268 en=0"
|
||||
|
||||
//Wait Link Down
|
||||
bcmcmd -n 0 -c "ps 268"
|
||||
bcmcmd -n 1 -c "ps 268"
|
||||
root@sonic:~# bcmcmd -n 0 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) !ena 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~# bcmcmd -n 1 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) !ena 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~#
|
||||
|
||||
|
||||
//Enable Port
|
||||
bcmcmd -n 0 -c "port 268 en=1"
|
||||
bcmcmd -n 1 -c "port 268 en=1"
|
||||
|
||||
//Wait Link Up
|
||||
bcmcmd -n 0 -c "ps 268"
|
||||
bcmcmd -n 1 -c "ps 268"
|
||||
root@sonic:~# bcmcmd -n 0 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) up 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~# bcmcmd -n 1 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) up 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~#
|
||||
|
||||
|
||||
//Collect DSC Data
|
||||
bcmcmd -n 0 -c "dsh -c 'phydiag 268 dsc'"
|
||||
bcmcmd -n 1 -c "dsh -c 'phydiag 268 dsc'"
|
||||
|
||||
//讀Retimer TX/RX
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 1
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 2
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 3
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 4
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 5
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 6
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 7
|
||||
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 1
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 2
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 3
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 4
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 5
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 6
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 7
|
||||
Reference in New Issue
Block a user