Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b911ccd658 | ||
|
|
dd558a7b0f | ||
|
|
690be5db7d | ||
|
|
a5b7c4757b | ||
|
|
f5694345fb | ||
|
|
7a1967bc9a | ||
|
|
496e3ab650 | ||
|
|
29ec4c2239 | ||
|
|
d73b119dc3 | ||
|
|
79a3604ac4 | ||
|
|
b84188322d | ||
|
|
b4148c9571 | ||
|
|
eacb218da8 | ||
|
|
4648f936c7 | ||
|
|
9cdd651d79 | ||
|
|
4f2f9ed090 |
@@ -31,3 +31,6 @@ publish/*
|
||||
|
||||
# 尚未上機驗證的腳本暫存區(不入 repo,測過再搬進 src/)
|
||||
tmp/
|
||||
|
||||
# mlucas 的 ELF binary(~99 MB),DUT image 已內含,不入 repo
|
||||
src/Script_ABC_Blanton/hammer/
|
||||
|
||||
+95
-20
@@ -112,7 +112,7 @@ Blanton_TTL_Script/
|
||||
│
|
||||
├── CLAUDE.md # 專案記憶 & 給 Claude 的指令
|
||||
├── ARCHITECTURE.md # 本架構文件
|
||||
├── .gitignore # secret/ + For_AI/ + tmp/ + logs + publish 產出
|
||||
├── .gitignore # secret/ + For_AI/ + tmp/ + hammer/ + logs + publish 產出
|
||||
├── .gitattributes # 換行符政策:.sh/.conf 釘 LF、.ttl 保持 CRLF
|
||||
│
|
||||
├── docs/ # 規格與狀態文件(客戶提供 / 專案產出)
|
||||
@@ -136,6 +136,7 @@ Blanton_TTL_Script/
|
||||
│ ├── 100G_PRBS.txt # 100G PRBS 原始指令(port_prbs_monitor.sh 的來源)
|
||||
│ ├── TR518.txt # 內建封包測試 tr 518 → blanton_tr518.sh
|
||||
│ ├── BER_Test.txt # 全機 PRBS BER 指令 → blanton_ber.sh
|
||||
│ ├── 100G_dsc_cycle.txt # 100G tuning + link 循環 + DSC → bcm_100g_dsc_cycle.sh
|
||||
│ ├── Multiphase_{high3,low3,comboA,comboB,Save}.txt # → blanton_multiphase_margin.sh
|
||||
│ ├── Multiphase_Netname # 38 條軌的 net 名稱(區塊順序對應)
|
||||
│ ├── 100G_Traffic.txt # ce0 VLAN 138 打流 → blanton_traffic_linespeed.sh
|
||||
@@ -161,7 +162,10 @@ Blanton_TTL_Script/
|
||||
│
|
||||
├── 4_Blanton_Script_thermal_safety.ttl # 熱 / 安規:獨立跑,不接 A→B→C
|
||||
├── 5_Blanton_Script_Reboot_Cycle.ttl # 重開機循環:獨立跑,while 1 無止境(未上機驗證)
|
||||
├── config.ttl # 全域設定:testcase 名、EN_Margin / EN_log、三個 prompt 字串
|
||||
├── config.ttl # 全域設定:testcase 名、EN_Margin / EN_TX_FIR / EN_log / EN_timestamp、
|
||||
│ # 風扇與 SWB 宣告、SSD/USB/BMC_USB 壓力秒數、三個 prompt 字串
|
||||
├── hammer/ # 🚫 gitignored — mlucas ELF binary(~99 MB),DUT image 自帶,
|
||||
│ # publish.sh 也排除;Script B / Script 4 只負責 chmod +x 後執行
|
||||
├── 1_Blanton_Script_A.ttl # Pre-test:登入 → 對時 → source 工具 → 清 AER → 抓基線
|
||||
├── 2_Blanton_Script_B.ttl # Stress:開壓力程序 → while 1 每輪抓 fan/temp/margin
|
||||
├── 3_Blanton_Script_C.ttl # Post-test:kill → 再抓一次 → 比對 stress log
|
||||
@@ -196,7 +200,13 @@ Blanton_TTL_Script/
|
||||
├── mgmt_ping_monitor.sh # 10G/1G 管理網路 ping 監控(iproute2)
|
||||
│ # eth0→TARGET_10G、eth1→TARGET_1G,兩張卡同時持續 ping
|
||||
├── port_prbs_monitor.sh # 100G uplink PRBS 測試(bcmcmd phy diag)
|
||||
│ # ⚠️ V1.1.9 起 A/B/C 內註解掉的呼叫已整段刪除,只剩手動執行
|
||||
├── blanton_ber.sh # 全機 SerDes BER 掃描(phydiag prbs / prbsstat)
|
||||
│ # V1.2.2 起含 port 268(100G ce0),summary 每 unit 分 200G/100G/TOTAL
|
||||
├── bcm_100g_dsc_cycle.sh # 100G link down/up 循環 + 每輪 DSC 與 retimer TX/RX 讀回(未驗證)
|
||||
├── bcm_dsc_scan.sh # 全機 SerDes DSC dump(phydiag <port> dsc,~445 port × 2 unit)
|
||||
│ # 由 EN_TX_FIR 控制,A 與 C 各掃一次;單次約 10 分鐘
|
||||
├── bcm_dsc_parse.sh # DSC raw log → <prefix>_lane.csv / _pm.csv(純 sh + awk)
|
||||
├── blanton_multiphase_margin.sh # 多相 VRM margin(原生 i2c,38 條軌)
|
||||
├── pwr_brick.sh # PDB / CPB power brick 讀值(VI2C ch4)
|
||||
├── nfc_polling.sh # NFC tag 背景輪詢 + PASS/FAIL 統計
|
||||
@@ -207,7 +217,7 @@ Blanton_TTL_Script/
|
||||
│ # ⚠️ 已納入交付,但 A/B/C 的呼叫仍註解掉(bring-up 中)
|
||||
├── usb_target.sh # 偵測 USB 裝置節點/掛載點(排除系統碟)
|
||||
├── bmc_monitor.sh # BMC 監控:free -m + I2C 寫入/讀回圖樣測試
|
||||
├── bmc_monitor_ddr.sh # ⚠️ 已停用:B/C 內呼叫處已註解,DDR 壓力改走 bgctl + bmc-manager
|
||||
├── bmc_monitor_ddr.sh # ⚠️ 已停用:V1.1.9 起 B/C 內的註解區塊已刪除,DDR 壓力走 bgctl + bmc-manager
|
||||
│ # ↑ 三支都要 chmod +x 才能跑(見 Key Constraints)
|
||||
└── LTC2980_Margin_Script/ # 電壓 margin 子專案(另有獨立 repo)
|
||||
├── margin.sh # margin_init/status/set/apply_profile/save
|
||||
@@ -367,10 +377,10 @@ PROFILE_CHANNELS=( "0:high:8" "1:low:8" ... ) # "channel:operation:change_perc
|
||||
|
||||
| 模組 | 進入點 | 功能 |
|
||||
|------|--------|------|
|
||||
| **Script A(Pre-test)** | `1_Blanton_Script_A.ttl` V1.1.7 | root 登入 → `date -s` 對時 → 清 job log → **收掉前一輪殘留**(`hw-test-session finish` → `bgctl reset --yes` → `bgctl stop --all` → `bgctl list` 確認清空)→ `hw-test-session start` → **設定風扇轉速** → **等資料面就緒**(`wait_init.ttl`)→ DUT 清單(boot/version/fwutil/syseeprom/ssdhealth/TPM/NVMe)→ BMC enroll + version/status → source bash 工具 → 清/讀 7 組 PCIe AER + `ras-mc-ctl --summary` → `lspci` → margin → PMON → dmesg → **10G/1G 管理網路 ping** → **NFC 基線**(start → 30s → stop → report)→ 100G uplink 設定與狀態 → **一輪流量基線** → `show uptime` |
|
||||
| **Script B(Stress)** | `2_Blanton_Script_B.ttl` | `mlucas-avx2 -cpu 0:15` + `bgctl run` 的 `memtester` / SSD / USB 壓力(v1.1.7 起 SSD/USB 的 `--runtime`/`--passes` 讀 `config.ttl` 的 `SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT`;BMC memtester 不帶次數 = 跑到被停) + BMC DDR(`bmc-manager run memtester`)→ BMC USB net test(探測 cdc_ncm 介面,`systemd-run` 掛 4 小時 ping)→ 啟動 mgmt ping 監控 → **啟動 NFC 輪詢** → `jobs` / `bgctl list` → traffic(依 `SWB_UNIT`)→ `while 1` 每輪 PMON + `bgctl list`/`jobs` + `pause 60`(**V1.1.4 起 margin 不在迴圈內**)|
|
||||
| **Script C(Post-test)** | `3_Blanton_Script_C.ttl` | `kill $(jobs -p)` + `bgctl stop --all` / `reset` + 停 BMC USB unit、mgmt ping 與 **NFC 輪詢** → PMON + dmesg → 7 組 AER + `ras-mc-ctl` → BMC USB `journalctl` **寫進 `log/bmc_usb_net.log`**(不再印在 console) → `ip -s link show eth0/eth1` → `cat` mlucas log → **NFC `report`** → traffic `stop` + `report` → **BER 掃描**(依 `SWB_UNIT`)→ NVMe 健康 → `show reboot-cause` → **margin 掃描**(`EN_Margin`,V1.1.4 由 Script B 移來)→ `uptime` → **job log 複製到 USB(帶時戳)** → `exit` + wait『Script done』→ `hw-test-session finish` |
|
||||
| **Script 4(Thermal / Safety)** | `4_Blanton_Script_thermal_safety.ttl` | **獨立腳本,不接 A→B→C。** Script A 式的環境建立(登入 / session / 風扇 / `wait_init` / DUT 清單 / BMC / PCIe 清除 / `pcie_bus` / PMON / dmesg)→ **滿載**(`mlucas-avx2 -cpu 0:15` 背景 + `blanton_tr518.sh start`)→ `while 1`:PMON + `TH6_SWB0/1_power_readback.sh` + `bgctl list` / `jobs`,`pause 60`。**沒有收尾段** —— 中斷後 DUT 仍在 `port cd lb=mac` / `l2 learn off` / `test mode nr=yes`,要手動 `blanton_tr518.sh stop` |
|
||||
| **Script A(Pre-test)** | `1_Blanton_Script_A.ttl` V1.2.2 | root 登入 → `date -s` 對時 → 清 job log → **收掉前一輪殘留**(`hw-test-session finish` → `bgctl reset --yes` → `bgctl stop --all` → `bgctl list` 確認清空)→ `hw-test-session start` → **設定風扇轉速** → **等資料面就緒**(`wait_init.ttl`)→ DUT 清單(boot/version/fwutil/syseeprom/ssdhealth/TPM/NVMe)→ BMC enroll + version/status → source bash 工具 → 清/讀 7 組 PCIe AER + `ras-mc-ctl --summary` → `lspci` → margin → PMON → dmesg → **10G/1G 管理網路 ping** → **NFC 基線**(start → 30s → stop → report)→ 100G uplink 設定與狀態 → **BER 基線掃描**(依 `SWB_UNIT`,V1.2.1 新增)→ **一輪流量基線** → **SerDes DSC 基線**(`EN_TX_FIR`)→ `show uptime` |
|
||||
| **Script B(Stress)** | `2_Blanton_Script_B.ttl` | **V1.1.8 起 `mlucas-avx2 -cpu 0:15` 由 `systemd-run --unit=mlucas-amm`(`Type=exec`、`Restart=always`)啟動,不再是 shell 的 `&` 背景 job** + `bgctl run` 的 `memtester` / SSD / USB 壓力(v1.1.7 起 SSD/USB 的 `--runtime`/`--passes` 讀 `config.ttl` 的 `SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT`;BMC memtester 不帶次數 = 跑到被停) + BMC DDR(`bmc-manager run memtester`)→ BMC USB net test(探測 cdc_ncm 介面,`systemd-run` 掛 `BMC_USB_RUNTIME` 秒的 ping,V1.1.8 前寫死 4 小時)→ 啟動 mgmt ping 監控 → **啟動 NFC 輪詢** → `jobs` / `systemctl is-active mlucas-amm` / `bgctl list` → traffic(依 `SWB_UNIT`)→ `while 1` 每輪 PMON + `bgctl list`/`jobs`/`systemctl is-active mlucas-amm` + `sleep 60`(**V1.1.4 起 margin 不在迴圈內**;V1.2.0 起等待走 DUT 端 `sleep` 而非 macro 端 `pause`)|
|
||||
| **Script C(Post-test)** | `3_Blanton_Script_C.ttl` | `kill $(jobs -p)` + **`systemctl stop mlucas-amm`**(systemd unit 不是 shell job,`jobs -p` 抓不到)+ `bgctl stop --all` / `reset` + 停 BMC USB unit、mgmt ping 與 **NFC 輪詢** → PMON + dmesg → 7 組 AER + `ras-mc-ctl` → BMC USB `journalctl` **寫進 `log/bmc_usb_net.log`**(不再印在 console) → `ip -s link show eth0/eth1` → `cat` mlucas log(並 `cp` 進 job 目錄一起帶走)→ **NFC `report`** → traffic `stop` + `report` → **SerDes DSC 收尾**(`EN_TX_FIR`)→ **BER 掃描**(依 `SWB_UNIT`)→ NVMe 健康 → `show reboot-cause` → **margin 掃描**(`EN_Margin`,V1.1.4 由 Script B 移來)→ `uptime` → **job log 複製到 USB(帶時戳)** → `exit` + wait『Script done』→ `hw-test-session finish` |
|
||||
| **Script 4(Thermal / Safety)** | `4_Blanton_Script_thermal_safety.ttl` | **獨立腳本,不接 A→B→C。** Script A 式的環境建立(登入 / session / 風扇 / `wait_init` / DUT 清單 / BMC / PCIe 清除 / `pcie_bus` / PMON / dmesg)→ **滿載**(`mlucas-avx2 -cpu 0:15`,V1.1.8 起與 Script B 同樣掛成 `mlucas-amm` systemd unit + `blanton_tr518.sh start`)→ `while 1`:PMON + `TH6_SWB0/1_power_readback.sh` + `bgctl list` / `jobs` / `systemctl is-active mlucas-amm`,`pause 60`。**沒有收尾段** —— 中斷後 DUT 仍在 `port cd lb=mac` / `l2 learn off` / `test mode nr=yes`,且 CPU 仍滿載,要手動 `blanton_tr518.sh stop` + `systemctl stop mlucas-amm` |
|
||||
| **Script 5(Reboot Cycle)** | `5_Blanton_Script_Reboot_Cycle.ttl` | **獨立腳本,不接 A→B→C。** `while 1`:登入 → `sudo -i` → 對時 → 風扇 → **Check Pt1~Pt13**(DUT/BMC info、mgmt ping、NFC、SSD、USB、BMC USB、port status + 15s traffic、BER、NVMe、platform data、dmesg i2c、PCIe error counters)→ `show uptime` → `reboot` → `wait "Booting in blind mode"`。`cnt_cycle` 每輪 +1 印在標頭。**沒有結束條件** —— 要停就關掉 Tera Term 的 macro。**尚未上機驗證** |
|
||||
| **SWB loopback 流量** | `blanton_traffic_linespeed <sub> [-u 0\|1\|all]` | `init`(VLAN 30..137,成對 `cdN`/`cdN+32`)→ `clear` → `start`(`tx 200 length=324 PatternRandom=yes`,v0.7.0 起)→ `stop` → `report`(per-pair TX/RX 交叉比對 PASS/FAIL);另有 `ps`(link/speed)與 `run`(一條龍)。**V0.6.0 起含 100G uplink `ce0`**(VLAN 138):它沒有 loopback 對象,靠 **ingress mirror 打回自己**維持循環,`report` 最後一列就是它,底下再接一份跨 unit summary |
|
||||
| **FPGA 暫存器存取** | `cb_fpga <fn> <addr> [data]`、`fpga_rescan` | pcimem 打 `resource0` + BAR 相對 offset,32-bit word;BDF 於 source 時自動偵測;`pmc/icb/swb0/swb1_fpga` 走 VSPI 窗口 |
|
||||
@@ -383,9 +393,11 @@ PROFILE_CHANNELS=( "0:high:8" "1:low:8" ... ) # "channel:operation:change_perc
|
||||
| **Margin 寫 NVM** | `margin_save` / `margin_save all` / `margin_save blanton` | ⚠️ 永久寫入。`0x15` 送 PMBus **send-byte**(不帶 data byte)。`all` = 依 `settings/` 走 9 顆、跑完還原原本載入的板子;`blanton` = 本平台捷徑,18 顆全走原生 i2c bus(`MARGIN_BLANTON_STORE` 表),不需 `margin_init`。每筆之間 `MARGIN_STORE_SETTLE`(0.5s),工具**不 poll busy bit** |
|
||||
| **管理網路 ping** | `mgmt_ping_monitor.sh {start\|stop\|status\|fg\|tail\|clear\|summary}` | 一輪 = 10G leg + 1G leg,各自 `ip address/route replace` 後 `ping -I <if>` 打自己的對端;兩張卡都保持 up。每 leg 記 `ip -s link show` 與該卡的 TX/RX packet delta,另出一行可 grep 的 `RESULT` |
|
||||
| **BMC 監控** | `bmc_monitor.sh {start\|stop\|status\|fg\|tail\|clear}` | 由 host 端 `bmc-manager run` 週期取樣 `free -m`,另加一組 **I2C 寫入/讀回圖樣測試**(bus 1、addr 0x41、offset 0x00C0,寫 `55AA55AA` 讀回、再寫 `AA55AA55` 讀回)。B 啟動、C 停止並 `cat log/bmc_poll.log` |
|
||||
| **BMC DDR 壓力** | `bmc_monitor_ddr.sh` | ⚠️ **已停用** —— B/C 內呼叫處已註解,改由 `bgctl run bmc-manager run memtester` 取代。檔案保留未刪 |
|
||||
| **100G PRBS** | `port_prbs_monitor.sh {start [both\|a\|b]\|stop\|status\|report\|clear}` | 兩個 uplink 各一個 worker,可同時或單獨跑。setup(`lpmode off` → `prbs set` → `prbsstat STArt`)→ 週期 `prbs get`(判定)+ `prbsstat Ber`(只記錄)→ teardown(`STOp` + `prbs clear`)。輸出含 `PRBS OK!` 才算 PASS。**⚠️ A/B/C 的呼叫目前註解掉** |
|
||||
| **SerDes BER 掃描** | `blanton_ber.sh {init\|clear\|start\|stop\|report} [-u 0\|1\|all]` | 全機每個對接 port 的 PRBS BER:`prbs set p=3` + `prbs get`(確認每條 lane 都鎖上)→ `prbsstat STArt Interval=30` →(等一個 interval)→ `prbsstat Ber` → `STOp` → `prbs clear`。`report` 一列一條 lane,**BER < 1e-6 才 PASS**,最後跨 unit 的 summary 帶最差 lane。⚠️ PRBS 期間 link 顯示 DOWN、不過流量 |
|
||||
| **BMC DDR 壓力** | `bmc_monitor_ddr.sh` | ⚠️ **已停用** —— V1.1.9 起 B/C 內的註解區塊已刪除,改由 `bgctl run bmc-manager run memtester` 取代。檔案保留未刪 |
|
||||
| **SerDes DSC 擷取** | `bcm_dsc_scan.sh [-u 0\|1] [-f <name>] [-F] [-n] [-C]` + `bcm_dsc_parse.sh -i <log> -o <prefix>` | `bcmcmd -n <unit> -c "dsh -c 'phydiag <port> dsc'"` 掃全清單(1 個 100G + 445 個 200G port)× 2 unit,raw log 寫進 `<script_dir>/log` 再 `cp` 到 `/host/hw-eval/jobs`。`-f` 支援 `%u` / `%t`;沒帶 `%u` 時自動插 `_u0`/`_u1`。成功與否**不看 `bcmcmd` 的 exit code**,改比對 `SERDES DISPLAY DIAG DATA END` 數量 + 實際迭代筆數,結尾印 `### DSC_SCAN_DONE rc=<n>`。parse 端輸出 `_lane.csv`(每 lane:TXEQ/RXFFE/DFE/EYE/SNR)與 `_pm.csv`(每 pm_id:溫度/PLL/lane 數/統計)。**由 `EN_TX_FIR` 控制,預設關** |
|
||||
| **100G PRBS** | `port_prbs_monitor.sh {start [both\|a\|b]\|stop\|status\|report\|clear}` | 兩個 uplink 各一個 worker,可同時或單獨跑。setup(`lpmode off` → `prbs set` → `prbsstat STArt`)→ 週期 `prbs get`(判定)+ `prbsstat Ber`(只記錄)→ teardown(`STOp` + `prbs clear`)。輸出含 `PRBS OK!` 才算 PASS。**⚠️ V1.1.9 起 A/B/C 內的註解呼叫已刪除,只能手動跑** |
|
||||
| **SerDes BER 掃描** | `blanton_ber.sh {init\|clear\|start\|stop\|report} [-u 0\|1\|all]` | 全機每個對接 port 的 PRBS BER:`prbs set p=3` + `prbs get`(確認每條 lane 都鎖上)→ `prbsstat STArt Interval=30` →(等一個 interval)→ `prbsstat Ber` → `STOp` → `prbs clear`。`report` 一列一條 lane,**BER < 1e-6 才 PASS**,最後跨 unit 的 summary 帶最差 lane。⚠️ PRBS 期間 link 顯示 DOWN、不過流量。V1.2.1 起 Script A 與 Script C 各跑一次(A 在流量之前、C 在流量報表之後),夾住 soak。**V1.2.2(腳本 v1.1.0)起清單納入 port 268(100G uplink ce0)**,summary 每個 unit 拆成 200G ports / 100G p268 / TOTAL 三列;某組零 lane 算 FAIL |
|
||||
| **100G link 循環 + DSC** | `bcm_100g_dsc_cycle.sh <cycles> [-d] [-T] [-P] [-n]` | tuning(`docker pause gbsyncd*` → `serdes-tx-set sys 0..7` → 每 lane `tx_taps set`)一次,之後每輪:`port 268 en=0` → 輪詢 `bcmcmd ps 268` 等 `!ena` → `en=1` → 等 `up` → 兩個 unit 的 `phydiag 268 dsc` + `serdes-tx-get`/`serdes-rx-get sys 0..7`。log 一次執行一份(`~/dsc_log/`),每輪一段 `### TEST CYCLE n/N`,結尾 `### DSC_CYCLE_DONE rc=<n>`。**全 ASCII 輸出**(DUT console 會把 UTF-8 弄成亂碼)。⚠️ 尚未上機驗證 |
|
||||
| **TR518 負載** | `blanton_tr518.sh {start\|stop} [-u 0\|1\|all]` | `port cd lb=mac` → `l2 learn off` → `test mode nr=yes` → `tr 518 ... testphase=1`,然後 settle `TR_SETTLE_SEC`(120s)再讓人量功耗。`stop` 只送 `port cd lb=none`(照 `TR518.txt`),L2 學習與 test mode **不會還原**,`TR_RESTORE_*=1` 才會 |
|
||||
| **PDB / CPB brick** | `pwr_brick.sh show [-r]` | 5 顆 brick 的 Vin / Iin / Vout / Iout / Temp(SWB0 `0x62`/`0x63`、SWB1 `0x60`/`0x61`、CPB `0x26`)。四顆 ADPM12200 **每個量各自一個 PMBus PAGE**(Vin 9、Iin 10、Vout 2、Iout 14、Temp 18),讀之前一定要先寫 PAGE —— `blanton_pwr_data.sh` 完全沒寫,所以整張 PDB 表都是 NA。換算用手冊 DIRECT 公式 `X=(1/m)(Y×10⁻ᴿ−b)`。每個值重讀到通過三道檢查為止(非 `0xFFFF`、非 `0x0000`、落在合理範圍)—— 這條匯流排**只壞高位元組**,`11.98 V` 會變成 `32.00 V`,全一過濾擋不住 |
|
||||
| **SWB 軌功耗** | `TH6_SWB{0,1}_power_readback.sh` | 讀 `show platform voltage/current` **各一次**,19 條軌 V×I 與總功耗。舊版每軌各讀一次(38 次呼叫、每軌不同時刻),總和是不同瞬間的加總 |
|
||||
@@ -479,6 +491,8 @@ pwr_data
|
||||
|
||||
1. **A → B → C 必須依序跑,且 B 之後才能跑 C。** Script B 的背景 job 只存在於那個 shell session,
|
||||
Script C 的 `kill $(jobs -p)` 依賴同一個 session;中途斷線就得手動收程序。
|
||||
V1.1.8 起 `mlucas-amm` 例外 —— 它是 systemd transient unit,**跨 session 存活**,
|
||||
所以 Script C 必須另外 `systemctl stop mlucas-amm`(見第 17 條)。
|
||||
2. **A / B / C 共用同一個 log 檔**:只有 Script A `logopen`,B / C 用 `logwrite` 續寫分隔線。
|
||||
跑 B/C 前若關掉了 Tera Term,log 就斷了。
|
||||
3. **PCIe AER 的判定是「差值」不是「絕對值」**:Script A 抓基線、Script C 抓結果,
|
||||
@@ -505,6 +519,11 @@ pwr_data
|
||||
`vlan create` 的 `Entry exists`(重跑 init)與 `vlan remove` 的 `not found`(init 前先 stop)
|
||||
當良性情況分開計數。另外 `bcmcmd` 一律以 `</dev/null` 執行 ——
|
||||
否則它會吃掉 while 迴圈的 stdin,導致**只有第一個 VLAN 被設定**(V0.3.1 修的 bug)。
|
||||
**2026-09-07 `bcm_dsc_scan.sh` 又踩了同一個坑**(port 清單走 stdin,445 個 port 只掃了 1 個),
|
||||
而且因為 `### PORT` marker 與 dsc 區塊數量一起變少,原本的比對檢查完全看不出來、`rc=0`。
|
||||
修法:迴圈改讀 fd 3(`done 3< "$PORTFILE"`)+ `bcmcmd ... < /dev/null`,再加一道
|
||||
「實際迭代筆數 == 清單筆數」的獨立檢查。**寫任何餵檔案給 `while` 又在裡面呼叫 `bcmcmd` 的迴圈,
|
||||
都要假設它會讀 stdin;而且完整性檢查不能只比對兩個會一起變少的數字。**
|
||||
11. **DUT 密碼不得進 git**:`1_Blanton_Script_A.ttl` 目前的 `sendln "YourPaSsWoRd"` 是 placeholder,
|
||||
實際帳密必須放 `secret/config_secret.ttl`(gitignored)並由 `config.ttl` include。
|
||||
12. **`.sh` / `.conf` 必須是 LF**:CRLF 的 shell script 在 SONiC 上直接掛
|
||||
@@ -535,6 +554,27 @@ pwr_data
|
||||
**ping 成功但 `nic_tx` 接近 0 就代表封包從另一張卡出去了**。兩張卡在不同網段時可設 0。
|
||||
16. **客戶 NDA**:`docs/` 內含客戶 FPGA 規格與 schematic 衍生資訊,**本 repo 只推 NAS + Gitea,
|
||||
絕不推 GitHub**。
|
||||
17. **改用 `systemd-run` 起壓力工作,就要自己補停的那一行。** `utils/kill_all_process.ttl` 的內容
|
||||
只有 `kill $(jobs -p)`,而 systemd unit 不是 shell job,它抓不到。V1.1.8 把 `mlucas-avx2`
|
||||
搬到 `mlucas-amm` unit(好處:掉線不會停、`jobs` 看不見的問題消失、`Restart=always` 會自動拉回),
|
||||
代價是 Script C 少了那行 `systemctl stop` 就會讓 CPU 滿載燒進收工量測與下一輪測試 ——
|
||||
而 `show interfaces status` 之類的檢查完全看不出來。目前 `mlucas-amm` 與
|
||||
`qfx5252-bmc-usb-net-test` 兩個 unit 都已在 Script C 收掉。
|
||||
⚠️ **`4_Blanton_Script_thermal_safety.ttl` 用同一個 `mlucas-amm` unit,但它沒有收尾段**,
|
||||
所以那行 `systemctl stop mlucas-amm` 必須由測試員手動下 —— 這是 Script 4 目前
|
||||
唯一一個「不下就會延續到下一輪」的負載。
|
||||
18. **`pause` 停的是 macro,`sleep` 停的是 DUT。** V1.2.0 起 A/B/C 的等待一律改成
|
||||
`sendln "sleep N"` + `wait prompt_sonic_root`。`pause N` 期間 macro 不理會 DUT,shell 是閒著的,
|
||||
等於用猜的估上一個指令要跑多久 —— DUT 一忙就會提早往下送指令;改成 DUT 端 `sleep` 之後,
|
||||
prompt 回來才繼續,兩邊不可能走散,而且這段等待會帶時戳進 log,事後看得出哪一輪跑久了。
|
||||
⚠️ **Script 4 與 Script 5 尚未轉換,仍在用 `pause`**;B/C 開頭的 `pause 1` 也保留
|
||||
(它在登入前執行,沒有 prompt 可等)。
|
||||
19. **`config.ttl` 少一個變數會讓 macro 當場中止**,不是當成 0。V1.1.8 新增 `EN_timestamp`
|
||||
(Script A `logopen` 的第 5 個參數)與 `BMC_USB_RUNTIME`(Script B 的 ping unit);
|
||||
V1.2.0 再加 `EN_TX_FIR`(A/C 的 DSC 掃描與 C 的四行 `cp`,預設 0)。
|
||||
只更新 `.ttl` 卻沿用舊的 `config.ttl`,Script A 會停在 `logopen` 那一行。
|
||||
另外 **時間戳的「格式」不在 macro 裡** —— 它在 Tera Term 的
|
||||
Setup → Additional settings → Log,存在那台電腦的 `TERATERM.INI`,會跟著 PC 而不是跟著腳本走。
|
||||
|
||||
---
|
||||
|
||||
@@ -768,7 +808,7 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
- [ ] `CHANGELOG.md`:延續三支腳本頂端的 Version History,集中成一份
|
||||
- [ ] 三支腳本的 header 版號與 `CHANGELOG.md` 對齊,`config.ttl` 加 `script_version` 並寫進 log 開頭
|
||||
- [ ] `docs/TTL_Script_Blanton_Status_*.xlsx` 更新為全 OK,或把剩餘項目移進 Future Extensions
|
||||
- [x] 打 git tag,push `nas` + `gitea`(已至 `V1.1.7`;`publish/` 依 Script A 檔頭版號打包)
|
||||
- [x] 打 git tag,push `nas` + `gitea`(已至 `V1.2.2`;`publish/` 依 Script A 檔頭版號打包)
|
||||
|
||||
**驗收條件:** 從 zip 解壓到一台乾淨的測試 NB,照 README 操作,不看原始碼就能完成一輪 A→B→C
|
||||
並產出 summary;`publish/` 內容與 git tag 一致。
|
||||
@@ -802,14 +842,23 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
Script A 既不往下走也不報錯。可加最大重試次數 + `messagebox` 提示。
|
||||
- **腳本的執行權**:`mgmt_ping_monitor.sh` 與 `bmc_monitor.sh`(以及已停用的 `bmc_monitor_ddr.sh`)需人工 `chmod +x`。
|
||||
可在 Script B 起它之前補一行(比照既有的 `chmod +x ~/hammer/tools/amd/mlucas-avx2`),或在 publish 時處理。
|
||||
- **`bmc_monitor_ddr.sh` 已停用**:B/C 內的呼叫已註解,DDR 壓力改由 `bgctl run bmc-manager run memtester`
|
||||
取代。檔案還留在 repo,待確認不再需要後刪除,或在檔頭標註停用。
|
||||
- **`bmc_monitor_ddr.sh` 已停用**:V1.1.9 起 B/C 內那段註解掉的呼叫已刪除,DDR 壓力由
|
||||
`bgctl run bmc-manager run memtester` 取代。檔案還留在 repo,待確認不再需要後刪除,或在檔頭標註停用。
|
||||
(`bmc_monitor.sh` 已於 2026-08-25 重新啟用,並加上 I2C 寫入/讀回圖樣測試。)
|
||||
- **`rm /host/hw-eval/jobs/*` 清不掉子目錄**:Script A 用的是不帶 `-r` 的 `rm`,但 Script C
|
||||
是 `cp -r` 那個目錄,代表裡面可能有子目錄。清不乾淨的話,上一輪的 job log 會被當成這輪的一起
|
||||
複製到 USB。要確認 `bgctl` 的產出結構,必要時改成 `rm -rf .../jobs/*`。
|
||||
- **100G PRBS 尚未接進 A/B/C**:`port_prbs_monitor.sh` 已隨包交付、可手動執行,但三支腳本裡的呼叫
|
||||
都還註解著(bring-up 中)。要啟用時記得 PRBS 期間 link 會顯示 down,`stop` 之後才會回來。
|
||||
- **100G PRBS 已從 A/B/C 移除**:`port_prbs_monitor.sh` 仍隨包交付、仍可手動執行,但 V1.1.9 起
|
||||
三支腳本裡那些註解掉的呼叫已整段刪除(連同 Script C 的 `bmc_monitor_ddr.sh` 區塊)。
|
||||
要接回去得重寫,不是把註解拿掉。啟用時記得 PRBS 期間 link 會顯示 down,`stop` 之後才會回來。
|
||||
- **DSC 擷取預設是關的**:`EN_TX_FIR = 0`,所以出貨設定不會跑 `bcm_dsc_scan.sh`。
|
||||
開啟後 Script A 與 Script C 各掃一次、整輪多約 20 分鐘;只要單邊資料可以在腳本裡改成 `-u 0`。
|
||||
掃描本身是唯讀(`phydiag ... dsc` 不改 SerDes 設定),但**它與 `blanton_ber.sh` /
|
||||
`blanton_traffic_linespeed` / `blanton_tr518.sh` 一樣都在動同一批 port,不要同時跑**。
|
||||
- **`bcm_100g_dsc_cycle.sh` 尚未上機驗證**:`port en=0/1` 之後 link 多久會動、以及
|
||||
`docker pause gbsyncd*` 期間的影響,都要上機才知道。等待是讀 SDK(`bcmcmd ps <port>`)而不是
|
||||
`show interfaces status`,所以理論上不受 gbsyncd 暫停影響,但沒實測過。
|
||||
結束時(含 Ctrl-C)會自動 `docker unpause`,`RESUME_GBSYNCD=0` 可關掉。
|
||||
- **Script 5(Reboot Cycle)尚未上機驗證**:`5_Blanton_Script_Reboot_Cycle.ttl` 隨 V1.1.7 交付,
|
||||
但還沒在真 DUT 上跑過一輪完整循環。兩個要現場確認的點:`wait "Booting in blind mode"` 是否
|
||||
真的在每一版開機訊息裡出現(沒出現就會永遠等下去,而且沒有 `timeout` 保護);以及每輪重跑
|
||||
@@ -817,6 +866,16 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
- **`tmp/blanton_reboottest.sh`(bash 版重開循環)也未上機驗證**:不用 Tera Term、由 systemd 在
|
||||
每次開機跑一輪並自動 `reboot`,登入下 `stop` 才停。整個 `tmp/` 是 gitignored,**不會進交付包** ——
|
||||
驗證過再搬進 `Blanton_Script/`。未驗證的腳本混進 `Blanton_Script/` 之後,測試員分不出哪支能信。
|
||||
- **交付包不含 `hammer/`,但 Script B 與 Script 4 都要用它**:V1.1.8 起 `hammer/` 是 gitignored
|
||||
且被 `publish.sh` 排除(~99 MB 的 mlucas binary,進 git 就永久留在 history)。這是刻意的 ——
|
||||
binary 由 DUT image 帶 —— 但代表**測試員解開交付包後 `~/hammer` 必須已經在 DUT 上**,
|
||||
否則 Script B 會在 `chmod +x ~/hammer/tools/amd/mlucas-avx2` 那一行失敗,而且 macro 不會停,
|
||||
接著的 `systemd-run` 只是起一個立刻失敗又被 `Restart=always` 反覆重拉的 unit。
|
||||
上機前先 `ls ~/hammer/tools/amd/mlucas-avx2` 確認一次。
|
||||
- **`mlucas-amm` 的 `~` 依賴 systemd 的環境**:unit 是
|
||||
`/bin/bash -c '~/hammer/... >> ~/hammer/mlucas_amm_log'`,system service 沒有 `User=` 時 systemd
|
||||
**不會設 `HOME`**,bash 才退回 passwd 的家目錄(root → `/root`)解出同一條路徑。目前會通,
|
||||
但那是剛好對上而不是保證;改成絕對路徑 `/root/hammer/...` 就不用賭。**尚未上機驗證。**
|
||||
- **Script C 的 USB 掛載走 `mount` 而非 `usb_target.sh -o mnt`**:目前是 `-o dev` 取節點再自己
|
||||
`mount <dev> /mnt/usb`,繞過了工具內建的 `mkdir -p`、exfat/ntfs `modprobe` 與可寫性驗證。
|
||||
掛載點不存在、檔案系統模組沒載、或媒體唯讀時只會失敗一行,job log 就沒帶出來。
|
||||
@@ -833,6 +892,8 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
(`SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT`,COUNT 直接對應 `--passes`)。
|
||||
`qfx5252-stress-*` 跑滿 RUNTIME 秒就結束,**`--passes` 不會把它延長**,所以
|
||||
`RUNTIME=600, COUNT=720` 是 10 分鐘的壓力而不是 120 小時。要跑滿一天就把 RUNTIME 填 `86400`。
|
||||
**V1.1.8 起出貨預設是 `RUNTIME=3600`(1 小時)**,`BMC_USB_RUNTIME` 則是 `172800`(2 天)——
|
||||
三個壓力源的長度目前刻意不一致,開跑前要照當次 soak 的時間各自對一次。
|
||||
壓力提早結束時 Script B 每輪的 `bgctl list` 數量會變少,但**沒有任何東西會主動報警**;
|
||||
要比 24 小時更久則得加一段重啟 job 的邏輯。
|
||||
- **BMC USB 的 service log 只剩隨身碟一份**:Script C 把 `journalctl` 導進
|
||||
@@ -853,16 +914,30 @@ margin status、全部 VRM 軌的 Vin/Vout/Iout/Temp、CB + ICB 溫感讀值,
|
||||
LTC2980 PMBus)摸同一批軌,各自都假設自己是唯一的寫入者。目前只靠文件與 help 提醒,
|
||||
沒有任何執行期檢查。
|
||||
- **Script 4 沒有收尾段**:`while 1` 之後就是檔案結尾,中斷時 DUT 停在 `port cd lb=mac` /
|
||||
`l2 learn off` / `test mode nr=yes`,而且 `mlucas-avx2` 還在背景跑。要手動
|
||||
`blanton_tr518.sh stop` + `kill`,否則下一輪任何測試都在被污染的狀態上開始
|
||||
`l2 learn off` / `test mode nr=yes`,而且 CPU 壓力還在跑。要手動
|
||||
`blanton_tr518.sh stop` + `systemctl stop mlucas-amm`(V1.1.8 起它是 systemd unit,
|
||||
關掉 Tera Term 或 `kill $(jobs -p)` 都收不掉),否則下一輪任何測試都在被污染的狀態上開始
|
||||
(Script A 的 `bgctl reset/stop --all` 收得掉 job,但收不掉 bcm 的 loopback 設定)。
|
||||
可考慮加一支對應的收尾腳本,或在檔尾用 `messagebox` 提示要下哪些指令。
|
||||
- **margin 在 soak 期間不再取樣**(V1.1.4):只剩 Script A 的起始值與 Script C 的結束值。
|
||||
漂移發生在中間就看不到了。若要恢復,比較好的做法是在 B 的迴圈內每 N 輪才掃一次,
|
||||
而不是每輪都掃。
|
||||
- **`blanton_ber.sh` 的 `sleep 30` 與 `BER_INTERVAL` 是兩處要同步改的數字**:Script C 寫死
|
||||
`sendln "sleep 30"`,腳本預設 `BER_INTERVAL=30`。改一邊沒改另一邊,report 會讀到還沒累積完
|
||||
一個 interval 的數字,而且不會有任何警告。
|
||||
- ✅ **`blanton_ber.sh` 不再把 lane 的量測結果當成 shell 錯誤**(v1.1.1,2026-09-15):
|
||||
`prbs get` 每條 lane 一行,沒鎖上的會印 `PRBS Error!`,而錯誤樣式是拿
|
||||
`grep -i 'ERROR|Invalid|...'` 掃整份輸出的 —— 於是正常的 init 會報「the BCM shell rejected」,
|
||||
而且提早 return,**把「哪幾條 lane 沒鎖」這個真正有用的資訊吞掉**。現在 per-lane 行排除在
|
||||
掃描外、改成大小寫敏感、真正的拒絕會把該行印出來。
|
||||
⚠️ **這類「完整性檢查自己看不到失敗」的錯在這個 repo 已經出現三次**
|
||||
(`bcm_dsc_scan.sh` v1.3.0 數 marker、`blanton_ber.sh` v1.1.1、以及 power cycle 報告的
|
||||
ping 解析把 100% 丟包當成「沒資料」)。寫檢查時要問:**失敗長什麼樣,這段程式碼看得到嗎?**
|
||||
- **`blanton_ber.sh` 的 `sleep 30` 與 `BER_INTERVAL` 是兩處要同步改的數字**:Script A(V1.2.1 起)
|
||||
與 Script C 都寫死 `sendln "sleep 30"`,腳本預設 `BER_INTERVAL=30`。改一邊沒改另一邊,report 會
|
||||
讀到還沒累積完一個 interval 的數字,而且不會有任何警告。現在有**三處**要一起改。
|
||||
- **Script A 的 BER `clear` 與第一輪流量之間沒有 settle**:V1.2.1 把 BER 掃描放在流量**之前**,
|
||||
而 PRBS 是把 lane 拉出正常運作、`clear` 才放回去的。Script A 的 `blanton_ber.sh clear` 之後
|
||||
緊接著就是 `blanton_traffic_linespeed ps` / `init`,中間沒有等待;Script C 的 BER 在流量報表
|
||||
之後,沒有這個順序。上機時比對兩份 traffic report —— Script A 有 NA/FAIL 而 Script C 沒有,
|
||||
就是 lane 尚未回穩,在 `clear` 後加一個 `sleep` 即可。**尚未上機驗證。**
|
||||
- **`ce0` 的 ingress mirror 尚未上機驗證**:`mirror port ce0 Mode=Ingress DestPort=ce0` 這條路是
|
||||
推論出來的(鏡像複本不受 egress filter 的 source-port knockout 限制),語法取自 DUT 的
|
||||
`help mirror`,但**還沒證明硬體真的接受 destport == srcport**。驗證方式是隔 10 秒看兩次
|
||||
|
||||
@@ -9,15 +9,24 @@ DUT 端的取數能力由一組 **bash 工具庫** 提供(CB FPGA `pcimem` →
|
||||
|
||||
完整架構見 [ARCHITECTURE.md](ARCHITECTURE.md)。
|
||||
|
||||
**目前狀態(2026-09-04)**:Script A 為 **V1.1.7**(A 集中記錄所有 TTL 的變更)。涵蓋
|
||||
**目前狀態(2026-09-15)**:Script A 為 **V1.2.2**(A 集中記錄所有 TTL 的變更)。涵蓋
|
||||
PCIe AER / DDR EDAC / PMON+TPM / 電壓 margin(9 顆 LTC2980)/ SWB loopback 線速流量 /
|
||||
10G+1G 管理網路 ping / BMC DDR 與 USB / NVMe 健康,壓力層走平台的 `bgctl`。
|
||||
switch unit 數量由 `config.ttl` 的 `SWB_UNIT0` / `SWB_UNIT1` 宣告,風扇轉速由 `FAN_SPEED` 指定
|
||||
(預設 100 = 全速;soak 是熱測試,散熱不該是變因)。
|
||||
測試 job log 收在 DUT 的 `/host/hw-eval/jobs/`,Script C 結束時掛好 USB 再帶時戳複製到 `/mnt/usb/`。
|
||||
100G PRBS(`port_prbs_monitor.sh`)已隨包交付,但 **A/B/C 的呼叫仍註解掉**,還在 bring-up。
|
||||
Script B 的 soak 迴圈每輪印 `bgctl list` + `jobs`,並 `pause 60`(V1.0.10 之前完全沒有 pause)。
|
||||
V1.1.0 新增全機 SerDes BER 掃描(`blanton_ber.sh`,**Script C 已接**,依 `SWB_UNIT` 分支)、
|
||||
100G PRBS(`port_prbs_monitor.sh`)已隨包交付、可手動執行,但 **V1.1.9 起 A/B/C 裡註解掉的呼叫已整段刪除**,
|
||||
要接回去得重寫(`bmc_monitor_ddr.sh` 在 Script C 的註解區塊同時移除)。
|
||||
V1.1.9 新增 **SerDes DSC 擷取**:`bcm_dsc_scan.sh`(DUT 端,`phydiag <port> dsc` 掃全機 ~445 port × 2 unit,
|
||||
約 10 分鐘)由 Script A 與 Script C 各跑一次夾住 soak,`bcm_dsc_parse.sh`(Host 端)把 raw log 轉成
|
||||
`_lane.csv` / `_pm.csv`。**由 `config.ttl` 的 `EN_TX_FIR` 控制,預設 0(關)** —— 開了整輪測試多 ~20 分鐘。
|
||||
Script B 的 soak 迴圈每輪印 `bgctl list` + `jobs` + `systemctl is-active mlucas-amm`,並 `sendln "sleep 60"`
|
||||
(V1.0.10 之前完全沒有間隔;V1.2.0 起 A/B/C 的等待一律改成 DUT 端 `sleep` 而非 macro 端 `pause`)。V1.1.8 起 AMD CPU 壓力(`mlucas-avx2`)由 `systemd-run` 掛成
|
||||
`mlucas-amm` transient unit(`Restart=always`),不再是 shell 的 `&` 背景 job —— 掉線也不會停,
|
||||
Script C 用 `systemctl stop mlucas-amm` 收掉並把 log 複製到隨身碟。
|
||||
V1.1.0 新增全機 SerDes BER 掃描(`blanton_ber.sh`,依 `SWB_UNIT` 分支);**V1.2.1 起 Script A 也接了一次**,
|
||||
**V1.2.2 起清單納入 port 268(100G uplink,ce0)**,summary 每個 unit 拆成「200G ports / 100G p268 / TOTAL」三列;
|
||||
在第一輪流量之前取基線,與 Script C 流量報表之後的收尾掃描一起夾住 soak(同 DSC 的做法)。另有
|
||||
TR518 負載(`blanton_tr518.sh`,**TTL 尚未接**)、SWB 軌功耗(`TH6_SWB{0,1}_power_readback.sh`),
|
||||
線速流量也納入 100G `ce0`(VLAN 138,靠 ingress mirror 維持循環)。
|
||||
Script A 開頭會先收掉前一輪殘留(`hw-test-session finish` + `bgctl reset/stop --all`),
|
||||
@@ -144,9 +153,38 @@ sleep 30 # 至少等一個 interval 再 rep
|
||||
./Blanton_Script/blanton_ber.sh report # 每 lane 一列,BER < 1e-6 才 PASS,最後 summary
|
||||
./Blanton_Script/blanton_ber.sh stop # prbsstat STOp
|
||||
./Blanton_Script/blanton_ber.sh clear # prbs clear(會解除 PRBS,要重測得再 init)
|
||||
# summary 每個 unit 三列:200G ports / 100G p268 / TOTAL(兩個 unit 各有自己的 ce0(268))
|
||||
# ⚠️ 某一組「一條 lane 都沒回報」算 FAIL 不算 PASS —— 待測的 port 沉默不是通過
|
||||
# -u 0 / -u 1 單邊;-c 1e-9 換門檻;-F 只列失敗;-f <log> 離線重解
|
||||
# ⚠️ 不能跟 blanton_traffic_linespeed / blanton_tr518 同時跑(都會接管同一批 port)
|
||||
|
||||
# ── SerDes DSC 擷取(TX_FIR / RXFFE / DFE / EYE / SNR)──
|
||||
./Blanton_Script/bcm_dsc_scan.sh # unit 0+1 全掃,約 10 分鐘
|
||||
./Blanton_Script/bcm_dsc_scan.sh -u 0 # 只掃一個 unit(時間減半)
|
||||
./Blanton_Script/bcm_dsc_scan.sh -f my.log -F # 自訂檔名 + 覆蓋
|
||||
./Blanton_Script/bcm_dsc_scan.sh -n # dry-run,只列要掃的 port
|
||||
# log: ~/Blanton_Script/log/,掃完自動 cp 到 /host/hw-eval/jobs(-C 可關)
|
||||
# 檔名沒帶 %u 時會自動插 _u0 / _u1,所以兩個 unit 不會互相覆蓋
|
||||
# 結尾固定印 "### DSC_SCAN_DONE rc=<n>";rc=0 才是全部 port 都拿到完整 dump
|
||||
# ⚠️ 由 TTL 的 EN_TX_FIR 控制,預設 0(關)—— A 與 C 各掃一次,開了整輪多 ~20 分鐘
|
||||
|
||||
# ── DSC raw log → CSV(離線解,Host 或 DUT 都可)──────
|
||||
sh ~/Blanton_Script/bcm_dsc_parse.sh -i blanton_dsc_ScriptA_u0.log -o u0
|
||||
# 產出 u0_lane.csv(每 lane 一列)+ u0_pm.csv(每 pm_id 一列)
|
||||
# 純 sh + awk(mawk 相容),欄位靠括號群組順序抓,SDK 改排版不會壞;
|
||||
# 群組數 != 9 的列標成 UNPARSED 印到 stderr,不會安靜解錯
|
||||
|
||||
# ── 100G link down/up 循環 + DSC(TTL 尚未接)────────
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh # 印說明
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 200 # 跑 200 輪(-c 200 亦可)
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 200 -d # 每行指令以灰色 [CMD] 印出來
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 20 -T # 跳過 tuning
|
||||
./Blanton_Script/bcm_100g_dsc_cycle.sh 1 -n # dry-run,不碰硬體也不 sleep
|
||||
# log: ~/dsc_log/100G_dsc_cycle_test_<ts>.log,每輪一段 '### TEST CYCLE n/N'
|
||||
# 每輪收:兩個 unit 的 phydiag dsc + serdes-tx-get / serdes-rx-get sys 0..7
|
||||
# 等 link 是輪詢 'bcmcmd ps 268'(每 -p 秒,上限 -w 秒),不是 show interfaces status
|
||||
# ⚠️ 尚未上機驗證;與 blanton_ber / blanton_traffic_linespeed / blanton_tr518 互斥
|
||||
|
||||
# ── TR518 負載 + 量功耗(TTL 尚未接)─────────────────
|
||||
./Blanton_Script/blanton_tr518.sh start # 兩 unit,跑完 settle 120s
|
||||
./Blanton_Script/TH6_SWB0_power_readback.sh # 19 條軌 V×I + TOTAL POWER
|
||||
@@ -255,7 +293,17 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
`SSD_RUNTIME`/`SSD_COUNT`、`USB_RUNTIME`/`USB_COUNT` 指定(COUNT 就是 `--passes`)。
|
||||
`qfx5252-stress-*` 跑滿 RUNTIME 秒就停,**`--passes` 不會把它延長** —— `RUNTIME=600` 就是
|
||||
10 分鐘,不是 600×COUNT。要 24 小時就填 `86400`。壓力提早結束時 `bgctl list` 數量變少是
|
||||
唯一線索,**不會報警**
|
||||
唯一線索,**不會報警**。V1.1.8 起出貨預設改為 `RUNTIME=3600`(1 小時),2 天 soak 要填 `172800`
|
||||
- ⚠️ **`utils/kill_all_process.ttl` 只有 `kill $(jobs -p)`,抓不到 systemd unit**:把壓力工作改成
|
||||
`systemd-run` 之後,Script C 一定要補對應的 `systemctl stop <unit>`,否則它會活過 Script C 燒進
|
||||
下一輪,**收工的溫度/功耗全部是在滿載下量的**。V1.1.8 已為 `mlucas-amm` 補上。
|
||||
Script 4 也改用同一個 unit,但它**沒有收尾段**,所以那行 `systemctl stop` 得由人手動下
|
||||
- ⚠️ **`config.ttl` 少一個變數會讓 macro 當場中止,不是當成 0**:`EN_timestamp`(V1.1.8 新增,
|
||||
餵 Script A `logopen` 的第 5 個參數)、`BMC_USB_RUNTIME` 都是必要項。只更新 `.ttl` 卻沿用舊的
|
||||
`config.ttl`,Script A 會停在 `logopen` 那一行
|
||||
- ⚠️ **log 時間戳的「格式」不在腳本裡**:`EN_timestamp` 只決定開不開,格式在 Tera Term 的
|
||||
Setup → Additional settings → Log,存在**那台電腦的 `TERATERM.INI`**。同一份 macro 在兩台機器上
|
||||
可以產生不同格式的 log,跨機比對前先確認
|
||||
- ⚠️ **多相 VRM 與 LTC2980 是同一批軌的兩條路**:`blanton_multiphase_margin.sh`(原生 i2c)
|
||||
和 `margin.sh`(LTC2980 PMBus)都假設自己是唯一寫入者,**沒有互斥檢查**,只靠人記得
|
||||
- ⚠️ **traffic 的計數器基準在 v0.7.0 變了**:`tx 100 length=512` → `tx 200 length=324`,
|
||||
@@ -263,7 +311,8 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
- ⚠️ **Script 4(thermal/safety)沒有收尾**:`while 1` 之後就是檔尾。中斷後 DUT 停在
|
||||
`port cd lb=mac` / `l2 learn off` / `test mode nr=yes`,`mlucas-avx2` 也還在背景。
|
||||
下一個測試會在被污染的狀態上開始,而且 **`show interfaces status` 看不出來**。
|
||||
收尾:`blanton_tr518.sh stop` + `kill $(jobs -p)`
|
||||
收尾:`blanton_tr518.sh stop` + `kill $(jobs -p)` + **`systemctl stop mlucas-amm`**
|
||||
(V1.1.8 起 CPU 壓力是 systemd unit,`kill $(jobs -p)` 收不掉,少這一行 CPU 會一直滿載)
|
||||
- ⚠️ **Script 4 與 Script B 不能共用同一台 DUT**:兩者都會接管 `port cd` 的 loopback
|
||||
- ⚠️ **`settings/*.conf` 的 `VNOM` 是實測中位數,不是規格值**(2026-08-30 起)。它同時是
|
||||
`margin_status` 偏差 % 的分母、也是 `margin_set` 推算 high/low/OV/UV 的基準,所以改它會
|
||||
@@ -290,9 +339,17 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
`Failed to execute the diagnostic command` 都照樣回 0),所以 `blanton_traffic_linespeed.sh`
|
||||
是**比對輸出**而非看 exit code。另外 `bcmcmd` 一律要 `</dev/null` ——
|
||||
否則它會吃掉 while 迴圈的 stdin,**只有第一個 VLAN 被設定**(V0.3.1 修過的 bug,很難看出來)。
|
||||
**2026-09-07 `bcm_dsc_scan.sh` 又踩了同一個坑**:port 清單走 stdin,第一次 `bcmcmd` 就把剩下的吃光,
|
||||
445 個 port 只掃了 1 個,而且 marker 與 dump 數量一起變少 → 比對通過、exit 0、log 看起來正常。
|
||||
修法是迴圈改讀 fd 3(`done 3< "$PORTFILE"`)+ `bcmcmd ... < /dev/null`,並加一道
|
||||
「實際迭代筆數 == 清單筆數」的檢查。**任何 `while ... done < file` 裡呼叫 `bcmcmd` 都要當它會讀 stdin。**
|
||||
- ⚠️ **`stress_hhmd.py` / `stress_pcie.py` 已於 2026-08-17 由 KC 移除**(Status 表 ScriptB #5/#7)。
|
||||
Script B 現在是 **8 個背景 job**(1 mlucas + 5 mem + ssd + usb),不是 10 個。
|
||||
I2C/PCIe 的持續讀取壓力目前沒有替代品。
|
||||
- ⚠️ **`pause` 是停 macro,`sleep` 是停 DUT**(V1.2.0 起 A/B/C 一律用後者):`pause N` 期間 macro 不理會
|
||||
DUT,shell 是閒著的,等於用猜的估上一個指令跑多久;`sendln "sleep N"` + `wait prompt_sonic_root`
|
||||
把等待放到 DUT 上,prompt 回來才繼續,兩邊不會走散,而且這一段會帶時戳進 log。
|
||||
**Script 4 / Script 5 尚未轉換,仍在用 `pause`**;B/C 開頭那個 `pause 1` 也保留(登入前沒有 prompt 可等)
|
||||
- ⚠️ **soak 迴圈沒有 `pause`**:取樣間隔不是 Status 表寫的 10 分鐘,而是取決於指令執行時間。
|
||||
(`-u 0` 已於 `491e807` 移除,A/B/C 現在都跑兩個 switch unit。)
|
||||
- 🔒 **`.sh` / `.conf` 一定要 LF**。CRLF 的 shell script 在 SONiC 上直接掛(`$'\r': command not found`);
|
||||
@@ -311,6 +368,15 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
- ⚠️ **兩張管理網卡同網段會 ARP flux**:`mgmt_ping_monitor.sh` 讓 eth0/eth1 同時 up,同一個 /24 下
|
||||
對端 ARP 可能由任一張卡回應,回包落在沒送封包的那張。`ARP_STRICT=1` 擋掉;判讀時看 `RESULT` 行的
|
||||
`nic_tx`/`nic_rx`(該卡自己的 packet delta)—— **ping 成功但 `nic_tx` 接近 0 = 封包從另一張卡出去的**
|
||||
- ⚠️ **`prbs get` 印的每一行是資料,不是 shell 的錯誤訊息**:沒鎖上的 lane 會印 `PRBS Error!`,
|
||||
而 `blanton_ber.sh` 從 v1.0.0 就用 `grep -i 'ERROR|Invalid|...'` 掃**整份輸出**判斷「指令被拒絕」——
|
||||
於是**跑得好好的 init 會報 `the BCM shell rejected 'phydiag ... prbs get'`**,而且因為提早 return,
|
||||
那段「數 PRBS OK / 總數」的檢查根本沒跑,**你看到一個假錯誤,卻看不到哪幾條 lane 真的沒鎖**。
|
||||
v1.1.1 已把 per-lane 行排除在錯誤掃描外、改成大小寫敏感,並把真正的拒絕訊息印出來
|
||||
- ⚠️ **BER 的 `clear` 才是把 lane 放回正常運作的動作**:V1.2.1 起 Script A 在第一輪流量**之前**跑 BER,
|
||||
而 `blanton_ber.sh clear` 與緊接著的 `blanton_traffic_linespeed ps` 之間**沒有任何 settle**。
|
||||
Script C 是流量報表之後才跑 BER,沒有這個順序問題。第一次上機時比對兩邊的 traffic report ——
|
||||
**Script A 有 NA/FAIL 而 Script C 沒有,就是 lane 還沒回穩**,在 `clear` 後補一個 `sleep` 即可
|
||||
- ⚠️ **PRBS 跑起來時 `show interfaces status` 會顯示 DOWN**,那是 link 離開正常運作模式,不是故障。
|
||||
`port_prbs_monitor.sh` 的 `start` 因此刻意不印 port status(避免測試員誤判),只在 `prbs clear` 之後的報表裡印
|
||||
- ⚠️ **`ip -s link` 的欄位順序是 `bytes packets errors ...`**,要 packet 數取的是**第 2 欄**不是第 1 欄
|
||||
@@ -355,5 +421,9 @@ Tera Term 連 COM port (115200-8-N-1)
|
||||
- `tmp/` — 🚫 gitignored:**還沒上機驗證**的腳本暫存區(目前 `blanton_reboottest.sh`)。
|
||||
上機測過才搬進 `src/Script_ABC_Blanton/Blanton_Script/` 並進 git —— 未驗證的東西不該混在
|
||||
交付包裡,publish 時測試員分不出哪支能信
|
||||
- `src/Script_ABC_Blanton/hammer/` — 🚫 gitignored:mlucas 的 ELF binary(~99 MB,avx512 / avx2 /
|
||||
avx / sse2 / generic-c)。Script B 與 Script 4 會 `chmod +x` 後執行,但**它由 DUT image 帶**,
|
||||
不從這個 repo 發 —— binary 一旦進 git 就永久留在 history,每個 clone 都要付這個代價。
|
||||
`publish.sh` 也把它排除,所以打包結果不會因為誰的工作目錄有沒有 binary 而變成 100 MB
|
||||
- `secret/` — 🚫 gitignored(除 `README.md` 與 `*.example`):DUT 帳密、per-unit BDF 覆寫、COM 設定
|
||||
- `For_AI/` — 🚫 gitignored:AI 協作素材(波形截圖、草稿筆記)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# v1.1.8 — The CPU stress no longer dies with your terminal
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**`mlucas` now runs as a systemd unit, so it survives a lost session**
|
||||
* The AMD CPU load is started as `systemd-run --unit=mlucas-amm --property=Type=exec --property=Restart=always --property=RestartSec=2`, instead of being backgrounded with `&` in the login shell. Script B and the thermal/safety macro (Script 4) both use it.
|
||||
* As a shell job it belonged to that one terminal: close Tera Term, drop the console, reconnect on a different session, and the load was gone — while `jobs` from the new session showed nothing to tell you. A multi-day soak could spend most of its length with an idle CPU and produce thermal numbers that looked fine because nothing was loading the part.
|
||||
* `Restart=always` brings it back if it exits on its own, and the log is appended (`>>`) inside the unit while being truncated (`>`) once before the unit starts, so a restart adds to the run rather than erasing it.
|
||||
* Script B now prints `systemctl is-active mlucas-amm` beside `jobs`, both in the one-shot check and in every pass of the monitor loop. One word — `active` — tells you the CPU is still loaded.
|
||||
|
||||
> The old `&` line is kept commented out directly above the new one, to fall back to if `systemd-run` is not available on a build.
|
||||
|
||||
**Script C stops the unit and brings its log home**
|
||||
* `systemctl stop mlucas-amm` was added next to the existing `kill $(jobs -p)`, and `mlucas_amm_log` is copied into `/host/hw-eval/jobs/` with the other stress logs.
|
||||
* `kill $(jobs -p)` cannot reach a systemd unit, so without this the load would have run straight through the post-test snapshot and into the next test — every temperature and power reading taken at full load, with nothing in `show interfaces status` to reveal it.
|
||||
|
||||
**`EN_timestamp` — turn the per-line log timestamp on or off from `config.ttl`**
|
||||
* Script A's `logopen` takes its 5th argument from the new flag: `logopen filename 0 1 1 EN_timestamp`. The argument order (`<file> <binary> <append> <PlainText> <timestamp>`) is now spelled out above the call.
|
||||
|
||||
**`BMC_USB_RUNTIME` — the BMC USB ping duration is a knob, not a constant**
|
||||
* It was hard-coded to `-w 14400` (4 hours), which no longer matched the soak lengths the SSD and USB knobs were being set to. The shipped default is `172800` (2 days).
|
||||
|
||||
**The shipped stress defaults are now one hour**
|
||||
* `SSD_RUNTIME` and `USB_RUNTIME` go from 600 to `3600`. v1.1.7 shipped a 10 minute stress by default.
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**`publish.sh` produced a different bundle depending on whose machine it ran on**
|
||||
* `rsync` copied whatever sat in `src/Script_ABC_Blanton/`, so a developer with the `hammer/` mlucas binaries in their working tree shipped a ~100 MB zip while a fresh clone shipped ~200 KB. `hammer/` is now excluded from the bundle and ignored by git.
|
||||
* `publish.sh -h` also printed the first lines of code along with the usage text, and would have truncated the usage block as the header grew. It now reads the header up to the first line of code.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.1.8.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
**After copying to the DUT:**
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh # see below
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there - see below
|
||||
```
|
||||
|
||||
`blanton_ber.sh`, `nfc_polling.sh`, `blanton_tr518.sh`, `blanton_multiphase_margin.sh`, `pwr_brick.sh` and the two `TH6_SWB*_power_readback.sh` are invoked through `./` or `~/`, so they need the execute bit along with `bmc_monitor.sh` and `usb_target.sh` — neither git nor a Windows/USB copy carries it reliably.
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Replace `config.ttl` along with the macros.** `EN_timestamp` and `BMC_USB_RUNTIME` are new, and Tera Term aborts the macro on an undefined variable rather than treating it as 0 — keeping an old `config.ttl` stops Script A at the `logopen` line.
|
||||
* **`hammer/` is not in the bundle.** The ~99 MB of mlucas binaries comes with the DUT image, not from this repo. If `~/hammer/tools/amd/mlucas-avx2` is missing, Script B does not stop — it starts a unit that fails and is restarted every 2 seconds. Check for the file before you start.
|
||||
* **The timestamp format is not in the macro.** `EN_timestamp` only decides whether there is one; the format lives in Tera Term's Setup → Additional settings → Log, stored in that PC's `TERATERM.INI`. Two benches running the same macro can produce differently stamped logs.
|
||||
* **The three stress durations are deliberately different**: SSD and USB at 1 hour, the BMC USB ping at 2 days. Set each to match the soak you are actually running.
|
||||
* **`--runtime` is the job's overall limit, not a per-pass one.** `qfx5252-stress-*` stops after `RUNTIME` seconds whatever `--passes` says. For a 24 hour soak set `RUNTIME` to `86400`.
|
||||
* **`mlucas-amm` has not been run on hardware yet.** The unit resolves `~` through bash's fallback to the passwd entry, because systemd does not set `HOME` for a system service without `User=`. It should land on `/root/hammer`, the same path the interactive shell uses — confirm it on the bench, or change both paths to an absolute `/root/hammer/...`.
|
||||
* **Script 4 uses the same unit but has no teardown.** The thermal/safety macro now starts `mlucas-amm` the same way Script B does, and it still ends at `while 1` — so stopping it means `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`. Closing Tera Term does not stop the CPU load any more.
|
||||
* **Script 5 (reboot cycle) is still unverified on hardware**, still has no exit condition, and `wait "Booting in blind mode"` still has no `timeout` guard.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh` — same rails, two access paths, no interlock.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **The BMC USB service log only exists on the USB stick.**
|
||||
* **These three cannot share the switch**: `blanton_ber.sh`, `blanton_tr518.sh`, `blanton_traffic_linespeed`.
|
||||
* **`FAN_SPEED` is 100.**
|
||||
* **Still unverified from v1.1.0**: the `ce0` ingress mirror, the 120 s settle in `blanton_tr518.sh`, and `test mode nr=no` on stop.
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
* [Blanton_Test_Flow.drawio](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/docs/Blanton_Test_Flow.drawio) — the scripts side by side
|
||||
|
||||
**Full changelog:** [V1.1.7...V1.1.8](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.1.7...V1.1.8)
|
||||
@@ -0,0 +1,76 @@
|
||||
# v1.2.0 — SerDes DSC capture, and the macro now waits on the DUT
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**Full-machine SerDes DSC capture, before and after the soak**
|
||||
* `bcm_dsc_scan.sh` runs `phydiag <port> dsc` through the Broadcom diag shell across the whole port list — one 100G port plus the 200G ranges, about 445 per unit, on both units — and writes the raw log under `Blanton_Script/log/`, then copies it to `/host/hw-eval/jobs/`.
|
||||
* Script A takes a baseline and Script C takes a closing dump, so the pair brackets the stress run and the two are directly comparable.
|
||||
* `-u 0` scans one unit, `-f <name>` names the log (`%u` → `u0`/`u1`, `%t` → timestamp), `-F` overwrites, `-n` dry runs, `-C` skips the copy. A name without `%u` gets `_u0` / `_u1` inserted automatically, so unit 1 can never overwrite unit 0.
|
||||
* Every run ends with `### DSC_SCAN_DONE rc=<n>`. `rc=0` means every port returned a complete dump.
|
||||
* `phydiag ... dsc` is read-only — it does not change any SerDes setting.
|
||||
|
||||
**`bcm_dsc_parse.sh` turns a raw DSC log into two CSVs**
|
||||
* `<prefix>_lane.csv` — one row per lane: TXEQ, RXFFE, DFE, EYE, SNR and the rest.
|
||||
* `<prefix>_pm.csv` — one row per `pm_id`: temperature, PLL, lane count, SNR/EYE statistics.
|
||||
* POSIX `sh` + `awk`, no Python, so it runs on the DUT as happily as on a laptop. Lane rows are read by the order of their bracket groups rather than by fixed column positions, so the SDK re-spacing its output does not break parsing; a row that does not yield 9 groups is reported as `UNPARSED` on stderr instead of being parsed wrongly.
|
||||
|
||||
**`EN_TX_FIR` in `config.ttl` — the DSC scan is off by default**
|
||||
* A full scan is about 10 minutes and both Script A and Script C do one, so the shipped default (`0`) keeps ~20 minutes out of a routine run. Set it to `1` when the SerDes data is what you are after.
|
||||
|
||||
**The macro now waits on the DUT instead of on itself**
|
||||
* Every `pause N` in Scripts A, B and C became `sendln "sleep N"` followed by `wait prompt_sonic_root`.
|
||||
* `pause` stopped the macro for N seconds without touching the DUT — the shell sat idle and the macro was guessing how long the previous command would take. On a loaded DUT the guess ran short and the next command went out before the previous one had finished, putting the whole run a beat ahead of the switch. Now the wait happens on the DUT and the macro resumes when the prompt actually comes back.
|
||||
* It also means the wait appears in the console log with a timestamp, so a round that ran long is visible afterwards.
|
||||
* Timings changed with it: the management-ping sample in Script A is 15 s (was 30), the settle after each traffic `clear` is 5 s (was 15). The traffic run itself stays 15 s and Script B's monitor loop stays 60 s.
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**The DSC scan only ever scanned one port — and reported success**
|
||||
* The scan loop read its port list on stdin, and `bcmcmd` reads stdin too, so the first call swallowed the rest of the file and the loop ended after a single port.
|
||||
* Nothing looked wrong. The log held one port marker and one dsc block, the marker-to-dump comparison passed because both numbers shrank together, the exit code was 0, and a 445-port scan simply finished in seconds.
|
||||
* The loop now reads on file descriptor 3 and `bcmcmd` is given `< /dev/null`, and there is a new check that the number of iterations matches the number of ports in the list — a check that cannot be fooled by the loop ending early.
|
||||
|
||||
**Dead call sites removed from Scripts A, B and C**
|
||||
* The commented-out 100G PRBS calls, a duplicated commented 100G port-status block in Script A, and the stale `bmc_monitor_ddr.sh` block in Script C have been deleted. They had been dead since bring-up.
|
||||
* `port_prbs_monitor.sh` still ships and still runs by hand — but re-enabling it means writing the calls again, not un-commenting them.
|
||||
|
||||
## 🔧 Changes
|
||||
|
||||
* **Script C no longer prints the BMC poll log or the mlucas log to the console.** Both files are still copied to `/host/hw-eval/jobs/` and out to the USB stick with everything else; only the master log got shorter.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.0.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
**After copying to the DUT:**
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh # bcm_dsc_scan.sh is invoked as ./Blanton_Script/... and needs it
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Replace `config.ttl` along with the macros.** `EN_TX_FIR` is new, and Tera Term aborts on an undefined variable rather than treating it as 0.
|
||||
* **`bcm_dsc_scan.sh` needs the execute bit.** Script A and Script C invoke it as `./Blanton_Script/bcm_dsc_scan.sh`. Without `chmod +x` it fails with `Permission denied`, the macro walks straight past it, and Script C's four `cp` lines then each report a missing file.
|
||||
* **The DSC scan shares the ports with `blanton_ber.sh`, `blanton_traffic_linespeed` and `blanton_tr518.sh`** — do not run it alongside any of them.
|
||||
* **Script 4 and Script 5 still use `pause`.** Only A, B and C were converted.
|
||||
* **`--runtime` is the job's overall limit, not a per-pass one.** `--passes` does not extend it; for a 24 hour soak set `RUNTIME` to `86400`.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **Script 5 (reboot cycle) is still unverified on hardware**, has no exit condition, and its `wait "Booting in blind mode"` has no `timeout` guard.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **The BMC USB service log only exists on the USB stick.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
* **Still unverified from v1.1.0**: the `ce0` ingress mirror, the 120 s settle in `blanton_tr518.sh`, and `test mode nr=no` on stop.
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
* [Blanton_Test_Flow.drawio](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/docs/Blanton_Test_Flow.drawio) — the scripts side by side
|
||||
|
||||
**Full changelog:** [V1.1.8...V1.2.0](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.1.8...V1.2.0)
|
||||
@@ -0,0 +1,49 @@
|
||||
# v1.2.1 — A BER baseline, so the SerDes numbers have something to be compared against
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**Script A now takes a SerDes BER sweep before the first traffic run**
|
||||
* `blanton_ber.sh init → start → sleep 30 → report → stop → clear`, branched on `SWB_UNIT0` / `SWB_UNIT1` exactly like the traffic block beside it.
|
||||
* Script C already ran the closing sweep after its traffic report. With this one at the front, the pair brackets the soak — the same shape as the DSC scans added in v1.2.0, and the same reason: a BER figure on its own says very little, a *change* in it says a lot.
|
||||
* The `sleep 30` matches `blanton_ber.sh`'s `BER_INTERVAL`. Reporting before one full interval has accumulated reads a window that has not finished, and nothing warns you about it.
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**The new block's comment described the wrong position**
|
||||
* It was copied out of Script C, which runs BER *after* the traffic report, and the comment came with it and still said "after". In Script A the block runs *before* traffic. The comment now says what the code does and why the ordering is what it is.
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Watch Script A's first traffic report on the first run.** PRBS holds the lanes out of normal operation and `blanton_ber.sh clear` is what releases them — and in Script A nothing settles between that `clear` and the `blanton_traffic_linespeed ps` / `init` that follows. Script C does not have this ordering, so the two reports are a built-in comparison: **if Script A shows NA or FAIL pairs that Script C does not, the lanes had not come back yet**, and a `sleep` after `clear` is the fix. This has not been run on hardware yet.
|
||||
* **`sleep 30` and `BER_INTERVAL` now live in three places** — Script A, Script C, and `blanton_ber.sh` itself. Changing one without the others silently reports a partial interval.
|
||||
* **BER, `blanton_traffic_linespeed` and `blanton_tr518.sh` cannot overlap.** They all take over the same ports. Script A now touches those ports twice before the soak even starts.
|
||||
* **`EN_TX_FIR` is still 0 by default**, so the DSC scans do not run unless you turn them on; a full scan is ~10 min in each of Script A and Script C.
|
||||
* **Replace `config.ttl` along with the macros.** Tera Term aborts the macro on an undefined variable rather than treating it as 0.
|
||||
* **`bcm_dsc_scan.sh` needs the execute bit**: `chmod +x ~/Blanton_Script/*.sh` after copying.
|
||||
* **`--runtime` is the job's overall limit, not a per-pass one.** `--passes` does not extend it.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **Script 5 (reboot cycle) has no exit condition.** A 199-cycle / 60-hour run has now been analysed: PCIe AER stayed at 0, all 218 traffic pairs and all 864 lanes passed every cycle, and the BMC never reset. Two things it did surface — `bmc-first-enroll` does not exist on that BMC build, and NFC returned FAIL on every cycle.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **The BMC USB service log only exists on the USB stick.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
* **Still unverified from v1.1.0**: the `ce0` ingress mirror, the 120 s settle in `blanton_tr518.sh`, and `test mode nr=no` on stop.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.1.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
|
||||
**Full changelog:** [V1.2.0...V1.2.1](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.2.0...V1.2.1)
|
||||
@@ -0,0 +1,63 @@
|
||||
# v1.2.2 — The 100G uplink joins the BER sweep, and a boot cycle that can pull the power
|
||||
|
||||
## ✨ New features
|
||||
|
||||
**The BER sweep now measures port 268, the 100G uplink**
|
||||
* The port list ran `256-259` straight to `270-308`, so `ce0(268)` — the 100G uplink — was never in it. It is now its own group alongside the 200G ports.
|
||||
* The summary gives **each unit three rows**: `200G ports`, `100G p268`, and `TOTAL`. Unit 0 and unit 1 each have their own `ce0(268)`, and a marginal uplink lane averaged into 432 others is invisible; on its own row it is not.
|
||||
* A group that reports no lane at all now counts as a **failure**, not a pass. Silence on the port under test is the easiest way to ship a green report that measured nothing.
|
||||
|
||||
**Script 5 is now a boot cycle, warm or cold**
|
||||
* `5_Blanton_Script_Reboot_Cycle.ttl` → `5_Blanton_Script_Boot_Cycle.ttl`, with `EN_Cold_Warm_boot` in `config.ttl` choosing: `1` = `reboot` (warm), `2` = a cold cycle by writing the CB FPGA, `0` = neither.
|
||||
* The log file and the per-round banner are named after the kind of boot: `Blanton_Reboot_Cycle_<ts>.log` / `Reboot Cycle : N`, or `Blanton_Power_Cycle_<ts>.log` / `Power Cycle : N`.
|
||||
* Both stress logs are removed before each run. `qfx5252-stress-*` appends to its `--log` and the macro `cat`s the whole thing, so every cycle was printing back everything the earlier cycles had written — 289 lines at cycle 1 and 36,555 by cycle 199. At roughly 6.2 ms a line on a 115200 console that accounted for **+212 s of the +214 s the cycle grew** over a 199-cycle run. The DUT was never getting slower.
|
||||
|
||||
**`bcm_100g_dsc_cycle.sh` — 100G link down/up cycling with SerDes capture**
|
||||
* Tuning once, then N cycles of `port 268 en=0` → wait → `en=1` → wait → `phydiag 268 dsc` on both units → retimer `serdes-tx-get` / `serdes-rx-get` for `sys 0..7`. One log per run under `~/dsc_log/`, split by `### TEST CYCLE n/N`.
|
||||
* Waiting is a **poll of `bcmcmd ps 268`**, not `show interfaces status`. `port en=` changes the SDK, and the SONiC table only follows once gbsyncd has synced it — and this test pauses gbsyncd.
|
||||
* `-d` echoes every command as a grey `[CMD]` line before running it; `-n` is a dry run that touches nothing; the cycle count can be a bare argument (`./bcm_100g_dsc_cycle.sh 200`).
|
||||
|
||||
## 🐛 Bug fixes
|
||||
|
||||
**`blanton_ber.sh init` reported a hard error on runs that were working**
|
||||
* `[ERR ] unit 1: the BCM shell rejected 'phydiag ... prbs get'` — except the command was fine. `prbs get` prints one line per lane, a lane that has not locked says `PRBS Error!`, and the rejection check grepped the **whole** output case-insensitively for `ERROR|Invalid|...`.
|
||||
* Worse than the false alarm: the function returned early, so the lock count that was meant to judge those very lanes never ran. The operator saw an error they could not act on, and did **not** see which lanes had failed to lock. That check has been there since v1.0.0.
|
||||
* Per-lane result lines are now excluded, the match is case-sensitive, and a genuine rejection prints the offending line.
|
||||
|
||||
## 🔧 Changes
|
||||
|
||||
* **Script A takes a BER baseline before its first traffic run** (v1.2.1), so together with Script C's closing sweep the pair brackets the soak.
|
||||
* **`EN_TX_FIR`** gates the SerDes DSC scans in Script A and Script C. Default `0`: a full scan is ~10 min and both scripts do one.
|
||||
|
||||
## 📦 Downloads
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `Script_ABC_Blanton_V1.2.2.zip` | The full Tera Term working directory, including `Blanton_Script/` to copy onto the DUT |
|
||||
|
||||
```bash
|
||||
chmod +x ~/Blanton_Script/*.sh
|
||||
grep -rlU $'\r' ~/Blanton_Script # expect no output
|
||||
ls ~/hammer/tools/amd/mlucas-avx2 # must already be there
|
||||
```
|
||||
|
||||
## ⚠️ Before you run
|
||||
|
||||
* **Replace `config.ttl` with the macros.** `EN_Cold_Warm_boot` is new, and Tera Term aborts on an undefined variable rather than treating it as 0.
|
||||
* **Do not run the boot cycle with `EN_Cold_Warm_boot = 0`.** Nothing reboots, the `wait "Booting in blind mode"` underneath has no timeout, and since v1.2.2 there is no `logopen` either — so it hangs *and* leaves no log showing where. `0` is the value for the A → B → C flow, where that macro is not used.
|
||||
* **The cold path writes a raw FPGA register.** `0x480` is `ICB_CTRL_REG`; the register map carries no bit definitions for it and `0xD0000` is not documented anywhere in this repo. The BDF `0000:05:00.0` is hard-coded while `blanton_fpga_pcimem.sh` auto-detects it because it moves per unit — **check the BDF on every board before running the cold path**.
|
||||
* **Anything that parses these logs must accept both banners**, `Reboot Cycle : N` and `Power Cycle : N`. A parser that greps only the first finds zero cycles in a power-cycle log and reports nothing without complaining.
|
||||
* **`bcm_100g_dsc_cycle.sh` has not been run on hardware.** It pauses `gbsyncd0/1` for tuning and unpauses them on exit, Ctrl-C included; `-P` leaves them alone.
|
||||
* **`sleep 30` and `BER_INTERVAL` now live in three places** — Script A, Script C and `blanton_ber.sh`.
|
||||
* **BER, the DSC cycle, `blanton_traffic_linespeed` and `blanton_tr518.sh` cannot overlap.** They all take the same ports.
|
||||
* **Script 4 has no teardown**: finish it with `blanton_tr518.sh stop` **and** `systemctl stop mlucas-amm`.
|
||||
* **`blanton_multiphase_margin.sh save` is permanent**, and must not be mixed with `margin.sh`.
|
||||
* **`READ_VIN` and `READ_IIN` are still not in the ADPM12200 coefficient table.**
|
||||
* **`FAN_SPEED` is 100.**
|
||||
|
||||
## 🔗 Links
|
||||
|
||||
* [ARCHITECTURE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/ARCHITECTURE.md) — test flow, data models, constraints
|
||||
* [CLAUDE.md](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/src/branch/main/CLAUDE.md) — commands and the bench gotchas
|
||||
|
||||
**Full changelog:** [V1.2.1...V1.2.2](https://etgit.et-wen.com/etwen/Blanton_TTL_Script/compare/V1.2.1...V1.2.2)
|
||||
+14
-2
@@ -7,6 +7,9 @@
|
||||
# publish/Script_ABC_Blanton_<Version>, taking <Version>
|
||||
# from the "; Version : Vx.y.z" header of
|
||||
# 1_Blanton_Script_A.ttl.
|
||||
# V1.0.1 2026-09-06 Exclude hammer/ (the mlucas binaries) from the bundle,
|
||||
# so the output no longer depends on whether the developer
|
||||
# has them in their working tree.
|
||||
# =============================================================================
|
||||
#
|
||||
# The version is READ FROM Script A rather than passed in, so the folder name
|
||||
@@ -33,7 +36,9 @@ info() { printf '[\033[34mINFO\033[0m] %s\n' "$*"; }
|
||||
ok() { printf '[\033[32m OK \033[0m] %s\n' "$*"; }
|
||||
die() { printf '[\033[31mERR \033[0m] %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
usage() { sed -n '2,26p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0; }
|
||||
# Print the header block above, up to (but not including) the first line of
|
||||
# code, so adding a Version History entry never truncates the usage text.
|
||||
usage() { sed -n '2,/^set -u/p' "${BASH_SOURCE[0]}" | sed '$d' | sed 's/^# \{0,1\}//'; exit 0; }
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
@@ -86,12 +91,19 @@ fi
|
||||
# Captured console logs are per-run and often carry DUT serials, so they are
|
||||
# never shipped; the empty logs/ directory itself is kept because Script A's
|
||||
# logopen writes into it.
|
||||
#
|
||||
# hammer/ is excluded as well: it is ~99 MB of mlucas ELF binaries that the DUT
|
||||
# image already carries, and it is gitignored. Without this exclude the bundle
|
||||
# would depend on whether the developer happened to have the binaries sitting in
|
||||
# src/ - 100 MB from one machine, 200 KB from a fresh clone.
|
||||
mkdir -p -- "${DEST}" || die "cannot create ${DEST}"
|
||||
if command -v rsync >/dev/null 2>&1; then
|
||||
rsync -a --exclude='*.log' "${SRC_DIR}/" "${DEST}/" || die "rsync failed"
|
||||
rsync -a --exclude='*.log' --exclude='hammer/' "${SRC_DIR}/" "${DEST}/" \
|
||||
|| die "rsync failed"
|
||||
else
|
||||
cp -a -- "${SRC_DIR}/." "${DEST}/" || die "cp failed"
|
||||
find "${DEST}" -type f -name '*.log' -delete
|
||||
rm -rf -- "${DEST}/hammer"
|
||||
fi
|
||||
|
||||
# Tera Term's logopen target, in case the source tree never had one
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
; =============================================================================
|
||||
; Script A for Blanton
|
||||
; Version : V1.1.7
|
||||
; Date : 2026-09-04
|
||||
; Version : V1.2.2
|
||||
; Date : 2026-09-15
|
||||
; Author : ETWen
|
||||
; =============================================================================
|
||||
; Version History:
|
||||
@@ -341,6 +341,357 @@
|
||||
; sprintf2 cmd_ssd ... SSD_RUNTIME SSD_COUNT, and the same
|
||||
; for USB; the old hard-coded lines are left commented out
|
||||
; next to them
|
||||
; V1.1.8 2026-09-06
|
||||
; ScriptA Drive logopen's timestamp flag from EN_timestamp
|
||||
; logopen filename 0 1 1 EN_timestamp, with the argument
|
||||
; order spelled out above the call: <file> <binary>
|
||||
; <append> <PlainText> <timestamp>
|
||||
; binary must stay 0. In binary mode Tera Term writes the
|
||||
; raw stream and both PlainText and the timestamp are
|
||||
; ignored
|
||||
; !! EN_timestamp has to exist in config.ttl. An undefined
|
||||
; variable aborts the macro at logopen, it is not read
|
||||
; as 0
|
||||
; !! the timestamp FORMAT lives in TERATERM.INI (Setup >
|
||||
; Additional settings > Log), so it follows the PC and
|
||||
; not this script. Two testers can produce differently
|
||||
; stamped logs from one macro
|
||||
; ScriptB Run mlucas as a systemd unit, not a shell background job
|
||||
; systemd-run --unit=mlucas-amm --property=Type=exec
|
||||
; --property=Restart=always --property=RestartSec=2
|
||||
; The '&' job belonged to the interactive shell: it died
|
||||
; with the session, and 'jobs' could not see it from a
|
||||
; reconnected one, so a soak that lost its terminal quietly
|
||||
; stopped loading the CPU
|
||||
; The log is truncated with '>' before the unit starts and
|
||||
; appended with '>>' inside it, so a restart adds to the
|
||||
; run instead of erasing it
|
||||
; 'systemctl is-active mlucas-amm' now sits beside 'jobs',
|
||||
; both in the one-shot check and in the monitor loop at the
|
||||
; end
|
||||
; !! the old '&' line is kept commented out directly above
|
||||
; it, to fall back to if systemd-run is unavailable
|
||||
; ScriptB Take the BMC USB ping duration from BMC_USB_RUNTIME
|
||||
; sprintf2 cmd '... -w %d ...' ncm_if BMC_USB_RUNTIME. It
|
||||
; was hard-coded to 14400 (4 h), which no longer matched
|
||||
; the soak the SSD and USB knobs were set for
|
||||
; ScriptC Stop mlucas-amm and copy its log to the USB stick
|
||||
; utils/kill_all_process.ttl is only "kill $(jobs -p)", and
|
||||
; a systemd unit is not a shell job
|
||||
; ScriptB's change on its own would have left mlucas
|
||||
; loading all 16 cores through ScriptC and into the next
|
||||
; run, so every temperature and power reading after it
|
||||
; would have been taken under load
|
||||
; cp ~/hammer/tools/amd/mlucas_amm_log /host/hw-eval/jobs/
|
||||
; joins the other stress logs
|
||||
; !! ScriptB and ScriptC change together. Whoever moves a
|
||||
; stress job to systemd owns the matching stop in
|
||||
; ScriptC
|
||||
; Script4 Use the same mlucas-amm systemd unit as ScriptB
|
||||
; The thermal / safety macro started mlucas the same way
|
||||
; ScriptB used to, with '&', so it had the same failure:
|
||||
; the load quietly ended when the console went away, in the
|
||||
; one test where a long steady full load is the whole point
|
||||
; 'systemctl is-active mlucas-amm' joins 'bgctl list' and
|
||||
; 'jobs' in the monitor loop
|
||||
; !! Script4 still has no teardown section, so the matching
|
||||
; stop is manual: blanton_tr518.sh stop AND systemctl
|
||||
; stop mlucas-amm. Closing Tera Term no longer stops the
|
||||
; CPU load, and 'kill $(jobs -p)' never reached a unit
|
||||
; config.ttl Add EN_timestamp and BMC_USB_RUNTIME, raise stress runtimes
|
||||
; EN_timestamp = 1 feeds ScriptA's logopen; BMC_USB_RUNTIME
|
||||
; = 172800 feeds ScriptB's ping unit
|
||||
; SSD_RUNTIME and USB_RUNTIME 600 -> 3600, so the shipped
|
||||
; default is a one hour stress rather than the ten minutes
|
||||
; V1.1.7 went out with
|
||||
; !! --runtime is still the job's OVERALL limit. RUNTIME
|
||||
; 3600 = one hour of stress; COUNT does not multiply it.
|
||||
; For a 2 day soak set RUNTIME 172800
|
||||
; .gitignore Ignore src/Script_ABC_Blanton/hammer/
|
||||
; ~99 MB of unstripped mlucas ELF binaries (avx512 / avx2 /
|
||||
; avx / sse2 / generic-c) that ScriptB and the thermal
|
||||
; macro chmod and run on the DUT
|
||||
; !! they are deliberately NOT in git. A binary tree that
|
||||
; large is permanent once committed and every clone pays
|
||||
; for it forever; the DUT image already carries hammer/
|
||||
; publish/publish.sh Exclude hammer/ from the delivery bundle
|
||||
; rsync copies whatever sits in src/, so a developer who
|
||||
; happened to have the binaries locally would have shipped
|
||||
; a 100 MB zip while a clone shipped 200 KB. The bundle is
|
||||
; now the same either way
|
||||
; V1.1.9 2026-09-07
|
||||
; Blanton_Script/bcm_dsc_scan.sh Created V1.2.0 - batch SerDes DSC dump
|
||||
; Runs bcmcmd -n <unit> -c "dsh -c 'phydiag <port> dsc'"
|
||||
; over the whole port list (1 x 100G + 445 x 200G ranges),
|
||||
; both units, and leaves the raw log in <script_dir>/log
|
||||
; before copying it to /host/hw-eval/jobs
|
||||
; -f names the log (%u -> u0/u1, %t -> timestamp), -F
|
||||
; overwrites, -u picks one unit, -n dry runs, -C skips the
|
||||
; copy. -f takes a filename only; the directory comes from
|
||||
; -o
|
||||
; bcmcmd exits 0 even when the diag shell rejects a
|
||||
; command, so success is judged by counting 'SERDES DISPLAY
|
||||
; DIAG DATA END' markers, not by the exit code - the same
|
||||
; trap blanton_traffic_linespeed.sh hit
|
||||
; Paths come from $0, not $HOME, because Script A runs it
|
||||
; after sudo -i and a literal ~ would depend on the
|
||||
; environment. Ends with '### DSC_SCAN_DONE rc=<n>' for a
|
||||
; TTL wait to key on
|
||||
; Non-tty output is one line per port instead of a progress
|
||||
; bar, so it does not flood the Tera Term log
|
||||
; !! phydiag dsc is read-only and does not change any
|
||||
; SerDes setting, but a full scan is ~10 min per run.
|
||||
; ScriptA and ScriptC each do one, so the test is ~20
|
||||
; min longer than before. Do not set DELAY to 0
|
||||
; Blanton_Script/bcm_dsc_parse.sh Created V1.0.0 - offline DSC log to CSV
|
||||
; Host side. -i <raw log> -o <prefix> produces
|
||||
; <prefix>_lane.csv (one row per lane: TXEQ / RXFFE / DFE /
|
||||
; EYE / SNR) and <prefix>_pm.csv (one row per pm_id:
|
||||
; temperature / PLL / lane count / SNR-EYE stats)
|
||||
; POSIX sh + awk, mawk compatible, no Python. Lane rows are
|
||||
; read by the order of their bracket groups rather than
|
||||
; fixed columns, so re-spacing by the SDK does not break
|
||||
; it; a row whose group count is not 9 is marked UNPARSED
|
||||
; on stderr instead of being parsed wrongly
|
||||
; SD / LCK sticky '*' flags are kept as separate columns
|
||||
; and not interpreted; a BER field of '!chk_en' means the
|
||||
; checker was off and is passed through unchanged
|
||||
; ScriptA Collect the SerDes DSC baseline before 'show uptime'
|
||||
; ./Blanton_Script/bcm_dsc_scan.sh -f
|
||||
; blanton_dsc_ScriptA.log -F
|
||||
; The name has no %u, so the script inserts _u0 / _u1
|
||||
; itself and unit 1 cannot overwrite unit 0 - that is where
|
||||
; the file names ScriptC copies come from
|
||||
; ScriptA / ScriptB / ScriptC Remove the commented-out 100G PRBS blocks
|
||||
; The port_prbs_monitor.sh calls had been commented out in
|
||||
; all three since bring-up, together with a second copy of
|
||||
; the 100G port status block in ScriptA and the stale
|
||||
; bmc_monitor_ddr.sh block in ScriptC
|
||||
; port_prbs_monitor.sh still ships and still runs by hand;
|
||||
; only the dead call sites are gone
|
||||
; ScriptC Collect the closing DSC dump and carry the four logs out
|
||||
; ./Blanton_Script/bcm_dsc_scan.sh -f
|
||||
; blanton_dsc_ScriptC.log -F, then cp of
|
||||
; blanton_dsc_Script{A,C}_u{0,1}.log into
|
||||
; /host/hw-eval/jobs/
|
||||
; The scan already copies its own output there; these four
|
||||
; are a second pass so a failed copy inside the scan still
|
||||
; leaves the data on the USB stick
|
||||
; Blanton_Script Both new scripts need the execute bit on the DUT
|
||||
; !! git records them 100644 like every other .sh here, and
|
||||
; ScriptA / ScriptC invoke them as
|
||||
; ./Blanton_Script/bcm_dsc_scan.sh, so they join
|
||||
; bmc_monitor.sh and friends in the chmod +x line after
|
||||
; deployment. Without it both scans fail with Permission
|
||||
; denied and the macro walks straight past it
|
||||
; V1.2.0 2026-09-07
|
||||
; Blanton_Script/bcm_dsc_scan.sh v1.3.0 bcmcmd was eating the port list
|
||||
; The scan loop read the port file on stdin, and bcmcmd
|
||||
; reads stdin too, so the first call swallowed the rest of
|
||||
; the file and the loop ended after ONE port
|
||||
; Nothing looked wrong: the log held one PORT marker and
|
||||
; one dsc block, markers == dumps, exit code 0. A 445 port
|
||||
; scan finished in seconds and reported success
|
||||
; Fixed by reading the loop on fd 3 (done 3< "$PORTFILE")
|
||||
; and giving bcmcmd < /dev/null
|
||||
; Added a check that the iteration count equals the port
|
||||
; list count. The marker / dump comparison cannot catch a
|
||||
; loop that ends early, because both numbers shrink
|
||||
; together; the Saved: line now reports dumps out of TOTAL,
|
||||
; not out of markers
|
||||
; !! this is the same bcmcmd trap
|
||||
; blanton_traffic_linespeed.sh hit at v0.3.1, where only
|
||||
; the first VLAN was configured. Every while ... done <
|
||||
; file loop that calls bcmcmd needs < /dev/null - assume
|
||||
; it will read stdin unless proven otherwise
|
||||
; ScriptA / ScriptB / ScriptC Wait on the DUT with 'sleep', not on the macro with 'pause'
|
||||
; pause N stops the macro for N seconds without touching
|
||||
; the DUT: the shell sits idle and the macro is guessing
|
||||
; how long the last command took. sendln "sleep N" + wait
|
||||
; prompt_sonic_root moves the wait onto the DUT, so the
|
||||
; macro resumes when the prompt actually comes back and the
|
||||
; two cannot drift apart
|
||||
; It also lands in the console log with a timestamp, so a
|
||||
; round that ran long is visible afterwards instead of
|
||||
; invisible
|
||||
; Durations changed with it: ScriptA mgmt ping 30 -> 15 s;
|
||||
; the settle after every traffic 'clear' 15 -> 5 s; the
|
||||
; traffic run itself stays 15 s; ScriptB's monitor loop
|
||||
; stays 60 s
|
||||
; !! Script4 and Script5 were NOT converted and still use
|
||||
; pause. The 'pause 1' at the top of ScriptB / ScriptC
|
||||
; also stays - it only spaces the log header and runs
|
||||
; before there is a prompt to wait for
|
||||
; config.ttl Add EN_TX_FIR to gate the SerDes DSC scan
|
||||
; EN_TX_FIR = 1 enables bcm_dsc_scan.sh in ScriptA and
|
||||
; ScriptC, and the four cp lines that carry
|
||||
; blanton_dsc_Script{A,C}_u{0,1}.log to /host/hw-eval/jobs.
|
||||
; Default is 0
|
||||
; A full scan is ~10 min and both scripts do one, so
|
||||
; leaving it off keeps ~20 min out of a routine run. Turn
|
||||
; it on when the SerDes data is what you are after
|
||||
; ScriptC Stop printing the BMC poll and mlucas logs to the console
|
||||
; cat bmc_poll.log and cat mlucas_amm_log are commented
|
||||
; out. Both files are still copied to /host/hw-eval/jobs
|
||||
; and out to the USB stick with everything else, so no data
|
||||
; is lost - the master log is just shorter
|
||||
; V1.2.1 2026-09-08
|
||||
; ScriptA Add a BER baseline sweep before the traffic run
|
||||
; blanton_ber.sh init / start / sleep 30 / report / stop /
|
||||
; clear, branched on SWB_UNIT0 and SWB_UNIT1 the same way
|
||||
; the traffic block is. ScriptC already took the closing
|
||||
; sweep after its traffic report, so the two now bracket
|
||||
; the soak the way the DSC scans do
|
||||
; sleep 30 is not arbitrary: it is blanton_ber.sh's
|
||||
; BER_INTERVAL. 'report' before one full interval has
|
||||
; accumulated reads a window that has not finished
|
||||
; The block was copied from ScriptC, which runs it AFTER
|
||||
; traffic, and its comment said so. In ScriptA it runs
|
||||
; BEFORE traffic, so the comment was rewritten to match
|
||||
; where the code actually is
|
||||
; !! nothing settles between 'blanton_ber.sh clear' and
|
||||
; 'blanton_traffic_linespeed ps' that follows it. PRBS
|
||||
; holds the lanes out of normal operation and 'clear' is
|
||||
; what releases them, so the first traffic run of
|
||||
; ScriptA now starts on lanes that have just been
|
||||
; released. ScriptC does not have this ordering. Watch
|
||||
; ScriptA's first traffic report for NA or FAIL pairs
|
||||
; that ScriptC's does not show - if they appear, a sleep
|
||||
; after 'clear' is the fix
|
||||
; !! ScriptA now runs BER twice as far as the switch is
|
||||
; concerned: once here and once inside ScriptC. Neither
|
||||
; can overlap blanton_traffic_linespeed or
|
||||
; blanton_tr518.sh - they all take over the same ports
|
||||
; V1.2.2 2026-09-15
|
||||
; 5_.._Boot_Cycle.ttl Renamed from 5_Blanton_Script_Reboot_Cycle.ttl
|
||||
; It does warm and cold now, so 'Reboot' was too narrow.
|
||||
; Same macro otherwise - git records it as a rename
|
||||
; 5_.._Boot_Cycle.ttl Warm or cold boot, chosen by EN_Cold_Warm_boot
|
||||
; 1 = 'reboot' (warm, what the macro always did). 2 = cold,
|
||||
; by writing the CB FPGA: pcimem
|
||||
; /sys/bus/pci/devices/0000:05:00.0/resource0 0x480 w
|
||||
; 0xD0000. 0 = neither
|
||||
; !! EN_Cold_Warm_boot = 0 hangs the macro. Neither branch
|
||||
; runs, so nothing reboots, and the 'wait "Booting in
|
||||
; blind mode"' underneath has no timeout - cycle 1
|
||||
; finishes and the macro then waits for a boot that will
|
||||
; never come. 0 is the value for the A/B/C flow, where
|
||||
; this macro is not used at all; do not run ScriptBoot
|
||||
; with it
|
||||
; !! 0x480 is ICB_CTRL_REG in
|
||||
; docs/Blantons_FPGA_Registers.md, and that table
|
||||
; carries no bit definitions for it. 0xD0000 is not
|
||||
; written down anywhere in this repo either. Whoever
|
||||
; knows what those bits do should put it in the register
|
||||
; doc before someone changes the value
|
||||
; !! the BDF 0000:05:00.0 is hard-coded here, while
|
||||
; blanton_fpga_pcimem.sh auto-detects the CB FPGA
|
||||
; precisely because it moves per unit (see the V1.0.x
|
||||
; notes and Key Constraints #4). On a DUT where the FPGA
|
||||
; is not at 05:00.0 this becomes a raw 32-bit write to
|
||||
; an unknown register on some other device. Verify the
|
||||
; BDF on every unit before running the cold path
|
||||
; !! NOT verified on hardware. The warm path has 199 cycles
|
||||
; behind it; the cold path has none
|
||||
; 5_.._Boot_Cycle.ttl Truncate the two stress logs before each run
|
||||
; rm -rf /host/hw-eval/jobs/qfx5252-stress-ssd.log and the
|
||||
; usb one, added ahead of each qfx5252-stress-* call
|
||||
; qfx5252-stress-* APPENDS to its --log file and the macro
|
||||
; cats the whole thing afterwards, so every cycle printed
|
||||
; back everything the ones before it had written
|
||||
; Measured on the 199 cycle log from 2026-09-04: cycle 1
|
||||
; printed 289 lines, cycle 199 printed 36,555. At ~6.2 ms a
|
||||
; line - which is what a 115200 baud console costs - that
|
||||
; is +212 s of the +214 s the cycle grew over the run. The
|
||||
; stress itself is fixed at --runtime 30, and the other
|
||||
; eleven segments between check points drifted by less than
|
||||
; 1.5 s each. The DUT was never getting slower; the test
|
||||
; was reading its own history aloud once per cycle
|
||||
; 5_.._Boot_Cycle.ttl Name the log file and the banner after the kind of boot
|
||||
; EN_Cold_Warm_boot = 1 gives
|
||||
; Logs\Blanton_Reboot_Cycle_<ts>.log and a 'Reboot Cycle :
|
||||
; N' banner; = 2 gives Blanton_Power_Cycle_<ts>.log and
|
||||
; 'Power Cycle : N'. Three sites now branch on the flag:
|
||||
; the log name, the banner and the boot itself
|
||||
; !! with 0 there is no longer a logopen either, so the run
|
||||
; that already hangs at wait "Booting in blind mode" now
|
||||
; also leaves no log to show where it stopped. EN_log =
|
||||
; 1 no longer guarantees a log in this macro -
|
||||
; EN_Cold_Warm_boot has to be 1 or 2 as well
|
||||
; !! anything that parses these logs has to accept BOTH
|
||||
; banners. The cycle banner is the record separator the
|
||||
; check-point and per-segment timing reports are built
|
||||
; on, so a parser that only greps 'Reboot Cycle :' finds
|
||||
; zero cycles in a power-cycle log and silently reports
|
||||
; nothing at all
|
||||
; The logopen timestamp flag is still a literal 1 here
|
||||
; rather than EN_timestamp the way ScriptA does it. Leaving
|
||||
; it on is the right call - the per-segment timing analysis
|
||||
; is derived from those timestamps and there is nothing
|
||||
; else to recover them from - but it does mean this macro
|
||||
; ignores EN_timestamp = 0
|
||||
; config.ttl Add EN_Cold_Warm_boot
|
||||
; 0 = normal test (A/B/C), 1 = reboot cycle, 2 = power
|
||||
; cycle. Default 0 so that opening the shared config cannot
|
||||
; reboot anyone's DUT by accident
|
||||
; Blanton_Script/blanton_ber.sh v1.1.0 Add port 268, the 100G uplink
|
||||
; The port list jumps 256-259 -> 270-308, so 268 (ce0) was
|
||||
; never measured. It is now its own group: BER_PORTS_200G
|
||||
; and BER_PORTS_100G, joined for the phydiag command but
|
||||
; reported apart
|
||||
; The summary gives each unit three rows - 200G ports /
|
||||
; 100G p268 / TOTAL - because unit 0 and unit 1 each have
|
||||
; their own ce0(268). A bad uplink lane averaged into 432
|
||||
; others is invisible; on its own row it is not
|
||||
; TOTAL is not a hand-added sum: every lane is folded into
|
||||
; its own group and into ALL separately, so the parts
|
||||
; cannot drift from the whole
|
||||
; !! a group that reports no lane at all is a FAILURE, not
|
||||
; a pass. Silence on the port under test is the easiest
|
||||
; way to ship a green report that measured nothing
|
||||
; Blanton_Script/blanton_ber.sh v1.1.1 Stop reading lane results as shell errors
|
||||
; init kept answering "the BCM shell rejected 'phydiag ...
|
||||
; prbs get'" on runs that were working. The check grepped
|
||||
; the WHOLE output case-insensitively for
|
||||
; ERROR|Invalid|..., and 'prbs get' prints one line per
|
||||
; lane - a lane that had not locked says PRBS Error!, which
|
||||
; matched
|
||||
; So a good run reported a hard error, and worse, the
|
||||
; function returned before the lock-count check that was
|
||||
; supposed to judge those very lanes. The operator saw a
|
||||
; failure and did NOT see which lanes were unlocked
|
||||
; Per-lane lines are now excluded from the check, the match
|
||||
; is case-sensitive, and a genuine rejection prints the
|
||||
; offending line so it can be read
|
||||
; !! this was in the file from v1.0.0. Any earlier init
|
||||
; that printed that error was probably fine, and any
|
||||
; earlier init that passed said nothing about lanes that
|
||||
; had failed to lock
|
||||
; Blanton_Script/bcm_100g_dsc_cycle.sh Created v1.2.0 - 100G link down/up cycle with DSC capture
|
||||
; Tuning once (gbsyncd pause, serdes-tx-set sys 0..7,
|
||||
; tx_taps per lane), then N cycles of: port en=0, wait,
|
||||
; port en=1, wait, phydiag dsc on both units, serdes-tx-get
|
||||
; and serdes-rx-get sys 0..7. One log per run under
|
||||
; ~/dsc_log, split by '### TEST CYCLE n/N'
|
||||
; Waiting polls 'bcmcmd ps <port>' every -p seconds up to
|
||||
; -w, and reads the state from the row whose first field
|
||||
; ends in (<port>). Asking the SDK matters here: 'port en='
|
||||
; changes the SDK, while 'show interfaces status' only
|
||||
; follows once gbsyncd has synced it - and this test pauses
|
||||
; gbsyncd
|
||||
; -d prints every command as [CMD] in grey before running
|
||||
; it, CMD_DELAY paces them, -n is a dry run that touches
|
||||
; nothing and sleeps not at all
|
||||
; !! all output is ASCII English on purpose. The DUT serial
|
||||
; console mangles UTF-8: a Chinese error message came
|
||||
; back as "ERROR: -c ? 1", which is worse than no
|
||||
; message
|
||||
; !! NOT verified on hardware
|
||||
; tools/100G_dsc_cycle.txt Created - the hand-typed command list behind that script
|
||||
; Same role as BER_Test.txt and TR518.txt: the bench
|
||||
; version of the commands, kept so the .sh can be checked
|
||||
; against what a person would actually type. Change one and
|
||||
; change the other
|
||||
; =============================================================================
|
||||
include "config.ttl"
|
||||
|
||||
@@ -353,7 +704,11 @@ getdir mdir
|
||||
|
||||
if EN_log = 1 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name strTestcase str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
; logopen <file> <binary> <append> <PlainText> <timestamp>
|
||||
; binary=0 text mode, so PlainText / timestamp are honoured
|
||||
; append=1 keep earlier cycles, do not overwrite
|
||||
; PlainText=1 strip the ANSI colour codes SONiC emits
|
||||
logopen filename 0 1 1 EN_timestamp
|
||||
endif
|
||||
|
||||
; ========== Root Login ==========
|
||||
@@ -502,7 +857,8 @@ include "utils/show_dmesg.ttl"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh start"
|
||||
wait prompt_sonic_root
|
||||
pause 30
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/mgmt_ping_monitor.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ~/Blanton_Script/log/mgmt_ping.log"
|
||||
@@ -537,36 +893,6 @@ sendln "config save -y"
|
||||
wait prompt_sonic_root
|
||||
sendln "show interfaces status Ethernet513,Ethernet514"
|
||||
|
||||
;=============================================================================================================
|
||||
; ========== 100G PRBS ==========
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh start"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "sleep 10"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh status"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh stop"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh report"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh clear"
|
||||
|
||||
; ========== 100G Port Status ==========
|
||||
;wait prompt_sonic_root
|
||||
;sendln "sudo sfputil lpmode off Ethernet513"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "sudo sfputil lpmode off Ethernet514"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "config interface -n asic0 startup Ethernet513"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "config interface -n asic1 startup Ethernet514"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "config save -y"
|
||||
|
||||
;wait prompt_sonic_root
|
||||
;sendln "show interfaces status Ethernet513,Ethernet514"
|
||||
;=============================================================================================================
|
||||
|
||||
; ========== First Traffic Test ==========
|
||||
; Silent MAC
|
||||
@@ -575,6 +901,56 @@ sendln "config feature state lldp disabled"
|
||||
wait prompt_sonic_root
|
||||
sendln "config save -y"
|
||||
|
||||
; ========== BER ==========
|
||||
; Baseline sweep, taken BEFORE the traffic run - ScriptC takes the closing
|
||||
; one after its traffic report, so the pair brackets the soak.
|
||||
; PRBS takes the SerDes out of normal operation; 'clear' is what puts the
|
||||
; lanes back, so it has to run before the traffic block below.
|
||||
; sleep 30 matches blanton_ber.sh BER_INTERVAL - reporting earlier reads an
|
||||
; interval that has not finished accumulating.
|
||||
if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear"
|
||||
elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear -u 0"
|
||||
elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh init -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh start -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 30"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh report -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh stop -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/blanton_ber.sh clear -u 1"
|
||||
else
|
||||
|
||||
endif
|
||||
|
||||
if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed ps"
|
||||
@@ -583,12 +959,14 @@ if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report"
|
||||
@@ -600,12 +978,14 @@ elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report -u 0"
|
||||
@@ -617,12 +997,14 @@ elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 15"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed stop -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed report -u 1"
|
||||
@@ -630,6 +1012,12 @@ else
|
||||
|
||||
endif
|
||||
|
||||
if EN_TX_FIR = 1 then
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptA.log -F"
|
||||
endif
|
||||
|
||||
; ========== Show Power on uptime ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "show uptime"
|
||||
|
||||
@@ -15,10 +15,15 @@ pause 1
|
||||
;sendln "command"
|
||||
|
||||
; ========== AMD CPU STRESS TEST ==========
|
||||
; Clear the log file first
|
||||
wait prompt_sonic_root
|
||||
sendln "> ~/hammer/tools/amd/mlucas_amm_log"
|
||||
wait prompt_sonic_root
|
||||
sendln 'chmod +x ~/hammer/tools/amd/mlucas-avx2'
|
||||
wait prompt_sonic_root
|
||||
sendln "~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 > ~/hammer/tools/amd/mlucas_amm_log 2>&1 &"
|
||||
;sendln "~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 > ~/hammer/tools/amd/mlucas_amm_log 2>&1 &"
|
||||
sendln "systemd-run --unit=mlucas-amm --property=Type=exec --property=Restart=always --property=RestartSec=2 /bin/bash -c '~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 >> ~/hammer/tools/amd/mlucas_amm_log 2>&1'"
|
||||
|
||||
|
||||
; ========== BMC DDR Stress Test ==========
|
||||
;wait prompt_sonic_root
|
||||
@@ -65,6 +70,8 @@ sendln cmd
|
||||
wait prompt_sonic_root
|
||||
sendln "jobs"
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl is-active mlucas-amm"
|
||||
wait prompt_sonic_root
|
||||
sendln "bgctl list"
|
||||
|
||||
; ========== BMC USB Test ==========
|
||||
@@ -89,7 +96,7 @@ wait prompt_sonic_root
|
||||
sendln 'systemctl stop qfx5252-bmc-usb-net-test.service 2>/dev/null || true'
|
||||
wait prompt_sonic_root
|
||||
sendln 'systemctl reset-failed qfx5252-bmc-usb-net-test.service 2>/dev/null || true'
|
||||
sprintf2 cmd 'systemd-run --unit=qfx5252-bmc-usb-net-test --property=Type=exec /usr/bin/ping -I %s -i 1 -W 2 -w 14400 192.168.200.200' ncm_if
|
||||
sprintf2 cmd 'systemd-run --unit=qfx5252-bmc-usb-net-test --property=Type=exec /usr/bin/ping -I %s -i 1 -W 2 -w %d 192.168.200.200' ncm_if BMC_USB_RUNTIME
|
||||
wait prompt_sonic_root
|
||||
sendln cmd
|
||||
|
||||
@@ -102,7 +109,8 @@ timeout = 0
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh start"
|
||||
wait prompt_sonic_root
|
||||
pause 3
|
||||
sendln "sleep 3"
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/mgmt_ping_monitor.sh status"
|
||||
|
||||
; ========== NFC Test ==========
|
||||
@@ -116,12 +124,6 @@ sendln "~/Blanton_Script/nfc_polling.sh status"
|
||||
wait prompt_sonic_root
|
||||
sendln "show interfaces status Ethernet513,Ethernet514"
|
||||
|
||||
;=============================================================================================================
|
||||
; ========== 100G PRBS ==========
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh start"
|
||||
;=============================================================================================================
|
||||
|
||||
|
||||
; ========== Traffic START ==========
|
||||
; Silent MAC
|
||||
@@ -138,7 +140,8 @@ if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start"
|
||||
@@ -150,7 +153,8 @@ elseif SWB_UNIT0 = 1 && SWB_UNIT1 = 0 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 0"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 0"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 0"
|
||||
@@ -162,7 +166,8 @@ elseif SWB_UNIT0 = 0 && SWB_UNIT1 = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed clear -u 1"
|
||||
wait prompt_sonic_root
|
||||
pause 15
|
||||
sendln "sleep 5"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed show -u 1"
|
||||
wait prompt_sonic_root
|
||||
sendln "blanton_traffic_linespeed start -u 1"
|
||||
@@ -172,12 +177,15 @@ endif
|
||||
|
||||
|
||||
|
||||
; ========== Monitor loop: platform data + background jobs (pause 60s per round) ==========
|
||||
; ========== Monitor loop: platform data + background jobs (sleep 60s per round) ==========
|
||||
while 1
|
||||
include "utils/setup_pmon.ttl"
|
||||
wait prompt_sonic_root
|
||||
sendln "bgctl list"
|
||||
wait prompt_sonic_root
|
||||
sendln "jobs"
|
||||
pause 60
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl is-active mlucas-amm"
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 60"
|
||||
endwhile
|
||||
|
||||
@@ -12,6 +12,11 @@ pause 1
|
||||
|
||||
; ========== Kill Process ==========
|
||||
include "utils/kill_all_process.ttl"
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl is-active mlucas-amm"
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl stop mlucas-amm"
|
||||
|
||||
; bgctrl all stop
|
||||
wait prompt_sonic_root
|
||||
sendln "bgctl stop --all"
|
||||
@@ -28,23 +33,6 @@ sendln "~/Blanton_Script/mgmt_ping_monitor.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "~/Blanton_Script/nfc_polling.sh stop"
|
||||
|
||||
;=============================================================================================================
|
||||
; 100G PRBS STOP
|
||||
;wait prompt_sonic_root
|
||||
;sendln "~/Blanton_Script/port_prbs_monitor.sh stop"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "sudo sfputil lpmode off Ethernet513"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "sudo sfputil lpmode off Ethernet514"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "config interface -n asic0 startup Ethernet513"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "config interface -n asic1 startup Ethernet514"
|
||||
;=============================================================================================================
|
||||
|
||||
|
||||
|
||||
|
||||
; ========== TAKE DATA ==========
|
||||
include "utils/setup_pmon.ttl"
|
||||
include "utils/show_dmesg.ttl"
|
||||
@@ -61,21 +49,12 @@ include "utils/show_pcie_error_reg_SSD.ttl"
|
||||
wait prompt_sonic_root
|
||||
sendln "/usr/sbin/ras-mc-ctl --summary"
|
||||
|
||||
; ========== BMC DDR TAKE DATA ==========
|
||||
;wait prompt_sonic_root
|
||||
;sendln "./Blanton_Script/bmc_monitor_ddr.sh stop"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "./Blanton_Script/bmc_monitor.sh stop"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "cat ./Blanton_Script/log/bmc_poll.log"
|
||||
;wait prompt_sonic_root
|
||||
;sendln "cat ./Blanton_Script/log/bmc_ddr.log"
|
||||
|
||||
; ========== BMC Monitor TAKE DATA ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bmc_monitor.sh stop"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ./Blanton_Script/log/bmc_poll.log"
|
||||
; wait prompt_sonic_root
|
||||
; sendln "cat ./Blanton_Script/log/bmc_poll.log"
|
||||
|
||||
; ========== BMC USB Test result ==========
|
||||
wait prompt_sonic_root
|
||||
@@ -85,8 +64,8 @@ sendln "journalctl -u qfx5252-bmc-usb-net-test.service --no-pager -o short-iso >
|
||||
|
||||
|
||||
; ========== CHECK Stress results ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "cat ~/hammer/tools/amd/mlucas_amm_log"
|
||||
; wait prompt_sonic_root
|
||||
; sendln "cat ~/hammer/tools/amd/mlucas_amm_log"
|
||||
wait prompt_sonic_root
|
||||
sendln "cat /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
@@ -103,10 +82,6 @@ sendln "~/Blanton_Script/nfc_polling.sh report"
|
||||
; ========== 100G Port ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "show interfaces status Ethernet513,Ethernet514"
|
||||
;=============================================================================================================
|
||||
;wait prompt_sonic_root
|
||||
;sendln "cat ~/Blanton_Script/log/port_prbs.log "
|
||||
;=============================================================================================================
|
||||
|
||||
; ========== CHECK Traffic counters ==========
|
||||
if SWB_UNIT0 = 1 && SWB_UNIT1 = 1 then
|
||||
@@ -175,6 +150,11 @@ else
|
||||
|
||||
endif
|
||||
|
||||
if EN_TX_FIR = 1 then
|
||||
; ========== Collect bcm dsc data ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "./Blanton_Script/bcm_dsc_scan.sh -f blanton_dsc_ScriptC.log -F"
|
||||
endif
|
||||
|
||||
; ========== NVME Err Info ==========
|
||||
wait prompt_sonic_root
|
||||
@@ -211,6 +191,8 @@ sendln cmd
|
||||
|
||||
; ========== Copy Stress Log to USB ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ~/hammer/tools/amd/mlucas_amm_log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/bmc_usb_net.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/bmc_poll.log /host/hw-eval/jobs/"
|
||||
@@ -219,7 +201,16 @@ sendln "cp ./Blanton_Script/log/mgmt_ping.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/nfc_poll.log /host/hw-eval/jobs/"
|
||||
|
||||
|
||||
if EN_TX_FIR = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptA_u1.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u0.log /host/hw-eval/jobs/"
|
||||
wait prompt_sonic_root
|
||||
sendln "cp ./Blanton_Script/log/blanton_dsc_ScriptC_u1.log /host/hw-eval/jobs/"
|
||||
endif
|
||||
|
||||
wait prompt_sonic_root
|
||||
getdate ts_date "%Y%m%d"
|
||||
|
||||
@@ -129,11 +129,15 @@ include "utils/setup_pmon.ttl"
|
||||
include "utils/show_dmesg.ttl"
|
||||
|
||||
; ========== Full Load ==========
|
||||
; AMD CPU STRESS TEST
|
||||
; AMD CPU STRESS TEST
|
||||
; Clear the log file first
|
||||
wait prompt_sonic_root
|
||||
sendln "> ~/hammer/tools/amd/mlucas_amm_log"
|
||||
wait prompt_sonic_root
|
||||
sendln 'chmod +x ~/hammer/tools/amd/mlucas-avx2'
|
||||
wait prompt_sonic_root
|
||||
sendln "~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 > ~/hammer/tools/amd/mlucas_amm_log 2>&1 &"
|
||||
;sendln "~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 > ~/hammer/tools/amd/mlucas_amm_log 2>&1 &"
|
||||
sendln "systemd-run --unit=mlucas-amm --property=Type=exec --property=Restart=always --property=RestartSec=2 /bin/bash -c '~/hammer/tools/amd/mlucas-avx2 -s l -cpu 0:15 >> ~/hammer/tools/amd/mlucas_amm_log 2>&1'"
|
||||
|
||||
; TR518 TEST
|
||||
wait prompt_sonic_root
|
||||
@@ -154,5 +158,7 @@ while 1
|
||||
sendln "bgctl list"
|
||||
wait prompt_sonic_root
|
||||
sendln "jobs"
|
||||
wait prompt_sonic_root
|
||||
sendln "systemctl is-active mlucas-amm"
|
||||
pause 60
|
||||
endwhile
|
||||
|
||||
+31
-7
@@ -8,8 +8,14 @@ gettime str_starttime "%Y%m%d-%H%M%S"
|
||||
getdir mdir
|
||||
|
||||
if EN_log = 1 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Reboot_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Reboot_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
sprintf2 filename '%s\Logs\%s_%s_%s.log' mdir project_name "Power_Cycle" str_starttime
|
||||
logopen filename 0 1 1 1
|
||||
else
|
||||
endif
|
||||
endif
|
||||
|
||||
cnt_cycle = 0
|
||||
@@ -26,9 +32,16 @@ while 1
|
||||
wait prompt_sonic_root
|
||||
sendln "sleep 10"
|
||||
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mReboot Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mReboot Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
wait prompt_sonic_root
|
||||
sprintf2 strTitle "echo -e '\033[1;32mPower Cycle : %d \033[0m'" cnt_cycle
|
||||
sendln strTitle
|
||||
else
|
||||
endif
|
||||
|
||||
; ========== Pre Setting ==========
|
||||
; Make log directory
|
||||
@@ -172,6 +185,8 @@ while 1
|
||||
sprintf2 strCKPt5 "echo -e '\033[36mCheck Pt5 : SSD Test\033[0m'"
|
||||
sendln strCKPt5
|
||||
|
||||
wait prompt_sonic_root
|
||||
sendln "rm -rf /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
sendln "qfx5252-stress-ssd 1G --runtime 30 --passes 2 --write --force --log /host/hw-eval/jobs/qfx5252-stress-ssd.log"
|
||||
wait prompt_sonic_root
|
||||
@@ -189,6 +204,8 @@ while 1
|
||||
waitregex '<<DEV=([^>]*)>>'
|
||||
usb_dev = groupmatchstr1
|
||||
|
||||
wait prompt_sonic_root
|
||||
sendln "rm -rf /host/hw-eval/jobs/qfx5252-stress-usb.log"
|
||||
wait prompt_sonic_root
|
||||
sprintf2 cmd "qfx5252-stress-usb %s 256M --runtime 30 --passes 2 --write --log /host/hw-eval/jobs/qfx5252-stress-usb.log" usb_dev
|
||||
sendln cmd
|
||||
@@ -412,8 +429,15 @@ while 1
|
||||
sendln "show uptime"
|
||||
|
||||
; ========== Show Power on uptime ==========
|
||||
wait prompt_sonic_root
|
||||
sendln "reboot"
|
||||
if EN_Cold_Warm_boot = 1 then
|
||||
wait prompt_sonic_root
|
||||
sendln "reboot"
|
||||
elseif EN_Cold_Warm_boot = 2 then
|
||||
wait prompt_sonic_root
|
||||
sendln "pcimem /sys/bus/pci/devices/0000:05:00.0/resource0 0x480 w 0xD0000"
|
||||
else
|
||||
endif
|
||||
|
||||
|
||||
wait "Booting in blind mode"
|
||||
endwhile
|
||||
@@ -0,0 +1,608 @@
|
||||
#!/bin/sh
|
||||
# =============================================================================
|
||||
# bcm_100g_dsc_cycle.sh
|
||||
#
|
||||
# Author : ETWen
|
||||
# Date : 20260909
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260909 Initial Version
|
||||
# V1.1.0 20260909 All runtime output is ASCII English - the DUT serial
|
||||
# console mangles UTF-8 (a Chinese error message came
|
||||
# back as "ERROR: -c ? 1"). Cycle count may be given as
|
||||
# a bare argument. Added -d to echo every command, and
|
||||
# CMD_DELAY to pace them.
|
||||
# V1.2.0 20260909 Wait on "bcmcmd ps <port>" instead of
|
||||
# "show interfaces status". The state comes from the
|
||||
# SDK, which is what "port en=" actually changes, so it
|
||||
# no longer depends on gbsyncd syncing into the DB.
|
||||
#
|
||||
# Description:
|
||||
# Link down/up cycling on the 100G uplink. After every re-link it collects a
|
||||
# SerDes DSC dump and reads the retimer back, so you can see whether the
|
||||
# tuned TX taps settle to the same place each time the link comes up.
|
||||
#
|
||||
# One run has two parts:
|
||||
# 1. Tuning -- once. pause gbsyncd -> serdes-tx-set sys 0..7
|
||||
# -> tx_taps set on four lanes of each unit
|
||||
# 2. Cycle -- repeated N times:
|
||||
# port en=0 -> wait until "ps <port>" reads !ena on both units
|
||||
# port en=1 -> wait until "ps <port>" reads up on both units
|
||||
# phydiag <port> dsc on each unit
|
||||
# serdes-tx-get / serdes-rx-get sys 0..7
|
||||
# all of it appended to one log, under a per-cycle heading
|
||||
#
|
||||
# Command source: tools/100G_dsc_cycle.txt (the hand-typed bench version).
|
||||
# Change that file and this one together.
|
||||
#
|
||||
# Deployment:
|
||||
# Script : /root/Blanton_Script/bcm_100g_dsc_cycle.sh
|
||||
# Log : ~/dsc_log/100G_dsc_cycle_test_<YYYYmmdd_HHMMSS>.log
|
||||
# One file per run, every cycle in it, split by "### TEST CYCLE n/N".
|
||||
#
|
||||
# Paths come from $HOME. Run as root (after sudo -i) and ~ is /root.
|
||||
#
|
||||
# Usage:
|
||||
# ./bcm_100g_dsc_cycle.sh # print this help
|
||||
# ./bcm_100g_dsc_cycle.sh 200 # 200 cycles
|
||||
# ./bcm_100g_dsc_cycle.sh -c 200 # same thing
|
||||
# ./bcm_100g_dsc_cycle.sh -c 200 -d # echo every command as it is issued
|
||||
# ./bcm_100g_dsc_cycle.sh -c 10 -T # skip tuning, cycle only
|
||||
# ./bcm_100g_dsc_cycle.sh -c 1 -n # dry run, print commands, touch nothing
|
||||
#
|
||||
# Exit code:
|
||||
# 0 = every cycle linked down and up as expected, and all data was collected
|
||||
# 1 = a cycle timed out waiting for link, or data was missing
|
||||
# 2 = bad argument / path
|
||||
# 3 = a required tool is missing (bcmcmd / show / blanton_barchetta_tool)
|
||||
#
|
||||
# The last line is always "### DSC_CYCLE_DONE rc=<n>", for a TTL wait to
|
||||
# key on.
|
||||
#
|
||||
# Design notes:
|
||||
# - POSIX sh. No bash, no Python.
|
||||
# - All runtime output is plain ASCII. The DUT console is not UTF-8 clean:
|
||||
# Chinese comes back as "M-^A"-style garbage, and an error you cannot read
|
||||
# is worse than no error at all. Comments are English for the same reason -
|
||||
# whoever debugs this on the bench is reading it through that console.
|
||||
# - Link state comes from "bcmcmd -n <unit> -c 'ps <port>'". Its header is
|
||||
# two stacked lines, so a column index cannot be derived from it the way
|
||||
# it can from a SONiC table. Instead the data row is found by its first
|
||||
# field ending in "(<port>)" - e.g. ce0(268) - and the state is the field
|
||||
# after it:
|
||||
# ce0(268) !ena 4 100G FD ... <- port disabled
|
||||
# ce0(268) up 4 100G FD ... <- link up
|
||||
# "down" is therefore tested as "not up", which covers !ena as well as an
|
||||
# enabled port whose link has not come back.
|
||||
# - Asking the SDK rather than "show interfaces status" matters here: "port
|
||||
# en=" changes the SDK, and the SONiC table only follows once gbsyncd and
|
||||
# orchagent have synced it into the DB. Since this test pauses gbsyncd,
|
||||
# the SONiC table was the wrong thing to wait on.
|
||||
# - bcmcmd reads stdin, and inside a while loop it will swallow the loop's
|
||||
# input. blanton_traffic_linespeed.sh hit this at v0.3.1 (only the first
|
||||
# VLAN was configured) and bcm_dsc_scan.sh hit it again at v1.3.0 (only
|
||||
# the first port was scanned). Every external command here gets
|
||||
# < /dev/null.
|
||||
# - bcmcmd exits 0 even when the diag shell rejects a command, so the DSC is
|
||||
# verified by counting "SERDES DISPLAY DIAG DATA END", and the retimer read
|
||||
# by counting sections that actually have output under them - not by exit
|
||||
# code, and not by counting headings this script prints itself.
|
||||
#
|
||||
# Cautions:
|
||||
# - !! While gbsyncd is paused the gearbox state stops being synced into the
|
||||
# DB. The wait reads the SDK directly ("ps <port>") so it is not affected,
|
||||
# but anything you check by hand with "show interfaces status" during the
|
||||
# run may be stale. Set PAUSE_GBSYNCD=0 (or -P) if that gets in the way.
|
||||
# - !! On exit, including Ctrl-C, both gbsyncd containers are unpaused again
|
||||
# so nothing is left paused. Set RESUME_GBSYNCD=0 to keep them paused.
|
||||
# - !! phydiag dsc is read-only, but port en=0/1 and tx_taps set do change
|
||||
# the hardware. Do not run this at the same time as blanton_ber.sh,
|
||||
# blanton_traffic_linespeed or blanton_tr518.sh - they take the same ports.
|
||||
# - !! NOT verified on hardware yet.
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Knobs - change these for a different DUT or port, not the flow below
|
||||
# ----------------------------------------------------------------------------
|
||||
PORT=268 # bcmcmd port number, not a port name
|
||||
UNITS="0 1" # units to drive
|
||||
SYS_LANES="0 1 2 3 4 5 6 7" # blanton_barchetta_tool sys lanes, tuning
|
||||
SYS_TX="pre=-10 main=87 post=0" # TX setting applied to each of them
|
||||
|
||||
# tx_taps, one line per lane: "<unit> <lane> <main> <pre1> <post1>"
|
||||
TX_TAPS="
|
||||
0 0 31 4 13
|
||||
0 1 31 4 13
|
||||
0 2 27 4 13
|
||||
0 3 29 4 13
|
||||
1 0 29 4 13
|
||||
1 1 30 4 13
|
||||
1 2 31 4 13
|
||||
1 3 30 4 13
|
||||
"
|
||||
|
||||
TX_LANES="0 1 2 3 4 5 6 7" # retimer TX lanes read back each cycle
|
||||
RX_LANES="0 1 2 3 4 5 6 7" # retimer RX lanes read back each cycle
|
||||
|
||||
SUDO="" # set to "sudo" if not running as root
|
||||
PAUSE_GBSYNCD=1 # pause gbsyncd0/1 before tuning
|
||||
RESUME_GBSYNCD=1 # unpause again on exit, Ctrl-C included
|
||||
GBSYNCD="gbsyncd0 gbsyncd1"
|
||||
|
||||
LINK_WAIT=90 # give up waiting for a link state after this
|
||||
POLL=2 # seconds between "ps <port>" reads while waiting
|
||||
SETTLE=2 # quiet time after link up, before collecting
|
||||
CMD_DELAY=0.2 # pause after every command issued to the DUT
|
||||
|
||||
OUTDIR="${HOME:-/root}/dsc_log"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
CYCLES=0
|
||||
DRYRUN=0
|
||||
DOTUNE=1
|
||||
DEBUG=0
|
||||
RC=0
|
||||
|
||||
usage() {
|
||||
cat >&2 << 'USAGE'
|
||||
Usage: bcm_100g_dsc_cycle.sh <cycles> [options]
|
||||
bcm_100g_dsc_cycle.sh -c <cycles> [options]
|
||||
|
||||
-c <n> how many cycles to run. May also be given as a bare argument.
|
||||
-o <dir> log directory (default: ~/dsc_log)
|
||||
-w <sec> how long to wait for a link state (default: 90)
|
||||
-p <sec> how often to re-read the port state (default: 2)
|
||||
-s <sec> quiet time after link up, before collecting (default: 2)
|
||||
-d debug: echo every command as [CMD] before running it
|
||||
-T skip tuning, go straight to the cycles
|
||||
-P leave gbsyncd alone (no pause, no unpause)
|
||||
-n dry run: print the commands, touch nothing
|
||||
-h this help
|
||||
|
||||
One log per run:
|
||||
~/dsc_log/100G_dsc_cycle_test_<YYYYmmdd_HHMMSS>.log
|
||||
|
||||
Each cycle is one "### TEST CYCLE n/N" section holding, in order: the "ps"
|
||||
output showing the port down, the same showing it up, phydiag dsc for both
|
||||
units, and the retimer serdes-tx-get / serdes-rx-get for sys 0..7.
|
||||
|
||||
Waiting for a port state is a poll, not a single read: "bcmcmd ps <port>" on
|
||||
every unit, every -p seconds, until both agree or -w seconds have passed.
|
||||
|
||||
Examples:
|
||||
./bcm_100g_dsc_cycle.sh 200
|
||||
./bcm_100g_dsc_cycle.sh 200 -d
|
||||
./bcm_100g_dsc_cycle.sh -c 20 -T -w 120
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ $# -eq 0 ] && usage
|
||||
|
||||
# getopts stops at the first argument that is not an option, so "200 -d" would
|
||||
# leave -d unread. Parse, take the bare count, then parse whatever followed it.
|
||||
# getopts inside a function reads the function's own arguments; OPTIND is
|
||||
# global, so it has to be reset on each entry.
|
||||
PARSED=0
|
||||
parse_opts() {
|
||||
OPTIND=1
|
||||
while getopts "c:o:w:p:s:dTPnh" opt; do
|
||||
case "$opt" in
|
||||
c) CYCLES="$OPTARG" ;;
|
||||
o) OUTDIR="$OPTARG" ;;
|
||||
w) LINK_WAIT="$OPTARG" ;;
|
||||
p) POLL="$OPTARG" ;;
|
||||
s) SETTLE="$OPTARG" ;;
|
||||
d) DEBUG=1 ;;
|
||||
T) DOTUNE=0 ;;
|
||||
P) PAUSE_GBSYNCD=0; RESUME_GBSYNCD=0 ;;
|
||||
n) DRYRUN=1 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
PARSED=$((OPTIND - 1))
|
||||
}
|
||||
|
||||
parse_opts "$@"
|
||||
shift "$PARSED"
|
||||
|
||||
# Allow the count as a bare argument: "... 200" == "... -c 200". That is what
|
||||
# most people type first, and answering it with an error helps nobody.
|
||||
if [ "$CYCLES" = "0" ] && [ $# -ge 1 ]; then
|
||||
case "$1" in
|
||||
-*) : ;; # an option; leave it for the second pass
|
||||
*) CYCLES="$1"; shift ;; # anything else is meant to be the count,
|
||||
esac # and the check below says so if it is not
|
||||
fi
|
||||
|
||||
# Anything after the count is options again: "200 -d -T"
|
||||
if [ $# -gt 0 ]; then
|
||||
parse_opts "$@"
|
||||
shift "$PARSED"
|
||||
fi
|
||||
|
||||
if [ $# -gt 0 ]; then
|
||||
echo "ERROR: unexpected argument '$1'" >&2
|
||||
echo " run with no argument to see the usage" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
case "$CYCLES" in
|
||||
''|*[!0-9]*)
|
||||
echo "ERROR: cycle count must be a positive integer (got: '$CYCLES')" >&2
|
||||
echo " try: $0 200 or $0 -c 200" >&2
|
||||
exit 2 ;;
|
||||
esac
|
||||
if [ "$CYCLES" -lt 1 ]; then
|
||||
echo "ERROR: cycle count must be at least 1 (got: $CYCLES)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Colour, only when stdout is a terminal, so a redirected run stays clean
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
|
||||
C_INFO=$(printf '\033[1;34m') # blue
|
||||
C_OK=$(printf '\033[1;32m') # green
|
||||
C_WARN=$(printf '\033[1;33m') # yellow
|
||||
C_ERR=$(printf '\033[1;31m') # red
|
||||
C_CMD=$(printf '\033[90m') # dark grey
|
||||
C_OFF=$(printf '\033[0m')
|
||||
else
|
||||
C_INFO=''; C_OK=''; C_WARN=''; C_ERR=''; C_CMD=''; C_OFF=''
|
||||
fi
|
||||
|
||||
# A dry run touches nothing, so it must not sleep either - otherwise
|
||||
# "-c 200 -n" sits there for as long as the real thing would.
|
||||
naptime() { [ "$DRYRUN" = "1" ] && return 0; [ "${1:-0}" = "0" ] && return 0; sleep "$1"; }
|
||||
|
||||
info() { printf '%s[info]%s %s\n' "$C_INFO" "$C_OFF" "$*"; }
|
||||
ok() { printf '%s[ OK ]%s %s\n' "$C_OK" "$C_OFF" "$*"; }
|
||||
warn() { printf '%s[warn]%s %s\n' "$C_WARN" "$C_OFF" "$*" >&2; }
|
||||
err() { printf '%s[ERR ]%s %s\n' "$C_ERR" "$C_OFF" "$*" >&2; }
|
||||
|
||||
# Both -d and -n print commands. During collection the whole block is
|
||||
# redirected into a temp file, so those lines would end up in the file with
|
||||
# nothing on screen. Bind fd 9 to the terminal up front and print there.
|
||||
if { [ "$DEBUG" = "1" ] || [ "$DRYRUN" = "1" ]; } && ( true > /dev/tty ) 2>/dev/null; then
|
||||
exec 9> /dev/tty
|
||||
else
|
||||
exec 9>&1
|
||||
fi
|
||||
# ':' is a special built-in in dash (the DUT's /bin/sh); a failed redirection
|
||||
# on one kills the whole shell with no message. Hence (true > /dev/tty) in a
|
||||
# subshell above - if that cannot open, only the subshell dies.
|
||||
|
||||
# Print a command the way you would type it: quote any argument that needs it,
|
||||
# so the line can be copied back to the prompt as is.
|
||||
# NOTE: sh has no local variables - every name assigned here is global. All
|
||||
# helper variables are therefore prefixed. _show_cmd used to assign "tag",
|
||||
# which silently overwrote the caller's cycle label and turned every later
|
||||
# message into "[info] [CMD] : Wait Port Link Down".
|
||||
_show_cmd() {
|
||||
_sc_tag="$1"; _sc_col="$2"; shift 2
|
||||
{
|
||||
printf '%s%s%s' "$_sc_col" "$_sc_tag" "$C_OFF"
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
''|*[!A-Za-z0-9_./=:,+-]*) printf " '%s'" "$a" ;;
|
||||
*) printf ' %s' "$a" ;;
|
||||
esac
|
||||
done
|
||||
printf '\n'
|
||||
} >&9
|
||||
}
|
||||
|
||||
# Every external command goes through here: never reads stdin, echoed when
|
||||
# asked, and paced by CMD_DELAY so the DUT is not hammered.
|
||||
run() {
|
||||
[ "$DEBUG" = "1" ] && [ "$DRYRUN" = "0" ] && _show_cmd '[CMD]' "$C_CMD" "$@"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" "$@"
|
||||
return 0
|
||||
fi
|
||||
"$@" < /dev/null
|
||||
_cmd_rc=$?
|
||||
naptime "$CMD_DELAY"
|
||||
return $_cmd_rc
|
||||
}
|
||||
|
||||
# Same, but discards output. Do not write "> /dev/null" at the call site or
|
||||
# the [CMD] line goes with it.
|
||||
runq() {
|
||||
[ "$DEBUG" = "1" ] && [ "$DRYRUN" = "0" ] && _show_cmd '[CMD]' "$C_CMD" "$@"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" "$@"
|
||||
return 0
|
||||
fi
|
||||
"$@" < /dev/null > /dev/null 2>&1
|
||||
_cmd_rc=$?
|
||||
naptime "$CMD_DELAY"
|
||||
return $_cmd_rc
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Preflight
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DRYRUN" = "0" ]; then
|
||||
command -v bcmcmd > /dev/null 2>&1 || { err "not found: bcmcmd"; exit 3; }
|
||||
# -T only skips tuning; the per-cycle retimer read still needs this tool
|
||||
if [ ! -x /usr/sbin/blanton_barchetta_tool ]; then
|
||||
err "not found: /usr/sbin/blanton_barchetta_tool (tuning and the per-cycle retimer read both use it)"
|
||||
exit 3
|
||||
fi
|
||||
fi
|
||||
|
||||
# busybox sleep may not take a fraction
|
||||
if [ "$DRYRUN" = "0" ] && [ "$CMD_DELAY" != "0" ] && ! sleep "$CMD_DELAY" 2>/dev/null; then
|
||||
warn "sleep '$CMD_DELAY' not supported, falling back to 1"
|
||||
CMD_DELAY=1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTDIR" 2>/dev/null || { err "cannot create outdir: $OUTDIR"; exit 2; }
|
||||
[ -w "$OUTDIR" ] || { err "outdir not writable: $OUTDIR"; exit 2; }
|
||||
|
||||
TS=$(date '+%Y%m%d_%H%M%S')
|
||||
LOG="${OUTDIR}/100G_dsc_cycle_test_${TS}.log"
|
||||
: > "$LOG" || { err "cannot write $LOG"; exit 2; }
|
||||
|
||||
STATUS_TMP=$(mktemp) || { err "mktemp failed"; exit 2; }
|
||||
CYCLE_TMP=$(mktemp) || { err "mktemp failed"; exit 2; }
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# gbsyncd must go back the way it was found, or the container stays paused
|
||||
# ----------------------------------------------------------------------------
|
||||
PAUSED=0
|
||||
resume_gbsyncd() {
|
||||
[ "$PAUSED" = "1" ] || return 0
|
||||
[ "$RESUME_GBSYNCD" = "1" ] || return 0
|
||||
PAUSED=0
|
||||
for _rg_c in $GBSYNCD; do
|
||||
if runq $SUDO docker unpause "$_rg_c"; then
|
||||
info "gbsyncd resumed: $_rg_c"
|
||||
else
|
||||
warn "docker unpause $_rg_c failed - check it by hand"
|
||||
fi
|
||||
done
|
||||
}
|
||||
on_exit() { rm -f "$STATUS_TMP" "$CYCLE_TMP"; resume_gbsyncd; }
|
||||
on_int() {
|
||||
echo ""
|
||||
warn "interrupted by user"
|
||||
resume_gbsyncd
|
||||
printf '\n### DSC_CYCLE_DONE rc=130 (INTERRUPTED)\n' >> "$LOG"
|
||||
printf '### DSC_CYCLE_DONE rc=130 (INTERRUPTED)\n'
|
||||
rm -f "$STATUS_TMP" "$CYCLE_TMP"
|
||||
exit 130
|
||||
}
|
||||
trap on_exit EXIT
|
||||
trap on_int INT TERM
|
||||
|
||||
tee_log() { tee -a "$LOG"; }
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Link state. Find which field "Oper" is from the header row, then read that
|
||||
# field on every data row. Returns 0 only when every interface matches.
|
||||
# The table itself is left in $STATUS_TMP for the caller to print.
|
||||
# ----------------------------------------------------------------------------
|
||||
# $1 = "up" or "down". Runs "ps <port>" on every unit, leaves the combined
|
||||
# output in $STATUS_TMP for the caller to print, and returns 0 only when every
|
||||
# unit agrees.
|
||||
link_is() {
|
||||
_li_want="$1"
|
||||
: > "$STATUS_TMP"
|
||||
for _li_u in $UNITS; do
|
||||
printf '\n[unit %s] bcmcmd -n %s -c "ps %s"\n' "$_li_u" "$_li_u" "$PORT" >> "$STATUS_TMP"
|
||||
if [ "$DRYRUN" = "1" ]; then
|
||||
_show_cmd ' +' "$C_CMD" bcmcmd -n "$_li_u" -c "ps $PORT"
|
||||
printf ' ce0(%s) %s 4 100G FD (dry-run)\n' \
|
||||
"$PORT" "$([ "$_li_want" = up ] && echo 'up ' || echo '!ena')" >> "$STATUS_TMP"
|
||||
continue
|
||||
fi
|
||||
[ "$DEBUG" = "1" ] && _show_cmd '[CMD]' "$C_CMD" bcmcmd -n "$_li_u" -c "ps $PORT"
|
||||
bcmcmd -n "$_li_u" -c "ps $PORT" < /dev/null >> "$STATUS_TMP" 2>&1
|
||||
naptime "$CMD_DELAY"
|
||||
done
|
||||
|
||||
# The data row is the one whose first field ends in "(<port>)"; the state
|
||||
# is the field right after it. "down" is "not up", so !ena counts.
|
||||
awk -v port="$PORT" -v want="$_li_want" -v units="$(echo "$UNITS" | wc -w)" '
|
||||
$1 ~ ("\\(" port "\\)$") {
|
||||
n++
|
||||
if (want == "up") { if ($2 != "up") bad++ }
|
||||
else { if ($2 == "up") bad++ }
|
||||
}
|
||||
END { exit (n >= units && bad == 0) ? 0 : 1 }
|
||||
' "$STATUS_TMP"
|
||||
}
|
||||
|
||||
# $1 = wanted state (Down/up), $2 = label for the messages
|
||||
wait_link() {
|
||||
_wl_want="$1"; _wl_tag="$2"
|
||||
_wl_waited=0
|
||||
while :; do
|
||||
if link_is "$_wl_want"; then
|
||||
{
|
||||
printf '\n--- port %s after %ss ---\n' "$_wl_want" "$_wl_waited"
|
||||
cat "$STATUS_TMP"
|
||||
} | tee_log
|
||||
ok "$_wl_tag : port $_wl_want after ${_wl_waited}s"
|
||||
return 0
|
||||
fi
|
||||
[ "$_wl_waited" -ge "$LINK_WAIT" ] && break
|
||||
naptime "$POLL"
|
||||
_wl_waited=$((_wl_waited + POLL))
|
||||
done
|
||||
{
|
||||
printf '\n--- TIMEOUT waiting for port %s (%ss) ---\n' "$_wl_want" "$LINK_WAIT"
|
||||
cat "$STATUS_TMP"
|
||||
} | tee_log
|
||||
err "$_wl_tag : TIMEOUT waiting for port $_wl_want (${LINK_WAIT}s)"
|
||||
return 1
|
||||
}
|
||||
|
||||
port_en() {
|
||||
for _pe_u in $UNITS; do
|
||||
runq bcmcmd -n "$_pe_u" -c "port ${PORT} en=$1"
|
||||
done
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Log header
|
||||
# ----------------------------------------------------------------------------
|
||||
{
|
||||
printf '=====================================================================\n'
|
||||
printf '100G DSC cycle test\n'
|
||||
printf ' script : bcm_100g_dsc_cycle.sh V1.2.0\n'
|
||||
printf ' started : %s\n' "$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
printf ' cycles : %s\n' "$CYCLES"
|
||||
printf ' port : %s units : %s\n' "$PORT" "$UNITS"
|
||||
printf ' tuning : %s\n' "$([ "$DOTUNE" = 1 ] && echo yes || echo 'skipped (-T)')"
|
||||
printf ' tx lanes : %s\n' "$TX_LANES"
|
||||
printf ' rx lanes : %s\n' "$RX_LANES"
|
||||
printf ' link wait: %ss poll %ss settle %ss cmd delay %ss\n' \
|
||||
"$LINK_WAIT" "$POLL" "$SETTLE" "$CMD_DELAY"
|
||||
printf '=====================================================================\n'
|
||||
} >> "$LOG"
|
||||
|
||||
info "log : $LOG"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 1. Tuning, once
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DOTUNE" = "1" ]; then
|
||||
info "Tuning"
|
||||
printf '\n### TUNING %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
if [ "$PAUSE_GBSYNCD" = "1" ]; then
|
||||
for c in $GBSYNCD; do
|
||||
if runq $SUDO docker pause "$c"; then
|
||||
PAUSED=1
|
||||
info "gbsyncd paused: $c"
|
||||
else
|
||||
warn "docker pause $c failed (container not running?)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0 2>&1 | tee_log
|
||||
|
||||
for l in $SYS_LANES; do
|
||||
# shellcheck disable=SC2086
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys "$l" $SYS_TX 2>&1 | tee_log
|
||||
done
|
||||
|
||||
echo "$TX_TAPS" | while read -r u lane main pre1 post1; do
|
||||
[ -n "${u:-}" ] || continue
|
||||
run bcmcmd -n "$u" -c \
|
||||
" dsh -c 'phydiag ${PORT} tx_taps set lane=${lane} main=${main} pre1=${pre1} post1=${post1}' " \
|
||||
2>&1 | tee_log
|
||||
done
|
||||
|
||||
ok "Tuning done"
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 2. Test cycles
|
||||
# ----------------------------------------------------------------------------
|
||||
n=1
|
||||
while [ "$n" -le "$CYCLES" ]; do
|
||||
tag="Test Cycle $n / $CYCLES"
|
||||
|
||||
printf '\n\n### TEST CYCLE %s/%s %s\n' "$n" "$CYCLES" \
|
||||
"$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
# ---- Disable Port ------------------------------------------------------
|
||||
info "$tag : Disable Port"
|
||||
port_en 0
|
||||
|
||||
info "$tag : Wait Port Link Down"
|
||||
wait_link down "$tag" || RC=1
|
||||
|
||||
# ---- Enable Port -------------------------------------------------------
|
||||
info "$tag : Enable Port"
|
||||
port_en 1
|
||||
|
||||
info "$tag : Wait Link up"
|
||||
if wait_link up "$tag"; then
|
||||
naptime "$SETTLE"
|
||||
else
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# ---- Collect -----------------------------------------------------------
|
||||
# Written to a per-cycle temp file first, then appended. Appending to the
|
||||
# log and then re-reading it to check would mean re-reading an ever larger
|
||||
# file every round - O(n^2) over 200 cycles.
|
||||
info "$tag : Collect Data"
|
||||
: > "$CYCLE_TMP"
|
||||
for u in $UNITS; do
|
||||
{
|
||||
printf '\n--- phydiag %s dsc : unit %s ---\n' "$PORT" "$u"
|
||||
run bcmcmd -n "$u" -c "dsh -c 'phydiag ${PORT} dsc'" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
info "$tag : Read Retimer TX"
|
||||
for l in $TX_LANES; do
|
||||
{
|
||||
printf '\n--- serdes-tx-get sys %s ---\n' "$l"
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys "$l" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
info "$tag : Read Retimer RX"
|
||||
for l in $RX_LANES; do
|
||||
{
|
||||
printf '\n--- serdes-rx-get sys %s ---\n' "$l"
|
||||
run $SUDO /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys "$l" 2>&1
|
||||
} >> "$CYCLE_TMP"
|
||||
done
|
||||
|
||||
cat "$CYCLE_TMP" >> "$LOG"
|
||||
|
||||
if [ "$DRYRUN" = "0" ]; then
|
||||
# bcmcmd exits 0 even when it failed, so count the end marker instead
|
||||
got=$(grep -c 'SERDES DISPLAY DIAG DATA END' "$CYCLE_TMP" 2>/dev/null || true)
|
||||
want=$(echo "$UNITS" | wc -w)
|
||||
if [ "${got:-0}" -lt "$want" ]; then
|
||||
err "$tag : got only ${got:-0}/${want} DSC dumps"
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# The section headings are printed by this script, so counting them
|
||||
# only proves the loop ran. Count sections that have output under them.
|
||||
for d in tx rx; do
|
||||
case "$d" in
|
||||
tx) wantl="$TX_LANES" ;;
|
||||
rx) wantl="$RX_LANES" ;;
|
||||
esac
|
||||
got_l=$(awk -v pat="--- serdes-${d}-get sys " '
|
||||
index($0, pat) == 1 { if (insec && body) ok++; insec = 1; body = 0; next }
|
||||
/^--- / || /^### / { if (insec && body) ok++; insec = 0; next }
|
||||
insec && NF { body = 1 }
|
||||
END { if (insec && body) ok++; print ok + 0 }
|
||||
' "$CYCLE_TMP")
|
||||
want_l=$(echo "$wantl" | wc -w)
|
||||
if [ "${got_l:-0}" -lt "$want_l" ]; then
|
||||
err "$tag : retimer $(echo "$d" | tr 'a-z' 'A-Z') returned data for only ${got_l:-0}/${want_l} lanes"
|
||||
RC=1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
n=$((n + 1))
|
||||
done
|
||||
|
||||
resume_gbsyncd
|
||||
|
||||
printf '\n### DSC_CYCLE_DONE rc=%s\n' "$RC" >> "$LOG"
|
||||
if [ "$RC" = "0" ]; then
|
||||
ok "all $CYCLES cycles done -> $LOG"
|
||||
else
|
||||
err "some cycles did not complete as expected - see $LOG"
|
||||
fi
|
||||
printf '### DSC_CYCLE_DONE rc=%s\n' "$RC"
|
||||
exit "$RC"
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/bin/sh
|
||||
# =============================================================================
|
||||
# bcm_dsc_parse_V1.0.0.sh
|
||||
#
|
||||
# Author : ETWen
|
||||
# Date : 20260906
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260906 Initial Version
|
||||
#
|
||||
# Description:
|
||||
# Parse Broadcom diag shell "phydiag <pm_id> dsc" raw log into CSV.
|
||||
# 支援多個 pm_id 串接在同一份 log,以 "pm_id =" 當 record separator。
|
||||
#
|
||||
# Output:
|
||||
# <prefix>_lane.csv : 每個 lane 一列(TXEQ / RXFFE / DFE / EYE / SNR ...)
|
||||
# <prefix>_pm.csv : 每個 pm_id 一列(溫度 / PLL / lane 數 / SNR-EYE 統計)
|
||||
#
|
||||
# Usage:
|
||||
# ./bcm_dsc_parse_V1.0.0.sh -i dsc_raw.log [-o out_prefix]
|
||||
#
|
||||
# Design notes:
|
||||
# - 純 POSIX sh + awk(mawk 相容),不依賴 Python。
|
||||
# - Lane row 採「括號群組出現順序」抽取,而非固定欄位位置,對排版空白
|
||||
# 變動免疫。SDK 若增減欄位,群組數不等於 9,該列會被標記 UNPARSED
|
||||
# 並輸出到 stderr,不會靜默解析錯誤。
|
||||
# - 已知 header 名稱數與資料欄數不一致之處:
|
||||
# * (P RX, CDRxN, UC_CFG, UC_STS, RST, STP) 6 個名字 / 7 個值
|
||||
# (RST 實際是 "0,0" 兩欄)
|
||||
# * TXEQ 尾端多一個逗號
|
||||
# * AVG_TMON 顯示為 "(11) 66C" 兩個 token
|
||||
# - SD / LCK 的 "*" 為 sticky 旗標,本版拆成獨立欄位保留,不做判讀。
|
||||
# - BER 欄若為 "!chk_en" 表示 checker 未開啟,值無效,原樣保留。
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
|
||||
IN=""
|
||||
PREFIX="dsc"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -i <raw_log> [-o <out_prefix>]" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts "i:o:h" opt; do
|
||||
case "$opt" in
|
||||
i) IN="$OPTARG" ;;
|
||||
o) PREFIX="$OPTARG" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$IN" ] || usage
|
||||
[ -f "$IN" ] || { echo "ERROR: input not found: $IN" >&2; exit 2; }
|
||||
|
||||
LANE_CSV="${PREFIX}_lane.csv"
|
||||
PM_CSV="${PREFIX}_pm.csv"
|
||||
|
||||
# 先去掉 CR(console / telnet log 常帶),再交給 awk
|
||||
tr -d '\r' < "$IN" | awk -v LANE_CSV="$LANE_CSV" -v PM_CSV="$PM_CSV" '
|
||||
function trim(s) { sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s); return s }
|
||||
|
||||
# 掃描一列,取出所有頂層括號群組到 G[],群組前的英數 prefix 到 P[],
|
||||
# 其餘純量文字留在 REST。回傳群組數。
|
||||
function split_groups(line, i, ch, d, buf, ng, rest, prefix) {
|
||||
split("", G, ":"); split("", P, ":")
|
||||
ng = 0; d = 0; buf = ""; rest = ""; prefix = ""
|
||||
for (i = 1; i <= length(line); i++) {
|
||||
ch = substr(line, i, 1)
|
||||
if (ch == "(") {
|
||||
d++
|
||||
if (d == 1) {
|
||||
buf = ""; prefix = ""
|
||||
if (match(rest, /[A-Za-z_]+$/)) {
|
||||
prefix = substr(rest, RSTART, RLENGTH)
|
||||
rest = substr(rest, 1, RSTART - 1)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if (ch == ")") {
|
||||
d--
|
||||
if (d == 0) { ng++; G[ng] = buf; P[ng] = prefix; rest = rest " "; continue }
|
||||
}
|
||||
if (d > 0) { buf = buf ch; continue }
|
||||
rest = rest ch
|
||||
}
|
||||
REST = rest
|
||||
return ng
|
||||
}
|
||||
|
||||
# 取括號群組的第 n 個子欄位(以逗號分隔)
|
||||
function gf(idx, n, a, c) {
|
||||
c = split(G[idx], a, ",")
|
||||
if (n > c) return ""
|
||||
return trim(a[n])
|
||||
}
|
||||
|
||||
function is_num(s) { return (s ~ /^[+-]?[0-9]+(\.[0-9]+)?$/) }
|
||||
|
||||
function reset_pm() {
|
||||
core = ""; sdtype = ""; hw_lanes = ""; lane_sel = ""
|
||||
live_temp = ""; avg_tmon = ""; pll_lock = ""
|
||||
n_lane = 0; snr_min = ""; snr_max = ""; eye_min = ""; all_lck = 1
|
||||
}
|
||||
|
||||
function flush_pm() {
|
||||
if (pm == "") return
|
||||
printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n",
|
||||
pm, core, sdtype, hw_lanes, lane_sel, n_lane,
|
||||
live_temp, avg_tmon, pll_lock,
|
||||
(snr_min == "" ? "NA" : snr_min),
|
||||
(eye_min == "" ? "NA" : eye_min),
|
||||
(n_lane == 0 ? "NA" : (all_lck ? "PASS" : "FAIL")) > PM_CSV
|
||||
}
|
||||
|
||||
BEGIN {
|
||||
pm = ""; in_lane = 0; core_row = 0; bad = 0
|
||||
print "pm_id,core,lane,serdes_type,sd,sd_sticky,lck,lck_sticky,rxppm,txppm," \
|
||||
"txeq_n3,txeq_n2,txeq_n1,txeq_m,txeq_p1,txeq_p2,txeq_sum," \
|
||||
"rxffe_n3,rxffe_n2,rxffe_n1,rxffe_m,rxffe_p1,rxffe_p2," \
|
||||
"dfe1,dfe2,pf_m,pf_l,pf_h,vga,dco,tp0,tp1,tp2,flt_m,flt_s," \
|
||||
"nlc_u,nlc_l,eye_u,eye_m,eye_l,link_time,snr,ber," \
|
||||
"live_temp,avg_tmon,pll_lock,rx_pol,cdr,uc_cfg,uc_sts,rst,stp" > LANE_CSV
|
||||
print "pm_id,core,serdes_type,hw_lanes,lane_select,active_lanes," \
|
||||
"live_temp,avg_tmon,pll_lock,snr_min,eye_min,lck_all" > PM_CSV
|
||||
}
|
||||
|
||||
/^pm_id[ \t]*=/ { flush_pm(); pm = $NF; reset_pm(); in_lane = 0; next }
|
||||
/^Core[ \t]*=/ { if (match($0, /Core[ \t]*=[ \t]*[0-9]+/)) { core = $0; sub(/.*Core[ \t]*=[ \t]*/, "", core); sub(/[^0-9].*/, "", core) } next }
|
||||
/^SerDes type[ \t]*=/ { sdtype = $NF; next }
|
||||
/^Rev ID # Lanes/ { hw_lanes = $NF; next }
|
||||
/^lane_select[ \t]*=/ { lane_sel = $NF; next }
|
||||
|
||||
/^CORE[ \t]+RST_ST/ { core_row = 1; next }
|
||||
core_row == 1 {
|
||||
line = $0
|
||||
gsub(/\([^)]*\)[ \t]*/, "", line) # 去掉 AVG_TMON 的 "(11)"
|
||||
n = split(trim(line), c, /[ \t]+/)
|
||||
if (n >= 15) { live_temp = c[9]; avg_tmon = c[10]; pll_lock = c[15] }
|
||||
core_row = 0
|
||||
next
|
||||
}
|
||||
|
||||
/^LN[ \t]*\(/ { in_lane = 1; next }
|
||||
/SERDES DISPLAY DIAG DATA END/ { in_lane = 0; next }
|
||||
|
||||
in_lane && /^[0-9]+[ \t]*\(/ {
|
||||
ng = split_groups($0)
|
||||
if (ng != 9) {
|
||||
printf "UNPARSED (groups=%d) pm_id=%s: %s\n", ng, pm, $0 > "/dev/stderr"
|
||||
bad++
|
||||
next
|
||||
}
|
||||
nr = split(trim(REST), r, /[ \t]+/)
|
||||
if (nr != 10) {
|
||||
printf "UNPARSED (scalars=%d) pm_id=%s: %s\n", nr, pm, $0 > "/dev/stderr"
|
||||
bad++
|
||||
next
|
||||
}
|
||||
|
||||
lane = r[1]
|
||||
sd = r[2]; sd_st = (sd ~ /\*/) ? 1 : 0; gsub(/\*/, "", sd)
|
||||
lck = r[3]; lck_st = (lck ~ /\*/) ? 1 : 0; gsub(/\*/, "", lck)
|
||||
rxppm = r[4]; vga = r[5]; dco = r[6]; txppm = r[7]
|
||||
link_time = r[8]; snr = r[9]; ber = r[10]
|
||||
|
||||
# TXEQ 六個 tap 與總和(僅在全為數值時計算)
|
||||
tsum = 0; tok = 1
|
||||
for (i = 1; i <= 6; i++) {
|
||||
tq[i] = gf(7, i)
|
||||
if (is_num(tq[i])) tsum += tq[i]; else tok = 0
|
||||
}
|
||||
|
||||
printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,", pm, core, lane, sdtype, sd, sd_st, lck, lck_st, rxppm, txppm > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s,%s,%s,", tq[1], tq[2], tq[3], tq[4], tq[5], tq[6], (tok ? tsum : "NA") > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s,%s,", gf(4,1), gf(4,2), gf(4,3), gf(4,4), gf(4,5), gf(4,6) > LANE_CSV
|
||||
printf "%s,%s,", gf(5,1), gf(5,2) > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s,", gf(2,1), gf(2,2), gf(2,3), vga, dco > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s,", gf(3,1), gf(3,2), gf(3,3), gf(6,1), gf(6,2) > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s,", gf(8,1), gf(8,2), gf(9,1), gf(9,2), gf(9,3) > LANE_CSV
|
||||
printf "%s,%s,%s,", link_time, snr, ber > LANE_CSV
|
||||
printf "%s,%s,%s,", live_temp, avg_tmon, pll_lock > LANE_CSV
|
||||
printf "%s,%s,%s,%s,%s/%s,%s\n", gf(1,1), gf(1,2), gf(1,3), gf(1,4), gf(1,5), gf(1,6), gf(1,7) > LANE_CSV
|
||||
|
||||
# per-PM 統計
|
||||
n_lane++
|
||||
if (lck != "1" || sd != "1") all_lck = 0
|
||||
if (is_num(snr)) { if (snr_min == "" || snr + 0 < snr_min + 0) snr_min = snr }
|
||||
for (i = 1; i <= 3; i++) {
|
||||
e = gf(9, i)
|
||||
if (is_num(e)) { if (eye_min == "" || e + 0 < eye_min + 0) eye_min = e }
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
END {
|
||||
flush_pm()
|
||||
if (bad > 0) printf "WARN: %d lane row(s) unparsed\n", bad > "/dev/stderr"
|
||||
}
|
||||
'
|
||||
|
||||
echo "OK: ${LANE_CSV}, ${PM_CSV}"
|
||||
@@ -0,0 +1,347 @@
|
||||
#!/bin/sh
|
||||
# =============================================================================
|
||||
# bcm_dsc_scan.sh
|
||||
#
|
||||
# Author : ETWen
|
||||
# Date : 20260906
|
||||
#
|
||||
# Version History
|
||||
# V1.0.0 20260906 Initial Version
|
||||
# V1.1.0 20260906 Log 預設輸出至 <script_dir>/log,掃描後自動複製到
|
||||
# /host/hw-eval/jobs/;非 tty 環境改用逐行進度輸出,
|
||||
# 避免灌爆 Tera Term logopen 的 log
|
||||
# V1.2.0 20260906 新增 -f 自訂 log 檔名(支援 %u / %t 佔位符)與
|
||||
# -F 強制覆蓋;預設不覆蓋既有檔案,自動加序號
|
||||
# V1.3.0 20260907 修正 bcmcmd 讀取 stdin 導致掃描迴圈在第一筆之後
|
||||
# 提前結束的問題(迴圈改用 fd 3,bcmcmd 導向
|
||||
# /dev/null);新增實際迭代筆數與清單筆數的檢查
|
||||
#
|
||||
# Description:
|
||||
# 在 Blanton SONiC DUT 上批次執行 Broadcom diag shell 的
|
||||
# "phydiag <port> dsc",收集 raw log 供離線解析。
|
||||
#
|
||||
# 實際下的指令為:
|
||||
# bcmcmd -n <unit> -c "dsh -c 'phydiag <port> dsc'"
|
||||
#
|
||||
# Deployment:
|
||||
# Script : /root/Blanton_Script/bcm_dsc_scan.sh
|
||||
# Log : /root/Blanton_Script/log/dsc_raw_u<unit>_<YYYYmmdd_HHMMSS>.log
|
||||
# Copy : /host/hw-eval/jobs/
|
||||
#
|
||||
# Log 檔名可用 -f 覆寫,支援兩個佔位符:
|
||||
# %u -> u0 / u1
|
||||
# %t -> YYYYmmdd_HHMMSS
|
||||
# -f 只接受檔名,不接受路徑;目錄一律由 -o 決定。
|
||||
# 同時掃兩個 unit 但檔名未含 %u 時,會自動在副檔名前插入 _u<unit>,
|
||||
# 避免第二個 unit 覆蓋掉第一個。
|
||||
#
|
||||
# 路徑由 $0 推導,不依賴 $HOME。Script A 以 sudo -i 執行時 ~ 為 /root,
|
||||
# 直接寫死 ~ 會有風險,故一律使用絕對路徑。
|
||||
#
|
||||
# Usage:
|
||||
# ./bcm_dsc_scan.sh # unit 0 與 1 都掃,掃完自動複製
|
||||
# ./bcm_dsc_scan.sh -u 0 # 只掃 unit 0
|
||||
# ./bcm_dsc_scan.sh -n # dry-run,只列出要掃的 port
|
||||
# ./bcm_dsc_scan.sh -C # 不複製到 /host/hw-eval/jobs
|
||||
# ./bcm_dsc_scan.sh -c /other/dir # 指定複製目的地
|
||||
# ./bcm_dsc_scan.sh -u 0 -f dsc_before_reboot.log
|
||||
# ./bcm_dsc_scan.sh -f blanton_dsc_%u_%t.log
|
||||
#
|
||||
# Exit code:
|
||||
# 0 = 全部 port 皆取得完整 dump
|
||||
# 1 = 有 port 未取得完整 dump,或複製失敗
|
||||
# 2 = 參數 / 路徑錯誤
|
||||
# 3 = 找不到 bcmcmd
|
||||
#
|
||||
# 結束時固定輸出一行 "### DSC_SCAN_DONE rc=<n>",供 TTL 端 wait 判斷。
|
||||
#
|
||||
# Design notes:
|
||||
# - 純 POSIX sh,不依賴 bash / Python。
|
||||
# - Port 清單以「範圍字串」定義在下方 PORTS_* 變數,展開後去重。
|
||||
# 100G 清單先展開,故重複的 port 會被判定為 100G(目前僅 port 268)。
|
||||
# - bcmcmd 即使指令失敗通常仍回傳 0,因此不以 exit code 判斷成功與否;
|
||||
# 改由 marker 行 + "SERDES DISPLAY DIAG DATA END" 數量比對。
|
||||
# - 先寫入本機 log 目錄再複製,而非直接寫遠端,避免中途失敗遺失整份資料。
|
||||
#
|
||||
# Assumptions:
|
||||
# - unit 0 與 unit 1 使用相同的 port 清單。
|
||||
# - phydiag 的參數為此清單中的 port 編號(非 port name)。
|
||||
# - 由 root 執行(Script A 的 sudo -i 之後),故不呼叫 sudo。
|
||||
#
|
||||
# Cautions:
|
||||
# - 全清單約 445 port x 2 unit,估計 10 分鐘,請勿把 DELAY 設為 0。
|
||||
# - phydiag ... dsc 為唯讀操作,不會變更 SerDes 設定。
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Port list (以空白分隔的範圍或單一值)
|
||||
# ----------------------------------------------------------------------------
|
||||
PORTS_100G="268"
|
||||
PORTS_200G="1-24 40-96 112-168 184-240 256-308 310-327 342-380 382-399 414-452 454-471 486-524 526-543 558-565"
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Paths / defaults
|
||||
# ----------------------------------------------------------------------------
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) || exit 2
|
||||
|
||||
UNITS="0 1"
|
||||
OUTDIR="${SCRIPT_DIR}/log"
|
||||
COPYDIR="/host/hw-eval/jobs"
|
||||
DELAY="0.2"
|
||||
DRYRUN=0
|
||||
DOCOPY=1
|
||||
LOGNAME="" # -f 指定的 log 檔名,空字串表示使用預設命名
|
||||
FORCE=0 # -F 允許覆蓋既有檔案
|
||||
PROGRESS_EVERY=50 # 非 tty 環境每 N 筆印一行進度
|
||||
|
||||
usage() {
|
||||
cat >&2 << 'USAGE'
|
||||
Usage: bcm_dsc_scan.sh [-u <0|1>] [-o <outdir>] [-f <logname>] [-F]
|
||||
[-c <copydir>] [-C] [-d <sec>] [-n] [-h]
|
||||
|
||||
-u <unit> 只掃指定 unit。省略則 unit 0 與 1 都掃。
|
||||
-o <dir> log 輸出目錄 (default: <script_dir>/log)
|
||||
-f <name> 自訂 log 檔名(僅檔名,不含路徑)
|
||||
佔位符: %u -> u0/u1 , %t -> YYYYmmdd_HHMMSS
|
||||
default: dsc_raw_%u_%t.log
|
||||
-F 允許覆蓋既有同名檔案 (default: 自動加序號 _1 / _2 ...)
|
||||
-c <dir> 掃描後複製目的地 (default: /host/hw-eval/jobs)
|
||||
-C 不複製
|
||||
-d <sec> 每筆 phydiag 之間的間隔 (default: 0.2)
|
||||
-n dry-run,列出展開後的 port 清單與指令,不實際執行
|
||||
-h 顯示此說明
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts "u:o:f:Fc:Cd:nh" opt; do
|
||||
case "$opt" in
|
||||
u)
|
||||
case "$OPTARG" in
|
||||
0|1) UNITS="$OPTARG" ;;
|
||||
*) echo "ERROR: -u 只接受 0 或 1 (got: $OPTARG)" >&2; exit 2 ;;
|
||||
esac
|
||||
;;
|
||||
o) OUTDIR="$OPTARG" ;;
|
||||
f)
|
||||
case "$OPTARG" in
|
||||
"") echo "ERROR: -f 不可為空" >&2; exit 2 ;;
|
||||
*/*) echo "ERROR: -f 只接受檔名,路徑請用 -o (got: $OPTARG)" >&2; exit 2 ;;
|
||||
*) LOGNAME="$OPTARG" ;;
|
||||
esac
|
||||
;;
|
||||
F) FORCE=1 ;;
|
||||
c) COPYDIR="$OPTARG" ;;
|
||||
C) DOCOPY=0 ;;
|
||||
d) DELAY="$OPTARG" ;;
|
||||
n) DRYRUN=1 ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$OUTDIR" 2>/dev/null || { echo "ERROR: cannot create outdir: $OUTDIR" >&2; exit 2; }
|
||||
[ -w "$OUTDIR" ] || { echo "ERROR: outdir not writable: $OUTDIR" >&2; exit 2; }
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# sleep 是否支援小數秒(busybox sleep 可能只吃整數)
|
||||
# ----------------------------------------------------------------------------
|
||||
if ! sleep "$DELAY" 2>/dev/null; then
|
||||
echo "WARN: sleep '$DELAY' 不支援,改用 sleep 1" >&2
|
||||
DELAY="1"
|
||||
fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# 展開 port 清單。100G 先展開,重複者以 100G 為準。
|
||||
# 輸出格式: "<port>,<speed>",依 port 數值排序。
|
||||
# ----------------------------------------------------------------------------
|
||||
PORTFILE=$(mktemp) || { echo "ERROR: mktemp failed" >&2; exit 2; }
|
||||
trap 'rm -f "$PORTFILE"' EXIT
|
||||
trap 'echo "" >&2; echo "### DSC_SCAN_DONE rc=130 (INTERRUPTED)" >&2; rm -f "$PORTFILE"; exit 130' INT TERM
|
||||
|
||||
awk -v a="$PORTS_100G" -v b="$PORTS_200G" '
|
||||
function expand(s, spd, n, i, j, p, k, lo, hi) {
|
||||
n = split(s, p, /[ \t]+/)
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (p[i] == "") continue
|
||||
if (split(p[i], k, "-") == 2) { lo = k[1] + 0; hi = k[2] + 0 }
|
||||
else { lo = p[i] + 0; hi = lo }
|
||||
if (hi < lo) { printf "ERROR: bad range %s\n", p[i] > "/dev/stderr"; continue }
|
||||
for (j = lo; j <= hi; j++) if (!(j in SPD)) SPD[j] = spd
|
||||
}
|
||||
}
|
||||
BEGIN {
|
||||
expand(a, "100G") # 先展開 100G,重複的 port 由 100G 取得
|
||||
expand(b, "200G")
|
||||
for (p in SPD) print p "," SPD[p]
|
||||
}' | sort -t, -k1,1n > "$PORTFILE"
|
||||
|
||||
TOTAL=$(wc -l < "$PORTFILE" | tr -d ' ')
|
||||
[ "$TOTAL" -gt 0 ] || { echo "ERROR: port list is empty" >&2; exit 2; }
|
||||
|
||||
N100=$(grep -c ',100G$' "$PORTFILE" || true)
|
||||
N200=$(grep -c ',200G$' "$PORTFILE" || true)
|
||||
|
||||
echo "Script dir : $SCRIPT_DIR" >&2
|
||||
echo "Log dir : $OUTDIR" >&2
|
||||
echo "Port list : $TOTAL ports (200G x2: $N200, 100G: $N100)" >&2
|
||||
echo "Units : $UNITS" >&2
|
||||
echo "Delay : ${DELAY}s" >&2
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Dry-run
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DRYRUN" -eq 1 ]; then
|
||||
for U in $UNITS; do
|
||||
echo "--- unit $U ---"
|
||||
while IFS=, read -r P SPD; do
|
||||
printf 'bcmcmd -n %s -c "dsh -c '\''phydiag %s dsc'\''" # %s\n' "$U" "$P" "$SPD"
|
||||
done < "$PORTFILE"
|
||||
done
|
||||
echo "### DSC_SCAN_DONE rc=0 (dry-run)" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
command -v bcmcmd > /dev/null 2>&1 || {
|
||||
echo "ERROR: bcmcmd not found in PATH" >&2
|
||||
echo "### DSC_SCAN_DONE rc=3" >&2
|
||||
exit 3
|
||||
}
|
||||
|
||||
# stderr 是 tty 才用 \r 進度條;被導向檔案(Tera Term logopen)時改逐行輸出
|
||||
if [ -t 2 ]; then TTY=1; else TTY=0; fi
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Log 檔名處理
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
# 純 POSIX 的字串取代,避免 sed 對檔名中的特殊字元誤判
|
||||
replace_all() { # $1=haystack $2=needle $3=replacement
|
||||
_s=$1; _n=$2; _r=$3; _out=""
|
||||
while :; do
|
||||
case "$_s" in
|
||||
*"$_n"*)
|
||||
_out="${_out}${_s%%"$_n"*}${_r}"
|
||||
_s=${_s#*"$_n"}
|
||||
;;
|
||||
*) _out="${_out}${_s}"; break ;;
|
||||
esac
|
||||
done
|
||||
printf '%s' "$_out"
|
||||
}
|
||||
|
||||
# 依 unit 組出 log 檔名。未指定 -f 時使用預設命名。
|
||||
build_logname() { # $1=unit
|
||||
_u=$1
|
||||
_n=${LOGNAME:-dsc_raw_%u_%t.log}
|
||||
case "$_n" in *%u*) _has_u=1 ;; *) _has_u=0 ;; esac
|
||||
_n=$(replace_all "$_n" '%u' "u${_u}")
|
||||
_n=$(replace_all "$_n" '%t' "$TS")
|
||||
# 掃兩個 unit 但檔名沒有 %u,第二個 unit 會覆蓋第一個,故自動加上 _u<unit>
|
||||
if [ "$_has_u" -eq 0 ] && [ "$MULTI" -eq 1 ]; then
|
||||
case "$_n" in
|
||||
*.*) _n="${_n%.*}_u${_u}.${_n##*.}" ;;
|
||||
*) _n="${_n}_u${_u}" ;;
|
||||
esac
|
||||
fi
|
||||
printf '%s' "$_n"
|
||||
}
|
||||
|
||||
# 檔案已存在時自動加序號,避免覆蓋掉先前的掃描結果(-F 可強制覆蓋)
|
||||
uniquify() { # $1=filename
|
||||
_f=$1
|
||||
[ -e "${OUTDIR}/${_f}" ] || { printf '%s' "$_f"; return; }
|
||||
if [ "$FORCE" -eq 1 ]; then
|
||||
echo "WARN: 覆蓋既有檔案 ${OUTDIR}/${_f}" >&2
|
||||
printf '%s' "$_f"; return
|
||||
fi
|
||||
case "$_f" in
|
||||
*.*) _b=${_f%.*}; _e=".${_f##*.}" ;;
|
||||
*) _b=$_f; _e="" ;;
|
||||
esac
|
||||
_i=1
|
||||
while [ -e "${OUTDIR}/${_b}_${_i}${_e}" ]; do _i=$((_i + 1)); done
|
||||
echo "WARN: ${_f} 已存在,改用 ${_b}_${_i}${_e}(-F 可強制覆蓋)" >&2
|
||||
printf '%s' "${_b}_${_i}${_e}"
|
||||
}
|
||||
|
||||
MULTI=0
|
||||
[ "$(echo "$UNITS" | wc -w | tr -d ' ')" -gt 1 ] && MULTI=1
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Scan
|
||||
# ----------------------------------------------------------------------------
|
||||
TS=$(date +%Y%m%d_%H%M%S)
|
||||
RC=0
|
||||
LOGS=""
|
||||
|
||||
for U in $UNITS; do
|
||||
LOG="${OUTDIR}/$(uniquify "$(build_logname "$U")")"
|
||||
: > "$LOG" || { echo "ERROR: cannot write $LOG" >&2; exit 2; }
|
||||
LOGS="$LOGS $LOG"
|
||||
|
||||
printf '### SCAN_BEGIN unit=%s time=%s total=%s script=bcm_dsc_scan_V1.3.0\n' \
|
||||
"$U" "$(date '+%Y-%m-%d %H:%M:%S')" "$TOTAL" >> "$LOG"
|
||||
|
||||
echo "[unit $U] scanning $TOTAL ports -> $LOG" >&2
|
||||
|
||||
SEQ=0
|
||||
while IFS=, read -r P SPD <&3; do
|
||||
SEQ=$((SEQ + 1))
|
||||
printf '### PORT unit=%s port=%s speed=%s seq=%s/%s\n' "$U" "$P" "$SPD" "$SEQ" "$TOTAL" >> "$LOG"
|
||||
# bcmcmd 會讀取 stdin,若不導成 /dev/null 會把 PORTFILE 剩餘內容吸走,
|
||||
# 導致迴圈在第一筆之後就讀到 EOF 而提前結束。
|
||||
bcmcmd -n "$U" -c "dsh -c 'phydiag $P dsc'" >> "$LOG" 2>&1 < /dev/null || RC=1
|
||||
|
||||
if [ "$TTY" -eq 1 ]; then
|
||||
printf '\r[unit %s] %s/%s port=%-5s' "$U" "$SEQ" "$TOTAL" "$P" >&2
|
||||
elif [ $((SEQ % PROGRESS_EVERY)) -eq 0 ] || [ "$SEQ" -eq "$TOTAL" ]; then
|
||||
printf '[unit %s] %s/%s port=%s\n' "$U" "$SEQ" "$TOTAL" "$P" >&2
|
||||
fi
|
||||
|
||||
sleep "$DELAY"
|
||||
done 3< "$PORTFILE"
|
||||
|
||||
[ "$TTY" -eq 1 ] && printf '\n' >&2
|
||||
printf '### SCAN_END unit=%s time=%s\n' "$U" "$(date '+%Y-%m-%d %H:%M:%S')" >> "$LOG"
|
||||
|
||||
# 檢查 1:實際迭代筆數是否等於 port 清單筆數。
|
||||
# 迴圈若因 stdin 被子程序吸走而提前結束,只靠下面的 marker/dump 比對抓不到。
|
||||
if [ "$SEQ" -ne "$TOTAL" ]; then
|
||||
echo "ERROR: unit $U 只跑了 $SEQ/$TOTAL 筆,迴圈提前結束" >&2
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# 檢查 2:每筆 PORT marker 應對應一個 dsc 區塊
|
||||
MARK=$(grep -c '^### PORT ' "$LOG" || true)
|
||||
DUMP=$(grep -c 'SERDES DISPLAY DIAG DATA END' "$LOG" || true)
|
||||
if [ "$MARK" -ne "$DUMP" ]; then
|
||||
echo "WARN: unit $U markers=$MARK dsc_blocks=$DUMP (有 port 未取得完整 dump)" >&2
|
||||
RC=1
|
||||
fi
|
||||
echo "Saved: $LOG (${DUMP}/${TOTAL} dumps)" >&2
|
||||
done
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Copy to job directory
|
||||
# ----------------------------------------------------------------------------
|
||||
if [ "$DOCOPY" -eq 1 ]; then
|
||||
if mkdir -p "$COPYDIR" 2>/dev/null && [ -w "$COPYDIR" ]; then
|
||||
for L in $LOGS; do
|
||||
if cp -p "$L" "$COPYDIR/"; then
|
||||
echo "Copied: $COPYDIR/$(basename "$L")" >&2
|
||||
else
|
||||
echo "ERROR: copy failed: $L -> $COPYDIR" >&2
|
||||
RC=1
|
||||
fi
|
||||
done
|
||||
sync
|
||||
else
|
||||
echo "ERROR: copydir not writable: $COPYDIR (log 仍保留在 $OUTDIR)" >&2
|
||||
RC=1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "### DSC_SCAN_DONE rc=${RC}" >&2
|
||||
exit "$RC"
|
||||
@@ -1,8 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# blanton_ber.sh -- SerDes BER test over every cabled port (PRBS via phydiag)
|
||||
# blanton_ber.sh -- SerDes BER test, cabled ports PLUS the 100G uplink
|
||||
# =============================================================================
|
||||
# Version History:
|
||||
# V1.1.1 2026-09-15 Fix: "the BCM shell rejected 'phydiag ... prbs get'" on
|
||||
# a run that was in fact working. The rejection check
|
||||
# grepped the WHOLE output, case-insensitively, for
|
||||
# ERROR|Invalid|... -- and 'prbs get' prints one line per
|
||||
# lane, so a lane that had not locked ("PRBS Error!")
|
||||
# matched. The command was fine; the output was data.
|
||||
# Two consequences: a good run reported a hard error, and
|
||||
# the lock-count check that was supposed to judge exactly
|
||||
# those lanes never ran. Per-lane result lines are now
|
||||
# excluded from the check, the match is case-sensitive,
|
||||
# and a real rejection prints the offending line.
|
||||
# The rejection check had been there since V1.0.0, so
|
||||
# every earlier run carried the same blind spot.
|
||||
# V1.1.0 2026-09-15 Adds port 268, the 100G uplink, which the old port list
|
||||
# deliberately skipped (it jumps 256-259 -> 270-308).
|
||||
# The per-lane table is unchanged; the summary now breaks
|
||||
# each unit into "200G ports" and "100G p268" plus a
|
||||
# TOTAL row, so the uplink cannot hide inside a 432-lane
|
||||
# average. A group that reports no lane at all counts as
|
||||
# a failure, not a pass - silence on the port under test
|
||||
# is not a result. Developed on the bench as
|
||||
# blanton_ber_test.sh and folded back over this file once
|
||||
# it was proven, so there is one BER script again.
|
||||
# V1.0.1 2026-08-28 Fix: every lane reported NA and the Port column showed
|
||||
# "NA". bcmcmd returns CRLF, and the BER is the last token
|
||||
# on its line, so the CR stayed glued to the number: it
|
||||
@@ -19,8 +42,24 @@
|
||||
# =============================================================================
|
||||
#
|
||||
# This is the WHOLE-SWITCH BER sweep: every cabled port, both lanes, PRBS-31
|
||||
# through the SerDes. Not to be confused with port_prbs_monitor.sh, which drives
|
||||
# the two 100G uplinks only and polls them continuously.
|
||||
# through the SerDes -- and, unlike blanton_ber.sh, the 100G uplink as well.
|
||||
# Not to be confused with port_prbs_monitor.sh, which drives the two 100G
|
||||
# uplinks only and polls them continuously.
|
||||
#
|
||||
# The port list is two groups:
|
||||
# BER_PORTS_200G the cabled 200G ports (432 lanes per unit)
|
||||
# BER_PORTS_100G the 100G uplink, port 268 = ce0, one per unit
|
||||
# They are measured in one phydiag pass, but the summary reports them
|
||||
# separately per unit, plus a TOTAL:
|
||||
#
|
||||
# Unit GROUP LANES PASS FAIL NA WORST BER ...
|
||||
# unit 0 200G ports 432 432 0 0 2.07e-08 PASS
|
||||
# unit 0 100G p268 4 4 0 0 1.10e-11 PASS
|
||||
# unit 0 TOTAL 436 436 0 0 2.07e-08 PASS
|
||||
# unit 1 ...
|
||||
#
|
||||
# A 100G row of "NO DATA" means the uplink reported no lane at all, and is
|
||||
# treated as a failure -- silence on the port under test is not a pass.
|
||||
#
|
||||
# Order of operations (matches tools/BER_Test.txt):
|
||||
# init phydiag <ports> prbs set p=3 arm PRBS on every lane
|
||||
@@ -51,7 +90,7 @@
|
||||
# -u|--unit <0|1|all|0,1> bcmcmd unit(s). Default: $BER_UNITS ("0 1")
|
||||
# -c|--criteria <ber> report: PASS threshold (default $BER_MAX)
|
||||
# -i|--interval <sec> start: prbsstat Interval (default $BER_INTERVAL)
|
||||
# -p|--ports <list> override the port list
|
||||
# -p|--ports <list> override the whole port list (both groups)
|
||||
# -F|--fail-only report: list only the lanes that FAIL
|
||||
# --tsv report: tab-separated (paste into Excel)
|
||||
# --raw report: also print the raw 'prbsstat Ber' output
|
||||
@@ -63,7 +102,8 @@
|
||||
#
|
||||
# Environment overrides:
|
||||
# BCMCMD_CMD=bcmcmd BER_UNITS="0 1" BER_MAX=1e-6 BER_INTERVAL=30
|
||||
# BER_PORTS="1-24,..." BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1
|
||||
# BER_PORTS="1-24,..." BER_PORTS_200G=... BER_PORTS_100G=268
|
||||
# BER_POLY=p=3 BER_LOG_DIR=/tmp DEBUG_MODE=1
|
||||
# =============================================================================
|
||||
|
||||
# === Config ==================================================================
|
||||
@@ -73,7 +113,18 @@ BER_UNITS="${BER_UNITS:-0 1}" # one DUT = two switch boards (0 and 1)
|
||||
# The cabled ports, in bcm logical-port numbering. Same list as
|
||||
# blanton_traffic_linespeed.sh's TL_PS_PORTS -- kept as one string so it can be
|
||||
# pasted straight from/into a manual phydiag command.
|
||||
BER_PORTS="${BER_PORTS:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}"
|
||||
#
|
||||
# Note the gap: the list runs 256-259 then 270-308, so 260..269 are not in it.
|
||||
# 268 is ce0, the 100G uplink, and it is the reason this fork exists.
|
||||
BER_PORTS_200G="${BER_PORTS_200G:-1-24,40-96,112-168,184-240,256-259,270-308,310-327,342-380,382-399,414-452,454-471,486-524,526-543,558-565}"
|
||||
|
||||
# The 100G uplink. One port per unit -- unit 0 and unit 1 each have their own
|
||||
# ce0(268), so a two-unit run measures two of them. Comma-separated if more are
|
||||
# ever added; the summary treats everything in here as the "100G" group.
|
||||
BER_PORTS_100G="${BER_PORTS_100G:-268}"
|
||||
|
||||
# What actually goes on the phydiag command line. -p|--ports overrides the lot.
|
||||
BER_PORTS="${BER_PORTS:-${BER_PORTS_200G},${BER_PORTS_100G}}"
|
||||
|
||||
BER_POLY="${BER_POLY:-p=3}" # passed to 'prbs set'
|
||||
BER_INTERVAL="${BER_INTERVAL:-30}" # passed to 'prbsstat STArt Interval='
|
||||
@@ -111,6 +162,24 @@ _ber_dbg() { [ "$DEBUG_MODE" = "1" ] && echo -e "\033[90m[DEBG] $*\033[0m" >&2;
|
||||
BER_ERR_PAT="${BER_ERR_PAT:-ERROR|Failed to execute|Invalid|Syntax error|Unavailable|Unknown command}"
|
||||
BER_BENIGN_PAT="${BER_BENIGN_PAT:-polling socket timeout}"
|
||||
|
||||
# A per-lane result line -- " 185 : PRBS OK!..." from 'prbs get', or
|
||||
# " 203[1] : 5.05e-09" from 'prbsstat Ber'. These are MEASUREMENTS, never
|
||||
# shell diagnostics, and they must be kept out of the rejection check:
|
||||
# a lane that did not lock prints its own wording (PRBS Error!, and similar),
|
||||
# which a case-insensitive match on "ERROR" reads as the BCM shell refusing
|
||||
# the command. That is what turned a perfectly good run into
|
||||
# [ERR ] unit 1: the BCM shell rejected 'phydiag ... prbs get'
|
||||
# while init/start/report were in fact working -- and it skipped the
|
||||
# lock-count check that was supposed to judge those very lanes.
|
||||
BER_LANE_LINE='^[ \t]*[0-9]+(\[[0-9]+\])?[ \t]*:'
|
||||
|
||||
# _ber_shell_error <output> -- echo the offending line(s), return 0 if any
|
||||
_ber_shell_error() {
|
||||
printf '%s\n' "$1" | tr -d '\r' \
|
||||
| grep -vE "$BER_LANE_LINE" \
|
||||
| grep -E "$BER_ERR_PAT"
|
||||
}
|
||||
|
||||
# _ber_bcm <unit> <phydiag args...> -- wraps in dsh -c 'phydiag <ports> ...'
|
||||
# echoes the DUT output on stdout, returns 0 ok / 1 failed
|
||||
_ber_bcm() {
|
||||
@@ -125,11 +194,14 @@ _ber_bcm() {
|
||||
local out rc
|
||||
out=$("$BCMCMD_CMD" -n "$unit" -c "$cmd" </dev/null 2>&1); rc=$?
|
||||
printf '%s\n' "$out"
|
||||
if echo "$out" | grep -qiE "$BER_ERR_PAT"; then
|
||||
local why
|
||||
why=$(_ber_shell_error "$out")
|
||||
if [ -n "$why" ]; then
|
||||
_ber_err "unit $unit: the BCM shell rejected 'phydiag ... $*'"
|
||||
printf '%s\n' "$why" | sed 's/^/ /' >&2
|
||||
return 1
|
||||
fi
|
||||
if [ "$rc" -ne 0 ] && ! echo "$out" | grep -qiE "$BER_BENIGN_PAT"; then
|
||||
if [ "$rc" -ne 0 ] && ! printf '%s\n' "$out" | grep -qiE "$BER_BENIGN_PAT"; then
|
||||
_ber_err "unit $unit: bcmcmd exited $rc on 'phydiag ... $*'"
|
||||
return 1
|
||||
fi
|
||||
@@ -243,13 +315,32 @@ _ber_render() {
|
||||
-v FAILONLY="$BER_FAIL_ONLY" \
|
||||
-v TSV="$BER_TSV" \
|
||||
-v COLOR="$color" \
|
||||
-v SIDE="$side" '
|
||||
-v SIDE="$side" \
|
||||
-v P100="$BER_PORTS_100G" '
|
||||
function vc(v) {
|
||||
if (!COLOR) return v
|
||||
if (v == "PASS") return "\033[32m" v "\033[0m"
|
||||
if (v == "FAIL") return "\033[31m" v "\033[0m"
|
||||
return "\033[33m" v "\033[0m"
|
||||
}
|
||||
# Fold one lane into a group tally. Called once for its own group and
|
||||
# and once for ALL, so the TOTAL row is never a hand-added sum that can
|
||||
# drift away from the parts.
|
||||
function acct(g, verdict, berstr, berval, label) {
|
||||
GN[g]++
|
||||
if (verdict == "PASS") GP[g]++
|
||||
else if (verdict == "FAIL") GF[g]++
|
||||
else GNA[g]++
|
||||
if (berval >= 0 && (!(g in GW) || berval > GW[g])) {
|
||||
GW[g] = berval; GWS[g] = berstr; GWL[g] = label
|
||||
}
|
||||
}
|
||||
function emit(g, lanes) {
|
||||
lanes = GN[g] + 0
|
||||
printf "SUM %s %d %d %d %d %s %s\n", g, lanes, GP[g] + 0, GF[g] + 0, GNA[g] + 0,
|
||||
(lanes > 0 && (g in GWS)) ? GWS[g] : "-",
|
||||
(lanes > 0 && (g in GWL)) ? GWL[g] : "-" > SIDE
|
||||
}
|
||||
BEGIN {
|
||||
LIM = MAXBER + 0
|
||||
NC = 4
|
||||
@@ -258,6 +349,18 @@ _ber_render() {
|
||||
HDR[3] = "BER"; AL[3] = 1
|
||||
HDR[4] = "RESULT"; AL[4] = 1
|
||||
worst = -1; worstlane = "-"
|
||||
|
||||
# Expand the 100G list ("268" or "268,270-271") into a lookup, so a lane
|
||||
# can be put in the right group by its port number alone.
|
||||
np = split(P100, pp, /[ ,\t]+/)
|
||||
for (i2 = 1; i2 <= np; i2++) {
|
||||
if (pp[i2] == "") continue
|
||||
if (split(pp[i2], rr, "-") == 2) {
|
||||
for (j2 = rr[1] + 0; j2 <= rr[2] + 0; j2++) IS100[j2] = 1
|
||||
} else {
|
||||
IS100[pp[i2] + 0] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
# bcmcmd hands back CRLF. On the cd/traffic report the counter is followed by
|
||||
# a delta, so the CR got trimmed with the rest of the line; here the BER is
|
||||
@@ -282,6 +385,8 @@ _ber_render() {
|
||||
seen++
|
||||
n++
|
||||
PORT[n] = port; LANE[n] = lane; BERS[n] = ber
|
||||
grp = ((port + 0) in IS100) ? "P100" : "P200"
|
||||
GRP[n] = grp
|
||||
# a lane that reported something non-numeric has no usable measurement
|
||||
if (ber ~ /^[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$/) {
|
||||
v = ber + 0
|
||||
@@ -290,7 +395,10 @@ _ber_render() {
|
||||
if (v > worst) { worst = v; worstlane = port "[" lane "]"; worststr = ber }
|
||||
} else {
|
||||
V[n] = "NA"; nna++
|
||||
v = -1
|
||||
}
|
||||
acct(grp, V[n], ber, v, port "[" lane "]")
|
||||
acct("ALL", V[n], ber, v, port "[" lane "]")
|
||||
}
|
||||
END {
|
||||
if (seen == 0) {
|
||||
@@ -327,19 +435,32 @@ _ber_render() {
|
||||
}
|
||||
|
||||
print ""
|
||||
printf "# lanes measured : %d\n", n
|
||||
printf "# lanes measured : %d (200G %d, 100G %d)\n", n, GN["P200"] + 0, GN["P100"] + 0
|
||||
printf "# %s %d %s %d %s %d (criteria BER < %s)\n", \
|
||||
vc("PASS"), npass + 0, vc("FAIL"), nfail + 0, vc("NA"), nna + 0, MAXBER
|
||||
if (worst >= 0)
|
||||
printf "# worst lane : %s %s\n", worstlane, worststr
|
||||
if (SIDE != "" && SIDE != "/dev/null")
|
||||
printf "SUM %d %d %d %d %s %s\n", n, npass + 0, nfail + 0, nna + 0, \
|
||||
(worst >= 0 ? worststr : "-"), worstlane > SIDE
|
||||
if (GN["P100"] + 0 == 0)
|
||||
printf "# %s the 100G port list (%s) matched no lane in this capture\n", vc("NA"), P100
|
||||
else
|
||||
printf "# 100G p%s : %s %d %s %d %s %d worst %s %s\n", P100,
|
||||
vc("PASS"), GP["P100"] + 0, vc("FAIL"), GF["P100"] + 0,
|
||||
vc("NA"), GNA["P100"] + 0,
|
||||
("P100" in GWS) ? GWS["P100"] : "-",
|
||||
("P100" in GWL) ? GWL["P100"] : "-"
|
||||
if (SIDE != "" && SIDE != "/dev/null") {
|
||||
emit("P200"); emit("P100"); emit("ALL")
|
||||
}
|
||||
exit (nfail || nna) ? 2 : 0
|
||||
}' "$log"
|
||||
}
|
||||
|
||||
# _ber_summary <stamp> -- one table for every unit, printed last
|
||||
# _ber_summary <stamp> -- one table for every unit, printed last.
|
||||
#
|
||||
# Each unit gets three rows: the 200G ports, the 100G uplink on its own, and a
|
||||
# TOTAL. Unit 0 and unit 1 each have their own ce0(268), so the 100G row is per
|
||||
# unit too -- a bad uplink on one board is what this fork exists to make
|
||||
# visible, and averaged into 432 other lanes it would not be.
|
||||
_ber_summary() {
|
||||
local stamp="$1" unit side g r y n worst=0
|
||||
g=$'\033[32m'; r=$'\033[31m'; y=$'\033[33m'; n=$'\033[0m'
|
||||
@@ -347,27 +468,41 @@ _ber_summary() {
|
||||
|
||||
echo ""
|
||||
echo "===== summary : all units ====="
|
||||
printf '%-8s %8s %8s %8s %8s %-12s %-10s %s\n' \
|
||||
"Unit" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT"
|
||||
printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %s\n' \
|
||||
"Unit" "GROUP" "LANES" "PASS" "FAIL" "NA" "WORST BER" "WORST LANE" "RESULT"
|
||||
|
||||
local tag lanes pass fail na wber wlane v any=0
|
||||
local tag grp lanes pass fail na wber wlane v any=0 label
|
||||
for unit in $BER_UNITS; do
|
||||
side="$BER_LOG_DIR/berside_unit${unit}_${stamp}"
|
||||
side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}"
|
||||
[ -r "$side" ] || continue
|
||||
read -r tag lanes pass fail na wber wlane < "$side" 2>/dev/null
|
||||
[ "$tag" = "SUM" ] || continue
|
||||
any=1
|
||||
if [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then
|
||||
v="${r}FAIL${n}"; worst=2
|
||||
else
|
||||
v="${g}PASS${n}"
|
||||
fi
|
||||
printf '%-8s %8s %8s %8s %8s %-12s %-10s %b\n' \
|
||||
"unit $unit" "$lanes" "$pass" "$fail" "$na" "$wber" "$wlane" "$v"
|
||||
while read -r tag grp lanes pass fail na wber wlane; do
|
||||
[ "$tag" = "SUM" ] || continue
|
||||
any=1
|
||||
case "$grp" in
|
||||
P200) label="200G ports" ;;
|
||||
P100) label="100G p${BER_PORTS_100G}" ;;
|
||||
ALL) label="TOTAL" ;;
|
||||
*) label="$grp" ;;
|
||||
esac
|
||||
# A group with no lanes is not a pass. On the 100G row it means the
|
||||
# uplink never reported -- exactly the case worth flagging.
|
||||
if [ "${lanes:-0}" -eq 0 ]; then
|
||||
v="${y}NO DATA${n}"
|
||||
[ "$grp" = "P100" ] && worst=2
|
||||
elif [ "${fail:-0}" -gt 0 ] || [ "${na:-0}" -gt 0 ]; then
|
||||
v="${r}FAIL${n}"; worst=2
|
||||
else
|
||||
v="${g}PASS${n}"
|
||||
fi
|
||||
printf '%-8s %-12s %7s %7s %7s %7s %-12s %-12s %b\n' \
|
||||
"unit $unit" "$label" "$lanes" "$pass" "$fail" "$na" \
|
||||
"$wber" "$wlane" "$v"
|
||||
done < "$side"
|
||||
done
|
||||
[ "$any" = "1" ] || echo " (no unit produced a result)"
|
||||
echo ""
|
||||
printf 'criteria: BER < %s\n' "$BER_MAX"
|
||||
printf '100G group: port(s) %s -- one per unit\n' "$BER_PORTS_100G"
|
||||
echo ""
|
||||
return "$worst"
|
||||
}
|
||||
@@ -391,7 +526,7 @@ blanton_ber_report() {
|
||||
stamp=$(date +%Y%m%d_%H%M%S 2>/dev/null || echo now)
|
||||
for unit in $BER_UNITS; do
|
||||
log="$BER_LOG_DIR/prbsber_unit${unit}_${stamp}.log"
|
||||
side="$BER_LOG_DIR/berside_unit${unit}_${stamp}"
|
||||
side="$BER_LOG_DIR/bertside_unit${unit}_${stamp}"
|
||||
: > "$side"
|
||||
if ! _ber_bcm "$unit" "prbsstat Ber" > "$log" 2>&1; then
|
||||
_ber_err "unit $unit: 'prbsstat Ber' failed (see $log)"; worst=1; continue
|
||||
@@ -409,22 +544,24 @@ blanton_ber_report() {
|
||||
|
||||
_ber_summary "$stamp"; rc=$?
|
||||
[ "$rc" -gt "$worst" ] && worst=$rc
|
||||
rm -f "$BER_LOG_DIR"/berside_unit*_"${stamp}"
|
||||
rm -f "$BER_LOG_DIR"/bertside_unit*_"${stamp}"
|
||||
return "$worst"
|
||||
}
|
||||
|
||||
# === Help ====================================================================
|
||||
blanton_ber_help() {
|
||||
echo -e "\033[1mblanton_ber.sh - SerDes BER test over every cabled port (PRBS via phydiag)\033[0m"
|
||||
echo -e "\033[1mblanton_ber.sh - SerDes BER test: cabled 200G ports + the 100G uplink\033[0m"
|
||||
echo ""
|
||||
echo -e " Units : \033[36m$BER_UNITS\033[0m poly: \033[36m$BER_POLY\033[0m interval: \033[36m${BER_INTERVAL}s\033[0m"
|
||||
echo -e " Criteria : BER < \033[36m$BER_MAX\033[0m"
|
||||
echo -e " Ports : \033[36m$BER_PORTS\033[0m"
|
||||
echo -e " 200G : \033[36m$BER_PORTS_200G\033[0m"
|
||||
echo -e " 100G : \033[36m$BER_PORTS_100G\033[0m (ce0, one per unit -- reported on its own row)"
|
||||
echo ""
|
||||
echo -e "\033[1mSubcommands\033[0m"
|
||||
echo -e " \033[33minit\033[0m prbs set $BER_POLY + prbs get (checks every lane locked)"
|
||||
echo -e " \033[33mstart\033[0m prbsstat STArt Interval=$BER_INTERVAL"
|
||||
echo -e " \033[33mreport\033[0m prbsstat Ber -> per-lane table + PASS/FAIL + summary"
|
||||
echo -e " summary splits each unit into 200G ports / 100G p$BER_PORTS_100G / TOTAL"
|
||||
echo -e " \033[33mstop\033[0m prbsstat STOp"
|
||||
echo -e " \033[33mclear\033[0m prbs clear (disarms PRBS -- 'init' again before measuring)"
|
||||
echo -e " \033[33mhelp\033[0m"
|
||||
@@ -433,7 +570,7 @@ blanton_ber_help() {
|
||||
echo -e " \033[33m-u|--unit\033[0m <0|1|all> bcmcmd unit(s); 'all' = 0 1 (default: $BER_UNITS)"
|
||||
echo -e " \033[33m-c|--criteria\033[0m <ber> PASS threshold (default $BER_MAX)"
|
||||
echo -e " \033[33m-i|--interval\033[0m <sec> prbsstat Interval (default $BER_INTERVAL)"
|
||||
echo -e " \033[33m-p|--ports\033[0m <list> override the port list"
|
||||
echo -e " \033[33m-p|--ports\033[0m <list> override the whole port list (both groups)"
|
||||
echo -e " \033[33m-F|--fail-only\033[0m report: list only failing lanes"
|
||||
echo -e " \033[33m--tsv\033[0m report: tab-separated"
|
||||
echo -e " \033[33m--raw\033[0m report: also print the raw output"
|
||||
|
||||
@@ -5,7 +5,10 @@ project_name = "Blanton"
|
||||
strTestcase="ENV" ;ENV,EMC,Margin...etc
|
||||
|
||||
EN_Margin = 0 ; 1: Enable Margin Test, 0: Disable Margin Test
|
||||
EN_TX_FIR = 0 ; 1: Enable bcm_dsc_scan.sh collect port TX_FIR, 0: Disable (saves ~20 min: ScriptA and ScriptC each scan for ~10 min)
|
||||
|
||||
EN_log = 1 ; 1: Enable log, 0: Disable log
|
||||
EN_timestamp = 1 ; 1: Enable timestamp in log, 0: Disable timestamp in log
|
||||
|
||||
;================================================================
|
||||
; FAN_SPEED
|
||||
@@ -20,10 +23,11 @@ SWB_UNIT1 = 1 ; 1 = this DUT has switch unit 1, wait for it ; 0 = skip
|
||||
;================================================================
|
||||
; Stress_Test
|
||||
;================================================================
|
||||
SSD_RUNTIME = 600 ; Unit seconds, 600 = 10 minutes
|
||||
SSD_COUNT = 720 ; Total time = SSD_RUNTIME * SSD_COUNT
|
||||
USB_RUNTIME = 600 ; Unit seconds, 600 = 10 minutes
|
||||
USB_COUNT = 720 ; Total time = USB_RUNTIME * USB_COUNT
|
||||
SSD_RUNTIME = 3600 ; Unit seconds, 3600 = 1 hour
|
||||
SSD_COUNT = 240 ; Total time = SSD_RUNTIME * SSD_COUNT
|
||||
USB_RUNTIME = 3600 ; Unit seconds, 3600 = 1 hour
|
||||
USB_COUNT = 240 ; Total time = USB_RUNTIME * USB_COUNT
|
||||
BMC_USB_RUNTIME = 172800 ; Unit seconds, 172800 = 2 days
|
||||
|
||||
;================================================================
|
||||
; Prompt Definitions
|
||||
@@ -31,3 +35,12 @@ USB_COUNT = 720 ; Total time = USB_RUNTIME * USB_COUNT
|
||||
prompt_login = "sonic login:"
|
||||
prompt_sonic = "admin@sonic:~$"
|
||||
prompt_sonic_root = "root@sonic:~#"
|
||||
|
||||
|
||||
;================================================================
|
||||
; Reboot or Power Cycle
|
||||
;================================================================
|
||||
; 0: Normal Test
|
||||
; 1: Enable Reboot Cycle
|
||||
; 2: Enable Power Cycle
|
||||
EN_Cold_Warm_boot = 0
|
||||
@@ -0,0 +1,90 @@
|
||||
cd /usr/sbin
|
||||
sudo docker pause gbsyncd1
|
||||
sudo docker pause gbsyncd0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0
|
||||
|
||||
//Tuning
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 0 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 1 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 2 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 3 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 4 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 5 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 6 pre=-10 main=87 post=0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-set sys 7 pre=-10 main=87 post=0
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=0 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=1 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=2 main=27 pre1=4 post1=13' "
|
||||
bcmcmd -n 0 -c " dsh -c 'phydiag 268 tx_taps set lane=3 main=29 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=0 main=29 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=1 main=30 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=2 main=31 pre1=4 post1=13' "
|
||||
bcmcmd -n 1 -c " dsh -c 'phydiag 268 tx_taps set lane=3 main=30 pre1=4 post1=13' "
|
||||
|
||||
以下Test Cycle 目標200次
|
||||
//Disable Port
|
||||
bcmcmd -n 0 -c "port 268 en=0"
|
||||
bcmcmd -n 1 -c "port 268 en=0"
|
||||
|
||||
//Wait Link Down
|
||||
bcmcmd -n 0 -c "ps 268"
|
||||
bcmcmd -n 1 -c "ps 268"
|
||||
root@sonic:~# bcmcmd -n 0 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) !ena 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~# bcmcmd -n 1 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) !ena 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~#
|
||||
|
||||
|
||||
//Enable Port
|
||||
bcmcmd -n 0 -c "port 268 en=1"
|
||||
bcmcmd -n 1 -c "port 268 en=1"
|
||||
|
||||
//Wait Link Up
|
||||
bcmcmd -n 0 -c "ps 268"
|
||||
bcmcmd -n 1 -c "ps 268"
|
||||
root@sonic:~# bcmcmd -n 0 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) up 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~# bcmcmd -n 1 -c "ps 268"
|
||||
ps 268
|
||||
ena/ speed/ link auto STP lrn max cut loop
|
||||
port link Lns duplex scan neg? state pause discrd ops medium frame thru FEC back
|
||||
ce0(268) up 4 100G FD SW No Forward Untag FA Backplane 9122 No RS528
|
||||
drivshell>
|
||||
root@sonic:~#
|
||||
|
||||
|
||||
//Collect DSC Data
|
||||
bcmcmd -n 0 -c "dsh -c 'phydiag 268 dsc'"
|
||||
bcmcmd -n 1 -c "dsh -c 'phydiag 268 dsc'"
|
||||
|
||||
//讀Retimer TX/RX
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 1
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 2
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 3
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 4
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 5
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 6
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-tx-get sys 7
|
||||
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 0
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 1
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 2
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 3
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 4
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 5
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 6
|
||||
sudo /usr/sbin/blanton_barchetta_tool --serdes-rx-get sys 7
|
||||
Reference in New Issue
Block a user