From 9f2b074c4a679d1627cbbde64bbee603b2c7ccd9 Mon Sep 17 00:00:00 2001 From: flamingcow Date: Fri, 14 Aug 2026 23:35:19 -0700 Subject: [PATCH] Mixed FS+Wiitek test pair: modules dispatched by EEPROM PN to a BCM or Go RollBall client (whitelist panics outside the wiitek proven-safe set), SNR now solely the wiitek's IEEE margins (BCM handler SNR dropped as unactionable), ECD-clears-AN-enable trap found and fixed (7.0 |= 0x1200 on every restart, explicit restart after each diag), mixed roles FS-slave/wiitek-auto-master, window re-arms at taken edge so the one CMD write per window sits far from the poll; X710 rejects the honest FS EEPROM so the noise pair runs Fibergaga+Wiitek --- docs/modules/README.md | 5 +- docs/modules/fs/README.md | 9 +- docs/modules/wiitek/README.md | 13 +- docs/state.md | 16 +- phy.go | 751 +++++++++++++++++++++++----------- phy_test.go | 36 +- 6 files changed, 565 insertions(+), 265 deletions(-) diff --git a/docs/modules/README.md b/docs/modules/README.md index d97c117..1b283af 100644 --- a/docs/modules/README.md +++ b/docs/modules/README.md @@ -26,6 +26,7 @@ Counts and current placement: [../state.md](../state.md). | 10Gtek | `SFP-10G-SR` part number | (not in the test set) | - `allow_unsupported_sfp=1` is mandatory on the X520 — and it's the *truthful* module that requires it ([../nics/x520/](../nics/x520/README.md)). +- The X710 rejects the FS outright: i40e firmware disables Rx/Tx ("unsupported SFP module type", logged only at driver bind) and i40e has no override parameter — the FS can only live behind ixgbe. The rejection is silent at insertion time; a dead port with `ethtool` reporting no supported link modes is the symptom. - Physical-layer reasoning must use 10GBASE-T: PAM16, LDPC FEC, self-synchronizing scrambler, 4 twisted pairs, distance/temperature sensitive — never an optical model. - The module PHYs keep the copper link trained on their own: an admin `ip link set down` does NOT drop the wire unless the i40e `link-down-on-close` priv flag is set (peer sees the drop in ~200 ms, relinks in ~0.9 s). @@ -39,7 +40,7 @@ Proven on the CUX3610 (before the originals died) and the Fibergaga; the BCM not | PCS 3.32/3.33 | Block-lock loss, BER saturate, errored blocks | Latched, clear-on-read | | AN 7.33 | Link-partner status / master-slave | | | PMA 1.147 | Fast retrain status/count | Per-run quality metric | -| AN 7.0 bit 9 | Restart AN — forces retrain, refreshes SNR minimums | Proven safe everywhere. An `ip link` bounce does not drop the copper line, so it won't retrain | +| AN 7.0 | Restart AN (`\|= 0x1200`, enable + restart) — forces retrain, refreshes SNR minimums | Proven safe everywhere. Always force the enable bit too: the BCM ECD can leave 7.0.12 cleared ([fs/](fs/README.md)), and a bare restart preserves it. An `ip link` bounce does not drop the copper line, so it won't retrain | ## RollBall protocol (Marvell and Aquantia modules) @@ -49,7 +50,7 @@ Clause-45 access to the internal PHY over I2C A2h, all messages ≤ 4 B, offset- 2. Page-select: A2h 0x7F → 3. 3. Mailbox: A2h 0x80 (cmd) / 0x81 (data), poll for DONE (0x04). -Split transactions (separate STOP per byte) are field-proven on both the Fibergaga and the original Wiiteks; the kernel `mdio-i2c` combined form is not required by the modules. +The full mailbox map (0x80 CMD, 0x81 devad, 0x82/0x83 reg, 0x84/0x85 value) is vendor-confirmed in [wiitek/10gbase-t-phy-register-operation.md](wiitek/10gbase-t-phy-register-operation.md). Split transactions (separate STOP per byte) are field-proven on both the Fibergaga and the original Wiiteks; the kernel `mdio-i2c` combined form is not required by the modules. cabletest carries a Go client (`phy.go`): split ≤4-byte offset writes, single-hold compound reads, whitelist-guarded against the Wiitek brick space — proven live over the patched-ixgbe `sff_i2c` on the X520. ## Cable-length physics (applies to all vendors) diff --git a/docs/modules/fs/README.md b/docs/modules/fs/README.md index c5cdf05..12dfd27 100644 --- a/docs/modules/fs/README.md +++ b/docs/modules/fs/README.md @@ -33,11 +33,11 @@ Protocol and full verified catalog: [bcm84891l-mdio-commands.md](bcm84891l-mdio- | Capability | How | Observed | |---|---|---| -| Per-pair SNR | `CMD_GET_SNR` 0x8030, **invoked bare** (writing the documented DATA1 display flag returns zeros; IEEE 1.133–1.140 never populate — constant 0x8080) | DATA2–5 = SNR A–D ×0.1 dB; ≈ 27–32 dB absolute on bench, 0.1–0.4 dB jitter. 10GBASE-T operating point ≈ 26.5 dB, so margin ≈ value − 26.5. cabletest shows the margin classified green ≥ 3 dB / amber ≥ 1 dB / red below — provisional thresholds until the graded-noise correlation run. **Each training re-converges differently**: back-to-back retrains with identical roles, temperature and noise moved one pair by 3.6 dB (roles pinned, launch power pinned — the convergence itself is the variable, and the two bench modules differ in stability: ≤1.5 dB spread on one, 3.6 dB on the other). A single training's absolute SNR is a weak qualification number; qualification wants median-of-N trainings, live SNR is honest as a within-session tracker | +| Per-pair SNR | `CMD_GET_SNR` 0x8030, **invoked bare** (writing the documented DATA1 display flag returns zeros; IEEE 1.133–1.140 never populate — constant 0x8080) | DATA2–5 = SNR A–D ×0.1 dB; ≈ 27–32 dB absolute on bench, 0.1–0.4 dB jitter. 10GBASE-T operating point ≈ 26.5 dB, so margin ≈ value − 26.5 — a derived number that never proved actionable, so **cabletest does not poll it** (the panel's SNR comes from an IEEE-margin end — modules/wiitek/ — and each GET costs handler traffic on this µC). **Each training re-converges differently**: back-to-back retrains with identical roles, temperature and noise moved one pair by 3.6 dB (roles pinned, launch power pinned — the convergence itself is the variable, and the two bench modules differ in stability: ≤1.5 dB spread on one, 3.6 dB on the other). A single training's absolute SNR is a weak qualification number; qualification wants median-of-N trainings | | Die temperature | `CMD_GET_CURRENT_TEMP` 0x8031 | ~68–70 °C on bench | | Supply rails | `GET_CURRENT_VOLTAGE` 0x802F | 0.8 V and 1.88 V rails, tenths of mV | | Error counters | IEEE PCS 3.32/3.33 — block lock, latched errored-block/BER, clear-on-read | The noise-stress error proxy | -| Retrain on demand | IEEE AN restart `7.0 \|= bit9` | Relinks in ~5–6 s | +| Retrain on demand | IEEE AN restart `7.0 \|= 0x1200` (enable + restart — the ECD can leave the enable bit cleared, see below) | Relinks in ~5–6 s | ## Probed configuration states @@ -52,7 +52,7 @@ Protocol and full verified catalog: [bcm84891l-mdio-commands.md](bcm84891l-mdio- | Jumbo | 0x801C/0x801D | GET 0x801D answers in DATA1 — scratch-prone, unusable — and bringup cannot assume a cable to probe through, so cabletest forces enable every boot (SET 0x801C + AN restart); the running mix's 9018-byte frames are the standing wire truth (`phy.go`). The FS at its 9K option passes them at line rate with zero loss/corrupt | | 1588 | 0x8004 | Disabled (engine registers undocumented — see asks below) | | Limited reach | 0x8006 | Disabled — its DATA3 "linked in LR mode" bit, the only handler-visible trace of the DSP length estimate, is unavailable without a config change | -| Master/slave role | IEEE 7.32 bits 15/14 | Left to AN the resolution is a per-training lottery; manual config writes stick exactly and the resolution follows deterministically (7.33.14, no config fault with complementary ends). cabletest forces A master / B slave every boot (`phy.go`) so sessions are measured under identical conditions | +| Master/slave role | IEEE 7.32 bits 15/14 | Left to AN the resolution is a per-training lottery; manual config writes stick exactly and the resolution follows deterministically (7.33.14, no config fault with complementary ends). cabletest forces A master / B slave every boot when both ends are BCM; with a non-BCM partner (whose manual config is unreachable) the BCM is forced slave and the partner auto-resolves master — proven linking in ~3 s against the Wiitek (`phy.go`) | | Launch power | IEEE 1.130, 1.0.11 | 1.130 reads 0x0003 and is firmware-pinned: host writes to the short-reach bit revert instantly and the value never changes across retrains, so launch power is constant and not host-influenceable. PMA low-power bit clear. The handler catalog has no TX-power command | ## Firmware-reliability notes @@ -61,7 +61,7 @@ Protocol and full verified catalog: [bcm84891l-mdio-commands.md](bcm84891l-mdio- **The internal poll's clock, measured on hardware.** One poll every 3.49–4.22 s, regime-dependent but rock-stable within a regime (±15 ms): ≈3.50 s warm and idle, ≈3.70 s, ≈4.01 s cool and idle, ≈4.21 s under continuous host mailbox load — host traffic *defers* the poll, never advances it, and nothing observed brings two polls closer than 3.49 s. The poll is link-independent: it runs straight through an AN restart and the whole retrain without missing a beat. Around each poll's service the bridge serves stale data for ~50 ms: every read returns the last value successfully fetched before the µC went busy (stuck-at-last-fetch, not one-behind), which is exactly how 3.33 inherits 1.147's 0x0011. The window closes *before* the poll is detectable — a CMD read returning the fresh resident 0x0031 is itself proof the µC serviced the fetch, so flip detection doubles as the all-clear. -**The windowed protocol (`phy.go`) — contention resolved by time-division.** Since the poll cannot be silenced, the host schedules around it. One goroutine per module owns every transport touch (requests execute one at a time on the loop — nothing else can reach the wire, by construction), and each request is admitted only inside a fixed 3.4 s window following an observed poll (CMD flipping to resident 0x0031, polled at 10 ms); at the cutoff the host goes quiet until the next flip. 3.4 s sits below every observed cadence, so a regime switch mid-run lands harmlessly in vacated time — no prediction, no period tracking, just the last observed flip plus two constants. **Re-lock — no window at assumed phase**: edge detection needs CMD ≠ 0x0031, and normal work leaves it armed (every handler command rewrites CMD), but after a gap with no commands — a diag plus its relink wait, an idle stretch, a poll that skipped its GET because the link bit read down — the resident is 0x0031 and the phase is unknown. Admission then arms first (a bare GET_PAIR_SWAP, resident 0x0000) and anchors only on the true edge that follows, so every window is edge-locked from first contact at boot; bringup pays one phase acquisition per module (~2–4 s). A missing heartbeat free-runs one window rather than stall (never observed — the heartbeat survives retrains). Before re-lock the post-diag window anchored blind on the stale resident, and every ghost observed under windowing sat in exactly that window; with re-lock, ghost-free across all bench diags to date. A poisoned batch is self-consistent and forensically visible: the stuck value clears the link bit (SNR skipped, panel shows "-", the ghost-SNR panic sidestepped) while 3.33 charges the same value as errored blocks (0x0011 → the "+17"); every corrected charge and link-down reading prints its raw 1.1/3.33/1.147 on the console, so events self-attribute — a ghost names its neighbor register, real correction decodes as nothing but itself. Validated on hardware in `~/work/phydiag-work/bcm_phaselock_bench.py`: aiming read bursts at the poll reproduces the poisoning on 100% of polls (the "+17" manufactured on demand); windowed operation ran 2,881 back-to-back batches — 16× the production rate, riding the cutoff — with zero stale values, and the production 1 Hz stream is unperturbed (blackout hides inside natural gaps, worst sample gap ~1.6 s). +**The windowed protocol (`phy.go`) — contention resolved by time-division.** Since the poll cannot be silenced, the host schedules around it. One goroutine per module owns every transport touch (requests execute one at a time on the loop — nothing else can reach the wire, by construction), and each request is admitted only inside a fixed 3.4 s window following an observed poll (CMD flipping to resident 0x0031, polled at 10 ms); at the cutoff the host goes quiet until the next flip. 3.4 s sits below every observed cadence, so a regime switch mid-run lands harmlessly in vacated time — no prediction, no period tracking, just the last observed flip plus two constants. **Re-lock — no window at assumed phase**: edge detection needs CMD ≠ 0x0031, and handler commands leave it armed (every one rewrites CMD); whenever CMD sits at the resident 0x0031 at expiry the phase is unknown and admission arms first, then anchors on the true edge that follows. Steady-state polling issues no handler commands of its own (the SNR GET is dropped), so each taken edge immediately re-arms with a bare GET_PAIR_SWAP: the single CMD write per window lands at the start of the quiet period, maximally far from the next poll — the wedge risk is writes near the poll — and every later expiry reads the phase without writing. One command per ~4 s window replaces a handler GET per second. Admission then arms first (a bare GET_PAIR_SWAP, resident 0x0000) and anchors only on the true edge that follows, so every window is edge-locked from first contact at boot; bringup pays one phase acquisition per module (~2–4 s). A missing heartbeat free-runs one window rather than stall (never observed — the heartbeat survives retrains). Before re-lock the post-diag window anchored blind on the stale resident, and every ghost observed under windowing sat in exactly that window; with re-lock, ghost-free across all bench diags to date. A poisoned batch is self-consistent and forensically visible: the stuck value clears the link bit (SNR skipped, panel shows "-", the ghost-SNR panic sidestepped) while 3.33 charges the same value as errored blocks (0x0011 → the "+17"); every corrected charge and link-down reading prints its raw 1.1/3.33/1.147 on the console, so events self-attribute — a ghost names its neighbor register, real correction decodes as nothing but itself. Validated on hardware in `~/work/phydiag-work/bcm_phaselock_bench.py`: aiming read bursts at the poll reproduces the poisoning on 100% of polls (the "+17" manufactured on demand); windowed operation ran 2,881 back-to-back batches — 16× the production rate, riding the cutoff — with zero stale values, and the production 1 Hz stream is unperturbed (blackout hides inside natural gaps, worst sample gap ~1.6 s). ## ECD — recovered from the OpenBCM SDK, proven on hardware @@ -76,6 +76,7 @@ The ECD register mechanism is absent from the handler catalog and the datasheet Observed on a plugged, linked, healthy cable: verdicts OK×4 and per-pair lengths of [45, 45, 41, 46] on a ~45 m cable — **meter-accurate with no calibration**, and this ECD reports length for healthy pairs, not just faults, resolving the terminated-far-end concern in [../README.md](../README.md). Caveats: - The run blips the link (PMA 1.1 latch-low catches a drop even with the break-link bit clear) — do not run mid-measurement until the disturbance is characterized. +- **The run can leave clause-45 AN disabled** (7.0.12 cleared — observed live after diags racing an in-progress retrain): with AN off the BCM emits no AN pulses, both ends report no link partner (7.1 LP-AN-able 0), and the link stays down through AN restarts, role changes and PMA resets until the bit is re-enabled. cabletest restarts AN with `7.0 |= 0x1200` (enable + restart) after every diag, and `devRestartAN` always forces the enable bit — a bare bit-9 restart preserves the cleared enable forever. - **An interrupted run can wedge the µC's SMI service** (observed after a process died mid-diag): the bridge ACKs I2C but serves 0x0000 for every register at any delay and silently drops MDIO writes — no in-band recovery, not even PMA reset — while the EEPROM path and the trained link keep working. Recovery: a driver unbind/rebind of the port (the SFP re-initialization resets the µC); a reseat would do the same. - Fault verdicts (open/short/inter-pair) are unexercised — deliberately: the product is a closed-loop tester, both ends always plugged. - Family constraints from the SDK: port must be enabled; unsupported at forced 100M. diff --git a/docs/modules/wiitek/README.md b/docs/modules/wiitek/README.md index b631ca7..69ff81c 100644 --- a/docs/modules/wiitek/README.md +++ b/docs/modules/wiitek/README.md @@ -21,14 +21,23 @@ The shared IEEE 802.3an set ([../README.md](../README.md)) plus two safe vendor | Registers | What | Reading | |---|---|---| -| PMA 1.133–1.136 / 1.137–1.140 | Per-pair SNR margin (offset-0x8000, 0.1 dB) | [8.8, 7.0, 7.3, 8.9] dB | +| PMA 1.133–1.136 / 1.137–1.140 | Per-pair SNR margin (offset-0x8000, 0.1 dB) | [8.8, 7.0, 7.3, 8.9] dB (as auto-master vs FS manual-slave: [9.2, 7.2, 7.0, 8.7]) | | PCS 3.32 / 3.33 | Errored-block / block-lock latches | 0x1001 / 0x7f00 | -| AN 7.33 | Master/slave, link partner | 0x3c62 | +| AN 7.33 | Master/slave, link partner | 0x3c62 (as master against the FS: 0x7c62) | | 1.0xC011 / 12 | Firmware version | 0x0c01 / 0x0500 (12.1.5.0) | | 1.0xC050 | Boot status | 0x087e | +| PMA 1.0 | Speed selection | 0x2040 (10G) — see the reset trap below | +| AN 7.0 / 7.1 / 7.32 / 7.60 | AN control/status, 10GBASE-T control, EEE advert | 0x3000 / 0x0008 idle · 0x00ad linked / 0x11e1 (auto role, 10G advert) / 0x0000 (no EEE advertised) | +| A2.250 (Table 3) | PHY part number ([10gbase-t-phy-register-operation.md](10gbase-t-phy-register-operation.md)) | 9 on the CUX3610 (1 is documented as 88X3310) | + +Proven-safe writes, all IEEE standard space: AN restart `7.0 |= 0x1200`, and PMA reset `1.0 = 0x8000` — **the PMA reset clears the speed-selection bits** (1.0 reads 0x0000 after, no speed selected, 10G training impossible) and does not touch 7.x config; write 1.0 = 0x2040 back after any reset or the link never returns. `rb_forensic.py` in phydiag-work performs this baseline and logs every sff_i2c op (PRE line fsync'd before each op, POST after) to `wiitek-ops.log` — a PRE with no POST names whatever killed the µC. `rollball_ixgbe.py` hard-guards the `*.0x??64` blacklist and raises before touching hardware. +## In the test set — mixed pair with the FS + +cabletest drives this module through its own Go RollBall client (`phy.go`): module type dispatched from the EEPROM vendor PN before any protocol touch, mailbox ops as split ≤4-byte offset writes plus single-hold compound reads over the patched-ixgbe `sff_i2c`, and a **whitelist guard** — every MDIO read/write is checked against the proven-safe set above and panics before touching hardware on anything else, so the client structurally cannot repeat the kill. The 1 Hz poller reads link (1.1), per-pair IEEE margins (1.133–136, used directly — no operating-point subtraction), corrected latches (3.33) and fast-retrain count (1.147); margins ride ghost bounds of −10…+25 dB. **This module's IEEE margins are the panel's only SNR source** — the FS's handler SNR is not polled at all (the derived number was never actionable and each GET costs BCM handler traffic), so an all-BCM pair shows no SNR. Roles in the mixed pair: the FS is forced manual-slave and this module auto-resolves master (its 7.32 reads auto and cannot be safely forced) — links in ~3 s. EEE stays down one-sided (this unit advertises none; the FS forces none). Module-level jumbo cannot be forced here; the running mix's 9018-byte frames pass at line rate regardless. Length comes from the FS end's ECD with this module as the far end: healthy per-pair verdicts and lengths within a few meters of the FS↔FS baseline on the same cable. + Not available in safe space: pre-FEC counter, cable length — those live in the µC vendor space that killed the modules. ## VCT/cable length: no confirmed-safe path diff --git a/docs/state.md b/docs/state.md index b3decc0..3db8028 100644 --- a/docs/state.md +++ b/docs/state.md @@ -2,7 +2,7 @@ ## Committed tree -AF_PACKET raw sockets everywhere (`sock.go`); flow-director steering; per-packet-MAC-rx-stamped rate buckets (`SO_TIMESTAMPING` cmsg, `rx_filter=ALL` as a hard host check — nics/README.md for what that demands of the NIC; **temporarily bypassed** in `ts.go` so BCM work can run on the X520, which cannot stamp — the check reports yellow and the panel rates read zero there; restore to fatal for the product NIC); read-time-stamped NIC-counter rates; test interfaces pinned to MTU 9000 with a 9018-byte jumbo in the size mix (the modules' jumbo path is exercised, not assumed); BCM module diagnostics (`phy.go`, over the patched-ixgbe `sff_i2c` debugfs, compound-op framing; one loop goroutine per module owns every transport touch, and all work is time-division-multiplexed with the firmware's own mailbox client — each request admitted only inside a 3.4 s edge-locked window after an observed internal temp poll, re-acquiring phase whenever it was lost — which ends the stale-read poisoning of the corrected channel; corrected charges and link-down reads print their raw registers so events self-attribute — modules/fs/ for the full trap list and validation): bringup identifies both modules and forces EEE off, jumbo on and the master/slave roles (A master / B slave — left to AN the role is a per-training lottery) every boot — no trustworthy readback exists and no cable is guaranteed to probe through; the ECD — per-pair verdicts, lengths and pair maps are the length/wiring path — runs through one async path at startup and on every reset, never blocking the UI, with counters re-baselining only after the diag's own link blip so it is never charged to the run; a 1 Hz poller feeds per-pair SNR margin (vs the ≈26.5 dB operating point; green ≥ 3 dB, amber ≥ 1 dB — provisional until the graded-noise run) and the corrected-error set (PCS 3.33 errored blocks/BER, PMA 1.147 fast-retrain count) to the panel and console; framebuffer UI; harness. +AF_PACKET raw sockets everywhere (`sock.go`); flow-director steering; per-packet-MAC-rx-stamped rate buckets (`SO_TIMESTAMPING` cmsg, `rx_filter=ALL` as a hard host check — nics/README.md for what that demands of the NIC; **temporarily bypassed** in `ts.go` so BCM work can run on the X520, which cannot stamp — the check reports yellow and the panel rates read zero there; restore to fatal for the product NIC); read-time-stamped NIC-counter rates; test interfaces pinned to MTU 9000 with a 9018-byte jumbo in the size mix (the modules' jumbo path is exercised, not assumed); module diagnostics (`phy.go`, over the patched-ixgbe `sff_i2c` debugfs; module type dispatched from the EEPROM vendor PN before any protocol touch, with a BCM client — compound-op framing, one loop goroutine per module owning every transport touch, all work time-division-multiplexed with the firmware's own mailbox client: each request admitted only inside a 3.4 s edge-locked window after an observed internal temp poll, re-acquiring phase whenever it was lost, which ends the stale-read poisoning of the corrected channel — and a RollBall client for the Wiitek — unlock/page/mailbox in Go, whitelist-guarded so any register outside the proven-safe set panics before touching hardware; corrected charges and link-down reads print their raw registers so events self-attribute — modules/fs/ and modules/wiitek/ for the trap lists and validation): bringup identifies both modules and forces EEE off, jumbo on and the master/slave roles on every BCM end every boot (A master / B slave when both ends are BCM; with a non-BCM partner the BCM is forced slave and the partner auto-resolves master — the proven mixed combination) — no trustworthy readback exists and no cable is guaranteed to probe through — and every AN restart forces the enable bit (`7.0 |= 0x1200`: the ECD can leave 7.0.12 cleared, which mutes AN entirely and holds the link down through restarts and resets), with an explicit AN restart on the ECD end after every diag; the ECD — per-pair verdicts, lengths and pair maps are the length/wiring path — runs through one async path at startup and on every reset, never blocking the UI, with counters re-baselining only after the diag's own link blip so it is never charged to the run; a 1 Hz poller feeds per-pair SNR margin and the corrected-error set (PCS 3.33 errored blocks/BER, PMA 1.147 fast-retrain count) to the panel and console — SNR comes only from ends that measure IEEE margins (the Wiitek's 1.133–136, the reason it is in the pair); the BCM's handler SNR is not polled (the derived number was never actionable and each GET costs handler traffic), so an all-BCM pair shows no SNR; green ≥ 3 dB, amber ≥ 1 dB — provisional until the graded-noise run; framebuffer UI; harness. ## Stashes @@ -17,20 +17,20 @@ AF_PACKET raw sockets everywhere (`sock.go`); flow-director steering; per-packet | Interface | Device | Role (rules: hardware.md) | |---|---|---| -| `enp1s0f0` | X520 port 0 (ixgbe) | Test pair — FS module | -| `enp1s0f1` | X520 port 1 (ixgbe) | Test pair — FS module | -| `enp3s0f0np0` / `enp3s0f1np1` | X710 (i40e) | Noise pair (has been `enp4s0f*` across reboots) | +| `enp1s0f0` | X520 port 0 (ixgbe) | Test pair — Wiitek module (SNR end) | +| `enp1s0f1` | X520 port 1 (ixgbe) | Test pair — FS module (length end) | +| `enp3s0f0np0` / `enp3s0f1np1` | X710 (i40e) | Noise pair (has been `enp4s0f*` across reboots). **The FS cannot live here**: X710 firmware disables Rx/Tx on the honest FS EEPROM ("unsupported SFP module type") and i40e has no `allow_unsupported_sfp` — the noise pair needs lying modules. Seated: Fibergaga (f0) + Wiitek SN …X170U (f1), both qualified, noise cycle live | | `enp88s0` | igc | LAN uplink, default route; sibling `enp89s0` is dark | ## Hardware | Item | Status | Notes | |---|---|---| -| X520-DA2 | **Installed** in the single PCIe slot (E810 out); PCIe 5 GT/s ×8 | Both ports FS (port 1 SN S2433774168); cable linked at 10G. Stock ixgbe needs `allow_unsupported_sfp=1` — the *FS* trips qualification (hardware.md) | +| X520-DA2 | **Installed** in the single PCIe slot (E810 out); PCIe 5 GT/s ×8 | Mixed test pair: port 0 Wiitek (SN WAMZ012606X039U), port 1 FS (SN S2433774168); cable linked at 10G. Stock ixgbe needs `allow_unsupported_sfp=1` — the *FS* trips qualification (hardware.md) | | ConnectX-5 | **Ordered** (dual SFP28, PCIe x8) — the product NIC candidate | mlx5 is the one driver meeting the full requirement set: stamps every packet, shared PHC across ports, native ETHER_FLOW steering. Open: MCIA diagnostics questions (nics/connectx-5/). Arrival notes: ports may ship in InfiniBand mode (`mlxconfig set LINK_TYPE_P1=2 LINK_TYPE_P2=2`); SFP+ drops into SFP28 cages at 10G; check `mlx5_ib` vs channel changes (the irdma lesson) | -| Replacement Wiiteks | Arrived; on the shelf | Originals bricked by register exploration — modules/wiitek/ trap first | -| FS SFP-10G-T-100 ×2 | Both in the X520 test pair | BCM84891L, documented, robust. **2× FS at both ends is the expected product module config** unless mixed ends prove wanted | -| Fibergaga SFP-10G-T-30M | In hand | Aquantia, RollBall, the documented oracle | +| Replacement Wiiteks | One in the X520 test pair (port 0), rest on the shelf | Originals bricked by register exploration — modules/wiitek/ trap first | +| FS SFP-10G-T-100 ×2 | One in the X520 test pair (port 1), one on the shelf (rejected by the X710 — ixgbe-only) | BCM84891L, documented, robust. The mixed FS+Wiitek pair is the configuration under evaluation: length from the FS ECD, IEEE per-pair SNR margins from the Wiitek — the best of both worlds a 2× FS pair cannot give (the BCM never populates IEEE SNR) | +| Fibergaga SFP-10G-T-30M | In the X710 noise pair (f0) | Aquantia, RollBall, the documented oracle | | 10Gtek | In hand | Claims SFP-10G-SR, still copper RJ45; filler, not in the test set | | E810 | Out of the box | Patched ice + `sff_i2c` remains useful only if it returns for read-side work | diff --git a/phy.go b/phy.go index 24f6bfc..2c4f53c 100644 --- a/phy.go +++ b/phy.go @@ -29,7 +29,6 @@ const ( bcmCmdGetPairSwap uint16 = 0x8000 bcmCmdSetEEEMode uint16 = 0x8009 bcmCmdSetJumbo uint16 = 0x801C - bcmCmdGetSNR uint16 = 0x8030 bcmRegECDCtrl uint16 = 0x4006 bcmRegECDResult uint16 = 0xA896 @@ -56,6 +55,9 @@ const ( ecdDeadline = 50 * time.Second pairIdentityMap = 0xE4 + + fsVendorPN = "SFP-10G-T-100" + wiitekVendorPN = "UF-RJ45-10G-100" ) const ( @@ -69,33 +71,209 @@ var pairVerdicts = map[int]string{ pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK", } -type bcm struct { +type sff struct { ifname string path string // Every transport touch happens on the loop goroutine: requests execute - // one at a time, each admitted into the quiet window first. - reqs chan func() - windowEnd time.Time + // one at a time, each admitted by the module type's own gate first. + reqs chan func() + admit func() } -func (b *bcm) loop() { +func openSFF(ifname string) (*sff, error) { + devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device") + if err != nil { + return nil, fmt.Errorf("%s: %w", ifname, err) + } + drv, err := ifDriver(ifname) + if err != nil { + return nil, fmt.Errorf("%s: %w", ifname, err) + } + if drv != "ixgbe" { + return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv) + } + s := &sff{ + ifname: ifname, + path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c", + reqs: make(chan func()), + admit: func() {}, + } + if _, err := os.Stat(s.path); err != nil { + return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err) + } + go s.loop() + return s, nil +} + +func (s *sff) loop() { defer holdPanic() - for fn := range b.reqs { - b.window() + for fn := range s.reqs { + s.admit() fn() } } -func (b *bcm) exec(fn func()) { +func (s *sff) exec(fn func()) { done := make(chan struct{}) - b.reqs <- func() { fn(); close(done) } + s.reqs <- func() { fn(); close(done) } <-done } +func (s *sff) name() string { return s.ifname } + +func (s *sff) op(cmd string) (string, error) { + fd, err := unix.Open(s.path, unix.O_RDWR, 0) + if err != nil { + return "", fmt.Errorf("%s: %w", s.path, err) + } + defer unix.Close(fd) + if _, err := unix.Write(fd, []byte(cmd)); err != nil { + return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err) + } + buf := make([]byte, 256) + n, err := unix.Read(fd, buf) + if err != nil { + return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err) + } + resp := strings.TrimSpace(string(buf[:n])) + if !strings.HasPrefix(resp, "ok") { + return "", fmt.Errorf("%s %q: %s", s.ifname, cmd, resp) + } + return strings.TrimSpace(resp[2:]), nil +} + +func parseHexBytes(s string, n int) ([]byte, error) { + fields := strings.Fields(s) + if len(fields) != n { + return nil, fmt.Errorf("want %d bytes, got %q", n, s) + } + out := make([]byte, n) + for i, f := range fields { + var v byte + if _, err := fmt.Sscanf(f, "%x", &v); err != nil { + return nil, fmt.Errorf("byte %q in %q", f, s) + } + out[i] = v + } + return out, nil +} + +// A single bus hold; split write/read ops would let the driver's own SFP +// traffic consume the bridge's pending read. +func (s *sff) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) { + var sb strings.Builder + fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n) + for _, v := range wdata { + fmt.Fprintf(&sb, " %02x", v) + } + resp, err := s.op(sb.String()) + if err != nil { + return nil, err + } + return parseHexBytes(resp, n) +} + +func (s *sff) eeprom(off byte, n int) ([]byte, error) { + return s.compound(0xA0, 0xA1, 500, n, []byte{off}) +} + +func (s *sff) vendorPN() (string, error) { + pn, err := s.eeprom(40, 16) + if err != nil { + return "", err + } + return strings.TrimSpace(string(pn)), nil +} + +type mdioDev interface { + name() string + exec(func()) + mdioRead(devad, reg uint16) (uint16, error) + mdioWrite(devad, reg, val uint16) error +} + +type phyDev interface { + mdioDev + identify() (string, error) +} + +type snrSource interface { + snrMargins() ([4]float64, error) +} + +// PMA 1.1 latches low, so the first read reports any drop since it was last +// read and the second reports the wire as it is now. +func devLinkUp(d mdioDev) (up bool, raw uint16, err error) { + d.exec(func() { + if _, err = d.mdioRead(1, 1); err != nil { + return + } + if raw, err = d.mdioRead(1, 1); err != nil { + return + } + up = raw&0x0004 != 0 + }) + return +} + +func devPCSLatch(d mdioDev) (blocks, ber uint64, raw uint16, err error) { + d.exec(func() { + if raw, err = d.mdioRead(3, 33); err != nil { + return + } + blocks, ber = uint64(raw&0xFF), uint64((raw>>8)&0x3F) + }) + return +} + +func devFastRetrain(d mdioDev) (count, raw uint16, err error) { + d.exec(func() { + if raw, err = d.mdioRead(1, 147); err != nil { + return + } + count = raw >> 11 + }) + return +} + +// AN enable is forced alongside the restart: the ECD can leave the BCM with +// 7.0.12 cleared (proven live — no AN pulses, both ends deaf, link down until +// power cycle), and a bare restart bit preserves the cleared enable. +func devRestartAN(d mdioDev) (err error) { + d.exec(func() { + var v uint16 + if v, err = d.mdioRead(7, 0); err != nil { + return + } + err = d.mdioWrite(7, 0, v|0x1200) + }) + return +} + +func devEEEAdvert(d mdioDev) (v uint16, err error) { + d.exec(func() { v, err = d.mdioRead(7, 60) }) + return +} + +type bcm struct { + *sff + windowEnd time.Time +} + +func newBCM(t *sff) *bcm { + b := &bcm{sff: t} + t.exec(func() { t.admit = b.window }) + return b +} + // The firmware's internal temp poll serves stale bridge reads for ~50 ms // around it; work stays inside 3.4 s of an observed poll. A resident 0x0031 at // expiry means the phase is unknown, so re-lock: arm, then take the true edge. +// Each taken edge immediately re-arms — the one CMD write per window lands at +// the start of the quiet period, maximally far from the next poll (writes near +// the poll are the µC-wedge risk), and every later expiry reads the phase +// without writing. func (b *bcm) window() { if time.Now().Add(bcmWindowFit).Before(b.windowEnd) { return @@ -110,6 +288,7 @@ func (b *bcm) window() { if v == bcmCmdResidentTemp { if armed { b.windowEnd = time.Now().Add(bcmWindow) + b.rearm() return } b.rearm() @@ -142,82 +321,6 @@ func (b *bcm) rearm() { } } -func openBCM(ifname string) (*bcm, error) { - devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device") - if err != nil { - return nil, fmt.Errorf("%s: %w", ifname, err) - } - drv, err := ifDriver(ifname) - if err != nil { - return nil, fmt.Errorf("%s: %w", ifname, err) - } - if drv != "ixgbe" { - return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv) - } - b := &bcm{ - ifname: ifname, - path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c", - reqs: make(chan func()), - } - if _, err := os.Stat(b.path); err != nil { - return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err) - } - go b.loop() - return b, nil -} - -func (b *bcm) op(cmd string) (string, error) { - fd, err := unix.Open(b.path, unix.O_RDWR, 0) - if err != nil { - return "", fmt.Errorf("%s: %w", b.path, err) - } - defer unix.Close(fd) - if _, err := unix.Write(fd, []byte(cmd)); err != nil { - return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err) - } - buf := make([]byte, 256) - n, err := unix.Read(fd, buf) - if err != nil { - return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err) - } - resp := strings.TrimSpace(string(buf[:n])) - if !strings.HasPrefix(resp, "ok") { - return "", fmt.Errorf("%s %q: %s", b.ifname, cmd, resp) - } - return strings.TrimSpace(resp[2:]), nil -} - -func parseHexBytes(s string, n int) ([]byte, error) { - fields := strings.Fields(s) - if len(fields) != n { - return nil, fmt.Errorf("want %d bytes, got %q", n, s) - } - out := make([]byte, n) - for i, f := range fields { - var v byte - if _, err := fmt.Sscanf(f, "%x", &v); err != nil { - return nil, fmt.Errorf("byte %q in %q", f, s) - } - out[i] = v - } - return out, nil -} - -// A single bus hold; split write/read ops would let the driver's own SFP -// traffic consume the bridge's pending read. -func (b *bcm) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) { - var sb strings.Builder - fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n) - for _, v := range wdata { - fmt.Fprintf(&sb, " %02x", v) - } - resp, err := b.op(sb.String()) - if err != nil { - return nil, err - } - return parseHexBytes(resp, n) -} - func (b *bcm) mdioReadDelay(devad, reg uint16, delayUs int) (uint16, error) { d, err := b.compound(bcmI2CWrite, bcmI2CRead, delayUs, 2, []byte{0x20 | byte(devad), byte(reg >> 8), byte(reg)}) @@ -243,10 +346,6 @@ func (b *bcm) mdioWrite(devad, reg, val uint16) error { return err } -func (b *bcm) eeprom(off byte, n int) ([]byte, error) { - return b.compound(0xA0, 0xA1, 500, n, []byte{off}) -} - // The datasheet's completion handshake: poll STATUS on its 100 ms cadence // until the wanted state, bounded by a deadline. func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) { @@ -329,21 +428,6 @@ func (b *bcm) identify() (ident string, err error) { return } -// PMA 1.1 latches low, so the first read reports any drop since it was last -// read and the second reports the wire as it is now. -func (b *bcm) linkUp() (up bool, raw uint16, err error) { - b.exec(func() { - if _, err = b.mdioRead(1, 1); err != nil { - return - } - if raw, err = b.mdioRead(1, 1); err != nil { - return - } - up = raw&0x0004 != 0 - }) - return -} - func (b *bcm) forceEEEOff() error { _, err := b.command(bcmCmdSetEEEMode, 0x0000, 0x0000, 0x7A12, 0x0480, 0x0000) return err @@ -374,22 +458,6 @@ func (b *bcm) forceRole(master bool) (err error) { return } -func (b *bcm) restartAN() (err error) { - b.exec(func() { - var v uint16 - if v, err = b.mdioRead(7, 0); err != nil { - return - } - err = b.mdioWrite(7, 0, v|0x0200) - }) - return -} - -func (b *bcm) eeeAdvert() (v uint16, err error) { - b.exec(func() { v, err = b.mdioRead(7, 60) }) - return -} - func (b *bcm) pairMap() (byte, error) { d, err := b.command(bcmCmdGetPairSwap) if err != nil { @@ -398,81 +466,172 @@ func (b *bcm) pairMap() (byte, error) { return byte(d[1]), nil } -func (b *bcm) snr() ([4]float64, error) { - var out [4]float64 - d, err := b.command(bcmCmdGetSNR) +const ( + rbI2CWrite = 0xA2 + rbI2CRead = 0xA3 + + rbOffPassword byte = 0x7B + rbOffPage byte = 0x7F + rbOffCmd byte = 0x80 + rbOffDevad byte = 0x81 + rbOffValHi byte = 0x84 + rbOffPartNum byte = 0xFA + + rbPageMailbox byte = 3 + + rbCmdWrite byte = 0x01 + rbCmdRead byte = 0x02 + rbCmdDone byte = 0x04 + + rbReadDelayUs = 500 + rbCmdPoll = 20 * time.Millisecond + // Matches the BCM allowance for a handler frozen by 10GBASE-T training. + rbCmdTimeout = 3 * time.Second + + rbPHYIDHi uint16 = 0x002B + rbPHYIDLo uint16 = 0x0BF4 + + // IEEE margins land near 7-9 dB on a healthy short cable; far outside is + // another register's data. + rbGhostLow = -10.0 + rbGhostHigh = 25.0 +) + +// Only the registers proven safe on this PHY (docs/modules/wiitek/): single +// reads in the vendor windows brick the µC permanently, so everything else +// refuses before touching hardware. +var rbReadSafe = map[uint16]map[uint16]bool{ + 1: {1: true, 2: true, 3: true, 133: true, 134: true, 135: true, 136: true, 147: true}, + 3: {32: true, 33: true}, + 7: {0: true, 33: true, 60: true}, +} + +var rbWriteSafe = map[uint16]map[uint16]bool{ + 7: {0: true}, +} + +type rollball struct { + *sff +} + +func (r *rollball) i2cWrite(off byte, data ...byte) error { + var sb strings.Builder + fmt.Fprintf(&sb, "w %02x %02x", rbI2CWrite, off) + for _, v := range data { + fmt.Fprintf(&sb, " %02x", v) + } + _, err := r.op(sb.String()) + return err +} + +func (r *rollball) i2cRead(off byte, n int) ([]byte, error) { + return r.compound(rbI2CWrite, rbI2CRead, rbReadDelayUs, n, []byte{off}) +} + +func (r *rollball) unlock() error { + if err := r.i2cWrite(rbOffPage, rbPageMailbox); err != nil { + return err + } + return r.i2cWrite(rbOffPassword, 0xFF, 0xFF, 0xFF, 0xFF) +} + +func (r *rollball) mbox(cmd byte, devad, reg, val uint16) error { + if err := r.unlock(); err != nil { + return err + } + if err := r.i2cWrite(rbOffDevad, byte(devad), byte(reg>>8), byte(reg)); err != nil { + return err + } + if cmd == rbCmdWrite { + if err := r.i2cWrite(rbOffValHi, byte(val>>8), byte(val)); err != nil { + return err + } + } + if err := r.i2cWrite(rbOffCmd, cmd); err != nil { + return err + } + deadline := time.Now().Add(rbCmdTimeout) + for { + d, err := r.i2cRead(rbOffCmd, 1) + if err != nil { + return err + } + if d[0] == rbCmdDone { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("%s: mailbox %d.%#04x stuck at %#02x", r.ifname, devad, reg, d[0]) + } + time.Sleep(rbCmdPoll) + } +} + +func rbGuard(safe map[uint16]map[uint16]bool, ifname, what string, devad, reg uint16) { + if !safe[devad][reg] { + panic(fmt.Sprintf("%s: refusing MDIO %s %d.%#04x: outside the proven-safe set", + ifname, what, devad, reg)) + } +} + +func (r *rollball) mdioRead(devad, reg uint16) (uint16, error) { + rbGuard(rbReadSafe, r.ifname, "read", devad, reg) + if err := r.mbox(rbCmdRead, devad, reg, 0); err != nil { + return 0, err + } + d, err := r.i2cRead(rbOffValHi, 2) if err != nil { - return out, err + return 0, err } - for i := range out { - out[i] = float64(d[i+1]) / 10 - } - return out, nil + return uint16(d[0])<<8 | uint16(d[1]), nil } -func (b *bcm) pcsLatch() (blocks, ber uint64, raw uint16, err error) { - b.exec(func() { - if raw, err = b.mdioRead(3, 33); err != nil { +func (r *rollball) mdioWrite(devad, reg, val uint16) error { + rbGuard(rbWriteSafe, r.ifname, "write", devad, reg) + return r.mbox(rbCmdWrite, devad, reg, val) +} + +func (r *rollball) identify() (ident string, err error) { + r.exec(func() { + var hi, lo uint16 + if hi, err = r.mdioRead(1, 2); err != nil { return } - blocks, ber = uint64(raw&0xFF), uint64((raw>>8)&0x3F) + if lo, err = r.mdioRead(1, 3); err != nil { + return + } + if hi != rbPHYIDHi || lo != rbPHYIDLo { + err = fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x", + r.ifname, hi, lo, rbPHYIDHi, rbPHYIDLo) + return + } + if err = r.unlock(); err != nil { + return + } + var part []byte + if part, err = r.i2cRead(rbOffPartNum, 1); err != nil { + return + } + var sn []byte + if sn, err = r.eeprom(68, 16); err != nil { + return + } + ident = fmt.Sprintf("CUX3610 sn %s (A2.250=%d)", strings.TrimSpace(string(sn)), part[0]) }) return } -func (b *bcm) fastRetrainCount() (count, raw uint16, err error) { - b.exec(func() { - if raw, err = b.mdioRead(1, 147); err != nil { - return - } - count = raw >> 11 - }) - return -} - -type ecdResult struct { - verdicts [4]int - metres [4]int -} - -func (b *bcm) cableDiag() (res ecdResult, err error) { - b.exec(func() { - var ctrl uint16 - if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil { - return - } - if err = b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil { - return - } - deadline := time.Now().Add(ecdDeadline) - for { - if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil { +func (r *rollball) snrMargins() (out [4]float64, err error) { + r.exec(func() { + for i := range out { + var v uint16 + if v, err = r.mdioRead(1, uint16(133+i)); err != nil { return } - if ctrl&0x0800 == 0 { - break + m := (float64(v) - 0x8000) / 10 + if m < rbGhostLow || m > rbGhostHigh { + panic(fmt.Sprintf("%s: ghost SNR margin %.1f dB (1.%d=%#04x)", r.ifname, m, 133+i, v)) } - if time.Now().After(deadline) { - err = fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline) - return - } - time.Sleep(ecdPoll) - } - b.window() - var v uint16 - if v, err = b.mdioRead(1, bcmRegECDResult); err != nil { - return - } - for i := range res.verdicts { - res.verdicts[i] = int(v>>(4*i)) & 0xF - if res.verdicts[i] > pairXtalk { - panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v)) - } - var m uint16 - if m, err = b.mdioRead(1, bcmRegECDLen+uint16(i)); err != nil { - return - } - res.metres[i] = int(m) + out[i] = m } }) return @@ -485,18 +644,12 @@ const ( linkWaitSpan = 25 * time.Second linkWaitPoll = time.Second - snrOperatingPoint = 26.5 - snrGoodMargin = 3.0 - snrWarnMargin = 1.0 - - // No trained link produces SNR outside this; readings there are another - // register's data (die temp ≈ 8, handler status ≈ 0.4) and the run dies. - snrGhostLow = 15.0 - snrGhostHigh = 50.0 + snrGoodMargin = 3.0 + snrWarnMargin = 1.0 ) type phyModule struct { - bcm *bcm + dev phyDev busy atomic.Bool mu sync.Mutex @@ -504,7 +657,7 @@ type phyModule struct { lastOK time.Time link bool haveSNR bool - snr [4]float64 + margins [4]float64 blocks uint64 ber uint64 retrains uint64 @@ -519,26 +672,23 @@ func (m *phyModule) poll() error { if m.busy.Load() { return nil } - link, linkRaw, err := m.bcm.linkUp() + link, linkRaw, err := devLinkUp(m.dev) if err != nil { return err } - var snr [4]float64 - if link { - if snr, err = m.bcm.snr(); err != nil { + var margins [4]float64 + haveSNR := false + if src, ok := m.dev.(snrSource); ok && link { + if margins, err = src.snrMargins(); err != nil { return err } - for _, s := range snr { - if s < snrGhostLow || s > snrGhostHigh { - panic(fmt.Sprintf("%s: ghost SNR %.1f dB", m.bcm.ifname, s)) - } - } + haveSNR = true } - blocks, ber, pcsRaw, err := m.bcm.pcsLatch() + blocks, ber, pcsRaw, err := devPCSLatch(m.dev) if err != nil { return err } - count, frRaw, err := m.bcm.fastRetrainCount() + count, frRaw, err := devFastRetrain(m.dev) if err != nil { return err } @@ -546,13 +696,13 @@ func (m *phyModule) poll() error { m.mu.Lock() if m.link && !link { m.notes = append(m.notes, - fmt.Sprintf("%s link read down: 1.1=0x%04x", m.bcm.ifname, linkRaw)) + fmt.Sprintf("%s link read down: 1.1=0x%04x", m.dev.name(), linkRaw)) } m.sampled = true m.lastOK = time.Now() m.link = link - m.haveSNR = link - m.snr = snr + m.haveSNR = haveSNR + m.margins = margins // The first poll after a baseline drains latches from the bringup/diag // retrain era, so it only sets the origin; the retrain counter is 5 bits. if m.primed { @@ -561,7 +711,7 @@ func (m *phyModule) poll() error { if delta > 0 { m.notes = append(m.notes, fmt.Sprintf( "%s corrected +%d raw: 3.33=0x%04x (blocks %d ber %d) 1.147=0x%04x (retrain +%d) 1.1=0x%04x", - m.bcm.ifname, delta, pcsRaw, blocks, ber, frRaw, rt, linkRaw)) + m.dev.name(), delta, pcsRaw, blocks, ber, frRaw, rt, linkRaw)) } m.blocks += blocks m.ber += ber @@ -614,6 +764,7 @@ func (m *phyModule) reset() { type phyModView struct { fresh bool link bool + haveSNR bool margins [4]float64 blocks uint64 ber uint64 @@ -626,7 +777,9 @@ func (m *phyModule) view() phyModView { defer m.mu.Unlock() v := phyModView{ fresh: m.sampled && time.Since(m.lastOK) < phyStale, - link: m.link && m.haveSNR, + link: m.link, + haveSNR: m.haveSNR, + margins: m.margins, blocks: m.blocks, ber: m.ber, retrain: m.retrains, @@ -634,15 +787,13 @@ func (m *phyModule) view() phyModView { if v.fresh { v.recent = m.recentDelta } - for i, s := range m.snr { - v.margins[i] = s - snrOperatingPoint - } return v } type cableInfo struct { - ecd ecdResult - maps [2]byte + ecd ecdResult + maps [2]byte + haveMaps [2]bool } func (c cableInfo) metresString() string { @@ -688,9 +839,15 @@ type phyDisplay struct { func pairLetter(i int) string { return string(rune('A' + i)) } -// Each end resolves MDI on its own, so a swap at either end counts. -func pairSwapped(i int, maps [2]byte) bool { - return int(maps[0]>>(2*i))&3 != i || int(maps[1]>>(2*i))&3 != i +// Each end resolves MDI on its own, so a swap at either known end counts; an +// end with no readable map abstains. +func pairSwapped(i int, c cableInfo) bool { + for e := range c.maps { + if c.haveMaps[e] && int(c.maps[e]>>(2*i))&3 != i { + return true + } + } + return false } func cableSummary(cable cableInfo, measuring bool) (string, int) { @@ -705,7 +862,7 @@ func cableSummary(cable cableInfo, measuring bool) (string, int) { if v != 0 && v != pairOK { anyFault = true } - if pairSwapped(i, cable.maps) { + if pairSwapped(i, cable) { anySwap = true } } @@ -721,17 +878,25 @@ func cableSummary(cable cableInfo, measuring bool) (string, int) { return s, clsGood } +// The margin is the worst pair across the ends that measure SNR (the Wiitek's +// IEEE 1.133–136), gated on the whole pair being fresh and linked. func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay { d := phyDisplay{ - haveSNR: a.fresh && b.fresh && a.link && b.link, + haveSNR: a.fresh && b.fresh && a.link && b.link && (a.haveSNR || b.haveSNR), corrected: a.blocks + a.ber + a.retrain + b.blocks + b.ber + b.retrain, recent: a.recent + b.recent, } if d.haveSNR { - d.worstMargin = min(a.margins[0], b.margins[0]) - for i := range a.margins { - if m := min(a.margins[i], b.margins[i]); m < d.worstMargin { - d.worstMargin = m + first := true + for _, v := range []phyModView{a, b} { + if !v.haveSNR { + continue + } + for _, m := range v.margins { + if first || m < d.worstMargin { + d.worstMargin = m + first = false + } } } } @@ -739,6 +904,63 @@ func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay return d } +type ecdResult struct { + verdicts [4]int + metres [4]int +} + +func (b *bcm) cableDiag() (res ecdResult, err error) { + b.exec(func() { + var ctrl uint16 + if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil { + return + } + if err = b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil { + return + } + deadline := time.Now().Add(ecdDeadline) + for { + if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil { + return + } + if ctrl&0x0800 == 0 { + break + } + if time.Now().After(deadline) { + err = fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline) + return + } + time.Sleep(ecdPoll) + } + b.window() + var v uint16 + if v, err = b.mdioRead(1, bcmRegECDResult); err != nil { + return + } + for i := range res.verdicts { + res.verdicts[i] = int(v>>(4*i)) & 0xF + if res.verdicts[i] > pairXtalk { + panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v)) + } + var m uint16 + if m, err = b.mdioRead(1, bcmRegECDLen+uint16(i)); err != nil { + return + } + res.metres[i] = int(m) + } + }) + return +} + +func bcmEnd(mods []*phyModule) *bcm { + for _, m := range mods { + if b, ok := m.dev.(*bcm); ok { + return b + } + } + panic("no BCM module in the pair: the ECD is the only length path") +} + // The pollers are held silent throughout; pair maps are read after the // relink, so the MDI resolution is the fresh one. func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableInfo, bool, error) { @@ -753,19 +975,28 @@ func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableI var c cableInfo var err error - c.ecd, err = mods[0].bcm.cableDiag() + end := bcmEnd(mods) + c.ecd, err = end.cableDiag() if err != nil { return c, false, err } + if err = devRestartAN(end); err != nil { + return c, false, err + } relinked := false if waitRelink { - names := [2]string{mods[0].bcm.ifname, mods[1].bcm.ifname} + names := [2]string{mods[0].dev.name(), mods[1].dev.name()} _, relinked = waitCarrier(names, done) } for i, m := range mods { - if c.maps[i], err = m.bcm.pairMap(); err != nil { + b, ok := m.dev.(*bcm) + if !ok { + continue + } + if c.maps[i], err = b.pairMap(); err != nil { return c, false, err } + c.haveMaps[i] = true } return c, relinked, nil } @@ -816,7 +1047,10 @@ func (c *cableDiag) kick(done *atomic.Bool) bool { return true } -func mapString(m byte) string { +func mapString(m byte, have bool) string { + if !have { + return "unread" + } if m == pairIdentityMap { return "straight" } @@ -849,15 +1083,28 @@ func openModules(names [2]string) ([]*phyModule, [2]string, error) { mods := make([]*phyModule, 0, 2) var idents [2]string for i, name := range names { - b, err := openBCM(name) + t, err := openSFF(name) if err != nil { return nil, idents, err } - idents[i], err = b.identify() + pn, err := t.vendorPN() if err != nil { return nil, idents, err } - m := &phyModule{bcm: b} + var dev phyDev + switch pn { + case fsVendorPN: + dev = newBCM(t) + case wiitekVendorPN: + dev = &rollball{sff: t} + default: + return nil, idents, fmt.Errorf("%s: unknown module PN %q", name, pn) + } + idents[i], err = dev.identify() + if err != nil { + return nil, idents, err + } + m := &phyModule{dev: dev} // Born busy: the pollers stay silent through bringup's SETs and // retrains until the first measure completes and lifts the gate. m.busy.Store(true) @@ -892,26 +1139,43 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult { return append(out, checkResult{item: item, err: err}) } + // With a non-BCM partner the BCM is forced slave: the partner's manual + // config is unreachable, and auto-resolves-master against manual-slave is + // the combination proven to link. + mixed := false + for _, m := range mods { + if _, ok := m.dev.(*bcm); !ok { + mixed = true + } + } + master := !mixed for i, m := range mods { + b, ok := m.dev.(*bcm) + if !ok { + out = append(out, checkResult{item: names[i] + " role", state: "auto"}) + continue + } + res := checkResult{item: names[i] + " eee", state: "forced off"} - if err := m.bcm.forceEEEOff(); err != nil { + if err := b.forceEEEOff(); err != nil { return fail(res.item, err) } out = append(out, res) res = checkResult{item: names[i] + " jumbo", state: "forced on"} - if err := m.bcm.forceJumbo(); err != nil { + if err := b.forceJumbo(); err != nil { return fail(res.item, err) } out = append(out, res) res = checkResult{item: names[i] + " role", state: "forced master"} - if i > 0 { + if !master { res.state = "forced slave" } - if err := m.bcm.forceRole(i == 0); err != nil { + if err := b.forceRole(master); err != nil { return fail(res.item, err) } + master = false out = append(out, res) } @@ -919,22 +1183,27 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult { // modules link to each other, so one restart puts both µCs into training, // and no read should race that. The restarts fire last, nothing after. res := checkResult{item: "eee advert"} - var adv [2]string + var adv [2]uint16 + var advs [2]string for i, m := range mods { - v, err := m.bcm.eeeAdvert() + v, err := devEEEAdvert(m.dev) if err != nil { return fail(res.item, err) } - adv[i] = fmt.Sprintf("%#04x", v) - if v != 0 { + adv[i] = v + advs[i] = fmt.Sprintf("%#04x", v) + if _, ok := m.dev.(*bcm); ok && v != 0 { res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v) } } - res.state = adv[0] + "/" + adv[1] + if res.err == nil && adv[0]&adv[1] != 0 { + res.err = fmt.Errorf("EEE would negotiate: common ability %#04x", adv[0]&adv[1]) + } + res.state = advs[0] + "/" + advs[1] out = append(out, res) for i, m := range mods { - if err := m.bcm.restartAN(); err != nil { + if err := devRestartAN(m.dev); err != nil { return fail(names[i]+" retrain", err) } } @@ -943,5 +1212,5 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult { func cableLine(c cableInfo) string { return fmt.Sprintf("%s; map %s / %s", - verdictString(c.ecd), mapString(c.maps[0]), mapString(c.maps[1])) + verdictString(c.ecd), mapString(c.maps[0], c.haveMaps[0]), mapString(c.maps[1], c.haveMaps[1])) } diff --git a/phy_test.go b/phy_test.go index 9ad4794..21d63ca 100644 --- a/phy_test.go +++ b/phy_test.go @@ -3,13 +3,14 @@ package main import "testing" func freshMod(margins [4]float64) phyModView { - return phyModView{fresh: true, link: true, margins: margins} + return phyModView{fresh: true, link: true, haveSNR: true, margins: margins} } func TestPhyDisplayWorstMarginAndFault(t *testing.T) { cable := cableInfo{ - ecd: ecdResult{verdicts: [4]int{pairOK, pairOpen, pairOK, pairOK}, metres: [4]int{45, 12, 41, 46}}, - maps: [2]byte{pairIdentityMap, pairIdentityMap}, + ecd: ecdResult{verdicts: [4]int{pairOK, pairOpen, pairOK, pairOK}, metres: [4]int{45, 12, 41, 46}}, + maps: [2]byte{pairIdentityMap, pairIdentityMap}, + haveMaps: [2]bool{true, true}, } a := freshMod([4]float64{5, 5, 2, 0.5}) b := freshMod([4]float64{4, 5, 5, 5}) @@ -23,6 +24,23 @@ func TestPhyDisplayWorstMarginAndFault(t *testing.T) { } } +func TestPhyDisplaySNREndOwnsMargin(t *testing.T) { + wiitek := freshMod([4]float64{9.2, 7.2, 7.0, 8.7}) + fs := phyModView{fresh: true, link: true} + d := phyDisplayFrom(cableInfo{}, false, wiitek, fs) + if !d.haveSNR || d.worstMargin != 7.0 { + t.Errorf("worstMargin = %v (have %v), want the SNR end's 7.0", d.worstMargin, d.haveSNR) + } + d = phyDisplayFrom(cableInfo{}, false, fs, wiitek) + if !d.haveSNR || d.worstMargin != 7.0 { + t.Errorf("swapped ends: worstMargin = %v, want 7.0", d.worstMargin) + } + d = phyDisplayFrom(cableInfo{}, false, fs, fs) + if d.haveSNR { + t.Error("a pair with no SNR end should withhold snr") + } +} + func TestCableSummary(t *testing.T) { healthy := ecdResult{verdicts: [4]int{pairOK, pairOK, pairOK, pairOK}, metres: [4]int{50, 48, 47, 51}} for _, c := range []struct { @@ -32,10 +50,11 @@ func TestCableSummary(t *testing.T) { want string class int }{ - {"clean", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, pairIdentityMap}}, false, "49", clsGood}, - {"far-end swap", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, 0xE1}}, false, "49", clsWarn}, + {"clean", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, pairIdentityMap}, haveMaps: [2]bool{true, true}}, false, "49", clsGood}, + {"far-end swap", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, 0xE1}, haveMaps: [2]bool{true, true}}, false, "49", clsWarn}, + {"one-end map only", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, 0}, haveMaps: [2]bool{true, false}}, false, "49", clsGood}, {"no diag yet", cableInfo{}, false, "-", clsNone}, - {"measuring", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, pairIdentityMap}}, true, "...", clsNone}, + {"measuring", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, pairIdentityMap}, haveMaps: [2]bool{true, true}}, true, "...", clsNone}, } { s, cl := cableSummary(c.cable, c.measuring) if s != c.want || cl != c.class { @@ -46,8 +65,9 @@ func TestCableSummary(t *testing.T) { func TestPhyDisplayStaleGoesDim(t *testing.T) { cable := cableInfo{ - ecd: ecdResult{verdicts: [4]int{pairOK, pairOK, pairOK, pairOK}, metres: [4]int{45, 45, 41, 46}}, - maps: [2]byte{pairIdentityMap, pairIdentityMap}, + ecd: ecdResult{verdicts: [4]int{pairOK, pairOK, pairOK, pairOK}, metres: [4]int{45, 45, 41, 46}}, + maps: [2]byte{pairIdentityMap, pairIdentityMap}, + haveMaps: [2]bool{true, true}, } d := phyDisplayFrom(cable, false, freshMod([4]float64{5, 5, 5, 5}), phyModView{}) if d.haveSNR {