From f9140ff8d9755afbe8ded39e4a9be4ae312ebfdc Mon Sep 17 00:00:00 2001 From: flamingcow Date: Thu, 13 Aug 2026 10:49:01 -0700 Subject: [PATCH] Bringup waits for nothing (no sleeps, no link waits, SETs fire cable-or-not with advert readback), errors held off the display while a measure is in flight, status poll is a deadline not a retry count, ghost signals (implausible SNR, undefined ECD verdict) panic outright; persistent stale-pipeline bridge fact documented --- docs/modules/fs/README.md | 4 +- docs/state.md | 2 +- main.go | 23 +++++++--- phy.go | 91 ++++++++++++++++----------------------- 4 files changed, 56 insertions(+), 64 deletions(-) diff --git a/docs/modules/fs/README.md b/docs/modules/fs/README.md index 8f6c6c3..a24b17e 100644 --- a/docs/modules/fs/README.md +++ b/docs/modules/fs/README.md @@ -24,8 +24,8 @@ Protocol and full verified catalog: [bcm84891l-mdio-commands.md](bcm84891l-mdio- - **GETs must be invoked bare** — pre-writing *any* DATA register, not just the documented DATA1 display flag, leaves the handler executing as a no-op with results never written (sentinel pre-fills survived GET_SNR untouched, proven on hardware). - **STATUS must never be written** — any user value (0x0000 and 0x0008 both tried) closes the mailbox: the next command is silently ignored until firmware restores it. - **PASS cannot prove completion.** The previous command's PASS stays latched while a slow firmware still executes, and until then the DATA registers are firmware scratch — the die temperature turns up in them (0x004E–0x0051 observed as "SNR" and "EEE mode"; the campaign's "anomalous 0x0047" was this same race). -- **Handler writes during firmware-busy windows wedge the µC permanently** — DATA/CMD writes landing while firmware does post-AN provisioning or training work (a window stretching ~10 s past relink) killed the SMI service four times; reads alone never once. Every handler write — SETs, GET commands, the ECD trigger — runs only in a quiet window: carrier up and stable ≥ 10 s (`phy.go`). -- **Bridge reads carry no fetch identity** — a fetch that outruns the delay leaves the previous transaction's data at 0xAD with no error. +- **Handler writes during firmware-busy windows wedge the µC permanently** — DATA/CMD writes landing while firmware does post-AN provisioning or training work (a window stretching ~10 s past relink) killed the SMI service four times; reads alone never once. Unmitigated — exposure accepted until the shared-mailbox contention is properly resolved. +- **Bridge reads carry no fetch identity** — a fetch that outruns the delay leaves the previous transaction's data at 0xAD with no error. A killed host can also leave the pipeline latched a response behind *persistently* (PHY ID served a stuck stale value across processes until driver re-init); every appliance boot clears it by loading the driver, so only bench runs reusing a loaded driver see it, and the identity check at startup dies loudly on it. - Poll STATUS ~100 ms; frozen up to 2 s during 10GBASE-T training — only run after link-up. ## Proven diagnostics diff --git a/docs/state.md b/docs/state.md index f44c746..b1a796f 100644 --- a/docs/state.md +++ b/docs/state.md @@ -2,7 +2,7 @@ ## Committed tree -AF_PACKET raw sockets everywhere (`sock.go`); flow-director steering; per-packet-MAC-rx-stamped rate buckets (`SO_TIMESTAMPING` cmsg, `rx_filter=ALL` as a hard host check — nics/README.md for what that demands of the NIC; **temporarily bypassed** in `ts.go` so BCM work can run on the X520, which cannot stamp — the check reports yellow and the panel rates read zero there; restore to fatal for the product NIC); read-time-stamped NIC-counter rates; test interfaces pinned to MTU 9000 with a 9018-byte jumbo in the size mix (the modules' jumbo path is exercised, not assumed); BCM module diagnostics (`phy.go`, over the patched-ixgbe `sff_i2c` debugfs, compound-op framing; one per-module lock around every whole operation, handler writes confined to quiet windows — the µC wedges otherwise; the firmware's own mailbox use still contends, unresolved — modules/fs/ for the full trap list): bringup identifies both modules and forces EEE off and jumbo on every boot — no trustworthy readback exists and no cable is guaranteed to probe through; the ECD — per-pair verdicts, lengths and pair maps are the length/wiring path — runs through one async path at startup and on every reset, never blocking the UI, with counters re-baselining only after the diag's own link blip so it is never charged to the run; a 1 Hz poller feeds per-pair SNR margin (vs the ≈26.5 dB operating point; green ≥ 3 dB, amber ≥ 1 dB — provisional until the graded-noise run) and the corrected-error set (PCS 3.33 errored blocks/BER, PMA 1.147 fast-retrain count) to the panel and console; framebuffer UI; harness. +AF_PACKET raw sockets everywhere (`sock.go`); flow-director steering; per-packet-MAC-rx-stamped rate buckets (`SO_TIMESTAMPING` cmsg, `rx_filter=ALL` as a hard host check — nics/README.md for what that demands of the NIC; **temporarily bypassed** in `ts.go` so BCM work can run on the X520, which cannot stamp — the check reports yellow and the panel rates read zero there; restore to fatal for the product NIC); read-time-stamped NIC-counter rates; test interfaces pinned to MTU 9000 with a 9018-byte jumbo in the size mix (the modules' jumbo path is exercised, not assumed); BCM module diagnostics (`phy.go`, over the patched-ixgbe `sff_i2c` debugfs, compound-op framing; one per-module lock around every whole operation; the firmware's own mailbox use still contends, unresolved — modules/fs/ for the full trap list): bringup identifies both modules and forces EEE off and jumbo on every boot — no trustworthy readback exists and no cable is guaranteed to probe through; the ECD — per-pair verdicts, lengths and pair maps are the length/wiring path — runs through one async path at startup and on every reset, never blocking the UI, with counters re-baselining only after the diag's own link blip so it is never charged to the run; a 1 Hz poller feeds per-pair SNR margin (vs the ≈26.5 dB operating point; green ≥ 3 dB, amber ≥ 1 dB — provisional until the graded-noise run) and the corrected-error set (PCS 3.33 errored blocks/BER, PMA 1.147 fast-retrain count) to the panel and console; framebuffer UI; harness. ## Stashes diff --git a/main.go b/main.go index 0fad486..4ce3dcd 100644 --- a/main.go +++ b/main.go @@ -299,6 +299,19 @@ func (d *direction) counters(now counterSet) view { } } +// Errors seen while a measure is in flight are the diag's own link blip and +// are re-based away at its completion; until then they are held off the +// display rather than shown as the cable's. +func measureView(diag *cableDiag, modules []*phyModule, v view) (view, phyDisplay) { + info, measuring := diag.snapshot() + phy := phyDisplayFrom(info, measuring, modules[0].view(), modules[1].view()) + if measuring { + v.window, v.since = errs{}, errs{} + phy.corrected, phy.recent = 0, 0 + } + return v, phy +} + func totalView(views []view) view { var t view for _, v := range views { @@ -769,9 +782,8 @@ func run(aName, bName string) (err error) { for i, d := range dirs { views[i] = d.displayView() } - info, measuring := diag.snapshot() - phy := phyDisplayFrom(info, measuring, modules[0].view(), modules[1].view()) - if err := disp.render(totalView(views), now.Sub(start), phy, + v, phy := measureView(diag, modules, totalView(views)) + if err := disp.render(v, now.Sub(start), phy, noise.missing()); err != nil { return err } @@ -780,9 +792,8 @@ func run(aName, bName string) (err error) { for i, d := range dirs { rows[i] = d.displayView() } - info, measuring := diag.snapshot() - phy := phyDisplayFrom(info, measuring, modules[0].view(), modules[1].view()) - for _, line := range stats.emit(totalRow(elapsed, totalView(rows), target, phy, + v, phy := measureView(diag, modules, totalView(rows)) + for _, line := range stats.emit(totalRow(elapsed, v, target, phy, noise.missing())) { fmt.Println(line) } diff --git a/phy.go b/phy.go index 36dc4c4..d8d2053 100644 --- a/phy.go +++ b/phy.go @@ -41,10 +41,9 @@ const ( bcmReadDelayUs = 3000 bcmRetryDelayUs = 10000 - bcmStatusPoll = 100 * time.Millisecond - bcmStatusTries = 30 - bcmSetSettle = time.Second - bcmQuiet = 10 * time.Second + bcmStatusPoll = 100 * time.Millisecond + // Covers the handler's documented 2 s freeze during 10GBASE-T training. + bcmStatusTimeout = 3 * time.Second ecdPoll = 200 * time.Millisecond ecdDeadline = 50 * time.Second @@ -175,25 +174,28 @@ func (b *bcm) eeprom(off byte, n int) ([]byte, error) { return b.compound(0xA0, 0xA1, 500, n, []byte{off}) } +// The datasheet's completion handshake: poll STATUS on its 100 ms cadence +// until the wanted state, bounded by a deadline. func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) { - var st uint16 - for i := 0; i < bcmStatusTries; i++ { - var err error - st, err = b.mdioRead(bcmMMDVendor, bcmRegStatus) + deadline := time.Now().Add(bcmStatusTimeout) + for { + st, err := b.mdioRead(bcmMMDVendor, bcmRegStatus) if err != nil { return 0, err } if want(st) { return st, nil } + if time.Now().After(deadline) { + return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st) + } time.Sleep(bcmStatusPoll) } - return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st) } // GETs must be invoked bare (pre-writing any DATA register leaves the handler // executing as a no-op); SETs must pass their full parameter set (the handler -// executes stale DATA) and get settle time in place of unprovable completion. +// executes stale DATA). func (b *bcm) command(code uint16, params ...uint16) ([5]uint16, error) { b.mu.Lock() defer b.mu.Unlock() @@ -212,9 +214,6 @@ func (b *bcm) command(code uint16, params ...uint16) ([5]uint16, error) { if err := b.mdioWrite(bcmMMDVendor, bcmRegCmd, code); err != nil { return data, err } - if len(params) > 0 { - time.Sleep(bcmSetSettle) - } st, err := b.waitStatus(func(st uint16) bool { return st == bcmStPass || st == bcmStError }) @@ -375,6 +374,9 @@ func (b *bcm) cableDiag() (ecdResult, error) { } for i := range res.verdicts { res.verdicts[i] = int(v>>(4*i)) & 0xF + if res.verdicts[i] > pairXtalk { + panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v)) + } m, err := b.mdioRead(1, bcmRegECDLen+uint16(i)) if err != nil { return res, err @@ -394,12 +396,16 @@ const ( snrOperatingPoint = 26.5 snrGoodMargin = 3.0 snrWarnMargin = 1.0 + + // No trained link produces SNR outside this; readings there are another + // register's data (die temp ≈ 8, handler status ≈ 0.4) and the run dies. + snrGhostLow = 15.0 + snrGhostHigh = 50.0 ) type phyModule struct { - bcm *bcm - busy atomic.Bool - upSince time.Time + bcm *bcm + busy atomic.Bool mu sync.Mutex sampled bool @@ -415,32 +421,25 @@ type phyModule struct { retrainCount uint16 } -// Silent while a measure owns the module, and no handler command outside a -// quiet window (carrier up and stable): writes landing on a µC busy with -// post-AN work have wedged its SMI service permanently. Reads are always safe. +// Silent while a measure owns the module. func (m *phyModule) poll() error { if m.busy.Load() { return nil } - if carrierUp(m.bcm.ifname) { - if m.upSince.IsZero() { - m.upSince = time.Now() - } - } else { - m.upSince = time.Time{} - } - quiet := !m.upSince.IsZero() && time.Since(m.upSince) >= bcmQuiet - link, err := m.bcm.linkUp() if err != nil { return err } - link = link && quiet var snr [4]float64 if link { if snr, err = m.bcm.snr(); err != nil { return err } + for _, s := range snr { + if s < snrGhostLow || s > snrGhostHigh { + panic(fmt.Sprintf("%s: ghost SNR %.1f dB", m.bcm.ifname, s)) + } + } } blocks, ber, err := m.bcm.pcsLatch() if err != nil { @@ -629,9 +628,8 @@ func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay return d } -// Every handler write inside runs in a quiet window: the ECD trigger only -// after the carrier has settled, the pair-map commands only after the run's -// own blip has settled too. The pollers are held silent throughout. +// The pollers are held silent throughout; pair maps are read after the +// relink, so the MDI resolution is the fresh one. func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableInfo, bool, error) { for _, m := range mods { m.busy.Store(true) @@ -641,11 +639,6 @@ func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableI m.busy.Store(false) } }() - names := [2]string{mods[0].bcm.ifname, mods[1].bcm.ifname} - time.Sleep(2 * phyInterval) - if _, up := waitCarrier(names, done); up { - time.Sleep(bcmQuiet) - } var c cableInfo var err error @@ -655,9 +648,9 @@ func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableI } relinked := false if waitRelink { + names := [2]string{mods[0].bcm.ifname, mods[1].bcm.ifname} _, relinked = waitCarrier(names, done) } - time.Sleep(bcmQuiet) for i, m := range mods { if c.maps[i], err = m.bcm.pairMap(); err != nil { return c, false, err @@ -780,17 +773,14 @@ func waitCarrier(names [2]string, done *atomic.Bool) (time.Duration, bool) { // cable is guaranteed at bringup, so both settings are forced every boot: the // one deterministic assurance. The handler freezes during training, so the // carrier settles — the host checks just reset the links — before any command. +// Never waits for a link: there may be no cable, and forcing config needs +// none — the AN restart applies it whenever training next happens. func moduleChecks(mods []*phyModule, names [2]string) []checkResult { var out []checkResult fail := func(item string, err error) []checkResult { return append(out, checkResult{item: item, err: err}) } - _, hadLink := waitCarrier(names, nil) - if hadLink { - time.Sleep(bcmQuiet) - } - for i, m := range mods { res := checkResult{item: names[i] + " eee", state: "forced off"} if err := m.bcm.forceEEEOff(); err != nil { @@ -809,12 +799,7 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult { } } - res := checkResult{item: "link retrain"} - var took time.Duration - up := false - if hadLink { - took, up = waitCarrier(names, nil) - } + res := checkResult{item: "eee advert"} var adv [2]string for i, m := range mods { v, err := m.bcm.eeeAdvert() @@ -822,15 +807,11 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult { return fail(res.item, err) } adv[i] = fmt.Sprintf("%#04x", v) - if up && v != 0 { + if v != 0 { res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v) } } - if up { - res.state = fmt.Sprintf("up in %.1fs, eee advert %s/%s", took.Seconds(), adv[0], adv[1]) - } else { - res.state = fmt.Sprintf("no link (cable unplugged?), eee advert %s/%s", adv[0], adv[1]) - } + res.state = adv[0] + "/" + adv[1] return append(out, res) }