Unplugged-cable behavior fixed end to end: heat grid rolls on the slot boundary (view computes the newest slot from the draw clock; history sampled every frame so slots color within a frame; noise phase displayed as commanded straight from the shared clock — radiating and the per-slot vote deleted), SNR ghost panic disambiguated via the PMA 1.1 latch (link dropped mid-read discards the batch, ghost on a proven-up link still dies), measure skips the 25s relink wait when the ECD verdict says faulted and the header says MEASURING instead of GOOD while a diag runs, RollBall mailbox deadline resized to 10s from bench measurement (<25ms linked, ~220ms no-link, >3s during a live plug's training; docs/modules/wiitek updated)

This commit is contained in:
flamingcow
2026-08-17 21:29:02 -07:00
parent 916c0150ef
commit be2bf56fa7
7 changed files with 81 additions and 40 deletions
+2
View File
@@ -40,6 +40,8 @@ cabletest drives this module through its own Go RollBall client (`phy.go`): modu
Not available in safe space: pre-FEC counter, cable length — those live in the µC vendor space that killed the modules. Not available in safe space: pre-FEC counter, cable length — those live in the µC vendor space that killed the modules.
**Mailbox latency is regime-dependent** — <25 ms per read linked (X520 host), ~220 ms steady with no link (bench-measured, `~/work/phydiag-work/rb_mbox_latency.py`), and beyond 3 s while 10GBASE-T training runs (observed live when a cable plug landed mid-read at bringup). cabletest's mailbox deadline (`rbCmdTimeout`) is sized past a full ~6 s training so a plug event reads busy, not dead.
**Mailbox trap: an orphaned command's completion answers as yours.** If a session dies mid-mailbox-op (killed process), the µC finishes the orphaned command on its own time, leaving CMD=DONE and that command's value in the block. A next session that issues a command and samples status immediately reads the orphan's DONE and value as its own — seen live: a PHY ID 1.2 read answered 0x0006, the orphaned EEE-advert (7.60) value of the killed session, while the very next read was correct. The client therefore never samples status before a full poll gap, reads the value from the same 6-byte block read as the status, and accepts a completion only when the block echoes the issued devad/reg (`phy.go` mbox). The split-transaction python clients always had the pre-sample gap, which is why the field campaign never hit this. **Mailbox trap: an orphaned command's completion answers as yours.** If a session dies mid-mailbox-op (killed process), the µC finishes the orphaned command on its own time, leaving CMD=DONE and that command's value in the block. A next session that issues a command and samples status immediately reads the orphan's DONE and value as its own — seen live: a PHY ID 1.2 read answered 0x0006, the orphaned EEE-advert (7.60) value of the killed session, while the very next read was correct. The client therefore never samples status before a full poll gap, reads the value from the same 6-byte block read as the status, and accepts a completion only when the block echoes the issued devad/reg (`phy.go` mbox). The split-transaction python clients always had the pre-sample gap, which is why the field campaign never hit this.
## VCT/cable length: no confirmed-safe path ## VCT/cable length: no confirmed-safe path
+13 -16
View File
@@ -5,10 +5,11 @@ import (
"time" "time"
) )
// The panel's recent-history ring: 18 slots of 5 seconds, sampled once a // The panel's recent-history ring: 18 slots of 5 seconds, sampled every drawn
// second from the same figures the console row prints. Slots are wall-time // frame from the same figures the panel paints, so a slot has data within one
// and survive a reset — recently is recently, whatever the counters were // frame of being born. Slots are wall-time and survive a reset — recently is
// re-based to — while the since-reset aggregates and fault ages clear. // recently, whatever the counters were re-based to — while the since-reset
// aggregates and fault ages clear.
const ( const (
histSlots = 18 histSlots = 18
histSlotSpan = 5 * time.Second histSlotSpan = 5 * time.Second
@@ -39,15 +40,12 @@ type histSlot struct {
rateLow bool rateLow bool
haveSNR bool haveSNR bool
snrMin float64 snrMin float64
noiseOn int
noiseN int
} }
type history struct { type history struct {
mu sync.Mutex mu sync.Mutex
start time.Time start time.Time
slots [histSlots]histSlot slots [histSlots]histSlot
seq int64
prevValid bool prevValid bool
prevSince errs prevSince errs
@@ -80,13 +78,12 @@ func delta(now, prev uint64) uint64 {
return now - prev return now - prev
} }
func (h *history) sample(now time.Time, v view, phy phyDisplay, nv noiseView, func (h *history) sample(now time.Time, v view, phy phyDisplay,
measuring bool, target float64) { measuring bool, target float64) {
h.mu.Lock() h.mu.Lock()
defer h.mu.Unlock() defer h.mu.Unlock()
h.seq = int64(now.Sub(h.start) / histSlotSpan) s := h.slot(int64(now.Sub(h.start) / histSlotSpan))
s := h.slot(h.seq)
if h.prevValid { if h.prevValid {
for i, c := range faultClasses { for i, c := range faultClasses {
@@ -118,10 +115,6 @@ func (h *history) sample(now time.Time, v view, phy phyDisplay, nv noiseView,
} }
} }
} }
s.noiseN++
if nv.on {
s.noiseOn++
}
} }
// The wall-time slots stay: recently is recently. Everything judged against // The wall-time slots stay: recently is recently. Everything judged against
@@ -146,12 +139,16 @@ type histView struct {
sinceSNRMin float64 sinceSNRMin float64
} }
func (h *history) view() histView { // The window's newest column is the slot containing now, taken from the draw
// clock rather than the last sample: the grid then rolls on the slot boundary
// itself — the same instant the noise cycle flips.
func (h *history) view(now time.Time) histView {
h.mu.Lock() h.mu.Lock()
defer h.mu.Unlock() defer h.mu.Unlock()
newest := int64(now.Sub(h.start) / histSlotSpan)
var out histView var out histView
for k := 0; k < histSlots; k++ { for k := 0; k < histSlots; k++ {
seq := h.seq - int64(histSlots-1-k) seq := newest - int64(histSlots-1-k)
if seq < 0 { if seq < 0 {
continue continue
} }
+3 -3
View File
@@ -403,8 +403,9 @@ func run(aName, bName string) (err error) {
} }
v := totalView(views) v := totalView(views)
phy, measuring := phyView(diag, modules) phy, measuring := phyView(diag, modules)
hist.sample(now, v, phy, measuring, target)
if err := disp.render(v, now.Sub(start), phy, measuring, if err := disp.render(v, now.Sub(start), phy, measuring,
noise.view(), hist.view(), target); err != nil { noise.view(), hist.view(now), target); err != nil {
return err return err
} }
case now := <-tick.C: case now := <-tick.C:
@@ -413,8 +414,7 @@ func run(aName, bName string) (err error) {
rows[i] = d.displayView() rows[i] = d.displayView()
} }
v := totalView(rows) v := totalView(rows)
phy, measuring := phyView(diag, modules) phy, _ := phyView(diag, modules)
hist.sample(now, v, phy, noise.view(), measuring, target)
for _, m := range modules { for _, m := range modules {
for _, n := range m.takeNotes() { for _, n := range m.takeNotes() {
fmt.Println(stats.rule(n)) fmt.Println(stats.rule(n))
+13 -6
View File
@@ -65,6 +65,16 @@ type noisePort struct {
frame []byte frame []byte
} }
// The cycle is a pure function of the shared slot clock: even slots up — ports
// admin-up and the wire loud, training or frames — odd slots down. Everything
// that shows the phase computes it from the clock through here rather than
// sampling the actuator, whose ioctls lag the boundary.
func slotUp(seq int64) bool { return seq%2 == 0 }
func noiseUp(now time.Time) bool {
return slotUp(int64(now.Sub(rateEpochStart) / histSlotSpan))
}
type noiser struct { type noiser struct {
eps [2]endpoint eps [2]endpoint
ports [2]noisePort ports [2]noisePort
@@ -73,8 +83,6 @@ type noiser struct {
// phase. Latched across the down phase, where the missing carrier is our // phase. Latched across the down phase, where the missing carrier is our
// own doing and says nothing about the cable. // own doing and says nothing about the cable.
connected atomic.Bool connected atomic.Bool
// The cycle's up phase: ports admin-up and the wire loud, training or frames.
radiating atomic.Bool
} }
func newNoiser() (*noiser, error) { func newNoiser() (*noiser, error) {
@@ -124,9 +132,9 @@ type noiseView struct {
// missing is zero while the cable was there at the last verdict, one while it // missing is zero while the cable was there at the last verdict, one while it
// was not: the shape the error cells already colour by, so absence paints as // was not: the shape the error cells already colour by, so absence paints as
// the fault it is. on is the cycle's phase, presence granted. // the fault it is. on is the commanded phase, straight from the clock.
func (n *noiser) view() noiseView { func (n *noiser) view() noiseView {
v := noiseView{on: n.radiating.Load()} v := noiseView{on: noiseUp(time.Now())}
if !n.connected.Load() { if !n.connected.Load() {
v.missing = 1 v.missing = 1
} }
@@ -188,7 +196,7 @@ func (n *noiser) run(done *atomic.Bool) {
applied, wasUp, linked := false, false, false applied, wasUp, linked := false, false, false
for !done.Load() { for !done.Load() {
<-tick.C <-tick.C
up := (time.Since(rateEpochStart)/histSlotSpan)%2 == 0 up := noiseUp(time.Now())
if !applied || up != wasUp { if !applied || up != wasUp {
// A whole up phase with no link is many times the ~1s the wire // A whole up phase with no link is many times the ~1s the wire
// needs to train, so at its end the silence is the cable's answer. // needs to train, so at its end the silence is the cable's answer.
@@ -196,7 +204,6 @@ func (n *noiser) run(done *atomic.Bool) {
n.connected.Store(linked) n.connected.Store(linked)
} }
n.setLinks(fd, up) n.setLinks(fd, up)
n.radiating.Store(up)
applied, wasUp, linked = true, up, false applied, wasUp, linked = true, up, false
} }
if !up || !n.bothUp() { if !up || !n.bothUp() {
+14 -4
View File
@@ -38,7 +38,7 @@ type phyDev interface {
} }
type snrSource interface { type snrSource interface {
snrMargins() ([4]float64, error) snrMargins() ([4]float64, bool, error)
} }
// PMA 1.1 latches low, so the first read reports any drop since it was last // PMA 1.1 latches low, so the first read reports any drop since it was last
@@ -137,10 +137,9 @@ func (m *phyModule) poll() error {
var margins [4]float64 var margins [4]float64
haveSNR := false haveSNR := false
if src, ok := m.dev.(snrSource); ok && link { if src, ok := m.dev.(snrSource); ok && link {
if margins, err = src.snrMargins(); err != nil { if margins, haveSNR, err = src.snrMargins(); err != nil {
return err return err
} }
haveSNR = true
} }
blocks, ber, pcsRaw, err := devPCSLatch(m.dev) blocks, ber, pcsRaw, err := devPCSLatch(m.dev)
if err != nil { if err != nil {
@@ -405,7 +404,18 @@ func measureCable(mods []*phyModule, done *atomic.Bool) (cableInfo, error) {
if err = devRestartAN(end); err != nil { if err = devRestartAN(end); err != nil {
return c, err return c, err
} }
waitCarrier([2]string{mods[0].dev.name(), mods[1].dev.name()}, done) // A pair the diag just called faulted is not going to relink; waiting the
// full carrier timeout would hold the verdict hostage for 25 s on an
// unplugged cable.
healthy := true
for _, v := range c.ecd.verdicts {
if v != pairOK {
healthy = false
}
}
if healthy {
waitCarrier([2]string{mods[0].dev.name(), mods[1].dev.name()}, done)
}
for i, m := range mods { for i, m := range mods {
b, ok := m.dev.(*bcm) b, ok := m.dev.(*bcm)
if !ok { if !ok {
+20 -4
View File
@@ -25,8 +25,10 @@ const (
rbReadDelayUs = 500 rbReadDelayUs = 500
rbCmdPoll = 20 * time.Millisecond rbCmdPoll = 20 * time.Millisecond
// Matches the BCM allowance for a handler frozen by 10GBASE-T training. // Bench-measured: <25 ms linked, ~220 ms steady with no link, and >3 s
rbCmdTimeout = 3 * time.Second // while a live plug event's training ran — sized past a full ~6 s
// training, so a cable plugged mid-read reads busy, not dead.
rbCmdTimeout = 10 * time.Second
rbPHYIDHi uint16 = 0x002B rbPHYIDHi uint16 = 0x002B
rbPHYIDLo uint16 = 0x0BF4 rbPHYIDLo uint16 = 0x0BF4
@@ -168,7 +170,12 @@ func (r *rollball) identify() (ident string, err error) {
return return
} }
func (r *rollball) snrMargins() (out [4]float64, err error) { // An untrained PHY honestly serves 0x0000 here, so an out-of-band value is
// only a ghost if the link stayed up across the reads. The caller drained the
// PMA 1.1 latch when it checked the link, so one latched read now answers
// that: a drop since means the unplug race — discard the batch, ok false, and
// the next poll reads the link down.
func (r *rollball) snrMargins() (out [4]float64, ok bool, err error) {
r.exec(func() { r.exec(func() {
for i := range out { for i := range out {
var v uint16 var v uint16
@@ -177,10 +184,19 @@ func (r *rollball) snrMargins() (out [4]float64, err error) {
} }
m := (float64(v) - 0x8000) / 10 m := (float64(v) - 0x8000) / 10
if m < rbGhostLow || m > rbGhostHigh { if m < rbGhostLow || m > rbGhostHigh {
panic(fmt.Sprintf("%s: ghost SNR margin %.1f dB (1.%d=%#04x)", r.ifname, m, 133+i, v)) var link uint16
if link, err = r.mdioRead(1, 1); err != nil {
return
}
if link&0x0004 == 0 {
return
}
panic(fmt.Sprintf("%s: ghost SNR margin %.1f dB (1.%d=%#04x) with the link up",
r.ifname, m, 133+i, v))
} }
out[i] = m out[i] = m
} }
ok = true
}) })
return return
} }
+16 -7
View File
@@ -326,8 +326,10 @@ func (d *display) heatCell(rowY, k int, c rgb) {
// The verdict names the newest thing wrong at any horizon: the cable's own // The verdict names the newest thing wrong at any horizon: the cable's own
// fault first, then the most recent traffic fault with its age, and only a // fault first, then the most recent traffic fault with its age, and only a
// clean record reads GOOD. // clean record reads GOOD. While a measure runs the tester is asserting
func (d *display) drawHeader(v view, phy phyDisplay, hv histView, now time.Time) { // nothing, and says so instead of GOOD.
func (d *display) drawHeader(v view, phy phyDisplay, hv histView, measuring bool,
now time.Time) {
good := true good := true
word, sub := "GOOD", []seg{{d.text, "no faults", uiInk2}} word, sub := "GOOD", []seg{{d.text, "no faults", uiInk2}}
if i, age, ok := hv.newestFault(now); ok { if i, age, ok := hv.newestFault(now); ok {
@@ -349,13 +351,18 @@ func (d *display) drawHeader(v view, phy phyDisplay, hv histView, now time.Time)
if !good { if !good {
col = uiCrit col = uiCrit
} }
if measuring {
word, sub, col = "MEASURING", []seg{{d.text, "cable diag running", uiInk2}}, uiMuted
}
rx, ry := uiMargin, headerTop rx, ry := uiMargin, headerTop
d.fb.roundRect(rx, ry, ringD, ringD, ringD/2, col) d.fb.roundRect(rx, ry, ringD, ringD, ringD/2, col)
d.fb.roundRect(rx+ringEdge, ry+ringEdge, ringD-2*ringEdge, ringD-2*ringEdge, d.fb.roundRect(rx+ringEdge, ry+ringEdge, ringD-2*ringEdge, ringD-2*ringEdge,
ringD/2-ringEdge, uiBg) ringD/2-ringEdge, uiBg)
if good { switch {
case measuring:
case good:
d.check(rx+ringD/2, ry+ringD/2, 44, col) d.check(rx+ringD/2, ry+ringD/2, 44, col)
} else { default:
d.cross(rx+ringD/2, ry+ringD/2, 40, col) d.cross(rx+ringD/2, ry+ringD/2, 40, col)
} }
@@ -476,11 +483,13 @@ func (d *display) drawRows(v view, phy phyDisplay, measuring bool, nv noiseView,
} else { } else {
d.colText(d.text, nowR, y, state, uiInk2) d.colText(d.text, nowR, y, state, uiInk2)
} }
// The lane shows the commanded phase, a pure function of each
// slot's parity; only slots the run never lived through are unknown.
for k, s := range hv.slots { for k, s := range hv.slots {
c := uiCellNA c := uiCellNA
if s.sampled && s.noiseN > 0 { if s.sampled {
c = uiNoise0 c = uiNoise0
if 2*s.noiseOn >= s.noiseN { if slotUp(s.seq) {
c = uiNoise c = uiNoise
} }
} }
@@ -548,7 +557,7 @@ func (d *display) render(v view, elapsed time.Duration, phy phyDisplay,
fb := d.fb fb := d.fb
fb.fill(uiBg) fb.fill(uiBg)
d.drawHeader(v, phy, hv, time.Now()) d.drawHeader(v, phy, hv, measuring, time.Now())
d.hairline(d.headerH - 1) d.hairline(d.headerH - 1)
d.drawColHeaders() d.drawColHeaders()
d.drawRows(v, phy, measuring, nv, hv, target) d.drawRows(v, phy, measuring, nv, hv, target)