package main import ( "fmt" "sync" "sync/atomic" "time" ) const ( pairIdentityMap = 0xE4 fsVendorPN = "SFP-10G-T-100" wiitekVendorPN = "UF-RJ45-10G-100" ) const ( pairOK = 1 pairOpen = 2 pairShort = 3 pairXtalk = 4 ) var pairVerdicts = map[int]string{ pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK", } type mdioDev interface { name() string exec(func()) mdioRead(devad, reg uint16) (uint16, error) mdioWrite(devad, reg, val uint16) error } type phyDev interface { mdioDev identify() (string, error) } type snrSource interface { snrMargins() ([4]float64, error) } // PMA 1.1 latches low, so the first read reports any drop since it was last // read and the second reports the wire as it is now. func devLinkUp(d mdioDev) (up bool, raw uint16, err error) { d.exec(func() { if _, err = d.mdioRead(1, 1); err != nil { return } if raw, err = d.mdioRead(1, 1); err != nil { return } up = raw&0x0004 != 0 }) return } func devPCSLatch(d mdioDev) (blocks, ber uint64, raw uint16, err error) { d.exec(func() { if raw, err = d.mdioRead(3, 33); err != nil { return } blocks, ber = uint64(raw&0xFF), uint64((raw>>8)&0x3F) }) return } func devFastRetrain(d mdioDev) (count, raw uint16, err error) { d.exec(func() { if raw, err = d.mdioRead(1, 147); err != nil { return } count = raw >> 11 }) return } // AN enable is forced alongside the restart: the ECD can leave the BCM with // 7.0.12 cleared (proven live — no AN pulses, both ends deaf, link down until // power cycle), and a bare restart bit preserves the cleared enable. func devRestartAN(d mdioDev) (err error) { d.exec(func() { var v uint16 if v, err = d.mdioRead(7, 0); err != nil { return } err = d.mdioWrite(7, 0, v|0x1200) }) return } func devEEEAdvert(d mdioDev) (v uint16, err error) { d.exec(func() { v, err = d.mdioRead(7, 60) }) return } const ( phyInterval = time.Second phyStale = 5 * time.Second phyMaxDark = 30 linkWaitSpan = 25 * time.Second linkWaitPoll = time.Second snrGoodMargin = 3.0 snrWarnMargin = 1.0 ) type phyModule struct { dev phyDev busy atomic.Bool mu sync.Mutex sampled bool lastOK time.Time link bool haveSNR bool margins [4]float64 blocks uint64 ber uint64 retrains uint64 recentDelta uint64 primed bool retrainCount uint16 notes []string } // Silent while a measure owns the module. func (m *phyModule) poll() error { if m.busy.Load() { return nil } link, linkRaw, err := devLinkUp(m.dev) if err != nil { return err } var margins [4]float64 haveSNR := false if src, ok := m.dev.(snrSource); ok && link { if margins, err = src.snrMargins(); err != nil { return err } haveSNR = true } blocks, ber, pcsRaw, err := devPCSLatch(m.dev) if err != nil { return err } count, frRaw, err := devFastRetrain(m.dev) if err != nil { return err } m.mu.Lock() if m.link && !link { m.notes = append(m.notes, fmt.Sprintf("%s link read down: 1.1=0x%04x", m.dev.name(), linkRaw)) } m.sampled = true m.lastOK = time.Now() m.link = link m.haveSNR = haveSNR m.margins = margins // The first poll after a baseline drains latches from the bringup/diag // retrain era, so it only sets the origin; the retrain counter is 5 bits. if m.primed { rt := uint64((count - m.retrainCount) & 0x1F) delta := blocks + ber + rt if delta > 0 { m.notes = append(m.notes, fmt.Sprintf( "%s corrected +%d raw: 3.33=0x%04x (blocks %d ber %d) 1.147=0x%04x (retrain +%d) 1.1=0x%04x", m.dev.name(), delta, pcsRaw, blocks, ber, frRaw, rt, linkRaw)) } m.blocks += blocks m.ber += ber m.retrains += rt m.recentDelta = delta } else { m.recentDelta = 0 m.primed = true } m.retrainCount = count m.mu.Unlock() return nil } func (m *phyModule) takeNotes() []string { m.mu.Lock() defer m.mu.Unlock() n := m.notes m.notes = nil return n } func (m *phyModule) run(done *atomic.Bool) { tick := time.NewTicker(phyInterval) defer tick.Stop() dark := 0 var lastErr error for !done.Load() { <-tick.C if err := m.poll(); err != nil { dark++ lastErr = err if dark >= phyMaxDark { panic(fmt.Sprintf("module diagnostics dark for %d polls: %v", dark, lastErr)) } continue } dark = 0 } } func (m *phyModule) reset() { m.mu.Lock() m.blocks, m.ber, m.retrains, m.recentDelta = 0, 0, 0, 0 m.primed = false m.mu.Unlock() } // Latches drained after a measure belong to its blip: un-priming makes the // next poll an origin only, without discarding the pre-measure totals. func (m *phyModule) forgive() { m.mu.Lock() m.primed = false m.recentDelta = 0 m.mu.Unlock() } type phyModView struct { fresh bool link bool haveSNR bool margins [4]float64 blocks uint64 ber uint64 retrain uint64 recent uint64 } func (m *phyModule) view() phyModView { m.mu.Lock() defer m.mu.Unlock() v := phyModView{ fresh: m.sampled && time.Since(m.lastOK) < phyStale, link: m.link, haveSNR: m.haveSNR, margins: m.margins, blocks: m.blocks, ber: m.ber, retrain: m.retrains, } if v.fresh { v.recent = m.recentDelta } return v } type cableInfo struct { ecd ecdResult maps [2]byte haveMaps [2]bool } func (c cableInfo) metresString() string { sum, n := 0, 0 for i, v := range c.ecd.verdicts { if v == pairOK { sum += c.ecd.metres[i] n++ } } if n == 0 { return "-" } return fmt.Sprintf("%d", (sum+n/2)/n) } const ( clsNone = iota clsGood clsWarn clsBad ) func snrClass(margin float64) int { switch { case margin >= snrGoodMargin: return clsGood case margin >= snrWarnMargin: return clsWarn default: return clsBad } } type phyDisplay struct { haveSNR bool worstMargin float64 corrected uint64 recent uint64 metres string metresClass int } // Each end resolves MDI on its own, so a swap at either known end counts; an // end with no readable map abstains. func pairSwapped(i int, c cableInfo) bool { for e := range c.maps { if c.haveMaps[e] && int(c.maps[e]>>(2*i))&3 != i { return true } } return false } // The whole cable verdict in one cell: the length when healthy, the first // faulted pair's verdict when not, "xover" for a pair swap. func cableSummary(cable cableInfo, measuring bool) (string, int) { if measuring { return "-", clsNone } anyData, anySwap := false, false for i, v := range cable.ecd.verdicts { if v != 0 { anyData = true } if v != 0 && v != pairOK { return pairVerdicts[v], clsBad } if pairSwapped(i, cable) { anySwap = true } } switch { case !anyData: return "-", clsNone case anySwap: return "xover", clsWarn } return cable.metresString(), clsGood } // The margin is the worst pair across the ends that measure SNR (the Wiitek's // IEEE 1.133–136), gated on the whole pair being fresh and linked. func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay { d := phyDisplay{ haveSNR: a.fresh && b.fresh && a.link && b.link && (a.haveSNR || b.haveSNR), corrected: a.blocks + a.ber + a.retrain + b.blocks + b.ber + b.retrain, recent: a.recent + b.recent, } if d.haveSNR { first := true for _, v := range []phyModView{a, b} { if !v.haveSNR { continue } for _, m := range v.margins { if first || m < d.worstMargin { d.worstMargin = m first = false } } } } d.metres, d.metresClass = cableSummary(cable, measuring) return d } type ecdResult struct { verdicts [4]int metres [4]int } func bcmEnd(mods []*phyModule) *bcm { for _, m := range mods { if b, ok := m.dev.(*bcm); ok { return b } } panic("no BCM module in the pair: the ECD is the only length path") } // The pollers are held silent throughout; pair maps are read after the // relink, so the MDI resolution is the fresh one. func measureCable(mods []*phyModule, done *atomic.Bool) (cableInfo, error) { for _, m := range mods { m.busy.Store(true) } defer func() { for _, m := range mods { m.forgive() m.busy.Store(false) } }() var c cableInfo var err error end := bcmEnd(mods) c.ecd, err = end.cableDiag() if err != nil { return c, err } if err = devRestartAN(end); err != nil { return c, err } waitCarrier([2]string{mods[0].dev.name(), mods[1].dev.name()}, done) for i, m := range mods { b, ok := m.dev.(*bcm) if !ok { continue } if c.maps[i], err = b.pairMap(); err != nil { return c, err } c.haveMaps[i] = true } return c, nil } type cableDiag struct { mods []*phyModule // While set, every failure counter is suppressed at its source: the // measure's own link blip is never counted anywhere, rather than counted, // hidden and reverted. measuring atomic.Bool mu sync.Mutex info cableInfo } func newCableDiag(mods []*phyModule, info cableInfo) *cableDiag { return &cableDiag{mods: mods, info: info} } func (c *cableDiag) snapshot() (cableInfo, bool) { c.mu.Lock() defer c.mu.Unlock() return c.info, c.measuring.Load() } // A measure that errors means the diag transport or the µC is broken, and a // tester that cannot run its diagnostics must not keep testing: the failure is // fatal, and the reboot's driver reload is also the wedged-µC recovery. func (c *cableDiag) kick(done *atomic.Bool) bool { if !c.measuring.CompareAndSwap(false, true) { return false } go func() { defer holdPanic() info, err := measureCable(c.mods, done) if err != nil { panic(fmt.Sprintf("cable measure: %v", err)) } c.mu.Lock() c.info = info c.mu.Unlock() c.measuring.Store(false) }() return true } func openModules(names [2]string) ([]*phyModule, [2]string, error) { mods := make([]*phyModule, 0, 2) var idents [2]string for i, name := range names { t, err := openSFF(name) if err != nil { return nil, idents, err } pn, err := t.vendorPN() if err != nil { return nil, idents, err } var dev phyDev switch pn { case fsVendorPN: dev = newBCM(t) case wiitekVendorPN: dev = &rollball{sff: t} default: return nil, idents, fmt.Errorf("%s: unknown module PN %q", name, pn) } idents[i], err = dev.identify() if err != nil { return nil, idents, err } m := &phyModule{dev: dev} // Born busy: the pollers stay silent through bringup's SETs and // retrains until the first measure completes and lifts the gate. m.busy.Store(true) mods = append(mods, m) } return mods, idents, nil } func waitCarrier(names [2]string, done *atomic.Bool) { deadline := time.Now().Add(linkWaitSpan) for !(carrierUp(names[0]) && carrierUp(names[1])) { if time.Now().After(deadline) || done.Load() { return } time.Sleep(linkWaitPoll) } } // No trustworthy config readback exists (DATA1 is firmware scratch) and no // cable is guaranteed at bringup, so both settings are forced every boot: the // one deterministic assurance. The handler freezes during training, so the // carrier settles — the host checks just reset the links — before any command. // Never waits for a link: there may be no cable, and forcing config needs // none — the AN restart applies it whenever training next happens. func moduleChecks(mods []*phyModule, names [2]string) []checkResult { var out []checkResult fail := func(item string, err error) []checkResult { return append(out, checkResult{item: item, err: err}) } // With a non-BCM partner the BCM is forced slave: the partner's manual // config is unreachable, and auto-resolves-master against manual-slave is // the combination proven to link. mixed := false for _, m := range mods { if _, ok := m.dev.(*bcm); !ok { mixed = true } } master := !mixed for i, m := range mods { b, ok := m.dev.(*bcm) if !ok { out = append(out, checkResult{item: names[i] + " role", state: "auto"}) continue } res := checkResult{item: names[i] + " eee", state: "forced off"} if err := b.forceEEEOff(); err != nil { return fail(res.item, err) } out = append(out, res) res = checkResult{item: names[i] + " jumbo", state: "forced on"} if err := b.forceJumbo(); err != nil { return fail(res.item, err) } out = append(out, res) res = checkResult{item: names[i] + " role", state: "forced master"} if !master { res.state = "forced slave" } if err := b.forceRole(master); err != nil { return fail(res.item, err) } master = false out = append(out, res) } // Both modules configured and verified before either AN restart: the // modules link to each other, so one restart puts both µCs into training, // and no read should race that. The restarts fire last, nothing after. res := checkResult{item: "eee advert"} var adv [2]uint16 var advs [2]string for i, m := range mods { v, err := devEEEAdvert(m.dev) if err != nil { return fail(res.item, err) } adv[i] = v advs[i] = fmt.Sprintf("%#04x", v) if _, ok := m.dev.(*bcm); ok && v != 0 { res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v) } } if res.err == nil && adv[0]&adv[1] != 0 { res.err = fmt.Errorf("EEE would negotiate: common ability %#04x", adv[0]&adv[1]) } res.state = advs[0] + "/" + advs[1] out = append(out, res) for i, m := range mods { if err := devRestartAN(m.dev); err != nil { return fail(names[i]+" retrain", err) } } return out }