package main import ( "fmt" "os" "path/filepath" "strings" "sync" "sync/atomic" "time" "golang.org/x/sys/unix" ) const ( bcmI2CWrite = 0xAC bcmI2CRead = 0xAD bcmMMDVendor uint16 = 0x1E bcmRegCmd uint16 = 0x4005 bcmRegStatus uint16 = 0x4037 bcmRegData1 uint16 = 0x4038 bcmStInProgress uint16 = 0x0002 bcmStPass uint16 = 0x0004 bcmStError uint16 = 0x0008 bcmStBusy uint16 = 0xBBBB bcmCmdGetPairSwap uint16 = 0x8000 bcmCmdSetEEEMode uint16 = 0x8009 bcmCmdGetSNR uint16 = 0x8030 bcmRegECDCtrl uint16 = 0x4006 bcmRegECDResult uint16 = 0xA896 bcmRegECDLen uint16 = 0xA897 bcmPHYIDHi = 0x3590 bcmPHYIDLo = 0x5081 bcmReadDelayUs = 3000 bcmRetryDelayUs = 10000 bcmStatusPoll = 100 * time.Millisecond bcmStatusTries = 30 ecdPoll = 200 * time.Millisecond ecdDeadline = 50 * time.Second pairIdentityMap = 0xE4 ) const ( pairOK = 1 pairOpen = 2 pairShort = 3 pairXtalk = 4 ) var pairVerdicts = map[int]string{ pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK", } type bcm struct { ifname string path string // Serializes the multi-op sequences — a handler command, an ECD run — that // would corrupt each other interleaved. Single register reads ride bare: // the compound op makes each one atomic on the wire. mu sync.Mutex } func openBCM(ifname string) (*bcm, error) { devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device") if err != nil { return nil, fmt.Errorf("%s: %w", ifname, err) } drv, err := ifDriver(ifname) if err != nil { return nil, fmt.Errorf("%s: %w", ifname, err) } if drv != "ixgbe" { return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv) } b := &bcm{ ifname: ifname, path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c", } if _, err := os.Stat(b.path); err != nil { return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err) } return b, nil } func (b *bcm) op(cmd string) (string, error) { fd, err := unix.Open(b.path, unix.O_RDWR, 0) if err != nil { return "", fmt.Errorf("%s: %w", b.path, err) } defer unix.Close(fd) if _, err := unix.Write(fd, []byte(cmd)); err != nil { return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err) } buf := make([]byte, 256) n, err := unix.Read(fd, buf) if err != nil { return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err) } resp := strings.TrimSpace(string(buf[:n])) if !strings.HasPrefix(resp, "ok") { return "", fmt.Errorf("%s %q: %s", b.ifname, cmd, resp) } return strings.TrimSpace(resp[2:]), nil } func parseHexBytes(s string, n int) ([]byte, error) { fields := strings.Fields(s) if len(fields) != n { return nil, fmt.Errorf("want %d bytes, got %q", n, s) } out := make([]byte, n) for i, f := range fields { var v byte if _, err := fmt.Sscanf(f, "%x", &v); err != nil { return nil, fmt.Errorf("byte %q in %q", f, s) } out[i] = v } return out, nil } // One write-STOP-delay-read transaction under a single bus hold, so the // driver's own SFP traffic can never consume the bridge's pending data. func (b *bcm) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) { var sb strings.Builder fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n) for _, v := range wdata { fmt.Fprintf(&sb, " %02x", v) } resp, err := b.op(sb.String()) if err != nil { return nil, err } return parseHexBytes(resp, n) } func (b *bcm) mdioReadDelay(devad, reg uint16, delayUs int) (uint16, error) { d, err := b.compound(bcmI2CWrite, bcmI2CRead, delayUs, 2, []byte{0x20 | byte(devad), byte(reg >> 8), byte(reg)}) if err != nil { return 0, err } return uint16(d[0])<<8 | uint16(d[1]), nil } // 0x0000 is also the bridge's not-ready signature, so a zero is read again at // a longer delay before being believed. func (b *bcm) mdioRead(devad, reg uint16) (uint16, error) { v, err := b.mdioReadDelay(devad, reg, bcmReadDelayUs) if err != nil || v != 0 { return v, err } return b.mdioReadDelay(devad, reg, bcmRetryDelayUs) } func (b *bcm) mdioWrite(devad, reg, val uint16) error { _, err := b.op(fmt.Sprintf("w %02x %02x %02x %02x %02x %02x", bcmI2CWrite, byte(devad), byte(reg>>8), byte(reg), byte(val>>8), byte(val))) return err } func (b *bcm) eeprom(off byte, n int) ([]byte, error) { return b.compound(0xA0, 0xA1, 500, n, []byte{off}) } func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) { var st uint16 for i := 0; i < bcmStatusTries; i++ { var err error st, err = b.mdioRead(bcmMMDVendor, bcmRegStatus) if err != nil { return 0, err } if want(st) { return st, nil } time.Sleep(bcmStatusPoll) } return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st) } // The handler never clears DATA registers it does not use, so every SET must // pass its full parameter set and every GET must pass none. func (b *bcm) command(code uint16, params ...uint16) ([5]uint16, error) { b.mu.Lock() defer b.mu.Unlock() var data [5]uint16 if _, err := b.waitStatus(func(st uint16) bool { return st != bcmStInProgress && st != bcmStBusy }); err != nil { return data, err } for i, p := range params { if err := b.mdioWrite(bcmMMDVendor, bcmRegData1+uint16(i), p); err != nil { return data, err } } if err := b.mdioWrite(bcmMMDVendor, bcmRegCmd, code); err != nil { return data, err } st, err := b.waitStatus(func(st uint16) bool { return st == bcmStPass || st == bcmStError }) if err != nil { return data, err } if st == bcmStError { return data, fmt.Errorf("%s: command %#04x returned ERROR", b.ifname, code) } for i := range data { data[i], err = b.mdioRead(bcmMMDVendor, bcmRegData1+uint16(i)) if err != nil { return data, err } } return data, nil } func (b *bcm) identify() (string, error) { hi, err := b.mdioRead(1, 2) if err != nil { return "", err } lo, err := b.mdioRead(1, 3) if err != nil { return "", err } if hi != bcmPHYIDHi || lo != bcmPHYIDLo { return "", fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x", b.ifname, hi, lo, bcmPHYIDHi, bcmPHYIDLo) } sn, err := b.eeprom(68, 16) if err != nil { return "", err } return "BCM84891L sn " + strings.TrimSpace(string(sn)), nil } // PMA 1.1 latches low, so the first read reports any drop since it was last // read and the second reports the wire as it is now. func (b *bcm) linkUp() (bool, error) { if _, err := b.mdioRead(1, 1); err != nil { return false, err } v, err := b.mdioRead(1, 1) if err != nil { return false, err } return v&0x0004 != 0, nil } func (b *bcm) forceEEEOff() error { _, err := b.command(bcmCmdSetEEEMode, 0x0000, 0x0000, 0x7A12, 0x0480, 0x0000) return err } func (b *bcm) restartAN() error { v, err := b.mdioRead(7, 0) if err != nil { return err } return b.mdioWrite(7, 0, v|0x0200) } func (b *bcm) eeeAdvert() (uint16, error) { return b.mdioRead(7, 60) } func (b *bcm) pairMap() (byte, error) { d, err := b.command(bcmCmdGetPairSwap) if err != nil { return 0, err } return byte(d[1]), nil } func (b *bcm) snr() ([4]float64, error) { var out [4]float64 d, err := b.command(bcmCmdGetSNR) if err != nil { return out, err } for i := range out { out[i] = float64(d[i+1]) / 10 } return out, nil } func (b *bcm) pcsLatch() (blocks, ber uint64, err error) { v, err := b.mdioRead(3, 33) if err != nil { return 0, 0, err } return uint64(v & 0xFF), uint64((v >> 8) & 0x3F), nil } func (b *bcm) fastRetrainCount() (uint16, error) { v, err := b.mdioRead(1, 147) if err != nil { return 0, err } return v >> 11, nil } type ecdResult struct { verdicts [4]int metres [4]int } func (b *bcm) cableDiag() (ecdResult, error) { b.mu.Lock() defer b.mu.Unlock() var res ecdResult ctrl, err := b.mdioRead(bcmMMDVendor, bcmRegECDCtrl) if err != nil { return res, err } if err := b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil { return res, err } deadline := time.Now().Add(ecdDeadline) for { ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl) if err != nil { return res, err } if ctrl&0x0800 == 0 { break } if time.Now().After(deadline) { return res, fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline) } time.Sleep(ecdPoll) } v, err := b.mdioRead(1, bcmRegECDResult) if err != nil { return res, err } for i := range res.verdicts { res.verdicts[i] = int(v>>(4*i)) & 0xF m, err := b.mdioRead(1, bcmRegECDLen+uint16(i)) if err != nil { return res, err } res.metres[i] = int(m) } return res, nil } const ( phyInterval = time.Second phyStale = 5 * time.Second phyMaxDark = 30 linkWaitSpan = 25 * time.Second linkWaitPoll = time.Second snrOperatingPoint = 26.5 snrGoodMargin = 3.0 snrWarnMargin = 1.0 ) type phyModule struct { bcm *bcm mu sync.Mutex sampled bool lastOK time.Time link bool haveSNR bool snr [4]float64 blocks uint64 ber uint64 retrains uint64 recentDelta uint64 primed bool retrainCount uint16 } func (m *phyModule) poll() error { link, err := m.bcm.linkUp() if err != nil { return err } var snr [4]float64 if link { if snr, err = m.bcm.snr(); err != nil { return err } } blocks, ber, err := m.bcm.pcsLatch() if err != nil { return err } count, err := m.bcm.fastRetrainCount() if err != nil { return err } m.mu.Lock() m.sampled = true m.lastOK = time.Now() m.link = link m.haveSNR = link m.snr = snr // The first poll after a baseline drains what the latches gathered during // the bringup or diag retrain, which predates the run: it only establishes // the origin. The retrain counter is 5 bits and rolls over, so only its // forward motion is kept. if m.primed { delta := blocks + ber + uint64((count-m.retrainCount)&0x1F) m.blocks += blocks m.ber += ber m.retrains += uint64((count - m.retrainCount) & 0x1F) m.recentDelta = delta } else { m.recentDelta = 0 m.primed = true } m.retrainCount = count m.mu.Unlock() return nil } // A tester that quietly loses its SNR eye goes on reporting a clean link, so a // transport that stays dark past every transient explanation stops the run. func (m *phyModule) run(done *atomic.Bool) { tick := time.NewTicker(phyInterval) defer tick.Stop() dark := 0 var lastErr error for !done.Load() { <-tick.C if err := m.poll(); err != nil { dark++ lastErr = err if dark >= phyMaxDark { panic(fmt.Sprintf("module diagnostics dark for %d polls: %v", dark, lastErr)) } continue } dark = 0 } } func (m *phyModule) reset() { m.mu.Lock() m.blocks, m.ber, m.retrains, m.recentDelta = 0, 0, 0, 0 m.primed = false m.mu.Unlock() } type phyModView struct { fresh bool link bool margins [4]float64 blocks uint64 ber uint64 retrain uint64 recent uint64 } func (m *phyModule) view() phyModView { m.mu.Lock() defer m.mu.Unlock() v := phyModView{ fresh: m.sampled && time.Since(m.lastOK) < phyStale, link: m.link && m.haveSNR, blocks: m.blocks, ber: m.ber, retrain: m.retrains, } if v.fresh { v.recent = m.recentDelta } for i, s := range m.snr { v.margins[i] = s - snrOperatingPoint } return v } type cableInfo struct { ecd ecdResult maps [2]byte } // The four pair lengths of one healthy cable disagree by a few metres of twist // rate, so the cable's length is shown as their mean. func (c cableInfo) metresString() string { sum, n := 0, 0 for i, v := range c.ecd.verdicts { if v == pairOK { sum += c.ecd.metres[i] n++ } } if n == 0 { return "-" } return fmt.Sprintf("%d", (sum+n/2)/n) } const ( clsNone = iota clsGood clsWarn clsBad ) func snrClass(margin float64) int { switch { case margin >= snrGoodMargin: return clsGood case margin >= snrWarnMargin: return clsWarn default: return clsBad } } type phyDisplay struct { haveSNR bool worstMargin float64 corrected uint64 recent uint64 metres string metresClass int } func pairLetter(i int) string { return string(rune('A' + i)) } // Each end resolves MDI on its own, so a swap at either end counts. func pairSwapped(i int, maps [2]byte) bool { return int(maps[0]>>(2*i))&3 != i || int(maps[1]>>(2*i))&3 != i } // The cable as one figure and one judgment: the mean length of its healthy // pairs, red when the diag found a fault, amber when a pair arrived swapped. // Per-pair detail stays on the console — pair letters don't correlate back to // wires by eye. func cableSummary(cable cableInfo, measuring bool) (string, int) { if measuring { return "...", clsNone } anyData, anyFault, anySwap := false, false, false for i, v := range cable.ecd.verdicts { if v != 0 { anyData = true } if v != 0 && v != pairOK { anyFault = true } if pairSwapped(i, cable.maps) { anySwap = true } } s := cable.metresString() switch { case !anyData: return "-", clsNone case anyFault: return s, clsBad case anySwap: return s, clsWarn } return s, clsGood } // The worse of the two receivers' margins, worst pair across the cable. func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay { d := phyDisplay{ haveSNR: a.fresh && b.fresh && a.link && b.link, corrected: a.blocks + a.ber + a.retrain + b.blocks + b.ber + b.retrain, recent: a.recent + b.recent, } if d.haveSNR { d.worstMargin = min(a.margins[0], b.margins[0]) for i := range a.margins { if m := min(a.margins[i], b.margins[i]); m < d.worstMargin { d.worstMargin = m } } } d.metres, d.metresClass = cableSummary(cable, measuring) return d } func waitLink(mods []*phyModule, done *atomic.Bool) (time.Duration, bool, error) { start := time.Now() deadline := start.Add(linkWaitSpan) for { up := true for _, m := range mods { v, err := m.bcm.linkUp() if err != nil { return 0, false, err } up = up && v } if up { return time.Since(start), true, nil } if time.Now().After(deadline) || (done != nil && done.Load()) { return time.Since(start), false, nil } time.Sleep(linkWaitPoll) } } // The whole cable picture in one pass: the ECD's per-pair verdicts and // lengths, then — after the blip it causes has settled — both ends' pair // maps, read post-link so the MDI resolution is the fresh one. func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableInfo, bool, error) { var c cableInfo var err error c.ecd, err = mods[0].bcm.cableDiag() if err != nil { return c, false, err } relinked := false if waitRelink { if _, relinked, err = waitLink(mods, done); err != nil { return c, false, err } } for i, m := range mods { if c.maps[i], err = m.bcm.pairMap(); err != nil { return c, false, err } } return c, relinked, nil } // Owns the cable picture after bringup: a reset re-measures — the cable under // a reset is usually a different cable — and the counters re-baseline only // once the diag's own link blip is over, so it is never charged to the run. type cableDiag struct { mods []*phyModule completed chan struct{} mu sync.Mutex info cableInfo running bool } func newCableDiag(mods []*phyModule, info cableInfo) *cableDiag { return &cableDiag{mods: mods, completed: make(chan struct{}, 1), info: info} } func (c *cableDiag) snapshot() (cableInfo, bool) { c.mu.Lock() defer c.mu.Unlock() return c.info, c.running } // Runs the re-measure off the display loop, so the panel keeps drawing while // the diag and the relink take their seconds. Reports whether one started; a // press while one is in flight is absorbed. func (c *cableDiag) kick(done *atomic.Bool) bool { c.mu.Lock() if c.running { c.mu.Unlock() return false } c.running = true c.mu.Unlock() go func() { defer holdPanic() info, _, err := measureCable(c.mods, true, done) if err != nil { panic(err) } c.mu.Lock() c.info = info c.running = false c.mu.Unlock() select { case c.completed <- struct{}{}: default: } }() return true } func mapString(m byte) string { if m == pairIdentityMap { return "straight" } out := make([]string, 4) for i := range out { out[i] = pairLetter(int(m>>(2*i)) & 3) } return "swapped to " + strings.Join(out, "") } func verdictString(r ecdResult) string { bad := []string{} for i, v := range r.verdicts { if v != pairOK { s, ok := pairVerdicts[v] if !ok { s = fmt.Sprintf("%d", v) } bad = append(bad, fmt.Sprintf("%s %s at %dm", pairLetter(i), s, r.metres[i])) } } if len(bad) > 0 { return strings.Join(bad, ", ") } return fmt.Sprintf("all pairs ok, %d/%d/%d/%d m", r.metres[0], r.metres[1], r.metres[2], r.metres[3]) } // Runs before any socket opens: forcing EEE off retrains the link and the ECD // blips it, and both belong before the baselines rather than under them. func moduleBringup(names [2]string) ([]*phyModule, cableInfo, []checkResult) { var out []checkResult var cable cableInfo mods := make([]*phyModule, 0, 2) fail := func(item string, err error) ([]*phyModule, cableInfo, []checkResult) { return nil, cable, append(out, checkResult{item: item, err: err}) } for _, name := range names { res := checkResult{item: name + " module"} b, err := openBCM(name) if err != nil { return fail(res.item, err) } res.state, err = b.identify() if err != nil { return fail(res.item, err) } out = append(out, res) mods = append(mods, &phyModule{bcm: b}) } for i, m := range mods { res := checkResult{item: names[i] + " eee", fixed: true} if err := m.bcm.forceEEEOff(); err != nil { return fail(res.item, err) } if err := m.bcm.restartAN(); err != nil { return fail(res.item, err) } res.state = "forced off, retraining" out = append(out, res) } res := checkResult{item: "link retrain"} took, up, err := waitLink(mods, nil) if err != nil { return fail(res.item, err) } if up { var adv [2]string for i, m := range mods { v, err := m.bcm.eeeAdvert() if err != nil { return fail(res.item, err) } adv[i] = fmt.Sprintf("%#04x", v) if v != 0 { res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v) } } res.state = fmt.Sprintf("up in %.1fs, eee advert %s/%s", took.Seconds(), adv[0], adv[1]) } else { res.state = "no link (cable unplugged?)" } out = append(out, res) if res.err != nil { return nil, cable, out } res = checkResult{item: "cable diag"} cable, relinked, err := measureCable(mods, up, nil) if err != nil { return fail(res.item, err) } res.state = verdictString(cable.ecd) if up && !relinked { res.err = fmt.Errorf("link did not return after cable diag") } out = append(out, res) for i := range mods { out = append(out, checkResult{ item: names[i] + " pair map", state: mapString(cable.maps[i]), }) } return mods, cable, out }