cabletest/phy.go

package main

import (
	"fmt"
	"sync"
	"sync/atomic"
	"time"
)

const (
	pairIdentityMap = 0xE4

	fsVendorPN     = "SFP-10G-T-100"
	wiitekVendorPN = "UF-RJ45-10G-100"
)

const (
	pairOK    = 1
	pairOpen  = 2
	pairShort = 3
	pairXtalk = 4
)

var pairVerdicts = map[int]string{
	pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK",
}

type mdioDev interface {
	name() string
	exec(func())
	mdioRead(devad, reg uint16) (uint16, error)
	mdioWrite(devad, reg, val uint16) error
}

type phyDev interface {
	mdioDev
	identify() (string, error)
}

type snrSource interface {
	snrMargins() ([4]float64, error)
}

// PMA 1.1 latches low, so the first read reports any drop since it was last
// read and the second reports the wire as it is now.
func devLinkUp(d mdioDev) (up bool, raw uint16, err error) {
	d.exec(func() {
		if _, err = d.mdioRead(1, 1); err != nil {
			return
		}
		if raw, err = d.mdioRead(1, 1); err != nil {
			return
		}
		up = raw&0x0004 != 0
	})
	return
}

func devPCSLatch(d mdioDev) (blocks, ber uint64, raw uint16, err error) {
	d.exec(func() {
		if raw, err = d.mdioRead(3, 33); err != nil {
			return
		}
		blocks, ber = uint64(raw&0xFF), uint64((raw>>8)&0x3F)
	})
	return
}

func devFastRetrain(d mdioDev) (count, raw uint16, err error) {
	d.exec(func() {
		if raw, err = d.mdioRead(1, 147); err != nil {
			return
		}
		count = raw >> 11
	})
	return
}

// AN enable is forced alongside the restart: the ECD can leave the BCM with
// 7.0.12 cleared (proven live — no AN pulses, both ends deaf, link down until
// power cycle), and a bare restart bit preserves the cleared enable.
func devRestartAN(d mdioDev) (err error) {
	d.exec(func() {
		var v uint16
		if v, err = d.mdioRead(7, 0); err != nil {
			return
		}
		err = d.mdioWrite(7, 0, v|0x1200)
	})
	return
}

func devEEEAdvert(d mdioDev) (v uint16, err error) {
	d.exec(func() { v, err = d.mdioRead(7, 60) })
	return
}

const (
	phyInterval  = time.Second
	phyStale     = 5 * time.Second
	phyMaxDark   = 30
	linkWaitSpan = 25 * time.Second
	linkWaitPoll = time.Second

	snrGoodMargin = 3.0
	snrWarnMargin = 1.0
)

type phyModule struct {
	dev  phyDev
	busy atomic.Bool

	mu           sync.Mutex
	sampled      bool
	lastOK       time.Time
	link         bool
	haveSNR      bool
	margins      [4]float64
	blocks       uint64
	ber          uint64
	retrains     uint64
	recentDelta  uint64
	primed       bool
	retrainCount uint16
	notes        []string
}

// Silent while a measure owns the module.
func (m *phyModule) poll() error {
	if m.busy.Load() {
		return nil
	}
	link, linkRaw, err := devLinkUp(m.dev)
	if err != nil {
		return err
	}
	var margins [4]float64
	haveSNR := false
	if src, ok := m.dev.(snrSource); ok && link {
		if margins, err = src.snrMargins(); err != nil {
			return err
		}
		haveSNR = true
	}
	blocks, ber, pcsRaw, err := devPCSLatch(m.dev)
	if err != nil {
		return err
	}
	count, frRaw, err := devFastRetrain(m.dev)
	if err != nil {
		return err
	}

	m.mu.Lock()
	if m.link && !link {
		m.notes = append(m.notes,
			fmt.Sprintf("%s link read down: 1.1=0x%04x", m.dev.name(), linkRaw))
	}
	m.sampled = true
	m.lastOK = time.Now()
	m.link = link
	m.haveSNR = haveSNR
	m.margins = margins
	// The first poll after a baseline drains latches from the bringup/diag
	// retrain era, so it only sets the origin; the retrain counter is 5 bits.
	if m.primed {
		rt := uint64((count - m.retrainCount) & 0x1F)
		delta := blocks + ber + rt
		if delta > 0 {
			m.notes = append(m.notes, fmt.Sprintf(
				"%s corrected +%d raw: 3.33=0x%04x (blocks %d ber %d) 1.147=0x%04x (retrain +%d) 1.1=0x%04x",
				m.dev.name(), delta, pcsRaw, blocks, ber, frRaw, rt, linkRaw))
		}
		m.blocks += blocks
		m.ber += ber
		m.retrains += rt
		m.recentDelta = delta
	} else {
		m.recentDelta = 0
		m.primed = true
	}
	m.retrainCount = count
	m.mu.Unlock()
	return nil
}

func (m *phyModule) takeNotes() []string {
	m.mu.Lock()
	defer m.mu.Unlock()
	n := m.notes
	m.notes = nil
	return n
}

func (m *phyModule) run(done *atomic.Bool) {
	tick := time.NewTicker(phyInterval)
	defer tick.Stop()

	dark := 0
	var lastErr error
	for !done.Load() {
		<-tick.C
		if err := m.poll(); err != nil {
			dark++
			lastErr = err
			if dark >= phyMaxDark {
				panic(fmt.Sprintf("module diagnostics dark for %d polls: %v", dark, lastErr))
			}
			continue
		}
		dark = 0
	}
}

func (m *phyModule) reset() {
	m.mu.Lock()
	m.blocks, m.ber, m.retrains, m.recentDelta = 0, 0, 0, 0
	m.primed = false
	m.mu.Unlock()
}

// Latches drained after a measure belong to its blip: un-priming makes the
// next poll an origin only, without discarding the pre-measure totals.
func (m *phyModule) forgive() {
	m.mu.Lock()
	m.primed = false
	m.recentDelta = 0
	m.mu.Unlock()
}

type phyModView struct {
	fresh   bool
	link    bool
	haveSNR bool
	margins [4]float64
	blocks  uint64
	ber     uint64
	retrain uint64
	recent  uint64
}

func (m *phyModule) view() phyModView {
	m.mu.Lock()
	defer m.mu.Unlock()
	v := phyModView{
		fresh:   m.sampled && time.Since(m.lastOK) < phyStale,
		link:    m.link,
		haveSNR: m.haveSNR,
		margins: m.margins,
		blocks:  m.blocks,
		ber:     m.ber,
		retrain: m.retrains,
	}
	if v.fresh {
		v.recent = m.recentDelta
	}
	return v
}

type cableInfo struct {
	ecd      ecdResult
	maps     [2]byte
	haveMaps [2]bool
}

func (c cableInfo) metresString() string {
	sum, n := 0, 0
	for i, v := range c.ecd.verdicts {
		if v == pairOK {
			sum += c.ecd.metres[i]
			n++
		}
	}
	if n == 0 {
		return "-"
	}
	return fmt.Sprintf("%d", (sum+n/2)/n)
}

const (
	clsNone = iota
	clsGood
	clsWarn
	clsBad
)

func snrClass(margin float64) int {
	switch {
	case margin >= snrGoodMargin:
		return clsGood
	case margin >= snrWarnMargin:
		return clsWarn
	default:
		return clsBad
	}
}

type phyDisplay struct {
	haveSNR     bool
	worstMargin float64
	corrected   uint64
	recent      uint64
	metres      string
	metresClass int
}

// Each end resolves MDI on its own, so a swap at either known end counts; an
// end with no readable map abstains.
func pairSwapped(i int, c cableInfo) bool {
	for e := range c.maps {
		if c.haveMaps[e] && int(c.maps[e]>>(2*i))&3 != i {
			return true
		}
	}
	return false
}

// The whole cable verdict in one cell: the length when healthy, the first
// faulted pair's verdict when not, "xover" for a pair swap.
func cableSummary(cable cableInfo, measuring bool) (string, int) {
	if measuring {
		return "-", clsNone
	}
	anyData, anySwap := false, false
	for i, v := range cable.ecd.verdicts {
		if v != 0 {
			anyData = true
		}
		if v != 0 && v != pairOK {
			return pairVerdicts[v], clsBad
		}
		if pairSwapped(i, cable) {
			anySwap = true
		}
	}
	switch {
	case !anyData:
		return "-", clsNone
	case anySwap:
		return "xover", clsWarn
	}
	return cable.metresString(), clsGood
}

// The margin is the worst pair across the ends that measure SNR (the Wiitek's
// IEEE 1.133–136), gated on the whole pair being fresh and linked.
func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay {
	d := phyDisplay{
		haveSNR:   a.fresh && b.fresh && a.link && b.link && (a.haveSNR || b.haveSNR),
		corrected: a.blocks + a.ber + a.retrain + b.blocks + b.ber + b.retrain,
		recent:    a.recent + b.recent,
	}
	if d.haveSNR {
		first := true
		for _, v := range []phyModView{a, b} {
			if !v.haveSNR {
				continue
			}
			for _, m := range v.margins {
				if first || m < d.worstMargin {
					d.worstMargin = m
					first = false
				}
			}
		}
	}
	d.metres, d.metresClass = cableSummary(cable, measuring)
	return d
}

type ecdResult struct {
	verdicts [4]int
	metres   [4]int
}

func bcmEnd(mods []*phyModule) *bcm {
	for _, m := range mods {
		if b, ok := m.dev.(*bcm); ok {
			return b
		}
	}
	panic("no BCM module in the pair: the ECD is the only length path")
}

// The pollers are held silent throughout; pair maps are read after the
// relink, so the MDI resolution is the fresh one.
func measureCable(mods []*phyModule, done *atomic.Bool) (cableInfo, error) {
	for _, m := range mods {
		m.busy.Store(true)
	}
	defer func() {
		for _, m := range mods {
			m.forgive()
			m.busy.Store(false)
		}
	}()

	var c cableInfo
	var err error
	end := bcmEnd(mods)
	c.ecd, err = end.cableDiag()
	if err != nil {
		return c, err
	}
	if err = devRestartAN(end); err != nil {
		return c, err
	}
	waitCarrier([2]string{mods[0].dev.name(), mods[1].dev.name()}, done)
	for i, m := range mods {
		b, ok := m.dev.(*bcm)
		if !ok {
			continue
		}
		if c.maps[i], err = b.pairMap(); err != nil {
			return c, err
		}
		c.haveMaps[i] = true
	}
	return c, nil
}

type cableDiag struct {
	mods []*phyModule

	// While set, every failure counter is suppressed at its source: the
	// measure's own link blip is never counted anywhere, rather than counted,
	// hidden and reverted.
	measuring atomic.Bool

	mu   sync.Mutex
	info cableInfo
}

func newCableDiag(mods []*phyModule, info cableInfo) *cableDiag {
	return &cableDiag{mods: mods, info: info}
}

func (c *cableDiag) snapshot() (cableInfo, bool) {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.info, c.measuring.Load()
}

// A measure that errors means the diag transport or the µC is broken, and a
// tester that cannot run its diagnostics must not keep testing: the failure is
// fatal, and the reboot's driver reload is also the wedged-µC recovery.
func (c *cableDiag) kick(done *atomic.Bool) bool {
	if !c.measuring.CompareAndSwap(false, true) {
		return false
	}
	go func() {
		defer holdPanic()
		info, err := measureCable(c.mods, done)
		if err != nil {
			panic(fmt.Sprintf("cable measure: %v", err))
		}
		c.mu.Lock()
		c.info = info
		c.mu.Unlock()
		c.measuring.Store(false)
	}()
	return true
}

func openModules(names [2]string) ([]*phyModule, [2]string, error) {
	mods := make([]*phyModule, 0, 2)
	var idents [2]string
	for i, name := range names {
		t, err := openSFF(name)
		if err != nil {
			return nil, idents, err
		}
		pn, err := t.vendorPN()
		if err != nil {
			return nil, idents, err
		}
		var dev phyDev
		switch pn {
		case fsVendorPN:
			dev = newBCM(t)
		case wiitekVendorPN:
			dev = &rollball{sff: t}
		default:
			return nil, idents, fmt.Errorf("%s: unknown module PN %q", name, pn)
		}
		idents[i], err = dev.identify()
		if err != nil {
			return nil, idents, err
		}
		m := &phyModule{dev: dev}
		// Born busy: the pollers stay silent through bringup's SETs and
		// retrains until the first measure completes and lifts the gate.
		m.busy.Store(true)
		mods = append(mods, m)
	}
	return mods, idents, nil
}

func waitCarrier(names [2]string, done *atomic.Bool) {
	deadline := time.Now().Add(linkWaitSpan)
	for !(carrierUp(names[0]) && carrierUp(names[1])) {
		if time.Now().After(deadline) || done.Load() {
			return
		}
		time.Sleep(linkWaitPoll)
	}
}

// No trustworthy config readback exists (DATA1 is firmware scratch) and no
// cable is guaranteed at bringup, so both settings are forced every boot: the
// one deterministic assurance. The handler freezes during training, so the
// carrier settles — the host checks just reset the links — before any command.
// Never waits for a link: there may be no cable, and forcing config needs
// none — the AN restart applies it whenever training next happens.
func moduleChecks(mods []*phyModule, names [2]string) []checkResult {
	var out []checkResult
	fail := func(item string, err error) []checkResult {
		return append(out, checkResult{item: item, err: err})
	}

	// With a non-BCM partner the BCM is forced slave: the partner's manual
	// config is unreachable, and auto-resolves-master against manual-slave is
	// the combination proven to link.
	mixed := false
	for _, m := range mods {
		if _, ok := m.dev.(*bcm); !ok {
			mixed = true
		}
	}
	master := !mixed
	for i, m := range mods {
		b, ok := m.dev.(*bcm)
		if !ok {
			out = append(out, checkResult{item: names[i] + " role", state: "auto"})
			continue
		}

		res := checkResult{item: names[i] + " eee", state: "forced off"}
		if err := b.forceEEEOff(); err != nil {
			return fail(res.item, err)
		}
		out = append(out, res)

		res = checkResult{item: names[i] + " jumbo", state: "forced on"}
		if err := b.forceJumbo(); err != nil {
			return fail(res.item, err)
		}
		out = append(out, res)

		res = checkResult{item: names[i] + " role", state: "forced master"}
		if !master {
			res.state = "forced slave"
		}
		if err := b.forceRole(master); err != nil {
			return fail(res.item, err)
		}
		master = false
		out = append(out, res)
	}

	// Both modules configured and verified before either AN restart: the
	// modules link to each other, so one restart puts both µCs into training,
	// and no read should race that. The restarts fire last, nothing after.
	res := checkResult{item: "eee advert"}
	var adv [2]uint16
	var advs [2]string
	for i, m := range mods {
		v, err := devEEEAdvert(m.dev)
		if err != nil {
			return fail(res.item, err)
		}
		adv[i] = v
		advs[i] = fmt.Sprintf("%#04x", v)
		if _, ok := m.dev.(*bcm); ok && v != 0 {
			res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v)
		}
	}
	if res.err == nil && adv[0]&adv[1] != 0 {
		res.err = fmt.Errorf("EEE would negotiate: common ability %#04x", adv[0]&adv[1])
	}
	res.state = advs[0] + "/" + advs[1]
	out = append(out, res)

	for i, m := range mods {
		if err := devRestartAN(m.dev); err != nil {
			return fail(names[i]+" retrain", err)
		}
	}
	return out
}