Files
cabletest/phy.go
T

880 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
)
const (
bcmI2CWrite = 0xAC
bcmI2CRead = 0xAD
bcmMMDVendor uint16 = 0x1E
bcmRegCmd uint16 = 0x4005
bcmRegStatus uint16 = 0x4037
bcmRegData1 uint16 = 0x4038
bcmStInProgress uint16 = 0x0002
bcmStPass uint16 = 0x0004
bcmStError uint16 = 0x0008
bcmStBusy uint16 = 0xBBBB
bcmCmdGetPairSwap uint16 = 0x8000
bcmCmdSetEEEMode uint16 = 0x8009
bcmCmdSetJumbo uint16 = 0x801C
bcmCmdGetSNR uint16 = 0x8030
bcmRegECDCtrl uint16 = 0x4006
bcmRegECDResult uint16 = 0xA896
bcmRegECDLen uint16 = 0xA897
bcmPHYIDHi = 0x3590
bcmPHYIDLo = 0x5081
bcmCmdResidentTemp uint16 = 0x0031
bcmReadDelayUs = 3000
bcmRetryDelayUs = 10000
bcmWindow = 3400 * time.Millisecond
bcmWindowFit = 100 * time.Millisecond
bcmFlipPoll = 10 * time.Millisecond
bcmFlipWait = 5 * time.Second
bcmStatusPoll = 100 * time.Millisecond
// Covers the handler's documented 2 s freeze during 10GBASE-T training.
bcmStatusTimeout = 3 * time.Second
ecdPoll = 200 * time.Millisecond
ecdDeadline = 50 * time.Second
pairIdentityMap = 0xE4
)
const (
pairOK = 1
pairOpen = 2
pairShort = 3
pairXtalk = 4
)
var pairVerdicts = map[int]string{
pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK",
}
type bcm struct {
ifname string
path string
// Every method holds it for its whole logical operation: exactly one
// host-side conversation with the module at a time, by construction.
mu sync.Mutex
windowEnd time.Time
windowed atomic.Bool
}
// The firmware's internal temp poll (every 3.54.2 s, never under 3.49) serves
// stale bridge reads for ~50 ms around it; work stays inside 3.4 s post-poll.
func (b *bcm) window() {
if !b.windowed.Load() || time.Now().Add(bcmWindowFit).Before(b.windowEnd) {
return
}
deadline := time.Now().Add(bcmFlipWait)
for {
v, err := b.mdioRead(bcmMMDVendor, bcmRegCmd)
if err != nil {
panic(fmt.Sprintf("%s: heartbeat poll: %v", b.ifname, err))
}
if v == bcmCmdResidentTemp || time.Now().After(deadline) {
b.windowEnd = time.Now().Add(bcmWindow)
return
}
time.Sleep(bcmFlipPoll)
}
}
func (b *bcm) acquire() func() {
b.mu.Lock()
b.window()
return b.mu.Unlock
}
func openBCM(ifname string) (*bcm, error) {
devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device")
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
drv, err := ifDriver(ifname)
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
if drv != "ixgbe" {
return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv)
}
b := &bcm{
ifname: ifname,
path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c",
}
if _, err := os.Stat(b.path); err != nil {
return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err)
}
return b, nil
}
func (b *bcm) op(cmd string) (string, error) {
fd, err := unix.Open(b.path, unix.O_RDWR, 0)
if err != nil {
return "", fmt.Errorf("%s: %w", b.path, err)
}
defer unix.Close(fd)
if _, err := unix.Write(fd, []byte(cmd)); err != nil {
return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err)
}
buf := make([]byte, 256)
n, err := unix.Read(fd, buf)
if err != nil {
return "", fmt.Errorf("%s %q: %w", b.ifname, cmd, err)
}
resp := strings.TrimSpace(string(buf[:n]))
if !strings.HasPrefix(resp, "ok") {
return "", fmt.Errorf("%s %q: %s", b.ifname, cmd, resp)
}
return strings.TrimSpace(resp[2:]), nil
}
func parseHexBytes(s string, n int) ([]byte, error) {
fields := strings.Fields(s)
if len(fields) != n {
return nil, fmt.Errorf("want %d bytes, got %q", n, s)
}
out := make([]byte, n)
for i, f := range fields {
var v byte
if _, err := fmt.Sscanf(f, "%x", &v); err != nil {
return nil, fmt.Errorf("byte %q in %q", f, s)
}
out[i] = v
}
return out, nil
}
// A single bus hold; split write/read ops would let the driver's own SFP
// traffic consume the bridge's pending read.
func (b *bcm) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) {
var sb strings.Builder
fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n)
for _, v := range wdata {
fmt.Fprintf(&sb, " %02x", v)
}
resp, err := b.op(sb.String())
if err != nil {
return nil, err
}
return parseHexBytes(resp, n)
}
func (b *bcm) mdioReadDelay(devad, reg uint16, delayUs int) (uint16, error) {
d, err := b.compound(bcmI2CWrite, bcmI2CRead, delayUs, 2,
[]byte{0x20 | byte(devad), byte(reg >> 8), byte(reg)})
if err != nil {
return 0, err
}
return uint16(d[0])<<8 | uint16(d[1]), nil
}
// 0x0000 is also the bridge's not-ready signature, so a zero is read again at
// a longer delay before being believed.
func (b *bcm) mdioRead(devad, reg uint16) (uint16, error) {
v, err := b.mdioReadDelay(devad, reg, bcmReadDelayUs)
if err != nil || v != 0 {
return v, err
}
return b.mdioReadDelay(devad, reg, bcmRetryDelayUs)
}
func (b *bcm) mdioWrite(devad, reg, val uint16) error {
_, err := b.op(fmt.Sprintf("w %02x %02x %02x %02x %02x %02x",
bcmI2CWrite, byte(devad), byte(reg>>8), byte(reg), byte(val>>8), byte(val)))
return err
}
func (b *bcm) eeprom(off byte, n int) ([]byte, error) {
return b.compound(0xA0, 0xA1, 500, n, []byte{off})
}
// The datasheet's completion handshake: poll STATUS on its 100 ms cadence
// until the wanted state, bounded by a deadline.
func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) {
deadline := time.Now().Add(bcmStatusTimeout)
for {
st, err := b.mdioRead(bcmMMDVendor, bcmRegStatus)
if err != nil {
return 0, err
}
if want(st) {
return st, nil
}
if time.Now().After(deadline) {
return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st)
}
time.Sleep(bcmStatusPoll)
}
}
// GETs must be invoked bare (pre-writing any DATA register leaves the handler
// executing as a no-op); SETs must pass their full parameter set (the handler
// executes stale DATA).
func (b *bcm) command(code uint16, params ...uint16) ([5]uint16, error) {
defer b.acquire()()
var data [5]uint16
if _, err := b.waitStatus(func(st uint16) bool {
return st != bcmStInProgress && st != bcmStBusy
}); err != nil {
return data, err
}
for i, p := range params {
if err := b.mdioWrite(bcmMMDVendor, bcmRegData1+uint16(i), p); err != nil {
return data, err
}
}
if err := b.mdioWrite(bcmMMDVendor, bcmRegCmd, code); err != nil {
return data, err
}
st, err := b.waitStatus(func(st uint16) bool {
return st == bcmStPass || st == bcmStError
})
if err != nil {
return data, err
}
if st == bcmStError {
return data, fmt.Errorf("%s: command %#04x returned ERROR", b.ifname, code)
}
if len(params) > 0 {
return data, nil
}
for i := range data {
if data[i], err = b.mdioRead(bcmMMDVendor, bcmRegData1+uint16(i)); err != nil {
return data, err
}
}
return data, nil
}
func (b *bcm) identify() (string, error) {
defer b.acquire()()
hi, err := b.mdioRead(1, 2)
if err != nil {
return "", err
}
lo, err := b.mdioRead(1, 3)
if err != nil {
return "", err
}
if hi != bcmPHYIDHi || lo != bcmPHYIDLo {
return "", fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x",
b.ifname, hi, lo, bcmPHYIDHi, bcmPHYIDLo)
}
sn, err := b.eeprom(68, 16)
if err != nil {
return "", err
}
return "BCM84891L sn " + strings.TrimSpace(string(sn)), nil
}
// PMA 1.1 latches low, so the first read reports any drop since it was last
// read and the second reports the wire as it is now.
func (b *bcm) linkUp() (bool, error) {
defer b.acquire()()
if _, err := b.mdioRead(1, 1); err != nil {
return false, err
}
v, err := b.mdioRead(1, 1)
if err != nil {
return false, err
}
return v&0x0004 != 0, nil
}
func (b *bcm) forceEEEOff() error {
_, err := b.command(bcmCmdSetEEEMode, 0x0000, 0x0000, 0x7A12, 0x0480, 0x0000)
return err
}
func (b *bcm) forceJumbo() error {
_, err := b.command(bcmCmdSetJumbo, 1, 0, 0, 0, 0)
return err
}
// Left to AN, master/slave is a per-training lottery and each training's DSP
// convergence moves per-pair SNR by up to ~3.6 dB; pinned roles at least keep
// every session measured under identical conditions.
func (b *bcm) forceRole(master bool) error {
defer b.acquire()()
v, err := b.mdioRead(7, 32)
if err != nil {
return err
}
v |= 0x8000
if master {
v |= 0x4000
} else {
v &^= 0x4000
}
return b.mdioWrite(7, 32, v)
}
func (b *bcm) restartAN() error {
defer b.acquire()()
v, err := b.mdioRead(7, 0)
if err != nil {
return err
}
return b.mdioWrite(7, 0, v|0x0200)
}
func (b *bcm) eeeAdvert() (uint16, error) {
defer b.acquire()()
return b.mdioRead(7, 60)
}
func (b *bcm) pairMap() (byte, error) {
d, err := b.command(bcmCmdGetPairSwap)
if err != nil {
return 0, err
}
return byte(d[1]), nil
}
func (b *bcm) snr() ([4]float64, error) {
var out [4]float64
d, err := b.command(bcmCmdGetSNR)
if err != nil {
return out, err
}
for i := range out {
out[i] = float64(d[i+1]) / 10
}
return out, nil
}
func (b *bcm) pcsLatch() (blocks, ber uint64, err error) {
defer b.acquire()()
v, err := b.mdioRead(3, 33)
if err != nil {
return 0, 0, err
}
return uint64(v & 0xFF), uint64((v >> 8) & 0x3F), nil
}
func (b *bcm) fastRetrainCount() (uint16, error) {
defer b.acquire()()
v, err := b.mdioRead(1, 147)
if err != nil {
return 0, err
}
return v >> 11, nil
}
type ecdResult struct {
verdicts [4]int
metres [4]int
}
func (b *bcm) cableDiag() (ecdResult, error) {
defer b.acquire()()
var res ecdResult
ctrl, err := b.mdioRead(bcmMMDVendor, bcmRegECDCtrl)
if err != nil {
return res, err
}
if err := b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil {
return res, err
}
deadline := time.Now().Add(ecdDeadline)
for {
ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl)
if err != nil {
return res, err
}
if ctrl&0x0800 == 0 {
break
}
if time.Now().After(deadline) {
return res, fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline)
}
time.Sleep(ecdPoll)
}
b.window()
v, err := b.mdioRead(1, bcmRegECDResult)
if err != nil {
return res, err
}
for i := range res.verdicts {
res.verdicts[i] = int(v>>(4*i)) & 0xF
if res.verdicts[i] > pairXtalk {
panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v))
}
m, err := b.mdioRead(1, bcmRegECDLen+uint16(i))
if err != nil {
return res, err
}
res.metres[i] = int(m)
}
return res, nil
}
const (
phyInterval = time.Second
phyStale = 5 * time.Second
phyMaxDark = 30
linkWaitSpan = 25 * time.Second
linkWaitPoll = time.Second
snrOperatingPoint = 26.5
snrGoodMargin = 3.0
snrWarnMargin = 1.0
// No trained link produces SNR outside this; readings there are another
// register's data (die temp ≈ 8, handler status ≈ 0.4) and the run dies.
snrGhostLow = 15.0
snrGhostHigh = 50.0
)
type phyModule struct {
bcm *bcm
busy atomic.Bool
mu sync.Mutex
sampled bool
lastOK time.Time
link bool
haveSNR bool
snr [4]float64
blocks uint64
ber uint64
retrains uint64
recentDelta uint64
primed bool
retrainCount uint16
}
// Silent while a measure owns the module.
func (m *phyModule) poll() error {
if m.busy.Load() {
return nil
}
link, err := m.bcm.linkUp()
if err != nil {
return err
}
var snr [4]float64
if link {
if snr, err = m.bcm.snr(); err != nil {
return err
}
for _, s := range snr {
if s < snrGhostLow || s > snrGhostHigh {
panic(fmt.Sprintf("%s: ghost SNR %.1f dB", m.bcm.ifname, s))
}
}
}
blocks, ber, err := m.bcm.pcsLatch()
if err != nil {
return err
}
count, err := m.bcm.fastRetrainCount()
if err != nil {
return err
}
m.mu.Lock()
m.sampled = true
m.lastOK = time.Now()
m.link = link
m.haveSNR = link
m.snr = snr
// The first poll after a baseline drains latches from the bringup/diag
// retrain era, so it only sets the origin; the retrain counter is 5 bits.
if m.primed {
delta := blocks + ber + uint64((count-m.retrainCount)&0x1F)
m.blocks += blocks
m.ber += ber
m.retrains += uint64((count - m.retrainCount) & 0x1F)
m.recentDelta = delta
} else {
m.recentDelta = 0
m.primed = true
}
m.retrainCount = count
m.mu.Unlock()
return nil
}
func (m *phyModule) run(done *atomic.Bool) {
tick := time.NewTicker(phyInterval)
defer tick.Stop()
dark := 0
var lastErr error
for !done.Load() {
<-tick.C
if err := m.poll(); err != nil {
dark++
lastErr = err
if dark >= phyMaxDark {
panic(fmt.Sprintf("module diagnostics dark for %d polls: %v", dark, lastErr))
}
continue
}
dark = 0
}
}
func (m *phyModule) reset() {
m.mu.Lock()
m.blocks, m.ber, m.retrains, m.recentDelta = 0, 0, 0, 0
m.primed = false
m.mu.Unlock()
}
type phyModView struct {
fresh bool
link bool
margins [4]float64
blocks uint64
ber uint64
retrain uint64
recent uint64
}
func (m *phyModule) view() phyModView {
m.mu.Lock()
defer m.mu.Unlock()
v := phyModView{
fresh: m.sampled && time.Since(m.lastOK) < phyStale,
link: m.link && m.haveSNR,
blocks: m.blocks,
ber: m.ber,
retrain: m.retrains,
}
if v.fresh {
v.recent = m.recentDelta
}
for i, s := range m.snr {
v.margins[i] = s - snrOperatingPoint
}
return v
}
type cableInfo struct {
ecd ecdResult
maps [2]byte
}
func (c cableInfo) metresString() string {
sum, n := 0, 0
for i, v := range c.ecd.verdicts {
if v == pairOK {
sum += c.ecd.metres[i]
n++
}
}
if n == 0 {
return "-"
}
return fmt.Sprintf("%d", (sum+n/2)/n)
}
const (
clsNone = iota
clsGood
clsWarn
clsBad
)
func snrClass(margin float64) int {
switch {
case margin >= snrGoodMargin:
return clsGood
case margin >= snrWarnMargin:
return clsWarn
default:
return clsBad
}
}
type phyDisplay struct {
haveSNR bool
worstMargin float64
corrected uint64
recent uint64
metres string
metresClass int
}
func pairLetter(i int) string { return string(rune('A' + i)) }
// Each end resolves MDI on its own, so a swap at either end counts.
func pairSwapped(i int, maps [2]byte) bool {
return int(maps[0]>>(2*i))&3 != i || int(maps[1]>>(2*i))&3 != i
}
func cableSummary(cable cableInfo, measuring bool) (string, int) {
if measuring {
return "...", clsNone
}
anyData, anyFault, anySwap := false, false, false
for i, v := range cable.ecd.verdicts {
if v != 0 {
anyData = true
}
if v != 0 && v != pairOK {
anyFault = true
}
if pairSwapped(i, cable.maps) {
anySwap = true
}
}
s := cable.metresString()
switch {
case !anyData:
return "-", clsNone
case anyFault:
return s, clsBad
case anySwap:
return s, clsWarn
}
return s, clsGood
}
func phyDisplayFrom(cable cableInfo, measuring bool, a, b phyModView) phyDisplay {
d := phyDisplay{
haveSNR: a.fresh && b.fresh && a.link && b.link,
corrected: a.blocks + a.ber + a.retrain + b.blocks + b.ber + b.retrain,
recent: a.recent + b.recent,
}
if d.haveSNR {
d.worstMargin = min(a.margins[0], b.margins[0])
for i := range a.margins {
if m := min(a.margins[i], b.margins[i]); m < d.worstMargin {
d.worstMargin = m
}
}
}
d.metres, d.metresClass = cableSummary(cable, measuring)
return d
}
// The pollers are held silent throughout; pair maps are read after the
// relink, so the MDI resolution is the fresh one.
func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableInfo, bool, error) {
for _, m := range mods {
m.busy.Store(true)
}
defer func() {
for _, m := range mods {
m.busy.Store(false)
}
}()
var c cableInfo
var err error
c.ecd, err = mods[0].bcm.cableDiag()
if err != nil {
return c, false, err
}
relinked := false
if waitRelink {
names := [2]string{mods[0].bcm.ifname, mods[1].bcm.ifname}
_, relinked = waitCarrier(names, done)
}
for i, m := range mods {
if c.maps[i], err = m.bcm.pairMap(); err != nil {
return c, false, err
}
}
return c, relinked, nil
}
type cableDiag struct {
mods []*phyModule
completed chan error
mu sync.Mutex
info cableInfo
running bool
}
func newCableDiag(mods []*phyModule, info cableInfo) *cableDiag {
return &cableDiag{mods: mods, completed: make(chan error, 1), info: info}
}
func (c *cableDiag) snapshot() (cableInfo, bool) {
c.mu.Lock()
defer c.mu.Unlock()
return c.info, c.running
}
func (c *cableDiag) kick(done *atomic.Bool) bool {
c.mu.Lock()
if c.running {
c.mu.Unlock()
return false
}
c.running = true
c.mu.Unlock()
go func() {
defer holdPanic()
info, _, err := measureCable(c.mods, true, done)
if err != nil {
info = cableInfo{}
}
c.mu.Lock()
c.info = info
c.running = false
c.mu.Unlock()
select {
case c.completed <- err:
default:
}
}()
return true
}
func mapString(m byte) string {
if m == pairIdentityMap {
return "straight"
}
out := make([]string, 4)
for i := range out {
out[i] = pairLetter(int(m>>(2*i)) & 3)
}
return "swapped to " + strings.Join(out, "")
}
func verdictString(r ecdResult) string {
bad := []string{}
for i, v := range r.verdicts {
if v != pairOK {
s, ok := pairVerdicts[v]
if !ok {
s = fmt.Sprintf("%d", v)
}
bad = append(bad, fmt.Sprintf("%s %s at %dm", pairLetter(i), s, r.metres[i]))
}
}
if len(bad) > 0 {
return strings.Join(bad, ", ")
}
return fmt.Sprintf("all pairs ok, %d/%d/%d/%d m",
r.metres[0], r.metres[1], r.metres[2], r.metres[3])
}
func openModules(names [2]string) ([]*phyModule, [2]string, error) {
mods := make([]*phyModule, 0, 2)
var idents [2]string
for i, name := range names {
b, err := openBCM(name)
if err != nil {
return nil, idents, err
}
idents[i], err = b.identify()
if err != nil {
return nil, idents, err
}
m := &phyModule{bcm: b}
// Born busy: the pollers stay silent through bringup's SETs and
// retrains until the first measure completes and lifts the gate.
m.busy.Store(true)
mods = append(mods, m)
}
return mods, idents, nil
}
func waitCarrier(names [2]string, done *atomic.Bool) (time.Duration, bool) {
start := time.Now()
deadline := start.Add(linkWaitSpan)
for {
if carrierUp(names[0]) && carrierUp(names[1]) {
return time.Since(start), true
}
if time.Now().After(deadline) || (done != nil && done.Load()) {
return time.Since(start), false
}
time.Sleep(linkWaitPoll)
}
}
// No trustworthy config readback exists (DATA1 is firmware scratch) and no
// cable is guaranteed at bringup, so both settings are forced every boot: the
// one deterministic assurance. The handler freezes during training, so the
// carrier settles — the host checks just reset the links — before any command.
// Never waits for a link: there may be no cable, and forcing config needs
// none — the AN restart applies it whenever training next happens.
func moduleChecks(mods []*phyModule, names [2]string) []checkResult {
var out []checkResult
fail := func(item string, err error) []checkResult {
return append(out, checkResult{item: item, err: err})
}
for i, m := range mods {
res := checkResult{item: names[i] + " eee", state: "forced off"}
if err := m.bcm.forceEEEOff(); err != nil {
return fail(res.item, err)
}
out = append(out, res)
res = checkResult{item: names[i] + " jumbo", state: "forced on"}
if err := m.bcm.forceJumbo(); err != nil {
return fail(res.item, err)
}
out = append(out, res)
res = checkResult{item: names[i] + " role", state: "forced master"}
if i > 0 {
res.state = "forced slave"
}
if err := m.bcm.forceRole(i == 0); err != nil {
return fail(res.item, err)
}
out = append(out, res)
if err := m.bcm.restartAN(); err != nil {
return fail(names[i]+" retrain", err)
}
}
res := checkResult{item: "eee advert"}
var adv [2]string
for i, m := range mods {
v, err := m.bcm.eeeAdvert()
if err != nil {
return fail(res.item, err)
}
adv[i] = fmt.Sprintf("%#04x", v)
if v != 0 {
res.err = fmt.Errorf("%s still advertises EEE %#04x", names[i], v)
}
}
res.state = adv[0] + "/" + adv[1]
for _, m := range mods {
m.bcm.windowed.Store(true)
}
return append(out, res)
}
func cableLine(c cableInfo) string {
return fmt.Sprintf("%s; map %s / %s",
verdictString(c.ecd), mapString(c.maps[0]), mapString(c.maps[1]))
}