theater/cabletest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bugs-evident cleanup: cable-measure failure is fatal with its reason (failed/fail pretend-path and dead relinked plumbing deleted), bcm.command refuses partial SETs (0 or 5 params, stale-DATA trap made structural), BOOT gains a patched-ixgbe check naming stock-driver boots, panel fault rows matched by label and console rows strict on column count, phy.go split into sff/bcm/rollball and main.go's direction machinery into direction.go (pure moves), input events decoded via encoding/binary, gofmt; verified on hardware (20.00G, zero errors, ECD 41m)

Browse Source
This commit is contained in:
flamingcow
2026-08-17 18:56:55 -07:00
parent 38c2cc4da2
commit 916c0150ef
12 changed files with 1109 additions and 1049 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bcm.go
+307
View File
@@ -0,0 +1,307 @@
package main
import (
"fmt"
"strings"
"time"
)
const (
bcmI2CWrite = 0xAC
bcmI2CRead = 0xAD
bcmMMDVendor uint16 = 0x1E
bcmRegCmd uint16 = 0x4005
bcmRegStatus uint16 = 0x4037
bcmRegData1 uint16 = 0x4038
bcmStInProgress uint16 = 0x0002
bcmStPass uint16 = 0x0004
bcmStError uint16 = 0x0008
bcmStBusy uint16 = 0xBBBB
bcmCmdGetPairSwap uint16 = 0x8000
bcmCmdSetEEEMode uint16 = 0x8009
bcmCmdSetJumbo uint16 = 0x801C
bcmRegECDCtrl uint16 = 0x4006
bcmRegECDResult uint16 = 0xA896
bcmRegECDLen uint16 = 0xA897
bcmPHYIDHi = 0x3590
bcmPHYIDLo = 0x5081
bcmCmdResidentTemp uint16 = 0x0031
bcmReadDelayUs = 3000
bcmRetryDelayUs = 10000
bcmWindow = 3400 * time.Millisecond
bcmWindowFit = 100 * time.Millisecond
bcmFlipPoll = 10 * time.Millisecond
bcmFlipWait = 5 * time.Second
bcmStatusPoll = 100 * time.Millisecond
// Covers the handler's documented 2 s freeze during 10GBASE-T training.
bcmStatusTimeout = 3 * time.Second
ecdPoll = 200 * time.Millisecond
ecdDeadline = 50 * time.Second
)
type bcm struct {
*sff
windowEnd time.Time
}
func newBCM(t *sff) *bcm {
b := &bcm{sff: t}
t.exec(func() { t.admit = b.window })
return b
}
// The firmware's internal temp poll serves stale bridge reads for ~50 ms
// around it; work stays inside 3.4 s of an observed poll. A resident 0x0031 at
// expiry means the phase is unknown, so re-lock: arm, then take the true edge.
// Each taken edge immediately re-arms — the one CMD write per window lands at
// the start of the quiet period, maximally far from the next poll (writes near
// the poll are the µC-wedge risk), and every later expiry reads the phase
// without writing.
func (b *bcm) window() {
if time.Now().Add(bcmWindowFit).Before(b.windowEnd) {
return
}
armed := false
deadline := time.Now().Add(bcmFlipWait)
for {
v, err := b.mdioRead(bcmMMDVendor, bcmRegCmd)
if err != nil {
panic(fmt.Sprintf("%s: heartbeat poll: %v", b.ifname, err))
}
if v == bcmCmdResidentTemp {
if armed {
b.windowEnd = time.Now().Add(bcmWindow)
b.rearm()
return
}
b.rearm()
armed = true
deadline = time.Now().Add(bcmFlipWait)
continue
}
armed = true
if time.Now().After(deadline) {
b.windowEnd = time.Now().Add(bcmWindow)
return
}
time.Sleep(bcmFlipPoll)
}
}
func (b *bcm) rearm() {
if _, err := b.waitStatus(func(st uint16) bool {
return st != bcmStInProgress && st != bcmStBusy
}); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
if err := b.mdioWrite(bcmMMDVendor, bcmRegCmd, bcmCmdGetPairSwap); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
if _, err := b.waitStatus(func(st uint16) bool {
return st == bcmStPass || st == bcmStError
}); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
}
func (b *bcm) mdioReadDelay(devad, reg uint16, delayUs int) (uint16, error) {
d, err := b.compound(bcmI2CWrite, bcmI2CRead, delayUs, 2,
[]byte{0x20 | byte(devad), byte(reg >> 8), byte(reg)})
if err != nil {
return 0, err
}
return uint16(d[0])<<8 | uint16(d[1]), nil
}
// 0x0000 is also the bridge's not-ready signature, so a zero is read again at
// a longer delay before being believed.
func (b *bcm) mdioRead(devad, reg uint16) (uint16, error) {
v, err := b.mdioReadDelay(devad, reg, bcmReadDelayUs)
if err != nil || v != 0 {
return v, err
}
return b.mdioReadDelay(devad, reg, bcmRetryDelayUs)
}
func (b *bcm) mdioWrite(devad, reg, val uint16) error {
_, err := b.op(fmt.Sprintf("w %02x %02x %02x %02x %02x %02x",
bcmI2CWrite, byte(devad), byte(reg>>8), byte(reg), byte(val>>8), byte(val)))
return err
}
// The datasheet's completion handshake: poll STATUS on its 100 ms cadence
// until the wanted state, bounded by a deadline.
func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) {
deadline := time.Now().Add(bcmStatusTimeout)
for {
st, err := b.mdioRead(bcmMMDVendor, bcmRegStatus)
if err != nil {
return 0, err
}
if want(st) {
return st, nil
}
if time.Now().After(deadline) {
return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st)
}
time.Sleep(bcmStatusPoll)
}
}
// GETs must be invoked bare (pre-writing any DATA register leaves the handler
// executing as a no-op); SETs must write every DATA register (the handler
// executes stale DATA), so a partial parameter set is refused outright.
func (b *bcm) command(code uint16, params ...uint16) (data [5]uint16, err error) {
if len(params) != 0 && len(params) != len(data) {
panic(fmt.Sprintf("%s: command %#04x with %d params: a SET must write all %d DATA registers",
b.ifname, code, len(params), len(data)))
}
b.exec(func() {
if _, err = b.waitStatus(func(st uint16) bool {
return st != bcmStInProgress && st != bcmStBusy
}); err != nil {
return
}
for i, p := range params {
if err = b.mdioWrite(bcmMMDVendor, bcmRegData1+uint16(i), p); err != nil {
return
}
}
if err = b.mdioWrite(bcmMMDVendor, bcmRegCmd, code); err != nil {
return
}
var st uint16
if st, err = b.waitStatus(func(st uint16) bool {
return st == bcmStPass || st == bcmStError
}); err != nil {
return
}
if st == bcmStError {
err = fmt.Errorf("%s: command %#04x returned ERROR", b.ifname, code)
return
}
if len(params) > 0 {
return
}
for i := range data {
if data[i], err = b.mdioRead(bcmMMDVendor, bcmRegData1+uint16(i)); err != nil {
return
}
}
})
return
}
func (b *bcm) identify() (ident string, err error) {
b.exec(func() {
var hi, lo uint16
if hi, err = b.mdioRead(1, 2); err != nil {
return
}
if lo, err = b.mdioRead(1, 3); err != nil {
return
}
if hi != bcmPHYIDHi || lo != bcmPHYIDLo {
err = fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x",
b.ifname, hi, lo, bcmPHYIDHi, bcmPHYIDLo)
return
}
var sn []byte
if sn, err = b.eeprom(68, 16); err != nil {
return
}
ident = "BCM84891L sn " + strings.TrimSpace(string(sn))
})
return
}
func (b *bcm) forceEEEOff() error {
_, err := b.command(bcmCmdSetEEEMode, 0x0000, 0x0000, 0x7A12, 0x0480, 0x0000)
return err
}
func (b *bcm) forceJumbo() error {
_, err := b.command(bcmCmdSetJumbo, 1, 0, 0, 0, 0)
return err
}
// Left to AN, master/slave is a per-training lottery and each training's DSP
// convergence moves per-pair SNR by up to ~3.6 dB; pinned roles at least keep
// every session measured under identical conditions.
func (b *bcm) forceRole(master bool) (err error) {
b.exec(func() {
var v uint16
if v, err = b.mdioRead(7, 32); err != nil {
return
}
v |= 0x8000
if master {
v |= 0x4000
} else {
v &^= 0x4000
}
err = b.mdioWrite(7, 32, v)
})
return
}
func (b *bcm) pairMap() (byte, error) {
d, err := b.command(bcmCmdGetPairSwap)
if err != nil {
return 0, err
}
return byte(d[1]), nil
}
func (b *bcm) cableDiag() (res ecdResult, err error) {
b.exec(func() {
var ctrl uint16
if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil {
return
}
if err = b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil {
return
}
deadline := time.Now().Add(ecdDeadline)
for {
if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil {
return
}
if ctrl&0x0800 == 0 {
break
}
if time.Now().After(deadline) {
err = fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline)
return
}
time.Sleep(ecdPoll)
}
b.window()
var v uint16
if v, err = b.mdioRead(1, bcmRegECDResult); err != nil {
return
}
for i := range res.verdicts {
res.verdicts[i] = int(v>>(4*i)) & 0xF
if res.verdicts[i] > pairXtalk {
panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v))
}
var m uint16
if m, err = b.mdioRead(1, bcmRegECDLen+uint16(i)); err != nil {
return
}
res.metres[i] = int(m)
}
})
return
}
direction.go
+420
View File
@@ -0,0 +1,420 @@
package main
import (
"fmt"
"math"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
)
const wireOverhead = 24
type direction struct {
specs []*frameSpec
txStats []*txStats
rxStats []*rxStats
streams []lossWindow
txFDs []int
rxFDs []int
statFD int
// While a cable measure runs, every failure counter in this direction is
// suppressed at its source rather than counted, hidden and reverted.
measuring *atomic.Bool
// Guards everything the sampler touches. The counters are read on their own
// clock and drawn on another, and the two must not read them at once:
// sampleDrops consumes what it reads, so a second caller would see a gap.
mu sync.Mutex
win *rateWindow
drops uint64
base counterSet
// The smeared receive bucket the display draws its rate from, refreshed by
// the sampler because the buckets are keyed by the shared read clock and
// staleness has to be judged against the wall.
rateFrames uint64
rateBytes uint64
epoch int64
epochAt time.Time
// The settled window: each completed bucket enters once, donates its
// excess backward once, and leaves for display once no later bucket can
// still refill it. Recomputing the smear from the raw ring every sample
// instead would show each excess twice — once as the donation and again,
// unspent, when its bucket reaches the display slot of a later window.
smFrames [smearWindow]uint64
smBytes [smearWindow]uint64
smLen int
smNext int64
nic atomic.Uint64
poller *nicPoller
}
// Everything the display reads, taken at one instant, so a pair of these
// describes both the rates and the errors over the span between them.
type counterSet struct {
t time.Time
s sample
drops uint64
nic uint64
}
func (d *direction) capture() counterSet {
d.sampleDrops()
s := d.snapshot()
return counterSet{t: time.Now(), s: s, drops: d.drops, nic: d.nic.Load()}
}
// What someone testing a cable is asking, rather than how each failure happened
// to be noticed.
type errs struct {
lost uint64
corrupt uint64
link uint64
internal uint64
}
func (e errs) add(o errs) errs {
return errs{
lost: e.lost + o.lost, corrupt: e.corrupt + o.corrupt,
link: e.link + o.link, internal: e.internal + o.internal,
}
}
// A ring of one bucket per drawn frame, spanning rateWindowSpan. Rates come
// from the gap between adjacent buckets and errors from the ends of the ring,
// so both slide forward every frame instead of stepping once a second.
type rateWindow struct {
buf []counterSet
idx int
filled bool
}
func newRateWindow(n int) *rateWindow {
return &rateWindow{buf: make([]counterSet, n)}
}
func (w *rateWindow) push(c counterSet) {
w.buf[w.idx] = c
w.idx++
if w.idx == len(w.buf) {
w.idx = 0
w.filled = true
}
}
func (w *rateWindow) count() int {
if w.filled {
return len(w.buf)
}
return w.idx
}
// Indexed oldest first, so a partly filled ring reads the same as a full one.
func (w *rateWindow) at(i int) counterSet {
if w.filled {
i += w.idx
}
return w.buf[i%len(w.buf)]
}
// How long the frontier may sit still before the wire is taken to have gone
// quiet. It only advances when frames arrive on every stream, so a frozen
// frontier means a stream has stopped delivering rather than an unchanged rate.
const rateStale = 100 * time.Millisecond
// An epoch is only reachable once a worker drains frames into it, so each
// stream's newest epoch is a frontier: everything that queue has been read
// through. Epochs behind the lowest frontier — the leader's would claim
// buckets the stragglers are still filling — are closed to further commits,
// so each is settled into the window exactly once. The display takes the
// window's oldest bucket, the one no later bucket can still refill, so the
// headline runs one window behind real time.
func (d *direction) readRateBucket(now time.Time) {
newest := int64(math.MaxInt64)
for _, r := range d.rxStats {
if e := r.newest.Load(); e < newest {
newest = e
}
}
if newest > d.epoch {
d.epoch, d.epochAt = newest, now
}
if d.smNext == 0 && d.epoch > 1 {
d.smNext = d.epoch - 1
}
for e := d.smNext; e > 0 && e < d.epoch; e++ {
d.settle(e)
d.smNext = e + 1
}
d.rateFrames, d.rateBytes = 0, 0
if d.smLen == 0 || now.Sub(d.epochAt) > rateStale {
return
}
d.rateFrames, d.rateBytes = d.smFrames[0], d.smBytes[0]
}
func (d *direction) settle(e int64) {
if d.smLen == smearWindow {
copy(d.smFrames[:], d.smFrames[1:])
copy(d.smBytes[:], d.smBytes[1:])
d.smLen--
}
var f, b uint64
for _, r := range d.rxStats {
rf, rb := r.bucket(e)
f += rf
b += rb
}
d.smFrames[d.smLen], d.smBytes[d.smLen] = f, b
d.smLen++
fillBack(d.smFrames[:d.smLen], d.smBytes[:d.smLen])
}
type sample struct {
rxFrames, rxBytes uint64
lost, late uint64
crcErr, badMagic uint64
badHdr uint64
badLen uint64
txErrs uint64
rxErrs uint64
}
func (d *direction) snapshot() sample {
var s sample
for _, t := range d.txStats {
s.txErrs += t.errs.Load()
}
for _, r := range d.rxStats {
s.rxFrames += r.frames.Load()
s.rxBytes += r.bytes.Load()
s.crcErr += r.crcErr.Load()
s.badMagic += r.badMagic.Load()
s.badHdr += r.badHdr.Load()
s.badLen += r.badLen.Load()
s.rxErrs += r.rxErrs.Load()
}
for i := range d.streams {
s.lost += d.streams[i].lost.Load()
s.late += d.streams[i].late.Load()
}
return s
}
// Counters keep climbing in the workers, so resetting just moves the origin
// everything is measured from. Rates and the rolling error window are about now
// rather than since the reset, so they keep running; the origin goes into the
// ring so the newest bucket never sits behind it.
func (d *direction) reset() {
d.mu.Lock()
d.base = d.capture()
d.win.push(d.base)
d.mu.Unlock()
}
// The socket statistic is read-and-clear, so it is always consumed; a drop
// during a cable measure is the blip's and is discarded at this source.
func (d *direction) sampleDrops() {
for _, fd := range d.rxFDs {
n := packetDrops(fd)
if !d.measuring.Load() {
d.drops += n
}
}
}
func gbps(bytes, frames uint64, secs float64) float64 {
return float64((bytes+frames*wireOverhead)*8) / secs / 1e9
}
// Shared by the console table and the framebuffer so both show the same
// figures.
type view struct {
rxPPS float64
rxGbps float64
rxFrames, rxBytes uint64
since errs
window errs
}
func errsBetween(b, n counterSet) errs {
return errs{
lost: n.s.lost - b.s.lost,
// Four ways of noticing one thing: a payload that does not match its
// checksum, a header that does not match its own, a header that is not
// ours, and a length that cannot be.
corrupt: (n.s.crcErr - b.s.crcErr) + (n.s.badHdr - b.s.badHdr) +
(n.s.badMagic - b.s.badMagic) + (n.s.badLen - b.s.badLen),
// What the hardware reported. Nothing the host declined to send is here,
// so this one going red means the cable.
link: (n.nic - b.nic) + (n.s.rxErrs - b.s.rxErrs),
// Ours rather than the cable's. A late frame is unreachable while each
// stream has a flow rule to its own queue, which is exactly why it is
// worth counting.
internal: (n.drops - b.drops) + (n.s.late - b.s.late) +
(n.s.txErrs - b.s.txErrs),
}
}
func (d *direction) counters(now counterSet) view {
return view{
rxFrames: now.s.rxFrames - d.base.s.rxFrames,
rxBytes: now.s.rxBytes - d.base.s.rxBytes,
since: errsBetween(d.base, now),
}
}
func totalView(views []view) view {
var t view
for _, v := range views {
t.rxPPS += v.rxPPS
t.rxGbps += v.rxGbps
t.rxFrames += v.rxFrames
t.rxBytes += v.rxBytes
t.since = t.since.add(v.since)
t.window = t.window.add(v.window)
}
return t
}
func (d *direction) sample() {
d.mu.Lock()
d.win.push(d.capture())
d.readRateBucket(time.Now())
d.mu.Unlock()
}
// Draws what the sampler last put in the ring rather than reading the counters
// again, so the display never participates in the measurement.
func (d *direction) displayView() view {
d.mu.Lock()
n := d.win.count()
if n == 0 {
d.mu.Unlock()
return view{}
}
v := d.counters(d.win.at(n - 1))
if n >= 2 {
v.window = errsBetween(d.win.at(0), d.win.at(n-1))
}
v.rxPPS = float64(d.rateFrames) / rateBucketSecs
v.rxGbps = gbps(d.rateBytes, d.rateFrames, rateBucketSecs)
d.mu.Unlock()
return v
}
// Whatever the interfaces counted before now is not ours, and no interval has
// elapsed yet, so every baseline starts here and nothing is reported until the
// first one completes.
func (d *direction) primeCounters() {
d.poller.prime()
d.reset()
}
func buildDirection(label string, tx, rx endpoint, measuring *atomic.Bool) (*direction, error) {
// Built before the windows, since each window judges sequence numbers against
// the frontier its own sender publishes.
txs := make([]*txStats, numStreams)
for i := range txs {
txs[i] = &txStats{}
}
d := &direction{
txStats: txs,
streams: newLossWindows(txs),
measuring: measuring,
}
// Held open for the life of the run: the stats ioctl is issued five times a
// second and reopening a socket for each one is pure overhead.
statFD, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil {
return nil, fmt.Errorf("%s stats socket: %w", label, err)
}
d.statFD = statFD
d.poller, err = newNICPoller(statFD, tx.name, rx.name, &d.nic, measuring)
if err != nil {
return nil, fmt.Errorf("%s: %w", label, err)
}
d.win = newRateWindow(int(rateWindowSpan/sampleInterval) + 1)
for i := 0; i < numStreams; i++ {
et := uint16(etherBase + i)
d.specs = append(d.specs, newFrameSpec(rx.mac, tx.mac, et, frameSizes))
fd, err := openTxSocket(tx.idx)
if err != nil {
return nil, fmt.Errorf("%s tx socket: %w", label, err)
}
d.txFDs = append(d.txFDs, fd)
fd, err = openRxSocket(rx.idx, et)
if err != nil {
return nil, fmt.Errorf("%s rx socket for 0x%04x: %w", label, et, err)
}
d.rxFDs = append(d.rxFDs, fd)
d.rxStats = append(d.rxStats, &rxStats{})
}
return d, nil
}
func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.WaitGroup, startTx <-chan struct{}) {
for i, fd := range d.txFDs {
w := &txWorker{
fd: fd,
stream: uint16(i),
spec: d.specs[i],
batch: batchSize,
stats: d.txStats[i],
measuring: d.measuring,
startTx: startTx,
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
w.run(done)
}()
}
for i, fd := range d.rxFDs {
w := &rxWorker{
fd: fd,
batch: batchSize,
stream: uint16(i),
spec: d.specs[i],
stats: d.rxStats[i],
loss: &d.streams[i],
measuring: d.measuring,
ready: rxReady,
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
w.run(done)
}()
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
d.poller.run(done, startTx)
}()
}
func (d *direction) close() {
for _, fd := range d.txFDs {
unix.Close(fd)
}
for _, fd := range d.rxFDs {
unix.Close(fd)
}
unix.Close(d.statFD)
}
docs/state.md
+1 -1
View File
@@ -46,5 +46,5 @@ In `~/work/`, ready to fold into the repo's `kernel/`:
## Every boot
1. `./load-ixgbe` — until then the FS port has no netdev (stock driver, error -95) and `driverPair("ixgbe")` cannot find 2 interfaces. Verify `sff_i2c` present for both ports.
1. `./load-ixgbe` — until then the FS port has no netdev (stock driver, error -95) and `driverPair("ixgbe")` cannot find 2 interfaces. cabletest's BOOT check names the condition (`patched ixgbe` fails when any bound port lacks `sff_i2c`).
2. Re-apply host tuning (hardware.md) before trusting results.
init.go
+30 -1
View File
@@ -2,6 +2,8 @@ package main
import (
"fmt"
"os"
"path/filepath"
"time"
"golang.org/x/sys/unix"
@@ -89,6 +91,32 @@ func mountFilesystems() []checkResult {
return out
}
// The sff_i2c debugfs node is the patched driver's signature. Checked up front
// because stock ixgbe also refuses the FS module's EEPROM and leaves its port
// with no netdev, so the missing patch would otherwise surface later as a
// baffling "want 2 ixgbe interfaces, found 1".
func checkPatchedIxgbe() checkResult {
res := checkResult{item: "patched ixgbe"}
devs, err := filepath.Glob("/sys/kernel/debug/ixgbe/*")
if err != nil {
res.err = err
return res
}
if len(devs) == 0 {
res.err = fmt.Errorf("no ixgbe devices in debugfs; run ./load-ixgbe")
return res
}
for _, d := range devs {
if _, err := os.Stat(d + "/sff_i2c"); err != nil {
res.err = fmt.Errorf("%s has no sff_i2c: stock ixgbe loaded; run ./load-ixgbe",
filepath.Base(d))
return res
}
}
res.state = fmt.Sprintf("sff_i2c on %d devices", len(devs))
return res
}
const (
touchTimeout = 30 * time.Second
touchPoll = 100 * time.Millisecond
@@ -123,9 +151,10 @@ func waitForTouchscreen() checkResult {
func bootstrap() []checkResult {
out := mountFilesystems()
// /dev/input/event* only exists once devtmpfs is mounted above.
// debugfs and /dev/input/event* only exist once the mounts above are in.
if out[len(out)-1].err != nil {
return out
}
out = append(out, checkPatchedIxgbe())
return append(out, waitForTouchscreen())
}
input.go
+4 -3
View File
@@ -1,6 +1,7 @@
package main
import (
"encoding/binary"
"fmt"
"os"
"path/filepath"
@@ -127,9 +128,9 @@ func watchTouch(w, h int) (*touchState, error) {
panic(fmt.Sprintf("reading touchscreen events: %v", err))
}
for o := 0; o+sizeofInputEvent <= n; o += sizeofInputEvent {
typ := *(*uint16)(unsafe.Pointer(&buf[o+16]))
code := *(*uint16)(unsafe.Pointer(&buf[o+18]))
val := *(*int32)(unsafe.Pointer(&buf[o+20]))
typ := binary.NativeEndian.Uint16(buf[o+16:])
code := binary.NativeEndian.Uint16(buf[o+18:])
val := int32(binary.NativeEndian.Uint32(buf[o+20:]))
switch typ {
case evAbs:
switch code {
main.go
+5 -415
View File
@@ -3,7 +3,6 @@ package main
import (
"flag"
"fmt"
"math"
"net"
"os"
"os/signal"
@@ -15,8 +14,6 @@ import (
"golang.org/x/sys/unix"
)
const wireOverhead = 24
type endpoint struct {
name string
tag string
@@ -30,180 +27,6 @@ func (e endpoint) macString() string {
e.mac[0], e.mac[1], e.mac[2], e.mac[3], e.mac[4], e.mac[5])
}
type direction struct {
specs []*frameSpec
txStats []*txStats
rxStats []*rxStats
streams []lossWindow
txFDs []int
rxFDs []int
statFD int
// While a cable measure runs, every failure counter in this direction is
// suppressed at its source rather than counted, hidden and reverted.
measuring *atomic.Bool
// Guards everything the sampler touches. The counters are read on their own
// clock and drawn on another, and the two must not read them at once:
// sampleDrops consumes what it reads, so a second caller would see a gap.
mu sync.Mutex
win *rateWindow
drops uint64
base counterSet
// The smeared receive bucket the display draws its rate from, refreshed by
// the sampler because the buckets are keyed by the shared read clock and
// staleness has to be judged against the wall.
rateFrames uint64
rateBytes uint64
epoch int64
epochAt time.Time
// The settled window: each completed bucket enters once, donates its
// excess backward once, and leaves for display once no later bucket can
// still refill it. Recomputing the smear from the raw ring every sample
// instead would show each excess twice — once as the donation and again,
// unspent, when its bucket reaches the display slot of a later window.
smFrames [smearWindow]uint64
smBytes [smearWindow]uint64
smLen int
smNext int64
nic atomic.Uint64
poller *nicPoller
}
// Everything the display reads, taken at one instant, so a pair of these
// describes both the rates and the errors over the span between them.
type counterSet struct {
t time.Time
s sample
drops uint64
nic uint64
}
func (d *direction) capture() counterSet {
d.sampleDrops()
s := d.snapshot()
return counterSet{t: time.Now(), s: s, drops: d.drops, nic: d.nic.Load()}
}
// What someone testing a cable is asking, rather than how each failure happened
// to be noticed.
type errs struct {
lost uint64
corrupt uint64
link uint64
internal uint64
}
func (e errs) add(o errs) errs {
return errs{
lost: e.lost + o.lost, corrupt: e.corrupt + o.corrupt,
link: e.link + o.link, internal: e.internal + o.internal,
}
}
// A ring of one bucket per drawn frame, spanning rateWindowSpan. Rates come
// from the gap between adjacent buckets and errors from the ends of the ring,
// so both slide forward every frame instead of stepping once a second.
type rateWindow struct {
buf []counterSet
idx int
filled bool
}
func newRateWindow(n int) *rateWindow {
return &rateWindow{buf: make([]counterSet, n)}
}
func (w *rateWindow) push(c counterSet) {
w.buf[w.idx] = c
w.idx++
if w.idx == len(w.buf) {
w.idx = 0
w.filled = true
}
}
func (w *rateWindow) count() int {
if w.filled {
return len(w.buf)
}
return w.idx
}
// Indexed oldest first, so a partly filled ring reads the same as a full one.
func (w *rateWindow) at(i int) counterSet {
if w.filled {
i += w.idx
}
return w.buf[i%len(w.buf)]
}
// How long the frontier may sit still before the wire is taken to have gone
// quiet. It only advances when frames arrive on every stream, so a frozen
// frontier means a stream has stopped delivering rather than an unchanged rate.
const rateStale = 100 * time.Millisecond
// An epoch is only reachable once a worker drains frames into it, so each
// stream's newest epoch is a frontier: everything that queue has been read
// through. Epochs behind the lowest frontier — the leader's would claim
// buckets the stragglers are still filling — are closed to further commits,
// so each is settled into the window exactly once. The display takes the
// window's oldest bucket, the one no later bucket can still refill, so the
// headline runs one window behind real time.
func (d *direction) readRateBucket(now time.Time) {
newest := int64(math.MaxInt64)
for _, r := range d.rxStats {
if e := r.newest.Load(); e < newest {
newest = e
}
}
if newest > d.epoch {
d.epoch, d.epochAt = newest, now
}
if d.smNext == 0 && d.epoch > 1 {
d.smNext = d.epoch - 1
}
for e := d.smNext; e > 0 && e < d.epoch; e++ {
d.settle(e)
d.smNext = e + 1
}
d.rateFrames, d.rateBytes = 0, 0
if d.smLen == 0 || now.Sub(d.epochAt) > rateStale {
return
}
d.rateFrames, d.rateBytes = d.smFrames[0], d.smBytes[0]
}
func (d *direction) settle(e int64) {
if d.smLen == smearWindow {
copy(d.smFrames[:], d.smFrames[1:])
copy(d.smBytes[:], d.smBytes[1:])
d.smLen--
}
var f, b uint64
for _, r := range d.rxStats {
rf, rb := r.bucket(e)
f += rf
b += rb
}
d.smFrames[d.smLen], d.smBytes[d.smLen] = f, b
d.smLen++
fillBack(d.smFrames[:d.smLen], d.smBytes[:d.smLen])
}
type sample struct {
rxFrames, rxBytes uint64
lost, late uint64
crcErr, badMagic uint64
badHdr uint64
badLen uint64
txErrs uint64
rxErrs uint64
}
func lookupEndpoint(name string) (endpoint, error) {
ifi, err := net.InterfaceByName(name)
if err != nil {
@@ -217,38 +40,6 @@ func lookupEndpoint(name string) (endpoint, error) {
return endpoint{name: name, idx: ifi.Index, mac: mac, mtu: ifi.MTU}, nil
}
func (d *direction) snapshot() sample {
var s sample
for _, t := range d.txStats {
s.txErrs += t.errs.Load()
}
for _, r := range d.rxStats {
s.rxFrames += r.frames.Load()
s.rxBytes += r.bytes.Load()
s.crcErr += r.crcErr.Load()
s.badMagic += r.badMagic.Load()
s.badHdr += r.badHdr.Load()
s.badLen += r.badLen.Load()
s.rxErrs += r.rxErrs.Load()
}
for i := range d.streams {
s.lost += d.streams[i].lost.Load()
s.late += d.streams[i].late.Load()
}
return s
}
// Counters keep climbing in the workers, so resetting just moves the origin
// everything is measured from. Rates and the rolling error window are about now
// rather than since the reset, so they keep running; the origin goes into the
// ring so the newest bucket never sits behind it.
func (d *direction) reset() {
d.mu.Lock()
d.base = d.capture()
d.win.push(d.base)
d.mu.Unlock()
}
// Returns the new start time, so the uptime shown alongside the totals counts
// from the reset rather than from launch; the elapsed clock restarting is the
// visible mark of the re-baseline.
@@ -262,19 +53,11 @@ func resetAll(dirs []*direction, mods []*phyModule) time.Time {
return time.Now()
}
// The socket statistic is read-and-clear, so it is always consumed; a drop
// during a cable measure is the blip's and is discarded at this source.
func (d *direction) sampleDrops() {
for _, fd := range d.rxFDs {
n := packetDrops(fd)
if !d.measuring.Load() {
d.drops += n
}
}
}
func gbps(bytes, frames uint64, secs float64) float64 {
return float64((bytes+frames*wireOverhead)*8) / secs / 1e9
// Nothing to hold or revert here: failures during a measure were never
// counted, so the view is always the counters as they stand.
func phyView(diag *cableDiag, modules []*phyModule) (phyDisplay, bool) {
info, measuring := diag.snapshot()
return phyDisplayFrom(info, measuring, modules[0].view(), modules[1].view()), measuring
}
var intervalCols = []colSpec{
@@ -298,89 +81,6 @@ var intervalCols = []colSpec{
{group: "OVERALL", title: "internal", width: 9, right: true},
}
// Shared by the console table and the framebuffer so both show the same
// figures.
type view struct {
rxPPS float64
rxGbps float64
rxFrames, rxBytes uint64
since errs
window errs
}
func errsBetween(b, n counterSet) errs {
return errs{
lost: n.s.lost - b.s.lost,
// Four ways of noticing one thing: a payload that does not match its
// checksum, a header that does not match its own, a header that is not
// ours, and a length that cannot be.
corrupt: (n.s.crcErr - b.s.crcErr) + (n.s.badHdr - b.s.badHdr) +
(n.s.badMagic - b.s.badMagic) + (n.s.badLen - b.s.badLen),
// What the hardware reported. Nothing the host declined to send is here,
// so this one going red means the cable.
link: (n.nic - b.nic) + (n.s.rxErrs - b.s.rxErrs),
// Ours rather than the cable's. A late frame is unreachable while each
// stream has a flow rule to its own queue, which is exactly why it is
// worth counting.
internal: (n.drops - b.drops) + (n.s.late - b.s.late) +
(n.s.txErrs - b.s.txErrs),
}
}
func (d *direction) counters(now counterSet) view {
return view{
rxFrames: now.s.rxFrames - d.base.s.rxFrames,
rxBytes: now.s.rxBytes - d.base.s.rxBytes,
since: errsBetween(d.base, now),
}
}
// Nothing to hold or revert here: failures during a measure were never
// counted, so the view is always the counters as they stand.
func phyView(diag *cableDiag, modules []*phyModule) (phyDisplay, bool) {
info, measuring := diag.snapshot()
return phyDisplayFrom(info, measuring, modules[0].view(), modules[1].view()), measuring
}
func totalView(views []view) view {
var t view
for _, v := range views {
t.rxPPS += v.rxPPS
t.rxGbps += v.rxGbps
t.rxFrames += v.rxFrames
t.rxBytes += v.rxBytes
t.since = t.since.add(v.since)
t.window = t.window.add(v.window)
}
return t
}
func (d *direction) sample() {
d.mu.Lock()
d.win.push(d.capture())
d.readRateBucket(time.Now())
d.mu.Unlock()
}
// Draws what the sampler last put in the ring rather than reading the counters
// again, so the display never participates in the measurement.
func (d *direction) displayView() view {
d.mu.Lock()
n := d.win.count()
if n == 0 {
d.mu.Unlock()
return view{}
}
v := d.counters(d.win.at(n - 1))
if n >= 2 {
v.window = errsBetween(d.win.at(0), d.win.at(n-1))
}
v.rxPPS = float64(d.rateFrames) / rateBucketSecs
v.rxGbps = gbps(d.rateBytes, d.rateFrames, rateBucketSecs)
d.mu.Unlock()
return v
}
// The same figures the panel draws, in the same order: the last second as
// rates and error flags with the noise cable riding at the end of them, then
// everything since the reset.
@@ -407,116 +107,6 @@ func totalRow(elapsed time.Duration, v view, target float64, phy phyDisplay, nv
}
}
// Whatever the interfaces counted before now is not ours, and no interval has
// elapsed yet, so every baseline starts here and nothing is reported until the
// first one completes.
func (d *direction) primeCounters() {
d.poller.prime()
d.reset()
}
func buildDirection(label string, tx, rx endpoint, measuring *atomic.Bool) (*direction, error) {
// Built before the windows, since each window judges sequence numbers against
// the frontier its own sender publishes.
txs := make([]*txStats, numStreams)
for i := range txs {
txs[i] = &txStats{}
}
d := &direction{
txStats: txs,
streams: newLossWindows(txs),
measuring: measuring,
}
// Held open for the life of the run: the stats ioctl is issued five times a
// second and reopening a socket for each one is pure overhead.
statFD, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil {
return nil, fmt.Errorf("%s stats socket: %w", label, err)
}
d.statFD = statFD
d.poller, err = newNICPoller(statFD, tx.name, rx.name, &d.nic, measuring)
if err != nil {
return nil, fmt.Errorf("%s: %w", label, err)
}
d.win = newRateWindow(int(rateWindowSpan/sampleInterval) + 1)
for i := 0; i < numStreams; i++ {
et := uint16(etherBase + i)
d.specs = append(d.specs, newFrameSpec(rx.mac, tx.mac, et, frameSizes))
fd, err := openTxSocket(tx.idx)
if err != nil {
return nil, fmt.Errorf("%s tx socket: %w", label, err)
}
d.txFDs = append(d.txFDs, fd)
fd, err = openRxSocket(rx.idx, et)
if err != nil {
return nil, fmt.Errorf("%s rx socket for 0x%04x: %w", label, et, err)
}
d.rxFDs = append(d.rxFDs, fd)
d.rxStats = append(d.rxStats, &rxStats{})
}
return d, nil
}
func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.WaitGroup, startTx <-chan struct{}) {
for i, fd := range d.txFDs {
w := &txWorker{
fd: fd,
stream: uint16(i),
spec: d.specs[i],
batch: batchSize,
stats: d.txStats[i],
measuring: d.measuring,
startTx: startTx,
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
w.run(done)
}()
}
for i, fd := range d.rxFDs {
w := &rxWorker{
fd: fd,
batch: batchSize,
stream: uint16(i),
spec: d.specs[i],
stats: d.rxStats[i],
loss: &d.streams[i],
measuring: d.measuring,
ready: rxReady,
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
w.run(done)
}()
}
wg.Add(1)
go func() {
defer wg.Done()
defer holdPanic()
d.poller.run(done, startTx)
}()
}
func (d *direction) close() {
for _, fd := range d.txFDs {
unix.Close(fd)
}
for _, fd := range d.rxFDs {
unix.Close(fd)
}
unix.Close(d.statFD)
}
const (
numStreams = 7
batchSize = 64
phy.go
+17 -621
View File
@@ -2,58 +2,12 @@ package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/sys/unix"
)
const (
bcmI2CWrite = 0xAC
bcmI2CRead = 0xAD
bcmMMDVendor uint16 = 0x1E
bcmRegCmd uint16 = 0x4005
bcmRegStatus uint16 = 0x4037
bcmRegData1 uint16 = 0x4038
bcmStInProgress uint16 = 0x0002
bcmStPass uint16 = 0x0004
bcmStError uint16 = 0x0008
bcmStBusy uint16 = 0xBBBB
bcmCmdGetPairSwap uint16 = 0x8000
bcmCmdSetEEEMode uint16 = 0x8009
bcmCmdSetJumbo uint16 = 0x801C
bcmRegECDCtrl uint16 = 0x4006
bcmRegECDResult uint16 = 0xA896
bcmRegECDLen uint16 = 0xA897
bcmPHYIDHi = 0x3590
bcmPHYIDLo = 0x5081
bcmCmdResidentTemp uint16 = 0x0031
bcmReadDelayUs = 3000
bcmRetryDelayUs = 10000
bcmWindow = 3400 * time.Millisecond
bcmWindowFit = 100 * time.Millisecond
bcmFlipPoll = 10 * time.Millisecond
bcmFlipWait = 5 * time.Second
bcmStatusPoll = 100 * time.Millisecond
// Covers the handler's documented 2 s freeze during 10GBASE-T training.
bcmStatusTimeout = 3 * time.Second
ecdPoll = 200 * time.Millisecond
ecdDeadline = 50 * time.Second
pairIdentityMap = 0xE4
fsVendorPN = "SFP-10G-T-100"
@@ -71,121 +25,6 @@ var pairVerdicts = map[int]string{
pairOK: "ok", pairOpen: "OPEN", pairShort: "SHORT", pairXtalk: "XTALK",
}
type sff struct {
ifname string
path string
// Every transport touch happens on the loop goroutine: requests execute
// one at a time, each admitted by the module type's own gate first.
reqs chan func()
admit func()
}
func openSFF(ifname string) (*sff, error) {
devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device")
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
drv, err := ifDriver(ifname)
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
if drv != "ixgbe" {
return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv)
}
s := &sff{
ifname: ifname,
path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c",
reqs: make(chan func()),
admit: func() {},
}
if _, err := os.Stat(s.path); err != nil {
return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err)
}
go s.loop()
return s, nil
}
func (s *sff) loop() {
defer holdPanic()
for fn := range s.reqs {
s.admit()
fn()
}
}
func (s *sff) exec(fn func()) {
done := make(chan struct{})
s.reqs <- func() { fn(); close(done) }
<-done
}
func (s *sff) name() string { return s.ifname }
func (s *sff) op(cmd string) (string, error) {
fd, err := unix.Open(s.path, unix.O_RDWR, 0)
if err != nil {
return "", fmt.Errorf("%s: %w", s.path, err)
}
defer unix.Close(fd)
if _, err := unix.Write(fd, []byte(cmd)); err != nil {
return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err)
}
buf := make([]byte, 256)
n, err := unix.Read(fd, buf)
if err != nil {
return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err)
}
resp := strings.TrimSpace(string(buf[:n]))
if !strings.HasPrefix(resp, "ok") {
return "", fmt.Errorf("%s %q: %s", s.ifname, cmd, resp)
}
return strings.TrimSpace(resp[2:]), nil
}
func parseHexBytes(s string, n int) ([]byte, error) {
fields := strings.Fields(s)
if len(fields) != n {
return nil, fmt.Errorf("want %d bytes, got %q", n, s)
}
out := make([]byte, n)
for i, f := range fields {
var v byte
if _, err := fmt.Sscanf(f, "%x", &v); err != nil {
return nil, fmt.Errorf("byte %q in %q", f, s)
}
out[i] = v
}
return out, nil
}
// A single bus hold; split write/read ops would let the driver's own SFP
// traffic consume the bridge's pending read.
func (s *sff) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) {
var sb strings.Builder
fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n)
for _, v := range wdata {
fmt.Fprintf(&sb, " %02x", v)
}
resp, err := s.op(sb.String())
if err != nil {
return nil, err
}
return parseHexBytes(resp, n)
}
func (s *sff) eeprom(off byte, n int) ([]byte, error) {
return s.compound(0xA0, 0xA1, 500, n, []byte{off})
}
func (s *sff) vendorPN() (string, error) {
pn, err := s.eeprom(40, 16)
if err != nil {
return "", err
}
return strings.TrimSpace(string(pn)), nil
}
type mdioDev interface {
name() string
exec(func())
@@ -256,395 +95,6 @@ func devEEEAdvert(d mdioDev) (v uint16, err error) {
return
}
type bcm struct {
*sff
windowEnd time.Time
}
func newBCM(t *sff) *bcm {
b := &bcm{sff: t}
t.exec(func() { t.admit = b.window })
return b
}
// The firmware's internal temp poll serves stale bridge reads for ~50 ms
// around it; work stays inside 3.4 s of an observed poll. A resident 0x0031 at
// expiry means the phase is unknown, so re-lock: arm, then take the true edge.
// Each taken edge immediately re-arms — the one CMD write per window lands at
// the start of the quiet period, maximally far from the next poll (writes near
// the poll are the µC-wedge risk), and every later expiry reads the phase
// without writing.
func (b *bcm) window() {
if time.Now().Add(bcmWindowFit).Before(b.windowEnd) {
return
}
armed := false
deadline := time.Now().Add(bcmFlipWait)
for {
v, err := b.mdioRead(bcmMMDVendor, bcmRegCmd)
if err != nil {
panic(fmt.Sprintf("%s: heartbeat poll: %v", b.ifname, err))
}
if v == bcmCmdResidentTemp {
if armed {
b.windowEnd = time.Now().Add(bcmWindow)
b.rearm()
return
}
b.rearm()
armed = true
deadline = time.Now().Add(bcmFlipWait)
continue
}
armed = true
if time.Now().After(deadline) {
b.windowEnd = time.Now().Add(bcmWindow)
return
}
time.Sleep(bcmFlipPoll)
}
}
func (b *bcm) rearm() {
if _, err := b.waitStatus(func(st uint16) bool {
return st != bcmStInProgress && st != bcmStBusy
}); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
if err := b.mdioWrite(bcmMMDVendor, bcmRegCmd, bcmCmdGetPairSwap); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
if _, err := b.waitStatus(func(st uint16) bool {
return st == bcmStPass || st == bcmStError
}); err != nil {
panic(fmt.Sprintf("%s: rearm: %v", b.ifname, err))
}
}
func (b *bcm) mdioReadDelay(devad, reg uint16, delayUs int) (uint16, error) {
d, err := b.compound(bcmI2CWrite, bcmI2CRead, delayUs, 2,
[]byte{0x20 | byte(devad), byte(reg >> 8), byte(reg)})
if err != nil {
return 0, err
}
return uint16(d[0])<<8 | uint16(d[1]), nil
}
// 0x0000 is also the bridge's not-ready signature, so a zero is read again at
// a longer delay before being believed.
func (b *bcm) mdioRead(devad, reg uint16) (uint16, error) {
v, err := b.mdioReadDelay(devad, reg, bcmReadDelayUs)
if err != nil || v != 0 {
return v, err
}
return b.mdioReadDelay(devad, reg, bcmRetryDelayUs)
}
func (b *bcm) mdioWrite(devad, reg, val uint16) error {
_, err := b.op(fmt.Sprintf("w %02x %02x %02x %02x %02x %02x",
bcmI2CWrite, byte(devad), byte(reg>>8), byte(reg), byte(val>>8), byte(val)))
return err
}
// The datasheet's completion handshake: poll STATUS on its 100 ms cadence
// until the wanted state, bounded by a deadline.
func (b *bcm) waitStatus(want func(uint16) bool) (uint16, error) {
deadline := time.Now().Add(bcmStatusTimeout)
for {
st, err := b.mdioRead(bcmMMDVendor, bcmRegStatus)
if err != nil {
return 0, err
}
if want(st) {
return st, nil
}
if time.Now().After(deadline) {
return 0, fmt.Errorf("%s: command handler stuck, status %#04x", b.ifname, st)
}
time.Sleep(bcmStatusPoll)
}
}
// GETs must be invoked bare (pre-writing any DATA register leaves the handler
// executing as a no-op); SETs must pass their full parameter set (the handler
// executes stale DATA).
func (b *bcm) command(code uint16, params ...uint16) (data [5]uint16, err error) {
b.exec(func() {
if _, err = b.waitStatus(func(st uint16) bool {
return st != bcmStInProgress && st != bcmStBusy
}); err != nil {
return
}
for i, p := range params {
if err = b.mdioWrite(bcmMMDVendor, bcmRegData1+uint16(i), p); err != nil {
return
}
}
if err = b.mdioWrite(bcmMMDVendor, bcmRegCmd, code); err != nil {
return
}
var st uint16
if st, err = b.waitStatus(func(st uint16) bool {
return st == bcmStPass || st == bcmStError
}); err != nil {
return
}
if st == bcmStError {
err = fmt.Errorf("%s: command %#04x returned ERROR", b.ifname, code)
return
}
if len(params) > 0 {
return
}
for i := range data {
if data[i], err = b.mdioRead(bcmMMDVendor, bcmRegData1+uint16(i)); err != nil {
return
}
}
})
return
}
func (b *bcm) identify() (ident string, err error) {
b.exec(func() {
var hi, lo uint16
if hi, err = b.mdioRead(1, 2); err != nil {
return
}
if lo, err = b.mdioRead(1, 3); err != nil {
return
}
if hi != bcmPHYIDHi || lo != bcmPHYIDLo {
err = fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x",
b.ifname, hi, lo, bcmPHYIDHi, bcmPHYIDLo)
return
}
var sn []byte
if sn, err = b.eeprom(68, 16); err != nil {
return
}
ident = "BCM84891L sn " + strings.TrimSpace(string(sn))
})
return
}
func (b *bcm) forceEEEOff() error {
_, err := b.command(bcmCmdSetEEEMode, 0x0000, 0x0000, 0x7A12, 0x0480, 0x0000)
return err
}
func (b *bcm) forceJumbo() error {
_, err := b.command(bcmCmdSetJumbo, 1, 0, 0, 0, 0)
return err
}
// Left to AN, master/slave is a per-training lottery and each training's DSP
// convergence moves per-pair SNR by up to ~3.6 dB; pinned roles at least keep
// every session measured under identical conditions.
func (b *bcm) forceRole(master bool) (err error) {
b.exec(func() {
var v uint16
if v, err = b.mdioRead(7, 32); err != nil {
return
}
v |= 0x8000
if master {
v |= 0x4000
} else {
v &^= 0x4000
}
err = b.mdioWrite(7, 32, v)
})
return
}
func (b *bcm) pairMap() (byte, error) {
d, err := b.command(bcmCmdGetPairSwap)
if err != nil {
return 0, err
}
return byte(d[1]), nil
}
const (
rbI2CWrite = 0xA2
rbI2CRead = 0xA3
rbOffPassword byte = 0x7B
rbOffPage byte = 0x7F
rbOffCmd byte = 0x80
rbOffDevad byte = 0x81
rbOffValHi byte = 0x84
rbOffPartNum byte = 0xFA
rbPageMailbox byte = 3
rbCmdWrite byte = 0x01
rbCmdRead byte = 0x02
rbCmdDone byte = 0x04
rbReadDelayUs = 500
rbCmdPoll = 20 * time.Millisecond
// Matches the BCM allowance for a handler frozen by 10GBASE-T training.
rbCmdTimeout = 3 * time.Second
rbPHYIDHi uint16 = 0x002B
rbPHYIDLo uint16 = 0x0BF4
// IEEE margins land near 7-9 dB on a healthy short cable; far outside is
// another register's data.
rbGhostLow = -10.0
rbGhostHigh = 25.0
)
// Only the registers proven safe on this PHY (docs/modules/wiitek/): single
// reads in the vendor windows brick the µC permanently, so everything else
// refuses before touching hardware.
var rbReadSafe = map[uint16]map[uint16]bool{
1: {1: true, 2: true, 3: true, 133: true, 134: true, 135: true, 136: true, 147: true},
3: {32: true, 33: true},
7: {0: true, 33: true, 60: true},
}
var rbWriteSafe = map[uint16]map[uint16]bool{
7: {0: true},
}
type rollball struct {
*sff
}
func (r *rollball) i2cWrite(off byte, data ...byte) error {
var sb strings.Builder
fmt.Fprintf(&sb, "w %02x %02x", rbI2CWrite, off)
for _, v := range data {
fmt.Fprintf(&sb, " %02x", v)
}
_, err := r.op(sb.String())
return err
}
func (r *rollball) i2cRead(off byte, n int) ([]byte, error) {
return r.compound(rbI2CWrite, rbI2CRead, rbReadDelayUs, n, []byte{off})
}
func (r *rollball) unlock() error {
if err := r.i2cWrite(rbOffPage, rbPageMailbox); err != nil {
return err
}
return r.i2cWrite(rbOffPassword, 0xFF, 0xFF, 0xFF, 0xFF)
}
// The µC can be mid-service of an earlier session's command when this one is
// issued: it completes the old one, leaving DONE and a stale value in the
// block, and a status sample taken before the new command commits reads them
// as this command's (seen live: PHY ID high word answered by an orphaned 7.60
// read). So status is never sampled before a full poll gap, the value rides
// in the same block read as the status, and a completion only counts when the
// block echoes this command's devad/reg.
func (r *rollball) mbox(cmd byte, devad, reg, val uint16) (out [2]byte, err error) {
if err = r.unlock(); err != nil {
return
}
if err = r.i2cWrite(rbOffDevad, byte(devad), byte(reg>>8), byte(reg)); err != nil {
return
}
if cmd == rbCmdWrite {
if err = r.i2cWrite(rbOffValHi, byte(val>>8), byte(val)); err != nil {
return
}
}
if err = r.i2cWrite(rbOffCmd, cmd); err != nil {
return
}
deadline := time.Now().Add(rbCmdTimeout)
for {
time.Sleep(rbCmdPoll)
var d []byte
if d, err = r.i2cRead(rbOffCmd, 6); err != nil {
return
}
if d[0] == rbCmdDone && d[1] == byte(devad) &&
d[2] == byte(reg>>8) && d[3] == byte(reg) {
out[0], out[1] = d[4], d[5]
return
}
if time.Now().After(deadline) {
err = fmt.Errorf("%s: mailbox %d.%#04x stuck at %#02x", r.ifname, devad, reg, d[0])
return
}
}
}
func rbGuard(safe map[uint16]map[uint16]bool, ifname, what string, devad, reg uint16) {
if !safe[devad][reg] {
panic(fmt.Sprintf("%s: refusing MDIO %s %d.%#04x: outside the proven-safe set",
ifname, what, devad, reg))
}
}
func (r *rollball) mdioRead(devad, reg uint16) (uint16, error) {
rbGuard(rbReadSafe, r.ifname, "read", devad, reg)
d, err := r.mbox(rbCmdRead, devad, reg, 0)
if err != nil {
return 0, err
}
return uint16(d[0])<<8 | uint16(d[1]), nil
}
func (r *rollball) mdioWrite(devad, reg, val uint16) error {
rbGuard(rbWriteSafe, r.ifname, "write", devad, reg)
_, err := r.mbox(rbCmdWrite, devad, reg, val)
return err
}
func (r *rollball) identify() (ident string, err error) {
r.exec(func() {
var hi, lo uint16
if hi, err = r.mdioRead(1, 2); err != nil {
return
}
if lo, err = r.mdioRead(1, 3); err != nil {
return
}
if hi != rbPHYIDHi || lo != rbPHYIDLo {
err = fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x",
r.ifname, hi, lo, rbPHYIDHi, rbPHYIDLo)
return
}
if err = r.unlock(); err != nil {
return
}
var part []byte
if part, err = r.i2cRead(rbOffPartNum, 1); err != nil {
return
}
var sn []byte
if sn, err = r.eeprom(68, 16); err != nil {
return
}
ident = fmt.Sprintf("CUX3610 sn %s (A2.250=%d)", strings.TrimSpace(string(sn)), part[0])
})
return
}
func (r *rollball) snrMargins() (out [4]float64, err error) {
r.exec(func() {
for i := range out {
var v uint16
if v, err = r.mdioRead(1, uint16(133+i)); err != nil {
return
}
m := (float64(v) - 0x8000) / 10
if m < rbGhostLow || m > rbGhostHigh {
panic(fmt.Sprintf("%s: ghost SNR margin %.1f dB (1.%d=%#04x)", r.ifname, m, 133+i, v))
}
out[i] = m
}
})
return
}
const (
phyInterval = time.Second
phyStale = 5 * time.Second
@@ -811,7 +261,6 @@ type cableInfo struct {
ecd ecdResult
maps [2]byte
haveMaps [2]bool
failed bool
}
func (c cableInfo) metresString() string {
@@ -867,15 +316,11 @@ func pairSwapped(i int, c cableInfo) bool {
}
// The whole cable verdict in one cell: the length when healthy, the first
// faulted pair's verdict when not, "xover" for a pair swap, "fail" for a diag
// that never delivered.
// faulted pair's verdict when not, "xover" for a pair swap.
func cableSummary(cable cableInfo, measuring bool) (string, int) {
if measuring {
return "-", clsNone
}
if cable.failed {
return "fail", clsBad
}
anyData, anySwap := false, false
for i, v := range cable.ecd.verdicts {
if v != 0 {
@@ -928,49 +373,6 @@ type ecdResult struct {
metres [4]int
}
func (b *bcm) cableDiag() (res ecdResult, err error) {
b.exec(func() {
var ctrl uint16
if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil {
return
}
if err = b.mdioWrite(bcmMMDVendor, bcmRegECDCtrl, ctrl&^0xF400|0x8400); err != nil {
return
}
deadline := time.Now().Add(ecdDeadline)
for {
if ctrl, err = b.mdioRead(bcmMMDVendor, bcmRegECDCtrl); err != nil {
return
}
if ctrl&0x0800 == 0 {
break
}
if time.Now().After(deadline) {
err = fmt.Errorf("%s: cable diag still busy after %s", b.ifname, ecdDeadline)
return
}
time.Sleep(ecdPoll)
}
b.window()
var v uint16
if v, err = b.mdioRead(1, bcmRegECDResult); err != nil {
return
}
for i := range res.verdicts {
res.verdicts[i] = int(v>>(4*i)) & 0xF
if res.verdicts[i] > pairXtalk {
panic(fmt.Sprintf("%s: ghost ECD verdict %#04x", b.ifname, v))
}
var m uint16
if m, err = b.mdioRead(1, bcmRegECDLen+uint16(i)); err != nil {
return
}
res.metres[i] = int(m)
}
})
return
}
func bcmEnd(mods []*phyModule) *bcm {
for _, m := range mods {
if b, ok := m.dev.(*bcm); ok {
@@ -982,7 +384,7 @@ func bcmEnd(mods []*phyModule) *bcm {
// The pollers are held silent throughout; pair maps are read after the
// relink, so the MDI resolution is the fresh one.
func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableInfo, bool, error) {
func measureCable(mods []*phyModule, done *atomic.Bool) (cableInfo, error) {
for _, m := range mods {
m.busy.Store(true)
}
@@ -998,27 +400,23 @@ func measureCable(mods []*phyModule, waitRelink bool, done *atomic.Bool) (cableI
end := bcmEnd(mods)
c.ecd, err = end.cableDiag()
if err != nil {
return c, false, err
return c, err
}
if err = devRestartAN(end); err != nil {
return c, false, err
}
relinked := false
if waitRelink {
names := [2]string{mods[0].dev.name(), mods[1].dev.name()}
_, relinked = waitCarrier(names, done)
return c, err
}
waitCarrier([2]string{mods[0].dev.name(), mods[1].dev.name()}, done)
for i, m := range mods {
b, ok := m.dev.(*bcm)
if !ok {
continue
}
if c.maps[i], err = b.pairMap(); err != nil {
return c, false, err
return c, err
}
c.haveMaps[i] = true
}
return c, relinked, nil
return c, nil
}
type cableDiag struct {
@@ -1043,15 +441,18 @@ func (c *cableDiag) snapshot() (cableInfo, bool) {
return c.info, c.measuring.Load()
}
// A measure that errors means the diag transport or the µC is broken, and a
// tester that cannot run its diagnostics must not keep testing: the failure is
// fatal, and the reboot's driver reload is also the wedged-µC recovery.
func (c *cableDiag) kick(done *atomic.Bool) bool {
if !c.measuring.CompareAndSwap(false, true) {
return false
}
go func() {
defer holdPanic()
info, _, err := measureCable(c.mods, true, done)
info, err := measureCable(c.mods, done)
if err != nil {
info = cableInfo{failed: true}
panic(fmt.Sprintf("cable measure: %v", err))
}
c.mu.Lock()
c.info = info
@@ -1095,15 +496,11 @@ func openModules(names [2]string) ([]*phyModule, [2]string, error) {
return mods, idents, nil
}
func waitCarrier(names [2]string, done *atomic.Bool) (time.Duration, bool) {
start := time.Now()
deadline := start.Add(linkWaitSpan)
for {
if carrierUp(names[0]) && carrierUp(names[1]) {
return time.Since(start), true
}
if time.Now().After(deadline) || (done != nil && done.Load()) {
return time.Since(start), false
func waitCarrier(names [2]string, done *atomic.Bool) {
deadline := time.Now().Add(linkWaitSpan)
for !(carrierUp(names[0]) && carrierUp(names[1])) {
if time.Now().After(deadline) || done.Load() {
return
}
time.Sleep(linkWaitPoll)
}
@@ -1191,4 +588,3 @@ func moduleChecks(mods []*phyModule, names [2]string) []checkResult {
}
return out
}
phy_test.go
-2
View File
@@ -55,7 +55,6 @@ func TestCableSummary(t *testing.T) {
{"one-end map only", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, 0}, haveMaps: [2]bool{true, false}}, false, "49", clsGood},
{"fault", cableInfo{ecd: ecdResult{verdicts: [4]int{pairOK, pairShort, pairXtalk, pairOK}, metres: [4]int{50, 12, 30, 51}}}, false, "SHORT", clsBad},
{"fault beats swap", cableInfo{ecd: ecdResult{verdicts: [4]int{pairOpen, pairOK, pairOK, pairOK}}, maps: [2]byte{pairIdentityMap, 0xE1}, haveMaps: [2]bool{true, true}}, false, "OPEN", clsBad},
{"diag failed", cableInfo{failed: true}, false, "fail", clsBad},
{"no diag yet", cableInfo{}, false, "-", clsNone},
{"measuring", cableInfo{ecd: healthy, maps: [2]byte{pairIdentityMap, pairIdentityMap}, haveMaps: [2]bool{true, true}}, true, "-", clsNone},
} {
@@ -106,4 +105,3 @@ func TestSNRClass(t *testing.T) {
}
}
}
render.go
+4 -5
View File
@@ -185,6 +185,9 @@ func (t *streamTable) rule(label string) string {
}
func (t *streamTable) emit(cells []string) []string {
if len(cells) != len(t.cols) {
panic(fmt.Sprintf("row has %d cells for %d columns", len(cells), len(t.cols)))
}
var out []string
if t.sinceHeader == 0 || (t.headerEvery > 0 && t.sinceHeader >= t.headerEvery) {
out = append(out, t.headerLines()...)
@@ -192,11 +195,7 @@ func (t *streamTable) emit(cells []string) []string {
}
var padded []string
for i, c := range t.cols {
v := ""
if i < len(cells) {
v = cells[i]
}
padded = append(padded, pad(v, c.width, c.right))
padded = append(padded, pad(cells[i], c.width, c.right))
}
t.sinceHeader++
return append(out, t.join(padded, paint(" │ ", cGrey)))
rollball.go
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"fmt"
"strings"
"time"
)
const (
rbI2CWrite = 0xA2
rbI2CRead = 0xA3
rbOffPassword byte = 0x7B
rbOffPage byte = 0x7F
rbOffCmd byte = 0x80
rbOffDevad byte = 0x81
rbOffValHi byte = 0x84
rbOffPartNum byte = 0xFA
rbPageMailbox byte = 3
rbCmdWrite byte = 0x01
rbCmdRead byte = 0x02
rbCmdDone byte = 0x04
rbReadDelayUs = 500
rbCmdPoll = 20 * time.Millisecond
// Matches the BCM allowance for a handler frozen by 10GBASE-T training.
rbCmdTimeout = 3 * time.Second
rbPHYIDHi uint16 = 0x002B
rbPHYIDLo uint16 = 0x0BF4
// IEEE margins land near 7-9 dB on a healthy short cable; far outside is
// another register's data.
rbGhostLow = -10.0
rbGhostHigh = 25.0
)
// Only the registers proven safe on this PHY (docs/modules/wiitek/): single
// reads in the vendor windows brick the µC permanently, so everything else
// refuses before touching hardware.
var rbReadSafe = map[uint16]map[uint16]bool{
1: {1: true, 2: true, 3: true, 133: true, 134: true, 135: true, 136: true, 147: true},
3: {32: true, 33: true},
7: {0: true, 33: true, 60: true},
}
var rbWriteSafe = map[uint16]map[uint16]bool{
7: {0: true},
}
type rollball struct {
*sff
}
func (r *rollball) i2cWrite(off byte, data ...byte) error {
var sb strings.Builder
fmt.Fprintf(&sb, "w %02x %02x", rbI2CWrite, off)
for _, v := range data {
fmt.Fprintf(&sb, " %02x", v)
}
_, err := r.op(sb.String())
return err
}
func (r *rollball) i2cRead(off byte, n int) ([]byte, error) {
return r.compound(rbI2CWrite, rbI2CRead, rbReadDelayUs, n, []byte{off})
}
func (r *rollball) unlock() error {
if err := r.i2cWrite(rbOffPage, rbPageMailbox); err != nil {
return err
}
return r.i2cWrite(rbOffPassword, 0xFF, 0xFF, 0xFF, 0xFF)
}
// The µC can be mid-service of an earlier session's command when this one is
// issued: it completes the old one, leaving DONE and a stale value in the
// block, and a status sample taken before the new command commits reads them
// as this command's (seen live: PHY ID high word answered by an orphaned 7.60
// read). So status is never sampled before a full poll gap, the value rides
// in the same block read as the status, and a completion only counts when the
// block echoes this command's devad/reg.
func (r *rollball) mbox(cmd byte, devad, reg, val uint16) (out [2]byte, err error) {
if err = r.unlock(); err != nil {
return
}
if err = r.i2cWrite(rbOffDevad, byte(devad), byte(reg>>8), byte(reg)); err != nil {
return
}
if cmd == rbCmdWrite {
if err = r.i2cWrite(rbOffValHi, byte(val>>8), byte(val)); err != nil {
return
}
}
if err = r.i2cWrite(rbOffCmd, cmd); err != nil {
return
}
deadline := time.Now().Add(rbCmdTimeout)
for {
time.Sleep(rbCmdPoll)
var d []byte
if d, err = r.i2cRead(rbOffCmd, 6); err != nil {
return
}
if d[0] == rbCmdDone && d[1] == byte(devad) &&
d[2] == byte(reg>>8) && d[3] == byte(reg) {
out[0], out[1] = d[4], d[5]
return
}
if time.Now().After(deadline) {
err = fmt.Errorf("%s: mailbox %d.%#04x stuck at %#02x", r.ifname, devad, reg, d[0])
return
}
}
}
func rbGuard(safe map[uint16]map[uint16]bool, ifname, what string, devad, reg uint16) {
if !safe[devad][reg] {
panic(fmt.Sprintf("%s: refusing MDIO %s %d.%#04x: outside the proven-safe set",
ifname, what, devad, reg))
}
}
func (r *rollball) mdioRead(devad, reg uint16) (uint16, error) {
rbGuard(rbReadSafe, r.ifname, "read", devad, reg)
d, err := r.mbox(rbCmdRead, devad, reg, 0)
if err != nil {
return 0, err
}
return uint16(d[0])<<8 | uint16(d[1]), nil
}
func (r *rollball) mdioWrite(devad, reg, val uint16) error {
rbGuard(rbWriteSafe, r.ifname, "write", devad, reg)
_, err := r.mbox(rbCmdWrite, devad, reg, val)
return err
}
func (r *rollball) identify() (ident string, err error) {
r.exec(func() {
var hi, lo uint16
if hi, err = r.mdioRead(1, 2); err != nil {
return
}
if lo, err = r.mdioRead(1, 3); err != nil {
return
}
if hi != rbPHYIDHi || lo != rbPHYIDLo {
err = fmt.Errorf("%s: PHY ID %#04x:%#04x, want %#04x:%#04x",
r.ifname, hi, lo, rbPHYIDHi, rbPHYIDLo)
return
}
if err = r.unlock(); err != nil {
return
}
var part []byte
if part, err = r.i2cRead(rbOffPartNum, 1); err != nil {
return
}
var sn []byte
if sn, err = r.eeprom(68, 16); err != nil {
return
}
ident = fmt.Sprintf("CUX3610 sn %s (A2.250=%d)", strings.TrimSpace(string(sn)), part[0])
})
return
}
func (r *rollball) snrMargins() (out [4]float64, err error) {
r.exec(func() {
for i := range out {
var v uint16
if v, err = r.mdioRead(1, uint16(133+i)); err != nil {
return
}
m := (float64(v) - 0x8000) / 10
if m < rbGhostLow || m > rbGhostHigh {
panic(fmt.Sprintf("%s: ghost SNR margin %.1f dB (1.%d=%#04x)", r.ifname, m, 133+i, v))
}
out[i] = m
}
})
return
}
sff.go
+125
View File
@@ -0,0 +1,125 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"golang.org/x/sys/unix"
)
type sff struct {
ifname string
path string
// Every transport touch happens on the loop goroutine: requests execute
// one at a time, each admitted by the module type's own gate first.
reqs chan func()
admit func()
}
func openSFF(ifname string) (*sff, error) {
devLink, err := os.Readlink("/sys/class/net/" + ifname + "/device")
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
drv, err := ifDriver(ifname)
if err != nil {
return nil, fmt.Errorf("%s: %w", ifname, err)
}
if drv != "ixgbe" {
return nil, fmt.Errorf("%s: no module I2C transport for driver %s", ifname, drv)
}
s := &sff{
ifname: ifname,
path: "/sys/kernel/debug/ixgbe/" + filepath.Base(devLink) + "/sff_i2c",
reqs: make(chan func()),
admit: func() {},
}
if _, err := os.Stat(s.path); err != nil {
return nil, fmt.Errorf("%s: %w (patched ixgbe?)", ifname, err)
}
go s.loop()
return s, nil
}
func (s *sff) loop() {
defer holdPanic()
for fn := range s.reqs {
s.admit()
fn()
}
}
func (s *sff) exec(fn func()) {
done := make(chan struct{})
s.reqs <- func() { fn(); close(done) }
<-done
}
func (s *sff) name() string { return s.ifname }
func (s *sff) op(cmd string) (string, error) {
fd, err := unix.Open(s.path, unix.O_RDWR, 0)
if err != nil {
return "", fmt.Errorf("%s: %w", s.path, err)
}
defer unix.Close(fd)
if _, err := unix.Write(fd, []byte(cmd)); err != nil {
return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err)
}
buf := make([]byte, 256)
n, err := unix.Read(fd, buf)
if err != nil {
return "", fmt.Errorf("%s %q: %w", s.ifname, cmd, err)
}
resp := strings.TrimSpace(string(buf[:n]))
if !strings.HasPrefix(resp, "ok") {
return "", fmt.Errorf("%s %q: %s", s.ifname, cmd, resp)
}
return strings.TrimSpace(resp[2:]), nil
}
func parseHexBytes(s string, n int) ([]byte, error) {
fields := strings.Fields(s)
if len(fields) != n {
return nil, fmt.Errorf("want %d bytes, got %q", n, s)
}
out := make([]byte, n)
for i, f := range fields {
var v byte
if _, err := fmt.Sscanf(f, "%x", &v); err != nil {
return nil, fmt.Errorf("byte %q in %q", f, s)
}
out[i] = v
}
return out, nil
}
// A single bus hold; split write/read ops would let the driver's own SFP
// traffic consume the bridge's pending read.
func (s *sff) compound(waddr, raddr byte, delayUs, n int, wdata []byte) ([]byte, error) {
var sb strings.Builder
fmt.Fprintf(&sb, "x %02x %02x %d %x", waddr, raddr, delayUs, n)
for _, v := range wdata {
fmt.Fprintf(&sb, " %02x", v)
}
resp, err := s.op(sb.String())
if err != nil {
return nil, err
}
return parseHexBytes(resp, n)
}
func (s *sff) eeprom(off byte, n int) ([]byte, error) {
return s.compound(0xA0, 0xA1, 500, n, []byte{off})
}
func (s *sff) vendorPN() (string, error) {
pn, err := s.eeprom(40, 16)
if err != nil {
return "", err
}
return strings.TrimSpace(string(pn)), nil
}
ui.go
+10 -1
View File
@@ -492,7 +492,16 @@ func (d *display) drawRows(v view, phy phyDisplay, measuring bool, nv noiseView,
d.colMark(resetR, y, markCheck, uiGood)
}
default:
f := i - 1
f := -1
for j, c := range faultClasses {
if c.label == label {
f = j
break
}
}
if f < 0 {
panic("panel row " + label + " has no fault class")
}
if n := faultClasses[f].get(v.window); n > 0 {
d.colText(d.textB, nowR, y, scaleCount(n), uiCrit)
} else {