cabletest/counters.go

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync/atomic"
	"time"
)

// Split by direction of travel so each counter belongs to exactly one
// direction: the transmitting interface owns the tx fields and the receiving one
// the rx fields. Reading both sets off both interfaces double counts every drop.
//
// The two sides come from different places, and deliberately not from both at
// once. dev_get_stats adds the core's own drop counts on top of whatever the
// driver reports, so a sysfs field is the driver's counter plus the stack's and
// overlaps any driver counter read beside it.
//
// Sending is the stack's story: what it refused, and what the qdisc discarded
// because we outran it, neither of which the driver ever sees.
var nicTxFields = []string{
	"tx_errors", "tx_dropped", "tx_fifo_errors",
	"tx_carrier_errors", "tx_aborted_errors", "tx_window_errors",
	"collisions",
}

// Receiving is the hardware's, and is taken from the driver's own array, which
// sysfs both flattens and overlaps: ice folds crc errors and jabbers into
// rx_errors, so the old sum of rx_errors alongside rx_crc_errors charged every
// bad frame check twice. Named individually per driver, so nothing contains
// anything else in its list.
var nicRxStatsByDriver = map[string][]string{
	// The reference set (E810). Below the frame: illegal_bytes is a 64b/66b
	// block that decoded to no legal symbol, and the faults are the ordered
	// sets the pcs sends when it loses sync. A cable going marginal moves
	// these while every frame still arrives intact, which is as close to a
	// bit error rate as this link will report.
	"ice": {
		"rx_crc_errors.nic",
		"rx_jabber.nic",
		"rx_undersize.nic",
		"rx_oversize.nic",
		"rx_fragments.nic",
		"rx_dropped.nic",
		"illegal_bytes.nic",
		"mac_local_faults.nic",
		"mac_remote_faults.nic",
	},
	// The 82599 exposes no jabber, fragment, illegal-byte or fault counters
	// through ethtool — this is the closest bench set, and one of the X520
	// divergences listed in docs/open-questions.md.
	"ixgbe": {
		"rx_crc_errors",
		"rx_missed_errors",
		"rx_length_errors",
		"rx_long_length_errors",
		"rx_short_length_errors",
	},
}

func ifDriver(name string) (string, error) {
	link, err := os.Readlink("/sys/class/net/" + name + "/device/driver")
	if err != nil {
		return "", err
	}
	return filepath.Base(link), nil
}

func readUint(path string) (uint64, bool) {
	b, err := os.ReadFile(path)
	if err != nil {
		return 0, false
	}
	v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64)
	if err != nil {
		return 0, false
	}
	return v, true
}

// Every counter here was read once before the test began, so one that stops
// reading now is the interface going away underneath us. Stopping is the only
// honest answer: a tester that quietly gives up counting errors goes on
// reporting a clean link, which is the one lie it must never tell.
func mustReadUint(path string) uint64 {
	v, ok := readUint(path)
	if !ok {
		panic("reading " + path)
	}
	return v
}

func sumFields(base string, fields []string) uint64 {
	var total uint64
	for _, f := range fields {
		total += mustReadUint(base + "/statistics/" + f)
	}
	return total
}

// A poll is an ioctl and a handful of sysfs attributes, each costing
// microseconds of open and read, so it runs here rather than on the goroutine
// that has a frame to draw every sixteen milliseconds. Off that path it can
// match the sampler's cadence, so a link error lands on the panel as quickly
// as a lost frame rather than stepping five times a second.
const nicInterval = sampleInterval

type nicPoller struct {
	txBase string
	rx     *statReader
	total  *atomic.Uint64
	raw    uint64
}

func newNICPoller(fd int, txName, rxName string, total *atomic.Uint64) (*nicPoller, error) {
	drv, err := ifDriver(rxName)
	if err != nil {
		return nil, err
	}
	want, ok := nicRxStatsByDriver[drv]
	if !ok {
		return nil, fmt.Errorf("%s: no rx error statistic set for driver %s", rxName, drv)
	}
	rx, err := newStatReader(fd, rxName, want)
	if err != nil {
		return nil, err
	}
	return &nicPoller{
		txBase: "/sys/class/net/" + txName,
		rx:     rx,
		total:  total,
	}, nil
}

// Carrier is a property of the netdev rather than a counter of either side, and
// is taken from the transmitting interface only: across both directions each
// interface transmits exactly once, so every flap is counted once.
func (p *nicPoller) read() uint64 {
	return sumFields(p.txBase, nicTxFields) + p.rx.sum() +
		mustReadUint(p.txBase+"/carrier_down_count")
}

// Nic counters run from boot and restart from zero whenever the driver resets
// its statistics, so only their forward motion is accumulated. Taking raw
// differences instead charges a boot's worth of errors to the first sample and
// turns a reset into a near-2^64 underflow.
func (p *nicPoller) poll() {
	raw := p.read()
	if raw > p.raw {
		p.total.Add(raw - p.raw)
	}
	p.raw = raw
}

// Whatever the interfaces counted before now is not ours, so the first poll has
// something to measure forward motion from and adds nothing.
func (p *nicPoller) prime() {
	p.raw = p.read()
}

// Held until the test starts, since polling before the baseline is primed would
// charge everything the interfaces counted since boot to the first sample.
func (p *nicPoller) run(done *atomic.Bool, startTx <-chan struct{}) {
	<-startTx

	tick := time.NewTicker(nicInterval)
	defer tick.Stop()

	for !done.Load() {
		<-tick.C
		p.poll()
	}
}