Drive a noise cable on the i40e pair, cycling its link down and up since the module PHYs ignore everything softer, and flag its absence beside the error columns

2026-08-05 10:49:23 -07:00
parent 88f4f7da80
commit 2fcab83756
5 changed files with 335 additions and 32 deletions
@@ -1779,7 +1779,7 @@ CONFIG_LIBIE_FWLOG=y
 # CONFIG_IGBVF is not set
 # CONFIG_IXGBE is not set
 # CONFIG_IXGBEVF is not set
-# CONFIG_I40E is not set
+CONFIG_I40E=y
 # CONFIG_I40EVF is not set
 CONFIG_ICE=y
 CONFIG_ICE_HWMON=y
@@ -252,6 +252,7 @@ var intervalCols = []colSpec{
 	{group: "NOW", title: "corrupt", width: 7, right: true},
 	{group: "NOW", title: "link", width: 7, right: true},
 	{group: "NOW", title: "internal", width: 8, right: true},
+	{group: "NOW", title: "noise", width: 7, right: true},
 	{group: "OVERALL", title: "elapsed", width: 9, right: true},
 	{group: "OVERALL", title: "packets", width: 9, right: true},
 	{group: "OVERALL", title: "bytes", width: 9, right: true},
@@ -340,9 +341,10 @@ func (d *direction) displayView() view {
 	return v
 }

-// The same figures the panel draws, in the same order: the last second as rates
-// and error flags, then everything since the reset.
-func totalRow(elapsed time.Duration, v view, target float64, length string) []string {
+// The same figures the panel draws, in the same order: the last second as
+// rates and error flags with the noise cable riding at the end of them, then
+// everything since the reset.
+func totalRow(elapsed time.Duration, v view, target float64, length string, noiseMissing uint64) []string {
 	return []string{
 		rateCell(v.rxGbps*1e9, target*1e9),
 		scaleSI(v.rxPPS),
@@ -350,6 +352,7 @@ func totalRow(elapsed time.Duration, v view, target float64, length string) []st
 		flagCell(v.window.corrupt),
 		flagCell(v.window.link),
 		flagCell(v.window.internal),
+		flagCell(noiseMissing),
 		scaleTime(elapsed),
 		scaleCount(v.rxFrames),
 		scaleCount(v.rxBytes),
@@ -513,6 +516,8 @@ const (

 	probeEther uint16 = etherBase + numStreams

+	testDriver = "ice"
+
 	// A constant rather than the negotiated speed, since this has to come up
 	// with no cable in the port and nothing to negotiate.
 	linkSpeed = 10.0
@@ -523,10 +528,11 @@ const (
 var frameSizes = []int{60, 128, 256, 512, 1024, 1280, 1514}

 func main() {
-	// The names the kernel gives the only two ports built into it, since as
-	// PID 1 there is no udev to rename them and no command line to pass.
-	aName := flag.String("a", "eth0", "first interface")
-	bName := flag.String("b", "eth1", "second interface")
+	// Left empty, the test pair is found by driver name instead: as PID 1 there
+	// is no udev to pin names and no command line to pass, and which port gets
+	// which ethN shifts with every driver built into the kernel.
+	aName := flag.String("a", "", "first interface (default: the ice pair)")
+	bName := flag.String("b", "", "second interface")
 	nsPerM := flag.Float64("ns-per-m", 4.85, "mean of both directions, per metre of cable")
 	flag.Parse()

@@ -580,6 +586,13 @@ func run(aName, bName string, nsPerM float64) error {
 		return err
 	}

+	if aName == "" || bName == "" {
+		var err error
+		aName, bName, err = driverPair(testDriver)
+		if err != nil {
+			return err
+		}
+	}
 	a, err := lookupEndpoint(aName)
 	if err != nil {
 		return err
@@ -588,6 +601,11 @@ func run(aName, bName string, nsPerM float64) error {
 	if err != nil {
 		return err
 	}
+	noise, err := newNoiser()
+	if err != nil {
+		return err
+	}
+	defer noise.close()
 	for _, e := range []endpoint{a, b} {
 		for _, s := range frameSizes {
 			if s > e.mtu+ethHdrLen {
@@ -596,7 +614,8 @@ func run(aName, bName string, nsPerM float64) error {
 		}
 	}

-	a.tag, b.tag = "A", "B"
+	a.tag, b.tag = "TEST A", "TEST B"
+	noise.eps[0].tag, noise.eps[1].tag = "NOISE A", "NOISE B"
 	ifnames := []string{a.name, b.name}

 	ethertypes := make([]uint16, numStreams)
@@ -604,7 +623,8 @@ func run(aName, bName string, nsPerM float64) error {
 		ethertypes[i] = uint16(etherBase + i)
 	}

-	if err := reportChecks("HOST SETTINGS", configureSystem(ifnames, ethertypes)); err != nil {
+	if err := reportChecks("HOST SETTINGS",
+		append(configureSystem(ifnames, ethertypes), configureNoise(noise.names())...)); err != nil {
 		return err
 	}

@@ -623,7 +643,7 @@ func run(aName, bName string, nsPerM float64) error {
 	}()

 	var linkRows [][]string
-	for _, e := range []endpoint{a, b} {
+	for _, e := range []endpoint{a, b, noise.eps[0], noise.eps[1]} {
 		linkRows = append(linkRows, []string{
 			paint(e.tag, cCyan), e.name, e.macString(), fmt.Sprintf("%d", e.mtu),
 		})
@@ -652,6 +672,13 @@ func run(aName, bName string, nsPerM float64) error {
 		defer wg.Done()
 		samp.run(&done, startTx)
 	}()
+	// Not gated on startTx: the cycle and the connected verdict are wanted the
+	// moment the panel is, and nothing it does touches the measurement.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		noise.run(&done)
+	}()
 	// Every return from here on stops the workers before the deferred closes
 	// pull their sockets out from under them: otherwise the sampler panics on a
 	// closed fd and can mask the error that actually ended the run. An error
@@ -719,7 +746,8 @@ func run(aName, bName string, nsPerM float64) error {
 			if m, ok := cableMetres(views, nsPerM); ok {
 				cable = fmt.Sprintf("%.1f", m)
 			}
-			if err := disp.render(totalView(views), now.Sub(start), cable); err != nil {
+			if err := disp.render(totalView(views), now.Sub(start), cable,
+				noise.missing()); err != nil {
 				return err
 			}
 		case now := <-tick.C:
@@ -733,7 +761,8 @@ func run(aName, bName string, nsPerM float64) error {
 			if m, ok := cableMetres(rows, nsPerM); ok {
 				length = fmt.Sprintf("%.1f", m)
 			}
-			for _, line := range stats.emit(totalRow(elapsed, totalView(rows), target, length)) {
+			for _, line := range stats.emit(totalRow(elapsed, totalView(rows), target, length,
+				noise.missing())) {
 				fmt.Println(line)
 			}
 		}
@@ -0,0 +1,206 @@
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// The noise cable: a deliberately bad cable twisted around the one under test,
+// there to radiate into it. Its ports are driven for the interference they
+// produce, not measured: nothing is ever received from them, and nothing they
+// count reaches the error columns.
+//
+// The wire cannot be quieted by going idle, because without EEE the PHYs
+// signal at full power whether or not frames flow, and these PHYs live inside
+// the SFP+ modules where no EEE control reaches them. Closing the port is the
+// one switch the host actually has, so the cycle is built on it: links up and
+// carrying frames for a spell, then administratively down for one. Every wake
+// re-runs 10GBASE-T training, which is as loud as this wire ever gets.
+const (
+	noiseDriver   = "i40e"
+	noiseFrameLen = 1514
+	noiseUpSpan   = 5 * time.Second
+	noiseDownSpan = 5 * time.Second
+	noiseFrameGap = 10 * time.Millisecond
+
+	noiseEther uint16 = probeEther + 1
+)
+
+// Kernel names shift with which drivers are built in, since ethN is handed out
+// in link order rather than by slot. The driver name is the one label a port
+// keeps across kernel configs, so pairs are found by it rather than named.
+func driverPair(driver string) (string, string, error) {
+	ents, err := os.ReadDir("/sys/class/net")
+	if err != nil {
+		return "", "", err
+	}
+	var names []string
+	for _, e := range ents {
+		link, err := os.Readlink("/sys/class/net/" + e.Name() + "/device/driver")
+		if err != nil {
+			continue
+		}
+		if filepath.Base(link) == driver {
+			names = append(names, e.Name())
+		}
+	}
+	if len(names) != 2 {
+		return "", "", fmt.Errorf("want 2 %s interfaces, found %d [%s]",
+			driver, len(names), strings.Join(names, " "))
+	}
+	sort.Strings(names)
+	return names[0], names[1], nil
+}
+
+type noisePort struct {
+	name  string
+	fd    int
+	frame []byte
+}
+
+type noiser struct {
+	eps   [2]endpoint
+	ports [2]noisePort
+
+	// Whether the cable is judged present: both carriers seen during an up
+	// phase. Latched across the down phase, where the missing carrier is our
+	// own doing and says nothing about the cable.
+	connected atomic.Bool
+}
+
+func newNoiser() (*noiser, error) {
+	aName, bName, err := driverPair(noiseDriver)
+	if err != nil {
+		return nil, fmt.Errorf("noise: %w", err)
+	}
+	a, err := lookupEndpoint(aName)
+	if err != nil {
+		return nil, fmt.Errorf("noise: %w", err)
+	}
+	b, err := lookupEndpoint(bName)
+	if err != nil {
+		return nil, fmt.Errorf("noise: %w", err)
+	}
+
+	n := &noiser{eps: [2]endpoint{a, b}}
+	for i, p := range [][2]endpoint{{a, b}, {b, a}} {
+		fd, err := openTxSocket(p[0].idx)
+		if err != nil {
+			return nil, fmt.Errorf("noise tx socket %s: %w", p[0].name, err)
+		}
+		// The payload is left zero: the PCS scrambles everything on the wire,
+		// so no pattern radiates differently from any other. The frame exists
+		// to occupy the link, not to say anything.
+		frame := make([]byte, noiseFrameLen)
+		copy(frame[0:6], p[1].mac[:])
+		copy(frame[6:12], p[0].mac[:])
+		binary.BigEndian.PutUint16(frame[12:14], noiseEther)
+		n.ports[i] = noisePort{name: p[0].name, fd: fd, frame: frame}
+	}
+	return n, nil
+}
+
+func (n *noiser) names() []string {
+	return []string{n.eps[0].name, n.eps[1].name}
+}
+
+// Zero while the cable was there at the last verdict, one while it was not:
+// the shape the error cells already colour by, so absence paints as the fault
+// it is and presence as the usual green.
+func (n *noiser) missing() uint64 {
+	if n.connected.Load() {
+		return 0
+	}
+	return 1
+}
+
+// The ports were reachable when the noiser was built, so one that stops taking
+// the ioctl now is the interface going away underneath us, the same fault the
+// counter reads stop for.
+func (n *noiser) setLinks(fd int, up bool) {
+	for i := range n.ports {
+		var ifr flagsIfreq
+		copy(ifr.name[:], n.ports[i].name)
+		if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
+			uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr))); errno != 0 {
+			panic(fmt.Sprintf("reading %s flags: %v", n.ports[i].name, errno))
+		}
+		if up {
+			ifr.flags |= unix.IFF_UP
+		} else {
+			ifr.flags &^= unix.IFF_UP
+		}
+		if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
+			uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr))); errno != 0 {
+			panic(fmt.Sprintf("setting %s flags: %v", n.ports[i].name, errno))
+		}
+	}
+}
+
+// Down reads as EINVAL rather than zero, and either way the answer is the
+// same: no carrier here now.
+func carrierUp(name string) bool {
+	v, ok := readUint("/sys/class/net/" + name + "/carrier")
+	return ok && v == 1
+}
+
+func (n *noiser) bothUp() bool {
+	return carrierUp(n.ports[0].name) && carrierUp(n.ports[1].name)
+}
+
+// Send results are deliberately dropped: the cable is bad on purpose, the link
+// comes and goes under the cycle, and a frame this side declined to send is as
+// good as one the wire mangled. What matters is only ever what the test cable
+// counted.
+func (n *noiser) run(done *atomic.Bool) {
+	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
+	if err != nil {
+		panic(fmt.Sprintf("noise ioctl socket: %v", err))
+	}
+	defer unix.Close(fd)
+
+	tick := time.NewTicker(noiseFrameGap)
+	defer tick.Stop()
+
+	for !done.Load() {
+		n.setLinks(fd, true)
+		linked := false
+		for end := time.Now().Add(noiseUpSpan); time.Now().Before(end) && !done.Load(); {
+			<-tick.C
+			if !n.bothUp() {
+				continue
+			}
+			linked = true
+			n.connected.Store(true)
+			for i := range n.ports {
+				unix.Write(n.ports[i].fd, n.ports[i].frame)
+			}
+		}
+		// A whole up phase with no link is many times the ~1s the wire needs
+		// to train, so by now the silence is the cable's answer.
+		n.connected.Store(linked)
+
+		n.setLinks(fd, false)
+		for end := time.Now().Add(noiseDownSpan); time.Now().Before(end) && !done.Load(); {
+			<-tick.C
+		}
+	}
+	// Left up rather than wherever the cycle stopped, so a run never strands
+	// the ports down for whoever looks next.
+	n.setLinks(fd, true)
+}
+
+func (n *noiser) close() {
+	for i := range n.ports {
+		unix.Close(n.ports[i].fd)
+	}
+}
@@ -335,6 +335,7 @@ func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) {

 const (
 	ethSSStats     = 1
+	ethSSPrivFlags = 2
 	ethGstringLen  = 32
 )

@@ -364,33 +365,33 @@ type ethtoolStatsHdr struct {
 // boundary. A []byte carries no such guarantee.
 func statsBuf(n int) []uint64 { return make([]uint64, n) }

-func statCount(fd int, ifname string) (uint32, error) {
-	req := ethtoolSsetInfo{cmd: unix.ETHTOOL_GSSET_INFO, ssetMask: 1 << ethSSStats}
+func stringSetCount(fd int, ifname string, set uint32) (uint32, error) {
+	req := ethtoolSsetInfo{cmd: unix.ETHTOOL_GSSET_INFO, ssetMask: 1 << set}
 	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&req)); err != nil {
 		return 0, err
 	}
 	// A cleared mask bit means the driver does not have that string set at all,
 	// in which case no count was written into the tail.
 	if req.ssetMask == 0 {
-		return 0, fmt.Errorf("driver has no ETH_SS_STATS string set")
+		return 0, fmt.Errorf("driver has no string set %d", set)
 	}
 	return req.data[0], nil
 }

-func statNames(fd int, ifname string) ([]string, error) {
-	n, err := statCount(fd, ifname)
+func stringSetNames(fd int, ifname string, set uint32) ([]string, error) {
+	n, err := stringSetCount(fd, ifname, set)
 	if err != nil {
 		return nil, err
 	}
 	if n == 0 {
-		return nil, fmt.Errorf("driver reports zero statistics")
+		return nil, fmt.Errorf("driver reports an empty string set %d", set)
 	}

 	hdrLen := int(unsafe.Sizeof(ethtoolGstrings{}))
 	buf := statsBuf((hdrLen + int(n)*ethGstringLen + 7) / 8)
 	hdr := (*ethtoolGstrings)(unsafe.Pointer(&buf[0]))
 	hdr.cmd = unix.ETHTOOL_GSTRINGS
-	hdr.stringSet = ethSSStats
+	hdr.stringSet = set
 	hdr.len = n
 	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&buf[0])); err != nil {
 		return nil, err
@@ -426,7 +427,7 @@ type statReader struct {
 }

 func newStatReader(fd int, ifname string, want []string) (*statReader, error) {
-	names, err := statNames(fd, ifname)
+	names, err := stringSetNames(fd, ifname, ethSSStats)
 	if err != nil {
 		return nil, fmt.Errorf("%s statistics: %w", ifname, err)
 	}
@@ -623,6 +624,67 @@ func withIoctlSocket(fn func(fd int) []checkResult) []checkResult {
 	return fn(fd)
 }

+type ethtoolValue struct {
+	cmd  uint32
+	data uint32
+}
+
+// i40e leaves the module transmitting when a port is closed, so the peer never
+// sees anything happen and the copper stays trained. This flag is what makes
+// an administrative down reach the wire, and the noise cycle is switched
+// entirely through it.
+func checkPrivFlag(fd int, ifname, flag string) checkResult {
+	res := checkResult{item: ifname + " " + flag}
+	names, err := stringSetNames(fd, ifname, ethSSPrivFlags)
+	if err != nil {
+		res.err = err
+		return res
+	}
+	bit := -1
+	for i, s := range names {
+		if s == flag {
+			bit = i
+			break
+		}
+	}
+	if bit < 0 {
+		res.err = fmt.Errorf("driver has no private flag %q", flag)
+		return res
+	}
+	v := ethtoolValue{cmd: unix.ETHTOOL_GPFLAGS}
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&v)); err != nil {
+		res.err = err
+		return res
+	}
+	if v.data&(1<<bit) != 0 {
+		res.state = "on"
+		return res
+	}
+	v.cmd = unix.ETHTOOL_SPFLAGS
+	v.data |= 1 << bit
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&v)); err != nil {
+		res.err = err
+		res.state = "could not set"
+		return res
+	}
+	res.fixed = true
+	res.state = "was off, now on"
+	return res
+}
+
+// The noise ports get none of the test pair's tuning: nothing is measured on
+// them, so rings, coalescing and steering are all beside the point. All they
+// need is for a closed port to really drop the link.
+func configureNoise(ifnames []string) []checkResult {
+	return withIoctlSocket(func(fd int) []checkResult {
+		var out []checkResult
+		for _, ifname := range ifnames {
+			out = append(out, checkPrivFlag(fd, ifname, "link-down-on-close"))
+		}
+		return out
+	})
+}
+
 // Nothing here waits for a carrier: the two ports are the two ends of the cable
 // under test, so with no cable there is never going to be one.
 func configureSystem(ifnames []string, ethertypes []uint16) []checkResult {
@@ -267,8 +267,9 @@ func (d *display) chipH() int { return d.grid.lineH + 2*chipPadY }

 func (d *display) countChipH() int { return d.chipH() + d.gridB.lineH + pairGap }

+// The error chips plus the noise cable's, which shares their row grid.
 func (d *display) chipsH() int {
-	return gridRows(len(errRows))*(d.chipH()+chipGap) - chipGap
+	return gridRows(len(errRows)+1)*(d.chipH()+chipGap) - chipGap
 }

 func (d *display) countsH() int {
@@ -277,8 +278,8 @@ func (d *display) countsH() int {

 // Outlined by drawing the border colour and sinking a smaller well of
 // background into it, so both curves get the same antialiasing.
-func (d *display) chipAt(i, x, w, y, h int, c rgb) (int, int, int) {
-	cx, cw := gridCell(i, len(errRows), x, w)
+func (d *display) chipAt(i, n, x, w, y, h int, c rgb) (int, int, int) {
+	cx, cw := gridCell(i, n, x, w)
 	cy := y + (i/gridCols)*(h+chipGap)

 	d.fb.roundRect(cx, cy, cw, h, chipRadius, c)
@@ -288,13 +289,18 @@ func (d *display) chipAt(i, x, w, y, h int, c rgb) (int, int, int) {
 }

 // Whether rather than how many: over a window this short a count changes faster
-// than it can be read.
-func (d *display) errChips(x, w, y int, e errs) int {
+// than it can be read. The noise chip rides along at the end, presence rather
+// than health: red is the cable missing, not the cable failing.
+func (d *display) errChips(x, w, y int, e errs, noiseMissing uint64) int {
+	n := len(errRows) + 1
 	for i, r := range errRows {
 		c := errColor(r.get(e))
-		cx, cw, cy := d.chipAt(i, x, w, y, d.chipH(), c)
+		cx, cw, cy := d.chipAt(i, n, x, w, y, d.chipH(), c)
 		d.centerIn(d.grid, cx, cw, cy+chipPadY, r.label, c)
 	}
+	c := errColor(noiseMissing)
+	cx, cw, cy := d.chipAt(len(errRows), n, x, w, y, d.chipH(), c)
+	d.centerIn(d.grid, cx, cw, cy+chipPadY, "noise", c)
 	return y + d.chipsH()
 }

@@ -302,7 +308,7 @@ func (d *display) errCounts(x, w, y int, e errs) int {
 	for i, r := range errRows {
 		n := r.get(e)
 		c := errColor(n)
-		cx, cw, cy := d.chipAt(i, x, w, y, d.countChipH(), c)
+		cx, cw, cy := d.chipAt(i, len(errRows), x, w, y, d.countChipH(), c)
 		ty := d.centerIn(d.gridB, cx, cw, cy+chipPadY, scaleCount(n), c)
 		d.centerIn(d.grid, cx, cw, ty, r.label, c)
 	}
@@ -329,7 +335,7 @@ func errColor(n uint64) rgb {
 	return uiRed
 }

-func (d *display) render(v view, elapsed time.Duration, cable string) error {
+func (d *display) render(v view, elapsed time.Duration, cable string, noiseMissing uint64) error {
 	fb := d.fb
 	fb.fill(uiBg)

@@ -338,7 +344,7 @@ func (d *display) render(v view, elapsed time.Duration, cable string) error {
 		{scaleSI(v.rxGbps * 1e9), "bits/s", uiFg},
 		{scaleSI(v.rxPPS), "packets/s", uiFg},
 	})
-	d.errChips(x, w, d.nowYs[1], v.window)
+	d.errChips(x, w, d.nowYs[1], v.window, noiseMissing)

 	x, w = d.panel(d.sincePanel, v.since)
 	d.stats(d.gridB, x, w, d.sinceYs[0], []statCell{