diff --git a/kernel/config b/kernel/config index ee969a0..17a4812 100644 --- a/kernel/config +++ b/kernel/config @@ -1779,7 +1779,7 @@ CONFIG_LIBIE_FWLOG=y # CONFIG_IGBVF is not set # CONFIG_IXGBE is not set # CONFIG_IXGBEVF is not set -# CONFIG_I40E is not set +CONFIG_I40E=y # CONFIG_I40EVF is not set CONFIG_ICE=y CONFIG_ICE_HWMON=y diff --git a/main.go b/main.go index c97d7b8..c116bbc 100644 --- a/main.go +++ b/main.go @@ -252,6 +252,7 @@ var intervalCols = []colSpec{ {group: "NOW", title: "corrupt", width: 7, right: true}, {group: "NOW", title: "link", width: 7, right: true}, {group: "NOW", title: "internal", width: 8, right: true}, + {group: "NOW", title: "noise", width: 7, right: true}, {group: "OVERALL", title: "elapsed", width: 9, right: true}, {group: "OVERALL", title: "packets", width: 9, right: true}, {group: "OVERALL", title: "bytes", width: 9, right: true}, @@ -340,9 +341,10 @@ func (d *direction) displayView() view { return v } -// The same figures the panel draws, in the same order: the last second as rates -// and error flags, then everything since the reset. -func totalRow(elapsed time.Duration, v view, target float64, length string) []string { +// The same figures the panel draws, in the same order: the last second as +// rates and error flags with the noise cable riding at the end of them, then +// everything since the reset. +func totalRow(elapsed time.Duration, v view, target float64, length string, noiseMissing uint64) []string { return []string{ rateCell(v.rxGbps*1e9, target*1e9), scaleSI(v.rxPPS), @@ -350,6 +352,7 @@ func totalRow(elapsed time.Duration, v view, target float64, length string) []st flagCell(v.window.corrupt), flagCell(v.window.link), flagCell(v.window.internal), + flagCell(noiseMissing), scaleTime(elapsed), scaleCount(v.rxFrames), scaleCount(v.rxBytes), @@ -513,6 +516,8 @@ const ( probeEther uint16 = etherBase + numStreams + testDriver = "ice" + // A constant rather than the negotiated speed, since this has to come up // with no cable in the port and nothing to negotiate. linkSpeed = 10.0 @@ -523,10 +528,11 @@ const ( var frameSizes = []int{60, 128, 256, 512, 1024, 1280, 1514} func main() { - // The names the kernel gives the only two ports built into it, since as - // PID 1 there is no udev to rename them and no command line to pass. - aName := flag.String("a", "eth0", "first interface") - bName := flag.String("b", "eth1", "second interface") + // Left empty, the test pair is found by driver name instead: as PID 1 there + // is no udev to pin names and no command line to pass, and which port gets + // which ethN shifts with every driver built into the kernel. + aName := flag.String("a", "", "first interface (default: the ice pair)") + bName := flag.String("b", "", "second interface") nsPerM := flag.Float64("ns-per-m", 4.85, "mean of both directions, per metre of cable") flag.Parse() @@ -580,6 +586,13 @@ func run(aName, bName string, nsPerM float64) error { return err } + if aName == "" || bName == "" { + var err error + aName, bName, err = driverPair(testDriver) + if err != nil { + return err + } + } a, err := lookupEndpoint(aName) if err != nil { return err @@ -588,6 +601,11 @@ func run(aName, bName string, nsPerM float64) error { if err != nil { return err } + noise, err := newNoiser() + if err != nil { + return err + } + defer noise.close() for _, e := range []endpoint{a, b} { for _, s := range frameSizes { if s > e.mtu+ethHdrLen { @@ -596,7 +614,8 @@ func run(aName, bName string, nsPerM float64) error { } } - a.tag, b.tag = "A", "B" + a.tag, b.tag = "TEST A", "TEST B" + noise.eps[0].tag, noise.eps[1].tag = "NOISE A", "NOISE B" ifnames := []string{a.name, b.name} ethertypes := make([]uint16, numStreams) @@ -604,7 +623,8 @@ func run(aName, bName string, nsPerM float64) error { ethertypes[i] = uint16(etherBase + i) } - if err := reportChecks("HOST SETTINGS", configureSystem(ifnames, ethertypes)); err != nil { + if err := reportChecks("HOST SETTINGS", + append(configureSystem(ifnames, ethertypes), configureNoise(noise.names())...)); err != nil { return err } @@ -623,7 +643,7 @@ func run(aName, bName string, nsPerM float64) error { }() var linkRows [][]string - for _, e := range []endpoint{a, b} { + for _, e := range []endpoint{a, b, noise.eps[0], noise.eps[1]} { linkRows = append(linkRows, []string{ paint(e.tag, cCyan), e.name, e.macString(), fmt.Sprintf("%d", e.mtu), }) @@ -652,6 +672,13 @@ func run(aName, bName string, nsPerM float64) error { defer wg.Done() samp.run(&done, startTx) }() + // Not gated on startTx: the cycle and the connected verdict are wanted the + // moment the panel is, and nothing it does touches the measurement. + wg.Add(1) + go func() { + defer wg.Done() + noise.run(&done) + }() // Every return from here on stops the workers before the deferred closes // pull their sockets out from under them: otherwise the sampler panics on a // closed fd and can mask the error that actually ended the run. An error @@ -719,7 +746,8 @@ func run(aName, bName string, nsPerM float64) error { if m, ok := cableMetres(views, nsPerM); ok { cable = fmt.Sprintf("%.1f", m) } - if err := disp.render(totalView(views), now.Sub(start), cable); err != nil { + if err := disp.render(totalView(views), now.Sub(start), cable, + noise.missing()); err != nil { return err } case now := <-tick.C: @@ -733,7 +761,8 @@ func run(aName, bName string, nsPerM float64) error { if m, ok := cableMetres(rows, nsPerM); ok { length = fmt.Sprintf("%.1f", m) } - for _, line := range stats.emit(totalRow(elapsed, totalView(rows), target, length)) { + for _, line := range stats.emit(totalRow(elapsed, totalView(rows), target, length, + noise.missing())) { fmt.Println(line) } } diff --git a/noise.go b/noise.go new file mode 100644 index 0000000..624b48c --- /dev/null +++ b/noise.go @@ -0,0 +1,206 @@ +package main + +import ( + "encoding/binary" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync/atomic" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// The noise cable: a deliberately bad cable twisted around the one under test, +// there to radiate into it. Its ports are driven for the interference they +// produce, not measured: nothing is ever received from them, and nothing they +// count reaches the error columns. +// +// The wire cannot be quieted by going idle, because without EEE the PHYs +// signal at full power whether or not frames flow, and these PHYs live inside +// the SFP+ modules where no EEE control reaches them. Closing the port is the +// one switch the host actually has, so the cycle is built on it: links up and +// carrying frames for a spell, then administratively down for one. Every wake +// re-runs 10GBASE-T training, which is as loud as this wire ever gets. +const ( + noiseDriver = "i40e" + noiseFrameLen = 1514 + noiseUpSpan = 5 * time.Second + noiseDownSpan = 5 * time.Second + noiseFrameGap = 10 * time.Millisecond + + noiseEther uint16 = probeEther + 1 +) + +// Kernel names shift with which drivers are built in, since ethN is handed out +// in link order rather than by slot. The driver name is the one label a port +// keeps across kernel configs, so pairs are found by it rather than named. +func driverPair(driver string) (string, string, error) { + ents, err := os.ReadDir("/sys/class/net") + if err != nil { + return "", "", err + } + var names []string + for _, e := range ents { + link, err := os.Readlink("/sys/class/net/" + e.Name() + "/device/driver") + if err != nil { + continue + } + if filepath.Base(link) == driver { + names = append(names, e.Name()) + } + } + if len(names) != 2 { + return "", "", fmt.Errorf("want 2 %s interfaces, found %d [%s]", + driver, len(names), strings.Join(names, " ")) + } + sort.Strings(names) + return names[0], names[1], nil +} + +type noisePort struct { + name string + fd int + frame []byte +} + +type noiser struct { + eps [2]endpoint + ports [2]noisePort + + // Whether the cable is judged present: both carriers seen during an up + // phase. Latched across the down phase, where the missing carrier is our + // own doing and says nothing about the cable. + connected atomic.Bool +} + +func newNoiser() (*noiser, error) { + aName, bName, err := driverPair(noiseDriver) + if err != nil { + return nil, fmt.Errorf("noise: %w", err) + } + a, err := lookupEndpoint(aName) + if err != nil { + return nil, fmt.Errorf("noise: %w", err) + } + b, err := lookupEndpoint(bName) + if err != nil { + return nil, fmt.Errorf("noise: %w", err) + } + + n := &noiser{eps: [2]endpoint{a, b}} + for i, p := range [][2]endpoint{{a, b}, {b, a}} { + fd, err := openTxSocket(p[0].idx) + if err != nil { + return nil, fmt.Errorf("noise tx socket %s: %w", p[0].name, err) + } + // The payload is left zero: the PCS scrambles everything on the wire, + // so no pattern radiates differently from any other. The frame exists + // to occupy the link, not to say anything. + frame := make([]byte, noiseFrameLen) + copy(frame[0:6], p[1].mac[:]) + copy(frame[6:12], p[0].mac[:]) + binary.BigEndian.PutUint16(frame[12:14], noiseEther) + n.ports[i] = noisePort{name: p[0].name, fd: fd, frame: frame} + } + return n, nil +} + +func (n *noiser) names() []string { + return []string{n.eps[0].name, n.eps[1].name} +} + +// Zero while the cable was there at the last verdict, one while it was not: +// the shape the error cells already colour by, so absence paints as the fault +// it is and presence as the usual green. +func (n *noiser) missing() uint64 { + if n.connected.Load() { + return 0 + } + return 1 +} + +// The ports were reachable when the noiser was built, so one that stops taking +// the ioctl now is the interface going away underneath us, the same fault the +// counter reads stop for. +func (n *noiser) setLinks(fd int, up bool) { + for i := range n.ports { + var ifr flagsIfreq + copy(ifr.name[:], n.ports[i].name) + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), + uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr))); errno != 0 { + panic(fmt.Sprintf("reading %s flags: %v", n.ports[i].name, errno)) + } + if up { + ifr.flags |= unix.IFF_UP + } else { + ifr.flags &^= unix.IFF_UP + } + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), + uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr))); errno != 0 { + panic(fmt.Sprintf("setting %s flags: %v", n.ports[i].name, errno)) + } + } +} + +// Down reads as EINVAL rather than zero, and either way the answer is the +// same: no carrier here now. +func carrierUp(name string) bool { + v, ok := readUint("/sys/class/net/" + name + "/carrier") + return ok && v == 1 +} + +func (n *noiser) bothUp() bool { + return carrierUp(n.ports[0].name) && carrierUp(n.ports[1].name) +} + +// Send results are deliberately dropped: the cable is bad on purpose, the link +// comes and goes under the cycle, and a frame this side declined to send is as +// good as one the wire mangled. What matters is only ever what the test cable +// counted. +func (n *noiser) run(done *atomic.Bool) { + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) + if err != nil { + panic(fmt.Sprintf("noise ioctl socket: %v", err)) + } + defer unix.Close(fd) + + tick := time.NewTicker(noiseFrameGap) + defer tick.Stop() + + for !done.Load() { + n.setLinks(fd, true) + linked := false + for end := time.Now().Add(noiseUpSpan); time.Now().Before(end) && !done.Load(); { + <-tick.C + if !n.bothUp() { + continue + } + linked = true + n.connected.Store(true) + for i := range n.ports { + unix.Write(n.ports[i].fd, n.ports[i].frame) + } + } + // A whole up phase with no link is many times the ~1s the wire needs + // to train, so by now the silence is the cable's answer. + n.connected.Store(linked) + + n.setLinks(fd, false) + for end := time.Now().Add(noiseDownSpan); time.Now().Before(end) && !done.Load(); { + <-tick.C + } + } + // Left up rather than wherever the cycle stopped, so a run never strands + // the ports down for whoever looks next. + n.setLinks(fd, true) +} + +func (n *noiser) close() { + for i := range n.ports { + unix.Close(n.ports[i].fd) + } +} diff --git a/system.go b/system.go index cd66b11..3121391 100644 --- a/system.go +++ b/system.go @@ -334,8 +334,9 @@ func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) { } const ( - ethSSStats = 1 - ethGstringLen = 32 + ethSSStats = 1 + ethSSPrivFlags = 2 + ethGstringLen = 32 ) // Each of these has a __u32 or __u8 tail the kernel fills past the end of the @@ -364,33 +365,33 @@ type ethtoolStatsHdr struct { // boundary. A []byte carries no such guarantee. func statsBuf(n int) []uint64 { return make([]uint64, n) } -func statCount(fd int, ifname string) (uint32, error) { - req := ethtoolSsetInfo{cmd: unix.ETHTOOL_GSSET_INFO, ssetMask: 1 << ethSSStats} +func stringSetCount(fd int, ifname string, set uint32) (uint32, error) { + req := ethtoolSsetInfo{cmd: unix.ETHTOOL_GSSET_INFO, ssetMask: 1 << set} if err := ethtoolCall(fd, ifname, unsafe.Pointer(&req)); err != nil { return 0, err } // A cleared mask bit means the driver does not have that string set at all, // in which case no count was written into the tail. if req.ssetMask == 0 { - return 0, fmt.Errorf("driver has no ETH_SS_STATS string set") + return 0, fmt.Errorf("driver has no string set %d", set) } return req.data[0], nil } -func statNames(fd int, ifname string) ([]string, error) { - n, err := statCount(fd, ifname) +func stringSetNames(fd int, ifname string, set uint32) ([]string, error) { + n, err := stringSetCount(fd, ifname, set) if err != nil { return nil, err } if n == 0 { - return nil, fmt.Errorf("driver reports zero statistics") + return nil, fmt.Errorf("driver reports an empty string set %d", set) } hdrLen := int(unsafe.Sizeof(ethtoolGstrings{})) buf := statsBuf((hdrLen + int(n)*ethGstringLen + 7) / 8) hdr := (*ethtoolGstrings)(unsafe.Pointer(&buf[0])) hdr.cmd = unix.ETHTOOL_GSTRINGS - hdr.stringSet = ethSSStats + hdr.stringSet = set hdr.len = n if err := ethtoolCall(fd, ifname, unsafe.Pointer(&buf[0])); err != nil { return nil, err @@ -426,7 +427,7 @@ type statReader struct { } func newStatReader(fd int, ifname string, want []string) (*statReader, error) { - names, err := statNames(fd, ifname) + names, err := stringSetNames(fd, ifname, ethSSStats) if err != nil { return nil, fmt.Errorf("%s statistics: %w", ifname, err) } @@ -623,6 +624,67 @@ func withIoctlSocket(fn func(fd int) []checkResult) []checkResult { return fn(fd) } +type ethtoolValue struct { + cmd uint32 + data uint32 +} + +// i40e leaves the module transmitting when a port is closed, so the peer never +// sees anything happen and the copper stays trained. This flag is what makes +// an administrative down reach the wire, and the noise cycle is switched +// entirely through it. +func checkPrivFlag(fd int, ifname, flag string) checkResult { + res := checkResult{item: ifname + " " + flag} + names, err := stringSetNames(fd, ifname, ethSSPrivFlags) + if err != nil { + res.err = err + return res + } + bit := -1 + for i, s := range names { + if s == flag { + bit = i + break + } + } + if bit < 0 { + res.err = fmt.Errorf("driver has no private flag %q", flag) + return res + } + v := ethtoolValue{cmd: unix.ETHTOOL_GPFLAGS} + if err := ethtoolCall(fd, ifname, unsafe.Pointer(&v)); err != nil { + res.err = err + return res + } + if v.data&(1<