package main import ( "bytes" "encoding/binary" "fmt" "os" "path/filepath" "strings" "unsafe" "golang.org/x/sys/unix" ) const ( ethtoolGRXRINGS = 0x2d ethtoolGRXCLSRLCNT = 0x2e ethtoolGRXCLSRULE = 0x2f ethtoolGRXCLSRLALL = 0x30 ethtoolSRXCLSRLDEL = 0x31 ethtoolSRXCLSRLINS = 0x32 etherFlow = 0x12 ) type ethtoolFlowExt struct { padding [2]byte hDest [6]byte vlanEtype uint16 vlanTci uint16 data [2]uint32 } type ethtoolRxFlowSpec struct { flowType uint32 hU [52]byte hExt ethtoolFlowExt mU [52]byte mExt ethtoolFlowExt _ [4]byte ringCookie uint64 location uint32 _ [4]byte } type ethtoolRxnfc struct { cmd uint32 flowType uint32 data uint64 fs ethtoolRxFlowSpec ruleCnt uint32 _ [4]byte } func rxRings(fd int, ifname string) (uint64, error) { nfc := ethtoolRxnfc{cmd: ethtoolGRXRINGS} if err := ethtoolCall(fd, ifname, unsafe.Pointer(&nfc)); err != nil { return 0, err } return nfc.data, nil } // Returns the installed rule locations and the total filter capacity. ice // only honours filters near the top of that range, which is why ethtool's own // rule manager allocates downwards from the end. func allRuleLocations(fd int, ifname string) ([]uint32, uint32, error) { cnt := ethtoolRxnfc{cmd: ethtoolGRXCLSRLCNT} if err := ethtoolCall(fd, ifname, unsafe.Pointer(&cnt)); err != nil { return nil, 0, err } capacity := uint32(cnt.data) if cnt.ruleCnt == 0 { return nil, capacity, nil } // rule_locs follows rule_cnt directly, before the struct's tail padding, // and that is where the kernel copies it to. locOff := unsafe.Offsetof(ethtoolRxnfc{}.ruleCnt) + 4 buf := make([]byte, int(unsafe.Sizeof(ethtoolRxnfc{}))+4*int(cnt.ruleCnt)) all := (*ethtoolRxnfc)(unsafe.Pointer(&buf[0])) all.cmd = ethtoolGRXCLSRLALL all.ruleCnt = cnt.ruleCnt if err := ethtoolCall(fd, ifname, unsafe.Pointer(&buf[0])); err != nil { return nil, 0, err } if all.ruleCnt > cnt.ruleCnt { return nil, 0, fmt.Errorf("%s reported %d filter locations into room for %d", ifname, all.ruleCnt, cnt.ruleCnt) } raw := buf[locOff:] locs := make([]uint32, 0, all.ruleCnt) for i := 0; i < int(all.ruleCnt); i++ { locs = append(locs, binary.LittleEndian.Uint32(raw[i*4:])) } return locs, capacity, nil } func ruleIsEther(fd int, ifname string, loc uint32) bool { get := ethtoolRxnfc{cmd: ethtoolGRXCLSRULE} get.fs.location = loc if err := ethtoolCall(fd, ifname, unsafe.Pointer(&get)); err != nil { return false } return get.fs.flowType&0xff == etherFlow } func deleteRule(fd int, ifname string, loc uint32) error { nfc := ethtoolRxnfc{cmd: ethtoolSRXCLSRLDEL} nfc.fs.location = loc return ethtoolCall(fd, ifname, unsafe.Pointer(&nfc)) } // ice rejects RX_CLS_LOC_ANY, so the caller must supply a free location. func insertEtherRule(fd int, ifname string, ethType uint16, queue uint64, loc uint32) error { nfc := ethtoolRxnfc{cmd: ethtoolSRXCLSRLINS} nfc.fs.flowType = etherFlow binary.BigEndian.PutUint16(nfc.fs.hU[12:14], ethType) // A set mask bit means that bit must match, so mask only the ethertype and // leave both MAC masks zero. Note ethtool -n prints the complement of this. binary.BigEndian.PutUint16(nfc.fs.mU[12:14], 0xffff) nfc.fs.ringCookie = queue nfc.fs.location = loc return ethtoolCall(fd, ifname, unsafe.Pointer(&nfc)) } // Our rules survive process exit, so stale ones are cleared before the queue // configuration is touched and fresh ones installed. func clearFlowRules(fd int, ifname string) checkResult { res := checkResult{item: ifname + " stale rules"} locs, _, err := allRuleLocations(fd, ifname) if err != nil { res.err = err return res } n := 0 for _, loc := range locs { if !ruleIsEther(fd, ifname, loc) { continue } if err := deleteRule(fd, ifname, loc); err != nil { res.err = fmt.Errorf("deleting rule %d: %w", loc, err) return res } n++ } res.fixed = n > 0 res.state = fmt.Sprintf("%d removed", n) return res } func checkFlowRules(fd int, ifname string, ethertypes []uint16) checkResult { res := checkResult{item: ifname + " flow rules"} rings, err := rxRings(fd, ifname) if err != nil { res.err = err return res } if uint64(len(ethertypes)) > rings { res.err = fmt.Errorf("%d streams needs %d rx rings, only %d available", len(ethertypes), len(ethertypes), rings) return res } locs, capacity, err := allRuleLocations(fd, ifname) if err != nil { res.err = err return res } if capacity < uint32(len(ethertypes)) { res.err = fmt.Errorf("filter capacity %d is below %d streams", capacity, len(ethertypes)) return res } taken := make(map[uint32]bool, len(locs)) for _, loc := range locs { taken[loc] = true } // Inserting at a taken location would evict it, and would then leave every // later ethertype evicting the one before it at that same location. next := capacity - 1 for i, et := range ethertypes { for taken[next] { if next == 0 { res.err = fmt.Errorf("no free filter location below %d for ethertype 0x%04x", capacity, et) return res } next-- } if err := insertEtherRule(fd, ifname, et, uint64(i), next); err != nil { res.err = fmt.Errorf("steering ethertype 0x%04x to queue %d at location %d: %w", et, i, next, err) return res } taken[next] = true } res.state = fmt.Sprintf("0x%04x-0x%04x to queues 0-%d of %d", ethertypes[0], ethertypes[len(ethertypes)-1], len(ethertypes)-1, rings) return res } // The ifreq shape used by every ioctl that passes its payload by pointer. type dataIfreq struct { name [unix.IFNAMSIZ]byte data unsafe.Pointer _ [16]byte } type flagsIfreq struct { name [unix.IFNAMSIZ]byte flags uint16 _ [22]byte } type mtuIfreq struct { name [unix.IFNAMSIZ]byte mtu int32 _ [20]byte } func checkMTU(fd int, ifname string, want int32) checkResult { res := checkResult{item: ifname + " mtu"} var ifr mtuIfreq copy(ifr.name[:], ifname) if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCGIFMTU), uintptr(unsafe.Pointer(&ifr))); errno != 0 { res.err = errno return res } if ifr.mtu == want { res.state = fmt.Sprintf("%d", want) return res } was := ifr.mtu ifr.mtu = want if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCSIFMTU), uintptr(unsafe.Pointer(&ifr))); errno != 0 { res.err = errno res.state = fmt.Sprintf("could not set %d", want) return res } res.fixed = true res.state = fmt.Sprintf("was %d, now %d (link reset)", was, want) return res } type ethtoolRingparam struct { cmd uint32 rxMaxPending uint32 rxMiniMaxPending uint32 rxJumboMaxPending uint32 txMaxPending uint32 rxPending uint32 rxMiniPending uint32 rxJumboPending uint32 txPending uint32 } type ethtoolCoalesce struct { cmd uint32 rxCoalesceUsecs uint32 rxMaxCoalescedFrames uint32 rxCoalesceUsecsIrq uint32 rxMaxCoalescedFramesIrq uint32 txCoalesceUsecs uint32 txMaxCoalescedFrames uint32 txCoalesceUsecsIrq uint32 txMaxCoalescedFramesIrq uint32 statsBlockCoalesceUsecs uint32 useAdaptiveRxCoalesce uint32 useAdaptiveTxCoalesce uint32 pktRateLow uint32 rxCoalesceUsecsLow uint32 rxMaxCoalescedFramesLow uint32 txCoalesceUsecsLow uint32 txMaxCoalescedFramesLow uint32 pktRateHigh uint32 rxCoalesceUsecsHigh uint32 rxMaxCoalescedFramesHigh uint32 txCoalesceUsecsHigh uint32 txMaxCoalescedFramesHigh uint32 rateSampleInterval uint32 } const ( wantGovernor = "performance" wantCoalesceUsecs = 25 wantRxRing = 8160 wantTxRing = 4096 wantIdleLatency = 10 wantMTU = 9000 ) type checkResult struct { item string state string fixed bool err error } func (r checkResult) status() string { switch { case r.err != nil: return paint("FAIL", cRed) case r.fixed: return paint("FIXED", cYellow) default: return paint("ok", cGreen) } } func (r checkResult) detail() string { if r.err != nil { if r.state == "" { return r.err.Error() } return fmt.Sprintf("%s: %v", r.state, r.err) } return r.state } func reportChecks(title string, results []checkResult) error { var rows [][]string var failed []string for _, r := range results { rows = append(rows, []string{r.item, r.status(), r.detail()}) if r.err != nil { failed = append(failed, r.item) } } fmt.Println(renderBox(title, []string{"CHECK", "STATUS", "DETAIL"}, []bool{false, false, false}, rows)) if len(failed) > 0 { return fmt.Errorf("cannot test with %s in this state", strings.Join(failed, ", ")) } return nil } func ethtoolCall(fd int, ifname string, data unsafe.Pointer) error { var ifr dataIfreq if len(ifname) >= unix.IFNAMSIZ { return fmt.Errorf("interface name %q too long", ifname) } copy(ifr.name[:], ifname) ifr.data = data _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCETHTOOL), uintptr(unsafe.Pointer(&ifr))) if errno != 0 { return errno } return nil } func getRings(fd int, ifname string) (ethtoolRingparam, error) { rp := ethtoolRingparam{cmd: unix.ETHTOOL_GRINGPARAM} err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp)) return rp, err } func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) { ec := ethtoolCoalesce{cmd: unix.ETHTOOL_GCOALESCE} err := ethtoolCall(fd, ifname, unsafe.Pointer(&ec)) return ec, err } const ( ethSSStats = 1 ethSSPrivFlags = 2 ethGstringLen = 32 ) // Each of these has a __u32 or __u8 tail the kernel fills past the end of the // struct, so they are only ever the header on a larger buffer. type ethtoolSsetInfo struct { cmd uint32 reserved uint32 ssetMask uint64 data [1]uint32 _ [4]byte } type ethtoolGstrings struct { cmd uint32 stringSet uint32 len uint32 } type ethtoolStatsHdr struct { cmd uint32 nStats uint32 } // The stats payload is an array of __u64 immediately after the header, so the // buffer is allocated as uint64 to guarantee it lands on an eight byte // boundary. A []byte carries no such guarantee. func statsBuf(n int) []uint64 { return make([]uint64, n) } func stringSetCount(fd int, ifname string, set uint32) (uint32, error) { req := ethtoolSsetInfo{cmd: unix.ETHTOOL_GSSET_INFO, ssetMask: 1 << set} if err := ethtoolCall(fd, ifname, unsafe.Pointer(&req)); err != nil { return 0, err } // A cleared mask bit means the driver does not have that string set at all, // in which case no count was written into the tail. if req.ssetMask == 0 { return 0, fmt.Errorf("driver has no string set %d", set) } return req.data[0], nil } func stringSetNames(fd int, ifname string, set uint32) ([]string, error) { n, err := stringSetCount(fd, ifname, set) if err != nil { return nil, err } if n == 0 { return nil, fmt.Errorf("driver reports an empty string set %d", set) } hdrLen := int(unsafe.Sizeof(ethtoolGstrings{})) buf := statsBuf((hdrLen + int(n)*ethGstringLen + 7) / 8) hdr := (*ethtoolGstrings)(unsafe.Pointer(&buf[0])) hdr.cmd = unix.ETHTOOL_GSTRINGS hdr.stringSet = set hdr.len = n if err := ethtoolCall(fd, ifname, unsafe.Pointer(&buf[0])); err != nil { return nil, err } raw := unsafe.Slice((*byte)(unsafe.Pointer(&buf[0])), len(buf)*8)[hdrLen:] names := make([]string, hdr.len) for i := range names { s := raw[i*ethGstringLen : (i+1)*ethGstringLen] if k := bytes.IndexByte(s, 0); k >= 0 { s = s[:k] } names[i] = string(s) } return names, nil } func statValues(fd int, ifname string, n uint32, into []uint64) error { hdr := (*ethtoolStatsHdr)(unsafe.Pointer(&into[0])) hdr.cmd = unix.ETHTOOL_GSTATS hdr.nStats = n return ethtoolCall(fd, ifname, unsafe.Pointer(&into[0])) } // The ioctl indexes counters by position, and the ordering is a property of the // driver build, so names are resolved to indices once rather than per poll. type statReader struct { fd int ifname string n uint32 want []int buf []uint64 } func newStatReader(fd int, ifname string, want []string) (*statReader, error) { names, err := stringSetNames(fd, ifname, ethSSStats) if err != nil { return nil, fmt.Errorf("%s statistics: %w", ifname, err) } idx := make(map[string]int, len(names)) for i, s := range names { idx[s] = i } r := &statReader{ fd: fd, ifname: ifname, n: uint32(len(names)), buf: statsBuf(1 + len(names)), } for _, w := range want { i, ok := idx[w] if !ok { return nil, fmt.Errorf("%s has no statistic %q", ifname, w) } r.want = append(r.want, i) } return r, nil } func (r *statReader) sum() uint64 { if err := statValues(r.fd, r.ifname, r.n, r.buf); err != nil { panic(fmt.Sprintf("reading %s statistics: %v", r.ifname, err)) } vals := r.buf[1:] var total uint64 for _, i := range r.want { total += vals[i] } return total } func checkGovernor(want string) checkResult { res := checkResult{item: "cpu governor"} paths, err := filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor") if err != nil || len(paths) == 0 { res.err = fmt.Errorf("no cpufreq governors found") return res } var wrong []string counts := map[string]int{} for _, p := range paths { b, err := os.ReadFile(p) if err != nil { res.err = err return res } got := strings.TrimSpace(string(b)) counts[got]++ if got != want { wrong = append(wrong, p) } } if len(wrong) == 0 { res.state = fmt.Sprintf("%s on all %d cpus", want, len(paths)) return res } var found []string for k, v := range counts { found = append(found, fmt.Sprintf("%s:%d", k, v)) } for _, p := range wrong { if err := os.WriteFile(p, []byte(want), 0o644); err != nil { res.err = err res.state = fmt.Sprintf("could not set %s", p) return res } } res.fixed = true res.state = fmt.Sprintf("was %s, now %s on all %d cpus", strings.Join(found, " "), want, len(paths)) return res } // Bounds how far into idle any cpu may sink, so a worker or interrupt never // waits behind a deep state's exit. Ten microseconds admits C1, whose exit is // noise under the interrupt coalescing already in the path, and blocks the // states whose exits are many times that window. The kernel honours the bound // for exactly as long as the fd stays open, which is why it is opened once and // deliberately never closed: as PID 1 its lifetime is the machine's, and // everything is restored the moment the process goes. func checkIdleLatency(usecs uint32) checkResult { res := checkResult{item: "cpu idle latency"} fd, err := unix.Open("/dev/cpu_dma_latency", unix.O_RDWR|unix.O_CLOEXEC, 0) if err != nil { res.err = fmt.Errorf("opening /dev/cpu_dma_latency: %w", err) return res } var buf [4]byte binary.NativeEndian.PutUint32(buf[:], usecs) if _, err := unix.Write(fd, buf[:]); err != nil { unix.Close(fd) res.err = fmt.Errorf("writing cpu latency bound: %w", err) return res } res.state = fmt.Sprintf("capped at %dus while running", usecs) return res } func checkLinkUp(fd int, ifname string) checkResult { res := checkResult{item: ifname + " link up"} var ifr flagsIfreq copy(ifr.name[:], ifname) _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr))) if errno != 0 { res.err = errno return res } if ifr.flags&unix.IFF_UP != 0 { res.state = "up" return res } ifr.flags |= unix.IFF_UP _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr))) if errno != 0 { res.err = errno res.state = "could not set IFF_UP" return res } res.fixed = true res.state = "was down, now up" return res } func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32) checkResult { res := checkResult{item: ifname + " coalesce"} ec, err := getCoalesce(fd, ifname) if err != nil { res.err = err return res } desc := func(e ethtoolCoalesce) string { return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d", e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs) } settled := func(e ethtoolCoalesce, tx uint32) bool { return e.useAdaptiveRxCoalesce == 0 && e.useAdaptiveTxCoalesce == 0 && e.rxCoalesceUsecs == rxUsecs && e.txCoalesceUsecs == tx } if settled(ec, txUsecs) { res.state = desc(ec) return res } was := desc(ec) set := func(tx uint32) error { s := ec s.cmd = unix.ETHTOOL_SCOALESCE s.useAdaptiveRxCoalesce = 0 s.useAdaptiveTxCoalesce = 0 s.rxCoalesceUsecs = rxUsecs s.txCoalesceUsecs = tx return ethtoolCall(fd, ifname, unsafe.Pointer(&s)) } err = set(txUsecs) // A driver running mixed rx/tx vectors has one moderation value per // vector and rejects a tx-specific one: rx-usecs covers both, and get // reports the tx side as zero. if err == unix.EINVAL { if settled(ec, 0) { res.state = desc(ec) + " (tx shares rx)" return res } if err = set(0); err == nil { res.fixed = true res.state = fmt.Sprintf("was %s, now adaptive off rx-usecs=%d shared with tx", was, rxUsecs) return res } } if err != nil { res.err = err res.state = "could not set" return res } res.fixed = true res.state = fmt.Sprintf("was %s, now adaptive off rx-usecs=%d tx-usecs=%d", was, rxUsecs, txUsecs) return res } func checkRings(fd int, ifname string, rxWant, txWant uint32) checkResult { res := checkResult{item: ifname + " rings"} rp, err := getRings(fd, ifname) if err != nil { res.err = err return res } rx := min(rxWant, rp.rxMaxPending) tx := min(txWant, rp.txMaxPending) if rp.rxPending == rx && rp.txPending == tx { res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) return res } was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) rp.cmd = unix.ETHTOOL_SRINGPARAM rp.rxPending = rx rp.txPending = tx if err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp)); err != nil { res.err = err res.state = "could not set" return res } res.fixed = true res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link reset)", was, rx, tx) return res } func withIoctlSocket(fn func(fd int) []checkResult) []checkResult { fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) if err != nil { return []checkResult{{item: "ioctl socket", err: err}} } defer unix.Close(fd) return fn(fd) } type ethtoolValue struct { cmd uint32 data uint32 } // i40e leaves the module transmitting when a port is closed, so the peer never // sees anything happen and the copper stays trained. This flag is what makes // an administrative down reach the wire, and the noise cycle is switched // entirely through it. func checkPrivFlag(fd int, ifname, flag string) checkResult { res := checkResult{item: ifname + " " + flag} names, err := stringSetNames(fd, ifname, ethSSPrivFlags) if err != nil { res.err = err return res } bit := -1 for i, s := range names { if s == flag { bit = i break } } if bit < 0 { res.err = fmt.Errorf("driver has no private flag %q", flag) return res } v := ethtoolValue{cmd: unix.ETHTOOL_GPFLAGS} if err := ethtoolCall(fd, ifname, unsafe.Pointer(&v)); err != nil { res.err = err return res } if v.data&(1<