diff --git a/main.go b/main.go index caffdc8..f836733 100644 --- a/main.go +++ b/main.go @@ -369,12 +369,10 @@ func main() { rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket") txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to") rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to") - tune = flag.Bool("tune", true, "check host settings at startup and correct them; without this they are only reported") governor = flag.String("governor", "performance", "required cpufreq governor") coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off") rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum") txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum") - setRings = flag.Bool("set-rings", true, "allow ring resizing at startup, which resets the link") color = flag.String("color", "auto", "colored output: auto, always, never") ) flag.Parse() @@ -385,13 +383,11 @@ func main() { } tcfg := tuneConfig{ - enabled: *tune, governor: *governor, rxUsecs: uint32(*coalesce), txUsecs: uint32(*coalesce), rxRing: uint32(*rxRing), txRing: uint32(*txRing), - setRings: *setRings, } if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs, diff --git a/tune.go b/tune.go index 8086bd8..f8f715f 100644 --- a/tune.go +++ b/tune.go @@ -63,13 +63,11 @@ type ethtoolCoalesce struct { } type tuneConfig struct { - enabled bool governor string rxUsecs uint32 txUsecs uint32 rxRing uint32 txRing uint32 - setRings bool } type tuneResult struct { @@ -128,13 +126,9 @@ func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) { return ec, err } -func governorPaths() ([]string, error) { - return filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor") -} - -func checkGovernor(want string, fix bool) tuneResult { +func checkGovernor(want string) tuneResult { res := tuneResult{item: "cpu governor"} - paths, err := governorPaths() + paths, err := filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor") if err != nil || len(paths) == 0 { res.err = fmt.Errorf("no cpufreq governors found") return res @@ -161,11 +155,6 @@ func checkGovernor(want string, fix bool) tuneResult { for k, v := range counts { found = append(found, fmt.Sprintf("%s:%d", k, v)) } - if !fix { - res.err = fmt.Errorf("want %s, found %s", want, strings.Join(found, " ")) - res.state = "not corrected" - return res - } for _, p := range wrong { if err := os.WriteFile(p, []byte(want), 0o644); err != nil { res.err = err @@ -178,7 +167,7 @@ func checkGovernor(want string, fix bool) tuneResult { return res } -func checkLinkUp(fd int, ifname string, fix bool) tuneResult { +func checkLinkUp(fd int, ifname string) tuneResult { res := tuneResult{item: ifname + " link up"} var ifr flagsIfreq copy(ifr.name[:], ifname) @@ -186,18 +175,13 @@ func checkLinkUp(fd int, ifname string, fix bool) tuneResult { uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr))) if errno != 0 { res.err = errno + res.fatal = true return res } if ifr.flags&unix.IFF_UP != 0 { res.state = "up" return res } - if !fix { - res.err = fmt.Errorf("interface is down") - res.state = "not corrected" - res.fatal = true - return res - } ifr.flags |= unix.IFF_UP _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr))) @@ -230,28 +214,22 @@ func checkCarrier(ifname string, wait time.Duration) tuneResult { } } -func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tuneResult { +func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32) tuneResult { res := tuneResult{item: ifname + " coalesce"} ec, err := getCoalesce(fd, ifname) if err != nil { res.err = err return res } - ok := ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 && - ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs desc := func(e ethtoolCoalesce) string { return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d", e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs) } - if ok { + if ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 && + ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs { res.state = desc(ec) return res } - if !fix { - res.err = fmt.Errorf("want adaptive off rx-usecs=%d tx-usecs=%d, have %s", rxUsecs, txUsecs, desc(ec)) - res.state = "not corrected" - return res - } was := desc(ec) ec.cmd = unix.ETHTOOL_SCOALESCE ec.useAdaptiveRxCoalesce = 0 @@ -268,7 +246,7 @@ func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tun return res } -func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneResult, bool) { +func checkRings(fd int, ifname string, rxWant, txWant uint32) (tuneResult, bool) { res := tuneResult{item: ifname + " rings"} rp, err := getRings(fd, ifname) if err != nil { @@ -281,11 +259,6 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) return res, false } - if !fix { - res.err = fmt.Errorf("want rx=%d tx=%d, have rx=%d tx=%d", rx, tx, rp.rxPending, rp.txPending) - res.state = "not corrected" - return res, false - } was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) rp.cmd = unix.ETHTOOL_SRINGPARAM rp.rxPending = rx @@ -296,7 +269,7 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes return res, false } res.fixed = true - res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link resets)", was, rx, tx) + res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link reset)", was, rx, tx) return res, true } @@ -324,57 +297,60 @@ func checkNoAddrs(ifname string) tuneResult { res.state = "no routable addresses" return res } - res.err = fmt.Errorf("has %s, something is still configuring this interface", strings.Join(routable, ",")) - res.state = "not corrected" + res.err = fmt.Errorf("has %s; NetworkManager or DHCP is configuring this interface", + strings.Join(routable, ",")) return res } -func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult { +func withIoctlSocket(fn func(fd int) []tuneResult) []tuneResult { fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) if err != nil { - return []tuneResult{{item: "ioctl socket", err: err}} + return []tuneResult{{item: "ioctl socket", err: err, fatal: true}} } defer unix.Close(fd) + return fn(fd) +} - var out []tuneResult - out = append(out, checkGovernor(cfg.governor, cfg.enabled)) +func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult { + return withIoctlSocket(func(fd int) []tuneResult { + out := []tuneResult{checkGovernor(cfg.governor)} + for _, ifname := range ifnames { + out = append(out, checkLinkUp(fd, ifname)) + out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs)) - for _, ifname := range ifnames { - out = append(out, checkLinkUp(fd, ifname, cfg.enabled)) - out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled)) - carrierWait := 3 * time.Second - if cfg.setRings { - r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, cfg.enabled) + carrierWait := 3 * time.Second + r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing) out = append(out, r) if reset { carrierWait = 10 * time.Second } + out = append(out, checkCarrier(ifname, carrierWait)) + out = append(out, checkNoAddrs(ifname)) } - out = append(out, checkCarrier(ifname, carrierWait)) - out = append(out, checkNoAddrs(ifname)) - } - return out + return out + }) } func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult { - fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) - if err != nil { - return []tuneResult{{item: "drift check", err: err}} - } - defer unix.Close(fd) - - var drifted []tuneResult - if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil { - drifted = append(drifted, r) - } - for _, ifname := range ifnames { - if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil { + return withIoctlSocket(func(fd int) []tuneResult { + var drifted []tuneResult + if r := checkGovernor(cfg.governor); r.fixed || r.err != nil { drifted = append(drifted, r) } - if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil { - r.state = "left alone, resizing would reset the link" - drifted = append(drifted, r) + for _, ifname := range ifnames { + if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs); r.fixed || r.err != nil { + drifted = append(drifted, r) + } + r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing) + if r.fixed || r.err != nil { + drifted = append(drifted, r) + } + if reset { + if c := checkCarrier(ifname, 10*time.Second); c.err != nil { + drifted = append(drifted, c) + } + } } - } - return drifted + return drifted + }) }