Always correct host settings instead of gating fixes behind flags

This commit is contained in:
flamingcow
2026-07-25 18:16:34 -07:00
parent 27f89d9916
commit 773724368e
2 changed files with 45 additions and 73 deletions
-4
View File
@@ -369,12 +369,10 @@ func main() {
rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket") rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket")
txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to") txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to")
rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to") rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to")
tune = flag.Bool("tune", true, "check host settings at startup and correct them; without this they are only reported")
governor = flag.String("governor", "performance", "required cpufreq governor") governor = flag.String("governor", "performance", "required cpufreq governor")
coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off") coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off")
rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum") rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum")
txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum") txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum")
setRings = flag.Bool("set-rings", true, "allow ring resizing at startup, which resets the link")
color = flag.String("color", "auto", "colored output: auto, always, never") color = flag.String("color", "auto", "colored output: auto, always, never")
) )
flag.Parse() flag.Parse()
@@ -385,13 +383,11 @@ func main() {
} }
tcfg := tuneConfig{ tcfg := tuneConfig{
enabled: *tune,
governor: *governor, governor: *governor,
rxUsecs: uint32(*coalesce), rxUsecs: uint32(*coalesce),
txUsecs: uint32(*coalesce), txUsecs: uint32(*coalesce),
rxRing: uint32(*rxRing), rxRing: uint32(*rxRing),
txRing: uint32(*txRing), txRing: uint32(*txRing),
setRings: *setRings,
} }
if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs, if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs,
+34 -58
View File
@@ -63,13 +63,11 @@ type ethtoolCoalesce struct {
} }
type tuneConfig struct { type tuneConfig struct {
enabled bool
governor string governor string
rxUsecs uint32 rxUsecs uint32
txUsecs uint32 txUsecs uint32
rxRing uint32 rxRing uint32
txRing uint32 txRing uint32
setRings bool
} }
type tuneResult struct { type tuneResult struct {
@@ -128,13 +126,9 @@ func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) {
return ec, err return ec, err
} }
func governorPaths() ([]string, error) { func checkGovernor(want string) tuneResult {
return filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
}
func checkGovernor(want string, fix bool) tuneResult {
res := tuneResult{item: "cpu governor"} res := tuneResult{item: "cpu governor"}
paths, err := governorPaths() paths, err := filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
if err != nil || len(paths) == 0 { if err != nil || len(paths) == 0 {
res.err = fmt.Errorf("no cpufreq governors found") res.err = fmt.Errorf("no cpufreq governors found")
return res return res
@@ -161,11 +155,6 @@ func checkGovernor(want string, fix bool) tuneResult {
for k, v := range counts { for k, v := range counts {
found = append(found, fmt.Sprintf("%s:%d", k, v)) found = append(found, fmt.Sprintf("%s:%d", k, v))
} }
if !fix {
res.err = fmt.Errorf("want %s, found %s", want, strings.Join(found, " "))
res.state = "not corrected"
return res
}
for _, p := range wrong { for _, p := range wrong {
if err := os.WriteFile(p, []byte(want), 0o644); err != nil { if err := os.WriteFile(p, []byte(want), 0o644); err != nil {
res.err = err res.err = err
@@ -178,7 +167,7 @@ func checkGovernor(want string, fix bool) tuneResult {
return res return res
} }
func checkLinkUp(fd int, ifname string, fix bool) tuneResult { func checkLinkUp(fd int, ifname string) tuneResult {
res := tuneResult{item: ifname + " link up"} res := tuneResult{item: ifname + " link up"}
var ifr flagsIfreq var ifr flagsIfreq
copy(ifr.name[:], ifname) copy(ifr.name[:], ifname)
@@ -186,18 +175,13 @@ func checkLinkUp(fd int, ifname string, fix bool) tuneResult {
uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr))) uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
if errno != 0 { if errno != 0 {
res.err = errno res.err = errno
res.fatal = true
return res return res
} }
if ifr.flags&unix.IFF_UP != 0 { if ifr.flags&unix.IFF_UP != 0 {
res.state = "up" res.state = "up"
return res return res
} }
if !fix {
res.err = fmt.Errorf("interface is down")
res.state = "not corrected"
res.fatal = true
return res
}
ifr.flags |= unix.IFF_UP ifr.flags |= unix.IFF_UP
_, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd), _, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr))) uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
@@ -230,28 +214,22 @@ func checkCarrier(ifname string, wait time.Duration) tuneResult {
} }
} }
func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tuneResult { func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32) tuneResult {
res := tuneResult{item: ifname + " coalesce"} res := tuneResult{item: ifname + " coalesce"}
ec, err := getCoalesce(fd, ifname) ec, err := getCoalesce(fd, ifname)
if err != nil { if err != nil {
res.err = err res.err = err
return res return res
} }
ok := ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs
desc := func(e ethtoolCoalesce) string { desc := func(e ethtoolCoalesce) string {
return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d", return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d",
e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs) e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs)
} }
if ok { if ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs {
res.state = desc(ec) res.state = desc(ec)
return res return res
} }
if !fix {
res.err = fmt.Errorf("want adaptive off rx-usecs=%d tx-usecs=%d, have %s", rxUsecs, txUsecs, desc(ec))
res.state = "not corrected"
return res
}
was := desc(ec) was := desc(ec)
ec.cmd = unix.ETHTOOL_SCOALESCE ec.cmd = unix.ETHTOOL_SCOALESCE
ec.useAdaptiveRxCoalesce = 0 ec.useAdaptiveRxCoalesce = 0
@@ -268,7 +246,7 @@ func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tun
return res return res
} }
func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneResult, bool) { func checkRings(fd int, ifname string, rxWant, txWant uint32) (tuneResult, bool) {
res := tuneResult{item: ifname + " rings"} res := tuneResult{item: ifname + " rings"}
rp, err := getRings(fd, ifname) rp, err := getRings(fd, ifname)
if err != nil { if err != nil {
@@ -281,11 +259,6 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes
res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
return res, false return res, false
} }
if !fix {
res.err = fmt.Errorf("want rx=%d tx=%d, have rx=%d tx=%d", rx, tx, rp.rxPending, rp.txPending)
res.state = "not corrected"
return res, false
}
was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending) was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
rp.cmd = unix.ETHTOOL_SRINGPARAM rp.cmd = unix.ETHTOOL_SRINGPARAM
rp.rxPending = rx rp.rxPending = rx
@@ -296,7 +269,7 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes
return res, false return res, false
} }
res.fixed = true res.fixed = true
res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link resets)", was, rx, tx) res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link reset)", was, rx, tx)
return res, true return res, true
} }
@@ -324,57 +297,60 @@ func checkNoAddrs(ifname string) tuneResult {
res.state = "no routable addresses" res.state = "no routable addresses"
return res return res
} }
res.err = fmt.Errorf("has %s, something is still configuring this interface", strings.Join(routable, ",")) res.err = fmt.Errorf("has %s; NetworkManager or DHCP is configuring this interface",
res.state = "not corrected" strings.Join(routable, ","))
return res return res
} }
func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult { func withIoctlSocket(fn func(fd int) []tuneResult) []tuneResult {
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
if err != nil { if err != nil {
return []tuneResult{{item: "ioctl socket", err: err}} return []tuneResult{{item: "ioctl socket", err: err, fatal: true}}
} }
defer unix.Close(fd) defer unix.Close(fd)
return fn(fd)
}
var out []tuneResult func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
out = append(out, checkGovernor(cfg.governor, cfg.enabled)) return withIoctlSocket(func(fd int) []tuneResult {
out := []tuneResult{checkGovernor(cfg.governor)}
for _, ifname := range ifnames { for _, ifname := range ifnames {
out = append(out, checkLinkUp(fd, ifname, cfg.enabled)) out = append(out, checkLinkUp(fd, ifname))
out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled)) out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs))
carrierWait := 3 * time.Second carrierWait := 3 * time.Second
if cfg.setRings { r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing)
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, cfg.enabled)
out = append(out, r) out = append(out, r)
if reset { if reset {
carrierWait = 10 * time.Second carrierWait = 10 * time.Second
} }
}
out = append(out, checkCarrier(ifname, carrierWait)) out = append(out, checkCarrier(ifname, carrierWait))
out = append(out, checkNoAddrs(ifname)) out = append(out, checkNoAddrs(ifname))
} }
return out return out
})
} }
func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult { func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) return withIoctlSocket(func(fd int) []tuneResult {
if err != nil {
return []tuneResult{{item: "drift check", err: err}}
}
defer unix.Close(fd)
var drifted []tuneResult var drifted []tuneResult
if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil { if r := checkGovernor(cfg.governor); r.fixed || r.err != nil {
drifted = append(drifted, r) drifted = append(drifted, r)
} }
for _, ifname := range ifnames { for _, ifname := range ifnames {
if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil { if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs); r.fixed || r.err != nil {
drifted = append(drifted, r) drifted = append(drifted, r)
} }
if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil { r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing)
r.state = "left alone, resizing would reset the link" if r.fixed || r.err != nil {
drifted = append(drifted, r) drifted = append(drifted, r)
} }
if reset {
if c := checkCarrier(ifname, 10*time.Second); c.err != nil {
drifted = append(drifted, c)
}
}
} }
return drifted return drifted
})
} }