Check and correct host tuning at startup and on drift
This commit is contained in:
flamingcow
@@ -329,11 +329,27 @@ func main() {
|
||||
rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket")
|
||||
txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to")
|
||||
rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to")
|
||||
tune = flag.Bool("tune", true, "check host settings at startup and correct them; without this they are only reported")
|
||||
governor = flag.String("governor", "performance", "required cpufreq governor")
|
||||
coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off")
|
||||
rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum")
|
||||
txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum")
|
||||
setRings = flag.Bool("set-rings", true, "allow ring resizing at startup, which resets the link")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
tcfg := tuneConfig{
|
||||
enabled: *tune,
|
||||
governor: *governor,
|
||||
rxUsecs: uint32(*coalesce),
|
||||
txUsecs: uint32(*coalesce),
|
||||
rxRing: uint32(*rxRing),
|
||||
txRing: uint32(*txRing),
|
||||
setRings: *setRings,
|
||||
}
|
||||
|
||||
if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs,
|
||||
*duration, *interval, *drain, *txN, *rxN, *batch, *sndbuf, *rcvbuf, *verify, *duplex); err != nil {
|
||||
*duration, *interval, *drain, *txN, *rxN, *batch, *sndbuf, *rcvbuf, *verify, *duplex, tcfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
@@ -341,7 +357,7 @@ func main() {
|
||||
|
||||
func run(aName, bName, sizesArg, patArg, fanout, txCPUsArg, rxCPUsArg string,
|
||||
duration, interval, drain time.Duration, txN, rxN, batch, sndbuf, rcvbuf int,
|
||||
verify, duplex bool) error {
|
||||
verify, duplex bool, tcfg tuneConfig) error {
|
||||
|
||||
if aName == "" || bName == "" {
|
||||
return fmt.Errorf("both -a and -b are required")
|
||||
@@ -378,6 +394,19 @@ func run(aName, bName, sizesArg, patArg, fanout, txCPUsArg, rxCPUsArg string,
|
||||
}
|
||||
}
|
||||
|
||||
ifnames := []string{a.name, b.name}
|
||||
fmt.Println("host settings:")
|
||||
var fatal []string
|
||||
for _, r := range applyTuning(tcfg, ifnames) {
|
||||
fmt.Println(r)
|
||||
if r.fatal {
|
||||
fatal = append(fatal, r.item)
|
||||
}
|
||||
}
|
||||
if len(fatal) > 0 {
|
||||
return fmt.Errorf("cannot test with %s in this state", strings.Join(fatal, ", "))
|
||||
}
|
||||
|
||||
cfg := config{
|
||||
txWorkers: txN,
|
||||
rxWorkers: rxN,
|
||||
@@ -464,6 +493,9 @@ loop:
|
||||
case now := <-tick.C:
|
||||
secs := now.Sub(last).Seconds()
|
||||
last = now
|
||||
for _, w := range verifyTuning(tcfg, ifnames) {
|
||||
fmt.Println(w)
|
||||
}
|
||||
for _, d := range dirs {
|
||||
fmt.Println(d.reportInterval(secs))
|
||||
if s := d.reportNIC(); s != "" {
|
||||
|
||||
@@ -0,0 +1,369 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
type ethtoolIfreq struct {
|
||||
name [unix.IFNAMSIZ]byte
|
||||
data unsafe.Pointer
|
||||
_ [16]byte
|
||||
}
|
||||
|
||||
type flagsIfreq struct {
|
||||
name [unix.IFNAMSIZ]byte
|
||||
flags uint16
|
||||
_ [22]byte
|
||||
}
|
||||
|
||||
type ethtoolRingparam struct {
|
||||
cmd uint32
|
||||
rxMaxPending uint32
|
||||
rxMiniMaxPending uint32
|
||||
rxJumboMaxPending uint32
|
||||
txMaxPending uint32
|
||||
rxPending uint32
|
||||
rxMiniPending uint32
|
||||
rxJumboPending uint32
|
||||
txPending uint32
|
||||
}
|
||||
|
||||
type ethtoolCoalesce struct {
|
||||
cmd uint32
|
||||
rxCoalesceUsecs uint32
|
||||
rxMaxCoalescedFrames uint32
|
||||
rxCoalesceUsecsIrq uint32
|
||||
rxMaxCoalescedFramesIrq uint32
|
||||
txCoalesceUsecs uint32
|
||||
txMaxCoalescedFrames uint32
|
||||
txCoalesceUsecsIrq uint32
|
||||
txMaxCoalescedFramesIrq uint32
|
||||
statsBlockCoalesceUsecs uint32
|
||||
useAdaptiveRxCoalesce uint32
|
||||
useAdaptiveTxCoalesce uint32
|
||||
pktRateLow uint32
|
||||
rxCoalesceUsecsLow uint32
|
||||
rxMaxCoalescedFramesLow uint32
|
||||
txCoalesceUsecsLow uint32
|
||||
txMaxCoalescedFramesLow uint32
|
||||
pktRateHigh uint32
|
||||
rxCoalesceUsecsHigh uint32
|
||||
rxMaxCoalescedFramesHigh uint32
|
||||
txCoalesceUsecsHigh uint32
|
||||
txMaxCoalescedFramesHigh uint32
|
||||
rateSampleInterval uint32
|
||||
}
|
||||
|
||||
type tuneConfig struct {
|
||||
enabled bool
|
||||
governor string
|
||||
rxUsecs uint32
|
||||
txUsecs uint32
|
||||
rxRing uint32
|
||||
txRing uint32
|
||||
setRings bool
|
||||
}
|
||||
|
||||
type tuneResult struct {
|
||||
item string
|
||||
state string
|
||||
fixed bool
|
||||
fatal bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (r tuneResult) String() string {
|
||||
switch {
|
||||
case r.err != nil:
|
||||
return fmt.Sprintf(" %-34s FAIL %s (%v)", r.item, r.state, r.err)
|
||||
case r.fixed:
|
||||
return fmt.Sprintf(" %-34s FIXED %s", r.item, r.state)
|
||||
default:
|
||||
return fmt.Sprintf(" %-34s ok %s", r.item, r.state)
|
||||
}
|
||||
}
|
||||
|
||||
func ethtoolCall(fd int, ifname string, data unsafe.Pointer) error {
|
||||
var ifr ethtoolIfreq
|
||||
if len(ifname) >= unix.IFNAMSIZ {
|
||||
return fmt.Errorf("interface name %q too long", ifname)
|
||||
}
|
||||
copy(ifr.name[:], ifname)
|
||||
ifr.data = data
|
||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
||||
uintptr(unix.SIOCETHTOOL), uintptr(unsafe.Pointer(&ifr)))
|
||||
if errno != 0 {
|
||||
return errno
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getRings(fd int, ifname string) (ethtoolRingparam, error) {
|
||||
rp := ethtoolRingparam{cmd: unix.ETHTOOL_GRINGPARAM}
|
||||
err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp))
|
||||
return rp, err
|
||||
}
|
||||
|
||||
func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) {
|
||||
ec := ethtoolCoalesce{cmd: unix.ETHTOOL_GCOALESCE}
|
||||
err := ethtoolCall(fd, ifname, unsafe.Pointer(&ec))
|
||||
return ec, err
|
||||
}
|
||||
|
||||
func governorPaths() ([]string, error) {
|
||||
return filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
|
||||
}
|
||||
|
||||
func checkGovernor(want string, fix bool) tuneResult {
|
||||
res := tuneResult{item: "cpu governor"}
|
||||
paths, err := governorPaths()
|
||||
if err != nil || len(paths) == 0 {
|
||||
res.err = fmt.Errorf("no cpufreq governors found")
|
||||
return res
|
||||
}
|
||||
var wrong []string
|
||||
counts := map[string]int{}
|
||||
for _, p := range paths {
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
res.err = err
|
||||
return res
|
||||
}
|
||||
got := strings.TrimSpace(string(b))
|
||||
counts[got]++
|
||||
if got != want {
|
||||
wrong = append(wrong, p)
|
||||
}
|
||||
}
|
||||
if len(wrong) == 0 {
|
||||
res.state = fmt.Sprintf("%s on all %d cpus", want, len(paths))
|
||||
return res
|
||||
}
|
||||
var found []string
|
||||
for k, v := range counts {
|
||||
found = append(found, fmt.Sprintf("%s:%d", k, v))
|
||||
}
|
||||
if !fix {
|
||||
res.err = fmt.Errorf("want %s, found %s", want, strings.Join(found, " "))
|
||||
res.state = "not corrected"
|
||||
return res
|
||||
}
|
||||
for _, p := range wrong {
|
||||
if err := os.WriteFile(p, []byte(want), 0o644); err != nil {
|
||||
res.err = err
|
||||
res.state = fmt.Sprintf("could not set %s", p)
|
||||
return res
|
||||
}
|
||||
}
|
||||
res.fixed = true
|
||||
res.state = fmt.Sprintf("was %s, now %s on all %d cpus", strings.Join(found, " "), want, len(paths))
|
||||
return res
|
||||
}
|
||||
|
||||
func checkLinkUp(fd int, ifname string, fix bool) tuneResult {
|
||||
res := tuneResult{item: ifname + " link up"}
|
||||
var ifr flagsIfreq
|
||||
copy(ifr.name[:], ifname)
|
||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
||||
uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
||||
if errno != 0 {
|
||||
res.err = errno
|
||||
return res
|
||||
}
|
||||
if ifr.flags&unix.IFF_UP != 0 {
|
||||
res.state = "up"
|
||||
return res
|
||||
}
|
||||
if !fix {
|
||||
res.err = fmt.Errorf("interface is down")
|
||||
res.state = "not corrected"
|
||||
res.fatal = true
|
||||
return res
|
||||
}
|
||||
ifr.flags |= unix.IFF_UP
|
||||
_, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
||||
uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
||||
if errno != 0 {
|
||||
res.err = errno
|
||||
res.state = "could not set IFF_UP"
|
||||
res.fatal = true
|
||||
return res
|
||||
}
|
||||
res.fixed = true
|
||||
res.state = "was down, now up"
|
||||
return res
|
||||
}
|
||||
|
||||
func checkCarrier(ifname string, wait time.Duration) tuneResult {
|
||||
res := tuneResult{item: ifname + " carrier"}
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
v, ok := readUint("/sys/class/net/" + ifname + "/carrier")
|
||||
if ok && v == 1 {
|
||||
res.state = "present"
|
||||
return res
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
res.err = fmt.Errorf("no carrier after %s", wait)
|
||||
res.fatal = true
|
||||
return res
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tuneResult {
|
||||
res := tuneResult{item: ifname + " coalesce"}
|
||||
ec, err := getCoalesce(fd, ifname)
|
||||
if err != nil {
|
||||
res.err = err
|
||||
return res
|
||||
}
|
||||
ok := ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
|
||||
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs
|
||||
desc := func(e ethtoolCoalesce) string {
|
||||
return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d",
|
||||
e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs)
|
||||
}
|
||||
if ok {
|
||||
res.state = desc(ec)
|
||||
return res
|
||||
}
|
||||
if !fix {
|
||||
res.err = fmt.Errorf("want adaptive off rx-usecs=%d tx-usecs=%d, have %s", rxUsecs, txUsecs, desc(ec))
|
||||
res.state = "not corrected"
|
||||
return res
|
||||
}
|
||||
was := desc(ec)
|
||||
ec.cmd = unix.ETHTOOL_SCOALESCE
|
||||
ec.useAdaptiveRxCoalesce = 0
|
||||
ec.useAdaptiveTxCoalesce = 0
|
||||
ec.rxCoalesceUsecs = rxUsecs
|
||||
ec.txCoalesceUsecs = txUsecs
|
||||
if err := ethtoolCall(fd, ifname, unsafe.Pointer(&ec)); err != nil {
|
||||
res.err = err
|
||||
res.state = "could not set"
|
||||
return res
|
||||
}
|
||||
res.fixed = true
|
||||
res.state = fmt.Sprintf("was %s, now adaptive off rx-usecs=%d tx-usecs=%d", was, rxUsecs, txUsecs)
|
||||
return res
|
||||
}
|
||||
|
||||
func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneResult, bool) {
|
||||
res := tuneResult{item: ifname + " rings"}
|
||||
rp, err := getRings(fd, ifname)
|
||||
if err != nil {
|
||||
res.err = err
|
||||
return res, false
|
||||
}
|
||||
rx := min(rxWant, rp.rxMaxPending)
|
||||
tx := min(txWant, rp.txMaxPending)
|
||||
if rp.rxPending == rx && rp.txPending == tx {
|
||||
res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
||||
return res, false
|
||||
}
|
||||
if !fix {
|
||||
res.err = fmt.Errorf("want rx=%d tx=%d, have rx=%d tx=%d", rx, tx, rp.rxPending, rp.txPending)
|
||||
res.state = "not corrected"
|
||||
return res, false
|
||||
}
|
||||
was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
||||
rp.cmd = unix.ETHTOOL_SRINGPARAM
|
||||
rp.rxPending = rx
|
||||
rp.txPending = tx
|
||||
if err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp)); err != nil {
|
||||
res.err = err
|
||||
res.state = "could not set"
|
||||
return res, false
|
||||
}
|
||||
res.fixed = true
|
||||
res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link resets)", was, rx, tx)
|
||||
return res, true
|
||||
}
|
||||
|
||||
func checkNoAddrs(ifname string) tuneResult {
|
||||
res := tuneResult{item: ifname + " unmanaged"}
|
||||
ifi, err := net.InterfaceByName(ifname)
|
||||
if err != nil {
|
||||
res.err = err
|
||||
return res
|
||||
}
|
||||
addrs, err := ifi.Addrs()
|
||||
if err != nil {
|
||||
res.err = err
|
||||
return res
|
||||
}
|
||||
var routable []string
|
||||
for _, a := range addrs {
|
||||
ipn, ok := a.(*net.IPNet)
|
||||
if !ok || ipn.IP.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
routable = append(routable, a.String())
|
||||
}
|
||||
if len(routable) == 0 {
|
||||
res.state = "no routable addresses"
|
||||
return res
|
||||
}
|
||||
res.err = fmt.Errorf("has %s, something is still configuring this interface", strings.Join(routable, ","))
|
||||
res.state = "not corrected"
|
||||
return res
|
||||
}
|
||||
|
||||
func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
||||
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
||||
if err != nil {
|
||||
return []tuneResult{{item: "ioctl socket", err: err}}
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
|
||||
var out []tuneResult
|
||||
out = append(out, checkGovernor(cfg.governor, cfg.enabled))
|
||||
|
||||
for _, ifname := range ifnames {
|
||||
out = append(out, checkLinkUp(fd, ifname, cfg.enabled))
|
||||
out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled))
|
||||
carrierWait := 3 * time.Second
|
||||
if cfg.setRings {
|
||||
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, cfg.enabled)
|
||||
out = append(out, r)
|
||||
if reset {
|
||||
carrierWait = 10 * time.Second
|
||||
}
|
||||
}
|
||||
out = append(out, checkCarrier(ifname, carrierWait))
|
||||
out = append(out, checkNoAddrs(ifname))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func verifyTuning(cfg tuneConfig, ifnames []string) []string {
|
||||
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("drift check failed: %v", err)}
|
||||
}
|
||||
defer unix.Close(fd)
|
||||
|
||||
var warnings []string
|
||||
if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil {
|
||||
warnings = append(warnings, "host config drifted:"+r.String())
|
||||
}
|
||||
for _, ifname := range ifnames {
|
||||
if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil {
|
||||
warnings = append(warnings, "host config drifted:"+r.String())
|
||||
}
|
||||
if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil {
|
||||
warnings = append(warnings, "host config drifted (not corrected, would reset link):"+r.String())
|
||||
}
|
||||
}
|
||||
return warnings
|
||||
}
|
||||
Reference in New Issue
Block a user