Always correct host settings instead of gating fixes behind flags
This commit is contained in:
@@ -369,12 +369,10 @@ func main() {
|
|||||||
rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket")
|
rcvbuf = flag.Int("rcvbuf", 64<<20, "SO_RCVBUFFORCE per rx socket")
|
||||||
txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to")
|
txCPUs = flag.String("txcpus", "", "comma-separated CPUs to pin tx workers to")
|
||||||
rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to")
|
rxCPUs = flag.String("rxcpus", "", "comma-separated CPUs to pin rx workers to")
|
||||||
tune = flag.Bool("tune", true, "check host settings at startup and correct them; without this they are only reported")
|
|
||||||
governor = flag.String("governor", "performance", "required cpufreq governor")
|
governor = flag.String("governor", "performance", "required cpufreq governor")
|
||||||
coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off")
|
coalesce = flag.Uint("coalesce-usecs", 25, "required fixed rx/tx coalesce usecs, with adaptive coalescing off")
|
||||||
rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum")
|
rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum")
|
||||||
txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum")
|
txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum")
|
||||||
setRings = flag.Bool("set-rings", true, "allow ring resizing at startup, which resets the link")
|
|
||||||
color = flag.String("color", "auto", "colored output: auto, always, never")
|
color = flag.String("color", "auto", "colored output: auto, always, never")
|
||||||
)
|
)
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
@@ -385,13 +383,11 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
tcfg := tuneConfig{
|
tcfg := tuneConfig{
|
||||||
enabled: *tune,
|
|
||||||
governor: *governor,
|
governor: *governor,
|
||||||
rxUsecs: uint32(*coalesce),
|
rxUsecs: uint32(*coalesce),
|
||||||
txUsecs: uint32(*coalesce),
|
txUsecs: uint32(*coalesce),
|
||||||
rxRing: uint32(*rxRing),
|
rxRing: uint32(*rxRing),
|
||||||
txRing: uint32(*txRing),
|
txRing: uint32(*txRing),
|
||||||
setRings: *setRings,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs,
|
if err := run(*aName, *bName, *sizesArg, *patArg, *fanout, *txCPUs, *rxCPUs,
|
||||||
|
|||||||
@@ -63,13 +63,11 @@ type ethtoolCoalesce struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type tuneConfig struct {
|
type tuneConfig struct {
|
||||||
enabled bool
|
|
||||||
governor string
|
governor string
|
||||||
rxUsecs uint32
|
rxUsecs uint32
|
||||||
txUsecs uint32
|
txUsecs uint32
|
||||||
rxRing uint32
|
rxRing uint32
|
||||||
txRing uint32
|
txRing uint32
|
||||||
setRings bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type tuneResult struct {
|
type tuneResult struct {
|
||||||
@@ -128,13 +126,9 @@ func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) {
|
|||||||
return ec, err
|
return ec, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func governorPaths() ([]string, error) {
|
func checkGovernor(want string) tuneResult {
|
||||||
return filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkGovernor(want string, fix bool) tuneResult {
|
|
||||||
res := tuneResult{item: "cpu governor"}
|
res := tuneResult{item: "cpu governor"}
|
||||||
paths, err := governorPaths()
|
paths, err := filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
|
||||||
if err != nil || len(paths) == 0 {
|
if err != nil || len(paths) == 0 {
|
||||||
res.err = fmt.Errorf("no cpufreq governors found")
|
res.err = fmt.Errorf("no cpufreq governors found")
|
||||||
return res
|
return res
|
||||||
@@ -161,11 +155,6 @@ func checkGovernor(want string, fix bool) tuneResult {
|
|||||||
for k, v := range counts {
|
for k, v := range counts {
|
||||||
found = append(found, fmt.Sprintf("%s:%d", k, v))
|
found = append(found, fmt.Sprintf("%s:%d", k, v))
|
||||||
}
|
}
|
||||||
if !fix {
|
|
||||||
res.err = fmt.Errorf("want %s, found %s", want, strings.Join(found, " "))
|
|
||||||
res.state = "not corrected"
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
for _, p := range wrong {
|
for _, p := range wrong {
|
||||||
if err := os.WriteFile(p, []byte(want), 0o644); err != nil {
|
if err := os.WriteFile(p, []byte(want), 0o644); err != nil {
|
||||||
res.err = err
|
res.err = err
|
||||||
@@ -178,7 +167,7 @@ func checkGovernor(want string, fix bool) tuneResult {
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkLinkUp(fd int, ifname string, fix bool) tuneResult {
|
func checkLinkUp(fd int, ifname string) tuneResult {
|
||||||
res := tuneResult{item: ifname + " link up"}
|
res := tuneResult{item: ifname + " link up"}
|
||||||
var ifr flagsIfreq
|
var ifr flagsIfreq
|
||||||
copy(ifr.name[:], ifname)
|
copy(ifr.name[:], ifname)
|
||||||
@@ -186,18 +175,13 @@ func checkLinkUp(fd int, ifname string, fix bool) tuneResult {
|
|||||||
uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
||||||
if errno != 0 {
|
if errno != 0 {
|
||||||
res.err = errno
|
res.err = errno
|
||||||
|
res.fatal = true
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
if ifr.flags&unix.IFF_UP != 0 {
|
if ifr.flags&unix.IFF_UP != 0 {
|
||||||
res.state = "up"
|
res.state = "up"
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
if !fix {
|
|
||||||
res.err = fmt.Errorf("interface is down")
|
|
||||||
res.state = "not corrected"
|
|
||||||
res.fatal = true
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
ifr.flags |= unix.IFF_UP
|
ifr.flags |= unix.IFF_UP
|
||||||
_, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
_, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
||||||
uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
||||||
@@ -230,28 +214,22 @@ func checkCarrier(ifname string, wait time.Duration) tuneResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tuneResult {
|
func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32) tuneResult {
|
||||||
res := tuneResult{item: ifname + " coalesce"}
|
res := tuneResult{item: ifname + " coalesce"}
|
||||||
ec, err := getCoalesce(fd, ifname)
|
ec, err := getCoalesce(fd, ifname)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
res.err = err
|
res.err = err
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
ok := ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
|
|
||||||
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs
|
|
||||||
desc := func(e ethtoolCoalesce) string {
|
desc := func(e ethtoolCoalesce) string {
|
||||||
return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d",
|
return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d",
|
||||||
e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs)
|
e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs)
|
||||||
}
|
}
|
||||||
if ok {
|
if ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
|
||||||
|
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs {
|
||||||
res.state = desc(ec)
|
res.state = desc(ec)
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
if !fix {
|
|
||||||
res.err = fmt.Errorf("want adaptive off rx-usecs=%d tx-usecs=%d, have %s", rxUsecs, txUsecs, desc(ec))
|
|
||||||
res.state = "not corrected"
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
was := desc(ec)
|
was := desc(ec)
|
||||||
ec.cmd = unix.ETHTOOL_SCOALESCE
|
ec.cmd = unix.ETHTOOL_SCOALESCE
|
||||||
ec.useAdaptiveRxCoalesce = 0
|
ec.useAdaptiveRxCoalesce = 0
|
||||||
@@ -268,7 +246,7 @@ func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tun
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneResult, bool) {
|
func checkRings(fd int, ifname string, rxWant, txWant uint32) (tuneResult, bool) {
|
||||||
res := tuneResult{item: ifname + " rings"}
|
res := tuneResult{item: ifname + " rings"}
|
||||||
rp, err := getRings(fd, ifname)
|
rp, err := getRings(fd, ifname)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -281,11 +259,6 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes
|
|||||||
res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
||||||
return res, false
|
return res, false
|
||||||
}
|
}
|
||||||
if !fix {
|
|
||||||
res.err = fmt.Errorf("want rx=%d tx=%d, have rx=%d tx=%d", rx, tx, rp.rxPending, rp.txPending)
|
|
||||||
res.state = "not corrected"
|
|
||||||
return res, false
|
|
||||||
}
|
|
||||||
was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
||||||
rp.cmd = unix.ETHTOOL_SRINGPARAM
|
rp.cmd = unix.ETHTOOL_SRINGPARAM
|
||||||
rp.rxPending = rx
|
rp.rxPending = rx
|
||||||
@@ -296,7 +269,7 @@ func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneRes
|
|||||||
return res, false
|
return res, false
|
||||||
}
|
}
|
||||||
res.fixed = true
|
res.fixed = true
|
||||||
res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link resets)", was, rx, tx)
|
res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link reset)", was, rx, tx)
|
||||||
return res, true
|
return res, true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -324,57 +297,60 @@ func checkNoAddrs(ifname string) tuneResult {
|
|||||||
res.state = "no routable addresses"
|
res.state = "no routable addresses"
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
res.err = fmt.Errorf("has %s, something is still configuring this interface", strings.Join(routable, ","))
|
res.err = fmt.Errorf("has %s; NetworkManager or DHCP is configuring this interface",
|
||||||
res.state = "not corrected"
|
strings.Join(routable, ","))
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
func withIoctlSocket(fn func(fd int) []tuneResult) []tuneResult {
|
||||||
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []tuneResult{{item: "ioctl socket", err: err}}
|
return []tuneResult{{item: "ioctl socket", err: err, fatal: true}}
|
||||||
}
|
}
|
||||||
defer unix.Close(fd)
|
defer unix.Close(fd)
|
||||||
|
return fn(fd)
|
||||||
|
}
|
||||||
|
|
||||||
var out []tuneResult
|
func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
||||||
out = append(out, checkGovernor(cfg.governor, cfg.enabled))
|
return withIoctlSocket(func(fd int) []tuneResult {
|
||||||
|
out := []tuneResult{checkGovernor(cfg.governor)}
|
||||||
for _, ifname := range ifnames {
|
for _, ifname := range ifnames {
|
||||||
out = append(out, checkLinkUp(fd, ifname, cfg.enabled))
|
out = append(out, checkLinkUp(fd, ifname))
|
||||||
out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled))
|
out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs))
|
||||||
|
|
||||||
carrierWait := 3 * time.Second
|
carrierWait := 3 * time.Second
|
||||||
if cfg.setRings {
|
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing)
|
||||||
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, cfg.enabled)
|
|
||||||
out = append(out, r)
|
out = append(out, r)
|
||||||
if reset {
|
if reset {
|
||||||
carrierWait = 10 * time.Second
|
carrierWait = 10 * time.Second
|
||||||
}
|
}
|
||||||
}
|
|
||||||
out = append(out, checkCarrier(ifname, carrierWait))
|
out = append(out, checkCarrier(ifname, carrierWait))
|
||||||
out = append(out, checkNoAddrs(ifname))
|
out = append(out, checkNoAddrs(ifname))
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
||||||
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
return withIoctlSocket(func(fd int) []tuneResult {
|
||||||
if err != nil {
|
|
||||||
return []tuneResult{{item: "drift check", err: err}}
|
|
||||||
}
|
|
||||||
defer unix.Close(fd)
|
|
||||||
|
|
||||||
var drifted []tuneResult
|
var drifted []tuneResult
|
||||||
if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil {
|
if r := checkGovernor(cfg.governor); r.fixed || r.err != nil {
|
||||||
drifted = append(drifted, r)
|
drifted = append(drifted, r)
|
||||||
}
|
}
|
||||||
for _, ifname := range ifnames {
|
for _, ifname := range ifnames {
|
||||||
if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil {
|
if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs); r.fixed || r.err != nil {
|
||||||
drifted = append(drifted, r)
|
drifted = append(drifted, r)
|
||||||
}
|
}
|
||||||
if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil {
|
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing)
|
||||||
r.state = "left alone, resizing would reset the link"
|
if r.fixed || r.err != nil {
|
||||||
drifted = append(drifted, r)
|
drifted = append(drifted, r)
|
||||||
}
|
}
|
||||||
|
if reset {
|
||||||
|
if c := checkCarrier(ifname, 10*time.Second); c.err != nil {
|
||||||
|
drifted = append(drifted, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return drifted
|
return drifted
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user