370 lines
9.5 KiB
Go
370 lines
9.5 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
"unsafe"
|
|
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
type ethtoolIfreq struct {
|
|
name [unix.IFNAMSIZ]byte
|
|
data unsafe.Pointer
|
|
_ [16]byte
|
|
}
|
|
|
|
type flagsIfreq struct {
|
|
name [unix.IFNAMSIZ]byte
|
|
flags uint16
|
|
_ [22]byte
|
|
}
|
|
|
|
type ethtoolRingparam struct {
|
|
cmd uint32
|
|
rxMaxPending uint32
|
|
rxMiniMaxPending uint32
|
|
rxJumboMaxPending uint32
|
|
txMaxPending uint32
|
|
rxPending uint32
|
|
rxMiniPending uint32
|
|
rxJumboPending uint32
|
|
txPending uint32
|
|
}
|
|
|
|
type ethtoolCoalesce struct {
|
|
cmd uint32
|
|
rxCoalesceUsecs uint32
|
|
rxMaxCoalescedFrames uint32
|
|
rxCoalesceUsecsIrq uint32
|
|
rxMaxCoalescedFramesIrq uint32
|
|
txCoalesceUsecs uint32
|
|
txMaxCoalescedFrames uint32
|
|
txCoalesceUsecsIrq uint32
|
|
txMaxCoalescedFramesIrq uint32
|
|
statsBlockCoalesceUsecs uint32
|
|
useAdaptiveRxCoalesce uint32
|
|
useAdaptiveTxCoalesce uint32
|
|
pktRateLow uint32
|
|
rxCoalesceUsecsLow uint32
|
|
rxMaxCoalescedFramesLow uint32
|
|
txCoalesceUsecsLow uint32
|
|
txMaxCoalescedFramesLow uint32
|
|
pktRateHigh uint32
|
|
rxCoalesceUsecsHigh uint32
|
|
rxMaxCoalescedFramesHigh uint32
|
|
txCoalesceUsecsHigh uint32
|
|
txMaxCoalescedFramesHigh uint32
|
|
rateSampleInterval uint32
|
|
}
|
|
|
|
type tuneConfig struct {
|
|
enabled bool
|
|
governor string
|
|
rxUsecs uint32
|
|
txUsecs uint32
|
|
rxRing uint32
|
|
txRing uint32
|
|
setRings bool
|
|
}
|
|
|
|
type tuneResult struct {
|
|
item string
|
|
state string
|
|
fixed bool
|
|
fatal bool
|
|
err error
|
|
}
|
|
|
|
func (r tuneResult) String() string {
|
|
switch {
|
|
case r.err != nil:
|
|
return fmt.Sprintf(" %-34s FAIL %s (%v)", r.item, r.state, r.err)
|
|
case r.fixed:
|
|
return fmt.Sprintf(" %-34s FIXED %s", r.item, r.state)
|
|
default:
|
|
return fmt.Sprintf(" %-34s ok %s", r.item, r.state)
|
|
}
|
|
}
|
|
|
|
func ethtoolCall(fd int, ifname string, data unsafe.Pointer) error {
|
|
var ifr ethtoolIfreq
|
|
if len(ifname) >= unix.IFNAMSIZ {
|
|
return fmt.Errorf("interface name %q too long", ifname)
|
|
}
|
|
copy(ifr.name[:], ifname)
|
|
ifr.data = data
|
|
_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
|
uintptr(unix.SIOCETHTOOL), uintptr(unsafe.Pointer(&ifr)))
|
|
if errno != 0 {
|
|
return errno
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getRings(fd int, ifname string) (ethtoolRingparam, error) {
|
|
rp := ethtoolRingparam{cmd: unix.ETHTOOL_GRINGPARAM}
|
|
err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp))
|
|
return rp, err
|
|
}
|
|
|
|
func getCoalesce(fd int, ifname string) (ethtoolCoalesce, error) {
|
|
ec := ethtoolCoalesce{cmd: unix.ETHTOOL_GCOALESCE}
|
|
err := ethtoolCall(fd, ifname, unsafe.Pointer(&ec))
|
|
return ec, err
|
|
}
|
|
|
|
func governorPaths() ([]string, error) {
|
|
return filepath.Glob("/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor")
|
|
}
|
|
|
|
func checkGovernor(want string, fix bool) tuneResult {
|
|
res := tuneResult{item: "cpu governor"}
|
|
paths, err := governorPaths()
|
|
if err != nil || len(paths) == 0 {
|
|
res.err = fmt.Errorf("no cpufreq governors found")
|
|
return res
|
|
}
|
|
var wrong []string
|
|
counts := map[string]int{}
|
|
for _, p := range paths {
|
|
b, err := os.ReadFile(p)
|
|
if err != nil {
|
|
res.err = err
|
|
return res
|
|
}
|
|
got := strings.TrimSpace(string(b))
|
|
counts[got]++
|
|
if got != want {
|
|
wrong = append(wrong, p)
|
|
}
|
|
}
|
|
if len(wrong) == 0 {
|
|
res.state = fmt.Sprintf("%s on all %d cpus", want, len(paths))
|
|
return res
|
|
}
|
|
var found []string
|
|
for k, v := range counts {
|
|
found = append(found, fmt.Sprintf("%s:%d", k, v))
|
|
}
|
|
if !fix {
|
|
res.err = fmt.Errorf("want %s, found %s", want, strings.Join(found, " "))
|
|
res.state = "not corrected"
|
|
return res
|
|
}
|
|
for _, p := range wrong {
|
|
if err := os.WriteFile(p, []byte(want), 0o644); err != nil {
|
|
res.err = err
|
|
res.state = fmt.Sprintf("could not set %s", p)
|
|
return res
|
|
}
|
|
}
|
|
res.fixed = true
|
|
res.state = fmt.Sprintf("was %s, now %s on all %d cpus", strings.Join(found, " "), want, len(paths))
|
|
return res
|
|
}
|
|
|
|
func checkLinkUp(fd int, ifname string, fix bool) tuneResult {
|
|
res := tuneResult{item: ifname + " link up"}
|
|
var ifr flagsIfreq
|
|
copy(ifr.name[:], ifname)
|
|
_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
|
uintptr(unix.SIOCGIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
|
if errno != 0 {
|
|
res.err = errno
|
|
return res
|
|
}
|
|
if ifr.flags&unix.IFF_UP != 0 {
|
|
res.state = "up"
|
|
return res
|
|
}
|
|
if !fix {
|
|
res.err = fmt.Errorf("interface is down")
|
|
res.state = "not corrected"
|
|
res.fatal = true
|
|
return res
|
|
}
|
|
ifr.flags |= unix.IFF_UP
|
|
_, _, errno = unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
|
|
uintptr(unix.SIOCSIFFLAGS), uintptr(unsafe.Pointer(&ifr)))
|
|
if errno != 0 {
|
|
res.err = errno
|
|
res.state = "could not set IFF_UP"
|
|
res.fatal = true
|
|
return res
|
|
}
|
|
res.fixed = true
|
|
res.state = "was down, now up"
|
|
return res
|
|
}
|
|
|
|
func checkCarrier(ifname string, wait time.Duration) tuneResult {
|
|
res := tuneResult{item: ifname + " carrier"}
|
|
deadline := time.Now().Add(wait)
|
|
for {
|
|
v, ok := readUint("/sys/class/net/" + ifname + "/carrier")
|
|
if ok && v == 1 {
|
|
res.state = "present"
|
|
return res
|
|
}
|
|
if time.Now().After(deadline) {
|
|
res.err = fmt.Errorf("no carrier after %s", wait)
|
|
res.fatal = true
|
|
return res
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func checkCoalesce(fd int, ifname string, rxUsecs, txUsecs uint32, fix bool) tuneResult {
|
|
res := tuneResult{item: ifname + " coalesce"}
|
|
ec, err := getCoalesce(fd, ifname)
|
|
if err != nil {
|
|
res.err = err
|
|
return res
|
|
}
|
|
ok := ec.useAdaptiveRxCoalesce == 0 && ec.useAdaptiveTxCoalesce == 0 &&
|
|
ec.rxCoalesceUsecs == rxUsecs && ec.txCoalesceUsecs == txUsecs
|
|
desc := func(e ethtoolCoalesce) string {
|
|
return fmt.Sprintf("adaptive rx=%d tx=%d rx-usecs=%d tx-usecs=%d",
|
|
e.useAdaptiveRxCoalesce, e.useAdaptiveTxCoalesce, e.rxCoalesceUsecs, e.txCoalesceUsecs)
|
|
}
|
|
if ok {
|
|
res.state = desc(ec)
|
|
return res
|
|
}
|
|
if !fix {
|
|
res.err = fmt.Errorf("want adaptive off rx-usecs=%d tx-usecs=%d, have %s", rxUsecs, txUsecs, desc(ec))
|
|
res.state = "not corrected"
|
|
return res
|
|
}
|
|
was := desc(ec)
|
|
ec.cmd = unix.ETHTOOL_SCOALESCE
|
|
ec.useAdaptiveRxCoalesce = 0
|
|
ec.useAdaptiveTxCoalesce = 0
|
|
ec.rxCoalesceUsecs = rxUsecs
|
|
ec.txCoalesceUsecs = txUsecs
|
|
if err := ethtoolCall(fd, ifname, unsafe.Pointer(&ec)); err != nil {
|
|
res.err = err
|
|
res.state = "could not set"
|
|
return res
|
|
}
|
|
res.fixed = true
|
|
res.state = fmt.Sprintf("was %s, now adaptive off rx-usecs=%d tx-usecs=%d", was, rxUsecs, txUsecs)
|
|
return res
|
|
}
|
|
|
|
func checkRings(fd int, ifname string, rxWant, txWant uint32, fix bool) (tuneResult, bool) {
|
|
res := tuneResult{item: ifname + " rings"}
|
|
rp, err := getRings(fd, ifname)
|
|
if err != nil {
|
|
res.err = err
|
|
return res, false
|
|
}
|
|
rx := min(rxWant, rp.rxMaxPending)
|
|
tx := min(txWant, rp.txMaxPending)
|
|
if rp.rxPending == rx && rp.txPending == tx {
|
|
res.state = fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
|
return res, false
|
|
}
|
|
if !fix {
|
|
res.err = fmt.Errorf("want rx=%d tx=%d, have rx=%d tx=%d", rx, tx, rp.rxPending, rp.txPending)
|
|
res.state = "not corrected"
|
|
return res, false
|
|
}
|
|
was := fmt.Sprintf("rx=%d tx=%d", rp.rxPending, rp.txPending)
|
|
rp.cmd = unix.ETHTOOL_SRINGPARAM
|
|
rp.rxPending = rx
|
|
rp.txPending = tx
|
|
if err := ethtoolCall(fd, ifname, unsafe.Pointer(&rp)); err != nil {
|
|
res.err = err
|
|
res.state = "could not set"
|
|
return res, false
|
|
}
|
|
res.fixed = true
|
|
res.state = fmt.Sprintf("was %s, now rx=%d tx=%d (link resets)", was, rx, tx)
|
|
return res, true
|
|
}
|
|
|
|
func checkNoAddrs(ifname string) tuneResult {
|
|
res := tuneResult{item: ifname + " unmanaged"}
|
|
ifi, err := net.InterfaceByName(ifname)
|
|
if err != nil {
|
|
res.err = err
|
|
return res
|
|
}
|
|
addrs, err := ifi.Addrs()
|
|
if err != nil {
|
|
res.err = err
|
|
return res
|
|
}
|
|
var routable []string
|
|
for _, a := range addrs {
|
|
ipn, ok := a.(*net.IPNet)
|
|
if !ok || ipn.IP.IsLinkLocalUnicast() {
|
|
continue
|
|
}
|
|
routable = append(routable, a.String())
|
|
}
|
|
if len(routable) == 0 {
|
|
res.state = "no routable addresses"
|
|
return res
|
|
}
|
|
res.err = fmt.Errorf("has %s, something is still configuring this interface", strings.Join(routable, ","))
|
|
res.state = "not corrected"
|
|
return res
|
|
}
|
|
|
|
func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult {
|
|
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
|
if err != nil {
|
|
return []tuneResult{{item: "ioctl socket", err: err}}
|
|
}
|
|
defer unix.Close(fd)
|
|
|
|
var out []tuneResult
|
|
out = append(out, checkGovernor(cfg.governor, cfg.enabled))
|
|
|
|
for _, ifname := range ifnames {
|
|
out = append(out, checkLinkUp(fd, ifname, cfg.enabled))
|
|
out = append(out, checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled))
|
|
carrierWait := 3 * time.Second
|
|
if cfg.setRings {
|
|
r, reset := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, cfg.enabled)
|
|
out = append(out, r)
|
|
if reset {
|
|
carrierWait = 10 * time.Second
|
|
}
|
|
}
|
|
out = append(out, checkCarrier(ifname, carrierWait))
|
|
out = append(out, checkNoAddrs(ifname))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func verifyTuning(cfg tuneConfig, ifnames []string) []string {
|
|
fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
|
|
if err != nil {
|
|
return []string{fmt.Sprintf("drift check failed: %v", err)}
|
|
}
|
|
defer unix.Close(fd)
|
|
|
|
var warnings []string
|
|
if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil {
|
|
warnings = append(warnings, "host config drifted:"+r.String())
|
|
}
|
|
for _, ifname := range ifnames {
|
|
if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil {
|
|
warnings = append(warnings, "host config drifted:"+r.String())
|
|
}
|
|
if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil {
|
|
warnings = append(warnings, "host config drifted (not corrected, would reset link):"+r.String())
|
|
}
|
|
}
|
|
return warnings
|
|
}
|