Give each stream its own ethertype steered to its own rx queue

system.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/binary"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -11,6 +12,179 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+const (
+	ethtoolGRXRINGS    = 0x2d
+	ethtoolGRXCLSRLCNT = 0x2e
+	ethtoolGRXCLSRULE  = 0x2f
+	ethtoolGRXCLSRLALL = 0x30
+	ethtoolSRXCLSRLDEL = 0x31
+	ethtoolSRXCLSRLINS = 0x32
+	etherFlow          = 0x12
+)
+
+type ethtoolFlowExt struct {
+	padding   [2]byte
+	hDest     [6]byte
+	vlanEtype uint16
+	vlanTci   uint16
+	data      [2]uint32
+}
+
+type ethtoolRxFlowSpec struct {
+	flowType   uint32
+	hU         [52]byte
+	hExt       ethtoolFlowExt
+	mU         [52]byte
+	mExt       ethtoolFlowExt
+	_          [4]byte
+	ringCookie uint64
+	location   uint32
+	_          [4]byte
+}
+
+type ethtoolRxnfc struct {
+	cmd      uint32
+	flowType uint32
+	data     uint64
+	fs       ethtoolRxFlowSpec
+	ruleCnt  uint32
+	_        [4]byte
+}
+
+func rxRings(fd int, ifname string) (uint64, error) {
+	nfc := ethtoolRxnfc{cmd: ethtoolGRXRINGS}
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&nfc)); err != nil {
+		return 0, err
+	}
+	return nfc.data, nil
+}
+
+// Returns the installed rule locations and the total filter capacity. ice
+// only honours filters near the top of that range, which is why ethtool's own
+// rule manager allocates downwards from the end.
+func allRuleLocations(fd int, ifname string) ([]uint32, uint32, error) {
+	cnt := ethtoolRxnfc{cmd: ethtoolGRXCLSRLCNT}
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&cnt)); err != nil {
+		return nil, 0, err
+	}
+	capacity := uint32(cnt.data)
+	if cnt.ruleCnt == 0 {
+		return nil, capacity, nil
+	}
+
+	// rule_locs follows rule_cnt directly, before the struct's tail padding,
+	// and that is where the kernel copies it to.
+	locOff := unsafe.Offsetof(ethtoolRxnfc{}.ruleCnt) + 4
+	buf := make([]byte, int(unsafe.Sizeof(ethtoolRxnfc{}))+4*int(cnt.ruleCnt))
+	all := (*ethtoolRxnfc)(unsafe.Pointer(&buf[0]))
+	all.cmd = ethtoolGRXCLSRLALL
+	all.ruleCnt = cnt.ruleCnt
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&buf[0])); err != nil {
+		return nil, 0, err
+	}
+
+	raw := buf[locOff:]
+	locs := make([]uint32, 0, all.ruleCnt)
+	for i := 0; i < int(all.ruleCnt); i++ {
+		locs = append(locs, binary.LittleEndian.Uint32(raw[i*4:]))
+	}
+	return locs, capacity, nil
+}
+
+func ruleIsEther(fd int, ifname string, loc uint32) bool {
+	get := ethtoolRxnfc{cmd: ethtoolGRXCLSRULE}
+	get.fs.location = loc
+	if err := ethtoolCall(fd, ifname, unsafe.Pointer(&get)); err != nil {
+		return false
+	}
+	return get.fs.flowType&0xff == etherFlow
+}
+
+func deleteRule(fd int, ifname string, loc uint32) error {
+	nfc := ethtoolRxnfc{cmd: ethtoolSRXCLSRLDEL}
+	nfc.fs.location = loc
+	return ethtoolCall(fd, ifname, unsafe.Pointer(&nfc))
+}
+
+// ice rejects RX_CLS_LOC_ANY, so the caller must supply a free location.
+func insertEtherRule(fd int, ifname string, ethType uint16, queue uint64, loc uint32) error {
+	nfc := ethtoolRxnfc{cmd: ethtoolSRXCLSRLINS}
+	nfc.fs.flowType = etherFlow
+	binary.BigEndian.PutUint16(nfc.fs.hU[12:14], ethType)
+	// A set mask bit means that bit must match, so mask only the ethertype and
+	// leave both MAC masks zero. Note ethtool -n prints the complement of this.
+	binary.BigEndian.PutUint16(nfc.fs.mU[12:14], 0xffff)
+	nfc.fs.ringCookie = queue
+	nfc.fs.location = loc
+	return ethtoolCall(fd, ifname, unsafe.Pointer(&nfc))
+}
+
+func checkFlowRules(fd int, ifname string, ethertypes []uint16) checkResult {
+	res := checkResult{item: ifname + " flow rules"}
+	rings, err := rxRings(fd, ifname)
+	if err != nil {
+		res.err = err
+		res.fatal = true
+		return res
+	}
+	if uint64(len(ethertypes)) > rings {
+		res.err = fmt.Errorf("%d streams needs %d rx rings, only %d available",
+			len(ethertypes), len(ethertypes), rings)
+		res.fatal = true
+		return res
+	}
+
+	locs, capacity, err := allRuleLocations(fd, ifname)
+	if err != nil {
+		res.err = err
+		res.fatal = true
+		return res
+	}
+	if capacity < uint32(len(ethertypes)) {
+		res.err = fmt.Errorf("filter capacity %d is below %d streams", capacity, len(ethertypes))
+		res.fatal = true
+		return res
+	}
+	var stale []uint32
+	taken := make(map[uint32]bool, len(locs))
+	for _, loc := range locs {
+		if ruleIsEther(fd, ifname, loc) {
+			stale = append(stale, loc)
+			continue
+		}
+		taken[loc] = true
+	}
+	for _, loc := range stale {
+		if err := deleteRule(fd, ifname, loc); err != nil {
+			res.err = fmt.Errorf("deleting stale rule %d: %w", loc, err)
+			res.fatal = true
+			return res
+		}
+	}
+
+	next := capacity - 1
+	for i, et := range ethertypes {
+		for taken[next] && next > 0 {
+			next--
+		}
+		if err := insertEtherRule(fd, ifname, et, uint64(i), next); err != nil {
+			res.err = fmt.Errorf("steering ethertype 0x%04x to queue %d at location %d: %w",
+				et, i, next, err)
+			res.fatal = true
+			return res
+		}
+		taken[next] = true
+	}
+
+	res.fixed = len(stale) > 0
+	res.state = fmt.Sprintf("0x%04x-0x%04x to queues 0-%d of %d",
+		ethertypes[0], ethertypes[len(ethertypes)-1], len(ethertypes)-1, rings)
+	if len(stale) > 0 {
+		res.state = fmt.Sprintf("replaced %d stale, %s", len(stale), res.state)
+	}
+	return res
+}
+
 type ethtoolIfreq struct {
 	name [unix.IFNAMSIZ]byte
 	data unsafe.Pointer
@@ -280,12 +454,13 @@ func withIoctlSocket(fn func(fd int) []checkResult) []checkResult {
 	return fn(fd)
 }
 
-func configureSystem(ifnames []string) []checkResult {
+func configureSystem(ifnames []string, ethertypes []uint16) []checkResult {
 	return withIoctlSocket(func(fd int) []checkResult {
 		out := []checkResult{checkGovernor(wantGovernor)}
 		for _, ifname := range ifnames {
 			out = append(out, checkLinkUp(fd, ifname))
 			out = append(out, checkCoalesce(fd, ifname, wantCoalesceUsecs, wantCoalesceUsecs))
+			out = append(out, checkFlowRules(fd, ifname, ethertypes))
 
 			carrierWait := 3 * time.Second
 			r, reset := checkRings(fd, ifname, wantRxRing, wantTxRing)