diff --git a/main.go b/main.go index d7797d5..caffdc8 100644 --- a/main.go +++ b/main.go @@ -26,14 +26,22 @@ const ( const wireOverhead = 24 type endpoint struct { - name string - idx int - mac [6]byte - mtu int + name string + tag string + idx int + mac [6]byte + mtu int + speed float64 +} + +func (e endpoint) macString() string { + return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", + e.mac[0], e.mac[1], e.mac[2], e.mac[3], e.mac[4], e.mac[5]) } type direction struct { label string + short string tx endpoint rx endpoint spec *frameSpec @@ -44,10 +52,12 @@ type direction struct { rxFDs []int reports chan string - prev sample - drops uint64 - nicTX nicCounters - nicRX nicCounters + prev sample + drops uint64 + nicTX nicCounters + nicRX nicCounters + nicTX0 nicCounters + nicRX0 nicCounters } type sample struct { @@ -69,7 +79,11 @@ func lookupEndpoint(name string) (endpoint, error) { } var mac [6]byte copy(mac[:], ifi.HardwareAddr) - return endpoint{name: name, idx: ifi.Index, mac: mac, mtu: ifi.MTU}, nil + e := endpoint{name: name, idx: ifi.Index, mac: mac, mtu: ifi.MTU} + if v, ok := readUint("/sys/class/net/" + name + "/speed"); ok { + e.speed = float64(v) / 1000 + } + return e, nil } func parseSizes(s string) ([]int, error) { @@ -151,12 +165,13 @@ func (d *direction) snapshot() sample { s.badLen += r.badLen.Load() } for i := range d.streams { + expected := d.streams[i].maxSeq.Load() + 1 c := d.streams[i].count.Load() if c == 0 { continue } s.count += c - s.expected += d.streams[i].maxSeq.Load() + 1 + s.expected += expected } return s } @@ -171,7 +186,20 @@ func gbps(bytes, frames uint64, secs float64) float64 { return float64((bytes+frames*wireOverhead)*8) / secs / 1e9 } -func (d *direction) reportInterval(secs float64) string { +var intervalCols = []colSpec{ + {title: "TIME", width: 6, right: true}, + {title: "DIR", width: 5}, + {title: "TX pps", width: 9, right: true}, + {title: "TX Gb/s", width: 7, right: true}, + {title: "RX pps", width: 9, right: true}, + {title: "RX Gb/s", width: 7, right: true}, + {title: "GAP", width: 9, right: true}, + {title: "CRC", width: 5, right: true}, + {title: "BADMAG", width: 6, right: true}, + {title: "KDROP", width: 7, right: true}, +} + +func (d *direction) intervalRow(elapsed, secs, target float64) []string { now := d.snapshot() p := d.prev d.prev = now @@ -181,43 +209,53 @@ func (d *direction) reportInterval(secs float64) string { rxF := now.rxFrames - p.rxFrames rxB := now.rxBytes - p.rxBytes - lost := int64(now.expected) - int64(now.count) - before := d.drops d.sampleDrops() - drops := d.drops - before - return fmt.Sprintf("%s tx %8.0f pps %6.2f Gb/s | rx %8.0f pps %6.2f Gb/s | lost(cum) %6d crc %d badmagic %d kdrop %d txshort %d txerr %d", - d.label, - float64(txF)/secs, gbps(txB, txF, secs), - float64(rxF)/secs, gbps(rxB, rxF, secs), - lost, - now.crcErr-p.crcErr, - now.badMagic-p.badMagic, - drops, - now.txShort-p.txShort, - now.txErrs-p.txErrs, - ) + return []string{ + fmt.Sprintf("%.0fs", elapsed), + paint(d.short, cCyan), + commas(uint64(float64(txF) / secs)), + rateCell(gbps(txB, txF, secs), target), + commas(uint64(float64(rxF) / secs)), + rateCell(gbps(rxB, rxF, secs), target), + gapCell(int64(now.expected) - int64(now.count)), + statusCell(now.crcErr - p.crcErr), + statusCell(now.badMagic - p.badMagic), + statusCell(d.drops - before), + } } -func (d *direction) reportNIC() string { +func (d *direction) reportNIC() []string { tx := readNIC(d.tx.name) rx := readNIC(d.rx.name) var parts []string if s := tx.diff(d.nicTX); s != "" { - parts = append(parts, " "+d.tx.name+"(tx): "+s) + parts = append(parts, paint(" ▲ "+d.tx.name+" tx-side: "+s, cYellow)) } if s := rx.diff(d.nicRX); s != "" { - parts = append(parts, " "+d.rx.name+"(rx): "+s) + parts = append(parts, paint(" ▲ "+d.rx.name+" rx-side: "+s, cYellow)) } d.nicTX = tx d.nicRX = rx - return strings.Join(parts, "\n") + return parts +} + +func (d *direction) nicRunTotals() string { + var parts []string + if s := readNIC(d.tx.name).diff(d.nicTX0); s != "" { + parts = append(parts, d.tx.name+" tx-side: "+s) + } + if s := readNIC(d.rx.name).diff(d.nicRX0); s != "" { + parts = append(parts, d.rx.name+" rx-side: "+s) + } + return strings.Join(parts, "; ") } func buildDirection(label string, tx, rx endpoint, patIdx int, sizes []int, cfg config) (*direction, error) { d := &direction{ label: label, + short: tx.tag + "→" + rx.tag, tx: tx, rx: rx, spec: newFrameSpec(patIdx, rx.mac, tx.mac, sizes), @@ -226,6 +264,8 @@ func buildDirection(label string, tx, rx endpoint, patIdx int, sizes []int, cfg } d.nicTX = readNIC(tx.name) d.nicRX = readNIC(rx.name) + d.nicTX0 = d.nicTX + d.nicRX0 = d.nicRX fm, err := fanoutMode(cfg.fanout, cfg.rxWorkers) if err != nil { @@ -335,9 +375,15 @@ func main() { rxRing = flag.Uint("rx-ring", 8160, "required rx ring size, clamped to hardware maximum") txRing = flag.Uint("tx-ring", 4096, "required tx ring size, clamped to hardware maximum") setRings = flag.Bool("set-rings", true, "allow ring resizing at startup, which resets the link") + color = flag.String("color", "auto", "colored output: auto, always, never") ) flag.Parse() + if err := initColor(*color); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + tcfg := tuneConfig{ enabled: *tune, governor: *governor, @@ -394,15 +440,20 @@ func run(aName, bName, sizesArg, patArg, fanout, txCPUsArg, rxCPUsArg string, } } + a.tag, b.tag = "A", "B" ifnames := []string{a.name, b.name} - fmt.Println("host settings:") + var fatal []string + var tuneRows [][]string for _, r := range applyTuning(tcfg, ifnames) { - fmt.Println(r) + tuneRows = append(tuneRows, []string{r.item, r.status(), r.detail()}) if r.fatal { fatal = append(fatal, r.item) } } + fmt.Println(renderBox("HOST SETTINGS", + []string{"CHECK", "STATUS", "DETAIL"}, + []bool{false, false, false}, tuneRows)) if len(fatal) > 0 { return fmt.Errorf("cannot test with %s in this state", strings.Join(fatal, ", ")) } @@ -442,18 +493,43 @@ func run(aName, bName, sizesArg, patArg, fanout, txCPUsArg, rxCPUsArg string, } }() - fmt.Printf("pattern=%s sizes=%v tx=%d rx=%d batch=%d verify=%v fanout=%s duplex=%v\n", - patterns[patIdx].name, sizes, txN, rxN, batch, verify, fanout, duplex) - for _, d := range dirs { - fmt.Printf(" %s %02x:%02x:%02x:%02x:%02x:%02x -> %02x:%02x:%02x:%02x:%02x:%02x ethertype 0x%04x\n", - d.label, - d.tx.mac[0], d.tx.mac[1], d.tx.mac[2], d.tx.mac[3], d.tx.mac[4], d.tx.mac[5], - d.rx.mac[0], d.rx.mac[1], d.rx.mac[2], d.rx.mac[3], d.rx.mac[4], d.rx.mac[5], - etherType) + var linkRows [][]string + for _, e := range []endpoint{a, b} { + linkRows = append(linkRows, []string{ + paint(e.tag, cCyan), e.name, e.macString(), + fmt.Sprintf("%.0f Gb/s", e.speed), fmt.Sprintf("%d", e.mtu), + }) } - fmt.Printf(" sndbuf %d rcvbuf %d (as granted by kernel)\n", - sockBufSize(dirs[0].txFDs[0], unix.SO_SNDBUF), - sockBufSize(dirs[0].rxFDs[0], unix.SO_RCVBUF)) + fmt.Println(renderBox("LINKS", + []string{"TAG", "INTERFACE", "MAC", "SPEED", "MTU"}, + []bool{false, false, false, true, true}, linkRows)) + + target := a.speed + if target <= 0 { + target = 10 + } + sizeStrs := make([]string, len(sizes)) + for i, s := range sizes { + sizeStrs[i] = fmt.Sprintf("%d", s) + } + fmt.Println(renderBox("TEST", + []string{"SETTING", "VALUE"}, + []bool{false, false}, [][]string{ + {"pattern", patterns[patIdx].name}, + {"frame sizes", strings.Join(sizeStrs, " ")}, + {"ethertype", fmt.Sprintf("0x%04x", etherType)}, + {"workers", fmt.Sprintf("%d tx, %d rx per direction", txN, rxN)}, + {"batch", fmt.Sprintf("%d frames per syscall", batch)}, + {"payload verify", fmt.Sprintf("%v", verify)}, + {"rx fanout", fanout}, + {"duplex", fmt.Sprintf("%v", duplex)}, + {"socket buffers", fmt.Sprintf("sndbuf %s, rcvbuf %s (granted)", + humanBytes(uint64(sockBufSize(dirs[0].txFDs[0], unix.SO_SNDBUF))), + humanBytes(uint64(sockBufSize(dirs[0].rxFDs[0], unix.SO_RCVBUF))))}, + })) + fmt.Println(paint("GAP counts frames below the high-water mark not yet drained from the rx queues;", cDim)) + fmt.Println(paint("it is provisional and settles at the drain. RESULTS at the end is authoritative.", cDim)) + fmt.Println() var doneTx, doneRx atomic.Bool var wg sync.WaitGroup @@ -483,6 +559,7 @@ func run(aName, bName, sizesArg, patArg, fanout, txCPUsArg, rxCPUsArg string, } last := time.Now() + stats := &streamTable{cols: intervalCols, headerEvery: 20} loop: for { select { @@ -493,18 +570,22 @@ loop: case now := <-tick.C: secs := now.Sub(last).Seconds() last = now - for _, w := range verifyTuning(tcfg, ifnames) { - fmt.Println(w) + elapsed := now.Sub(start).Seconds() + for _, r := range verifyTuning(tcfg, ifnames) { + fmt.Printf(" %s host config drifted: %s — %s\n", + paint("!", cYellow), r.item, r.detail()) } for _, d := range dirs { - fmt.Println(d.reportInterval(secs)) - if s := d.reportNIC(); s != "" { - fmt.Println(s) + for _, line := range stats.emit(d.intervalRow(elapsed, secs, target)) { + fmt.Println(line) + } + for _, line := range d.reportNIC() { + fmt.Println(line) } for { select { case msg := <-d.reports: - fmt.Println(" " + d.label + ": " + msg) + fmt.Println(paint(" ✗ "+d.short+": "+msg, cRed)) continue default: } @@ -520,22 +601,65 @@ loop: doneRx.Store(true) wg.Wait() - fmt.Println("---") + fmt.Println() + var resultRows, nicRows [][]string + var problems []string for _, d := range dirs { d.sampleDrops() s := d.snapshot() lost := int64(s.expected) - int64(s.count) - fmt.Printf("%s total tx %d frames %.2f GB | rx %d frames %.2f GB | lost %d (%.3g%%) crc %d badmagic %d badlen %d kdrop %d\n", - d.label, - s.txFrames, float64(s.txBytes)/1e9, - s.rxFrames, float64(s.rxBytes)/1e9, - lost, 100*float64(lost)/float64(max(s.expected, 1)), - s.crcErr, s.badMagic, s.badLen, d.drops) - fmt.Printf("%s avg tx %.2f Gb/s rx %.2f Gb/s over %.1fs\n", - d.label, - gbps(s.txBytes, s.txFrames, elapsed), - gbps(s.rxBytes, s.rxFrames, elapsed), - elapsed) + lostPct := 100 * float64(lost) / float64(max(s.expected, 1)) + + resultRows = append(resultRows, []string{ + paint(d.short, cCyan), + commas(s.txFrames), + commas(s.rxFrames), + humanBytes(s.rxBytes), + rateCell(gbps(s.txBytes, s.txFrames, elapsed), target), + rateCell(gbps(s.rxBytes, s.rxFrames, elapsed), target), + lostCell(lost), + fmt.Sprintf("%.4f", lostPct), + statusCell(s.crcErr), + statusCell(s.badMagic), + statusCell(s.badLen), + statusCell(d.drops), + }) + + if n := d.nicRunTotals(); n != "" { + nicRows = append(nicRows, []string{paint(d.short, cCyan), n}) + } + if lost != 0 { + problems = append(problems, fmt.Sprintf("%s lost %d frames (%.4f%%)", d.short, lost, lostPct)) + } + if s.crcErr != 0 { + problems = append(problems, fmt.Sprintf("%s had %d payload CRC failures", d.short, s.crcErr)) + } + if d.drops != 0 { + problems = append(problems, fmt.Sprintf("%s dropped %d frames in the kernel (host too slow, not the cable)", d.short, d.drops)) + } + } + + fmt.Println(renderBox(fmt.Sprintf("RESULTS after %.1fs", elapsed), + []string{"DIR", "TX FRAMES", "RX FRAMES", "RX DATA", "TX Gb/s", "RX Gb/s", "LOST", "LOST %", "CRC", "BAD", "BADLEN", "KDROP"}, + []bool{false, true, true, true, true, true, true, true, true, true, true, true}, + resultRows)) + + if len(nicRows) > 0 { + fmt.Println(renderBox("NIC COUNTER CHANGES DURING RUN", + []string{"DIR", "COUNTERS"}, []bool{false, false}, nicRows)) + problems = append(problems, "NIC error counters moved during the run") + } + + fmt.Println() + if len(problems) == 0 { + fmt.Println(paint(" PASS ", cBold+"\x1b[42m\x1b[30m") + " " + + paint(fmt.Sprintf("every frame arrived, no errors, %s each way at %.2f Gb/s", + humanBytes(dirs[0].snapshot().rxBytes), target), cGreen)) + } else { + fmt.Println(paint(" FAIL ", cBold+"\x1b[41m\x1b[37m")) + for _, p := range problems { + fmt.Println(paint(" ✗ "+p, cRed)) + } } return nil } diff --git a/render.go b/render.go new file mode 100644 index 0000000..f1185a5 --- /dev/null +++ b/render.go @@ -0,0 +1,231 @@ +package main + +import ( + "fmt" + "os" + "strings" +) + +const ( + cReset = "\x1b[0m" + cBold = "\x1b[1m" + cDim = "\x1b[2m" + cRed = "\x1b[31m" + cGreen = "\x1b[32m" + cYellow = "\x1b[33m" + cCyan = "\x1b[36m" + cGrey = "\x1b[90m" +) + +var useColor bool + +func initColor(mode string) error { + switch mode { + case "always": + useColor = true + case "never": + useColor = false + case "auto": + if os.Getenv("NO_COLOR") != "" || os.Getenv("TERM") == "dumb" { + useColor = false + return nil + } + st, err := os.Stdout.Stat() + useColor = err == nil && st.Mode()&os.ModeCharDevice != 0 + default: + return fmt.Errorf("unknown color mode %q (want auto, always, never)", mode) + } + return nil +} + +func paint(s, code string) string { + if !useColor || s == "" { + return s + } + return code + s + cReset +} + +func stripANSI(s string) string { + var b strings.Builder + for i := 0; i < len(s); { + if s[i] == 0x1b { + for i < len(s) && s[i] != 'm' { + i++ + } + if i < len(s) { + i++ + } + continue + } + b.WriteByte(s[i]) + i++ + } + return b.String() +} + +func visWidth(s string) int { + return len([]rune(stripANSI(s))) +} + +func pad(s string, w int, right bool) string { + gap := w - visWidth(s) + if gap < 0 { + gap = 0 + } + if right { + return strings.Repeat(" ", gap) + s + } + return s + strings.Repeat(" ", gap) +} + +func commas(v uint64) string { + s := fmt.Sprintf("%d", v) + if len(s) <= 3 { + return s + } + var parts []string + for len(s) > 3 { + parts = append([]string{s[len(s)-3:]}, parts...) + s = s[:len(s)-3] + } + return strings.Join(append([]string{s}, parts...), ",") +} + +func humanBytes(b uint64) string { + const unit = 1000.0 + v := float64(b) + for _, suffix := range []string{"B", "kB", "MB", "GB", "TB"} { + if v < unit { + return fmt.Sprintf("%.1f %s", v, suffix) + } + v /= unit + } + return fmt.Sprintf("%.1f PB", v) +} + +type colSpec struct { + title string + width int + right bool +} + +type streamTable struct { + cols []colSpec + sinceHeader int + headerEvery int +} + +func (t *streamTable) headerLines() []string { + var titles, rules []string + for _, c := range t.cols { + titles = append(titles, pad(c.title, c.width, c.right)) + rules = append(rules, strings.Repeat("─", c.width)) + } + return []string{ + paint(strings.Join(titles, " "), cBold), + paint(strings.Join(rules, " "), cGrey), + } +} + +func (t *streamTable) emit(cells []string) []string { + var out []string + if t.sinceHeader == 0 || (t.headerEvery > 0 && t.sinceHeader >= t.headerEvery) { + out = append(out, t.headerLines()...) + t.sinceHeader = 0 + } + var padded []string + for i, c := range t.cols { + v := "" + if i < len(cells) { + v = cells[i] + } + padded = append(padded, pad(v, c.width, c.right)) + } + t.sinceHeader++ + return append(out, strings.Join(padded, " ")) +} + +func renderBox(title string, headers []string, rights []bool, rows [][]string) string { + n := len(headers) + widths := make([]int, n) + for i, h := range headers { + widths[i] = visWidth(h) + } + for _, r := range rows { + for i := 0; i < n && i < len(r); i++ { + if w := visWidth(r[i]); w > widths[i] { + widths[i] = w + } + } + } + + line := func(l, m, r string) string { + var parts []string + for _, w := range widths { + parts = append(parts, strings.Repeat("─", w+2)) + } + return paint(l+strings.Join(parts, m)+r, cGrey) + } + rowText := func(cells []string, bold bool) string { + var parts []string + for i := 0; i < n; i++ { + v := "" + if i < len(cells) { + v = cells[i] + } + if bold { + v = paint(v, cBold) + } + parts = append(parts, " "+pad(v, widths[i], rights[i])+" ") + } + bar := paint("│", cGrey) + return bar + strings.Join(parts, bar) + bar + } + + var b strings.Builder + if title != "" { + b.WriteString(paint(title, cBold+cCyan) + "\n") + } + b.WriteString(line("┌", "┬", "┐") + "\n") + b.WriteString(rowText(headers, true) + "\n") + b.WriteString(line("├", "┼", "┤") + "\n") + for _, r := range rows { + b.WriteString(rowText(r, false) + "\n") + } + b.WriteString(line("└", "┴", "┘")) + return b.String() +} + +func statusCell(v uint64) string { + s := commas(v) + if v == 0 { + return paint(s, cGreen) + } + return paint(s, cRed) +} + +func gapCell(v int64) string { + if v <= 0 { + return paint("0", cGreen) + } + return paint(commas(uint64(v)), cYellow) +} + +func lostCell(v int64) string { + if v <= 0 { + return paint("0", cGreen) + } + return paint(commas(uint64(v)), cRed) +} + +func rateCell(gb float64, target float64) string { + s := fmt.Sprintf("%.2f", gb) + switch { + case gb >= target*0.995: + return paint(s, cGreen) + case gb >= target*0.90: + return paint(s, cYellow) + default: + return paint(s, cRed) + } +} diff --git a/tune.go b/tune.go index 4add685..8086bd8 100644 --- a/tune.go +++ b/tune.go @@ -80,17 +80,27 @@ type tuneResult struct { err error } -func (r tuneResult) String() string { +func (r tuneResult) status() string { switch { case r.err != nil: - return fmt.Sprintf(" %-34s FAIL %s (%v)", r.item, r.state, r.err) + return paint("FAIL", cRed) case r.fixed: - return fmt.Sprintf(" %-34s FIXED %s", r.item, r.state) + return paint("FIXED", cYellow) default: - return fmt.Sprintf(" %-34s ok %s", r.item, r.state) + return paint("ok", cGreen) } } +func (r tuneResult) detail() string { + if r.err != nil { + if r.state == "" { + return r.err.Error() + } + return fmt.Sprintf("%s: %v", r.state, r.err) + } + return r.state +} + func ethtoolCall(fd int, ifname string, data unsafe.Pointer) error { var ifr ethtoolIfreq if len(ifname) >= unix.IFNAMSIZ { @@ -346,24 +356,25 @@ func applyTuning(cfg tuneConfig, ifnames []string) []tuneResult { return out } -func verifyTuning(cfg tuneConfig, ifnames []string) []string { +func verifyTuning(cfg tuneConfig, ifnames []string) []tuneResult { fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0) if err != nil { - return []string{fmt.Sprintf("drift check failed: %v", err)} + return []tuneResult{{item: "drift check", err: err}} } defer unix.Close(fd) - var warnings []string + var drifted []tuneResult if r := checkGovernor(cfg.governor, cfg.enabled); r.fixed || r.err != nil { - warnings = append(warnings, "host config drifted:"+r.String()) + drifted = append(drifted, r) } for _, ifname := range ifnames { if r := checkCoalesce(fd, ifname, cfg.rxUsecs, cfg.txUsecs, cfg.enabled); r.fixed || r.err != nil { - warnings = append(warnings, "host config drifted:"+r.String()) + drifted = append(drifted, r) } if r, _ := checkRings(fd, ifname, cfg.rxRing, cfg.txRing, false); r.err != nil { - warnings = append(warnings, "host config drifted (not corrected, would reset link):"+r.String()) + r.state = "left alone, resizing would reset the link" + drifted = append(drifted, r) } } - return warnings + return drifted }