Hold a fatal error on the framebuffer and reboot in five minutes rather than exiting, since init's death drops the kernel's panic screen over the cause

fatal.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const fatalRebootDelay = 5 * time.Minute
+
+var fatalCh = make(chan any, 1)
+
+// Deferred in every goroutine that can panic: as PID 1 the message is shown by
+// the run loop instead of killing init; anywhere else the panic keeps its stack.
+func holdPanic() {
+	p := recover()
+	if p == nil {
+		return
+	}
+	if os.Getpid() != 1 {
+		panic(p)
+	}
+	select {
+	case fatalCh <- p:
+	default:
+	}
+}
+
+// Exiting would take init with it and the kernel's panic screen would cover
+// the cause, so it is drawn and held instead, with a reboot to retry.
+func fatal(cause error) {
+	if os.Getpid() != 1 {
+		panic(cause)
+	}
+	if err := drawFatal(cause); err != nil {
+		panic(fmt.Sprintf("%v; drawing it: %v", cause, err))
+	}
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
+	select {
+	case <-sig:
+	case <-time.After(fatalRebootDelay):
+	}
+	if err := unix.Reboot(unix.LINUX_REBOOT_CMD_RESTART); err != nil {
+		panic(err)
+	}
+}
+
+func drawFatal(cause error) error {
+	fb, err := openFramebuffer()
+	if err != nil {
+		return err
+	}
+	title, err := loadFace(true, 40)
+	if err != nil {
+		return err
+	}
+	body, err := loadFace(false, 34)
+	if err != nil {
+		return err
+	}
+
+	fb.fill(uiBg)
+	noteY := fb.h - spaceRow - body.lineH
+	y := spaceRow
+	title.draw(fb, spaceRow, y-title.capTop, "FATAL", uiRed)
+	y += title.lineH + spaceRow
+	for _, line := range wrapWords(cause.Error(), (fb.w-2*spaceRow)/body.cellW) {
+		if y+body.lineH > noteY-spaceRow {
+			body.draw(fb, spaceRow, y-body.capTop, "...", uiFg)
+			break
+		}
+		body.draw(fb, spaceRow, y-body.capTop, line, uiFg)
+		y += body.lineH + spaceTight
+	}
+	note := fmt.Sprintf("rebooting in %d minutes", int(fatalRebootDelay.Minutes()))
+	body.draw(fb, spaceRow, noteY-body.capTop, note, uiDim)
+	return fb.flush()
+}
+
+func wrapWords(s string, cols int) []string {
+	var lines []string
+	line := ""
+	for _, w := range strings.Fields(s) {
+		for len(w) > cols {
+			if line != "" {
+				lines = append(lines, line)
+				line = ""
+			}
+			lines = append(lines, w[:cols])
+			w = w[cols:]
+		}
+		switch {
+		case line == "":
+			line = w
+		case len(line)+1+len(w) <= cols:
+			line += " " + w
+		default:
+			lines = append(lines, line)
+			line = w
+		}
+	}
+	if line != "" {
+		lines = append(lines, line)
+	}
+	return lines
+}

fb.go

@@ -268,7 +268,10 @@ func openFramebuffer() (*framebuffer, error) {
 		return nil, fmt.Errorf("set crtc: %w", err)
 	}
 
-	go fb.readEvents()
+	go func() {
+		defer holdPanic()
+		fb.readEvents()
+	}()
 	// Nothing has been flipped yet, so the first frame is owed its turn.
 	fb.flips <- struct{}{}
 	return fb, nil

input.go

@@ -113,6 +113,7 @@ func watchTouch(w, h int) (*touchState, error) {
 
 	state := &touchState{}
 	go func() {
+		defer holdPanic()
 		defer f.Close()
 		buf := make([]byte, sizeofInputEvent*32)
 		var rawX, rawY int32

main.go

@@ -457,6 +457,7 @@ func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.W
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
+			defer holdPanic()
 			w.run(done)
 		}()
 	}
@@ -473,6 +474,7 @@ func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.W
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
+			defer holdPanic()
 			w.run(done)
 		}()
 	}
@@ -481,6 +483,7 @@ func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.W
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		defer holdPanic()
 		sender.run(done, startTx)
 	}()
 
@@ -488,12 +491,14 @@ func (d *direction) start(wg *sync.WaitGroup, done *atomic.Bool, rxReady *sync.W
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		defer holdPanic()
 		receiver.run(done)
 	}()
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		defer holdPanic()
 		d.poller.run(done, startTx)
 	}()
 }
@@ -536,10 +541,8 @@ func main() {
 	nsPerM := flag.Float64("ns-per-m", 4.85, "mean of both directions, per metre of cable")
 	flag.Parse()
 
-	// Nothing here is recoverable by the time it reaches this point, and as PID 1
-	// a plain exit would panic the kernel anyway with less to show for it.
 	if err := run(*aName, *bName, *nsPerM); err != nil {
-		panic(err)
+		fatal(err)
 	}
 	// A clean return is ctrl-alt-delete, which the kernel hands PID 1 as a
 	// SIGINT. Exiting on it would panic the kernel over the reboot it was asking
@@ -581,7 +584,16 @@ func (s *sampler) run(done *atomic.Bool, startTx <-chan struct{}) {
 	}
 }
 
-func run(aName, bName string, nsPerM float64) error {
+func run(aName, bName string, nsPerM float64) (err error) {
+	defer func() {
+		if p := recover(); p != nil {
+			if os.Getpid() != 1 {
+				panic(p)
+			}
+			err = fmt.Errorf("%v", p)
+		}
+	}()
+
 	if err := reportChecks("BOOT", bootstrap()); err != nil {
 		return err
 	}
@@ -670,6 +682,7 @@ func run(aName, bName string, nsPerM float64) error {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		defer holdPanic()
 		samp.run(&done, startTx)
 	}()
 	// Not gated on startTx: the cycle and the connected verdict are wanted the
@@ -677,6 +690,7 @@ func run(aName, bName string, nsPerM float64) error {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
+		defer holdPanic()
 		noise.run(&done)
 	}()
 	// Every return from here on stops the workers before the deferred closes
@@ -693,7 +707,17 @@ func run(aName, bName string, nsPerM float64) error {
 		}
 		wg.Wait()
 	}()
+	// A worker that panics before signalling ready would hang a bare Wait.
+	ready := make(chan struct{})
+	go func() {
 		rxReady.Wait()
+		close(ready)
+	}()
+	select {
+	case <-ready:
+	case p := <-fatalCh:
+		return fmt.Errorf("%v", p)
+	}
 
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
@@ -726,6 +750,8 @@ func run(aName, bName string, nsPerM float64) error {
 	stats := &streamTable{cols: intervalCols, headerEvery: 20}
 	for {
 		select {
+		case p := <-fatalCh:
+			return fmt.Errorf("%v", p)
 		case <-sig:
 			return nil
 		case <-space: