Run tasks on Codex beside Claude; drop turn caps and mid-turn snapshots

This commit is contained in:
Ian Gulliver
2026-10-15 14:55:58 -07:00
parent 17898b0191
commit 74707f6398
53 changed files with 1904 additions and 1295 deletions
+86
View File
@@ -18,6 +18,8 @@ import (
"sync"
"testing"
"time"
"g.fc.run/misc/byre/internal/mcp"
)
const (
@@ -40,6 +42,9 @@ var scenarios = []scenario{
{"Resume", scenarioResume},
{"ModelSwitch", scenarioModelSwitch},
{"Compact", scenarioCompact},
{"Tools", scenarioTools},
{"Fork", scenarioFork},
{"Steer", scenarioSteer},
}
func TestLive(t *testing.T) {
@@ -267,6 +272,87 @@ func scenarioCompact(h *harness) {
}
}
func lookupServer(calls *[]string) *mcp.Server {
return &mcp.Server{Name: "byre", Version: "0.0.0", Tools: []mcp.Tool{{
Name: "lookup",
Description: "Look up the current codeword for a topic.",
InputSchema: json.RawMessage(`{"type":"object","properties":{"topic":{"type":"string"}},"required":["topic"]}`),
Handler: func(ctx context.Context, args json.RawMessage) (string, error) {
*calls = append(*calls, string(args))
return "The codeword is TEAL-9.", nil
},
}}}
}
func scenarioTools(h *harness) {
calls := []string{}
srv := lookupServer(&calls)
first := h.start("first", Options{MCPServers: []*mcp.Server{srv}, Reminder: "This is a byre test session."})
h.send(first, "Call byre's lookup tool with topic birds, then reply with just the codeword it gives.")
h.waitTurn(first, nil)
if len(calls) != 1 || !strings.Contains(h.lastText(first), "TEAL-9") {
h.t.Fatalf("tool calls %v; assistant said %q", calls, h.lastText(first))
}
id := first.ID()
h.close(first)
resumed := h.start("resumed", Options{Resume: id, MCPServers: []*mcp.Server{srv}})
h.send(resumed, "Call byre's lookup tool again with topic fish, then reply with just the codeword it gives.")
h.waitTurn(resumed, nil)
if len(calls) != 2 || !strings.Contains(h.lastText(resumed), "TEAL-9") {
h.t.Fatalf("resumed thread lost the tool: calls %v; assistant said %q", calls, h.lastText(resumed))
}
}
func scenarioFork(h *harness) {
src := h.start("source", Options{})
h.send(src, "Remember the codeword AZURE-17. Reply with the single word ok.")
keep := h.waitTurn(src, nil)
h.send(src, "The codeword is now BEIGE-3; forget the old one. Reply with the single word ok.")
h.waitTurn(src, nil)
id := src.ID()
h.close(src)
fork := h.start("fork", Options{ForkFrom: id, ForkAt: keep.ID})
if fork.ID() == "" || fork.ID() == id {
h.t.Fatalf("fork thread id %q, source %q", fork.ID(), id)
}
h.send(fork, "What is the codeword? Reply with just the codeword.")
h.waitTurn(fork, nil)
text := h.lastText(fork)
if !strings.Contains(text, "AZURE-17") || strings.Contains(text, "BEIGE-3") {
h.t.Fatalf("fork didn't stop after the first turn; assistant said %q", text)
}
}
func scenarioSteer(h *harness) {
s := h.start("", Options{})
h.send(s, "Run the shell command `sleep 5 && echo finished`, then reply with what it printed.")
steered := false
h.waitTurn(s, func(ev Event) {
if steered || ev.Method != MethodItemStarted {
return
}
if decode[*ItemStartedNotification](h.t, ev).Item.Type != ThreadItemTypeCommandExecution {
return
}
steered = true
err := s.Steer(h.ctx, "Also end your reply with the word PELICAN.")
if err != nil {
h.t.Errorf("steer: %v", err)
}
})
if !steered {
h.t.Fatalf("never saw a command to steer during")
}
if !strings.Contains(h.lastText(s), "PELICAN") {
h.t.Fatalf("steer didn't reach the turn; assistant said %q", h.lastText(s))
}
}
type harness struct {
t *testing.T
name string