Run tasks on Codex beside Claude; drop turn caps and mid-turn snapshots
This commit is contained in:
@@ -18,6 +18,8 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"g.fc.run/misc/byre/internal/mcp"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -40,6 +42,9 @@ var scenarios = []scenario{
|
||||
{"Resume", scenarioResume},
|
||||
{"ModelSwitch", scenarioModelSwitch},
|
||||
{"Compact", scenarioCompact},
|
||||
{"Tools", scenarioTools},
|
||||
{"Fork", scenarioFork},
|
||||
{"Steer", scenarioSteer},
|
||||
}
|
||||
|
||||
func TestLive(t *testing.T) {
|
||||
@@ -267,6 +272,87 @@ func scenarioCompact(h *harness) {
|
||||
}
|
||||
}
|
||||
|
||||
func lookupServer(calls *[]string) *mcp.Server {
|
||||
return &mcp.Server{Name: "byre", Version: "0.0.0", Tools: []mcp.Tool{{
|
||||
Name: "lookup",
|
||||
Description: "Look up the current codeword for a topic.",
|
||||
InputSchema: json.RawMessage(`{"type":"object","properties":{"topic":{"type":"string"}},"required":["topic"]}`),
|
||||
Handler: func(ctx context.Context, args json.RawMessage) (string, error) {
|
||||
*calls = append(*calls, string(args))
|
||||
return "The codeword is TEAL-9.", nil
|
||||
},
|
||||
}}}
|
||||
}
|
||||
|
||||
func scenarioTools(h *harness) {
|
||||
calls := []string{}
|
||||
srv := lookupServer(&calls)
|
||||
|
||||
first := h.start("first", Options{MCPServers: []*mcp.Server{srv}, Reminder: "This is a byre test session."})
|
||||
h.send(first, "Call byre's lookup tool with topic birds, then reply with just the codeword it gives.")
|
||||
h.waitTurn(first, nil)
|
||||
if len(calls) != 1 || !strings.Contains(h.lastText(first), "TEAL-9") {
|
||||
h.t.Fatalf("tool calls %v; assistant said %q", calls, h.lastText(first))
|
||||
}
|
||||
id := first.ID()
|
||||
h.close(first)
|
||||
|
||||
resumed := h.start("resumed", Options{Resume: id, MCPServers: []*mcp.Server{srv}})
|
||||
h.send(resumed, "Call byre's lookup tool again with topic fish, then reply with just the codeword it gives.")
|
||||
h.waitTurn(resumed, nil)
|
||||
if len(calls) != 2 || !strings.Contains(h.lastText(resumed), "TEAL-9") {
|
||||
h.t.Fatalf("resumed thread lost the tool: calls %v; assistant said %q", calls, h.lastText(resumed))
|
||||
}
|
||||
}
|
||||
|
||||
func scenarioFork(h *harness) {
|
||||
src := h.start("source", Options{})
|
||||
h.send(src, "Remember the codeword AZURE-17. Reply with the single word ok.")
|
||||
keep := h.waitTurn(src, nil)
|
||||
h.send(src, "The codeword is now BEIGE-3; forget the old one. Reply with the single word ok.")
|
||||
h.waitTurn(src, nil)
|
||||
id := src.ID()
|
||||
h.close(src)
|
||||
|
||||
fork := h.start("fork", Options{ForkFrom: id, ForkAt: keep.ID})
|
||||
if fork.ID() == "" || fork.ID() == id {
|
||||
h.t.Fatalf("fork thread id %q, source %q", fork.ID(), id)
|
||||
}
|
||||
h.send(fork, "What is the codeword? Reply with just the codeword.")
|
||||
h.waitTurn(fork, nil)
|
||||
text := h.lastText(fork)
|
||||
if !strings.Contains(text, "AZURE-17") || strings.Contains(text, "BEIGE-3") {
|
||||
h.t.Fatalf("fork didn't stop after the first turn; assistant said %q", text)
|
||||
}
|
||||
}
|
||||
|
||||
func scenarioSteer(h *harness) {
|
||||
s := h.start("", Options{})
|
||||
h.send(s, "Run the shell command `sleep 5 && echo finished`, then reply with what it printed.")
|
||||
|
||||
steered := false
|
||||
h.waitTurn(s, func(ev Event) {
|
||||
if steered || ev.Method != MethodItemStarted {
|
||||
return
|
||||
}
|
||||
if decode[*ItemStartedNotification](h.t, ev).Item.Type != ThreadItemTypeCommandExecution {
|
||||
return
|
||||
}
|
||||
steered = true
|
||||
err := s.Steer(h.ctx, "Also end your reply with the word PELICAN.")
|
||||
if err != nil {
|
||||
h.t.Errorf("steer: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if !steered {
|
||||
h.t.Fatalf("never saw a command to steer during")
|
||||
}
|
||||
if !strings.Contains(h.lastText(s), "PELICAN") {
|
||||
h.t.Fatalf("steer didn't reach the turn; assistant said %q", h.lastText(s))
|
||||
}
|
||||
}
|
||||
|
||||
type harness struct {
|
||||
t *testing.T
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user