Open websites in the Page view and let agents drive them
This commit is contained in:
Ian Gulliver
+122
-4
@@ -3,8 +3,10 @@ package agents
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -94,19 +96,45 @@ func (m *Manager) clientTools(l *live) []mcp.Tool {
|
||||
},
|
||||
{
|
||||
Name: "show_page",
|
||||
Description: "Show the human a file from the Mac byre runs on in byre's app on a client Mac, rendered in the page view beside this task's transcript, as a browser would show it: an HTML page (loading the stylesheets, scripts and images beside it), a PDF, an image. " +
|
||||
Description: "Show the human a file from the Mac byre runs on, or a website, in byre's app on a client Mac, rendered in the page view beside this task's transcript, as a browser would show it: an HTML page (loading the stylesheets, scripts and images beside it), a PDF, an image, or (with url) a web page in a new tab. " +
|
||||
"Whenever you write or change a file the human should see as it looks (an HTML mockup or preview, a PDF, an image, an SVG), show it with this before your turn ends, without being asked or asking first. " +
|
||||
"When work needs a website (a developer console, a dashboard, a settings page), open it with url and work in it with drive_page rather than asking the human to open it; the page view keeps its sign-ins, so ask the human only to sign in. " +
|
||||
"Show it on the Mac the human's latest message was sent from.",
|
||||
InputSchema: json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"client": {"type": "string", "description": "The client's name from list_clients, or its id when several are connected from that Mac."},
|
||||
"path": {"type": "string", "description": "The file, relative to your worktree, absolute, or starting with ~/."}
|
||||
"path": {"type": "string", "description": "The file, relative to your worktree, absolute, or starting with ~/."},
|
||||
"url": {"type": "string", "description": "Instead of path: an http or https address to open in a new tab."}
|
||||
},
|
||||
"required": ["client", "path"]
|
||||
"required": ["client"]
|
||||
}`),
|
||||
Handler: func(ctx context.Context, raw json.RawMessage) (string, error) {
|
||||
return m.show(ctx, l, raw, "show_page", api.OpenPage)
|
||||
return m.showPage(ctx, l, raw)
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "drive_page",
|
||||
Description: "Work in the tabs of this task's page view in byre's app on a client Mac, which show_page opened: the human sees each step there as it happens. " +
|
||||
"It needs no mouse or keyboard, so it never disturbs the human. " +
|
||||
"ops: tabs (list the tabs: number, title, address, and which one shows), navigate (load url in the tab and wait for it), " +
|
||||
"js (run code in the tab's page: the body of an async function, so await works and return gives the result; strings come back as they are, other values as JSON; use it to read the DOM, click (element.click()) and fill fields (set value, then dispatch input and change events)), " +
|
||||
"capture (see the tab as it's drawn; only the tab the page view shows can be captured, and only while the human has the page view open). " +
|
||||
"Without tab, it acts on the tab the page view shows. Links that open a new window open as new tabs. " +
|
||||
"A page's alert, confirm or prompt asks the human, and the js call waits for the answer; while the tab isn't showing, they answer Cancel at once.",
|
||||
InputSchema: json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"client": {"type": "string", "description": "The client's name from list_clients, or its id when several are connected from that Mac."},
|
||||
"op": {"type": "string", "enum": ["tabs", "navigate", "js", "capture"]},
|
||||
"tab": {"type": "integer", "description": "The tab's number from tabs (default: the tab the page view shows)."},
|
||||
"url": {"type": "string", "description": "For navigate."},
|
||||
"code": {"type": "string", "description": "For js."}
|
||||
},
|
||||
"required": ["client", "op"]
|
||||
}`),
|
||||
Rich: func(ctx context.Context, raw json.RawMessage) mcp.Result {
|
||||
return m.drivePage(ctx, l, raw)
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -391,6 +419,96 @@ func (m *Manager) clientOpen(ctx context.Context, l *live, raw json.RawMessage)
|
||||
return fmt.Sprintf("Opened %s on %s, copied to %s.", clean, args.Client, dest), nil
|
||||
}
|
||||
|
||||
type showPageArgs struct {
|
||||
Client string `json:"client"`
|
||||
Path string `json:"path"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
func (m *Manager) showPage(ctx context.Context, l *live, raw json.RawMessage) (string, error) {
|
||||
args := showPageArgs{}
|
||||
err := json.Unmarshal(raw, &args)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("malformed show_page call: %w", err)
|
||||
}
|
||||
if (args.Path == "") == (args.URL == "") {
|
||||
return "", fmt.Errorf("show_page takes path or url, not both or neither")
|
||||
}
|
||||
if args.Path != "" {
|
||||
return m.show(ctx, l, raw, "show_page", api.OpenPage)
|
||||
}
|
||||
u, err := url.Parse(args.URL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("url %q: %w", args.URL, err)
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return "", fmt.Errorf("url %q isn't http or https", args.URL)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, fileTimeout)
|
||||
defer cancel()
|
||||
reply, err := m.clients.Call(cctx, args.Client, api.ClientRequest{Op: api.OpenPage, Task: &l.taskID, Url: &args.URL})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if reply.Output == nil {
|
||||
return "", fmt.Errorf("%s's reply has no output", args.Client)
|
||||
}
|
||||
return fmt.Sprintf("Showing %s on %s: %s", args.URL, args.Client, *reply.Output), nil
|
||||
}
|
||||
|
||||
type drivePageArgs struct {
|
||||
Client string `json:"client"`
|
||||
Op string `json:"op"`
|
||||
Tab *int `json:"tab"`
|
||||
URL string `json:"url"`
|
||||
Code string `json:"code"`
|
||||
}
|
||||
|
||||
func (m *Manager) drivePage(ctx context.Context, l *live, raw json.RawMessage) mcp.Result {
|
||||
args := drivePageArgs{}
|
||||
err := json.Unmarshal(raw, &args)
|
||||
if err != nil {
|
||||
return mcp.TextResult(fmt.Sprintf("malformed drive_page call: %s", err), true)
|
||||
}
|
||||
req := api.ClientRequest{Task: &l.taskID, Tab: args.Tab}
|
||||
switch args.Op {
|
||||
case "tabs":
|
||||
req.Op = api.PageTabs
|
||||
case "navigate":
|
||||
if args.URL == "" {
|
||||
return mcp.TextResult("navigate needs url", true)
|
||||
}
|
||||
req.Op = api.PageNavigate
|
||||
req.Url = &args.URL
|
||||
case "js":
|
||||
if args.Code == "" {
|
||||
return mcp.TextResult("js needs code", true)
|
||||
}
|
||||
req.Op = api.PageJs
|
||||
req.Code = &args.Code
|
||||
case "capture":
|
||||
req.Op = api.PageCapture
|
||||
default:
|
||||
return mcp.TextResult(fmt.Sprintf("unknown op %q", args.Op), true)
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, handsTimeout)
|
||||
defer cancel()
|
||||
reply, err := m.clients.Call(cctx, args.Client, req)
|
||||
if err != nil {
|
||||
return mcp.TextResult(err.Error(), true)
|
||||
}
|
||||
if req.Op == api.PageCapture {
|
||||
if reply.Data == nil {
|
||||
return mcp.TextResult(fmt.Sprintf("%s's reply has no data", args.Client), true)
|
||||
}
|
||||
return mcp.Result{Content: []mcp.Content{{Type: "image", Data: base64.StdEncoding.EncodeToString(*reply.Data), MimeType: "image/png"}}}
|
||||
}
|
||||
if reply.Output == nil {
|
||||
return mcp.TextResult(fmt.Sprintf("%s's reply has no output", args.Client), true)
|
||||
}
|
||||
return mcp.TextResult(*reply.Output, false)
|
||||
}
|
||||
|
||||
func (m *Manager) show(ctx context.Context, l *live, raw json.RawMessage, tool string, op api.ClientRequestOp) (string, error) {
|
||||
args := clientOpenArgs{}
|
||||
err := json.Unmarshal(raw, &args)
|
||||
|
||||
Reference in New Issue
Block a user