mirror of
https://github.com/andrewyng/openworker.git
synced 2026-09-11 14:50:14 +00:00
Devika Verma
0b5294ac41
# Conflicts: # coworker/agent.py # coworker/connectors/tool_defs.py # coworker/engine.py # coworker/permissions.py # coworker/personas/manifest.py # coworker/server/manager.py # surfaces/gui/src/App.tsx # surfaces/gui/src/types.ts # tests/test_connectors.py # tests/test_send_target_resolution.py
621 lines
21 KiB
Python
621 lines
21 KiB
Python
"""Playwright-backed browser automation tools for Cowork.
|
|
|
|
The dependency is optional. If Playwright or its browser binaries are not installed, the
|
|
tools return a clear setup error instead of breaking engine construction.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Optional
|
|
|
|
import aisuite as ai
|
|
|
|
from ..web.guard import check_url
|
|
|
|
|
|
def _meta(
|
|
name: str, *, approval: bool = False, capabilities: Optional[list[str]] = None
|
|
):
|
|
return ai.ToolMetadata(
|
|
name=name,
|
|
category="connector",
|
|
risk_level="medium" if approval else "low",
|
|
capabilities=capabilities or ["browser"],
|
|
requires_approval=approval,
|
|
)
|
|
|
|
|
|
def _schema(
|
|
name: str, description: str, properties: dict[str, Any], required: list[str]
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"type": "function",
|
|
"function": {
|
|
"name": name,
|
|
"description": description,
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": properties,
|
|
"required": required,
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
def _attach(fn: Callable[..., Any], schema: dict[str, Any], *, approval: bool = True):
|
|
from .tool_defs import approval_for_tool
|
|
|
|
name = schema["function"]["name"]
|
|
# §36: the tool registry's read/write kind wins for registered tools — reads never gate.
|
|
approval = approval_for_tool(name, default=approval)
|
|
fn.__coworker_schema__ = schema
|
|
fn.__aisuite_tool_metadata__ = _meta(name, approval=approval)
|
|
fn.__doc__ = schema["function"]["description"]
|
|
return fn
|
|
|
|
|
|
class _BrowserController:
|
|
def __init__(self) -> None:
|
|
self._lock = threading.RLock()
|
|
self._playwright = None
|
|
self._browser = None
|
|
self._context = None
|
|
self._page = None
|
|
self._error: Optional[str] = None
|
|
self._executor = ThreadPoolExecutor(
|
|
max_workers=1, thread_name_prefix="coworker-browser"
|
|
)
|
|
self._state: dict[str, Any] = {
|
|
"open": False,
|
|
"url": "",
|
|
"title": "",
|
|
"status": "closed",
|
|
"last_action": "",
|
|
"last_result": "",
|
|
"last_error": "",
|
|
"screenshot_data_url": "",
|
|
"updated_at": None,
|
|
"controls": [],
|
|
}
|
|
|
|
def _touch(self, **changes: Any) -> None:
|
|
self._state.update(changes)
|
|
self._state["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
def _refresh_page_state(self) -> None:
|
|
if self._page is None:
|
|
self._touch(open=False, status="closed", url="", title="", controls=[])
|
|
return
|
|
try:
|
|
snap = _snapshot(self._page, 2000)
|
|
self._touch(
|
|
open=True,
|
|
status="open",
|
|
url=self._page.url,
|
|
title=self._page.title(),
|
|
controls=snap.get("controls", [])[:30],
|
|
)
|
|
except Exception as exc:
|
|
self._touch(open=True, status="error", last_error=str(exc))
|
|
|
|
def _setup_error(self, exc: Exception) -> dict[str, str]:
|
|
return {
|
|
"error": (
|
|
"Interactive browser automation requires Playwright. Install it with "
|
|
"`pip install playwright` and `python -m playwright install chromium`."
|
|
),
|
|
"details": str(exc),
|
|
}
|
|
|
|
def page(self):
|
|
with self._lock:
|
|
if self._error:
|
|
return None, {"error": self._error}
|
|
if self._page is not None:
|
|
return self._page, None
|
|
try:
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
self._playwright = sync_playwright().start()
|
|
self._browser = self._playwright.chromium.launch(headless=False)
|
|
self._context = self._browser.new_context(
|
|
viewport={"width": 1280, "height": 900}
|
|
)
|
|
self._page = self._context.new_page()
|
|
self._touch(
|
|
open=True, status="open", last_action="open browser", last_error=""
|
|
)
|
|
return self._page, None
|
|
except Exception as exc:
|
|
self._touch(open=False, status="error", last_error=str(exc))
|
|
return None, self._setup_error(exc)
|
|
|
|
def _submit(self, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
|
|
return self._executor.submit(fn).result()
|
|
|
|
def close(self) -> dict[str, Any]:
|
|
return self._submit(self._close_locked)
|
|
|
|
def _close_locked(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
try:
|
|
if self._context is not None:
|
|
self._context.close()
|
|
if self._browser is not None:
|
|
self._browser.close()
|
|
if self._playwright is not None:
|
|
self._playwright.stop()
|
|
except Exception as exc:
|
|
return {"error": str(exc)}
|
|
finally:
|
|
self._playwright = None
|
|
self._browser = None
|
|
self._context = None
|
|
self._page = None
|
|
self._touch(open=False, status="closed", url="", title="", controls=[])
|
|
return {"ok": True}
|
|
|
|
def state(self) -> dict[str, Any]:
|
|
return self._submit(self._state_locked)
|
|
|
|
def _state_locked(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
self._refresh_page_state()
|
|
return dict(self._state)
|
|
|
|
def screenshot(self) -> dict[str, Any]:
|
|
return self._submit(self._screenshot_locked)
|
|
|
|
def _screenshot_locked(self) -> dict[str, Any]:
|
|
with self._lock:
|
|
page, err = self.page()
|
|
if err:
|
|
return err
|
|
try:
|
|
png = page.screenshot(full_page=False)
|
|
data_url = "data:image/png;base64," + base64.b64encode(png).decode(
|
|
"ascii"
|
|
)
|
|
self._touch(
|
|
screenshot_data_url=data_url,
|
|
last_action="screenshot",
|
|
last_result="ok",
|
|
last_error="",
|
|
)
|
|
self._refresh_page_state()
|
|
return {"ok": True, **dict(self._state)}
|
|
except Exception as exc:
|
|
self._touch(
|
|
last_action="screenshot", last_result="error", last_error=str(exc)
|
|
)
|
|
return {"error": str(exc)}
|
|
|
|
def call(self, action: str, fn: Callable[[Any], dict[str, Any]]) -> dict[str, Any]:
|
|
def run() -> dict[str, Any]:
|
|
with self._lock:
|
|
page, err = self.page()
|
|
if err:
|
|
return err
|
|
self._touch(last_action=action, last_result="running", last_error="")
|
|
try:
|
|
out = fn(page)
|
|
except Exception as exc:
|
|
out = {"error": str(exc)}
|
|
if "error" in out:
|
|
self._touch(
|
|
last_action=action,
|
|
last_result="error",
|
|
last_error=str(out["error"]),
|
|
)
|
|
else:
|
|
self._refresh_page_state()
|
|
self._touch(last_action=action, last_result="ok", last_error="")
|
|
return out
|
|
|
|
return self._submit(run)
|
|
|
|
|
|
_BROWSER = _BrowserController()
|
|
|
|
|
|
def browser_state() -> dict[str, Any]:
|
|
return _BROWSER.state()
|
|
|
|
|
|
def browser_take_screenshot() -> dict[str, Any]:
|
|
return _BROWSER.screenshot()
|
|
|
|
|
|
def browser_close_session() -> dict[str, Any]:
|
|
return _BROWSER.close()
|
|
|
|
|
|
def _cap(value: int, default: int = 20000, upper: int = 100000) -> int:
|
|
try:
|
|
return max(1, min(int(value or default), upper))
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def _target_locator(page, target: str):
|
|
target = target.strip()
|
|
if target.startswith("text="):
|
|
return page.get_by_text(target[5:], exact=False).first
|
|
if target.startswith("role="):
|
|
role_name = target[5:]
|
|
role, _, name = role_name.partition(":")
|
|
return page.get_by_role(role.strip(), name=name.strip() or None).first
|
|
try:
|
|
return page.locator(target).first
|
|
except Exception:
|
|
return page.get_by_text(target, exact=False).first
|
|
|
|
|
|
def _safe_call(fn: Callable[[], Any]) -> dict[str, Any]:
|
|
try:
|
|
return fn()
|
|
except Exception as exc:
|
|
return {"error": str(exc)}
|
|
|
|
|
|
def _browser_call(action: str, fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
|
|
return _BROWSER.call(action, lambda _page: fn())
|
|
|
|
|
|
_SNAPSHOT_JS = """
|
|
() => {
|
|
const visible = (el) => {
|
|
const style = window.getComputedStyle(el);
|
|
const rect = el.getBoundingClientRect();
|
|
return style && style.visibility !== 'hidden' && style.display !== 'none' && rect.width > 0 && rect.height > 0;
|
|
};
|
|
const labelFor = (el) => {
|
|
if (el.labels && el.labels.length) return Array.from(el.labels).map(l => l.innerText.trim()).filter(Boolean).join(' ');
|
|
const id = el.getAttribute('id');
|
|
if (id) {
|
|
const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
|
|
if (label) return label.innerText.trim();
|
|
}
|
|
return '';
|
|
};
|
|
const describe = (el, i) => ({
|
|
index: i,
|
|
tag: el.tagName.toLowerCase(),
|
|
type: el.getAttribute('type') || '',
|
|
id: el.getAttribute('id') || '',
|
|
name: el.getAttribute('name') || '',
|
|
role: el.getAttribute('role') || '',
|
|
aria: el.getAttribute('aria-label') || '',
|
|
label: labelFor(el),
|
|
placeholder: el.getAttribute('placeholder') || '',
|
|
text: (el.innerText || el.value || '').trim().slice(0, 200),
|
|
href: el.getAttribute('href') || '',
|
|
selectorHint: el.getAttribute('id') ? `#${CSS.escape(el.getAttribute('id'))}` : (el.getAttribute('name') ? `[name="${el.getAttribute('name')}"]` : '')
|
|
});
|
|
const controls = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role="button"],[contenteditable="true"]'))
|
|
.filter(visible)
|
|
.slice(0, 120)
|
|
.map(describe);
|
|
return {
|
|
title: document.title,
|
|
url: location.href,
|
|
text: document.body ? document.body.innerText : '',
|
|
controls
|
|
};
|
|
}
|
|
"""
|
|
|
|
|
|
def _snapshot(page, max_chars: int) -> dict[str, Any]:
|
|
data = page.evaluate(_SNAPSHOT_JS)
|
|
text = re.sub(r"\n{3,}", "\n\n", str(data.get("text") or ""))
|
|
cap = _cap(max_chars)
|
|
return {
|
|
"title": data.get("title"),
|
|
"url": data.get("url"),
|
|
"text": text[:cap],
|
|
"truncated": len(text) > cap,
|
|
"controls": data.get("controls") or [],
|
|
}
|
|
|
|
|
|
def redirect_refusal(requested: str, final: str) -> Optional[str]:
|
|
"""A refusal reason if navigation LANDED somewhere the address guard would refuse.
|
|
|
|
`check_url` vets the URL the model supplied; Playwright then follows redirects, and the
|
|
hop that actually loads is a different address the guard never saw (OPE-124). A public
|
|
shortener can land on the cloud metadata endpoint or a router admin page, and the
|
|
approval the user gave was for the first URL, not this one.
|
|
|
|
The request has already gone out by the time this runs — it cannot be prevented here.
|
|
What it prevents is the agent READING the page or interacting with it. Later
|
|
JavaScript- or meta-refresh-driven navigation is still unchecked; only a proxy that
|
|
vets every hop closes that, which is the larger design this defers."""
|
|
if not final or final == requested:
|
|
return None
|
|
return check_url(final)
|
|
|
|
|
|
def make_browser_automation_tools(
|
|
*, roots: Optional[list[Any]] = None
|
|
) -> list[Callable[..., Any]]:
|
|
tools: list[Callable[..., Any]] = []
|
|
|
|
def _readable_source(raw: str) -> tuple[Any, dict[str, Any] | None]:
|
|
"""A local file to upload, resolved inside a granted root (OPE-122).
|
|
|
|
These tools touch the filesystem but classify EXTERNAL, so the permission engine's
|
|
root scoping — which only runs for WRITE_LOCAL — never sees them. Without this
|
|
check the only thing between `~/.ssh/id_rsa` and a web form is someone reading the
|
|
approval card. Mirrors `email_send`'s attachment rule, which solves the same
|
|
problem for outgoing mail."""
|
|
allowed = [r.path for r in (roots or [])]
|
|
if not allowed:
|
|
return None, {"error": "no session directory is available to upload from"}
|
|
path = Path(str(raw)).expanduser().resolve()
|
|
if not any(path.is_relative_to(root) for root in allowed):
|
|
return None, {"error": f"{path} is outside the session's directories"}
|
|
return path, None
|
|
|
|
def _writable_target(raw: str) -> tuple[Any, dict[str, Any] | None]:
|
|
"""Where a screenshot may land: inside a WRITABLE granted root. An unnamed target
|
|
keeps the temp-file default, which is not a place the user asked us to protect."""
|
|
writable = [r.path for r in (roots or []) if r.writable]
|
|
if not writable:
|
|
return None, {"error": "no writable session directory for the screenshot"}
|
|
path = Path(str(raw)).expanduser().resolve()
|
|
if not any(path.is_relative_to(root) for root in writable):
|
|
return None, {
|
|
"error": f"{path} is outside the session's writable directories"
|
|
}
|
|
return path, None
|
|
|
|
def browser_open_url(
|
|
url: str, wait_until: str = "domcontentloaded"
|
|
) -> dict[str, Any]:
|
|
if not url.lower().startswith(("http://", "https://")):
|
|
return {"error": "url must start with http:// or https://"}
|
|
# Same address guard as web_fetch. This is approval gated, so it is defense in
|
|
# depth, not the primary control. It checks the initial model supplied URL only;
|
|
# redirects that the browser follows internally are not hop checked here.
|
|
blocked = check_url(url)
|
|
if blocked:
|
|
return {"error": blocked}
|
|
|
|
def _open(page):
|
|
page.goto(url, wait_until=wait_until, timeout=30000)
|
|
landed = redirect_refusal(url, page.url)
|
|
if landed:
|
|
# Leave nothing readable behind: the next snapshot/get_text must not be
|
|
# able to lift content off a page we just refused.
|
|
final = page.url
|
|
page.goto("about:blank")
|
|
return {"error": f"redirected to {final} — {landed}"}
|
|
return {"ok": True, "url": page.url}
|
|
|
|
return _BROWSER.call("open_url", _open)
|
|
|
|
browser_open_url.__name__ = "browser_open_url"
|
|
tools.append(
|
|
_attach(
|
|
browser_open_url,
|
|
_schema(
|
|
"browser_open_url",
|
|
"Open a URL in the local Playwright browser session.",
|
|
{"url": {"type": "string"}, "wait_until": {"type": "string"}},
|
|
["url"],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_read_page(max_chars: int = 20000) -> dict[str, Any]:
|
|
return _BROWSER.call("snapshot", lambda page: _snapshot(page, max_chars))
|
|
|
|
browser_read_page.__name__ = "browser_read_page"
|
|
tools.append(
|
|
_attach(
|
|
browser_read_page,
|
|
_schema(
|
|
"browser_read_page",
|
|
"Read the current page: its text plus visible controls and selector "
|
|
"hints (for browser_click/browser_type). Not an image — use "
|
|
"browser_screenshot for pixels.",
|
|
{"max_chars": {"type": "integer"}},
|
|
[],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_click(target: str) -> dict[str, Any]:
|
|
return _BROWSER.call(
|
|
"click",
|
|
lambda page: (
|
|
_target_locator(page, target).click(timeout=10000),
|
|
{"ok": True, "url": page.url},
|
|
)[1],
|
|
)
|
|
|
|
browser_click.__name__ = "browser_click"
|
|
tools.append(
|
|
_attach(
|
|
browser_click,
|
|
_schema(
|
|
"browser_click",
|
|
"Click a visible page element by CSS selector, text=label, role=button:Name, or text fallback. Requires approval.",
|
|
{"target": {"type": "string"}},
|
|
["target"],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_type(target: str, text: str, clear: bool = True) -> dict[str, Any]:
|
|
def run(page):
|
|
loc = _target_locator(page, target)
|
|
if clear:
|
|
loc.fill(text, timeout=10000)
|
|
else:
|
|
loc.type(text, timeout=10000)
|
|
return {"ok": True, "url": page.url}
|
|
|
|
return _BROWSER.call("type", run)
|
|
|
|
browser_type.__name__ = "browser_type"
|
|
tools.append(
|
|
_attach(
|
|
browser_type,
|
|
_schema(
|
|
"browser_type",
|
|
"Fill or type into an input, textarea, or editable element. Requires approval.",
|
|
{
|
|
"target": {"type": "string"},
|
|
"text": {"type": "string"},
|
|
"clear": {"type": "boolean"},
|
|
},
|
|
["target", "text"],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_select(target: str, value: str) -> dict[str, Any]:
|
|
return _BROWSER.call(
|
|
"select",
|
|
lambda page: (
|
|
_target_locator(page, target).select_option(value, timeout=10000),
|
|
{"ok": True, "url": page.url},
|
|
)[1],
|
|
)
|
|
|
|
browser_select.__name__ = "browser_select"
|
|
tools.append(
|
|
_attach(
|
|
browser_select,
|
|
_schema(
|
|
"browser_select",
|
|
"Select an option in a dropdown by selector and option value/label. Requires approval.",
|
|
{"target": {"type": "string"}, "value": {"type": "string"}},
|
|
["target", "value"],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_upload_file(target: str, path: str) -> dict[str, Any]:
|
|
file_path, err = _readable_source(path)
|
|
if err:
|
|
return err
|
|
if not file_path.exists():
|
|
return {"error": f"file not found: {file_path}"}
|
|
return _BROWSER.call(
|
|
"upload_file",
|
|
lambda page: (
|
|
_target_locator(page, target).set_input_files(
|
|
str(file_path), timeout=10000
|
|
),
|
|
{"ok": True, "path": str(file_path)},
|
|
)[1],
|
|
)
|
|
|
|
browser_upload_file.__name__ = "browser_upload_file"
|
|
tools.append(
|
|
_attach(
|
|
browser_upload_file,
|
|
_schema(
|
|
"browser_upload_file",
|
|
"Upload a local file through a file input. Requires approval.",
|
|
{"target": {"type": "string"}, "path": {"type": "string"}},
|
|
["target", "path"],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_wait(milliseconds: int = 1000, target: str = "") -> dict[str, Any]:
|
|
def run(page):
|
|
if target:
|
|
_target_locator(page, target).wait_for(
|
|
timeout=max(1, int(milliseconds or 1000))
|
|
)
|
|
else:
|
|
page.wait_for_timeout(max(1, min(int(milliseconds or 1000), 30000)))
|
|
return {"ok": True, "url": page.url}
|
|
|
|
return _BROWSER.call("wait", run)
|
|
|
|
browser_wait.__name__ = "browser_wait"
|
|
tools.append(
|
|
_attach(
|
|
browser_wait,
|
|
_schema(
|
|
"browser_wait",
|
|
"Wait for a duration or for a target element to appear.",
|
|
{"milliseconds": {"type": "integer"}, "target": {"type": "string"}},
|
|
[],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_screenshot(path: str = "") -> dict[str, Any]:
|
|
if path:
|
|
_target, target_err = _writable_target(path)
|
|
if target_err:
|
|
return target_err
|
|
|
|
def run(page):
|
|
out = (
|
|
_target
|
|
if path
|
|
else (
|
|
Path(tempfile.gettempdir()) / "coworker-browser-screenshot.png"
|
|
).resolve()
|
|
)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
page.screenshot(path=str(out), full_page=True)
|
|
return {"ok": True, "path": str(out), "url": page.url}
|
|
|
|
return _BROWSER.call("screenshot", run)
|
|
|
|
browser_screenshot.__name__ = "browser_screenshot"
|
|
tools.append(
|
|
_attach(
|
|
browser_screenshot,
|
|
_schema(
|
|
"browser_screenshot",
|
|
"Save a full-page screenshot of the current browser page and return the local path.",
|
|
{"path": {"type": "string"}},
|
|
[],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
def browser_close() -> dict[str, Any]:
|
|
return browser_close_session()
|
|
|
|
browser_close.__name__ = "browser_close"
|
|
tools.append(
|
|
_attach(
|
|
browser_close,
|
|
_schema(
|
|
"browser_close",
|
|
"Close the local Playwright browser session.",
|
|
{},
|
|
[],
|
|
),
|
|
approval=True,
|
|
)
|
|
)
|
|
|
|
return tools
|