Files
openworker/tests/test_ws_guard.py
Devika Verma b50283ed36 fix(server): a websocket client that stopped reading no longer hangs shutdown
Ctrl+C hung at "Shutting down" when a browser tab had stopped reading its
websocket (a frozen or discarded tab, or after sleep) while the server kept
pushing events: the connection could never send its close frame, uvicorn
waited for it, and since Python 3.12 Server.wait_closed() waits for every
transport, so a second Ctrl+C could not force past it either.

ws_guard.py: a send on the GUI's websockets (/ws/session, /ws/events) that
cannot finish in 15 s (COWORKER_WS_SEND_TIMEOUT) aborts that connection, and
the launcher's GuardedServer aborts every websocket still open before uvicorn
waits for connections. Machine links keep no send timeout; they carry large
frames over slow networks.

Checked with a frozen browser tab on a live session: before, still running
40 s after Ctrl+Break; now it exits in 0.5 s.
2026-11-05 16:50:45 +05:30

141 lines
5.0 KiB
Python

"""T3-3: a websocket whose client stopped reading must not hold the server up. A GUI websocket
send that cannot finish in time drops the connection, and at shutdown every open websocket is
aborted before uvicorn waits for its connections."""
from __future__ import annotations
import asyncio
import pytest
from fastapi.testclient import TestClient
from starlette.websockets import WebSocketDisconnect
from coworker.server import ws_guard
class _Transport(asyncio.BaseTransport):
def __init__(self) -> None:
super().__init__()
self.aborted = False
def abort(self) -> None:
self.aborted = True
def is_closing(self) -> bool:
return self.aborted
class _Protocol:
"""Stands in for uvicorn's websocket protocol: its `send` is a bound method, and it holds
the connection's transport."""
def __init__(self, stuck: bool) -> None:
self.transport = _Transport()
self.stuck = stuck
self.sent: list[dict] = []
async def send(self, message: dict) -> None:
if self.stuck and message.get("type") == "websocket.send":
await asyncio.sleep(3600) # the client stopped reading: the buffer never drains
self.sent.append(message)
async def _receive() -> dict:
return {"type": "websocket.connect"}
def _app(log: list):
async def app(scope, receive, send):
assert ws_guard.open_count() == 1 # tracked while it is open
await send({"type": "websocket.accept"})
try:
await send({"type": "websocket.send", "text": "x"})
except WebSocketDisconnect as exc:
log.append(exc.code)
return
log.append("sent")
return app
def test_a_gui_websocket_whose_client_stopped_reading_is_dropped():
log: list = []
proto = _Protocol(stuck=True)
guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
asyncio.run(guard({"type": "websocket", "path": "/ws/session/s1"}, _receive, proto.send))
assert proto.transport.aborted and log == [1011]
assert ws_guard.open_count() == 0 # no longer tracked
def test_a_reading_client_is_untouched():
log: list = []
proto = _Protocol(stuck=False)
guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
asyncio.run(guard({"type": "websocket", "path": "/ws/events"}, _receive, proto.send))
assert not proto.transport.aborted and log == ["sent"]
def test_every_websocket_is_tracked_only_while_open():
log: list = []
proto = _Protocol(stuck=False)
guard = ws_guard.WebsocketGuard(_app(log), send_timeout=0.05)
asyncio.run(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))
assert log == ["sent"] and ws_guard.open_count() == 0 and proto.transport not in ws_guard._open
def test_machine_links_have_no_send_timeout():
"""/ws/machines carries large frames over slow links: no timeout there, only the
shutdown abort."""
async def go():
proto = _Protocol(stuck=True)
guard = ws_guard.WebsocketGuard(_app([]), send_timeout=0.05)
task = asyncio.create_task(guard({"type": "websocket", "path": "/ws/machines/m1/p"}, _receive, proto.send))
await asyncio.sleep(0.3)
assert not task.done() and not proto.transport.aborted # still waiting, not dropped
assert ws_guard.abort_open_websockets() == 1 # what shutdown does
assert proto.transport.aborted
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
asyncio.run(go())
def test_the_server_aborts_open_websockets_before_it_waits(monkeypatch):
order: list[str] = []
transport = _Transport()
ws_guard._open.add(transport)
async def base_shutdown(self, sockets=None):
order.append("uvicorn waits" + (" (none open)" if ws_guard.open_count() == 0 else " (still open!)"))
monkeypatch.setattr(ws_guard.uvicorn.Server, "shutdown", base_shutdown)
server = ws_guard.GuardedServer.__new__(ws_guard.GuardedServer)
asyncio.run(server.shutdown())
assert transport.aborted and order == ["uvicorn waits (none open)"]
def test_the_send_timeout_can_be_set_and_falls_back(monkeypatch):
monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", "2.5")
assert ws_guard._send_timeout() == 2.5
for bad in ("0", "-1", "soon"):
monkeypatch.setenv("COWORKER_WS_SEND_TIMEOUT", bad)
assert ws_guard._send_timeout() == ws_guard.SEND_TIMEOUT_SECONDS
def test_the_app_still_serves_its_websockets_through_the_guard(tmp_path, monkeypatch):
"""Under the test client there is no transport to track; the session websocket works as
before."""
from coworker.server import SessionManager, create_app
monkeypatch.setenv("COWORKER_STATE_DIR", str(tmp_path / "state"))
manager = SessionManager(workspace=tmp_path)
client = TestClient(create_app(manager))
with client.websocket_connect("/ws/events"):
pass
with client.websocket_connect("/ws/session/g1") as ws:
assert ws.receive_json()["type"] in ("ready", "export_notice")
if hasattr(manager, "observability"):
manager.observability.close()