jyapayne/mnemosyne
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: remove blocking rate_limiter.acquire() from request paths

Browse source 
The rate limiter's time.sleep() blocked the single uvicorn worker
thread, deadlocking the entire server (health endpoint, dashboard,
all requests). Removed acquire() from both streaming and non-streaming
paths. The rate limiter still records 429s for circuit breaker stats
but no longer blocks.
This commit is contained in:
Joey Yakimowich-Payne 2026-03-19 07:50:20 -06:00
commit f529816c13
1 changed files with 7 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/mnemosyne/gateway.py
View file

@ -2707,17 +2707,17 @@ def create_app(
sse_buffer = bytearray() sse_buffer = bytearray()
usage: dict[str, Any] = {} usage: dict[str, Any] = {}
try: try:
rate_limiter: OutboundRateLimiter = app.state.rate_limiter
rate_limiter.acquire()
with client.stream( with client.stream(
"POST", upstream_path, json=outgoing_body, headers=headers "POST", upstream_path, json=outgoing_body, headers=headers
) as resp: ) as resp:
status_code = resp.status_code status_code = resp.status_code
retry_after = None # Record 429s for circuit breaker (non-blocking)
if status_code == 429: if status_code == 429:
ra = resp.headers.get("retry-after") ra = resp.headers.get("retry-after")
retry_after = float(ra) if ra else 1.0 retry_after = float(ra) if ra else 1.0
rate_limiter.record_response(status_code, retry_after) app.state.rate_limiter.record_response(429, retry_after)
else:
app.state.rate_limiter.record_response(status_code)
if status_code >= 400: if status_code >= 400:
body = resp.read() body = resp.read()
bytes_out = len(body) bytes_out = len(body)
@ -2797,14 +2797,12 @@ def create_app(
# Non-streaming # Non-streaming
try: try:
rate_limiter: OutboundRateLimiter = app.state.rate_limiter
rate_limiter.acquire()
resp = client.post(upstream_path, json=outgoing_body, headers=headers) resp = client.post(upstream_path, json=outgoing_body, headers=headers)
retry_after = None
if resp.status_code == 429: if resp.status_code == 429:
ra = resp.headers.get("retry-after") ra = resp.headers.get("retry-after")
retry_after = float(ra) if ra else 1.0 app.state.rate_limiter.record_response(429, float(ra) if ra else 1.0)
rate_limiter.record_response(resp.status_code, retry_after) else:
app.state.rate_limiter.record_response(resp.status_code)
except httpx.HTTPError as e: except httpx.HTTPError as e:
emit_event( emit_event(
"provider_error", "provider_error",