perf: batch embeddings in background thread to fix SSE timeouts

Root cause: 306 embedding calls at 61ms each blocked the request
thread for ~19s before forwarding to Anthropic.

- Batch all admitted objects into single embed_batch() call
- Run in background thread (non-blocking)
- store_object accepts pre-computed embeddings
- Goal detection uses turn heuristic instead of blocking embed
This commit is contained in:
Joey Yakimowich-Payne 2026-03-15 19:29:23 -06:00
commit 4f50359d01
2 changed files with 54 additions and 36 deletions

View file

@ -1983,6 +1983,7 @@ def create_app(
new_objects = segmented[len(session._segmented_objects) :]
admitted_count = 0
rejected_count = 0
admitted_objects = []
for seg_obj in new_objects:
# Phase 4d: admission control — score and gate
has_dup = False
@ -2005,24 +2006,46 @@ def create_app(
rejected_count += 1
continue
admitted_count += 1
with Timer(session.benchmark.latency["embedding"]):
_run_async(
session.object_store.store_object(
session_id=session.id,
content=seg_obj.content,
object_type=seg_obj.object_type,
source_tool=seg_obj.source_tool,
source_key=seg_obj.source_key,
stub=seg_obj.stub,
tags=seg_obj.tags,
key_entities=seg_obj.key_entities,
turn=seg_obj.turn_start,
)
admitted_objects.append(seg_obj)
# Batch embed + store in a single background thread.
# One embed_batch() call replaces N individual embed() calls.
if admitted_objects:
_batch = list(admitted_objects) # snapshot
def _store_batch(s=session, objs=_batch):
try:
embedder = s.object_store._embedder
embeddings: list[list[float]] = []
if embedder is not None:
texts = [o.content for o in objs]
with Timer(s.benchmark.latency["embedding"]):
embeddings = embedder.embed_batch(texts)
for i, so in enumerate(objs):
emb = embeddings[i] if i < len(embeddings) else []
_run_async(
s.object_store.store_object(
session_id=s.id,
content=so.content,
object_type=so.object_type,
source_tool=so.source_tool,
source_key=so.source_key,
stub=so.stub,
tags=so.tags,
key_entities=so.key_entities,
turn=so.turn_start,
embedding=emb,
)
)
except Exception:
pass # non-critical — object store is for search
_threading.Thread(target=_store_batch, daemon=True).start()
for _so in admitted_objects:
session.benchmark.segmentation.record_object(
_so.object_type,
max(1, len(_so.content) // 4),
)
session.benchmark.segmentation.record_object(
seg_obj.object_type,
max(1, len(seg_obj.content) // 4),
)
session._segmented_objects = segmented
parts = [
f"{admitted_count} new objects ({len(segmented)} total)",
@ -2104,21 +2127,13 @@ def create_app(
)
break
if user_text and session.object_store._embedder is not None:
from mnemosyne.object_store import _cosine_similarity
embedder = session.object_store._embedder
current_embedding = embedder.embed(user_text)
# Check for topic shift
goal_changed = False
if session._last_user_embedding is not None:
sim = _cosine_similarity(current_embedding, session._last_user_embedding)
goal_changed = sim < 0.5
else:
goal_changed = True # First message — classify
session._last_user_embedding = current_embedding
# Skip embedding-based goal detection in the hot path — it takes
# ~60ms per call and blocks the request thread. Use turn-based
# heuristic instead: classify goal every 10 turns.
goal_changed = False
turn = session.token_state.get("turn", 0)
if turn <= 1 or turn % 10 == 0:
goal_changed = True
if goal_changed and helper_llm is not None:
session.benchmark.goals.record_topic_shift()

View file

@ -409,10 +409,12 @@ class ObjectStore:
tags: list[str] | None = None,
key_entities: list[str] | None = None,
turn: int | None = None,
embedding: list[float] | None = None,
) -> StoredObject:
"""Create and store a new semantic object.
Auto-generates: ID, timestamps, token estimates, embedding, stub.
Pass ``embedding`` to skip the per-object embed call (for batch use).
"""
now = datetime.now(timezone.utc).isoformat()
object_id = uuid.uuid4().hex[:16]
@ -422,10 +424,11 @@ class ObjectStore:
preview = content[:80].replace("\n", " ")
stub = f"{object_type}: {preview}..."
# Compute embedding
embedding: list[float] = []
if self._embedder is not None:
embedding = self._embedder.embed(content)
# Use pre-computed embedding or compute one
if embedding is None:
embedding = []
if self._embedder is not None:
embedding = self._embedder.embed(content)
obj = StoredObject(
id=object_id,