"""Tests for the entropy-gated faulting module. Tests the EntropyDetector's ability to detect hedging, uncertainty, evicted entity references, and the should_fault decision logic. """ from __future__ import annotations from mnemosyne.entropy import ( DEFAULT_FAULT_THRESHOLD, EntropyDetector, EntropySignal, ) # ── Hedging detection ──────────────────────────────────────────────────── def test_entropy_hedging_detected(): """Hedging language like 'I think' and 'probably' is detected.""" detector = EntropyDetector() signal = detector.analyze_response( "I think the function is probably in utils.py", evicted_entities=[], ) assert signal.has_hedging is True assert signal.score > 0 def test_entropy_multiple_patterns(): """Multiple hedging patterns increase the score.""" detector = EntropyDetector() signal_one = detector.analyze_response( "I think it might work.", evicted_entities=[], ) signal_many = detector.analyze_response( "I think it probably might be in the file, if I recall correctly. I believe so.", evicted_entities=[], ) assert signal_many.score >= signal_one.score # ── Uncertainty detection ──────────────────────────────────────────────── def test_entropy_uncertainty_detected(): """Explicit uncertainty like 'I'm not sure' is detected.""" detector = EntropyDetector() signal = detector.analyze_response( "I'm not sure about the exact implementation. It's unclear to me.", evicted_entities=[], ) assert signal.has_uncertainty is True assert signal.score > 0 # ── No signals ─────────────────────────────────────────────────────────── def test_entropy_no_signals(): """Clean, confident response has no entropy signals.""" detector = EntropyDetector() signal = detector.analyze_response( "The function `calculate_total` is defined in src/utils.py at line 42. " "It takes two arguments: price and quantity.", evicted_entities=[], ) assert signal.has_hedging is False assert signal.has_uncertainty is False assert signal.references_evicted is False assert signal.has_hallucination_risk is False assert signal.score == 0.0 # ── Evicted entity references ─────────────────────────────────────────── def test_entropy_evicted_references(): """References to evicted entities are detected.""" detector = EntropyDetector() signal = detector.analyze_response( "The config is in settings.py and the handler is in views.py", evicted_entities=["settings.py", "views.py", "models.py"], ) assert signal.references_evicted is True assert "settings.py" in signal.referenced_entities assert "views.py" in signal.referenced_entities assert "models.py" not in signal.referenced_entities def test_entropy_hallucination_risk(): """Hedging + evicted references = hallucination risk.""" detector = EntropyDetector() signal = detector.analyze_response( "I think the config is probably in settings.py somewhere", evicted_entities=["settings.py"], ) assert signal.has_hedging is True assert signal.references_evicted is True assert signal.has_hallucination_risk is True # Hallucination risk adds a bonus to the score assert signal.score > 0.3 # ── Composite score ────────────────────────────────────────────────────── def test_entropy_composite_score(): """Score combines hedging, uncertainty, and evicted references.""" detector = EntropyDetector() # All signals present signal = detector.analyze_response( "I'm not sure, but I think the code is probably in utils.py", evicted_entities=["utils.py"], ) assert signal.has_hedging is True assert signal.has_uncertainty is True assert signal.references_evicted is True assert signal.score > 0.5 # Should be high with all signals # ── should_fault decisions ─────────────────────────────────────────────── def test_entropy_should_fault_above_threshold(): """Faulting triggers when score >= threshold AND references evicted.""" detector = EntropyDetector(fault_threshold=0.4) signal = detector.analyze_response( "I think the code is probably in utils.py, if I recall correctly. I believe so.", evicted_entities=["utils.py"], ) # Score should be above 0.4 with hedging + evicted ref + hallucination bonus entities = detector.should_fault(signal) assert len(entities) > 0 assert "utils.py" in entities def test_entropy_should_fault_below_threshold(): """No faulting when score is below threshold.""" detector = EntropyDetector(fault_threshold=0.99) # Very high threshold signal = detector.analyze_response( "I think it might be there.", evicted_entities=[], ) entities = detector.should_fault(signal) assert entities == [] def test_entropy_should_fault_no_evicted_refs(): """No faulting when there are no evicted entity references, even with high hedging.""" detector = EntropyDetector(fault_threshold=0.1) # Very low threshold signal = detector.analyze_response( "I think it probably might be somewhere, if I recall. I believe so.", evicted_entities=[], ) # Even though hedging score is high, no evicted refs → no fault entities = detector.should_fault(signal) assert entities == [] # ── Stats tracking ─────────────────────────────────────────────────────── def test_entropy_stats_tracking(): """Stats correctly count analyzed responses and triggered faults.""" detector = EntropyDetector(fault_threshold=0.3) assert detector.stats == {"analyzed": 0, "faults_triggered": 0} # Analyze a clean response detector.analyze_response("The function is in utils.py.", evicted_entities=[]) assert detector.stats["analyzed"] == 1 assert detector.stats["faults_triggered"] == 0 # Analyze and fault a hedging response with evicted refs signal = detector.analyze_response( "I think the code is probably in utils.py, if I recall correctly. I believe so.", evicted_entities=["utils.py"], ) assert detector.stats["analyzed"] == 2 detector.should_fault(signal) assert detector.stats["faults_triggered"] == 1