@@ -0,0 +1,77 @@ |
| 1 | +"""Tests for runtime-owned confidence and verification services.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +from types import SimpleNamespace |
| 6 | + |
| 7 | +import pytest |
| 8 | + |
| 9 | +from loader.llm.base import CompletionResponse |
| 10 | +from loader.runtime.reasoning_service import RuntimeReasoningService |
| 11 | +from tests.helpers.runtime_harness import ScriptedBackend |
| 12 | + |
| 13 | + |
| 14 | +def build_config(*, use_quick_confidence: bool = True, use_quick_verification: bool = True): |
| 15 | + return SimpleNamespace( |
| 16 | + reasoning=SimpleNamespace( |
| 17 | + use_quick_confidence=use_quick_confidence, |
| 18 | + use_quick_verification=use_quick_verification, |
| 19 | + ) |
| 20 | + ) |
| 21 | + |
| 22 | + |
| 23 | +@pytest.mark.asyncio |
| 24 | +async def test_runtime_reasoning_service_uses_quick_confidence_without_backend() -> None: |
| 25 | + backend = ScriptedBackend() |
| 26 | + service = RuntimeReasoningService(backend, build_config()) |
| 27 | + |
| 28 | + assessment = await service.assess_confidence( |
| 29 | + "read", |
| 30 | + {"file_path": "README.md"}, |
| 31 | + "Please inspect the docs first.", |
| 32 | + ) |
| 33 | + |
| 34 | + assert assessment.score >= 3 |
| 35 | + assert assessment.reasoning == "Quick heuristic assessment" |
| 36 | + assert backend.invocations == [] |
| 37 | + |
| 38 | + |
| 39 | +@pytest.mark.asyncio |
| 40 | +async def test_runtime_reasoning_service_calls_backend_for_low_confidence_actions() -> None: |
| 41 | + backend = ScriptedBackend( |
| 42 | + completions=[ |
| 43 | + CompletionResponse( |
| 44 | + content=( |
| 45 | + '{"confidence": 2, "reasoning": "This is risky.", ' |
| 46 | + '"risks": ["Potential deletion"], "mitigations": ["Inspect first"]}' |
| 47 | + ) |
| 48 | + ) |
| 49 | + ] |
| 50 | + ) |
| 51 | + service = RuntimeReasoningService(backend, build_config()) |
| 52 | + |
| 53 | + assessment = await service.assess_confidence( |
| 54 | + "bash", |
| 55 | + {"command": "rm -rf build"}, |
| 56 | + "Need to clean up artifacts.", |
| 57 | + ) |
| 58 | + |
| 59 | + assert assessment.score == 2 |
| 60 | + assert assessment.reasoning == "This is risky." |
| 61 | + assert len(backend.invocations) == 1 |
| 62 | + |
| 63 | + |
| 64 | +@pytest.mark.asyncio |
| 65 | +async def test_runtime_reasoning_service_uses_quick_verification_without_backend() -> None: |
| 66 | + backend = ScriptedBackend() |
| 67 | + service = RuntimeReasoningService(backend, build_config()) |
| 68 | + |
| 69 | + verification = await service.verify_action( |
| 70 | + "read", |
| 71 | + {"file_path": "README.md"}, |
| 72 | + "loader docs", |
| 73 | + ) |
| 74 | + |
| 75 | + assert verification.verified is True |
| 76 | + assert verification.verification_method == "quick_heuristic" |
| 77 | + assert backend.invocations == [] |
