| 1 | """Tests for runtime capability profile resolution.""" |
| 2 | |
| 3 | from loader.runtime.capabilities import ( |
| 4 | CapabilityProfile, |
| 5 | resolve_backend_capability_profile, |
| 6 | resolve_capability_profile, |
| 7 | ) |
| 8 | |
| 9 | |
| 10 | def test_explicit_override_wins() -> None: |
| 11 | override = CapabilityProfile( |
| 12 | model_name="override-model", |
| 13 | supports_native_tools=True, |
| 14 | supports_streaming=True, |
| 15 | context_window=32000, |
| 16 | preferred_tool_call_format="native", |
| 17 | verification_strictness="strict", |
| 18 | notes=["explicit override"], |
| 19 | ) |
| 20 | |
| 21 | resolved = resolve_capability_profile("phi3", override=override) |
| 22 | assert resolved is override |
| 23 | |
| 24 | |
| 25 | def test_exact_match_registry_resolution() -> None: |
| 26 | resolved = resolve_capability_profile("deepseek-r1") |
| 27 | |
| 28 | assert not resolved.supports_native_tools |
| 29 | assert resolved.preferred_tool_call_format == "json_tag" |
| 30 | assert resolved.verification_strictness == "strict" |
| 31 | |
| 32 | |
| 33 | def test_family_heuristic_resolution_uses_model_details() -> None: |
| 34 | resolved = resolve_capability_profile( |
| 35 | "custom-qwen-build", |
| 36 | model_details={"details": {"families": ["qwen2.5"]}}, |
| 37 | ) |
| 38 | |
| 39 | assert resolved.supports_native_tools |
| 40 | assert resolved.preferred_tool_call_format == "native" |
| 41 | assert "heuristic" in resolved.notes[0].lower() |
| 42 | |
| 43 | |
| 44 | def test_model_details_context_window_overrides_registry_default() -> None: |
| 45 | resolved = resolve_capability_profile( |
| 46 | "gpt-oss:20b", |
| 47 | model_details={ |
| 48 | "model_info": { |
| 49 | "gptoss.context_length": 131072, |
| 50 | } |
| 51 | }, |
| 52 | ) |
| 53 | |
| 54 | assert resolved.context_window == 131072 |
| 55 | assert resolved.supports_native_tools |
| 56 | |
| 57 | |
| 58 | def test_unknown_models_default_to_safe_react_profile() -> None: |
| 59 | resolved = resolve_capability_profile("mystery-model") |
| 60 | |
| 61 | assert not resolved.supports_native_tools |
| 62 | assert resolved.preferred_tool_call_format == "json_tag" |
| 63 | assert "defaulting to safe" in resolved.notes[0].lower() |
| 64 | |
| 65 | |
| 66 | def test_qwen25_coder_inherits_strict_verification() -> None: |
| 67 | resolved = resolve_capability_profile("qwen2.5-coder:32b") |
| 68 | |
| 69 | assert resolved.supports_native_tools |
| 70 | assert resolved.preferred_tool_call_format == "native" |
| 71 | assert resolved.verification_strictness == "strict" |
| 72 | |
| 73 | |
| 74 | def test_devstral_resolves_native_tools() -> None: |
| 75 | resolved = resolve_capability_profile("devstral:24b") |
| 76 | |
| 77 | assert resolved.supports_native_tools |
| 78 | assert resolved.preferred_tool_call_format == "native" |
| 79 | |
| 80 | |
| 81 | def test_gpt_oss_resolves_native_tools_via_registry() -> None: |
| 82 | resolved = resolve_capability_profile("gpt-oss") |
| 83 | |
| 84 | assert resolved.supports_native_tools |
| 85 | assert resolved.preferred_tool_call_format == "native" |
| 86 | |
| 87 | |
| 88 | def test_gpt_oss_custom_variant_resolves_native_via_family() -> None: |
| 89 | resolved = resolve_capability_profile("gpt-oss-custom:20b") |
| 90 | |
| 91 | assert resolved.supports_native_tools |
| 92 | assert resolved.preferred_tool_call_format == "native" |
| 93 | assert "heuristic" in resolved.notes[0].lower() |
| 94 | |
| 95 | |
| 96 | def test_backend_capability_profile_prefers_explicit_backend_surface() -> None: |
| 97 | class DummyBackend: |
| 98 | def supports_native_tools(self) -> bool: |
| 99 | return True |
| 100 | |
| 101 | resolved = resolve_backend_capability_profile(DummyBackend()) |
| 102 | |
| 103 | assert resolved.supports_native_tools |
| 104 | assert resolved.preferred_tool_call_format == "native" |
| 105 | assert "backend capability surface" in resolved.notes[0].lower() |