Stabilize export smoke coverage
Authored by
mfwolffe <wolffemf@dukes.jmu.edu>
- SHA
19042dba2924653d552fc2b0ac4db75e7991a65d- Parents
-
1c7561e - Tree
7d3212c
19042db
19042dba2924653d552fc2b0ac4db75e7991a65d1c7561e
7d3212c| Status | File | + | - |
|---|---|---|---|
| M |
tests/unit/export/targets/test_mlx_serve_argv.py
|
92 | 1 |
| M |
tests/unit/export/targets/test_vllm_argv.py
|
218 | 0 |
| A |
tests/unit/export/test_dispatch.py
|
340 | 0 |
| M |
tests/unit/export/test_smoke.py
|
354 | 118 |
tests/unit/export/targets/test_mlx_serve_argv.pymodified@@ -7,11 +7,14 @@ from pathlib import Path | ||
| 7 | 7 | import pytest |
| 8 | 8 | |
| 9 | 9 | from dlm.base_models import BASE_MODELS |
| 10 | -from dlm.export.errors import ExportError | |
| 10 | +from dlm.export.errors import ExportError, TargetSmokeError | |
| 11 | 11 | from dlm.export.manifest import load_export_manifest |
| 12 | +from dlm.export.targets.base import TargetResult | |
| 12 | 13 | from dlm.export.targets.mlx_serve import ( |
| 13 | 14 | LAUNCH_SCRIPT_FILENAME, |
| 14 | 15 | MLX_SERVE_TARGET, |
| 16 | + _quote_script_arg, | |
| 17 | + _require_prepared_int, | |
| 15 | 18 | finalize_mlx_serve_export, |
| 16 | 19 | prepare_mlx_serve_export, |
| 17 | 20 | ) |
@@ -60,6 +63,10 @@ def _setup_named_store(tmp_path: Path) -> object: | ||
| 60 | 63 | |
| 61 | 64 | |
| 62 | 65 | class TestPrepareMlxServeExport: |
| 66 | + def test_prepare_method_is_not_used_directly(self) -> None: | |
| 67 | + with pytest.raises(NotImplementedError, match="prepare_mlx_serve_export"): | |
| 68 | + MLX_SERVE_TARGET.prepare(object()) | |
| 69 | + | |
| 63 | 70 | def test_prepare_writes_launch_script_and_manifest( |
| 64 | 71 | self, tmp_path: Path, monkeypatch: object |
| 65 | 72 | ) -> None: |
@@ -139,6 +146,50 @@ class TestPrepareMlxServeExport: | ||
| 139 | 146 | declared_adapter_names=None, |
| 140 | 147 | ) |
| 141 | 148 | |
| 149 | + def test_refuses_without_mlx_extra(self, tmp_path: Path, monkeypatch: object) -> None: | |
| 150 | + store = _setup_flat_store(tmp_path) | |
| 151 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.is_apple_silicon", lambda: True) | |
| 152 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.mlx_available", lambda: False) | |
| 153 | + | |
| 154 | + with pytest.raises(ExportError, match="mlx extra"): | |
| 155 | + prepare_mlx_serve_export( | |
| 156 | + store=store, | |
| 157 | + spec=_SPEC, | |
| 158 | + adapter_name=None, | |
| 159 | + adapter_path_override=None, | |
| 160 | + declared_adapter_names=None, | |
| 161 | + ) | |
| 162 | + | |
| 163 | + def test_missing_named_adapter_raises(self, tmp_path: Path, monkeypatch: object) -> None: | |
| 164 | + store = _setup_named_store(tmp_path) | |
| 165 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.is_apple_silicon", lambda: True) | |
| 166 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.mlx_available", lambda: True) | |
| 167 | + | |
| 168 | + with pytest.raises(ExportError, match="no current adapter under"): | |
| 169 | + prepare_mlx_serve_export( | |
| 170 | + store=store, | |
| 171 | + spec=_SPEC, | |
| 172 | + adapter_name="missing", | |
| 173 | + adapter_path_override=None, | |
| 174 | + declared_adapter_names=None, | |
| 175 | + ) | |
| 176 | + | |
| 177 | + def test_missing_default_adapter_raises(self, tmp_path: Path, monkeypatch: object) -> None: | |
| 178 | + store = for_dlm("01EMPTYMLX", home=tmp_path) | |
| 179 | + store.ensure_layout() | |
| 180 | + save_manifest(store.manifest, Manifest(dlm_id="01EMPTYMLX", base_model=_SPEC.key)) | |
| 181 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.is_apple_silicon", lambda: True) | |
| 182 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.mlx_available", lambda: True) | |
| 183 | + | |
| 184 | + with pytest.raises(ExportError, match="no current adapter under"): | |
| 185 | + prepare_mlx_serve_export( | |
| 186 | + store=store, | |
| 187 | + spec=_SPEC, | |
| 188 | + adapter_name=None, | |
| 189 | + adapter_path_override=None, | |
| 190 | + declared_adapter_names=None, | |
| 191 | + ) | |
| 192 | + | |
| 142 | 193 | |
| 143 | 194 | class TestMlxServeSmoke: |
| 144 | 195 | def test_smoke_uses_absolute_runtime_paths(self, tmp_path: Path, monkeypatch: object) -> None: |
@@ -171,3 +222,43 @@ class TestMlxServeSmoke: | ||
| 171 | 222 | assert "$SCRIPT_DIR" not in " ".join(argv) |
| 172 | 223 | assert _SPEC.hf_id in argv |
| 173 | 224 | assert str(prepared.export_dir / "adapter") in argv |
| 225 | + | |
| 226 | + def test_smoke_failure_returns_failed_result(self, tmp_path: Path, monkeypatch: object) -> None: | |
| 227 | + store = _setup_flat_store(tmp_path) | |
| 228 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.is_apple_silicon", lambda: True) | |
| 229 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.mlx_available", lambda: True) | |
| 230 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.stage_mlx_adapter_dir", _fake_stage_mlx) | |
| 231 | + prepared = prepare_mlx_serve_export( | |
| 232 | + store=store, | |
| 233 | + spec=_SPEC, | |
| 234 | + adapter_name=None, | |
| 235 | + adapter_path_override=None, | |
| 236 | + declared_adapter_names=None, | |
| 237 | + ) | |
| 238 | + | |
| 239 | + def _fake_smoke(argv: list[str], **_: object) -> str: | |
| 240 | + _ = argv | |
| 241 | + raise TargetSmokeError("boom") | |
| 242 | + | |
| 243 | + monkeypatch.setattr("dlm.export.targets.mlx_serve.smoke_openai_compat_server", _fake_smoke) | |
| 244 | + | |
| 245 | + result = MLX_SERVE_TARGET.smoke_test(prepared) | |
| 246 | + | |
| 247 | + assert result.attempted is True | |
| 248 | + assert result.ok is False | |
| 249 | + assert result.detail == "boom" | |
| 250 | + | |
| 251 | + | |
| 252 | +class TestMlxServeHelpers: | |
| 253 | + def test_quote_script_arg_and_int_validation(self) -> None: | |
| 254 | + assert _quote_script_arg("$SCRIPT_DIR/adapter") == '"$SCRIPT_DIR/adapter"' | |
| 255 | + assert _quote_script_arg("plain value") == "'plain value'" | |
| 256 | + | |
| 257 | + prepared = TargetResult( | |
| 258 | + name="mlx-serve", | |
| 259 | + export_dir=Path("/tmp/export"), | |
| 260 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 261 | + extras={"adapter_version": "bad"}, | |
| 262 | + ) | |
| 263 | + with pytest.raises(ExportError, match="missing int extra"): | |
| 264 | + _require_prepared_int(prepared, "adapter_version") | |
tests/unit/export/targets/test_vllm_argv.pymodified@@ -5,11 +5,23 @@ from __future__ import annotations | ||
| 5 | 5 | import json |
| 6 | 6 | from pathlib import Path |
| 7 | 7 | |
| 8 | +import pytest | |
| 9 | + | |
| 8 | 10 | from dlm.base_models import BASE_MODELS |
| 11 | +from dlm.export.errors import ExportError, TargetSmokeError | |
| 9 | 12 | from dlm.export.manifest import load_export_manifest |
| 13 | +from dlm.export.targets.base import TargetResult | |
| 10 | 14 | from dlm.export.targets.vllm import ( |
| 11 | 15 | VLLM_CONFIG_FILENAME, |
| 12 | 16 | VLLM_TARGET, |
| 17 | + LoraModule, | |
| 18 | + _default_runtime_env, | |
| 19 | + _optional_prepared_int, | |
| 20 | + _render_launch_script, | |
| 21 | + _require_module_specs, | |
| 22 | + _require_prepared_int, | |
| 23 | + _require_prepared_str, | |
| 24 | + _runtime_env, | |
| 13 | 25 | finalize_vllm_export, |
| 14 | 26 | prepare_vllm_export, |
| 15 | 27 | ) |
@@ -53,6 +65,10 @@ def _setup_named_store(tmp_path: Path) -> object: | ||
| 53 | 65 | |
| 54 | 66 | |
| 55 | 67 | class TestPrepareVllmExport: |
| 68 | + def test_prepare_method_is_not_used_directly(self) -> None: | |
| 69 | + with pytest.raises(NotImplementedError, match="prepare_vllm_export"): | |
| 70 | + VLLM_TARGET.prepare(object()) | |
| 71 | + | |
| 56 | 72 | def test_flat_export_writes_config_manifest_and_launch_script(self, tmp_path: Path) -> None: |
| 57 | 73 | store = _setup_flat_store(tmp_path) |
| 58 | 74 | |
@@ -197,6 +213,81 @@ class TestPrepareVllmExport: | ||
| 197 | 213 | "VLLM_METAL_USE_PAGED_ATTENTION": "0", |
| 198 | 214 | } |
| 199 | 215 | |
| 216 | + def test_prepare_requires_at_least_one_module( | |
| 217 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 218 | + ) -> None: | |
| 219 | + store = _setup_flat_store(tmp_path) | |
| 220 | + monkeypatch.setattr("dlm.export.targets.vllm._stage_modules", lambda **kwargs: []) | |
| 221 | + | |
| 222 | + with pytest.raises(ExportError, match="at least one adapter module"): | |
| 223 | + prepare_vllm_export( | |
| 224 | + store=store, | |
| 225 | + spec=_SPEC, | |
| 226 | + served_model_name="dlm-flat", | |
| 227 | + training_sequence_len=2048, | |
| 228 | + adapter_name=None, | |
| 229 | + adapter_path_override=None, | |
| 230 | + declared_adapter_names=None, | |
| 231 | + ) | |
| 232 | + | |
| 233 | + def test_missing_named_adapter_raises(self, tmp_path: Path) -> None: | |
| 234 | + store = _setup_named_store(tmp_path) | |
| 235 | + | |
| 236 | + with pytest.raises(ExportError, match="no current adapter under"): | |
| 237 | + prepare_vllm_export( | |
| 238 | + store=store, | |
| 239 | + spec=_SPEC, | |
| 240 | + served_model_name="dlm-missing", | |
| 241 | + training_sequence_len=2048, | |
| 242 | + adapter_name="missing", | |
| 243 | + adapter_path_override=None, | |
| 244 | + declared_adapter_names=None, | |
| 245 | + ) | |
| 246 | + | |
| 247 | + def test_missing_default_adapter_raises(self, tmp_path: Path) -> None: | |
| 248 | + store = for_dlm("01EMPTYVLLM", home=tmp_path) | |
| 249 | + store.ensure_layout() | |
| 250 | + save_manifest(store.manifest, Manifest(dlm_id="01EMPTYVLLM", base_model=_SPEC.key)) | |
| 251 | + | |
| 252 | + with pytest.raises(ExportError, match="no current adapter under"): | |
| 253 | + prepare_vllm_export( | |
| 254 | + store=store, | |
| 255 | + spec=_SPEC, | |
| 256 | + served_model_name="dlm-empty", | |
| 257 | + training_sequence_len=2048, | |
| 258 | + adapter_name=None, | |
| 259 | + adapter_path_override=None, | |
| 260 | + declared_adapter_names=None, | |
| 261 | + ) | |
| 262 | + | |
| 263 | + def test_missing_declared_named_adapter_raises(self, tmp_path: Path) -> None: | |
| 264 | + store = _setup_named_store(tmp_path) | |
| 265 | + | |
| 266 | + with pytest.raises(ExportError, match="no current adapter under"): | |
| 267 | + prepare_vllm_export( | |
| 268 | + store=store, | |
| 269 | + spec=_SPEC, | |
| 270 | + served_model_name="dlm-multi", | |
| 271 | + training_sequence_len=2048, | |
| 272 | + adapter_name=None, | |
| 273 | + adapter_path_override=None, | |
| 274 | + declared_adapter_names=("knowledge", "missing"), | |
| 275 | + ) | |
| 276 | + | |
| 277 | + def test_missing_adapter_override_raises(self, tmp_path: Path) -> None: | |
| 278 | + store = _setup_flat_store(tmp_path) | |
| 279 | + | |
| 280 | + with pytest.raises(ExportError, match="adapter_path_override"): | |
| 281 | + prepare_vllm_export( | |
| 282 | + store=store, | |
| 283 | + spec=_SPEC, | |
| 284 | + served_model_name="dlm-mixed", | |
| 285 | + training_sequence_len=2048, | |
| 286 | + adapter_name=None, | |
| 287 | + adapter_path_override=tmp_path / "missing", | |
| 288 | + declared_adapter_names=None, | |
| 289 | + ) | |
| 290 | + | |
| 200 | 291 | |
| 201 | 292 | class TestVllmSmoke: |
| 202 | 293 | def test_smoke_uses_absolute_runtime_paths(self, tmp_path: Path, monkeypatch: object) -> None: |
@@ -237,3 +328,130 @@ class TestVllmSmoke: | ||
| 237 | 328 | } |
| 238 | 329 | assert f"knowledge={prepared.export_dir / 'adapters' / 'knowledge'}" in argv |
| 239 | 330 | assert f"tone={prepared.export_dir / 'adapters' / 'tone'}" in argv |
| 331 | + | |
| 332 | + def test_smoke_failure_returns_failed_result( | |
| 333 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 334 | + ) -> None: | |
| 335 | + store = _setup_flat_store(tmp_path) | |
| 336 | + prepared = prepare_vllm_export( | |
| 337 | + store=store, | |
| 338 | + spec=_SPEC, | |
| 339 | + served_model_name="dlm-flat", | |
| 340 | + training_sequence_len=2048, | |
| 341 | + adapter_name=None, | |
| 342 | + adapter_path_override=None, | |
| 343 | + declared_adapter_names=None, | |
| 344 | + ) | |
| 345 | + monkeypatch.setattr( | |
| 346 | + "dlm.export.targets.vllm.smoke_openai_compat_server", | |
| 347 | + lambda argv, **kwargs: (_ for _ in ()).throw(TargetSmokeError("boom")), | |
| 348 | + ) | |
| 349 | + | |
| 350 | + result = VLLM_TARGET.smoke_test(prepared) | |
| 351 | + | |
| 352 | + assert result.attempted is True | |
| 353 | + assert result.ok is False | |
| 354 | + assert result.detail == "boom" | |
| 355 | + | |
| 356 | + | |
| 357 | +class TestVllmHelpers: | |
| 358 | + def test_default_runtime_env_is_empty_off_apple_silicon( | |
| 359 | + self, monkeypatch: pytest.MonkeyPatch | |
| 360 | + ) -> None: | |
| 361 | + monkeypatch.setattr("dlm.export.targets.vllm._sys_platform", lambda: "linux") | |
| 362 | + monkeypatch.setattr("dlm.export.targets.vllm._machine", lambda: "x86_64") | |
| 363 | + | |
| 364 | + assert _default_runtime_env() == {} | |
| 365 | + | |
| 366 | + def test_runtime_env_validation(self) -> None: | |
| 367 | + empty = TargetResult( | |
| 368 | + name="vllm", | |
| 369 | + export_dir=Path("/tmp/export"), | |
| 370 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 371 | + ) | |
| 372 | + assert _runtime_env(empty) == {} | |
| 373 | + | |
| 374 | + prepared = TargetResult( | |
| 375 | + name="vllm", | |
| 376 | + export_dir=Path("/tmp/export"), | |
| 377 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 378 | + extras={"runtime_env": {"A": "B"}}, | |
| 379 | + ) | |
| 380 | + assert _runtime_env(prepared) == {"A": "B"} | |
| 381 | + | |
| 382 | + bad = TargetResult( | |
| 383 | + name="vllm", | |
| 384 | + export_dir=Path("/tmp/export"), | |
| 385 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 386 | + extras={"runtime_env": {"A": 3}}, | |
| 387 | + ) | |
| 388 | + with pytest.raises(ExportError, match="dict\\[str, str\\]"): | |
| 389 | + _runtime_env(bad) | |
| 390 | + | |
| 391 | + def test_optional_int_and_module_validation(self) -> None: | |
| 392 | + prepared = TargetResult( | |
| 393 | + name="vllm", | |
| 394 | + export_dir=Path("/tmp/export"), | |
| 395 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 396 | + extras={"context_length": None}, | |
| 397 | + ) | |
| 398 | + assert _optional_prepared_int(prepared, "context_length") is None | |
| 399 | + | |
| 400 | + bad_int = TargetResult( | |
| 401 | + name="vllm", | |
| 402 | + export_dir=Path("/tmp/export"), | |
| 403 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 404 | + extras={"context_length": "bad"}, | |
| 405 | + ) | |
| 406 | + with pytest.raises(ExportError, match="must be an int"): | |
| 407 | + _optional_prepared_int(bad_int, "context_length") | |
| 408 | + | |
| 409 | + bad_modules = TargetResult( | |
| 410 | + name="vllm", | |
| 411 | + export_dir=Path("/tmp/export"), | |
| 412 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 413 | + extras={"module_specs": ("bad",)}, | |
| 414 | + ) | |
| 415 | + with pytest.raises(ExportError, match="LoraModule tuple"): | |
| 416 | + _require_module_specs(bad_modules) | |
| 417 | + | |
| 418 | + good_modules = TargetResult( | |
| 419 | + name="vllm", | |
| 420 | + export_dir=Path("/tmp/export"), | |
| 421 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 422 | + extras={"module_specs": (LoraModule("adapter", Path("/tmp/a"), 1),)}, | |
| 423 | + ) | |
| 424 | + assert _require_module_specs(good_modules)[0].name == "adapter" | |
| 425 | + | |
| 426 | + bad_str = TargetResult( | |
| 427 | + name="vllm", | |
| 428 | + export_dir=Path("/tmp/export"), | |
| 429 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 430 | + extras={"model": ""}, | |
| 431 | + ) | |
| 432 | + with pytest.raises(ExportError, match="missing string extra"): | |
| 433 | + _require_prepared_str(bad_str, "model") | |
| 434 | + | |
| 435 | + bad_required_int = TargetResult( | |
| 436 | + name="vllm", | |
| 437 | + export_dir=Path("/tmp/export"), | |
| 438 | + manifest_path=Path("/tmp/export/export_manifest.json"), | |
| 439 | + extras={"context_length": "bad"}, | |
| 440 | + ) | |
| 441 | + with pytest.raises(ExportError, match="missing int extra"): | |
| 442 | + _require_prepared_int(bad_required_int, "context_length") | |
| 443 | + | |
| 444 | + def test_render_launch_script_quotes_inline_script_dir_modules(self) -> None: | |
| 445 | + rendered = _render_launch_script( | |
| 446 | + [ | |
| 447 | + "vllm", | |
| 448 | + "serve", | |
| 449 | + "model", | |
| 450 | + "knowledge=$SCRIPT_DIR/adapters/knowledge", | |
| 451 | + "$SCRIPT_DIR/direct", | |
| 452 | + ], | |
| 453 | + {}, | |
| 454 | + ) | |
| 455 | + | |
| 456 | + assert 'knowledge="$SCRIPT_DIR/adapters/knowledge"' in rendered | |
| 457 | + assert '"$SCRIPT_DIR/direct"' in rendered | |
tests/unit/export/test_dispatch.pyadded@@ -0,0 +1,340 @@ | ||
| 1 | +"""Unit coverage for modality-aware export dispatch.""" | |
| 2 | + | |
| 3 | +from __future__ import annotations | |
| 4 | + | |
| 5 | +from pathlib import Path | |
| 6 | +from types import SimpleNamespace | |
| 7 | + | |
| 8 | +import pytest | |
| 9 | + | |
| 10 | +from dlm.base_models import BASE_MODELS | |
| 11 | +from dlm.export.arch_probe import ArchProbeResult, SupportLevel | |
| 12 | +from dlm.export.dispatch import ( | |
| 13 | + DispatchResult, | |
| 14 | + _load_processor_or_raise, | |
| 15 | + dispatch_audio_export, | |
| 16 | + dispatch_vl_export, | |
| 17 | + emit_vl_snapshot, | |
| 18 | +) | |
| 19 | +from dlm.export.errors import ( | |
| 20 | + ExportError, | |
| 21 | + ProcessorLoadError, | |
| 22 | + VendoringError, | |
| 23 | + VlGgufUnsupportedError, | |
| 24 | +) | |
| 25 | + | |
| 26 | +_VL_SPEC = BASE_MODELS["qwen2-vl-2b-instruct"] | |
| 27 | +_AUDIO_SPEC = BASE_MODELS["qwen2-audio-7b-instruct"] | |
| 28 | + | |
| 29 | + | |
| 30 | +def _snapshot_result(tmp_path: Path, dirname: str) -> object: | |
| 31 | + export_dir = tmp_path / dirname | |
| 32 | + export_dir.mkdir(parents=True, exist_ok=True) | |
| 33 | + manifest_path = export_dir / "export_manifest.json" | |
| 34 | + manifest_path.write_text("{}", encoding="utf-8") | |
| 35 | + adapter_dir = export_dir / "adapter" | |
| 36 | + adapter_dir.mkdir() | |
| 37 | + artifact = export_dir / "artifact.txt" | |
| 38 | + artifact.write_text("ok", encoding="utf-8") | |
| 39 | + return SimpleNamespace( | |
| 40 | + export_dir=export_dir, | |
| 41 | + manifest_path=manifest_path, | |
| 42 | + adapter_dir=adapter_dir, | |
| 43 | + artifacts=[artifact], | |
| 44 | + ) | |
| 45 | + | |
| 46 | + | |
| 47 | +class TestLoadProcessorOrRaise: | |
| 48 | + def test_wraps_loader_errors(self, monkeypatch: pytest.MonkeyPatch) -> None: | |
| 49 | + monkeypatch.setattr( | |
| 50 | + "dlm.train.loader.load_processor", | |
| 51 | + lambda spec: (_ for _ in ()).throw(RuntimeError("missing cache")), | |
| 52 | + ) | |
| 53 | + | |
| 54 | + with pytest.raises(ProcessorLoadError, match="missing cache"): | |
| 55 | + _load_processor_or_raise(_VL_SPEC) | |
| 56 | + | |
| 57 | + | |
| 58 | +class TestEmitVlSnapshot: | |
| 59 | + def test_emits_snapshot_and_warns_about_gguf_only_flags( | |
| 60 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 61 | + ) -> None: | |
| 62 | + processor = object() | |
| 63 | + monkeypatch.setattr("dlm.train.loader.load_processor", lambda spec: processor) | |
| 64 | + monkeypatch.setattr( | |
| 65 | + "dlm.export.vl_snapshot.run_vl_snapshot_export", | |
| 66 | + lambda store, spec, *, adapter_name, processor: _snapshot_result( | |
| 67 | + tmp_path, "vl-snapshot" | |
| 68 | + ), | |
| 69 | + ) | |
| 70 | + | |
| 71 | + result = emit_vl_snapshot( | |
| 72 | + store=object(), | |
| 73 | + spec=_VL_SPEC, | |
| 74 | + adapter_name="named", | |
| 75 | + quant="Q4_K_M", | |
| 76 | + merged=True, | |
| 77 | + adapter_mix_raw="tone:0.5", | |
| 78 | + ) | |
| 79 | + | |
| 80 | + assert result.extras["path"] == "hf-snapshot" | |
| 81 | + assert any("ignoring GGUF-only flags" in line for line in result.banner_lines) | |
| 82 | + assert any("HF snapshot written" in line for line in result.banner_lines) | |
| 83 | + | |
| 84 | + def test_skip_warning_suppresses_flag_banner( | |
| 85 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 86 | + ) -> None: | |
| 87 | + monkeypatch.setattr("dlm.train.loader.load_processor", lambda spec: object()) | |
| 88 | + monkeypatch.setattr( | |
| 89 | + "dlm.export.vl_snapshot.run_vl_snapshot_export", | |
| 90 | + lambda store, spec, *, adapter_name, processor: _snapshot_result( | |
| 91 | + tmp_path, "vl-snapshot-skip-warning" | |
| 92 | + ), | |
| 93 | + ) | |
| 94 | + | |
| 95 | + result = emit_vl_snapshot( | |
| 96 | + store=object(), | |
| 97 | + spec=_VL_SPEC, | |
| 98 | + adapter_name=None, | |
| 99 | + quant="Q4_K_M", | |
| 100 | + merged=True, | |
| 101 | + adapter_mix_raw="tone:0.5", | |
| 102 | + skip_gguf_flag_warning=True, | |
| 103 | + ) | |
| 104 | + | |
| 105 | + assert not any("ignoring GGUF-only flags" in line for line in result.banner_lines) | |
| 106 | + | |
| 107 | + | |
| 108 | +class TestDispatchVlExport: | |
| 109 | + def test_probe_vendoring_failure_falls_back_to_snapshot( | |
| 110 | + self, monkeypatch: pytest.MonkeyPatch | |
| 111 | + ) -> None: | |
| 112 | + expected = DispatchResult( | |
| 113 | + export_dir=Path("/tmp/vl"), | |
| 114 | + manifest_path=Path("/tmp/vl/export_manifest.json"), | |
| 115 | + artifacts=[], | |
| 116 | + banner_lines=["snapshot"], | |
| 117 | + extras={"path": "hf-snapshot"}, | |
| 118 | + ) | |
| 119 | + monkeypatch.setattr( | |
| 120 | + "dlm.export.arch_probe.probe_gguf_arch", | |
| 121 | + lambda architecture: (_ for _ in ()).throw(VendoringError("missing submodule")), | |
| 122 | + ) | |
| 123 | + monkeypatch.setattr("dlm.export.dispatch.emit_vl_snapshot", lambda **kwargs: expected) | |
| 124 | + | |
| 125 | + result = dispatch_vl_export( | |
| 126 | + store=object(), | |
| 127 | + spec=_VL_SPEC, | |
| 128 | + adapter_name=None, | |
| 129 | + quant=None, | |
| 130 | + merged=False, | |
| 131 | + adapter_mix_raw=None, | |
| 132 | + ) | |
| 133 | + | |
| 134 | + assert result.banner_lines[0].startswith( | |
| 135 | + "[yellow]export:[/yellow] llama.cpp probe unavailable" | |
| 136 | + ) | |
| 137 | + assert result.banner_lines[-1] == "snapshot" | |
| 138 | + | |
| 139 | + @pytest.mark.parametrize( | |
| 140 | + ("support", "expected_text"), | |
| 141 | + [ | |
| 142 | + (SupportLevel.UNSUPPORTED, "is not covered by the vendored llama.cpp"), | |
| 143 | + (SupportLevel.PARTIAL, "has PARTIAL llama.cpp coverage"), | |
| 144 | + ], | |
| 145 | + ) | |
| 146 | + def test_unsupported_or_partial_verdicts_fall_back_to_snapshot( | |
| 147 | + self, | |
| 148 | + monkeypatch: pytest.MonkeyPatch, | |
| 149 | + support: SupportLevel, | |
| 150 | + expected_text: str, | |
| 151 | + ) -> None: | |
| 152 | + verdict = ArchProbeResult( | |
| 153 | + arch_class=_VL_SPEC.architecture, | |
| 154 | + support=support, | |
| 155 | + reason="probe result", | |
| 156 | + llama_cpp_tag="b1234", | |
| 157 | + ) | |
| 158 | + expected = DispatchResult( | |
| 159 | + export_dir=Path("/tmp/vl"), | |
| 160 | + manifest_path=Path("/tmp/vl/export_manifest.json"), | |
| 161 | + artifacts=[], | |
| 162 | + banner_lines=["snapshot"], | |
| 163 | + extras={"path": "hf-snapshot"}, | |
| 164 | + ) | |
| 165 | + monkeypatch.setattr("dlm.export.arch_probe.probe_gguf_arch", lambda architecture: verdict) | |
| 166 | + monkeypatch.setattr("dlm.export.dispatch.emit_vl_snapshot", lambda **kwargs: expected) | |
| 167 | + | |
| 168 | + result = dispatch_vl_export( | |
| 169 | + store=object(), | |
| 170 | + spec=_VL_SPEC, | |
| 171 | + adapter_name=None, | |
| 172 | + quant=None, | |
| 173 | + merged=False, | |
| 174 | + adapter_mix_raw=None, | |
| 175 | + ) | |
| 176 | + | |
| 177 | + assert expected_text in result.banner_lines[0] | |
| 178 | + assert result.banner_lines[-1] == "snapshot" | |
| 179 | + | |
| 180 | + def test_supported_without_context_falls_back_to_snapshot( | |
| 181 | + self, monkeypatch: pytest.MonkeyPatch | |
| 182 | + ) -> None: | |
| 183 | + verdict = ArchProbeResult( | |
| 184 | + arch_class=_VL_SPEC.architecture, | |
| 185 | + support=SupportLevel.SUPPORTED, | |
| 186 | + reason="probe result", | |
| 187 | + llama_cpp_tag="b1234", | |
| 188 | + ) | |
| 189 | + expected = DispatchResult( | |
| 190 | + export_dir=Path("/tmp/vl"), | |
| 191 | + manifest_path=Path("/tmp/vl/export_manifest.json"), | |
| 192 | + artifacts=[], | |
| 193 | + banner_lines=["snapshot"], | |
| 194 | + extras={"path": "hf-snapshot"}, | |
| 195 | + ) | |
| 196 | + monkeypatch.setattr("dlm.export.arch_probe.probe_gguf_arch", lambda architecture: verdict) | |
| 197 | + monkeypatch.setattr("dlm.export.dispatch.emit_vl_snapshot", lambda **kwargs: expected) | |
| 198 | + | |
| 199 | + result = dispatch_vl_export( | |
| 200 | + store=object(), | |
| 201 | + spec=_VL_SPEC, | |
| 202 | + adapter_name="named", | |
| 203 | + quant="Q4_K_M", | |
| 204 | + merged=False, | |
| 205 | + adapter_mix_raw=None, | |
| 206 | + gguf_emission_context=None, | |
| 207 | + ) | |
| 208 | + | |
| 209 | + assert "without GGUF plan context" in result.banner_lines[0] | |
| 210 | + | |
| 211 | + def test_supported_verdict_returns_vl_gguf_result( | |
| 212 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 213 | + ) -> None: | |
| 214 | + verdict = ArchProbeResult( | |
| 215 | + arch_class=_VL_SPEC.architecture, | |
| 216 | + support=SupportLevel.SUPPORTED, | |
| 217 | + reason="probe result", | |
| 218 | + llama_cpp_tag="b5678", | |
| 219 | + ) | |
| 220 | + export_dir = tmp_path / "vl-gguf" | |
| 221 | + export_dir.mkdir() | |
| 222 | + manifest_path = export_dir / "export_manifest.json" | |
| 223 | + manifest_path.write_text("{}", encoding="utf-8") | |
| 224 | + gguf_path = export_dir / "model.gguf" | |
| 225 | + gguf_path.write_bytes(b"gguf") | |
| 226 | + modelfile_path = export_dir / "Modelfile" | |
| 227 | + modelfile_path.write_text("FROM base", encoding="utf-8") | |
| 228 | + mmproj_path = export_dir / "mmproj.gguf" | |
| 229 | + mmproj_path.write_bytes(b"mmproj") | |
| 230 | + | |
| 231 | + monkeypatch.setattr("dlm.export.arch_probe.probe_gguf_arch", lambda architecture: verdict) | |
| 232 | + monkeypatch.setattr( | |
| 233 | + "dlm.export.vl_gguf.run_vl_gguf_export", | |
| 234 | + lambda *args, **kwargs: SimpleNamespace( | |
| 235 | + export_dir=export_dir, | |
| 236 | + manifest_path=manifest_path, | |
| 237 | + gguf_path=gguf_path, | |
| 238 | + modelfile_path=modelfile_path, | |
| 239 | + mmproj_path=mmproj_path, | |
| 240 | + quant="Q4_K_M", | |
| 241 | + llama_cpp_tag="b5678", | |
| 242 | + artifacts=[gguf_path, modelfile_path], | |
| 243 | + ), | |
| 244 | + ) | |
| 245 | + | |
| 246 | + result = dispatch_vl_export( | |
| 247 | + store=object(), | |
| 248 | + spec=_VL_SPEC, | |
| 249 | + adapter_name=None, | |
| 250 | + quant="Q4_K_M", | |
| 251 | + merged=True, | |
| 252 | + adapter_mix_raw=None, | |
| 253 | + gguf_emission_context={ | |
| 254 | + "plan": object(), | |
| 255 | + "cached_base_dir": tmp_path / "cache", | |
| 256 | + "source_dlm_path": tmp_path / "doc.dlm", | |
| 257 | + "dlm_version": "test", | |
| 258 | + "training_sequence_len": 1024, | |
| 259 | + }, | |
| 260 | + ) | |
| 261 | + | |
| 262 | + assert result.extras["path"] == "vl-gguf" | |
| 263 | + assert result.extras["gguf_path"] == gguf_path | |
| 264 | + assert any( | |
| 265 | + "attempting single-file VL GGUF emission" in line for line in result.banner_lines | |
| 266 | + ) | |
| 267 | + assert any("VL GGUF written" in line for line in result.banner_lines) | |
| 268 | + | |
| 269 | + @pytest.mark.parametrize( | |
| 270 | + "error", | |
| 271 | + [ | |
| 272 | + VlGgufUnsupportedError("plan refused"), | |
| 273 | + VendoringError("missing binary"), | |
| 274 | + ExportError("subprocess failed"), | |
| 275 | + ], | |
| 276 | + ) | |
| 277 | + def test_supported_verdict_falls_back_after_gguf_failure( | |
| 278 | + self, monkeypatch: pytest.MonkeyPatch, error: Exception | |
| 279 | + ) -> None: | |
| 280 | + verdict = ArchProbeResult( | |
| 281 | + arch_class=_VL_SPEC.architecture, | |
| 282 | + support=SupportLevel.SUPPORTED, | |
| 283 | + reason="probe result", | |
| 284 | + llama_cpp_tag="b1234", | |
| 285 | + ) | |
| 286 | + expected = DispatchResult( | |
| 287 | + export_dir=Path("/tmp/vl"), | |
| 288 | + manifest_path=Path("/tmp/vl/export_manifest.json"), | |
| 289 | + artifacts=[], | |
| 290 | + banner_lines=["snapshot"], | |
| 291 | + extras={"path": "hf-snapshot"}, | |
| 292 | + ) | |
| 293 | + monkeypatch.setattr("dlm.export.arch_probe.probe_gguf_arch", lambda architecture: verdict) | |
| 294 | + monkeypatch.setattr( | |
| 295 | + "dlm.export.vl_gguf.run_vl_gguf_export", | |
| 296 | + lambda *args, **kwargs: (_ for _ in ()).throw(error), | |
| 297 | + ) | |
| 298 | + monkeypatch.setattr("dlm.export.dispatch.emit_vl_snapshot", lambda **kwargs: expected) | |
| 299 | + | |
| 300 | + result = dispatch_vl_export( | |
| 301 | + store=object(), | |
| 302 | + spec=_VL_SPEC, | |
| 303 | + adapter_name=None, | |
| 304 | + quant="Q4_K_M", | |
| 305 | + merged=True, | |
| 306 | + adapter_mix_raw=None, | |
| 307 | + gguf_emission_context={ | |
| 308 | + "plan": object(), | |
| 309 | + "cached_base_dir": Path("/tmp/cache"), | |
| 310 | + }, | |
| 311 | + ) | |
| 312 | + | |
| 313 | + assert "falling back to HF-snapshot" in "\n".join(result.banner_lines) | |
| 314 | + | |
| 315 | + | |
| 316 | +class TestDispatchAudioExport: | |
| 317 | + def test_audio_export_uses_snapshot_path( | |
| 318 | + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch | |
| 319 | + ) -> None: | |
| 320 | + monkeypatch.setattr("dlm.train.loader.load_processor", lambda spec: object()) | |
| 321 | + monkeypatch.setattr( | |
| 322 | + "dlm.export.audio_snapshot.run_audio_snapshot_export", | |
| 323 | + lambda store, spec, *, adapter_name, processor: _snapshot_result( | |
| 324 | + tmp_path, "audio-snapshot" | |
| 325 | + ), | |
| 326 | + ) | |
| 327 | + | |
| 328 | + result = dispatch_audio_export( | |
| 329 | + store=object(), | |
| 330 | + spec=_AUDIO_SPEC, | |
| 331 | + adapter_name="named", | |
| 332 | + quant="Q4_K_M", | |
| 333 | + merged=True, | |
| 334 | + adapter_mix_raw="tone:0.5", | |
| 335 | + ) | |
| 336 | + | |
| 337 | + assert result.extras["path"] == "audio-snapshot" | |
| 338 | + assert any("audio-language" in line for line in result.banner_lines) | |
| 339 | + assert any("ignoring GGUF-only flags" in line for line in result.banner_lines) | |
| 340 | + assert any("HF audio snapshot written" in line for line in result.banner_lines) | |
tests/unit/export/test_smoke.pymodified@@ -1,142 +1,378 @@ | ||
| 1 | -"""Shared OpenAI-compatible smoke harness.""" | |
| 1 | +"""Deterministic unit coverage for the shared OpenAI-compatible smoke helper.""" | |
| 2 | 2 | |
| 3 | 3 | from __future__ import annotations |
| 4 | 4 | |
| 5 | -import socket | |
| 6 | -import sys | |
| 7 | -from pathlib import Path | |
| 5 | +import io | |
| 6 | +import json | |
| 7 | +import subprocess | |
| 8 | +import urllib.error | |
| 9 | +from collections.abc import Callable, Iterator | |
| 8 | 10 | |
| 9 | 11 | import pytest |
| 10 | 12 | |
| 13 | +from dlm.export import smoke as smoke_mod | |
| 11 | 14 | from dlm.export.errors import TargetSmokeError |
| 12 | -from dlm.export.smoke import smoke_openai_compat_server | |
| 13 | - | |
| 14 | - | |
| 15 | -def _require_loopback_bind() -> None: | |
| 16 | - try: | |
| 17 | - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: | |
| 18 | - sock.bind(("127.0.0.1", 0)) | |
| 19 | - except PermissionError as exc: | |
| 20 | - pytest.skip(f"loopback bind blocked on this host: {exc}") | |
| 21 | - | |
| 22 | - | |
| 23 | -def _write_server_script(tmp_path: Path, *, mode: str) -> Path: | |
| 24 | - script = tmp_path / f"fake_server_{mode}.py" | |
| 25 | - script.write_text( | |
| 26 | - ( | |
| 27 | - "from __future__ import annotations\n" | |
| 28 | - "import argparse\n" | |
| 29 | - "import os\n" | |
| 30 | - "import json\n" | |
| 31 | - "from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer\n" | |
| 32 | - "\n" | |
| 33 | - "parser = argparse.ArgumentParser()\n" | |
| 34 | - "parser.add_argument('--host', required=True)\n" | |
| 35 | - "parser.add_argument('--port', required=True, type=int)\n" | |
| 36 | - "parser.add_argument('--mode', required=True)\n" | |
| 37 | - "args = parser.parse_args()\n" | |
| 38 | - "\n" | |
| 39 | - "if args.mode == 'exit':\n" | |
| 40 | - " raise SystemExit(3)\n" | |
| 41 | - "if args.mode == 'env' and os.environ.get('FAKE_SMOKE_TOKEN') != 'ready':\n" | |
| 42 | - " raise SystemExit(4)\n" | |
| 43 | - "\n" | |
| 44 | - "class Handler(BaseHTTPRequestHandler):\n" | |
| 45 | - " def do_GET(self) -> None:\n" | |
| 46 | - " if self.path != '/v1/models':\n" | |
| 47 | - " self.send_response(404)\n" | |
| 48 | - " self.end_headers()\n" | |
| 49 | - " return\n" | |
| 50 | - " body = json.dumps({'data': [{'id': 'fake-model'}]}).encode('utf-8')\n" | |
| 51 | - " self.send_response(200)\n" | |
| 52 | - " self.send_header('Content-Type', 'application/json')\n" | |
| 53 | - " self.send_header('Content-Length', str(len(body)))\n" | |
| 54 | - " self.end_headers()\n" | |
| 55 | - " self.wfile.write(body)\n" | |
| 56 | - "\n" | |
| 57 | - " def do_POST(self) -> None:\n" | |
| 58 | - " if self.path != '/v1/chat/completions':\n" | |
| 59 | - " self.send_response(404)\n" | |
| 60 | - " self.end_headers()\n" | |
| 61 | - " return\n" | |
| 62 | - " _ = self.rfile.read(int(self.headers.get('Content-Length', '0')))\n" | |
| 63 | - " if args.mode == 'empty':\n" | |
| 64 | - " payload = {'choices': [{'message': {'content': ''}}]}\n" | |
| 65 | - " else:\n" | |
| 66 | - " payload = {'choices': [{'message': {'content': 'hello from fake server'}}]}\n" | |
| 67 | - " body = json.dumps(payload).encode('utf-8')\n" | |
| 68 | - " self.send_response(200)\n" | |
| 69 | - " self.send_header('Content-Type', 'application/json')\n" | |
| 70 | - " self.send_header('Content-Length', str(len(body)))\n" | |
| 71 | - " self.end_headers()\n" | |
| 72 | - " self.wfile.write(body)\n" | |
| 73 | - "\n" | |
| 74 | - " def log_message(self, format: str, *args: object) -> None:\n" | |
| 75 | - " return\n" | |
| 76 | - "\n" | |
| 77 | - "server = ThreadingHTTPServer((args.host, args.port), Handler)\n" | |
| 78 | - "server.serve_forever()\n" | |
| 79 | - ), | |
| 80 | - encoding="utf-8", | |
| 81 | - ) | |
| 82 | - return script | |
| 15 | + | |
| 16 | + | |
| 17 | +class _FakeProc: | |
| 18 | + def __init__(self, *, returncode: int | None = None, kill_times_out: bool = False) -> None: | |
| 19 | + self.returncode = returncode | |
| 20 | + self.kill_times_out = kill_times_out | |
| 21 | + self.terminated = False | |
| 22 | + self.killed = False | |
| 23 | + self.wait_calls = 0 | |
| 24 | + | |
| 25 | + def poll(self) -> int | None: | |
| 26 | + return self.returncode | |
| 27 | + | |
| 28 | + def terminate(self) -> None: | |
| 29 | + self.terminated = True | |
| 30 | + | |
| 31 | + def wait(self, timeout: float) -> None: | |
| 32 | + self.wait_calls += 1 | |
| 33 | + if self.kill_times_out and self.wait_calls == 1: | |
| 34 | + raise subprocess.TimeoutExpired(cmd="fake", timeout=timeout) | |
| 35 | + | |
| 36 | + def kill(self) -> None: | |
| 37 | + self.killed = True | |
| 38 | + | |
| 39 | + | |
| 40 | +class _FakeResponse: | |
| 41 | + def __init__(self, payload: object) -> None: | |
| 42 | + self._payload = payload | |
| 43 | + | |
| 44 | + def __enter__(self) -> _FakeResponse: | |
| 45 | + return self | |
| 46 | + | |
| 47 | + def __exit__(self, *_exc: object) -> None: | |
| 48 | + return None | |
| 49 | + | |
| 50 | + def read(self) -> bytes: | |
| 51 | + return json.dumps(self._payload).encode("utf-8") | |
| 52 | + | |
| 53 | + | |
| 54 | +def _urlopen_with(payload: object) -> Callable[..., _FakeResponse]: | |
| 55 | + def _fake_urlopen(*_args: object, **_kwargs: object) -> _FakeResponse: | |
| 56 | + return _FakeResponse(payload) | |
| 57 | + | |
| 58 | + return _fake_urlopen | |
| 83 | 59 | |
| 84 | 60 | |
| 85 | 61 | class TestSmokeOpenAiCompatServer: |
| 86 | - def test_returns_first_response_line(self, tmp_path: Path) -> None: | |
| 87 | - _require_loopback_bind() | |
| 88 | - script = _write_server_script(tmp_path, mode="ok") | |
| 62 | + def test_returns_first_response_line(self, monkeypatch: pytest.MonkeyPatch) -> None: | |
| 63 | + popen_argv: list[list[str]] = [] | |
| 64 | + popen_env: list[dict[str, str] | None] = [] | |
| 65 | + stopped: list[_FakeProc] = [] | |
| 89 | 66 | |
| 90 | - first_line = smoke_openai_compat_server( | |
| 91 | - [sys.executable, str(script), "--mode", "ok", "--host", "127.0.0.1", "--port", "8000"] | |
| 67 | + def _fake_popen(argv: list[str], **kwargs: object) -> _FakeProc: | |
| 68 | + popen_argv.append(list(argv)) | |
| 69 | + env = kwargs.get("env") | |
| 70 | + popen_env.append(env if isinstance(env, dict) else None) | |
| 71 | + return _FakeProc() | |
| 72 | + | |
| 73 | + monkeypatch.setattr(smoke_mod, "reserve_local_port", lambda host: 43123) | |
| 74 | + monkeypatch.setattr(smoke_mod.subprocess, "Popen", _fake_popen) | |
| 75 | + monkeypatch.setattr(smoke_mod, "_wait_for_models", lambda *args, **kwargs: "fake-model") | |
| 76 | + monkeypatch.setattr( | |
| 77 | + smoke_mod, | |
| 78 | + "_chat_completion", | |
| 79 | + lambda *args, **kwargs: "\n hello from fake server \nsecond line", | |
| 80 | + ) | |
| 81 | + monkeypatch.setattr(smoke_mod, "_stop_process", lambda proc: stopped.append(proc)) | |
| 82 | + | |
| 83 | + first_line = smoke_mod.smoke_openai_compat_server( | |
| 84 | + ["fake-server", "--mode", "ok", "--host", "0.0.0.0", "--port", "8000"], | |
| 85 | + env={"FAKE_SMOKE_TOKEN": "ready"}, | |
| 92 | 86 | ) |
| 93 | 87 | |
| 94 | 88 | assert first_line == "hello from fake server" |
| 89 | + assert popen_argv == [ | |
| 90 | + ["fake-server", "--mode", "ok", "--host", "127.0.0.1", "--port", "43123"] | |
| 91 | + ] | |
| 92 | + assert popen_env[0] is not None | |
| 93 | + assert popen_env[0]["FAKE_SMOKE_TOKEN"] == "ready" | |
| 94 | + assert len(stopped) == 1 | |
| 95 | 95 | |
| 96 | - def test_empty_content_raises(self, tmp_path: Path) -> None: | |
| 97 | - _require_loopback_bind() | |
| 98 | - script = _write_server_script(tmp_path, mode="empty") | |
| 99 | - | |
| 100 | - with pytest.raises(TargetSmokeError, match="non-empty"): | |
| 101 | - smoke_openai_compat_server( | |
| 102 | - [ | |
| 103 | - sys.executable, | |
| 104 | - str(script), | |
| 105 | - "--mode", | |
| 106 | - "empty", | |
| 107 | - "--host", | |
| 108 | - "127.0.0.1", | |
| 109 | - "--port", | |
| 110 | - "8000", | |
| 111 | - ] | |
| 112 | - ) | |
| 96 | + def test_empty_content_raises(self, monkeypatch: pytest.MonkeyPatch) -> None: | |
| 97 | + monkeypatch.setattr(smoke_mod, "reserve_local_port", lambda host: 42000) | |
| 98 | + monkeypatch.setattr( | |
| 99 | + smoke_mod.subprocess, | |
| 100 | + "Popen", | |
| 101 | + lambda argv, **kwargs: _FakeProc(), | |
| 102 | + ) | |
| 103 | + monkeypatch.setattr(smoke_mod, "_wait_for_models", lambda *args, **kwargs: "fake-model") | |
| 104 | + monkeypatch.setattr(smoke_mod, "_chat_completion", lambda *args, **kwargs: " \n ") | |
| 105 | + monkeypatch.setattr(smoke_mod, "_stop_process", lambda proc: None) | |
| 106 | + | |
| 107 | + with pytest.raises(TargetSmokeError, match="empty assistant content"): | |
| 108 | + smoke_mod.smoke_openai_compat_server(["fake-server"]) | |
| 109 | + | |
| 110 | + def test_retries_dynamic_port_after_target_smoke_error( | |
| 111 | + self, monkeypatch: pytest.MonkeyPatch | |
| 112 | + ) -> None: | |
| 113 | + ports = iter((41001, 41002)) | |
| 114 | + popen_argv: list[list[str]] = [] | |
| 115 | + wait_calls = 0 | |
| 116 | + | |
| 117 | + def _fake_popen(argv: list[str], **kwargs: object) -> _FakeProc: | |
| 118 | + popen_argv.append(list(argv)) | |
| 119 | + return _FakeProc() | |
| 120 | + | |
| 121 | + def _fake_wait(*args: object, **kwargs: object) -> str | None: | |
| 122 | + nonlocal wait_calls | |
| 123 | + wait_calls += 1 | |
| 124 | + if wait_calls == 1: | |
| 125 | + raise TargetSmokeError("port raced") | |
| 126 | + return None | |
| 127 | + | |
| 128 | + monkeypatch.setattr(smoke_mod, "reserve_local_port", lambda host: next(ports)) | |
| 129 | + monkeypatch.setattr(smoke_mod.subprocess, "Popen", _fake_popen) | |
| 130 | + monkeypatch.setattr(smoke_mod, "_wait_for_models", _fake_wait) | |
| 131 | + monkeypatch.setattr(smoke_mod, "_chat_completion", lambda *args, **kwargs: "hello") | |
| 132 | + monkeypatch.setattr(smoke_mod, "_stop_process", lambda proc: None) | |
| 133 | + | |
| 134 | + first_line = smoke_mod.smoke_openai_compat_server(["fake-server"], startup_attempts=2) | |
| 135 | + | |
| 136 | + assert first_line == "hello" | |
| 137 | + assert popen_argv == [ | |
| 138 | + ["fake-server", "--host", "127.0.0.1", "--port", "41001"], | |
| 139 | + ["fake-server", "--host", "127.0.0.1", "--port", "41002"], | |
| 140 | + ] | |
| 141 | + | |
| 142 | + def test_fixed_port_does_not_retry_after_target_smoke_error( | |
| 143 | + self, monkeypatch: pytest.MonkeyPatch | |
| 144 | + ) -> None: | |
| 145 | + popen_argv: list[list[str]] = [] | |
| 146 | + | |
| 147 | + def _fake_popen(argv: list[str], **kwargs: object) -> _FakeProc: | |
| 148 | + popen_argv.append(list(argv)) | |
| 149 | + return _FakeProc() | |
| 150 | + | |
| 151 | + monkeypatch.setattr(smoke_mod.subprocess, "Popen", _fake_popen) | |
| 152 | + monkeypatch.setattr( | |
| 153 | + smoke_mod, | |
| 154 | + "_wait_for_models", | |
| 155 | + lambda *args, **kwargs: (_ for _ in ()).throw(TargetSmokeError("boom")), | |
| 156 | + ) | |
| 157 | + monkeypatch.setattr(smoke_mod, "_stop_process", lambda proc: None) | |
| 158 | + | |
| 159 | + with pytest.raises(TargetSmokeError, match="boom"): | |
| 160 | + smoke_mod.smoke_openai_compat_server(["fake-server"], port=49999, startup_attempts=3) | |
| 161 | + | |
| 162 | + assert popen_argv == [["fake-server", "--host", "127.0.0.1", "--port", "49999"]] | |
| 163 | + | |
| 164 | + def test_invalid_startup_attempts_raise_value_error(self) -> None: | |
| 165 | + with pytest.raises(ValueError, match="startup_attempts"): | |
| 166 | + smoke_mod.smoke_openai_compat_server(["fake-server"], startup_attempts=0) | |
| 167 | + | |
| 168 | + | |
| 169 | +class TestWaitForModels: | |
| 170 | + def test_returns_model_id_after_retryable_fetch_error( | |
| 171 | + self, monkeypatch: pytest.MonkeyPatch | |
| 172 | + ) -> None: | |
| 173 | + proc = _FakeProc() | |
| 174 | + seen_sleeps: list[float] = [] | |
| 175 | + responses: Iterator[object] = iter( | |
| 176 | + [ | |
| 177 | + urllib.error.URLError("warming up"), | |
| 178 | + "fake-model", | |
| 179 | + ] | |
| 180 | + ) | |
| 113 | 181 | |
| 114 | - def test_early_exit_raises_with_readiness_message(self, tmp_path: Path) -> None: | |
| 115 | - _require_loopback_bind() | |
| 116 | - script = _write_server_script(tmp_path, mode="exit") | |
| 182 | + def _fake_fetch(**kwargs: object) -> str | None: | |
| 183 | + outcome = next(responses) | |
| 184 | + if isinstance(outcome, Exception): | |
| 185 | + raise outcome | |
| 186 | + return outcome | |
| 187 | + | |
| 188 | + monkeypatch.setattr(smoke_mod, "_fetch_model_id", _fake_fetch) | |
| 189 | + monkeypatch.setattr(smoke_mod.time, "sleep", lambda seconds: seen_sleeps.append(seconds)) | |
| 190 | + | |
| 191 | + model_id = smoke_mod._wait_for_models( | |
| 192 | + proc, | |
| 193 | + io.StringIO(""), | |
| 194 | + host="127.0.0.1", | |
| 195 | + port=41000, | |
| 196 | + startup_timeout=1.0, | |
| 197 | + request_timeout=0.1, | |
| 198 | + poll_interval=0.25, | |
| 199 | + ) | |
| 200 | + | |
| 201 | + assert model_id == "fake-model" | |
| 202 | + assert seen_sleeps == [0.25] | |
| 203 | + | |
| 204 | + def test_raises_when_process_exits_before_readiness(self) -> None: | |
| 205 | + proc = _FakeProc(returncode=3) | |
| 117 | 206 | |
| 118 | 207 | with pytest.raises(TargetSmokeError, match="exited before readiness"): |
| 119 | - smoke_openai_compat_server( | |
| 120 | - [ | |
| 121 | - sys.executable, | |
| 122 | - str(script), | |
| 123 | - "--mode", | |
| 124 | - "exit", | |
| 125 | - "--host", | |
| 126 | - "127.0.0.1", | |
| 127 | - "--port", | |
| 128 | - "8000", | |
| 129 | - ], | |
| 208 | + smoke_mod._wait_for_models( | |
| 209 | + proc, | |
| 210 | + io.StringIO("first\nsecond"), | |
| 211 | + host="127.0.0.1", | |
| 212 | + port=41000, | |
| 130 | 213 | startup_timeout=1.0, |
| 214 | + request_timeout=0.1, | |
| 215 | + poll_interval=0.1, | |
| 131 | 216 | ) |
| 132 | 217 | |
| 133 | - def test_passes_environment_to_subprocess(self, tmp_path: Path) -> None: | |
| 134 | - _require_loopback_bind() | |
| 135 | - script = _write_server_script(tmp_path, mode="env") | |
| 218 | + def test_raises_timeout_with_last_error(self, monkeypatch: pytest.MonkeyPatch) -> None: | |
| 219 | + proc = _FakeProc() | |
| 220 | + monotonic_values = iter((0.0, 0.05, 0.11)) | |
| 136 | 221 | |
| 137 | - first_line = smoke_openai_compat_server( | |
| 138 | - [sys.executable, str(script), "--mode", "env", "--host", "127.0.0.1", "--port", "8000"], | |
| 139 | - env={"FAKE_SMOKE_TOKEN": "ready"}, | |
| 222 | + monkeypatch.setattr(smoke_mod.time, "monotonic", lambda: next(monotonic_values)) | |
| 223 | + monkeypatch.setattr( | |
| 224 | + smoke_mod, | |
| 225 | + "_fetch_model_id", | |
| 226 | + lambda **kwargs: (_ for _ in ()).throw(TimeoutError("late reply")), | |
| 140 | 227 | ) |
| 228 | + monkeypatch.setattr(smoke_mod.time, "sleep", lambda seconds: None) | |
| 141 | 229 | |
| 142 | - assert first_line == "hello from fake server" | |
| 230 | + with pytest.raises(TargetSmokeError, match="late reply"): | |
| 231 | + smoke_mod._wait_for_models( | |
| 232 | + proc, | |
| 233 | + io.StringIO(""), | |
| 234 | + host="127.0.0.1", | |
| 235 | + port=41000, | |
| 236 | + startup_timeout=0.1, | |
| 237 | + request_timeout=0.1, | |
| 238 | + poll_interval=0.05, | |
| 239 | + ) | |
| 240 | + | |
| 241 | + | |
| 242 | +class TestFetchModelId: | |
| 243 | + @pytest.mark.parametrize( | |
| 244 | + ("payload", "expected"), | |
| 245 | + [ | |
| 246 | + ({"data": [{"id": "model-1"}]}, "model-1"), | |
| 247 | + ({"data": []}, None), | |
| 248 | + ({"data": ["not-a-dict"]}, None), | |
| 249 | + ({"data": [{"id": " "}]}, None), | |
| 250 | + ], | |
| 251 | + ) | |
| 252 | + def test_fetch_model_id_parses_payload( | |
| 253 | + self, | |
| 254 | + monkeypatch: pytest.MonkeyPatch, | |
| 255 | + payload: object, | |
| 256 | + expected: str | None, | |
| 257 | + ) -> None: | |
| 258 | + monkeypatch.setattr( | |
| 259 | + smoke_mod.urllib.request, | |
| 260 | + "urlopen", | |
| 261 | + _urlopen_with(payload), | |
| 262 | + ) | |
| 263 | + | |
| 264 | + assert ( | |
| 265 | + smoke_mod._fetch_model_id(host="127.0.0.1", port=41000, request_timeout=0.1) == expected | |
| 266 | + ) | |
| 267 | + | |
| 268 | + | |
| 269 | +class TestChatCompletion: | |
| 270 | + def test_returns_string_or_list_content(self, monkeypatch: pytest.MonkeyPatch) -> None: | |
| 271 | + monkeypatch.setattr( | |
| 272 | + smoke_mod.urllib.request, | |
| 273 | + "urlopen", | |
| 274 | + _urlopen_with( | |
| 275 | + { | |
| 276 | + "choices": [ | |
| 277 | + { | |
| 278 | + "message": { | |
| 279 | + "content": [ | |
| 280 | + {"text": " first "}, | |
| 281 | + {"not_text": "ignored"}, | |
| 282 | + {"text": "second"}, | |
| 283 | + ] | |
| 284 | + } | |
| 285 | + } | |
| 286 | + ] | |
| 287 | + } | |
| 288 | + ), | |
| 289 | + ) | |
| 290 | + | |
| 291 | + assert ( | |
| 292 | + smoke_mod._chat_completion( | |
| 293 | + host="127.0.0.1", | |
| 294 | + port=41000, | |
| 295 | + model_id=None, | |
| 296 | + prompt="Hello", | |
| 297 | + request_timeout=0.1, | |
| 298 | + ) | |
| 299 | + == "first\nsecond" | |
| 300 | + ) | |
| 301 | + | |
| 302 | + @pytest.mark.parametrize( | |
| 303 | + ("payload", "match"), | |
| 304 | + [ | |
| 305 | + ({}, "missing choices"), | |
| 306 | + ({"choices": ["bad"]}, "non-object"), | |
| 307 | + ({"choices": [{}]}, "missing choices\\[0\\]\\.message"), | |
| 308 | + ( | |
| 309 | + {"choices": [{"message": {"content": ""}}]}, | |
| 310 | + "missing non-empty choices\\[0\\]\\.message\\.content", | |
| 311 | + ), | |
| 312 | + ], | |
| 313 | + ) | |
| 314 | + def test_raises_for_invalid_response_shapes( | |
| 315 | + self, | |
| 316 | + monkeypatch: pytest.MonkeyPatch, | |
| 317 | + payload: object, | |
| 318 | + match: str, | |
| 319 | + ) -> None: | |
| 320 | + monkeypatch.setattr(smoke_mod.urllib.request, "urlopen", _urlopen_with(payload)) | |
| 321 | + | |
| 322 | + with pytest.raises(TargetSmokeError, match=match): | |
| 323 | + smoke_mod._chat_completion( | |
| 324 | + host="127.0.0.1", | |
| 325 | + port=41000, | |
| 326 | + model_id="model-1", | |
| 327 | + prompt="Hello", | |
| 328 | + request_timeout=0.1, | |
| 329 | + ) | |
| 330 | + | |
| 331 | + | |
| 332 | +class TestSmokeHelpers: | |
| 333 | + def test_normalize_message_content(self) -> None: | |
| 334 | + assert smoke_mod._normalize_message_content(" hello ") == "hello" | |
| 335 | + assert ( | |
| 336 | + smoke_mod._normalize_message_content( | |
| 337 | + [{"text": " first "}, {"skip": True}, {"text": "second"}] | |
| 338 | + ) | |
| 339 | + == "first\nsecond" | |
| 340 | + ) | |
| 341 | + assert smoke_mod._normalize_message_content([{"text": " "}]) is None | |
| 342 | + assert smoke_mod._normalize_message_content(3) is None | |
| 343 | + | |
| 344 | + def test_replace_or_append_flag_and_first_non_empty_line(self) -> None: | |
| 345 | + assert smoke_mod._replace_or_append_flag(["cmd"], "--host", "127.0.0.1") == [ | |
| 346 | + "cmd", | |
| 347 | + "--host", | |
| 348 | + "127.0.0.1", | |
| 349 | + ] | |
| 350 | + assert smoke_mod._replace_or_append_flag(["cmd", "--port"], "--port", "8000") == [ | |
| 351 | + "cmd", | |
| 352 | + "--port", | |
| 353 | + "8000", | |
| 354 | + ] | |
| 355 | + assert smoke_mod._first_non_empty_line("\n \nhello\nworld\n") == "hello" | |
| 356 | + assert smoke_mod._first_non_empty_line(" \n\t") == "" | |
| 357 | + | |
| 358 | + def test_stop_process_kills_after_timeout(self) -> None: | |
| 359 | + proc = _FakeProc(kill_times_out=True) | |
| 360 | + | |
| 361 | + smoke_mod._stop_process(proc) | |
| 362 | + | |
| 363 | + assert proc.terminated is True | |
| 364 | + assert proc.killed is True | |
| 365 | + | |
| 366 | + def test_stop_process_is_noop_when_already_exited(self) -> None: | |
| 367 | + proc = _FakeProc(returncode=0) | |
| 368 | + | |
| 369 | + smoke_mod._stop_process(proc) | |
| 370 | + | |
| 371 | + assert proc.terminated is False | |
| 372 | + assert proc.killed is False | |
| 373 | + | |
| 374 | + def test_log_tail_and_merged_env(self) -> None: | |
| 375 | + log = io.StringIO("line1\nline2\nline3") | |
| 376 | + | |
| 377 | + assert "--- server log tail ---" in smoke_mod._log_tail(log, lines=2) | |
| 378 | + assert smoke_mod._merged_env({"FAKE_SMOKE_TOKEN": "ready"})["FAKE_SMOKE_TOKEN"] == "ready" | |