`20cfe6f`

Enable more slow integration tests

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 2 weeks ago

SHA: 20cfe6f88862ac844cae71b785f168831e8a643d
Parents: 510e78c
Tree: eb852e9

6 changed files

Status	File	+	-
M	`tests/integration/train/multi_adapter/test_prompt_and_export_named.py`	6	13
M	`tests/integration/train/multi_adapter/test_two_adapters.py`	2	7
M	`tests/integration/train/multi_adapter/test_weighted_merge.py`	2	7
M	`tests/integration/train/preference/test_dpo_tinymodel.py`	7	7
M	`tests/integration/train/preference/test_orpo_tinymodel.py`	7	7
M	`tests/integration/watch/test_watch_retrain.py`	13	8

tests/integration/train/multi_adapter/test_prompt_and_export_named.pymodified

  def _train_two(tmp_path_factory: pytest.TempPathFactory) -> Path:
      """Train knowledge + tone on a prose doc; return the .dlm path."""
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.store.manifest import Manifest, save_manifest
      from dlm.store.paths import for_dlm
      from dlm.train.multi_adapter.trainer import run_all
      from tests.fixtures.dlm_factory import make_dlm, prose
+-
 -    plan = doctor().plan
 -    if plan is None:
 -        pytest.skip("doctor() returned no viable training plan on this host")
 +    from tests.fixtures.planning import resolve_spec_and_plan
      for key in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"):
          os.environ.pop(key, None)
      doc.write_text(raw[:fm_end] + "\n" + _PROSE, encoding="utf-8")
      parsed = parse_file(doc)
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 +    spec, plan, _caps = resolve_spec_and_plan(parsed, accept_license=True)
      store = for_dlm(parsed.frontmatter.dlm_id)
      store.ensure_layout()
      save_manifest(
      _skip_if_deps_missing()
      _skip_if_tiny_model_unavailable()
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.inference.loader import load_for_inference, resolve_adapter_path
      from dlm.store.paths import for_dlm
 +    from tests.fixtures.planning import resolve_spec_and_plan
      doc = _train_two(tmp_path_factory)
      parsed = parse_file(doc)
      assert k_path.is_dir()
      assert t_path.is_dir()
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 -    caps = doctor().capabilities
 +    spec, _plan, caps = resolve_spec_and_plan(parsed, accept_license=True)
      # Full load exercises the PEFT adapter-load path on both names.
      loaded_k = load_for_inference(store, spec, caps, adapter_name="knowledge")
      _skip_if_tiny_model_unavailable()
      try:
 -        from dlm.export.vendoring import resolve_llama_cpp_paths
 +        from dlm.export.vendoring import llama_quantize_bin
 -        resolve_llama_cpp_paths()  # probes the submodule; raises if missing
 +        llama_quantize_bin()  # probes the vendored quantize binary; raises if missing
      except Exception as exc:
          pytest.skip(f"llama.cpp vendoring unavailable: {exc}")

tests/integration/train/multi_adapter/test_two_adapters.pymodified

      except Exception as exc:
          pytest.skip(f"tiny-model fixture unavailable: {exc}")
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.store.manifest import Manifest, load_manifest, save_manifest
      from dlm.store.paths import for_dlm
      from dlm.train.multi_adapter.trainer import run_all
      from tests.fixtures.dlm_factory import make_dlm, prose
+-
 -    plan = doctor().plan
 -    if plan is None:
 -        pytest.skip("doctor() returned no viable training plan on this host")
 +    from tests.fixtures.planning import resolve_spec_and_plan
      # Unset offline env so weights can download on cold caches.
      for key in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"):
      assert parsed.frontmatter.training.adapters is not None
      assert set(parsed.frontmatter.training.adapters) == {"knowledge", "tone"}
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 +    spec, plan, _caps = resolve_spec_and_plan(parsed, accept_license=True)
      store = for_dlm(parsed.frontmatter.dlm_id)
      store.ensure_layout()
      save_manifest(

tests/integration/train/multi_adapter/test_weighted_merge.pymodified

      except Exception as exc:
          pytest.skip(f"tiny-model fixture unavailable: {exc}")
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.store.manifest import Manifest, save_manifest
      from dlm.store.paths import for_dlm
      from dlm.train.multi_adapter.trainer import run_all
      from tests.fixtures.dlm_factory import make_dlm, prose
+-
 -    plan = doctor().plan
 -    if plan is None:
 -        pytest.skip("doctor() returned no viable training plan on this host")
 +    from tests.fixtures.planning import resolve_spec_and_plan
      for key in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"):
          os.environ.pop(key, None)
      doc.write_text(raw[:fm_end] + "\n" + _PROSE, encoding="utf-8")
      parsed = parse_file(doc)
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 +    spec, plan, _caps = resolve_spec_and_plan(parsed, accept_license=True)
      store = for_dlm(parsed.frontmatter.dlm_id)
      store.ensure_layout()
      save_manifest(

tests/integration/train/preference/test_dpo_tinymodel.pymodified

  @pytest.mark.slow
  def test_dpo_phase_writes_second_adapter_version(trained_store) -> None:  # type: ignore[no-untyped-def]
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.store.manifest import load_manifest
      from dlm.train.preference.phase_orchestrator import run_phases
 +    from tests.fixtures.planning import resolve_spec_and_plan
      store = trained_store.store
      dlm_path = trained_store.doc
      _append_preference_section(dlm_path, terse_preferences)
      parsed = parse_file(dlm_path)
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 -    plan = doctor().plan
 -    if plan is None:
 -        pytest.skip("no viable plan on this host — DPO body needs a real trainer")
 +    spec, plan, capabilities = resolve_spec_and_plan(
 +        parsed,
 +        accept_license=True,
 +        skip_reason="no viable plan on this host — DPO body needs a real trainer",
 +    )
      prior_manifest = load_manifest(store.manifest)
      assert prior_manifest.adapter_version == 1
          spec,
          plan,
          phase="preference",
 -        capabilities=doctor().capabilities,
 +        capabilities=capabilities,
+     )
      assert [r.phase for r in results] == ["preference"]
      dpo_result = results[0].result

tests/integration/train/preference/test_orpo_tinymodel.pymodified

  @pytest.mark.slow
  def test_orpo_phase_writes_second_adapter_version(trained_store) -> None:  # type: ignore[no-untyped-def]
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
      from dlm.doc.serializer import serialize
 -    from dlm.hardware import doctor
      from dlm.store.manifest import load_manifest
      from dlm.train.preference.phase_orchestrator import run_phases
 +    from tests.fixtures.planning import resolve_spec_and_plan
      store = trained_store.store
      dlm_path = trained_store.doc
      parsed = parse_file(dlm_path)
      assert parsed.frontmatter.training.preference.method == "orpo"
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 -    plan = doctor().plan
 -    if plan is None:
 -        pytest.skip("no viable plan on this host — ORPO body needs a real trainer")
 +    spec, plan, capabilities = resolve_spec_and_plan(
 +        parsed,
 +        accept_license=True,
 +        skip_reason="no viable plan on this host — ORPO body needs a real trainer",
 +    )
      prior_manifest = load_manifest(store.manifest)
      prior_runs = len(prior_manifest.training_runs)
          spec,
          plan,
          phase="preference",
 -        capabilities=doctor().capabilities,
 +        capabilities=capabilities,
          lock_mode="ignore",
+     )
      assert [r.phase for r in results] == ["preference"]

tests/integration/watch/test_watch_retrain.pymodified

      tmp_path: Path,
  ) -> None:
      """`do_one_cycle` on a doc with new content runs the trainer and bumps version."""
 -    from dlm.base_models import resolve as resolve_base_model
      from dlm.doc.parser import parse_file
 -    from dlm.hardware import doctor
      from dlm.train.trainer import run as trainer_run
      from dlm.watch.loop import do_one_cycle
 +    from tests.fixtures.planning import resolve_spec_and_plan
 -    doc_path = trained_store.doc_path
 +    doc_path = trained_store.doc
      store = trained_store.store
      initial_adapter = store.resolve_current_adapter()
      assert initial_adapter is not None
      # Append a new section to the doc so the ChangeSet sees `new`.
      original = doc_path.read_text(encoding="utf-8")
      doc_path.write_text(
 -        original + "\n\n## Added by watch test\n\nThis is new content.\n",
 +        original
 +        + "\n\n::instruction::\n"
 +        + "### Q\n"
 +        + "What changed in the watch test?\n"
 +        + "### A\n"
 +        + "A new instruction section was appended.\n",
          encoding="utf-8",
+     )
      parsed = parse_file(doc_path)
 -    spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)
 -    plan = doctor(training_config=parsed.frontmatter.training).plan
 -    if plan is None:
 -        pytest.skip("no viable plan on this host — watch retrain needs a real trainer")
 +    spec, plan, _caps = resolve_spec_and_plan(
 +        parsed,
 +        accept_license=True,
 +        skip_reason="no viable plan on this host — watch retrain needs a real trainer",
 +    )
      result = do_one_cycle(
          doc_path=doc_path,