tenseleyflow/documentlanguagemodel / b6deabb

 # will take so `--help` reflects the shipping surface — even though the

 # stub body discards them.

 "src/dlm/cli/commands.py" = ["ARG001"]

 [tool.ruff.format]

 quote-style = "double"

 # --- aggregate ---------------------------------------------------------------

     """Run every probe; aggregate into a `ProbeReport`.

     `GatedModelError` from an individual probe propagates immediately —

         from dlm.templates import TemplateError, apply_template

         try:

         except TemplateError as exc:

             console.print(f"[red]init:[/red] {exc}")

             raise typer.Exit(code=1) from exc

     parsed = parse_file(path)

     store = for_dlm(parsed.frontmatter.dlm_id)

     if run_id is not None:

         # Drill-down: show this run's steps + evals.

             writer.writerow(["step", "loss", "lr", "grad_norm", "val_loss"])

             eval_by_step = {e.step: e.val_loss for e in evals}

             for s in steps:

             return

         console.print(

             f"[green]run_id={run.run_id}[/green]  phase={run.phase}  "

         writer = csv.writer(sys.stdout)

         writer.writerow(["run_id", "phase", "seed", "status", "started_at", "ended_at"])

         for r in runs:

         return

     if not runs:

         raise typer.Exit(code=1) from exc

     store = for_dlm(parsed.frontmatter.dlm_id)

     )

     # Store may not exist yet (no `dlm train` run). Treat that as an

     # informational state rather than an error — useful after `dlm init`.

     return f"{n} PB"

     """Best-effort resolution of `training.sources` for `dlm show`.

     Returns None when the frontmatter declares no directives; returns

                 "has_ignore": bool(dc.ignore_rules),

                 "include": list(dc.config.include) if dc.config else [],

                 "exclude": list(dc.config.exclude) if dc.config else [],

                 "metadata": dict(dc.config.metadata) if dc.config else {},

                 "ignore_rules": len(dc.ignore_rules),

             }

     return records, discovered_records

     """Return a JSON-friendly snapshot of the tokenized-section cache.

     None when the cache dir doesn't exist (store never trained with

         console.print(f"    last hit rate:  {float(rate):.1%}")

     from rich.console import Console

     assert isinstance(console, Console)

         bool,

         typer.Option("--sign", help="Sign the pack with minisign before upload."),

     ] = False,

     include_base: Annotated[bool, typer.Option("--include-base")] = False,

     include_logs: Annotated[bool, typer.Option("--include-logs")] = False,

     licensee: Annotated[

         raise typer.Exit(code=1) from exc

     size_mb = result.bytes_sent / (1024 * 1024)

     if result.sink_kind.value == "hf":

         console.print(f"[dim]install:[/dim] dlm pull {result.destination}")

     if result.detail:

         raise typer.Exit(code=1) from exc

     size_mb = result.bytes_received / (1024 * 1024)

     status = result.verification.status

     if status == VerifyStatus.VERIFIED:

     store = for_dlm(dlm_id)

     if not store.manifest.exists():

         console.print(

         )

         raise typer.Exit(code=1)

     seconds = _parse_duration(older_than)

     if seconds is None:

         console.print(

         )

         raise typer.Exit(code=2)

     if not target.exists():

         raise ScaffoldError(f"target does not exist: {target}", path=target)

     if not target.is_dir():

     dlm_dir = target / _SCAFFOLD_DIR

     existing = sorted(dlm_dir.glob("*.dlm")) if dlm_dir.is_dir() else []

             return ScaffoldResult(dlm_path=named_match, scaffolded=False, dlm_id=dlm_id)

         if name_is_default and len(existing) == 1:

             dlm_id = _dlm_id_from_file(existing[0])

         if name_is_default and len(existing) > 1:

             raise ScaffoldError(

                 f"multiple .dlm files found under {target / _SCAFFOLD_DIR}; "

                 f"pass --name to pick one:\n{listing}",

         )

     dlm_path = dlm_dir / f"{name}.dlm"

     dlm_id = existing_id or mint_ulid()

     dlm_dir.mkdir(parents=True, exist_ok=True)

         policy=policy,

         target=target,

     )

     return ScaffoldResult(dlm_path=dlm_path, scaffolded=True, dlm_id=dlm_id)

         [

             "---",

             "",

             "",

         ]

     )

     atomic_write_text(dlm_path, "\n".join(lines))

     """Map `--include` flags + `--recursive` to frontmatter globs.

     Empty `--include` + `--recursive` → `["**/*"]`: train on every

             if not isinstance(text, str):

                 raise DataFormatError(f"`text` field must be str, got {type(text).__name__}")

             return text

             raise DataFormatError(

                 "preference rows (prompt/chosen/rejected) must be routed to DPOTrainer, "

                 "not SFTTrainer's formatting_func"

     # ---- Open / construct --------------------------------------------

     @classmethod

         """Open (or create) a cache at `root`.

         Creates the directory layout idempotently. Missing manifest →

         mid-put fallback.

         """

         cutoff = time.time() - older_than_seconds

         for key_str in stale_keys:

             entry = self._manifest[key_str]

             self._entry_path(entry).unlink(missing_ok=True)

             del self._manifest[key_str]

         if stale_keys:

         return len(stale_keys)

     def clear(self) -> int:

         birthday threshold. The full sha is persisted in the manifest

         for verification if a collision ever occurs in practice.

         """

     def shard(self) -> str:

         """First 2 hex chars of section_id — the directory shard."""

         anchor = dlm_dir.parent

         config = _load_training_yaml(dlm_dir / _CONFIG_FILENAME)

         ignore_rules = _load_ignore(dlm_dir / _IGNORE_FILENAME)

     discovered.sort(key=lambda d: len(d.anchor.as_posix()))

     return tuple(discovered)

     try:

         return DlmTrainingConfig.model_validate(raw)

     except ValidationError as exc:

         return None

     if not directives:

         return ExpandResult(sections=(), provenance=(), discovered=())

     strict = training.sources_policy == "strict"

     sections: list[Section] = []

     provenance: list[SourceProvenance] = []

     header_root = resolved_root if resolved_root.is_dir() else resolved_root.parent

     for file_path in _iter_candidates(resolved_root):

             _LOG.info(

                 "directive: hit max_files=%d for %s; truncating deterministically",

                 directive.max_files,

             _LOG.warning("directive: stat failed for %s: %s; skipping", file_path, exc)

             continue

             _LOG.info(

                 "directive: %s (%d bytes) exceeds max_bytes_per_file=%d; skipping",

                 file_path,

             continue

         if is_probably_binary(raw):

             skipped_binary += 1

             continue

         relpath = file_path.relative_to(header_root).as_posix()

         content = f"# source: {relpath}\n\n{text}"

         total_bytes += len(raw)

     return sections, SourceProvenance(

     """Return DiscoveredConfigs whose anchor is an ancestor of file_path,

     sorted shallowest → deepest. Includes the direct-parent anchor."""

     abs_file = file_path.resolve()

     result.sort(key=lambda d: len(d.anchor.as_posix()))

     return tuple(result)

             yield candidate

     """Match rel_path against include (any) and exclude (none)."""

     if any(_compile_glob(pat).fullmatch(rel_path) for pat in exclude):

         return False

             match = _FENCE_RE.match(line)

             if match:

                 fence_name = match.group(1)

                 flush()

                 current_type = fence_type

                 current_adapter = fence_adapter

     return sections

     """Map a fence name to `(SectionType, adapter_name|None)` or raise.

     Multi-adapter fences carry a `#<adapter>` suffix; the adapter part is

         section_type = SectionType(base)

     except ValueError as exc:

         raise FenceError(

             path=path,

             line=line,

             col=1,

     enabled: bool = False

     method: Literal["dpo", "orpo"] = "dpo"

     # DPO-only fields — ignored for ORPO but kept on the config so a

     # user switching methods doesn't have to delete them.

     loss_type: Literal["sigmoid", "hinge", "ipo"] = "sigmoid"

             "target_modules": "auto",

             "learning_rate": 2e-4,

         }

         if drift:

             raise ValueError(

                 "training.adapters is declared; flat per-adapter fields "

             lines.append(f"{pad}{field_name}:")

             lines.extend(nested)

             continue

         ):

             # `dict[str, BaseModel]` (e.g. training.adapters) — emit

             # each entry as a nested mapping. The key is the dict

         if not adapter_path.exists():

             from dlm.export.errors import ExportError

     elif adapter_name is None:

         resolved = store.resolve_current_adapter()

         pointer = store.adapter_current_pointer

             from dlm.export.errors import ExportError

             raise ExportError(

             )

         adapter_path = resolved

     else:

     for piece in raw.split(","):

         token = piece.strip()

         if not token:

         if ":" not in token:

             raise InvalidMixSpecError(

             )

         name, _, weight_str = token.rpartition(":")

         name = name.strip()

         weight_str = weight_str.strip()

         if not _NAME_RE.fullmatch(name):

             raise InvalidMixSpecError(

             )

         if name in seen:

         seen.add(name)

         try:

             weight = float(weight_str)

         except ValueError as exc:

             raise InvalidMixSpecError(

             ) from exc

         if weight < 0:

             raise InvalidMixSpecError(

             )

         entries.append(MixEntry(name=name, weight=weight))

     return entries

     """Refuse mix entries that reference adapters not in `training.adapters`.

     Single source of error messaging so the CLI and the runner both

     first = entries[0]

     first_path = _resolve_or_raise(store, first.name)

     for extra in entries[1:]:

         path = _resolve_or_raise(store, extra.name)

         model.load_adapter(str(path), adapter_name=extra.name)

     single-valued), so any source is interchangeable — we pick the first.

     """

     if not entries:

     return _resolve_or_raise(store, entries[0].name)

     import shutil

     tmp_dir.mkdir(parents=True, exist_ok=True)

     # PEFT nests under the adapter name; that's where run_export

     # expects to find adapter_config.json + safetensors.

     return psutil.virtual_memory().total / (1024**3)

     if backend == Backend.CUDA:

         return sm is not None and sm >= (8, 0)

     if backend == Backend.ROCM:

         raise ValueError(f"world_size must be >= 1, got {world_size}")

     if world_size > 1:

         check_multi_gpu_refusals(caps, world_size)

     use_qlora = _should_qlora(training, caps)

     precision = _pick_precision(caps, override=training.precision)

         per_adapter_gb = max(0.1, base_params * avg_lora_r / (1e9 * 64))

         activations_gb = base_params * 2.0 / 1e9 * 0.25

         qlora_adapter_count = _qlora_adapter_count(training, num_adapters)

         budget = caps.vram_gb * 0.85

         if est_peak > budget:

             offenders = _qlora_adapter_names(training)

             raise ResolutionError(

                 "Multi-adapter QLoRA would exceed VRAM "

                 f"(~{est_peak:.1f} GB estimated vs {budget:.1f} GB budget "

         )

     if caps.backend == Backend.CPU:

         raise ResolutionError(

         )

     if caps.backend == Backend.ROCM:

         raise ResolutionError(

     """Average LoRA rank across declared adapters (fallback: flat lora_r)."""

     if training.adapters is None or not training.adapters:

         return float(training.lora_r)

 def _qlora_adapter_count(training: TrainingConfig, fallback: int) -> int:

     src_config = peft_adapter_dir / _ADAPTER_CONFIG_FILENAME

     if not src_config.exists():

         raise MlxConversionError(

         )

     if not (peft_adapter_dir / "adapter_model.safetensors").exists():

         raise MlxConversionError(

     return lookup.get(precision, torch.float16)

     """Return the on-disk adapter version dir for inference.

     Single entry point for both the flat (unnamed) and named-adapter

     return [EvalRow(*row) for row in rows]

     """The tokenization row for `run_id`, or None when absent.

     Returns None when the table is empty for this run (i.e. the run

             import wandb  # type: ignore[import-not-found]

         except ImportError as exc:  # pragma: no cover - depends on install

             raise RuntimeError(

             ) from exc

         wandb_dir = store_root / "wandb"

             exist_ok=True,

         )

     except HfHubHTTPError as exc:

     # Upload the pack.

     if progress is not None:

         signature = self._sign(self.dlm_id, expiry_iso, nonce)

         # Wire format: nonce || len(expiry_iso) || expiry_iso || sig

         payload = (

         )

         return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=")

             while self.requests and self.requests[0] < window_start:

                 self.requests.popleft()

             if len(self.requests) >= self.rate_limit_per_min:

             if self.active >= self.max_concurrency:

                 raise RateLimitError(

                     f"rate limit: max concurrent connections ({self.max_concurrency}) exceeded"

         pack_path = staging / "incoming.dlm.pack"

         sig_path = pack_path.with_suffix(pack_path.suffix + ".minisig")

         # Verify signature BEFORE unpack so users learn the trust

         # status even if unpack then fails for an unrelated reason.

 def _log_verification(source: str, result: VerifyResult) -> None:

     if result.status == VerifyStatus.VERIFIED:

     elif result.status == VerifyStatus.UNVERIFIED:

         _LOG.warning(

             "pull: signature present but could not verify (%s). "

     _LOG.info("push: signed %s → %s", pack_path, sig_path)

     if spec.kind == SinkKind.HF:

         from dlm.share.hf_sink import push_hf

         import zstandard as zstd

         fields: dict[str, str] = {}

             for member in tar:

                 if member.name.endswith("header.json"):

                     import json

                         header = json.loads(data.read().decode("utf-8"))

                         fields["dlm_id"] = str(header.get("dlm_id", ""))

                         fields["base_model"] = str(header.get("base_model", ""))

                     break

         return fields

     except (OSError, ValueError, ImportError) as exc:

     sig_path = target.with_suffix(target.suffix + ".minisig")

     if not sig_path.is_file():

         raise ShareError(

         )

     return sig_path

     if not minisign_available():

         _LOG.warning(

             signature,

         )

         return VerifyResult(

     keys_dir = trusted_keys_dir or _DEFAULT_TRUSTED_KEYS_DIR

     if not keys_dir.is_dir():

         _LOG.warning(

             keys_dir,

         )

         return VerifyResult(

     if source.startswith("hf:"):

         rest = source[len("hf:") :]

         if not rest or "/" not in rest:

         return SinkSpec(kind=SinkKind.HF, target=rest)

     if source.startswith(("http://", "https://")):

     if source.startswith("peer://"):

         rest = source[len("peer://") :]

         if not rest:

         return SinkSpec(kind=SinkKind.PEER, target=rest)

     # Fall through to local path. Accept absolute, relative, and `~`.

     if path_looking:

         return SinkSpec(kind=SinkKind.LOCAL, target=source)

         headers=_build_headers(total, content_type="application/octet-stream"),

     )

     try:

             status = resp.status

             if status < 200 or status >= 300:

                 raise SinkError(f"url push: HTTP {status} from {url}")

         raise SinkError(f"url pull: I/O error writing {out_path}: {exc}") from exc

     headers = {"User-Agent": _USER_AGENT}

     if content_type is not None:

         headers["Content-Type"] = content_type

     return headers

     """Streaming read adapter for urllib's `data=` parameter.

     urllib accepts a bytes-or-bytes-iterable. We return the full bytes

     return data

     written = 0

     with out_path.open("wb") as dst:

         while True:

         if not name.startswith(_VERSION_DIR_PREFIX):

             continue

         try:

         except ValueError:

             continue

         highest = max(highest, n)

 def _validate_adapter_name(name: str) -> None:

     if not _ADAPTER_NAME_RE.fullmatch(name):

         raise ValueError(

         )

             ) from exc

         from dlm.io.atomic import write_text as _atomic_write_text

     def export_quant_dir(self, quant: str) -> Path:

         """Return `exports/<quant>/` (does NOT create it)."""

         ) from exc

     if is_gated(spec) and not accept_license:

         raise TemplateApplyError(

         )

     parsed = parse_text(template.dlm_text)

 _VERSION_PREFIX = "v"

     """Return the next empty `adapter/[<name>/]versions/vNNNN/` path.

     Creates the directory (and any missing parents). When `adapter_name`

             )

         except OSError:

             _LOG.exception(

                 pending,

             )

         raise

         os.close(fd)

     """Return version dirs that exist on disk but aren't the current pointer.

     Used by the trainer's startup routine to detect crash-before-flip

     return [version_for(n) for n in sorted(existing) if n != current_n]

     base = (

     )

     if not base.is_dir():

         return []

         self._active: bool = False

     def on_train_begin(

     ) -> None:

         if self.n_steps <= 0:

             return

         self._active = False

     def on_step_end(

     ) -> None:

         if self._active and state.global_step >= self.n_steps:

             self._restore()

     def on_train_end(

     ) -> None:

         self._restore()

     if warmup_steps < 0:

         raise ValueError(f"warmup_steps must be non-negative, got {warmup_steps}")

     if warmup_steps >= total_steps:

     if not 0.0 <= floor_ratio <= 1.0:

     if step < 0:

         raise ValueError(f"step must be non-negative, got {step}")

     """

     if len(token_ids) != len(decoded_tokens):

         raise ValueError(

         )

     if top_n < 0:

         raise ValueError(f"top_n must be non-negative, got {top_n}")

     total_words = _count_words(text)

     tpw = total_tokens / total_words if total_words else 0.0

     counts: Counter[str] = Counter(decoded_tokens)

     top_tokens = counts.most_common(top_n)

     )

     """Run the base tokenizer over `text` and compute the fit report.

     Heavy-import shell around `compute_vocab_gap` — covered by the slow

         f"  <unk> hits      : {r.unk_hits}",

     ]

     if r.has_unk:

     if r.top_tokens:

         lines.append("  top tokens:")

         width = max(len(t) for t, _ in r.top_tokens)

     try:

         value = int(raw)

     except ValueError as exc:

     if value < 1:

         return 1

     return value

         try:

             value = int(raw)

         except ValueError as exc:

         if value < 0:

             return 0

         return value

     plan = build_plan(parsed)

     if adapter_name not in plan.by_adapter:

         raise UnknownAdapterError(

         )

     return plan.by_adapter[adapter_name]

     # Policy: base + the SFT-trained adapter as trainable.

     base_model = load_base_model(spec, plan)

     adapter_dir = store.adapter_version(reference_adapter_version)

     # Reference: frozen per preference.reference mode. We reload a

     # clean base for the reference rather than sharing `base_model` so

     doc_ds = build_dpo_dataset(list(parsed.sections))

     rng = _random.Random(seed + reference_adapter_version)

     now = datetime.now(UTC).replace(tzinfo=None, microsecond=0)

     if replay_rows:

         replay_ds = Dataset.from_list(replay_rows)

         train_ds = concatenate_datasets([doc_ds, replay_ds])

         try:

             ref = PeftModel.from_pretrained(model, str(adapter_path), is_trainable=False)

         except Exception as exc:

         _freeze(ref)

         return ref

     adapter-version path that couldn't be opened."""

     def __init__(self, *, adapter_path: str, cause: str) -> None:

         self.adapter_path = adapter_path

         self.cause = cause

 )

 if TYPE_CHECKING:

     from dlm.base_models import BaseModelSpec

     from dlm.doc.parser import ParsedDlm

     from dlm.hardware.capabilities import Capabilities

     base_model = load_base_model(spec, plan)

     adapter_dir = store.adapter_version(reference_adapter_version)

     tok_bringup = prepare_tokenizer(spec.hf_id, spec.revision)

     doc_ds = build_dpo_dataset(list(parsed.sections))

     rng = _random.Random(seed + reference_adapter_version)

     now = datetime.now(UTC).replace(tzinfo=None, microsecond=0)

     if replay_rows:

         replay_ds = Dataset.from_list(replay_rows)

         train_ds = concatenate_datasets([doc_ds, replay_ds])

       explicitly request DPO, skip with a warning instead of raising.

     """

     sections = list(parsed.sections)

     results: list[PhaseResult] = []

     sft_fn = sft_runner or _real_sft_runner()

         sft_result = sft_fn(store, parsed, spec, plan, **sft_kwargs)

         results.append(PhaseResult(phase="sft", result=sft_result))

     if should_run_pref:

         if not has_preference_content(sections):

             if phase == "preference":

     if parsed.frontmatter.training.sources is None:

         return parsed, ()

     result = expand_sources(parsed, base_path=base_path)

     if not result.sections:

         return parsed, result.provenance

             ),

         )

         assert run1.adapter_version == 1

         # --- Second invocation: reuse scaffolded .dlm ---------------------

         # Train again — should produce adapter v0002 in the same store.

         parsed2 = parse_file(result2.dlm_path)

         assert run2.adapter_version == 2

         manifest = load_manifest(store.manifest)

 def _build_tree(root: Path) -> None:

     """Build a repo fixture:

           .dlm/

     """

     (root / ".dlm").mkdir()

     (root / ".dlm" / "training.yaml").write_text(

     (root / "src" / "vendor").mkdir()

     (root / "src" / "vendor" / ".dlm").mkdir()

     (root / "src" / "vendor" / ".dlm" / "training.yaml").write_text(

         encoding="utf-8",

     )

     (root / "src" / "vendor" / "dep.py").write_text("def dep(): pass\n")

         tree = home / "src"

         tree.mkdir()

         (tree / "a.py").write_text(

             encoding="utf-8",

         )

         (tree / "b.py").write_text(

     runs = recent_runs(trained_store.store.root, limit=10)

     assert runs, "trainer.run() did not record any runs"

     latest = runs[0]

     steps = steps_for_run(trained_store.store.root, latest.run_id)

     # The tiny-model fixture runs at least one step.

     import json

     k_cfg = json.loads(

     )

     t_cfg = json.loads(

     )

     assert k_cfg["r"] == 8, f"knowledge lora_r: {k_cfg['r']}"

     assert t_cfg["r"] == 4, f"tone lora_r: {t_cfg['r']}"

         make_dlm(

             sections=[prose(_PROSE)],

             base_model="smollm2-135m",

         ),

         encoding="utf-8",

     )

     spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)

     cached = download_spec(spec, local_files_only=True)

     entries = [

         MixEntry(name="knowledge", weight=1.0),

     spec = resolve_base_model(parsed.frontmatter.base_model, accept_license=True)

     cached = download_spec(spec, local_files_only=True)

     entries = [

         MixEntry(name="knowledge", weight=0.7),

     the rejected one — the direction DPO should push completions."""

     pairs = [

         ("What is 2 + 2?", "4.", "The sum of two and two is four, a basic arithmetic fact."),

         ("Is water wet?", "Yes.", "Water is generally considered wet in most everyday contexts."),

         ("What's 10 - 3?", "7.", "Ten minus three equals seven in standard arithmetic."),

     ]