tenseleyflow/documentlanguagemodel / 95402d2

 class GatedModelError(BaseModelError):

     """Model requires HuggingFace license acceptance and the user hasn't accepted.

     """

     def __init__(self, hf_id: str, license_url: str | None) -> None:

 The `BaseModelSpec` schema already carries `requires_acceptance`,

 `LicenseAcceptance` rides on two load-bearing files:

 """

 from __future__ import annotations

     `via` records *how* acceptance was captured:

     - `"cli_flag"` — `--i-accept-license` on init/train (explicit).

     The `license_url` is captured at acceptance time so a later

     upstream URL change is auditable (the recorded URL stays the

     """

     model_config = ConfigDict(extra="forbid", frozen=True)

    `spec.architecture`. Catches model-surgery mismatches and wrong

    revisions.

 2. `probe_chat_template` — tokenizer has a non-empty `chat_template`

 3. `probe_gguf_arch_supported` — scans the vendored

    `convert_hf_to_gguf.py` for a `@Model.register("<arch>")` matching

 4. `probe_pretokenizer_label` — reads `vendor/llama_cpp_pretokenizer_hashes.json`

    (populated by `scripts/bump-llama-cpp.sh`) and checks the spec's

    `tokenizer_pre` is a known **label**. Silent drift here causes

    silent GGUF export failures per findings §9; the probe catches it

    early. This is the offline fast-check.

    CLAUDE.md pitfall #5). Tokenizes `_LLAMA_CPP_CHKTXT` and compares

    the sha256 of the stringified token sequence against a vendored

    per-label fingerprint table. Detects silent upstream tokenization

 _LOG = logging.getLogger(__name__)

 _REPO_ROOT: Final[Path] = Path(__file__).resolve().parents[3]

 VENDOR_LLAMA_CPP_DEFAULT: Final[Path] = _REPO_ROOT / "vendor" / "llama.cpp"

 VENDOR_PRETOKENIZER_HASHES_DEFAULT: Final[Path] = (

 # stringify the resulting token-id list, sha256 it — that digest is

 # the fingerprint llama.cpp maps to one of its pre-tokenizer types.

 # Keep verbatim; any edit here desynchronizes us from llama.cpp's

 _LLAMA_CPP_CHKTXT: Final[str] = (

     "\n \n\n \n\n\n \t \t\t \t\n  \n   \n    \n     \n"

     "🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ "

     """Scan vendored ``convert_hf_to_gguf.py`` for

     ``@Model.register("<gguf_arch>")`` or ``@ModelBase.register(...)``.

     """

     script = (vendor_path or VENDOR_LLAMA_CPP_DEFAULT) / "convert_hf_to_gguf.py"

     if not script.exists():

         return ProbeResult(

             name="gguf_arch",

             passed=True,

             skipped=True,

         )

     The vendored table is a JSON array of label strings that llama.cpp

     recognizes in `get_vocab_base_pre()`. Missing table → skip.

     """

     path = hashes_path or VENDOR_PRETOKENIZER_HASHES_DEFAULT

     if not path.exists():

 ) -> ProbeResult:

     """Compute the real llama.cpp pre-tokenizer fingerprint and compare.

     revision, silently different merges), the fingerprint drifts and

     this probe fails loudly *before* a broken GGUF reaches Ollama.

     vendored llama.cpp can absorb (e.g. brand-new Qwen3 on a llama.cpp

     pin from last month). They forfeit `dlm export` to Ollama until

     the vendored copy catches up. VL bases auto-opt-out of export

     """

     from dlm.modality import modality_for

     else:

         core = (*core, probe_chat_template(spec))

     is_media = dispatch.requires_processor

     if skip_export_probes or is_media:

         return ProbeReport(hf_id=spec.hf_id, results=core)

   with <100M MAU). We record it as `license_spdx="Other"` and surface

   the URL via `license_url`; it remains `redistributable=True` because

   the license permits bundling + redistribution with attribution.

   `redistributable_conditions: str | None` field on `BaseModelSpec`

   plus a pack-time attestation checkbox would encode this properly —

 - Llama-3.2 models are gated on HuggingFace (`requires_acceptance=True`)

   and their license does NOT permit bundling into a `.dlm.pack`

 - SmolLM2 and Phi-3.5-mini are permissive (Apache-2.0 / MIT).

 - `size_gb_fp16` is approximate; the hardware doctor uses it to seed

 """

 from __future__ import annotations

         context_length=131_072,

         recommended_seq_len=2048,

     ),

     # PaliGemma-3B-mix-224 — Google's instruction-tuned VL base built on

     # Gemma-2B + SigLIP-So400m. Gated under the Gemma license; cannot

     # redistribute inside a `.dlm.pack` (same pattern as Llama-3.2).

     # Training targets Gemma's transformer blocks; the vision tower is

     # trained jointly when modules_to_save expands to ["embed_tokens",

     #

     # `gguf_arch` / `tokenizer_pre` are set to tags the current vendored

     # llama.cpp doesn't recognize; the export probes surface

     BaseModelSpec(

         key="paligemma-3b-mix-224",

         hf_id="google/paligemma-3b-mix-224",

         # it as drift; a maintainer pastes in the observed SHA from

         # the script's output. Offline probe tests skip cleanly

         # until then (see tests/unit/base_models/test_vl_registry.py).

         #     uv run python scripts/refresh-registry.py --check

         revision="8d2f7bc9c15d71a00c14f9eb7e4c7b99c79e0a11",

         architecture="PaliGemmaForConditionalGeneration",

         ),

     ),

     # Qwen2-VL-2B-Instruct — Alibaba's Apache-2.0 VL base with dynamic-

     #

     # 672×672 with Qwen2-VL's 28-pixel patch-merger grid yields 24×24 =

     # 576 vision tokens per image. `<|image_pad|>` is the runtime

             num_image_tokens=256,

         ),

     ),

     # Qwen2-Audio-7B-Instruct — Alibaba's open audio-text model. Uses

     # the Qwen2 LLM backbone + a dedicated audio encoder. Apache-2.0

     # but the 7B checkpoint is gated on HF via license acceptance, so

     # Apache-2.0, but not-bundled-by-default because the pack size

     # (~14 GB fp16) dominates the tarball.

     #

     #

     # Placeholder SHA flagged the same way as paligemma — the weekly

     # `scripts/refresh-registry.py --check` run surfaces drift and a

 Gated models (`requires_acceptance=True`) raise `GatedModelError` unless

 the caller has already accepted the license (signalled via

 """

 from __future__ import annotations

   same spec pin at exactly the same weights.

 - `target_modules`: per-architecture LoRA target list (see findings §8;

   `"all-linear"` is avoided because it bloats small models).

   registry for Modelfile generation.

 - `gguf_arch` / `tokenizer_pre`: identifiers the llama.cpp converter

 """

 from __future__ import annotations

     preflight checks + cache keying.

     `target_size` is `(height, width)` in pixels. `resize_policy`

     """

     model_config = ConfigDict(extra="forbid", frozen=True)

 class AudioPreprocessorPlan(BaseModel):

     Mirrors `VlPreprocessorPlan` — pinned at registry-build time so

     non-target `sample_rate`; resampling lands as a follow-up.

     `sample_rate` is the model's training rate in Hz (Qwen2-Audio:

     gguf_arch: str = Field(..., min_length=1, description="Name llama.cpp's converter uses.")

     tokenizer_pre: str = Field(..., min_length=1, description="Pre-tokenizer label.")

     license_spdx: str = Field(..., min_length=1)

     license_url: str | None = None

     requires_acceptance: bool = False

 {#

 ChatML reference template — used by Qwen 2.5, SmolLM2, and compatible

 Render `{"messages": [...]}` where each message has `role` in

 {"system", "user", "assistant"} and a `content` string. Call with

 {#

 Llama 3 / 3.1 / 3.2 reference template. Uses header-id framing and the

 `<|eot_id|>` end-of-turn marker (distinct from the EOS). Source-of-truth

 Required tokens in the tokenizer:

   <|begin_of_text|>  <|start_header_id|>  <|end_header_id|>  <|eot_id|>

 prepended to the first user message by convention.

 Vendored for future Mistral-family entries in the registry; not used by

 #}

 {%- if messages[0]['role'] == 'system' -%}

     {%- set system_prompt = messages[0]['content'] -%}

 {#

 Phi-3 / Phi-3.5 reference template. Uses `<|role|>` opener and `<|end|>`

 closer; finishes with `<|endoftext|>` on assistant turns (handled by

 Roles accepted: "system", "user", "assistant".

 #}