@@ -103,6 +103,128 @@ def _handle_with_many_instruction_probes(n: int) -> DlmHandle: |
| 103 | 103 | ) |
| 104 | 104 | |
| 105 | 105 | |
| 106 | +class TestSkippedProbesRollup: |
| 107 | + """F07 (Audit 03) — ``_render_annotated_yaml`` prepends a |
| 108 | + ``# skipped: <probe> (<reason>)`` block so users see which probes |
| 109 | + the autogen intentionally omitted, without diffing this module's |
| 110 | + docstring.""" |
| 111 | + |
| 112 | + def test_prose_only_handle_omits_instruction_heavy_probes(self) -> None: |
| 113 | + """A .dlm with only PROSE sections skips adapter_revert + |
| 114 | + paraphrase_invariance + preference_flip + (with 1 section) |
| 115 | + section_internalization.""" |
| 116 | + from dlm_sway.integrations.dlm.autogen import collect_skipped_probe_reasons |
| 117 | + |
| 118 | + handle = DlmHandle( |
| 119 | + dlm_id="x", |
| 120 | + base_model="b", |
| 121 | + adapter_path=Path("/tmp/a"), |
| 122 | + sections=( |
| 123 | + Section( |
| 124 | + id="s1", |
| 125 | + kind="prose", |
| 126 | + content="One paragraph of prose. Second sentence.", |
| 127 | + ), |
| 128 | + ), |
| 129 | + doc_text="doc", |
| 130 | + ) |
| 131 | + skipped = collect_skipped_probe_reasons(handle) |
| 132 | + skipped_kinds = {k for k, _ in skipped} |
| 133 | + assert "adapter_revert" in skipped_kinds |
| 134 | + assert "paraphrase_invariance" in skipped_kinds |
| 135 | + assert "preference_flip" in skipped_kinds |
| 136 | + assert "section_internalization" in skipped_kinds |
| 137 | + # delta_kl should NOT be skipped — prose provides a fallback |
| 138 | + # prompt pool. |
| 139 | + assert "delta_kl" not in skipped_kinds |
| 140 | + |
| 141 | + def test_instruction_only_handle_omits_prose_heavy_probes(self) -> None: |
| 142 | + """An instruction-only doc skips external_perplexity + leakage.""" |
| 143 | + from dlm_sway.core.sections import SectionProbe |
| 144 | + from dlm_sway.integrations.dlm.autogen import collect_skipped_probe_reasons |
| 145 | + |
| 146 | + handle = DlmHandle( |
| 147 | + dlm_id="x", |
| 148 | + base_model="b", |
| 149 | + adapter_path=Path("/tmp/a"), |
| 150 | + sections=( |
| 151 | + Section( |
| 152 | + id="i1", |
| 153 | + kind="instruction", |
| 154 | + content="Q/A", |
| 155 | + probes=(SectionProbe(prompt="Q?", gold="A"),), |
| 156 | + ), |
| 157 | + ), |
| 158 | + doc_text=None, |
| 159 | + ) |
| 160 | + skipped = collect_skipped_probe_reasons(handle) |
| 161 | + skipped_kinds = {k for k, _ in skipped} |
| 162 | + assert "external_perplexity" in skipped_kinds |
| 163 | + assert "leakage" in skipped_kinds |
| 164 | + |
| 165 | + def test_rendered_yaml_carries_skipped_block(self, tmp_path: Path) -> None: |
| 166 | + """End-to-end: on a minimal prose-only .dlm, the rendered YAML |
| 167 | + header has the ``# skipped:`` lines.""" |
| 168 | + from dlm_sway.integrations.dlm.autogen import ( |
| 169 | + _render_annotated_yaml, |
| 170 | + build_spec_dict, |
| 171 | + collect_skipped_probe_reasons, |
| 172 | + ) |
| 173 | + |
| 174 | + handle = DlmHandle( |
| 175 | + dlm_id="x", |
| 176 | + base_model="b", |
| 177 | + adapter_path=Path("/tmp/a"), |
| 178 | + sections=(Section(id="s1", kind="prose", content="Short prose."),), |
| 179 | + doc_text="doc", |
| 180 | + ) |
| 181 | + dlm_path = tmp_path / "demo.dlm" |
| 182 | + dlm_path.write_text("# empty") |
| 183 | + spec = build_spec_dict(handle, dlm_source="demo.dlm") |
| 184 | + skipped = collect_skipped_probe_reasons(handle) |
| 185 | + rendered = _render_annotated_yaml(spec, handle, dlm_path, skipped=skipped) |
| 186 | + assert "# skipped: adapter_revert" in rendered |
| 187 | + assert "# skipped: preference_flip" in rendered |
| 188 | + assert "(no " in rendered # reasons start with "no ..." |
| 189 | + |
| 190 | + def test_rendered_yaml_omits_skipped_block_when_all_probes_fit(self) -> None: |
| 191 | + """A heavily-populated doc that triggers every probe emits no |
| 192 | + ``# skipped:`` lines.""" |
| 193 | + from dlm_sway.core.sections import SectionPreference, SectionProbe |
| 194 | + from dlm_sway.integrations.dlm.autogen import ( |
| 195 | + _render_annotated_yaml, |
| 196 | + build_spec_dict, |
| 197 | + collect_skipped_probe_reasons, |
| 198 | + ) |
| 199 | + |
| 200 | + probes = tuple(SectionProbe(prompt=f"Q{i}?", gold=f"A{i}") for i in range(25)) |
| 201 | + preferences = (SectionPreference(prompt="P1", chosen="good", rejected="bad"),) |
| 202 | + handle = DlmHandle( |
| 203 | + dlm_id="x", |
| 204 | + base_model="b", |
| 205 | + adapter_path=Path("/tmp/a"), |
| 206 | + sections=( |
| 207 | + Section(id="i1", kind="instruction", content="Q/A", probes=probes), |
| 208 | + Section( |
| 209 | + id="p1", |
| 210 | + kind="prose", |
| 211 | + content="A first prose sentence. A second. A third.", |
| 212 | + ), |
| 213 | + Section( |
| 214 | + id="pref1", |
| 215 | + kind="preference", |
| 216 | + content="pref", |
| 217 | + preferences=preferences, |
| 218 | + ), |
| 219 | + ), |
| 220 | + doc_text="doc", |
| 221 | + ) |
| 222 | + spec = build_spec_dict(handle) |
| 223 | + skipped = collect_skipped_probe_reasons(handle) |
| 224 | + rendered = _render_annotated_yaml(spec, handle, Path("/tmp/demo.dlm"), skipped=skipped) |
| 225 | + assert "# skipped:" not in rendered |
| 226 | + |
| 227 | + |
| 106 | 228 | class TestPortableDlmSource: |
| 107 | 229 | """F09 (Audit 03) — ``_portable_dlm_source`` emits a cwd-relative |
| 108 | 230 | path when the ``.dlm`` lives inside the cwd (survives CI checkout), |
