loader Public

Watch 0 Fork 0 Star 0
Python · 22004 bytes Raw Blame History
1 """Tests for definition-of-done state and persistence."""
2
3 from pathlib import Path
4
5 from loader.llm.base import ToolCall
6 from loader.runtime.dod import (
7 DefinitionOfDoneStore,
8 VerificationEvidence,
9 all_planned_artifact_outputs_exist,
10 all_planned_artifacts_exist,
11 begin_new_verification_attempt,
12 build_verification_summary,
13 collect_planned_artifact_targets,
14 create_definition_of_done,
15 derive_verification_commands,
16 determine_task_size,
17 ensure_active_verification_attempt,
18 record_successful_tool_call,
19 )
20
21
22 def test_determine_task_size_boundaries() -> None:
23 assert determine_task_size(1, 10) == "small"
24 assert determine_task_size(3, 99) == "small"
25 assert determine_task_size(4, 99) == "standard"
26 assert determine_task_size(15, 499) == "standard"
27 assert determine_task_size(16, 499) == "large"
28 assert determine_task_size(15, 500) == "large"
29
30
31 def test_definition_of_done_round_trip(tmp_path: Path) -> None:
32 store = DefinitionOfDoneStore(tmp_path)
33 dod = create_definition_of_done(
34 "Create hello.py and verify it runs.",
35 retry_budget=2,
36 )
37 dod.status = "fixing"
38 dod.retry_count = 1
39 dod.verification_commands = ["python hello.py"]
40 dod.touched_files = [str(tmp_path / "hello.py")]
41 attempt = begin_new_verification_attempt(dod)
42 saved_path = store.save(dod)
43
44 reloaded = store.load(saved_path)
45
46 assert reloaded.task_statement == dod.task_statement
47 assert reloaded.status == "fixing"
48 assert reloaded.retry_count == 1
49 assert reloaded.verification_commands == ["python hello.py"]
50 assert reloaded.touched_files == [str(tmp_path / "hello.py")]
51 assert reloaded.active_verification_attempt_id == attempt.attempt_id
52 assert reloaded.active_verification_attempt_number == attempt.attempt_number
53
54
55 def test_ensure_active_verification_attempt_rehydrates_missing_active_attempt() -> None:
56 dod = create_definition_of_done("Verify the runtime output.")
57 dod.verification_attempt_counter = 2
58
59 attempt = ensure_active_verification_attempt(dod)
60
61 assert attempt.attempt_id == "verification-attempt-2"
62 assert attempt.attempt_number == 2
63 assert dod.active_verification_attempt_id == "verification-attempt-2"
64 assert dod.active_verification_attempt_number == 2
65
66
67 def test_verification_command_derivation_prefers_runtime_evidence(tmp_path: Path) -> None:
68 project_root = tmp_path
69 dod = create_definition_of_done("Create hello.py and make sure it runs.")
70 hello_path = project_root / "hello.py"
71 record_successful_tool_call(
72 dod,
73 ToolCall(
74 id="write-1",
75 name="write",
76 arguments={"file_path": str(hello_path), "content": "print('hi')\n"},
77 ),
78 )
79 record_successful_tool_call(
80 dod,
81 ToolCall(
82 id="bash-1",
83 name="bash",
84 arguments={"command": "python hello.py"},
85 ),
86 )
87
88 commands = derive_verification_commands(
89 dod,
90 project_root=project_root,
91 task_statement=dod.task_statement,
92 )
93
94 assert commands == ["python hello.py"]
95
96
97 def test_record_successful_tool_call_preserves_absolute_path_string(tmp_path: Path) -> None:
98 dod = create_definition_of_done("Create hello.py and verify it exists.")
99 absolute_path = tmp_path / "hello.py"
100
101 record_successful_tool_call(
102 dod,
103 ToolCall(
104 id="write-1",
105 name="write",
106 arguments={"file_path": str(absolute_path), "content": "print('hi')\n"},
107 ),
108 )
109
110 assert dod.touched_files == [str(absolute_path)]
111
112
113 def test_derive_verification_commands_adds_semantic_html_toc_check(tmp_path: Path) -> None:
114 chapters = tmp_path / "chapters"
115 chapters.mkdir()
116 (chapters / "01-introduction.html").write_text(
117 "<h1>Chapter 1: Introduction to Fortran</h1>\n"
118 )
119 index = tmp_path / "index.html"
120 index.write_text(
121 "\n".join(
122 [
123 '<ul class="chapter-list">',
124 ' <li><a href="chapters/01-introduction.html">Chapter 1: Introduction to Fortran</a></li>',
125 "</ul>",
126 ]
127 )
128 )
129
130 dod = create_definition_of_done(
131 "Update index.html so the table of contents links and hrefs are correct."
132 )
133 dod.acceptance_criteria = [
134 "All table of contents links in index.html point to existing chapter files.",
135 "All link texts match the actual chapter titles.",
136 ]
137 dod.touched_files = [str(index)]
138
139 commands = derive_verification_commands(
140 dod,
141 project_root=tmp_path,
142 task_statement=dod.task_statement,
143 )
144
145 assert any(command.startswith("python3 - <<'PY'") for command in commands)
146 assert not any(command == f"test -f {index}" for command in commands)
147
148
149 def test_derive_verification_commands_avoids_repo_defaults_for_external_artifacts(
150 tmp_path: Path,
151 ) -> None:
152 (tmp_path / "pyproject.toml").write_text("[project]\nname='loader'\n")
153 (tmp_path / "package.json").write_text("{}\n")
154 external_root = tmp_path.parent / "external-guide"
155 external_root.mkdir(exist_ok=True)
156 external_index = external_root / "index.html"
157 external_index.write_text("<html></html>\n")
158
159 dod = create_definition_of_done("Create an external nginx guide.")
160 dod.task_size = "standard"
161 dod.touched_files = [str(external_index)]
162
163 commands = derive_verification_commands(
164 dod,
165 project_root=tmp_path,
166 task_statement=dod.task_statement,
167 )
168
169 assert commands == [f"test -f {external_index}"]
170
171
172 def test_derive_verification_commands_adds_generic_local_html_link_check(
173 tmp_path: Path,
174 ) -> None:
175 docs = tmp_path / "docs"
176 docs.mkdir()
177 index = docs / "index.html"
178 index.write_text('<a href="chapters/01-intro.html">Intro</a>\n')
179
180 dod = create_definition_of_done("Create a small multi-page HTML guide.")
181 dod.touched_files = [str(index)]
182
183 commands = derive_verification_commands(
184 dod,
185 project_root=tmp_path,
186 task_statement=dod.task_statement,
187 supplement_existing=True,
188 )
189
190 assert any("Missing local HTML links:" in command for command in commands)
191
192
193 def test_derive_verification_commands_adds_planned_artifact_existence_checks(
194 tmp_path: Path,
195 ) -> None:
196 implementation_plan = tmp_path / "implementation.md"
197 implementation_plan.write_text(
198 "\n".join(
199 [
200 "# Implementation Plan",
201 "",
202 "## File Changes",
203 "- `docs/index.html`",
204 "- `docs/chapters/01-intro.html`",
205 "- `docs/chapters/02-installation.html`",
206 "- `docs/chapters/`",
207 ]
208 )
209 )
210
211 dod = create_definition_of_done("Create a multi-page HTML guide.")
212 dod.implementation_plan = str(implementation_plan)
213
214 commands = derive_verification_commands(
215 dod,
216 project_root=tmp_path,
217 task_statement=dod.task_statement,
218 supplement_existing=True,
219 )
220
221 assert f"test -f {tmp_path / 'docs/index.html'}" in commands
222 assert f"test -f {tmp_path / 'docs/chapters/01-intro.html'}" in commands
223 assert f"test -f {tmp_path / 'docs/chapters/02-installation.html'}" in commands
224 assert f"test -d {tmp_path / 'docs/chapters'}" in commands
225
226
227 def test_derive_verification_commands_adds_html_guide_quality_check_for_thorough_guides(
228 tmp_path: Path,
229 ) -> None:
230 docs = tmp_path / "docs"
231 chapters = docs / "chapters"
232 chapters.mkdir(parents=True)
233 implementation_plan = tmp_path / "implementation.md"
234 implementation_plan.write_text(
235 "\n".join(
236 [
237 "# Implementation Plan",
238 "",
239 "## File Changes",
240 f"- `{docs / 'index.html'}`",
241 f"- `{chapters / '01-introduction.html'}`",
242 f"- `{chapters / '02-installation.html'}`",
243 f"- `{chapters / '03-configuration.html'}`",
244 f"- `{chapters / '04-troubleshooting.html'}`",
245 "",
246 ]
247 )
248 )
249
250 dod = create_definition_of_done(
251 "Create an equally thorough multi-page HTML guide with chapter files."
252 )
253 dod.implementation_plan = str(implementation_plan)
254
255 commands = derive_verification_commands(
256 dod,
257 project_root=tmp_path,
258 task_statement=dod.task_statement,
259 supplement_existing=True,
260 )
261
262 assert any("HTML guide content quality issues:" in command for command in commands)
263
264
265 def test_derive_verification_commands_flags_insufficient_pages_for_broad_thorough_guide(
266 tmp_path: Path,
267 ) -> None:
268 guide = tmp_path / "guide"
269 chapters = guide / "chapters"
270 chapters.mkdir(parents=True)
271 (guide / "index.html").write_text("<html></html>\n")
272 (chapters / "01-introduction.html").write_text("<h1>Intro</h1>\n")
273
274 implementation_plan = tmp_path / "implementation.md"
275 implementation_plan.write_text(
276 "\n".join(
277 [
278 "# Implementation Plan",
279 "",
280 "## File Changes",
281 f"- `{guide / 'index.html'}`",
282 f"- `{chapters}/` (directory for chapter files)",
283 "",
284 "## Execution Order",
285 "- Create chapter files with appropriate content",
286 ]
287 )
288 )
289
290 dod = create_definition_of_done(
291 "Create an equally thorough multi-page HTML guide with chapter files."
292 )
293 dod.implementation_plan = str(implementation_plan)
294
295 commands = derive_verification_commands(
296 dod,
297 project_root=tmp_path,
298 task_statement=dod.task_statement,
299 supplement_existing=True,
300 )
301
302 assert any("insufficient HTML page count" in command for command in commands)
303
304
305 def test_collect_planned_artifact_targets_ignores_prose_path_fragments_in_refreshed_plan(
306 tmp_path: Path,
307 ) -> None:
308 implementation_plan = tmp_path / "implementation.md"
309 touched_index = tmp_path / "external" / "guides" / "nginx" / "index.html"
310 touched_index.parent.mkdir(parents=True)
311 touched_index.write_text("<html></html>\n")
312 implementation_plan.write_text(
313 "\n".join(
314 [
315 "# Implementation Plan",
316 "",
317 "## File Changes",
318 "- Created main index.html file with proper structure and navigation",
319 "- Created the nginx guide directory structure (chapters/)",
320 "- Created the first chapter file (01-introduction.html) with appropriate content",
321 "",
322 "## Confirmed Progress",
323 f"- Already touched during execution: `{touched_index}`.",
324 ]
325 )
326 )
327
328 dod = create_definition_of_done("Create an external nginx guide.")
329 dod.implementation_plan = str(implementation_plan)
330
331 targets = collect_planned_artifact_targets(dod, project_root=tmp_path)
332
333 assert (tmp_path / "chapters", True) not in targets
334 assert (tmp_path / "01-introduction.html", False) not in targets
335 assert targets == [(touched_index, False)]
336
337
338 def test_collect_planned_artifact_targets_resolves_nested_file_changes_relative_to_parent_directory(
339 tmp_path: Path,
340 ) -> None:
341 implementation_plan = tmp_path / "implementation.md"
342 implementation_plan.write_text(
343 "\n".join(
344 [
345 "# Implementation Plan",
346 "",
347 "## File Changes",
348 f"- `{tmp_path / 'guide' / 'index.html'}`",
349 f"- Create chapter files in `{tmp_path / 'guide' / 'chapters'}/`:",
350 " - `00-introduction.html`",
351 " - `01-installation.html`",
352 " - `02-configuration.html`",
353 "",
354 ]
355 )
356 )
357
358 dod = create_definition_of_done("Create a multi-page guide.")
359 dod.implementation_plan = str(implementation_plan)
360
361 targets = collect_planned_artifact_targets(dod, project_root=tmp_path)
362
363 assert targets == [
364 (tmp_path / "guide" / "index.html", False),
365 (tmp_path / "guide" / "chapters", True),
366 (tmp_path / "guide" / "chapters" / "00-introduction.html", False),
367 (tmp_path / "guide" / "chapters" / "01-installation.html", False),
368 (tmp_path / "guide" / "chapters" / "02-configuration.html", False),
369 ]
370
371
372 def test_collect_planned_artifact_targets_ignores_read_only_reference_paths(
373 tmp_path: Path,
374 ) -> None:
375 implementation_plan = tmp_path / "implementation.md"
376 implementation_plan.write_text(
377 "\n".join(
378 [
379 "# Implementation Plan",
380 "",
381 "## File Changes",
382 f"- `{tmp_path / 'Loader' / 'guides' / 'nginx' / 'index.html'}`",
383 f"- `{tmp_path / 'Loader' / 'guides' / 'nginx' / 'chapters'}/`",
384 "- Read `~/Loader/guides/fortran/index.html`",
385 "- Read files in `~/Loader/guides/fortran/chapters/`",
386 "",
387 ]
388 )
389 )
390
391 dod = create_definition_of_done("Create an nginx guide from a Fortran reference.")
392 dod.implementation_plan = str(implementation_plan)
393
394 targets = collect_planned_artifact_targets(dod, project_root=tmp_path)
395
396 assert targets == [
397 (tmp_path / "Loader" / "guides" / "nginx" / "index.html", False),
398 (tmp_path / "Loader" / "guides" / "nginx" / "chapters", True),
399 ]
400
401
402 def test_collect_planned_artifact_targets_ignores_nested_read_only_reference_paths(
403 tmp_path: Path,
404 ) -> None:
405 implementation_plan = tmp_path / "implementation.md"
406 implementation_plan.write_text(
407 "\n".join(
408 [
409 "# Implementation Plan",
410 "",
411 "## File Changes",
412 "1. Create directory structure for nginx guide:",
413 f" - `{tmp_path / 'Loader' / 'guides' / 'nginx' / 'index.html'}`",
414 f" - `{tmp_path / 'Loader' / 'guides' / 'nginx' / 'chapters'}/`",
415 "2. Analyze existing fortran guide structure to understand the format:",
416 " - `~/Loader/guides/fortran/`",
417 " - `~/Loader/guides/fortran/chapters/`",
418 "3. Create nginx guide content following the same structure and cadence as the fortran guide",
419 "",
420 ]
421 )
422 )
423
424 dod = create_definition_of_done("Create an nginx guide from a Fortran reference.")
425 dod.implementation_plan = str(implementation_plan)
426
427 targets = collect_planned_artifact_targets(dod, project_root=tmp_path)
428
429 assert targets == [
430 (tmp_path / "Loader" / "guides" / "nginx" / "index.html", False),
431 (tmp_path / "Loader" / "guides" / "nginx" / "chapters", True),
432 ]
433
434
435 def test_all_planned_artifacts_exist_requires_file_contents_for_planned_output_directory(
436 tmp_path: Path,
437 ) -> None:
438 implementation_plan = tmp_path / "implementation.md"
439 implementation_plan.write_text(
440 "\n".join(
441 [
442 "# Implementation Plan",
443 "",
444 "## File Changes",
445 f"- `{tmp_path / 'guide' / 'index.html'}`",
446 f"- `{tmp_path / 'guide' / 'chapters'}/` (directory for chapter files)",
447 "",
448 "## Execution Order",
449 "- Create chapter files with appropriate content",
450 ]
451 )
452 )
453
454 guide_root = tmp_path / "guide"
455 chapters = guide_root / "chapters"
456 guide_root.mkdir()
457 chapters.mkdir()
458 (guide_root / "index.html").write_text("<html></html>\n")
459
460 dod = create_definition_of_done("Create a multi-file guide with chapters.")
461 dod.implementation_plan = str(implementation_plan)
462 dod.completed_items = ["Create chapter files with appropriate content"]
463
464 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is False
465
466
467 def test_all_planned_artifacts_exist_stays_false_for_substantive_guide_with_only_one_chapter(
468 tmp_path: Path,
469 ) -> None:
470 implementation_plan = tmp_path / "implementation.md"
471 implementation_plan.write_text(
472 "\n".join(
473 [
474 "# Implementation Plan",
475 "",
476 "## File Changes",
477 f"- `{tmp_path / 'guide' / 'index.html'}`",
478 f"- `{tmp_path / 'guide' / 'chapters'}/` (directory for chapter files)",
479 "",
480 "## Execution Order",
481 "- Create chapter files with appropriate content",
482 ]
483 )
484 )
485
486 guide_root = tmp_path / "guide"
487 chapters = guide_root / "chapters"
488 chapters.mkdir(parents=True)
489 (guide_root / "index.html").write_text("<html></html>\n")
490 (chapters / "01-introduction.html").write_text("<h1>Intro</h1>\n")
491
492 dod = create_definition_of_done("Create an equally thorough guide with chapters.")
493 dod.implementation_plan = str(implementation_plan)
494 dod.completed_items = ["Create chapter files with appropriate content"]
495 dod.touched_files = [
496 str(guide_root / "index.html"),
497 str(chapters / "01-introduction.html"),
498 ]
499
500 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is False
501
502
503 def test_all_planned_artifacts_exist_respects_nested_file_change_entries(
504 tmp_path: Path,
505 ) -> None:
506 implementation_plan = tmp_path / "implementation.md"
507 implementation_plan.write_text(
508 "\n".join(
509 [
510 "# Implementation Plan",
511 "",
512 "## File Changes",
513 f"- `{tmp_path / 'guide' / 'index.html'}`",
514 f"- Create chapter files in `{tmp_path / 'guide' / 'chapters'}/`:",
515 " - `00-introduction.html`",
516 " - `01-installation.html`",
517 "",
518 ]
519 )
520 )
521
522 guide = tmp_path / "guide"
523 chapters = guide / "chapters"
524 chapters.mkdir(parents=True)
525 (guide / "index.html").write_text("<html></html>\n")
526 (chapters / "00-introduction.html").write_text("<html></html>\n")
527
528 dod = create_definition_of_done("Create a multi-page guide.")
529 dod.implementation_plan = str(implementation_plan)
530
531 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is False
532
533 (chapters / "01-installation.html").write_text("<h1>Installation</h1>\n")
534
535 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is True
536
537
538 def test_all_planned_artifact_outputs_stay_false_while_root_declares_missing_html_outputs(
539 tmp_path: Path,
540 ) -> None:
541 implementation_plan = tmp_path / "implementation.md"
542 implementation_plan.write_text(
543 "\n".join(
544 [
545 "# Implementation Plan",
546 "",
547 "## File Changes",
548 f"- `{tmp_path / 'guide' / 'index.html'}`",
549 f"- `{tmp_path / 'guide' / 'chapters'}/` (directory for chapter files)",
550 "",
551 "## Execution Order",
552 "- Create chapter files with appropriate content",
553 ]
554 )
555 )
556
557 guide_root = tmp_path / "guide"
558 chapters = guide_root / "chapters"
559 guide_root.mkdir()
560 chapters.mkdir()
561 index = guide_root / "index.html"
562 index.write_text(
563 '<a href="chapters/01-introduction.html">Intro</a>\n'
564 '<a href="chapters/02-setup.html">Setup</a>\n'
565 )
566 (chapters / "01-introduction.html").write_text("<h1>Intro</h1>\n")
567
568 dod = create_definition_of_done("Create a multi-file guide with chapters.")
569 dod.implementation_plan = str(implementation_plan)
570 dod.touched_files = [str(index), str(chapters / "01-introduction.html")]
571 dod.completed_items = ["Create chapter files with appropriate content"]
572
573 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is False
574 assert all_planned_artifact_outputs_exist(dod, project_root=tmp_path) is False
575
576 (chapters / "02-setup.html").write_text("<h1>Setup</h1>\n")
577
578 assert all_planned_artifacts_exist(dod, project_root=tmp_path) is True
579 assert all_planned_artifact_outputs_exist(dod, project_root=tmp_path) is True
580
581
582 def test_collect_missing_declared_html_outputs_accepts_root_html_file_target(
583 tmp_path: Path,
584 ) -> None:
585 implementation_plan = tmp_path / "implementation.md"
586 implementation_plan.write_text(
587 "\n".join(
588 [
589 "# Implementation Plan",
590 "",
591 "## File Changes",
592 f"- `{tmp_path / 'guide' / 'index.html'}`",
593 f"- `{tmp_path / 'guide' / 'chapters'}/` (directory for chapter files)",
594 ]
595 )
596 )
597
598 guide_root = tmp_path / "guide"
599 chapters = guide_root / "chapters"
600 chapters.mkdir(parents=True)
601 index = guide_root / "index.html"
602 index.write_text(
603 '<a href="chapters/01-introduction.html">Intro</a>\n'
604 '<a href="chapters/02-setup.html">Setup</a>\n'
605 )
606 (chapters / "01-introduction.html").write_text("<h1>Intro</h1>\n")
607
608 dod = create_definition_of_done("Create a multi-file guide with chapters.")
609 dod.implementation_plan = str(implementation_plan)
610 dod.touched_files = [str(index), str(chapters / "01-introduction.html")]
611
612 assert all_planned_artifact_outputs_exist(dod, project_root=tmp_path) is False
613
614
615 def test_build_verification_summary_keeps_concrete_missing_link_details() -> None:
616 summary = build_verification_summary(
617 [
618 VerificationEvidence(
619 command="python3 - <<'PY' ... PY",
620 passed=False,
621 stderr=(
622 "Missing links:\n"
623 "chapters/05-control-structures.html -> missing\n"
624 "chapters/06-input-output.html -> missing\n"
625 ),
626 )
627 ]
628 )
629
630 assert "Missing links:" in summary
631 assert "chapters/05-control-structures.html -> missing" in summary
632 assert "chapters/06-input-output.html -> missing" in summary