`9d7e370`

Derive guide quality floors

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 1 week ago

SHA: 9d7e3708ac86db0aba9efa59cc8413b342209d80
Parents: aa9c340
Tree: 6fb71f0

2 changed files

Status	File	+	-
M	`src/loader/runtime/dod.py`	207	2
M	`tests/test_dod.py`	74	0

src/loader/runtime/dod.pymodified

      "developing",
+ )
  _MINIMUM_SUBSTANTIVE_HTML_GUIDE_PAGES = 4
++_MINIMUM_SUBSTANTIVE_HTML_INDEX_CHARS = 400
++_MINIMUM_SUBSTANTIVE_HTML_INDEX_BLOCKS = 4
++_MINIMUM_SUBSTANTIVE_HTML_CHAPTER_CHARS = 900
++_MINIMUM_SUBSTANTIVE_HTML_CHAPTER_BLOCKS = 8
++_REFERENCE_HTML_INDEX_CHAR_FRACTION = 0.35
++_REFERENCE_HTML_INDEX_BLOCK_FRACTION = 0.35
++_REFERENCE_HTML_INDEX_CHAR_CAP = 1400
++_REFERENCE_HTML_INDEX_BLOCK_CAP = 12
++_REFERENCE_HTML_CHAPTER_CHAR_FRACTION = 0.50
++_REFERENCE_HTML_CHAPTER_BLOCK_FRACTION = 0.45
++_REFERENCE_HTML_CHAPTER_CHAR_CAP = 2200
++_REFERENCE_HTML_CHAPTER_BLOCK_CAP = 18
  @dataclass
      kind: VerificationKind = "runtime"
++@dataclass(frozen=True)
++class HtmlGuideQualityFloor:
++    """Minimum content density expected for substantive generated HTML guides."""
++
++    index_chars: int = _MINIMUM_SUBSTANTIVE_HTML_INDEX_CHARS
++    index_blocks: int = _MINIMUM_SUBSTANTIVE_HTML_INDEX_BLOCKS
++    chapter_chars: int = _MINIMUM_SUBSTANTIVE_HTML_CHAPTER_CHARS
++    chapter_blocks: int = _MINIMUM_SUBSTANTIVE_HTML_CHAPTER_BLOCKS
++
++
  @dataclass
  class DefinitionOfDone:
      """Single source of truth for task completion state."""
          return None
      path_literals = ", ".join(repr(str(path)) for path in html_paths)
++    quality_floor = _derive_html_guide_quality_floor(
++        task_statement,
++        output_paths=html_paths,
++        project_root=project_root,
++    )
      return "\n".join(
+         [
              "python3 - <<'PY'",
              "import re",
              "",
              f"paths = [{path_literals}]",
++            f"minimum_index_chars = {quality_floor.index_chars}",
++            f"minimum_index_blocks = {quality_floor.index_blocks}",
++            f"minimum_chapter_chars = {quality_floor.chapter_chars}",
++            f"minimum_chapter_blocks = {quality_floor.chapter_blocks}",
              "tag_pattern = re.compile(r'<[^>]+>')",
              "content_block_pattern = re.compile(r'<(p|li|pre|code|section|article|table|h2|h3|h4)\\b', re.IGNORECASE)",
              "issues = []",
              "    plain = re.sub(r'\\s+', ' ', plain).strip()",
              "    content_blocks = len(content_block_pattern.findall(text))",
              "    has_h1 = bool(re.search(r'<h1\\b', text, re.IGNORECASE))",
--            "    minimum_chars = 180 if path.name.lower() == 'index.html' else 220",
++            "    minimum_chars = minimum_index_chars if path.name.lower() == 'index.html' else minimum_chapter_chars",
--            "    minimum_blocks = 2 if path.name.lower() == 'index.html' else 3",
++            "    minimum_blocks = minimum_index_blocks if path.name.lower() == 'index.html' else minimum_chapter_blocks",
              "    if not has_h1:",
              "        issues.append(f'{path}: missing <h1>')",
              "    if len(plain) < minimum_chars:",
+     )
++def _derive_html_guide_quality_floor(
++    task_statement: str,
++    *,
++    output_paths: list[Path],
++    project_root: Path,
++) -> HtmlGuideQualityFloor:
++    reference_paths = _reference_html_guide_paths_from_task(
++        task_statement,
++        output_paths=output_paths,
++        project_root=project_root,
++    )
++    if not reference_paths:
++        return HtmlGuideQualityFloor()
++
++    index_chars: list[int] = []
++    index_blocks: list[int] = []
++    chapter_chars: list[int] = []
++    chapter_blocks: list[int] = []
++    for path in reference_paths:
++        try:
++            text = path.read_text()
++        except OSError:
++            continue
++        plain = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", text)).strip()
++        blocks = len(
++            re.findall(
++                r"<(p|li|pre|code|section|article|table|h2|h3|h4)\b",
++                text,
++                re.IGNORECASE,
++            )
++        )
++        if path.name.lower() == "index.html":
++            index_chars.append(len(plain))
++            index_blocks.append(blocks)
++        else:
++            chapter_chars.append(len(plain))
++            chapter_blocks.append(blocks)
++
++    if not chapter_chars and not index_chars:
++        return HtmlGuideQualityFloor()
++
++    return HtmlGuideQualityFloor(
++        index_chars=_reference_quality_floor(
++            _lower_quartile(index_chars) or _lower_quartile(chapter_chars),
++            fraction=_REFERENCE_HTML_INDEX_CHAR_FRACTION,
++            minimum=_MINIMUM_SUBSTANTIVE_HTML_INDEX_CHARS,
++            cap=_REFERENCE_HTML_INDEX_CHAR_CAP,
++        ),
++        index_blocks=_reference_quality_floor(
++            _lower_quartile(index_blocks) or _lower_quartile(chapter_blocks),
++            fraction=_REFERENCE_HTML_INDEX_BLOCK_FRACTION,
++            minimum=_MINIMUM_SUBSTANTIVE_HTML_INDEX_BLOCKS,
++            cap=_REFERENCE_HTML_INDEX_BLOCK_CAP,
++        ),
++        chapter_chars=_reference_quality_floor(
++            _lower_quartile(chapter_chars) or _lower_quartile(index_chars),
++            fraction=_REFERENCE_HTML_CHAPTER_CHAR_FRACTION,
++            minimum=_MINIMUM_SUBSTANTIVE_HTML_CHAPTER_CHARS,
++            cap=_REFERENCE_HTML_CHAPTER_CHAR_CAP,
++        ),
++        chapter_blocks=_reference_quality_floor(
++            _lower_quartile(chapter_blocks) or _lower_quartile(index_blocks),
++            fraction=_REFERENCE_HTML_CHAPTER_BLOCK_FRACTION,
++            minimum=_MINIMUM_SUBSTANTIVE_HTML_CHAPTER_BLOCKS,
++            cap=_REFERENCE_HTML_CHAPTER_BLOCK_CAP,
++        ),
++    )
++
++
++def _reference_html_guide_paths_from_task(
++    task_statement: str,
++    *,
++    output_paths: list[Path],
++    project_root: Path,
++) -> list[Path]:
++    output_roots = _html_output_scope_roots(output_paths)
++    paths: list[Path] = []
++    seen: set[str] = set()
++    for raw_path in _extract_task_path_mentions(task_statement):
++        path = Path(raw_path).expanduser()
++        if not path.is_absolute():
++            path = project_root / path
++        try:
++            resolved = path.resolve(strict=False)
++        except (OSError, RuntimeError, ValueError):
++            continue
++        if any(_path_is_within_root(resolved, root) for root in output_roots):
++            continue
++        for candidate in _collect_reference_html_paths(resolved):
++            key = str(candidate)
++            if key in seen:
++                continue
++            seen.add(key)
++            paths.append(candidate)
++    return paths
++
++
++def _html_output_scope_roots(output_paths: list[Path]) -> tuple[Path, ...]:
++    roots: list[Path] = []
++    seen: set[str] = set()
++    for path in output_paths:
++        try:
++            resolved = path.expanduser().resolve(strict=False)
++        except (OSError, RuntimeError, ValueError):
++            continue
++        root = resolved.parent
++        if root.name.lower() in {"chapters", "pages", "sections"}:
++            root = root.parent
++        key = str(root)
++        if key in seen:
++            continue
++        seen.add(key)
++        roots.append(root)
++    return tuple(roots)
++
++
++def _extract_task_path_mentions(task_statement: str) -> list[str]:
++    candidates = re.findall(r"`([^`]+)`", task_statement)
++    candidates.extend(
++        re.findall(
++            r"(?:~|/|\./|\../)[A-Za-z0-9_./~+-]+",
++            task_statement,
++        )
++    )
++
++    paths: list[str] = []
++    seen: set[str] = set()
++    for candidate in candidates:
++        cleaned = candidate.strip().strip("`'\",.:;()[]{}")
++        if not cleaned or cleaned in seen:
++            continue
++        if not _looks_like_path_literal(cleaned):
++            continue
++        seen.add(cleaned)
++        paths.append(cleaned)
++    return paths
++
++
++def _collect_reference_html_paths(path: Path) -> list[Path]:
++    if path.is_file() and path.suffix.lower() in {".html", ".htm"}:
++        return [path]
++    if not path.is_dir():
++        return []
++
++    candidates: list[Path] = []
++    index = path / "index.html"
++    if index.is_file():
++        candidates.append(index)
++    try:
++        candidates.extend(sorted(child for child in path.rglob("*.html") if child.is_file()))
++    except OSError:
++        return candidates
++    return list(dict.fromkeys(candidates[:32]))
++
++
++def _lower_quartile(values: list[int]) -> int:
++    if not values:
++        return 0
++    ordered = sorted(values)
++    return ordered[max(0, (len(ordered) - 1) // 4)]
++
++
++def _reference_quality_floor(
++    value: int,
++    *,
++    fraction: float,
++    minimum: int,
++    cap: int,
++) -> int:
++    if value <= 0:
++        return minimum
++    return max(minimum, min(cap, int(value * fraction)))
++
++
  def collect_planned_artifact_targets(
      dod: DefinitionOfDone,
      *,

tests/test_dod.pymodified

  """Tests for definition-of-done state and persistence."""
++import subprocess
  from pathlib import Path
  from loader.llm.base import ToolCall
      assert any("HTML guide content quality issues:" in command for command in commands)
++def test_derive_verification_commands_uses_reference_guide_depth_floor(
++    tmp_path: Path,
++) -> None:
++    reference = tmp_path / "reference"
++    reference_chapters = reference / "chapters"
++    reference_chapters.mkdir(parents=True)
++    (reference / "index.html").write_text("<h1>Reference</h1>" + "<p>" + "i" * 1600 + "</p>")
++    for index in range(1, 5):
++        (reference_chapters / f"0{index}-topic.html").write_text(
++            "<h1>Reference Chapter</h1>"
++            + "".join(f"<h2>Section {section}</h2><p>{'x' * 300}</p>" for section in range(10))
++        )
++
++    guide = tmp_path / "guide"
++    chapters = guide / "chapters"
++    chapters.mkdir(parents=True)
++    (guide / "index.html").write_text("<h1>Guide</h1>" + "<p>" + "i" * 1000 + "</p>")
++    (chapters / "01-introduction.html").write_text(
++        "<h1>Intro</h1>"
++        + "".join(f"<h2>Section {section}</h2><p>{'x' * 110}</p>" for section in range(10))
++    )
++    for index in range(2, 5):
++        (chapters / f"0{index}-topic.html").write_text(
++            "<h1>Topic</h1>"
++            + "".join(f"<h2>Section {section}</h2><p>{'x' * 220}</p>" for section in range(10))
++        )
++
++    implementation_plan = tmp_path / "implementation.md"
++    implementation_plan.write_text(
++        "\n".join(
++            [
++                "# Implementation Plan",
++                "",
++                "## File Changes",
++                f"- `{guide / 'index.html'}`",
++                f"- `{chapters / '01-introduction.html'}`",
++                f"- `{chapters / '02-topic.html'}`",
++                f"- `{chapters / '03-topic.html'}`",
++                f"- `{chapters / '04-topic.html'}`",
++                "",
++            ]
++        )
++    )
++
++    dod = create_definition_of_done(
++        f"Create an equally thorough HTML guide modeled on {reference} at {guide}."
++    )
++    dod.implementation_plan = str(implementation_plan)
++
++    commands = derive_verification_commands(
++        dod,
++        project_root=tmp_path,
++        task_statement=dod.task_statement,
++        supplement_existing=True,
++    )
++    quality_command = next(
++        command for command in commands if "HTML guide content quality issues:" in command
++    )
++
++    result = subprocess.run(
++        quality_command,
++        shell=True,
++        cwd=tmp_path,
++        capture_output=True,
++        text=True,
++        check=False,
++    )
++
++    assert result.returncode == 1
++    assert "01-introduction.html: thin content" in result.stdout
++    assert "expected at least 15" in result.stdout
++
++
  def test_derive_verification_commands_flags_insufficient_pages_for_broad_thorough_guide(
      tmp_path: Path,
  ) -> None: