`803dde9`

Make completion policy attempt-aware

Authored by

espadonne 1 month ago

SHA: 803dde9c8ddcaf0e0275f0c330edaaf53e7683fd
Parents: 15460af
Tree: af93e54

3 changed files

Status	File	+	-
M	`src/loader/runtime/completion_policy.py`	12	2
M	`src/loader/runtime/task_completion.py`	87	0
M	`tests/test_completion_policy.py`	61	9

src/loader/runtime/completion_policy.pymodified

  from .evidence_provenance import EvidenceProvenance
  from .reasoning_types import TaskCompletionCheck
  from .task_completion import assess_completion_follow_through_with_provenance
 -from .verification_observations import VerificationObservation, VerificationObservationStatus
 +from .verification_observations import (
 +    VerificationObservation,
 +    VerificationObservationStatus,
 +    describe_verification_attempt,
 +)
  EventSink = Callable[[AgentEvent], Awaitable[None]]
      @staticmethod
      def _render_observation(entry: VerificationObservation) -> str:
          summary = entry.summary.strip()
 +        details: list[str] = []
          if entry.detail:
              detail = entry.detail.strip()
              if detail and detail not in summary:
 -                return f"{summary} [{detail}]"
 +                details.append(detail)
 +        attempt = describe_verification_attempt(entry)
 +        if attempt and attempt not in summary:
 +            details.append(attempt)
 +        if details:
 +            return f"{summary} [{'; '.join(details)}]"
          return summary

src/loader/runtime/task_completion.pymodified

  from .evidence_provenance import EvidenceProvenance, EvidenceProvenanceStatus
  from .reasoning_types import TaskCompletionCheck
  from .verification_observations import (
 +    VerificationAttempt,
      VerificationObservation,
      VerificationObservationStatus,
 +    verification_attempt_id,
+ )
  _ACTION_VERBS = ("create", "write", "make", "edit", "fix", "add", "delete", "run")
+     ]
 +def _active_verification_attempt(
 +    dod: DefinitionOfDone,
 +) -> VerificationAttempt | None:
 +    if (
 +        dod.active_verification_attempt_id
 +        and dod.active_verification_attempt_number is not None
 +    ):
 +        return VerificationAttempt(
 +            attempt_id=dod.active_verification_attempt_id,
 +            attempt_number=dod.active_verification_attempt_number,
 +        )
 +    if dod.active_verification_attempt_number is not None:
 +        return VerificationAttempt(
 +            attempt_id=verification_attempt_id(
 +                dod.active_verification_attempt_number
 +            ),
 +            attempt_number=dod.active_verification_attempt_number,
 +        )
 +    return None
++
++
 +def _stale_verification_attempt(
 +    dod: DefinitionOfDone,
 +) -> VerificationAttempt | None:
 +    active_attempt = _active_verification_attempt(dod)
 +    if active_attempt is None or active_attempt.attempt_number <= 1:
 +        return None
 +    stale_attempt_number = active_attempt.attempt_number - 1
 +    return VerificationAttempt(
 +        attempt_id=verification_attempt_id(stale_attempt_number),
 +        attempt_number=stale_attempt_number,
 +        supersedes_attempt_id=active_attempt.attempt_id,
 +    )
++
++
  def _observed_completion_verification(
      *,
      dod: DefinitionOfDone | None,
      if dod is None or not requires_verification:
          return []
 +    active_attempt = _active_verification_attempt(dod)
      observations: list[VerificationObservation] = []
      observed_commands: set[str] = set()
      for evidence in dod.evidence:
                  kind=evidence.kind,
                  exit_code=evidence.exit_code,
                  detail=_verification_detail(evidence),
 +                attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                attempt_number=(
 +                    active_attempt.attempt_number if active_attempt else None
 +                ),
+             )
+         )
                      status=VerificationObservationStatus.MISSING.value,
                      summary=f"verification did not produce an observed result for `{command}`",
                      command=command,
 +                    attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                    attempt_number=(
 +                        active_attempt.attempt_number if active_attempt else None
 +                    ),
+                 )
+             )
          return observations
                      status=VerificationObservationStatus.PENDING.value,
                      summary=f"verification pending for `{verification_command}`",
                      command=verification_command,
 +                    attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                    attempt_number=(
 +                        active_attempt.attempt_number if active_attempt else None
 +                    ),
+                 )
+             ]
          return [
              VerificationObservation(
                  status=VerificationObservationStatus.PENDING.value,
                  summary="verification is pending for the active command set",
 +                attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                attempt_number=(
 +                    active_attempt.attempt_number if active_attempt else None
 +                ),
+             )
+         ]
                      status=VerificationObservationStatus.PLANNED.value,
                      summary=f"verification planned for `{verification_command}`",
                      command=verification_command,
 +                    attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                    attempt_number=(
 +                        active_attempt.attempt_number if active_attempt else None
 +                    ),
+                 )
+             ]
          return [
              VerificationObservation(
                  status=VerificationObservationStatus.PLANNED.value,
                  summary="verification is planned but has not run yet",
 +                attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                attempt_number=(
 +                    active_attempt.attempt_number if active_attempt else None
 +                ),
+             )
+         ]
      if dod.last_verification_result == VerificationObservationStatus.STALE.value:
 +        stale_attempt = _stale_verification_attempt(dod)
          if verification_command:
              return [
                  VerificationObservation(
                          f"`{verification_command}` after new mutating work"
                      ),
                      command=verification_command,
 +                    attempt_id=(
 +                        stale_attempt.attempt_id if stale_attempt else None
 +                    ),
 +                    attempt_number=(
 +                        stale_attempt.attempt_number if stale_attempt else None
 +                    ),
 +                    supersedes_attempt_id=(
 +                        stale_attempt.supersedes_attempt_id
 +                        if stale_attempt
 +                        else None
 +                    ),
+                 )
+             ]
          return [
              VerificationObservation(
                  status=VerificationObservationStatus.STALE.value,
                  summary="previous verification became stale after new mutating work",
 +                attempt_id=stale_attempt.attempt_id if stale_attempt else None,
 +                attempt_number=(
 +                    stale_attempt.attempt_number if stale_attempt else None
 +                ),
 +                supersedes_attempt_id=(
 +                    stale_attempt.supersedes_attempt_id
 +                    if stale_attempt
 +                    else None
 +                ),
+             )
+         ]
                      f"`{verification_command}`"
                  ),
                  command=verification_command,
 +                attempt_id=active_attempt.attempt_id if active_attempt else None,
 +                attempt_number=(
 +                    active_attempt.attempt_number if active_attempt else None
 +                ),
+             )
+         ]
      return []

tests/test_completion_policy.pymodified

      detect_premature_completion,
      get_continuation_prompt,
+ )
 -from loader.runtime.verification_observations import VerificationObservationStatus
 +from loader.runtime.verification_observations import (
 +    VerificationObservationStatus,
 +    verification_attempt_id,
 +)
  from loader.tools.base import create_default_registry
  from tests.helpers.runtime_harness import ScriptedBackend
      dod = create_definition_of_done("Run pytest -q and make sure it works.")
      dod.verification_commands = ["pytest -q"]
      dod.last_verification_result = "stale"
 +    dod.verification_attempt_counter = 2
 +    dod.active_verification_attempt_id = verification_attempt_id(2)
 +    dod.active_verification_attempt_number = 2
      check = assess_completion_follow_through(
          task="Run pytest -q and make sure it works.",
+     ]
 +def test_completion_assessment_projects_superseded_verification_attempt_for_stale_result() -> None:
 +    dod = create_definition_of_done("Run pytest -q and make sure it works.")
 +    dod.verification_commands = ["pytest -q"]
 +    dod.last_verification_result = "stale"
 +    dod.verification_attempt_counter = 2
 +    dod.active_verification_attempt_id = verification_attempt_id(2)
 +    dod.active_verification_attempt_number = 2
++
 +    assessment = assess_completion_follow_through_with_provenance(
 +        task="Run pytest -q and make sure it works.",
 +        response="The tests were already handled.",
 +        actions_taken=["write: README.md"],
 +        dod=dod,
 +    )
++
 +    assert [item.status for item in assessment.verification_observations] == [
 +        VerificationObservationStatus.STALE.value
 +    ]
 +    assert assessment.verification_observations[0].attempt_id == verification_attempt_id(1)
 +    assert assessment.verification_observations[0].attempt_number == 1
 +    assert assessment.verification_observations[0].supersedes_attempt_id == (
 +        verification_attempt_id(2)
 +    )
++
++
  def test_completion_assessment_attaches_typed_verification_provenance() -> None:
      dod = create_definition_of_done("Run pytest -q and make sure it works.")
      dod.verification_commands = ["pytest -q"]
+         )
+     ]
      dod.last_verification_result = "failed"
 +    dod.verification_attempt_counter = 2
 +    dod.active_verification_attempt_id = verification_attempt_id(2)
 +    dod.active_verification_attempt_number = 2
      events = []
      async def emit(event) -> None:
      assert decision.decision_code == "continuation_budget_exhausted"
      assert decision.decision_summary == (
          "stopped because the continuation budget was exhausted while observed "
 -        "verification still showed verification failed for `pytest -q` [1 failed]"
 +        "verification still showed verification failed for `pytest -q` "
 +        "[1 failed; attempt 2]"
+     )
      assert decision.completion_check is not None
      assert decision.completion_check.missing_evidence == [
+     ]
      assert decision.final_response == (
          "I stopped because the continuation budget was exhausted and observed "
 -        "verification still showed: verification failed for `pytest -q` [1 failed]."
 +        "verification still showed: verification failed for `pytest -q` "
 +        "[1 failed; attempt 2]."
+     )
      assert events[0].type == "completion_check"
      assert [item.status for item in decision.evidence_provenance] == [
      assert [item.status for item in decision.verification_observations] == [
          VerificationObservationStatus.FAILED.value
+     ]
 +    assert decision.verification_observations[0].attempt_number == 2
  @pytest.mark.asyncio
      dod = create_definition_of_done("Run pytest -q and make sure it works.")
      dod.verification_commands = ["pytest -q"]
      dod.last_verification_result = "failed"
 +    dod.verification_attempt_counter = 3
 +    dod.active_verification_attempt_id = verification_attempt_id(3)
 +    dod.active_verification_attempt_number = 3
      events = []
      async def emit(event) -> None:
      assert decision.decision_summary == (
          "stopped because the continuation budget was exhausted while observed "
          "verification still showed verification did not produce an observed "
 -        "result for `pytest -q`"
 +        "result for `pytest -q` [attempt 3]"
+     )
      assert decision.final_response == (
          "I stopped because the continuation budget was exhausted and observed "
          "verification still showed: verification did not produce an observed "
 -        "result for `pytest -q`."
 +        "result for `pytest -q` [attempt 3]."
+     )
      assert [item.status for item in decision.verification_observations] == [
          VerificationObservationStatus.MISSING.value
+     ]
 +    assert decision.verification_observations[0].attempt_number == 3
      assert events[0].type == "completion_check"
      dod = create_definition_of_done("Run pytest -q and make sure it works.")
      dod.verification_commands = ["pytest -q"]
      dod.last_verification_result = "pending"
 +    dod.verification_attempt_counter = 4
 +    dod.active_verification_attempt_id = verification_attempt_id(4)
 +    dod.active_verification_attempt_number = 4
      events = []
      async def emit(event) -> None:
      assert decision.decision_code == "continuation_budget_exhausted"
      assert decision.decision_summary == (
          "stopped because the continuation budget was exhausted while observed "
 -        "verification still showed verification pending for `pytest -q`"
 +        "verification still showed verification pending for `pytest -q` [attempt 4]"
+     )
      assert decision.final_response == (
          "I stopped because the continuation budget was exhausted and observed "
 -        "verification still showed: verification pending for `pytest -q`."
 +        "verification still showed: verification pending for `pytest -q` [attempt 4]."
+     )
      assert [item.status for item in decision.verification_observations] == [
          VerificationObservationStatus.PENDING.value
+     ]
 +    assert decision.verification_observations[0].attempt_number == 4
      assert events[0].type == "completion_check"
      dod = create_definition_of_done("Run pytest -q and make sure it works.")
      dod.verification_commands = ["pytest -q"]
      dod.last_verification_result = "stale"
 +    dod.verification_attempt_counter = 2
 +    dod.active_verification_attempt_id = verification_attempt_id(2)
 +    dod.active_verification_attempt_number = 2
      events = []
      async def emit(event) -> None:
      assert decision.decision_summary == (
          "stopped because the continuation budget was exhausted while observed "
          "verification still showed verification became stale for `pytest -q` "
 -        "after new mutating work"
 +        "after new mutating work [attempt 1 -> attempt 2]"
+     )
      assert decision.final_response == (
          "I stopped because the continuation budget was exhausted and observed "
          "verification still showed: verification became stale for `pytest -q` "
 -        "after new mutating work."
 +        "after new mutating work [attempt 1 -> attempt 2]."
+     )
      assert [item.status for item in decision.verification_observations] == [
          VerificationObservationStatus.STALE.value
+     ]
 +    assert decision.verification_observations[0].attempt_number == 1
 +    assert decision.verification_observations[0].supersedes_attempt_id == (
 +        verification_attempt_id(2)
 +    )
      assert events[0].type == "completion_check"