tenseleyflow/loader / 53cf9bf

Browse files

feat: add error recovery system (Phase 4)

- Add RecoveryContext for tracking retry attempts
- Categorize errors (file not found, permission, syntax, etc.)
- Provide recovery hints based on error type
- Loop detection to prevent infinite retries
- Max 3 retries per tool call with graceful failure
Authored by mfwolffe <wolffemf@dukes.jmu.edu>
SHA
53cf9bff5c24c18df82e289b0fe73d38120b179d
Parents
dbbb638
Tree
d2f7ced

2 changed files

StatusFile+-
A src/loader/agent/recovery.py 223 0
A tests/test_recovery.py 163 0
src/loader/agent/recovery.pyadded
@@ -0,0 +1,223 @@
1
+"""Error recovery system for the agent.
2
+
3
+Provides intelligent retry logic with adaptation when tools fail.
4
+"""
5
+
6
+from dataclasses import dataclass, field
7
+from enum import Enum, auto
8
+from typing import Any
9
+
10
+
11
+class ErrorCategory(Enum):
12
+    """Categories of errors for recovery strategies."""
13
+    FILE_NOT_FOUND = auto()
14
+    PERMISSION_DENIED = auto()
15
+    SYNTAX_ERROR = auto()
16
+    COMMAND_NOT_FOUND = auto()
17
+    TIMEOUT = auto()
18
+    INVALID_ARGUMENTS = auto()
19
+    NETWORK_ERROR = auto()
20
+    UNKNOWN = auto()
21
+
22
+
23
+@dataclass
24
+class ToolAttempt:
25
+    """Record of a single tool execution attempt."""
26
+    tool_name: str
27
+    arguments: dict[str, Any]
28
+    error: str
29
+    category: ErrorCategory
30
+
31
+
32
+@dataclass
33
+class RecoveryContext:
34
+    """Tracks recovery state for a tool execution."""
35
+    original_tool: str
36
+    original_args: dict[str, Any]
37
+    attempts: list[ToolAttempt] = field(default_factory=list)
38
+    max_retries: int = 3
39
+
40
+    def add_attempt(self, tool_name: str, args: dict[str, Any], error: str) -> None:
41
+        """Record an attempted tool execution."""
42
+        category = categorize_error(error)
43
+        self.attempts.append(ToolAttempt(
44
+            tool_name=tool_name,
45
+            arguments=args,
46
+            error=error,
47
+            category=category,
48
+        ))
49
+
50
+    def can_retry(self) -> bool:
51
+        """Check if more retries are allowed."""
52
+        return len(self.attempts) < self.max_retries
53
+
54
+    def attempts_summary(self) -> str:
55
+        """Summarize what's been tried for the LLM."""
56
+        if not self.attempts:
57
+            return ""
58
+
59
+        lines = ["Previous attempts:"]
60
+        for i, attempt in enumerate(self.attempts, 1):
61
+            args_str = ", ".join(f"{k}={v!r}" for k, v in attempt.arguments.items())
62
+            lines.append(f"{i}. {attempt.tool_name}({args_str})")
63
+            lines.append(f"   Error: {attempt.error}")
64
+        return "\n".join(lines)
65
+
66
+    def was_tried(self, tool_name: str, args: dict[str, Any]) -> bool:
67
+        """Check if this exact tool+args combination was already tried."""
68
+        for attempt in self.attempts:
69
+            if attempt.tool_name == tool_name and attempt.arguments == args:
70
+                return True
71
+        return False
72
+
73
+
74
+def categorize_error(error_message: str) -> ErrorCategory:
75
+    """Categorize an error message for recovery strategy selection."""
76
+    error_lower = error_message.lower()
77
+
78
+    if any(x in error_lower for x in ["no such file", "file not found", "does not exist"]):
79
+        return ErrorCategory.FILE_NOT_FOUND
80
+
81
+    if any(x in error_lower for x in ["permission denied", "access denied", "not permitted"]):
82
+        return ErrorCategory.PERMISSION_DENIED
83
+
84
+    if any(x in error_lower for x in ["syntax error", "invalid syntax", "parse error"]):
85
+        return ErrorCategory.SYNTAX_ERROR
86
+
87
+    if any(x in error_lower for x in ["command not found", "not recognized", "no such command"]):
88
+        return ErrorCategory.COMMAND_NOT_FOUND
89
+
90
+    if any(x in error_lower for x in ["timeout", "timed out"]):
91
+        return ErrorCategory.TIMEOUT
92
+
93
+    if any(x in error_lower for x in ["invalid argument", "missing required", "expected"]):
94
+        return ErrorCategory.INVALID_ARGUMENTS
95
+
96
+    if any(x in error_lower for x in ["network", "connection", "unreachable"]):
97
+        return ErrorCategory.NETWORK_ERROR
98
+
99
+    return ErrorCategory.UNKNOWN
100
+
101
+
102
+def get_recovery_hints(category: ErrorCategory, tool_name: str) -> str:
103
+    """Get hints for recovering from a specific error category."""
104
+    hints = {
105
+        ErrorCategory.FILE_NOT_FOUND: [
106
+            "Check if the file path is correct (use glob to search for it)",
107
+            "The file might be in a different directory",
108
+            "Check for typos in the filename",
109
+            "List the directory contents first",
110
+        ],
111
+        ErrorCategory.PERMISSION_DENIED: [
112
+            "The file might be read-only or owned by another user",
113
+            "Try reading the file instead of modifying it",
114
+            "Check if the directory exists and is writable",
115
+        ],
116
+        ErrorCategory.SYNTAX_ERROR: [
117
+            "Review the edit content for syntax errors",
118
+            "Check that the old_string matches exactly (including whitespace)",
119
+            "Read the file first to verify current contents",
120
+        ],
121
+        ErrorCategory.COMMAND_NOT_FOUND: [
122
+            "Check if the command is installed",
123
+            "Try using the full path to the command",
124
+            "Use a different command that achieves the same goal",
125
+        ],
126
+        ErrorCategory.TIMEOUT: [
127
+            "The operation is taking too long",
128
+            "Try a simpler or more targeted operation",
129
+            "Break the task into smaller steps",
130
+        ],
131
+        ErrorCategory.INVALID_ARGUMENTS: [
132
+            "Review the tool parameters",
133
+            "Check that required arguments are provided",
134
+            "Verify argument types and formats",
135
+        ],
136
+        ErrorCategory.NETWORK_ERROR: [
137
+            "Network operations may be unavailable",
138
+            "Try an offline alternative if possible",
139
+        ],
140
+        ErrorCategory.UNKNOWN: [
141
+            "Try a different approach",
142
+            "Read related files for more context",
143
+            "Break the task into smaller steps",
144
+        ],
145
+    }
146
+
147
+    category_hints = hints.get(category, hints[ErrorCategory.UNKNOWN])
148
+
149
+    # Add tool-specific hints
150
+    if tool_name == "edit" and category == ErrorCategory.FILE_NOT_FOUND:
151
+        category_hints.insert(0, "Use 'write' instead of 'edit' to create a new file")
152
+
153
+    if tool_name == "bash" and category == ErrorCategory.COMMAND_NOT_FOUND:
154
+        category_hints.insert(0, "Check available commands with 'which' or 'type'")
155
+
156
+    return "\n".join(f"- {hint}" for hint in category_hints)
157
+
158
+
159
+RECOVERY_PROMPT = """The tool execution failed. Analyze the error and try an alternative approach.
160
+
161
+Tool: {tool_name}
162
+Arguments: {args}
163
+Error: {error}
164
+Category: {category}
165
+
166
+{attempts_summary}
167
+
168
+Recovery hints:
169
+{hints}
170
+
171
+IMPORTANT:
172
+- Do NOT repeat the exact same tool call that just failed
173
+- Try a different approach or gather more information first
174
+- If you've tried {attempt_count}/{max_retries} times, consider explaining the issue to the user
175
+
176
+What would you like to try instead?"""
177
+
178
+
179
+def format_recovery_prompt(
180
+    context: RecoveryContext,
181
+    tool_name: str,
182
+    args: dict[str, Any],
183
+    error: str,
184
+) -> str:
185
+    """Format a prompt asking the LLM to recover from an error."""
186
+    category = categorize_error(error)
187
+    hints = get_recovery_hints(category, tool_name)
188
+    args_str = ", ".join(f"{k}={v!r}" for k, v in args.items())
189
+
190
+    return RECOVERY_PROMPT.format(
191
+        tool_name=tool_name,
192
+        args=args_str,
193
+        error=error,
194
+        category=category.name.replace("_", " ").title(),
195
+        attempts_summary=context.attempts_summary(),
196
+        hints=hints,
197
+        attempt_count=len(context.attempts),
198
+        max_retries=context.max_retries,
199
+    )
200
+
201
+
202
+def format_failure_message(context: RecoveryContext) -> str:
203
+    """Format a message when all retries are exhausted."""
204
+    lines = [
205
+        f"Failed to complete the operation after {len(context.attempts)} attempts.",
206
+        "",
207
+        "What was tried:",
208
+    ]
209
+
210
+    for i, attempt in enumerate(context.attempts, 1):
211
+        args_str = ", ".join(f"{k}={v!r}" for k, v in attempt.arguments.items())
212
+        lines.append(f"{i}. {attempt.tool_name}({args_str})")
213
+        lines.append(f"   Error: {attempt.error}")
214
+
215
+    lines.extend([
216
+        "",
217
+        "You may need to:",
218
+        "- Manually check the file/directory structure",
219
+        "- Verify permissions",
220
+        "- Try a completely different approach",
221
+    ])
222
+
223
+    return "\n".join(lines)
tests/test_recovery.pyadded
@@ -0,0 +1,163 @@
1
+"""Tests for the error recovery system."""
2
+
3
+import pytest
4
+from loader.agent.recovery import (
5
+    ErrorCategory,
6
+    RecoveryContext,
7
+    ToolAttempt,
8
+    categorize_error,
9
+    format_failure_message,
10
+    format_recovery_prompt,
11
+    get_recovery_hints,
12
+)
13
+
14
+
15
+class TestCategorizeError:
16
+    """Tests for error categorization."""
17
+
18
+    def test_file_not_found(self):
19
+        assert categorize_error("No such file or directory") == ErrorCategory.FILE_NOT_FOUND
20
+        assert categorize_error("file not found: test.py") == ErrorCategory.FILE_NOT_FOUND
21
+        assert categorize_error("Path does not exist") == ErrorCategory.FILE_NOT_FOUND
22
+
23
+    def test_permission_denied(self):
24
+        assert categorize_error("Permission denied") == ErrorCategory.PERMISSION_DENIED
25
+        assert categorize_error("Access denied to file") == ErrorCategory.PERMISSION_DENIED
26
+        assert categorize_error("Operation not permitted") == ErrorCategory.PERMISSION_DENIED
27
+
28
+    def test_syntax_error(self):
29
+        assert categorize_error("SyntaxError: invalid syntax") == ErrorCategory.SYNTAX_ERROR
30
+        assert categorize_error("Parse error at line 5") == ErrorCategory.SYNTAX_ERROR
31
+
32
+    def test_command_not_found(self):
33
+        assert categorize_error("command not found: foo") == ErrorCategory.COMMAND_NOT_FOUND
34
+        assert categorize_error("'bar' is not recognized") == ErrorCategory.COMMAND_NOT_FOUND
35
+
36
+    def test_timeout(self):
37
+        assert categorize_error("Operation timed out") == ErrorCategory.TIMEOUT
38
+        assert categorize_error("Connection timeout") == ErrorCategory.TIMEOUT
39
+
40
+    def test_invalid_arguments(self):
41
+        assert categorize_error("Invalid argument: path") == ErrorCategory.INVALID_ARGUMENTS
42
+        assert categorize_error("Missing required parameter") == ErrorCategory.INVALID_ARGUMENTS
43
+
44
+    def test_network_error(self):
45
+        assert categorize_error("Network unreachable") == ErrorCategory.NETWORK_ERROR
46
+        assert categorize_error("Connection refused") == ErrorCategory.NETWORK_ERROR
47
+
48
+    def test_unknown(self):
49
+        assert categorize_error("Something weird happened") == ErrorCategory.UNKNOWN
50
+        assert categorize_error("") == ErrorCategory.UNKNOWN
51
+
52
+
53
+class TestRecoveryContext:
54
+    """Tests for RecoveryContext tracking."""
55
+
56
+    def test_add_attempt(self):
57
+        ctx = RecoveryContext(
58
+            original_tool="read",
59
+            original_args={"path": "test.py"},
60
+        )
61
+        assert len(ctx.attempts) == 0
62
+
63
+        ctx.add_attempt("read", {"path": "test.py"}, "File not found")
64
+        assert len(ctx.attempts) == 1
65
+        assert ctx.attempts[0].tool_name == "read"
66
+        assert ctx.attempts[0].category == ErrorCategory.FILE_NOT_FOUND
67
+
68
+    def test_can_retry(self):
69
+        ctx = RecoveryContext(
70
+            original_tool="read",
71
+            original_args={"path": "test.py"},
72
+            max_retries=3,
73
+        )
74
+        assert ctx.can_retry()
75
+
76
+        ctx.add_attempt("read", {"path": "test.py"}, "Error 1")
77
+        assert ctx.can_retry()
78
+
79
+        ctx.add_attempt("read", {"path": "test2.py"}, "Error 2")
80
+        assert ctx.can_retry()
81
+
82
+        ctx.add_attempt("read", {"path": "test3.py"}, "Error 3")
83
+        assert not ctx.can_retry()
84
+
85
+    def test_was_tried(self):
86
+        ctx = RecoveryContext(
87
+            original_tool="read",
88
+            original_args={"path": "test.py"},
89
+        )
90
+        assert not ctx.was_tried("read", {"path": "test.py"})
91
+
92
+        ctx.add_attempt("read", {"path": "test.py"}, "Error")
93
+        assert ctx.was_tried("read", {"path": "test.py"})
94
+        assert not ctx.was_tried("read", {"path": "other.py"})
95
+        assert not ctx.was_tried("write", {"path": "test.py"})
96
+
97
+    def test_attempts_summary(self):
98
+        ctx = RecoveryContext(
99
+            original_tool="read",
100
+            original_args={"path": "test.py"},
101
+        )
102
+        assert ctx.attempts_summary() == ""
103
+
104
+        ctx.add_attempt("read", {"path": "test.py"}, "File not found")
105
+        summary = ctx.attempts_summary()
106
+        assert "Previous attempts:" in summary
107
+        assert "read" in summary
108
+        assert "File not found" in summary
109
+
110
+
111
+class TestGetRecoveryHints:
112
+    """Tests for recovery hints."""
113
+
114
+    def test_file_not_found_hints(self):
115
+        hints = get_recovery_hints(ErrorCategory.FILE_NOT_FOUND, "read")
116
+        assert "glob" in hints.lower()
117
+        assert "directory" in hints.lower()
118
+
119
+    def test_edit_file_not_found_special_hint(self):
120
+        hints = get_recovery_hints(ErrorCategory.FILE_NOT_FOUND, "edit")
121
+        assert "write" in hints.lower()
122
+
123
+    def test_bash_command_not_found_special_hint(self):
124
+        hints = get_recovery_hints(ErrorCategory.COMMAND_NOT_FOUND, "bash")
125
+        assert "which" in hints.lower()
126
+
127
+
128
+class TestFormatRecoveryPrompt:
129
+    """Tests for recovery prompt formatting."""
130
+
131
+    def test_format_recovery_prompt(self):
132
+        ctx = RecoveryContext(
133
+            original_tool="read",
134
+            original_args={"path": "test.py"},
135
+        )
136
+        ctx.add_attempt("read", {"path": "test.py"}, "No such file")
137
+
138
+        prompt = format_recovery_prompt(ctx, "read", {"path": "test.py"}, "No such file")
139
+        assert "Tool: read" in prompt
140
+        assert "No such file" in prompt
141
+        assert "1/3" in prompt
142
+        assert "Do NOT repeat" in prompt
143
+
144
+
145
+class TestFormatFailureMessage:
146
+    """Tests for failure message formatting."""
147
+
148
+    def test_format_failure_message(self):
149
+        ctx = RecoveryContext(
150
+            original_tool="read",
151
+            original_args={"path": "test.py"},
152
+        )
153
+        ctx.add_attempt("read", {"path": "test.py"}, "Error 1")
154
+        ctx.add_attempt("glob", {"pattern": "*.py"}, "Error 2")
155
+        ctx.add_attempt("read", {"path": "src/test.py"}, "Error 3")
156
+
157
+        msg = format_failure_message(ctx)
158
+        assert "3 attempts" in msg
159
+        assert "read" in msg
160
+        assert "glob" in msg
161
+        assert "Error 1" in msg
162
+        assert "Error 2" in msg
163
+        assert "Error 3" in msg