loader Public

Watch 0 Fork 0 Star 0
Python · 13102 bytes Raw Blame History
1 """Tests for the error recovery system."""
2
3 from loader.runtime.recovery import (
4 ErrorCategory,
5 RecoveryContext,
6 categorize_error,
7 format_failure_message,
8 format_recovery_prompt,
9 get_recovery_hints,
10 )
11
12
13 class TestCategorizeError:
14 """Tests for error categorization."""
15
16 def test_file_not_found(self):
17 assert categorize_error("No such file or directory") == ErrorCategory.FILE_NOT_FOUND
18 assert categorize_error("file not found: test.py") == ErrorCategory.FILE_NOT_FOUND
19 assert categorize_error("Path does not exist") == ErrorCategory.FILE_NOT_FOUND
20
21 def test_permission_denied(self):
22 assert categorize_error("Permission denied") == ErrorCategory.PERMISSION_DENIED
23 assert categorize_error("Access denied to file") == ErrorCategory.PERMISSION_DENIED
24 assert categorize_error("Operation not permitted") == ErrorCategory.PERMISSION_DENIED
25
26 def test_syntax_error(self):
27 assert categorize_error("SyntaxError: invalid syntax") == ErrorCategory.SYNTAX_ERROR
28 assert categorize_error("Parse error at line 5") == ErrorCategory.SYNTAX_ERROR
29
30 def test_command_not_found(self):
31 assert categorize_error("command not found: foo") == ErrorCategory.COMMAND_NOT_FOUND
32 assert categorize_error("'bar' is not recognized") == ErrorCategory.COMMAND_NOT_FOUND
33
34 def test_timeout(self):
35 assert categorize_error("Operation timed out") == ErrorCategory.TIMEOUT
36 assert categorize_error("Connection timeout") == ErrorCategory.TIMEOUT
37
38 def test_invalid_arguments(self):
39 assert categorize_error("Invalid argument: path") == ErrorCategory.INVALID_ARGUMENTS
40 assert categorize_error("Missing required parameter") == ErrorCategory.INVALID_ARGUMENTS
41 assert (
42 categorize_error(
43 "WriteTool.execute() missing 1 required positional argument: 'content'"
44 )
45 == ErrorCategory.INVALID_ARGUMENTS
46 )
47 assert (
48 categorize_error(
49 "Error patching file: structured patch hunk consumed a different "
50 "number of original lines than declared"
51 )
52 == ErrorCategory.INVALID_ARGUMENTS
53 )
54
55 def test_network_error(self):
56 assert categorize_error("Network unreachable") == ErrorCategory.NETWORK_ERROR
57 assert categorize_error("Connection refused") == ErrorCategory.CONNECTION_REFUSED
58
59 def test_unknown(self):
60 assert categorize_error("Something weird happened") == ErrorCategory.UNKNOWN
61 assert categorize_error("") == ErrorCategory.UNKNOWN
62
63
64 class TestRecoveryContext:
65 """Tests for RecoveryContext tracking."""
66
67 def test_add_attempt(self):
68 ctx = RecoveryContext(
69 original_tool="read",
70 original_args={"path": "test.py"},
71 )
72 assert len(ctx.attempts) == 0
73
74 ctx.add_attempt("read", {"path": "test.py"}, "File not found")
75 assert len(ctx.attempts) == 1
76 assert ctx.attempts[0].tool_name == "read"
77 assert ctx.attempts[0].category == ErrorCategory.FILE_NOT_FOUND
78
79 def test_can_retry(self):
80 ctx = RecoveryContext(
81 original_tool="read",
82 original_args={"path": "test.py"},
83 max_retries=3,
84 )
85 assert ctx.can_retry()
86
87 ctx.add_attempt("read", {"path": "test.py"}, "Error 1")
88 assert ctx.can_retry()
89
90 ctx.add_attempt("read", {"path": "test2.py"}, "Error 2")
91 assert ctx.can_retry()
92
93 ctx.add_attempt("read", {"path": "test3.py"}, "Error 3")
94 assert not ctx.can_retry()
95
96 def test_was_tried(self):
97 ctx = RecoveryContext(
98 original_tool="read",
99 original_args={"path": "test.py"},
100 )
101 assert not ctx.was_tried("read", {"path": "test.py"})
102
103 ctx.add_attempt("read", {"path": "test.py"}, "Error")
104 assert ctx.was_tried("read", {"path": "test.py"})
105 assert not ctx.was_tried("read", {"path": "other.py"})
106 assert not ctx.was_tried("write", {"path": "test.py"})
107
108 def test_attempts_summary(self):
109 ctx = RecoveryContext(
110 original_tool="read",
111 original_args={"path": "test.py"},
112 )
113 assert ctx.attempts_summary() == ""
114
115 ctx.add_attempt("read", {"path": "test.py"}, "File not found")
116 summary = ctx.attempts_summary()
117 assert "Previous attempts:" in summary
118 assert "read" in summary
119 assert "File not found" in summary
120
121
122 class TestGetRecoveryHints:
123 """Tests for recovery hints."""
124
125 def test_file_not_found_hints(self):
126 hints = get_recovery_hints(ErrorCategory.FILE_NOT_FOUND, "read")
127 assert "glob" in hints.lower()
128 assert "directory" in hints.lower()
129
130 def test_edit_file_not_found_special_hint(self):
131 hints = get_recovery_hints(ErrorCategory.FILE_NOT_FOUND, "edit")
132 assert "write" in hints.lower()
133
134 def test_bash_command_not_found_special_hint(self):
135 hints = get_recovery_hints(ErrorCategory.COMMAND_NOT_FOUND, "bash")
136 assert "which" in hints.lower()
137
138 def test_bash_text_rewrite_hint_prefers_file_tools(self):
139 hints = get_recovery_hints(
140 ErrorCategory.UNKNOWN,
141 "bash",
142 {"command": "sed -i '1,3c\\updated' index.html"},
143 )
144 assert "edit/patch/write" in hints.lower()
145 assert "index.html" in hints
146
147 def test_write_metadata_only_hint_requests_real_content_payload(self):
148 hints = get_recovery_hints(
149 ErrorCategory.INVALID_ARGUMENTS,
150 "write",
151 {
152 "file_path": "~/Loader/guides/nginx/index.html",
153 "content_chars": 1354,
154 "content_lines": 30,
155 },
156 )
157 assert "content='...'" in hints
158 assert "content_chars" in hints
159 assert "index.html" in hints
160
161
162 class TestFormatRecoveryPrompt:
163 """Tests for recovery prompt formatting."""
164
165 def test_format_recovery_prompt(self):
166 ctx = RecoveryContext(
167 original_tool="read",
168 original_args={"path": "test.py"},
169 )
170 ctx.add_attempt("read", {"path": "test.py"}, "No such file")
171
172 prompt = format_recovery_prompt(ctx, "read", {"path": "test.py"}, "No such file")
173 assert "Failed Command" in prompt
174 assert "read(path='test.py')" in prompt
175 assert "No such file" in prompt
176 assert "1/3" in prompt
177 assert "retry the same command with slight variations" in prompt
178
179 def test_format_recovery_prompt_for_failed_shell_rewrite_points_to_file_tools(self):
180 ctx = RecoveryContext(
181 original_tool="bash",
182 original_args={"command": "sed -i '1,3c\\updated' index.html"},
183 )
184 ctx.add_attempt(
185 "bash",
186 {"command": "sed -i '1,3c\\updated' index.html"},
187 "Exit code 1",
188 )
189
190 prompt = format_recovery_prompt(
191 ctx,
192 "bash",
193 {"command": "sed -i '1,3c\\updated' index.html"},
194 "Exit code 1",
195 )
196
197 assert "edit/patch/write" in prompt.lower()
198 assert "index.html" in prompt
199
200 def test_format_recovery_prompt_for_metadata_only_write_requests_real_payload(self):
201 ctx = RecoveryContext(
202 original_tool="write",
203 original_args={
204 "file_path": "~/Loader/guides/nginx/index.html",
205 "content_chars": 1354,
206 "content_lines": 30,
207 },
208 )
209 ctx.add_attempt(
210 "write",
211 {
212 "file_path": "~/Loader/guides/nginx/index.html",
213 "content_chars": 1354,
214 "content_lines": 30,
215 },
216 "WriteTool.execute() missing 1 required positional argument: 'content'",
217 )
218
219 prompt = format_recovery_prompt(
220 ctx,
221 "write",
222 {
223 "file_path": "~/Loader/guides/nginx/index.html",
224 "content_chars": 1354,
225 "content_lines": 30,
226 },
227 "WriteTool.execute() missing 1 required positional argument: 'content'",
228 )
229
230 assert "content='...'" in prompt
231 assert "content_chars" in prompt
232 assert "index.html" in prompt
233
234 def test_format_recovery_prompt_for_old_string_miss_prefers_current_text(self):
235 ctx = RecoveryContext(
236 original_tool="edit",
237 original_args={
238 "file_path": "~/Loader/guides/nginx/chapters/02-installation.html",
239 "old_string": "<h1>Installation</h1>",
240 "new_string": "<h1>Installation</h1><p>Expanded.</p>",
241 },
242 )
243 ctx.add_attempt(
244 "edit",
245 ctx.original_args,
246 "old_string not found in file. Make sure it matches exactly.",
247 )
248
249 prompt = format_recovery_prompt(
250 ctx,
251 "edit",
252 ctx.original_args,
253 "old_string not found in file. Make sure it matches exactly.",
254 )
255
256 assert "`old_string` is stale" in prompt
257 assert "current on-disk file" in prompt
258 assert "write` with the complete replacement content" in prompt
259
260 def test_format_recovery_prompt_for_patch_context_miss_prefers_replacement(self):
261 ctx = RecoveryContext(
262 original_tool="patch",
263 original_args={
264 "file_path": "~/Loader/guides/nginx/chapters/05-load-balancing.html",
265 "hunks": [
266 {
267 "old_start": 64,
268 "old_lines": 1,
269 "lines": ["<h2>More</h2>"],
270 }
271 ],
272 },
273 )
274 ctx.add_attempt(
275 "patch",
276 ctx.original_args,
277 (
278 "Error patching file: structured patch hunk consumed a different "
279 "number of original lines than declared (0 vs 1)"
280 ),
281 )
282
283 prompt = format_recovery_prompt(
284 ctx,
285 "patch",
286 ctx.original_args,
287 (
288 "Error patching file: structured patch hunk consumed a different "
289 "number of original lines than declared (0 vs 1)"
290 ),
291 )
292
293 assert "did not match the current file" in prompt
294 assert "Displayed wrapping is not source line numbering" in prompt
295 assert "write(file_path=..., content=...)" in prompt
296 assert "notepad/project memory" in prompt
297
298
299 class TestFormatFailureMessage:
300 """Tests for failure message formatting."""
301
302 def test_format_failure_message(self):
303 ctx = RecoveryContext(
304 original_tool="read",
305 original_args={"path": "test.py"},
306 )
307 ctx.add_attempt("read", {"path": "test.py"}, "Error 1")
308 ctx.add_attempt("glob", {"pattern": "*.py"}, "Error 2")
309 ctx.add_attempt("read", {"path": "src/test.py"}, "Error 3")
310
311 msg = format_failure_message(ctx)
312 assert "3 attempts" in msg
313 assert "read" in msg
314 assert "glob" in msg
315 assert "Error 1" in msg
316 assert "Error 2" in msg
317 assert "Error 3" in msg
318
319 def test_format_failure_message_for_old_string_miss_is_specific(self):
320 ctx = RecoveryContext(
321 original_tool="edit",
322 original_args={
323 "file_path": "guide.html",
324 "old_string": "<h1>Old</h1>",
325 "new_string": "<h1>New</h1>",
326 },
327 max_retries=2,
328 )
329 ctx.add_attempt("edit", ctx.original_args, "old_string not found in file")
330 ctx.add_attempt("edit", ctx.original_args, "old_string not found in file")
331
332 msg = format_failure_message(ctx)
333
334 assert "`old_string` was stale" in msg
335 assert "exact current on-disk text" in msg
336 assert "Try a completely different approach" not in msg
337
338 def test_format_failure_message_for_patch_context_miss_is_specific(self):
339 ctx = RecoveryContext(
340 original_tool="patch",
341 original_args={
342 "file_path": "guide.html",
343 "hunks": [
344 {
345 "old_start": 12,
346 "old_lines": 1,
347 "lines": ["<h2>More</h2>"],
348 }
349 ],
350 },
351 max_retries=2,
352 )
353 ctx.add_attempt(
354 "patch",
355 ctx.original_args,
356 "Error patching file: structured patch references lines past the end of the file",
357 )
358 ctx.add_attempt(
359 "patch",
360 ctx.original_args,
361 "Error patching file: structured patch references lines past the end of the file",
362 )
363
364 msg = format_failure_message(ctx)
365
366 assert "Patch hunk coordinates/context were stale" in msg
367 assert "write(file_path=..., content=...)" in msg
368 assert "Try a completely different approach" not in msg