| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Fix MANUAL_REVIEW items in auto-generated YAML test files. |
| 4 | |
| 5 | This script applies heuristics to estimate expected outputs for common patterns. |
| 6 | """ |
| 7 | |
| 8 | import re |
| 9 | import yaml |
| 10 | from pathlib import Path |
| 11 | import subprocess |
| 12 | import sys |
| 13 | |
| 14 | |
| 15 | def fix_variable_echo(command: str) -> str: |
| 16 | """ |
| 17 | Fix: VAR=value; echo $VAR -> expect "value" |
| 18 | Also handles: VAR=value; echo "$VAR" |
| 19 | """ |
| 20 | # Pattern: VAR=value; echo $VAR or VAR=value; echo "$VAR" |
| 21 | match = re.match(r'(\w+)=([^;]+);\s*echo\s+\$\{?\1\}?', command) |
| 22 | if match: |
| 23 | var_name, value = match.groups() |
| 24 | # Remove quotes from value if present |
| 25 | value = value.strip('\'"') |
| 26 | return value |
| 27 | |
| 28 | # Pattern with quotes: VAR=value; echo "$VAR" |
| 29 | match = re.match(r'(\w+)=([^;]+);\s*echo\s+"?\$\1"?', command) |
| 30 | if match: |
| 31 | var_name, value = match.groups() |
| 32 | value = value.strip('\'"') |
| 33 | return value |
| 34 | |
| 35 | return None |
| 36 | |
| 37 | |
| 38 | def fix_parameter_expansion(command: str) -> str: |
| 39 | """ |
| 40 | Fix parameter expansion patterns. |
| 41 | """ |
| 42 | # ${VAR:-default} when VAR is unset -> expect "default" |
| 43 | match = re.match(r'echo\s+"\$\{(\w+):-([^}]+)\}"', command) |
| 44 | if match: |
| 45 | var_name, default = match.groups() |
| 46 | if var_name.startswith('UN') or 'UNSET' in var_name: # Likely unset |
| 47 | return default |
| 48 | |
| 49 | # ${#VAR} - string length |
| 50 | match = re.match(r'(\w+)=([^;]+);\s*echo\s+"\$\{#\1\}"', command) |
| 51 | if match: |
| 52 | var_name, value = match.groups() |
| 53 | value = value.strip('\'"') |
| 54 | return str(len(value)) |
| 55 | |
| 56 | # ${VAR#pattern} - remove prefix |
| 57 | match = re.match(r'(\w+)=([^;]+);\s*echo\s+"\$\{\1#([^}]+)\}"', command) |
| 58 | if match: |
| 59 | var_name, value, pattern = match.groups() |
| 60 | value = value.strip('\'"') |
| 61 | # Simple pattern matching (this is a heuristic) |
| 62 | if pattern == '*.': |
| 63 | # Remove shortest prefix matching *. |
| 64 | if '.' in value: |
| 65 | return value.split('.', 1)[1] |
| 66 | return None # Complex pattern, keep MANUAL_REVIEW |
| 67 | |
| 68 | return None |
| 69 | |
| 70 | |
| 71 | def fix_command_substitution(command: str) -> str: |
| 72 | """ |
| 73 | Fix command substitution patterns. |
| 74 | """ |
| 75 | # echo $(echo value) -> expect "value" |
| 76 | match = re.match(r'echo\s+\$\(echo\s+(\w+)\)', command) |
| 77 | if match: |
| 78 | return match.group(1) |
| 79 | |
| 80 | # Backtick version: echo `echo value` |
| 81 | match = re.match(r'echo\s+`echo\s+(\w+)`', command) |
| 82 | if match: |
| 83 | return match.group(1) |
| 84 | |
| 85 | return None |
| 86 | |
| 87 | |
| 88 | def fix_simple_echo(command: str) -> str: |
| 89 | """ |
| 90 | Fix simple echo commands with unquoted strings. |
| 91 | """ |
| 92 | # echo hello -> "hello" |
| 93 | match = re.match(r'echo\s+(\w+)$', command) |
| 94 | if match: |
| 95 | return match.group(1) |
| 96 | |
| 97 | # echo one two three -> "one two three" |
| 98 | match = re.match(r'echo\s+(.+)$', command) |
| 99 | if match: |
| 100 | content = match.group(1) |
| 101 | # If no special characters, return as-is |
| 102 | if not any(c in content for c in ['$', '`', '(', ')', '{', '}', '\\', '"', "'"]): |
| 103 | return content |
| 104 | |
| 105 | return None |
| 106 | |
| 107 | |
| 108 | def try_run_command(command: str, shell: str = '/bin/sh') -> str: |
| 109 | """ |
| 110 | Actually run the command in a POSIX shell and capture output. |
| 111 | This is the most reliable way but slower. |
| 112 | """ |
| 113 | try: |
| 114 | result = subprocess.run( |
| 115 | [shell, '-c', command], |
| 116 | capture_output=True, |
| 117 | text=True, |
| 118 | timeout=5 |
| 119 | ) |
| 120 | output = result.stdout.strip() |
| 121 | # Only use if successful and output is reasonable (allow multi-line and longer output) |
| 122 | if result.returncode == 0 and len(output) < 500: |
| 123 | return output if output else "" # Allow empty output |
| 124 | except Exception as e: |
| 125 | # Silently skip errors for dangerous commands |
| 126 | pass |
| 127 | return None |
| 128 | |
| 129 | |
| 130 | def fix_test(test: dict, use_shell: bool = False) -> dict: |
| 131 | """ |
| 132 | Fix a single test by estimating expected output. |
| 133 | |
| 134 | Args: |
| 135 | test: Test dictionary |
| 136 | use_shell: If True, actually run commands in /bin/sh to get output |
| 137 | |
| 138 | Returns: |
| 139 | Updated test dictionary |
| 140 | """ |
| 141 | if test.get('expect_output') != 'MANUAL_REVIEW': |
| 142 | return test |
| 143 | |
| 144 | # Get the command |
| 145 | steps = test.get('steps', []) |
| 146 | if not steps or 'send_line' not in steps[0]: |
| 147 | return test |
| 148 | |
| 149 | command = steps[0]['send_line'] |
| 150 | |
| 151 | # Try different fixing strategies |
| 152 | expected = None |
| 153 | |
| 154 | # Strategy 1: Pattern matching heuristics (fast) |
| 155 | expected = fix_variable_echo(command) |
| 156 | if expected: |
| 157 | test['expect_output'] = expected |
| 158 | return test |
| 159 | |
| 160 | expected = fix_parameter_expansion(command) |
| 161 | if expected: |
| 162 | test['expect_output'] = expected |
| 163 | return test |
| 164 | |
| 165 | expected = fix_command_substitution(command) |
| 166 | if expected: |
| 167 | test['expect_output'] = expected |
| 168 | return test |
| 169 | |
| 170 | expected = fix_simple_echo(command) |
| 171 | if expected: |
| 172 | test['expect_output'] = expected |
| 173 | return test |
| 174 | |
| 175 | # Strategy 2: Actually run the command (slower but more reliable) |
| 176 | if use_shell: |
| 177 | expected = try_run_command(command) |
| 178 | if expected: |
| 179 | test['expect_output'] = expected |
| 180 | test['auto_fixed'] = 'shell_execution' |
| 181 | return test |
| 182 | |
| 183 | # Couldn't fix, leave as MANUAL_REVIEW |
| 184 | return test |
| 185 | |
| 186 | |
| 187 | def main(): |
| 188 | if len(sys.argv) < 2: |
| 189 | print(f"Usage: {sys.argv[0]} <yaml_file> [--use-shell]") |
| 190 | print("\nFixes MANUAL_REVIEW items in auto-generated YAML test files.") |
| 191 | print("--use-shell: Actually run commands in /bin/sh to get expected output (slower)") |
| 192 | sys.exit(1) |
| 193 | |
| 194 | yaml_file = Path(sys.argv[1]) |
| 195 | use_shell = '--use-shell' in sys.argv |
| 196 | |
| 197 | if not yaml_file.exists(): |
| 198 | print(f"Error: File not found: {yaml_file}") |
| 199 | sys.exit(1) |
| 200 | |
| 201 | # Load YAML |
| 202 | with open(yaml_file, 'r') as f: |
| 203 | data = yaml.safe_load(f) |
| 204 | |
| 205 | # Count before |
| 206 | before_manual = sum(1 for t in data['tests'] if t.get('expect_output') == 'MANUAL_REVIEW') |
| 207 | |
| 208 | # Fix tests |
| 209 | for i, test in enumerate(data['tests']): |
| 210 | data['tests'][i] = fix_test(test, use_shell=use_shell) |
| 211 | |
| 212 | # Count after |
| 213 | after_manual = sum(1 for t in data['tests'] if t.get('expect_output') == 'MANUAL_REVIEW') |
| 214 | fixed = before_manual - after_manual |
| 215 | |
| 216 | # Write back |
| 217 | with open(yaml_file, 'w') as f: |
| 218 | yaml.dump(data, f, default_flow_style=False, sort_keys=False) |
| 219 | |
| 220 | print(f"✓ Fixed {fixed}/{before_manual} MANUAL_REVIEW items") |
| 221 | print(f" Remaining: {after_manual}") |
| 222 | print(f" Total tests: {len(data['tests'])}") |
| 223 | |
| 224 | if after_manual > 0: |
| 225 | print(f"\n⚠️ {after_manual} tests still need manual review") |
| 226 | print(" Consider using --use-shell for better auto-fixing") |
| 227 | |
| 228 | |
| 229 | if __name__ == '__main__': |
| 230 | main() |