bensch Public

Watch 0 Fork 0 Star 0
Text · 6288 bytes Raw Blame History
1 #!/bin/sh
2 # bensch — POSIX shell testing framework
3 # Usage: bensch --shell /path/to/shell [--profile name] [--suite list] [--ref-shell path]
4 set -e
5
6 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
7 SHELL_BIN=""
8 PROFILE=""
9 SUITES="all"
10 REF_SHELL="${REF_SHELL:-bash}"
11 REPORT=""
12 VERBOSE=""
13
14 usage() {
15 cat <<EOF
16 bensch — POSIX shell testing framework
17
18 Usage:
19 bensch --shell /path/to/shell [options]
20
21 Options:
22 --shell PATH Path to shell binary to test (required)
23 --profile NAME Shell profile (auto-detected from binary name if omitted)
24 --suite LIST Comma-separated suites: posix,interactive,builtins,all (default: all)
25 --ref-shell PATH Reference shell for comparison (default: bash)
26 --report PATH Write markdown report to file
27 --verbose Verbose output
28 --list-suites List available test suites
29 --list-profiles List available shell profiles
30 -h, --help Show this help
31
32 Examples:
33 bensch --shell /bin/bash --suite posix
34 bensch --shell ./bin/fortsh --profile fortsh --suite interactive
35 bensch --shell /bin/zsh --suite all --report reports/zsh.md
36 EOF
37 exit 0
38 }
39
40 list_suites() {
41 echo "Available test suites:"
42 echo " posix POSIX compliance shell scripts (~3800 assertions)"
43 echo " interactive PTY-based interactive tests (~1014 tests)"
44 echo " builtins Builtin command tests (~650 assertions)"
45 echo " all Run all suites"
46 echo ""
47 echo "Interactive sub-suites:"
48 for d in "$SCRIPT_DIR"/suites/interactive/*/; do
49 name=$(basename "$d")
50 count=$(find "$d" -name "*.yaml" | xargs grep -c "^ - name:" 2>/dev/null | awk -F: '{s+=$2} END {print s+0}')
51 echo " interactive/$name ($count tests)"
52 done
53 exit 0
54 }
55
56 list_profiles() {
57 echo "Available shell profiles:"
58 for f in "$SCRIPT_DIR"/profiles/*.yaml; do
59 name=$(basename "$f" .yaml)
60 display=$(grep "display_name:" "$f" | head -1 | sed 's/.*: *//' | tr -d '"')
61 printf " %-12s %s\n" "$name" "$display"
62 done
63 exit 0
64 }
65
66 # Parse arguments
67 while [ $# -gt 0 ]; do
68 case "$1" in
69 --shell) SHELL_BIN="$2"; shift 2 ;;
70 --profile) PROFILE="$2"; shift 2 ;;
71 --suite) SUITES="$2"; shift 2 ;;
72 --ref-shell) REF_SHELL="$2"; shift 2 ;;
73 --report) REPORT="$2"; shift 2 ;;
74 --verbose) VERBOSE=1; shift ;;
75 --list-suites) list_suites ;;
76 --list-profiles) list_profiles ;;
77 -h|--help) usage ;;
78 *) echo "Unknown option: $1"; usage ;;
79 esac
80 done
81
82 if [ -z "$SHELL_BIN" ]; then
83 echo "Error: --shell is required"
84 echo "Run: bensch --help"
85 exit 1
86 fi
87
88 if [ ! -x "$SHELL_BIN" ]; then
89 echo "Error: $SHELL_BIN is not executable"
90 exit 1
91 fi
92
93 # Auto-detect profile if not specified
94 if [ -z "$PROFILE" ]; then
95 shell_basename=$(basename "$SHELL_BIN" | tr '[:upper:]' '[:lower:]')
96 case "$shell_basename" in
97 bash*) PROFILE="bash" ;;
98 zsh*) PROFILE="zsh" ;;
99 dash*) PROFILE="dash" ;;
100 ksh*|mksh*|pdksh*) PROFILE="ksh" ;;
101 fortsh*) PROFILE="fortsh" ;;
102 *) PROFILE="generic" ;;
103 esac
104 fi
105
106 PROFILE_FILE="$SCRIPT_DIR/profiles/$PROFILE.yaml"
107 if [ ! -f "$PROFILE_FILE" ]; then
108 echo "Warning: profile '$PROFILE' not found, using generic"
109 PROFILE="generic"
110 PROFILE_FILE="$SCRIPT_DIR/profiles/generic.yaml"
111 fi
112
113 # Export for child processes
114 export SHELL_BIN
115 export REF_SHELL
116 export BASH_REF="$REF_SHELL" # Alias for POSIX scripts that use BASH_REF
117 export BENSCH_PROFILE="$PROFILE"
118
119 # Build RC_DISABLE_ENV from profile
120 RC_DISABLE_ENV=$(python3 -c "
121 import yaml, sys
122 with open('$PROFILE_FILE') as f:
123 p = yaml.safe_load(f)
124 env = p.get('rc_disable', {}).get('env', {})
125 print(' '.join(f'{k}={v}' for k, v in env.items()))
126 " 2>/dev/null || echo "")
127 export RC_DISABLE_ENV
128
129 echo "╔══════════════════════════════════════════════════════════════╗"
130 echo "║ bensch — POSIX Shell Testing Framework ║"
131 echo "╚══════════════════════════════════════════════════════════════╝"
132 echo ""
133 echo " Shell: $SHELL_BIN"
134 echo " Profile: $PROFILE"
135 echo " Ref: $REF_SHELL"
136 echo " Suites: $SUITES"
137 echo ""
138
139 TOTAL_PASS=0
140 TOTAL_FAIL=0
141
142 # Run requested suites
143 run_posix() {
144 echo "═══ POSIX Compliance Tests ═══"
145 if [ -x "$SCRIPT_DIR/suites/posix/run_posix_tests.sh" ]; then
146 SHELL_BIN="$SHELL_BIN" REF_SHELL="$REF_SHELL" RC_DISABLE_ENV="$RC_DISABLE_ENV" \
147 "$SCRIPT_DIR/suites/posix/run_posix_tests.sh"
148 else
149 echo " [SKIP] run_posix_tests.sh not found"
150 fi
151 }
152
153 run_interactive() {
154 echo "═══ Interactive PTY Tests ═══"
155 if [ ! -d "$SCRIPT_DIR/framework" ]; then
156 echo " [SKIP] framework/ not found"
157 return
158 fi
159
160 # Set up venv if needed
161 if [ ! -d "$SCRIPT_DIR/.venv" ]; then
162 python3 -m venv "$SCRIPT_DIR/.venv"
163 . "$SCRIPT_DIR/.venv/bin/activate"
164 pip install -q -r "$SCRIPT_DIR/framework/requirements.txt"
165 else
166 . "$SCRIPT_DIR/.venv/bin/activate"
167 fi
168
169 cd "$SCRIPT_DIR/framework"
170 python3 runner.py --shell "$SHELL_BIN" --spec-dir "$SCRIPT_DIR/suites/interactive"
171 cd "$SCRIPT_DIR"
172 }
173
174 run_builtins() {
175 echo "═══ Builtin Tests ═══"
176 if [ -x "$SCRIPT_DIR/suites/builtins/run_builtin_tests.sh" ]; then
177 SHELL_BIN="$SHELL_BIN" REF_SHELL="$REF_SHELL" \
178 "$SCRIPT_DIR/suites/builtins/run_builtin_tests.sh"
179 else
180 echo " [SKIP] run_builtin_tests.sh not found"
181 fi
182 }
183
184 case "$SUITES" in
185 all)
186 run_posix
187 run_interactive
188 run_builtins
189 ;;
190 *)
191 IFS=','
192 for suite in $SUITES; do
193 case "$suite" in
194 posix) run_posix ;;
195 interactive) run_interactive ;;
196 builtins) run_builtins ;;
197 *) echo "Unknown suite: $suite" ;;
198 esac
199 done
200 unset IFS
201 ;;
202 esac
203
204 echo ""
205 echo "Done."