tenseleyflow/bensch / 6d6715b

Browse files

create bensch entry point

Executable POSIX sh script that wires a shell binary to test suites.

Usage: bensch --shell /bin/bash --suite posix,interactive

Features:
- Auto-detects profile from shell binary name
- Exports SHELL_BIN, REF_SHELL, RC_DISABLE_ENV for child processes
- Dispatches to POSIX (shell scripts), interactive (Python PTY),
and builtin (shell scripts) runners
- --list-suites and --list-profiles for discovery
- Sets up Python venv automatically for interactive tests
Authored by espadonne
SHA
6d6715b78b177bd2e4427a35726f1fee1e8dc942
Parents
dad0c73
Tree
d7d2382

1 changed file

StatusFile+-
A bensch 204 0
benschadded
@@ -0,0 +1,204 @@
1
+#!/bin/sh
2
+# bensch — POSIX shell testing framework
3
+# Usage: bensch --shell /path/to/shell [--profile name] [--suite list] [--ref-shell path]
4
+set -e
5
+
6
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
7
+SHELL_BIN=""
8
+PROFILE=""
9
+SUITES="all"
10
+REF_SHELL="${REF_SHELL:-bash}"
11
+REPORT=""
12
+VERBOSE=""
13
+
14
+usage() {
15
+    cat <<EOF
16
+bensch — POSIX shell testing framework
17
+
18
+Usage:
19
+  bensch --shell /path/to/shell [options]
20
+
21
+Options:
22
+  --shell PATH       Path to shell binary to test (required)
23
+  --profile NAME     Shell profile (auto-detected from binary name if omitted)
24
+  --suite LIST       Comma-separated suites: posix,interactive,builtins,all (default: all)
25
+  --ref-shell PATH   Reference shell for comparison (default: bash)
26
+  --report PATH      Write markdown report to file
27
+  --verbose          Verbose output
28
+  --list-suites      List available test suites
29
+  --list-profiles    List available shell profiles
30
+  -h, --help         Show this help
31
+
32
+Examples:
33
+  bensch --shell /bin/bash --suite posix
34
+  bensch --shell ./bin/fortsh --profile fortsh --suite interactive
35
+  bensch --shell /bin/zsh --suite all --report reports/zsh.md
36
+EOF
37
+    exit 0
38
+}
39
+
40
+list_suites() {
41
+    echo "Available test suites:"
42
+    echo "  posix        POSIX compliance shell scripts (~3800 assertions)"
43
+    echo "  interactive  PTY-based interactive tests (~1014 tests)"
44
+    echo "  builtins     Builtin command tests (~650 assertions)"
45
+    echo "  all          Run all suites"
46
+    echo ""
47
+    echo "Interactive sub-suites:"
48
+    for d in "$SCRIPT_DIR"/suites/interactive/*/; do
49
+        name=$(basename "$d")
50
+        count=$(find "$d" -name "*.yaml" | xargs grep -c "^  - name:" 2>/dev/null | awk -F: '{s+=$2} END {print s+0}')
51
+        echo "  interactive/$name  ($count tests)"
52
+    done
53
+    exit 0
54
+}
55
+
56
+list_profiles() {
57
+    echo "Available shell profiles:"
58
+    for f in "$SCRIPT_DIR"/profiles/*.yaml; do
59
+        name=$(basename "$f" .yaml)
60
+        display=$(grep "display_name:" "$f" | head -1 | sed 's/.*: *//' | tr -d '"')
61
+        printf "  %-12s %s\n" "$name" "$display"
62
+    done
63
+    exit 0
64
+}
65
+
66
+# Parse arguments
67
+while [ $# -gt 0 ]; do
68
+    case "$1" in
69
+        --shell)      SHELL_BIN="$2"; shift 2 ;;
70
+        --profile)    PROFILE="$2"; shift 2 ;;
71
+        --suite)      SUITES="$2"; shift 2 ;;
72
+        --ref-shell)  REF_SHELL="$2"; shift 2 ;;
73
+        --report)     REPORT="$2"; shift 2 ;;
74
+        --verbose)    VERBOSE=1; shift ;;
75
+        --list-suites)   list_suites ;;
76
+        --list-profiles) list_profiles ;;
77
+        -h|--help)    usage ;;
78
+        *)            echo "Unknown option: $1"; usage ;;
79
+    esac
80
+done
81
+
82
+if [ -z "$SHELL_BIN" ]; then
83
+    echo "Error: --shell is required"
84
+    echo "Run: bensch --help"
85
+    exit 1
86
+fi
87
+
88
+if [ ! -x "$SHELL_BIN" ]; then
89
+    echo "Error: $SHELL_BIN is not executable"
90
+    exit 1
91
+fi
92
+
93
+# Auto-detect profile if not specified
94
+if [ -z "$PROFILE" ]; then
95
+    shell_basename=$(basename "$SHELL_BIN" | tr '[:upper:]' '[:lower:]')
96
+    case "$shell_basename" in
97
+        bash*)    PROFILE="bash" ;;
98
+        zsh*)     PROFILE="zsh" ;;
99
+        dash*)    PROFILE="dash" ;;
100
+        ksh*|mksh*|pdksh*) PROFILE="ksh" ;;
101
+        fortsh*)  PROFILE="fortsh" ;;
102
+        *)        PROFILE="generic" ;;
103
+    esac
104
+fi
105
+
106
+PROFILE_FILE="$SCRIPT_DIR/profiles/$PROFILE.yaml"
107
+if [ ! -f "$PROFILE_FILE" ]; then
108
+    echo "Warning: profile '$PROFILE' not found, using generic"
109
+    PROFILE="generic"
110
+    PROFILE_FILE="$SCRIPT_DIR/profiles/generic.yaml"
111
+fi
112
+
113
+# Export for child processes
114
+export SHELL_BIN
115
+export REF_SHELL
116
+export BENSCH_PROFILE="$PROFILE"
117
+
118
+# Build RC_DISABLE_ENV from profile
119
+RC_DISABLE_ENV=$(python3 -c "
120
+import yaml, sys
121
+with open('$PROFILE_FILE') as f:
122
+    p = yaml.safe_load(f)
123
+env = p.get('rc_disable', {}).get('env', {})
124
+print(' '.join(f'{k}={v}' for k, v in env.items()))
125
+" 2>/dev/null || echo "")
126
+export RC_DISABLE_ENV
127
+
128
+echo "╔══════════════════════════════════════════════════════════════╗"
129
+echo "║  bensch — POSIX Shell Testing Framework                     ║"
130
+echo "╚══════════════════════════════════════════════════════════════╝"
131
+echo ""
132
+echo "  Shell:   $SHELL_BIN"
133
+echo "  Profile: $PROFILE"
134
+echo "  Ref:     $REF_SHELL"
135
+echo "  Suites:  $SUITES"
136
+echo ""
137
+
138
+TOTAL_PASS=0
139
+TOTAL_FAIL=0
140
+
141
+# Run requested suites
142
+run_posix() {
143
+    echo "═══ POSIX Compliance Tests ═══"
144
+    if [ -x "$SCRIPT_DIR/suites/posix/run_posix_tests.sh" ]; then
145
+        SHELL_BIN="$SHELL_BIN" REF_SHELL="$REF_SHELL" RC_DISABLE_ENV="$RC_DISABLE_ENV" \
146
+            "$SCRIPT_DIR/suites/posix/run_posix_tests.sh"
147
+    else
148
+        echo "  [SKIP] run_posix_tests.sh not found"
149
+    fi
150
+}
151
+
152
+run_interactive() {
153
+    echo "═══ Interactive PTY Tests ═══"
154
+    if [ ! -d "$SCRIPT_DIR/framework" ]; then
155
+        echo "  [SKIP] framework/ not found"
156
+        return
157
+    fi
158
+
159
+    # Set up venv if needed
160
+    if [ ! -d "$SCRIPT_DIR/.venv" ]; then
161
+        python3 -m venv "$SCRIPT_DIR/.venv"
162
+        . "$SCRIPT_DIR/.venv/bin/activate"
163
+        pip install -q -r "$SCRIPT_DIR/framework/requirements.txt"
164
+    else
165
+        . "$SCRIPT_DIR/.venv/bin/activate"
166
+    fi
167
+
168
+    cd "$SCRIPT_DIR/framework"
169
+    python3 runner.py --shell "$SHELL_BIN" --spec-dir "$SCRIPT_DIR/suites/interactive"
170
+    cd "$SCRIPT_DIR"
171
+}
172
+
173
+run_builtins() {
174
+    echo "═══ Builtin Tests ═══"
175
+    if [ -x "$SCRIPT_DIR/suites/builtins/run_builtin_tests.sh" ]; then
176
+        SHELL_BIN="$SHELL_BIN" REF_SHELL="$REF_SHELL" \
177
+            "$SCRIPT_DIR/suites/builtins/run_builtin_tests.sh"
178
+    else
179
+        echo "  [SKIP] run_builtin_tests.sh not found"
180
+    fi
181
+}
182
+
183
+case "$SUITES" in
184
+    all)
185
+        run_posix
186
+        run_interactive
187
+        run_builtins
188
+        ;;
189
+    *)
190
+        IFS=','
191
+        for suite in $SUITES; do
192
+            case "$suite" in
193
+                posix)       run_posix ;;
194
+                interactive) run_interactive ;;
195
+                builtins)    run_builtins ;;
196
+                *)           echo "Unknown suite: $suite" ;;
197
+            esac
198
+        done
199
+        unset IFS
200
+        ;;
201
+esac
202
+
203
+echo ""
204
+echo "Done."