tests/golden: scaffold — 2-probe spec (delta_kl + calibration_drift) for S18 golden
- SHA
9fe7f3258ae54fb097e1ba5931bf74a30df18860- Parents
-
50b26bf - Tree
750940c
tenseleyflow/sway
/
9fe7f32
Browse files
2 changed files
| Status | File | + | - |
|---|---|---|---|
| A |
tests/golden/__init__.py
|
9 | 0 |
| A |
tests/golden/spec.yaml
|
46 | 0 |
tests/golden/__init__.pyadded@@ -0,0 +1,9 @@ | ||
| 1 | +"""Cross-platform determinism golden fixtures (S18). | |
| 2 | + | |
| 3 | +Contains the minimal spec + platform-pinned ``expected_<platform>.json`` | |
| 4 | +files that ``tests/integration/test_determinism_golden.py`` diffs against. | |
| 5 | + | |
| 6 | +Regeneration: set ``SWAY_UPDATE_GOLDENS=1`` when running the golden | |
| 7 | +test, or dispatch the ``determinism-golden`` CI workflow with | |
| 8 | +``regenerate_goldens=true`` and commit the resulting artifact. | |
| 9 | +""" | |
tests/golden/spec.yamladded@@ -0,0 +1,46 @@ | ||
| 1 | +# Determinism golden spec (S18). | |
| 2 | +# | |
| 3 | +# Pins a minimal 2-probe suite the CI runs on both ubuntu-latest and | |
| 4 | +# macos-latest-xlarge to catch silent algorithm drift. Intentionally | |
| 5 | +# small: one delta_kl probe + one calibration_drift probe, 4 prompts | |
| 6 | +# each, no null_adapter (null adds runtime without improving the | |
| 7 | +# drift-detection signal — the golden already encodes the exact | |
| 8 | +# raw/score values). | |
| 9 | +# | |
| 10 | +# ``models.base.base`` and ``models.ft.adapter`` are placeholders — | |
| 11 | +# the golden test's fixture substitutes paths at runtime so the | |
| 12 | +# checked-in spec stays portable. | |
| 13 | + | |
| 14 | +version: 1 | |
| 15 | +models: | |
| 16 | + base: | |
| 17 | + base: placeholder-base | |
| 18 | + kind: hf | |
| 19 | + adapter: placeholder-base-adapter | |
| 20 | + dtype: fp32 | |
| 21 | + device: cpu | |
| 22 | + ft: | |
| 23 | + base: placeholder-base | |
| 24 | + kind: hf | |
| 25 | + adapter: placeholder-ft-adapter | |
| 26 | + dtype: fp32 | |
| 27 | + device: cpu | |
| 28 | +defaults: | |
| 29 | + seed: 0 | |
| 30 | + differential: true | |
| 31 | +suite: | |
| 32 | + - name: dk_golden | |
| 33 | + kind: delta_kl | |
| 34 | + prompts: | |
| 35 | + - The capital of France is | |
| 36 | + - Water boils at a temperature of | |
| 37 | + - The sun rises in the | |
| 38 | + - Python decorators are useful for | |
| 39 | + divergence: js | |
| 40 | + assert_mean_gte: 0.0 | |
| 41 | + - name: cal_golden | |
| 42 | + kind: calibration_drift | |
| 43 | + items_limit: 20 | |
| 44 | + regression_nats: 1.0 | |
| 45 | + assert_mean_delta_gte: -0.5 | |
| 46 | + assert_fraction_regressed_lt: 0.15 | |