`648121e`

Add exact file and phase repro oracles

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 1 month ago

SHA: 648121e729a8d7a5045bef87db75dea3612ec935
Parents: 27bc208
Tree: 927ddb9

6 changed files

Status	File	+	-
M	`.docs/testing/testing02.md`	6	1
M	`README.md`	2	1
M	`test_programs/function_call.f90`	1	1
M	`test_programs/io_rewind.f90`	1	0
M	`tests/README.md`	1	0
M	`tests/run_programs.rs`	211	12

.docs/testing/testing02.mdmodified

    families without hardcoding every byte of formatted output
  - `FILE_RERUN_MODE` makes overwrite-vs-append intent explicit instead of
    relying on a human to infer it from the source
 +- `FILE_SET_EXACT` lets a test pin the full runtime side-effect surface,
 +  not just a handful of named files
  - `REPRO_CHECK: run_same_sandbox` makes repeated-run side effects explicit,
    so append-vs-replace and stale-file bugs can be tested deliberately
  - `PHASE_TRIANGULATE: ... | clean` requires compile-only surfaces to leave
    only their requested artifact in a private phase sandbox
 +- `PHASE_TRIANGULATE: ... | repro` requires compile-only surfaces to stay
 +  byte-identical across repeated materialization
  - that lets a runtime-side-effecting test prove `--emit-ir`, `-S`, and `-c`
 -  do not accidentally create the files that only linked execution should create
 +  do not accidentally create the files that only linked execution should create,
 +  while also keeping those compile-only outputs deterministic
  ## Goal

README.mdmodified

  - `! FILE_EXISTS:` / `! FILE_MISSING:` for explicit sandbox presence or absence
  - `! FILE_LINE_COUNT:` for structural file-shape assertions
  - `! FILE_RERUN_MODE:` for explicit overwrite vs append intent across reruns
 +- `! FILE_SET_EXACT:` for exact runtime side-effect file sets
  - `! REPRO_CHECK:` for per-test asm/object/run reproducibility
  - `! OPT_EQ:` for explicit cross-opt invariants
 -- `! PHASE_TRIANGULATE:` for same-opt IR/ASM/object availability and compile-cleanliness oracles
 +- `! PHASE_TRIANGULATE:` for same-opt IR/ASM/object availability, compile-cleanliness, and compile-only reproducibility oracles
  - `! IR_CHECK:` / `! IR_NOT:` for IR shape
  Those source comments are the canonical leaf-assertion language for the

test_programs/function_call.f90modified

  ! CHECK: 25
 -! PHASE_TRIANGULATE: ir|asm|obj
 +! PHASE_TRIANGULATE: ir|asm|obj|repro
  program test_func
      implicit none
      integer :: x, y

test_programs/io_rewind.f90modified

  ! FILE_MISSING: afs_rewind.tmp
  ! FILE_LINE_COUNT: afs_rewind.dat => 1
  ! FILE_RERUN_MODE: afs_rewind.dat => stable
 +! FILE_SET_EXACT: afs_rewind.dat
  program test_io_rewind
      implicit none
      integer :: x

tests/README.mdmodified

  - `! FILE_EXISTS:` / `! FILE_MISSING:`
  - `! FILE_LINE_COUNT:`
  - `! FILE_RERUN_MODE:`
 +- `! FILE_SET_EXACT:`
  - `! REPRO_CHECK:`
  - `! OPT_EQ:`
  - `! PHASE_TRIANGULATE:`

tests/run_programs.rsmodified

  //! should use stable substrings that are intentionally expected
  //! across the requested matrix.
  //!
 -//! ## FILE_CHECK / FILE_NOT / FILE_EXISTS / FILE_MISSING / FILE_LINE_COUNT / FILE_RERUN_MODE annotations
 +//! ## FILE_CHECK / FILE_NOT / FILE_EXISTS / FILE_MISSING / FILE_LINE_COUNT / FILE_RERUN_MODE / FILE_SET_EXACT annotations
  //!
  //! Runtime tests can assert on files created inside their per-test
  //! sandbox:
  //!     program is executed twice in the same sandbox, the named file must
  //!     either be byte-identical after both runs (`stable`) or grow by
  //!     strict append (`append`).
 +//!   * `! FILE_SET_EXACT: <relative-path>[,<relative-path>...]` — the
 +//!     final sandbox file set must match exactly, with no extra side
 +//!     effects beyond the listed relative paths.
  //!
  //! Paths are sandbox-relative on purpose. The harness runs each binary
  //! in a private temp directory, so file assertions pin side effects
  //!
  //!   * `! PHASE_TRIANGULATE: ir|asm|obj`
  //!   * `! PHASE_TRIANGULATE: ir|asm|obj|clean`
 +//!   * `! PHASE_TRIANGULATE: ir|asm|obj|repro`
  //!
  //! The linked runtime path is the anchor. If the program runs correctly but
  //! one of the requested extra surfaces fails to compile or produces an empty
  //! only their explicit output artifact in a private phase sandbox. That lets a
  //! runtime-side-effecting program assert that `--emit-ir`, `-S`, and `-c` do
  //! not accidentally create the files that only linked execution should create.
 +//!
 +//! `repro` strengthens it in a different direction: each requested compile-only
 +//! phase must produce byte-identical output across two independent compilations.
 +//! This keeps pipeline oracles from checking only "exists" when what we really
 +//! need is "exists, stays clean, and stays deterministic".
  use std::collections::BTreeMap;
  use std::fs;
      mode: FileRerunMode,
+ }
 +struct FileSetExactCheck {
 +    line_num: usize,
 +    rel_paths: Vec<String>,
 +}
++
  #[derive(Debug, Clone, Copy, PartialEq, Eq)]
  enum ReproStage {
      Asm,
      Asm,
      Obj,
      Clean,
 +    Repro,
+ }
  #[derive(Debug, Clone, PartialEq, Eq)]
      Ok(checks)
+ }
 +fn extract_file_set_exact(
 +    source: &str,
 +    filename: &str,
 +) -> Result<Option<FileSetExactCheck>, String> {
 +    let mut found: Option<FileSetExactCheck> = None;
 +    for (i, line) in source.lines().enumerate() {
 +        let line_num = i + 1;
 +        let trimmed = line.trim();
 +        let Some(rest) = trimmed.strip_prefix("! FILE_SET_EXACT:") else {
 +            continue;
 +        };
 +        if let Some(existing) = &found {
 +            return Err(format!(
 +                "{}:{}: multiple FILE_SET_EXACT annotations are not allowed (another at line {})",
 +                filename, line_num, existing.line_num
 +            ));
 +        }
++
 +        let mut rel_paths = Vec::new();
 +        for token in rest.trim().split(',') {
 +            let rel_path = token.trim();
 +            if rel_path.is_empty() {
 +                return Err(format!(
 +                    "{}:{}: FILE_SET_EXACT paths cannot be empty",
 +                    filename, line_num
 +                ));
 +            }
 +            if Path::new(rel_path).is_absolute() {
 +                return Err(format!(
 +                    "{}:{}: FILE_SET_EXACT paths must be relative, got '{}'",
 +                    filename, line_num, rel_path
 +                ));
 +            }
 +            if !rel_paths
 +                .iter()
 +                .any(|existing: &String| existing == rel_path)
 +            {
 +                rel_paths.push(rel_path.to_string());
 +            }
 +        }
 +        if rel_paths.is_empty() {
 +            return Err(format!(
 +                "{}:{}: FILE_SET_EXACT needs at least one relative path",
 +                filename, line_num
 +            ));
 +        }
 +        rel_paths.sort();
++
 +        found = Some(FileSetExactCheck {
 +            line_num,
 +            rel_paths,
 +        });
 +    }
 +    Ok(found)
 +}
++
  fn extract_repro_checks(source: &str, filename: &str) -> Result<Vec<ReproStage>, String> {
      let mut stages = Vec::new();
      for (i, line) in source.lines().enumerate() {
                  "asm" => PhaseSurface::Asm,
                  "obj" => PhaseSurface::Obj,
                  "clean" => PhaseSurface::Clean,
 +                "repro" => PhaseSurface::Repro,
                  other => {
                      return Err(format!(
 -                    "{}:{}: PHASE_TRIANGULATE surfaces must be ir, asm, obj, or clean; got '{}'",
 +                    "{}:{}: PHASE_TRIANGULATE surfaces must be ir, asm, obj, clean, or repro; got '{}'",
                      filename, line_num, other
                  ))
+                 }
+         }
          if surfaces
              .iter()
 -            .all(|surface| *surface == PhaseSurface::Clean)
 +            .all(|surface| matches!(surface, PhaseSurface::Clean | PhaseSurface::Repro))
+         {
              return Err(format!(
 -                "{}:{}: PHASE_TRIANGULATE(clean) needs at least one of ir, asm, or obj",
 +                "{}:{}: PHASE_TRIANGULATE policy-only annotations need at least one of ir, asm, or obj",
                  filename, line_num
              ));
+         }
      Ok(())
+ }
 +fn match_file_set_exact(
 +    check: &FileSetExactCheck,
 +    files: &BTreeMap<String, Vec<u8>>,
 +    filename: &str,
 +) -> Result<(), String> {
 +    let actual = files.keys().cloned().collect::<Vec<_>>();
 +    if actual != check.rel_paths {
 +        return Err(format!(
 +            "{}:{}: FILE_SET_EXACT failed: expected {:?}, got {:?}",
 +            filename, check.line_num, check.rel_paths, actual
 +        ));
 +    }
 +    Ok(())
 +}
++
  fn collect_declared_runtime_paths(
      file_checks: &[FileCheck],
      file_presence_checks: &[FilePresenceCheck],
      file_line_count_checks: &[FileLineCountCheck],
      file_rerun_mode_checks: &[FileRerunModeCheck],
 +    file_set_exact: Option<&FileSetExactCheck>,
  ) -> BTreeMap<String, String> {
      let mut paths = BTreeMap::new();
      for check in file_checks {
              .entry(check.rel_path.clone())
              .or_insert_with(|| "FILE_RERUN_MODE".to_string());
+     }
 +    if let Some(check) = file_set_exact {
 +        for rel_path in &check.rel_paths {
 +            paths
 +                .entry(rel_path.clone())
 +                .or_insert_with(|| "FILE_SET_EXACT".to_string());
 +        }
 +    }
      paths
+ }
          PhaseSurface::Asm => "asm",
          PhaseSurface::Obj => "obj",
          PhaseSurface::Clean => unreachable!("clean is a triangulation policy, not an artifact"),
 +        PhaseSurface::Repro => unreachable!("repro is a triangulation policy, not an artifact"),
      };
      let phase_sandbox = unique_temp_path(
          "phase_sandbox",
          PhaseSurface::Asm => ("phase-output.s", &["-S"]),
          PhaseSurface::Obj => ("phase-output.o", &["-c"]),
          PhaseSurface::Clean => unreachable!(),
 +        PhaseSurface::Repro => unreachable!(),
      };
      let output_path = phase_sandbox.join(output_name);
              PhaseSurface::Asm => "asm",
              PhaseSurface::Obj => "obj",
              PhaseSurface::Clean => "clean",
 +            PhaseSurface::Repro => "repro",
          })
          .collect::<Vec<_>>()
          .join("|")
      declared_runtime_paths: &BTreeMap<String, String>,
  ) -> Result<(), String> {
      let require_clean = triangulation.surfaces.contains(&PhaseSurface::Clean);
 +    let require_repro = triangulation.surfaces.contains(&PhaseSurface::Repro);
      for surface in &triangulation.surfaces {
          match surface {
              PhaseSurface::Ir | PhaseSurface::Asm | PhaseSurface::Obj => {
                          PhaseSurface::Asm => "assembly",
                          PhaseSurface::Obj => "object",
                          PhaseSurface::Clean => unreachable!(),
 +                        PhaseSurface::Repro => unreachable!(),
                      };
                      return Err(format!(
                          "{}:{}: PHASE_TRIANGULATE({}) produced empty {} output at {}",
                          ));
+                     }
+                 }
 +                if require_repro {
 +                    let second_artifact =
 +                        compile_phase_artifact(compiler, source, opt_flag, *surface, filename)?;
 +                    if require_clean {
 +                        let second_file_keys: Vec<&str> = second_artifact
 +                            .sandbox_files
 +                            .keys()
 +                            .map(|key| key.as_str())
 +                            .collect();
 +                        if second_file_keys != vec![second_artifact.output_rel_path.as_str()] {
 +                            return Err(format!(
 +                                "{}:{}: PHASE_TRIANGULATE({}) failed at {}: repeated compile-only phase left unexpected files {:?} (expected only '{}')",
 +                                filename,
 +                                triangulation.line_num,
 +                                render_phase_surfaces(&triangulation.surfaces),
 +                                opt_flag,
 +                                second_file_keys,
 +                                second_artifact.output_rel_path,
 +                            ));
 +                        }
 +                    }
 +                    if artifact.output_bytes != second_artifact.output_bytes {
 +                        let surface_name = match surface {
 +                            PhaseSurface::Ir => "IR",
 +                            PhaseSurface::Asm => "assembly",
 +                            PhaseSurface::Obj => "object",
 +                            PhaseSurface::Clean => unreachable!(),
 +                            PhaseSurface::Repro => unreachable!(),
 +                        };
 +                        return Err(format!(
 +                            "{}:{}: PHASE_TRIANGULATE({}) failed at {}: {} output changed across repeated compile-only runs",
 +                            filename,
 +                            triangulation.line_num,
 +                            render_phase_surfaces(&triangulation.surfaces),
 +                            opt_flag,
 +                            surface_name,
 +                        ));
 +                    }
 +                }
+             }
 -            PhaseSurface::Clean => {}
 +            PhaseSurface::Clean | PhaseSurface::Repro => {}
+         }
+     }
      Ok(())
          Ok(checks) => checks,
          Err(e) => return TestOutcome::Fail(e),
      };
 +    let file_set_exact = match extract_file_set_exact(&source_text, filename) {
 +        Ok(check) => check,
 +        Err(e) => return TestOutcome::Fail(e),
 +    };
      let repro_checks = match extract_repro_checks(&source_text, filename) {
          Ok(checks) => checks,
          Err(e) => return TestOutcome::Fail(e),
          && file_presence_checks.is_empty()
          && file_line_count_checks.is_empty()
          && file_rerun_mode_checks.is_empty()
 +        && file_set_exact.is_none()
          && repro_checks.is_empty()
          && opt_eq_rules.is_empty()
          && phase_triangulation.is_none()
          // Programs with no runtime or shape assertions, no XFAIL marker,
          // and no ERROR marker are mis-configured tests, not test failures.
          return TestOutcome::Fail(format!(
 -            "{}: no CHECK / STDERR_CHECK / EXIT_CODE / IR_CHECK / ASM_CHECK / FILE_CHECK / FILE_EXISTS / FILE_MISSING / FILE_LINE_COUNT / FILE_RERUN_MODE / REPRO_CHECK / XFAIL / ERROR_EXPECTED / ERROR_SPAN annotations",
 +            "{}: no CHECK / STDERR_CHECK / EXIT_CODE / IR_CHECK / ASM_CHECK / FILE_CHECK / FILE_EXISTS / FILE_MISSING / FILE_LINE_COUNT / FILE_RERUN_MODE / FILE_SET_EXACT / REPRO_CHECK / XFAIL / ERROR_EXPECTED / ERROR_SPAN annotations",
              filename,
          ));
+     }
              let _ = fs::remove_dir_all(&sandbox);
              return Err(e);
+         }
 +        if let Some(check) = &file_set_exact {
 +            if let Err(e) = match_file_set_exact(check, &snapshot.files, &label) {
 +                let _ = fs::remove_file(&binary);
 +                let _ = fs::remove_dir_all(&sandbox);
 +                return Err(e);
 +            }
 +        }
          if !file_rerun_mode_checks.is_empty() {
              let second = run_binary_in_sandbox(&binary, &sandbox, filename)?;
              if snapshot.exit_code != second.exit_code {
                  &file_presence_checks,
                  &file_line_count_checks,
                  &file_rerun_mode_checks,
 +                file_set_exact.as_ref(),
              );
              run_phase_triangulation(
                  compiler,
      assert_eq!(checks[1].mode, FileRerunMode::Append);
+ }
 +#[test]
 +fn extract_file_set_exact_accepts_relative_paths() {
 +    let check = extract_file_set_exact("! FILE_SET_EXACT: out.txt, log.txt\n", "inline.f90")
 +        .unwrap()
 +        .unwrap();
 +    assert_eq!(check.rel_paths, vec!["log.txt", "out.txt"]);
 +}
++
  #[test]
  fn file_rerun_mode_matcher_accepts_strict_append_growth() {
      let checks = vec![FileRerunModeCheck {
+ }
  #[test]
 -fn extract_phase_triangulation_accepts_ir_asm_obj_and_clean() {
 -    let source = "! PHASE_TRIANGULATE: ir|asm|obj|clean\n";
 +fn extract_phase_triangulation_accepts_ir_asm_obj_clean_and_repro() {
 +    let source = "! PHASE_TRIANGULATE: ir|asm|obj|clean|repro\n";
      let rule = extract_phase_triangulation(source, "inline.f90")
          .unwrap()
          .unwrap();
              PhaseSurface::Ir,
              PhaseSurface::Asm,
              PhaseSurface::Obj,
 -            PhaseSurface::Clean
 +            PhaseSurface::Clean,
 +            PhaseSurface::Repro
+         ]
      );
+ }
  fn extract_phase_triangulation_rejects_unknown_surface() {
      let source = "! PHASE_TRIANGULATE: run\n";
      let err = extract_phase_triangulation(source, "inline.f90").unwrap_err();
 -    assert!(err.contains("ir, asm, obj, or clean"));
 +    assert!(err.contains("ir, asm, obj, clean, or repro"));
+ }
  #[test]
  fn extract_phase_triangulation_rejects_clean_only() {
 -    let source = "! PHASE_TRIANGULATE: clean\n";
 +    let source = "! PHASE_TRIANGULATE: clean|repro\n";
      let err = extract_phase_triangulation(source, "inline.f90").unwrap_err();
 -    assert!(err.contains("needs at least one of ir, asm, or obj"));
 +    assert!(err.contains("policy-only annotations"));
+ }
  #[test]
+     }
+ }
 +#[test]
 +fn file_set_exact_allows_rewind_single_output() {
 +    let compiler = find_compiler();
 +    let test_dir = find_test_programs();
 +    let source = test_dir.join("io_rewind.f90");
 +    assert!(
 +        source.exists(),
 +        "io_rewind.f90 missing — needed for FILE_SET_EXACT coverage"
 +    );
++
 +    match run_test(&compiler, &source, "-O0") {
 +        TestOutcome::Pass => {}
 +        other => panic!(
 +            "io_rewind.f90 should pass with FILE_SET_EXACT coverage, got {:?}",
 +            other
 +        ),
 +    }
 +}
++
  #[test]
  fn file_rerun_mode_append_fixture_tracks_current_compiler_gap() {
      let compiler = find_compiler();
+     }
+ }
 +#[test]
 +fn phase_triangulation_repro_keeps_function_call_compile_surfaces_stable() {
 +    let compiler = find_compiler();
 +    let test_dir = find_test_programs();
 +    let source = test_dir.join("function_call.f90");
 +    assert!(
 +        source.exists(),
 +        "function_call.f90 missing — needed for PHASE_TRIANGULATE(repro) coverage"
 +    );
++
 +    match run_test(&compiler, &source, "-O0") {
 +        TestOutcome::Pass => {}
 +        other => panic!(
 +            "function_call.f90 should pass with PHASE_TRIANGULATE(repro) coverage, got {:?}",
 +            other
 +        ),
 +    }
 +}
++
  #[test]
  fn phase_triangulation_clean_keeps_compile_phases_free_of_runtime_files() {
      let compiler = find_compiler();