| 1 | mod compiler; |
| 2 | |
| 3 | use std::collections::{BTreeMap, BTreeSet, VecDeque}; |
| 4 | use std::fs; |
| 5 | use std::path::{Path, PathBuf}; |
| 6 | use std::process::Command; |
| 7 | use std::sync::atomic::{AtomicU64, Ordering}; |
| 8 | |
| 9 | use crate::compiler::{ |
| 10 | linked_capture_available, object_snapshot_text, ArmfortasAdapters, ArmfortasCliAdapter, |
| 11 | CaptureBackend, CaptureFailure, CaptureRequest, CaptureResult, CapturedStage, |
| 12 | CliObservableCaptureBackend, EmitMode, FailureStage, OptLevel, RunCapture, Stage, |
| 13 | }; |
| 14 | use bencch_core::{ |
| 15 | ArtifactDifference, ArtifactKey, ArtifactValue, ComparisonResult, CompilerCapabilities, |
| 16 | CompilerObservation, CompilerSpec, NamedCompiler, ObservationProvenance, |
| 17 | }; |
| 18 | |
| 19 | const SUITE_EXTENSION: &str = "afs"; |
| 20 | |
| 21 | static REPORT_COUNTER: AtomicU64 = AtomicU64::new(0); |
| 22 | |
| 23 | #[derive(Debug, Clone)] |
| 24 | struct SuiteSpec { |
| 25 | name: String, |
| 26 | path: PathBuf, |
| 27 | cases: Vec<CaseSpec>, |
| 28 | } |
| 29 | |
| 30 | #[derive(Debug, Clone)] |
| 31 | struct CaseSpec { |
| 32 | name: String, |
| 33 | source: PathBuf, |
| 34 | graph_files: Vec<PathBuf>, |
| 35 | requested: BTreeSet<Stage>, |
| 36 | generic_introspect: Option<GenericIntrospectCase>, |
| 37 | generic_compare: Option<GenericCompareCase>, |
| 38 | opt_levels: Vec<OptLevel>, |
| 39 | repeat_count: usize, |
| 40 | reference_compilers: Vec<ReferenceCompiler>, |
| 41 | consistency_checks: Vec<ConsistencyCheck>, |
| 42 | expectations: Vec<Expectation>, |
| 43 | status_rules: Vec<StatusRule>, |
| 44 | capability_policy: Option<CapabilityPolicy>, |
| 45 | } |
| 46 | |
| 47 | impl CaseSpec { |
| 48 | fn is_graph(&self) -> bool { |
| 49 | !self.graph_files.is_empty() |
| 50 | } |
| 51 | |
| 52 | fn is_generic_introspect(&self) -> bool { |
| 53 | self.generic_introspect.is_some() |
| 54 | } |
| 55 | |
| 56 | fn is_generic_compare(&self) -> bool { |
| 57 | self.generic_compare.is_some() |
| 58 | } |
| 59 | |
| 60 | fn source_label(&self) -> String { |
| 61 | if self.is_graph() { |
| 62 | format!( |
| 63 | "graph entry {} ({} files)", |
| 64 | self.source.display(), |
| 65 | self.graph_files.len() |
| 66 | ) |
| 67 | } else { |
| 68 | self.source.display().to_string() |
| 69 | } |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | #[derive(Debug, Clone)] |
| 74 | struct GenericIntrospectCase { |
| 75 | compiler: CompilerSpec, |
| 76 | artifacts: BTreeSet<ArtifactKey>, |
| 77 | } |
| 78 | |
| 79 | #[derive(Debug, Clone)] |
| 80 | struct GenericCompareCase { |
| 81 | left: CompilerSpec, |
| 82 | right: CompilerSpec, |
| 83 | artifacts: BTreeSet<ArtifactKey>, |
| 84 | } |
| 85 | |
| 86 | #[derive(Debug, Clone)] |
| 87 | struct PreparedInput { |
| 88 | compiler_source: PathBuf, |
| 89 | generated_source: Option<PathBuf>, |
| 90 | temp_root: Option<PathBuf>, |
| 91 | } |
| 92 | |
| 93 | #[derive(Debug, Clone)] |
| 94 | struct StatusRule { |
| 95 | kind: StatusKind, |
| 96 | selector: OptSelector, |
| 97 | reason: String, |
| 98 | } |
| 99 | |
| 100 | #[derive(Debug, Clone)] |
| 101 | struct CapabilityPolicy { |
| 102 | kind: StatusKind, |
| 103 | reason: String, |
| 104 | } |
| 105 | |
| 106 | #[derive(Debug, Clone)] |
| 107 | enum PendingStatusRule { |
| 108 | Explicit(StatusRule), |
| 109 | XfailSourceComments, |
| 110 | } |
| 111 | |
| 112 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 113 | enum StatusKind { |
| 114 | Xfail, |
| 115 | Future, |
| 116 | } |
| 117 | |
| 118 | #[derive(Debug, Clone)] |
| 119 | enum OptSelector { |
| 120 | All, |
| 121 | Only(Vec<OptLevel>), |
| 122 | } |
| 123 | |
| 124 | impl OptSelector { |
| 125 | fn matches(&self, opt_level: OptLevel) -> bool { |
| 126 | match self { |
| 127 | Self::All => true, |
| 128 | Self::Only(levels) => levels.contains(&opt_level), |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | #[derive(Debug, Clone)] |
| 134 | enum EffectiveStatus { |
| 135 | Normal, |
| 136 | Xfail(String), |
| 137 | Future(String), |
| 138 | } |
| 139 | |
| 140 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] |
| 141 | enum ConsistencyCheck { |
| 142 | CliObjVsSystemAs, |
| 143 | CliAsmReproducible, |
| 144 | CliObjReproducible, |
| 145 | CliRunReproducible, |
| 146 | CaptureAsmVsCliAsm, |
| 147 | CaptureObjVsCliObj, |
| 148 | CaptureRunVsCliRun, |
| 149 | CaptureAsmReproducible, |
| 150 | CaptureObjReproducible, |
| 151 | CaptureRunReproducible, |
| 152 | } |
| 153 | |
| 154 | impl ConsistencyCheck { |
| 155 | fn parse(name: &str) -> Option<Self> { |
| 156 | match name.trim().to_ascii_lowercase().as_str() { |
| 157 | "cli_obj_vs_system_as" | "cli-obj-vs-system-as" => Some(Self::CliObjVsSystemAs), |
| 158 | "cli_asm_reproducible" | "cli-asm-reproducible" => Some(Self::CliAsmReproducible), |
| 159 | "cli_obj_reproducible" | "cli-obj-reproducible" => Some(Self::CliObjReproducible), |
| 160 | "cli_run_reproducible" | "cli-run-reproducible" => Some(Self::CliRunReproducible), |
| 161 | "capture_asm_vs_cli_asm" | "capture-asm-vs-cli-asm" => Some(Self::CaptureAsmVsCliAsm), |
| 162 | "capture_obj_vs_cli_obj" | "capture-obj-vs-cli-obj" => Some(Self::CaptureObjVsCliObj), |
| 163 | "capture_run_vs_cli_run" | "capture-run-vs-cli-run" => Some(Self::CaptureRunVsCliRun), |
| 164 | "capture_asm_reproducible" | "capture-asm-reproducible" => { |
| 165 | Some(Self::CaptureAsmReproducible) |
| 166 | } |
| 167 | "capture_obj_reproducible" | "capture-obj-reproducible" => { |
| 168 | Some(Self::CaptureObjReproducible) |
| 169 | } |
| 170 | "capture_run_reproducible" | "capture-run-reproducible" => { |
| 171 | Some(Self::CaptureRunReproducible) |
| 172 | } |
| 173 | _ => None, |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | fn as_str(&self) -> &'static str { |
| 178 | match self { |
| 179 | Self::CliObjVsSystemAs => "cli_obj_vs_system_as", |
| 180 | Self::CliAsmReproducible => "cli_asm_reproducible", |
| 181 | Self::CliObjReproducible => "cli_obj_reproducible", |
| 182 | Self::CliRunReproducible => "cli_run_reproducible", |
| 183 | Self::CaptureAsmVsCliAsm => "capture_asm_vs_cli_asm", |
| 184 | Self::CaptureObjVsCliObj => "capture_obj_vs_cli_obj", |
| 185 | Self::CaptureRunVsCliRun => "capture_run_vs_cli_run", |
| 186 | Self::CaptureAsmReproducible => "capture_asm_reproducible", |
| 187 | Self::CaptureObjReproducible => "capture_obj_reproducible", |
| 188 | Self::CaptureRunReproducible => "capture_run_reproducible", |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | fn required_stage(&self) -> Option<Stage> { |
| 193 | match self { |
| 194 | Self::CliObjVsSystemAs |
| 195 | | Self::CliAsmReproducible |
| 196 | | Self::CliObjReproducible |
| 197 | | Self::CliRunReproducible => None, |
| 198 | Self::CaptureAsmVsCliAsm | Self::CaptureAsmReproducible => Some(Stage::Asm), |
| 199 | Self::CaptureObjVsCliObj | Self::CaptureObjReproducible => Some(Stage::Obj), |
| 200 | Self::CaptureRunVsCliRun | Self::CaptureRunReproducible => Some(Stage::Run), |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | fn requires_capture_result(&self) -> bool { |
| 205 | matches!( |
| 206 | self, |
| 207 | Self::CaptureAsmVsCliAsm |
| 208 | | Self::CaptureObjVsCliObj |
| 209 | | Self::CaptureRunVsCliRun |
| 210 | | Self::CaptureAsmReproducible |
| 211 | | Self::CaptureObjReproducible |
| 212 | | Self::CaptureRunReproducible |
| 213 | ) |
| 214 | } |
| 215 | |
| 216 | fn supports_generic_introspect(&self) -> bool { |
| 217 | matches!( |
| 218 | self, |
| 219 | Self::CliAsmReproducible | Self::CliObjReproducible | Self::CliRunReproducible |
| 220 | ) |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | #[derive(Debug, Clone)] |
| 225 | enum Expectation { |
| 226 | CheckComments(Target), |
| 227 | Contains { target: Target, needle: String }, |
| 228 | NotContains { target: Target, needle: String }, |
| 229 | Equals { target: Target, value: String }, |
| 230 | IntEquals { target: Target, value: i32 }, |
| 231 | FailContains { stage: FailureStage, needle: String }, |
| 232 | FailEquals { stage: FailureStage, value: String }, |
| 233 | FailSourceComments, |
| 234 | FailCommentPatterns(Vec<String>), |
| 235 | } |
| 236 | |
| 237 | #[derive(Debug, Clone)] |
| 238 | enum Target { |
| 239 | Stage(Stage), |
| 240 | Artifact(ArtifactKey), |
| 241 | CompareStatus, |
| 242 | CompareClassification, |
| 243 | CompareChangedArtifacts, |
| 244 | CompareDifferenceCount, |
| 245 | CompareBasis, |
| 246 | RunStdout, |
| 247 | RunStderr, |
| 248 | RunExitCode, |
| 249 | } |
| 250 | |
| 251 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] |
| 252 | enum ReferenceCompiler { |
| 253 | Gfortran, |
| 254 | FlangNew, |
| 255 | } |
| 256 | |
| 257 | impl ReferenceCompiler { |
| 258 | fn parse(name: &str) -> Option<Self> { |
| 259 | match name.trim().to_ascii_lowercase().as_str() { |
| 260 | "gfortran" => Some(Self::Gfortran), |
| 261 | "flang-new" | "flang_new" | "flang" => Some(Self::FlangNew), |
| 262 | _ => None, |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | fn binary_name(&self) -> &'static str { |
| 267 | match self { |
| 268 | Self::Gfortran => "gfortran", |
| 269 | Self::FlangNew => "flang-new", |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | fn as_str(&self) -> &'static str { |
| 274 | self.binary_name() |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 279 | struct ToolchainConfig { |
| 280 | armfortas: ArmfortasCliAdapter, |
| 281 | gfortran: String, |
| 282 | flang_new: String, |
| 283 | lfortran: String, |
| 284 | ifort: String, |
| 285 | ifx: String, |
| 286 | nvfortran: String, |
| 287 | system_as: String, |
| 288 | otool: String, |
| 289 | nm: String, |
| 290 | } |
| 291 | |
| 292 | impl ToolchainConfig { |
| 293 | fn from_env() -> Self { |
| 294 | Self { |
| 295 | armfortas: match std::env::var("BENCCH_ARMFORTAS_BIN") { |
| 296 | Ok(value) if !value.trim().is_empty() => ArmfortasCliAdapter::External(value), |
| 297 | _ if linked_capture_available() => ArmfortasCliAdapter::Linked, |
| 298 | _ => ArmfortasCliAdapter::External("armfortas".into()), |
| 299 | }, |
| 300 | gfortran: tool_override("BENCCH_GFORTRAN_BIN", "gfortran"), |
| 301 | flang_new: tool_override("BENCCH_FLANG_BIN", "flang-new"), |
| 302 | lfortran: tool_override("BENCCH_LFORTRAN_BIN", "lfortran"), |
| 303 | ifort: tool_override("BENCCH_IFORT_BIN", "ifort"), |
| 304 | ifx: tool_override("BENCCH_IFX_BIN", "ifx"), |
| 305 | nvfortran: tool_override("BENCCH_NVFORTRAN_BIN", "nvfortran"), |
| 306 | system_as: tool_override("BENCCH_AS_BIN", "as"), |
| 307 | otool: tool_override("BENCCH_OTOOL_BIN", "otool"), |
| 308 | nm: tool_override("BENCCH_NM_BIN", "nm"), |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | fn armfortas_adapters(&self) -> ArmfortasAdapters { |
| 313 | ArmfortasAdapters::new(self.armfortas.clone()) |
| 314 | } |
| 315 | |
| 316 | fn cli_observable_capture_backend(&self, work_root: PathBuf) -> CliObservableCaptureBackend { |
| 317 | CliObservableCaptureBackend::new( |
| 318 | self.armfortas.clone(), |
| 319 | work_root, |
| 320 | self.otool.clone(), |
| 321 | self.nm.clone(), |
| 322 | ) |
| 323 | } |
| 324 | |
| 325 | fn reference_binary(&self, compiler: ReferenceCompiler) -> &str { |
| 326 | match compiler { |
| 327 | ReferenceCompiler::Gfortran => &self.gfortran, |
| 328 | ReferenceCompiler::FlangNew => &self.flang_new, |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | fn system_as_bin(&self) -> &str { |
| 333 | &self.system_as |
| 334 | } |
| 335 | |
| 336 | fn otool_bin(&self) -> &str { |
| 337 | &self.otool |
| 338 | } |
| 339 | |
| 340 | fn nm_bin(&self) -> &str { |
| 341 | &self.nm |
| 342 | } |
| 343 | |
| 344 | fn named_compiler_binary(&self, compiler: NamedCompiler) -> Option<String> { |
| 345 | match compiler { |
| 346 | NamedCompiler::Armfortas => match &self.armfortas { |
| 347 | ArmfortasCliAdapter::Linked => None, |
| 348 | ArmfortasCliAdapter::External(binary) => Some(binary.clone()), |
| 349 | }, |
| 350 | NamedCompiler::Gfortran => Some(self.gfortran.clone()), |
| 351 | NamedCompiler::FlangNew => Some(self.flang_new.clone()), |
| 352 | NamedCompiler::LFortran => Some(self.lfortran.clone()), |
| 353 | NamedCompiler::Ifort => Some(self.ifort.clone()), |
| 354 | NamedCompiler::Ifx => Some(self.ifx.clone()), |
| 355 | NamedCompiler::Nvfortran => Some(self.nvfortran.clone()), |
| 356 | } |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | fn generic_external_capabilities(spec: CompilerSpec) -> CompilerCapabilities { |
| 361 | CompilerCapabilities::new(spec).support_all([ |
| 362 | ArtifactKey::Diagnostics, |
| 363 | ArtifactKey::ExitCode, |
| 364 | ArtifactKey::Stdout, |
| 365 | ArtifactKey::Stderr, |
| 366 | ArtifactKey::Asm, |
| 367 | ArtifactKey::Obj, |
| 368 | ArtifactKey::Executable, |
| 369 | ArtifactKey::Runtime, |
| 370 | ]) |
| 371 | } |
| 372 | |
| 373 | fn armfortas_capabilities(tools: &ToolchainConfig) -> CompilerCapabilities { |
| 374 | let mut capabilities = |
| 375 | generic_external_capabilities(CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 376 | let linked_reason = "linked armfortas capture is unavailable in this build; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary".to_string(); |
| 377 | let capture_available = tools.armfortas_adapters().capture_mode_name() != "unavailable"; |
| 378 | for stage in Stage::ALL { |
| 379 | if matches!(stage, Stage::Asm | Stage::Obj | Stage::Run) { |
| 380 | continue; |
| 381 | } |
| 382 | let artifact = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 383 | capabilities = if capture_available { |
| 384 | capabilities.support(artifact) |
| 385 | } else { |
| 386 | capabilities.mark_unavailable(artifact, linked_reason.clone()) |
| 387 | }; |
| 388 | } |
| 389 | capabilities |
| 390 | } |
| 391 | |
| 392 | fn compiler_capabilities(spec: &CompilerSpec, tools: &ToolchainConfig) -> CompilerCapabilities { |
| 393 | match spec { |
| 394 | CompilerSpec::Named(NamedCompiler::Armfortas) => armfortas_capabilities(tools), |
| 395 | CompilerSpec::Named(named) => generic_external_capabilities(CompilerSpec::Named(*named)), |
| 396 | CompilerSpec::Binary(path) => { |
| 397 | generic_external_capabilities(CompilerSpec::Binary(path.clone())) |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | fn capability_extra_summary(extras: &BTreeMap<String, Vec<String>>) -> String { |
| 403 | if extras.is_empty() { |
| 404 | return "none".to_string(); |
| 405 | } |
| 406 | |
| 407 | extras |
| 408 | .iter() |
| 409 | .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 410 | .collect::<Vec<_>>() |
| 411 | .join(", ") |
| 412 | } |
| 413 | |
| 414 | fn capability_unavailable_summary(capabilities: &CompilerCapabilities) -> String { |
| 415 | if capabilities.unavailable_artifacts.is_empty() { |
| 416 | return "none".to_string(); |
| 417 | } |
| 418 | |
| 419 | let mut grouped = BTreeMap::<String, Vec<String>>::new(); |
| 420 | for artifact in capabilities.unavailable_artifacts.keys() { |
| 421 | if let Some((namespace, local_name)) = artifact.extra_parts() { |
| 422 | grouped |
| 423 | .entry(namespace.to_string()) |
| 424 | .or_insert_with(Vec::new) |
| 425 | .push(local_name.to_string()); |
| 426 | } else { |
| 427 | grouped |
| 428 | .entry("generic".to_string()) |
| 429 | .or_insert_with(Vec::new) |
| 430 | .push(artifact.as_str().to_string()); |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | grouped |
| 435 | .iter() |
| 436 | .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 437 | .collect::<Vec<_>>() |
| 438 | .join(", ") |
| 439 | } |
| 440 | |
| 441 | fn named_compiler_status_value( |
| 442 | named: NamedCompiler, |
| 443 | tools: &ToolchainConfig, |
| 444 | capture_root: Option<&PathBuf>, |
| 445 | ) -> String { |
| 446 | match named { |
| 447 | NamedCompiler::Armfortas => match &tools.armfortas { |
| 448 | ArmfortasCliAdapter::Linked => capture_root |
| 449 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 450 | .unwrap_or_else(|| { |
| 451 | "linked adapter requested but unavailable in this build".to_string() |
| 452 | }), |
| 453 | ArmfortasCliAdapter::External(binary) => tool_probe_status(binary, false), |
| 454 | }, |
| 455 | _ => tool_probe_status( |
| 456 | &tools |
| 457 | .named_compiler_binary(named) |
| 458 | .unwrap_or_else(|| named.as_str().to_string()), |
| 459 | false, |
| 460 | ), |
| 461 | } |
| 462 | } |
| 463 | |
| 464 | fn append_named_compiler_fields( |
| 465 | fields: &mut Vec<(String, String)>, |
| 466 | named: NamedCompiler, |
| 467 | tools: &ToolchainConfig, |
| 468 | capture_root: Option<&PathBuf>, |
| 469 | ) { |
| 470 | let prefix = format!("named_compiler.{}", named.as_str()); |
| 471 | let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 472 | if named == NamedCompiler::Armfortas { |
| 473 | let armfortas = tools.armfortas_adapters(); |
| 474 | fields.push(( |
| 475 | prefix.clone(), |
| 476 | format!( |
| 477 | "cli={} capture={}", |
| 478 | armfortas.cli_mode_name(), |
| 479 | armfortas.capture_mode_name() |
| 480 | ), |
| 481 | )); |
| 482 | } else { |
| 483 | fields.push(( |
| 484 | prefix.clone(), |
| 485 | named_compiler_status_value(named, tools, capture_root), |
| 486 | )); |
| 487 | } |
| 488 | fields.push(( |
| 489 | format!("{}.accepted_names", prefix), |
| 490 | named.accepted_names().join(", "), |
| 491 | )); |
| 492 | fields.push(( |
| 493 | format!("{}.candidate_binaries", prefix), |
| 494 | named.candidate_binaries().join(", "), |
| 495 | )); |
| 496 | fields.push(( |
| 497 | format!("{}.generic_artifacts", prefix), |
| 498 | format_artifact_name_list(&capabilities.generic_artifacts()), |
| 499 | )); |
| 500 | fields.push(( |
| 501 | format!("{}.adapter_extras", prefix), |
| 502 | capability_extra_summary(&capabilities.adapter_extras()), |
| 503 | )); |
| 504 | fields.push(( |
| 505 | format!("{}.unavailable_artifacts", prefix), |
| 506 | capability_unavailable_summary(&capabilities), |
| 507 | )); |
| 508 | } |
| 509 | |
| 510 | fn compiler_capability_backend(spec: &CompilerSpec, tools: &ToolchainConfig) -> (String, String) { |
| 511 | match spec { |
| 512 | CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 513 | let adapters = tools.armfortas_adapters(); |
| 514 | ( |
| 515 | adapters.capture_mode_name().to_string(), |
| 516 | adapters.capture_description().to_string(), |
| 517 | ) |
| 518 | } |
| 519 | CompilerSpec::Named(named) => { |
| 520 | let binary = tools |
| 521 | .named_compiler_binary(*named) |
| 522 | .unwrap_or_else(|| named.as_str().to_string()); |
| 523 | ( |
| 524 | "external-driver".to_string(), |
| 525 | format!("generic external driver adapter using {}", binary), |
| 526 | ) |
| 527 | } |
| 528 | CompilerSpec::Binary(path) => ( |
| 529 | "external-driver".to_string(), |
| 530 | format!("generic external driver adapter using {}", path.display()), |
| 531 | ), |
| 532 | } |
| 533 | } |
| 534 | |
| 535 | fn observation_from_capability_mismatch( |
| 536 | spec: &CompilerSpec, |
| 537 | program: &Path, |
| 538 | opt_level: OptLevel, |
| 539 | requested: BTreeSet<ArtifactKey>, |
| 540 | backend_mode: String, |
| 541 | backend_detail: String, |
| 542 | detail: String, |
| 543 | ) -> ObservedProgram { |
| 544 | ObservedProgram { |
| 545 | observation: CompilerObservation { |
| 546 | compiler: spec.clone(), |
| 547 | program: program.to_path_buf(), |
| 548 | opt_level, |
| 549 | compile_exit_code: 1, |
| 550 | artifacts: BTreeMap::from([(ArtifactKey::Diagnostics, ArtifactValue::Text(detail))]), |
| 551 | provenance: ObservationProvenance { |
| 552 | compiler_identity: spec.display_name(), |
| 553 | adapter_kind: match spec { |
| 554 | CompilerSpec::Named(_) => "named".into(), |
| 555 | CompilerSpec::Binary(_) => "explicit-path".into(), |
| 556 | }, |
| 557 | backend_mode, |
| 558 | backend_detail, |
| 559 | artifacts_captured: vec!["diagnostics".into()], |
| 560 | comparison_basis: None, |
| 561 | failure_stage: None, |
| 562 | }, |
| 563 | }, |
| 564 | requested_artifacts: requested, |
| 565 | } |
| 566 | } |
| 567 | |
| 568 | fn preflight_introspection_request( |
| 569 | spec: &CompilerSpec, |
| 570 | program: &Path, |
| 571 | opt_level: OptLevel, |
| 572 | requested: &BTreeSet<ArtifactKey>, |
| 573 | tools: &ToolchainConfig, |
| 574 | ) -> Option<ObservedProgram> { |
| 575 | let capabilities = compiler_capabilities(spec, tools); |
| 576 | let (backend_mode, backend_detail) = compiler_capability_backend(spec, tools); |
| 577 | |
| 578 | let unavailable = capabilities.unavailable_requests(requested); |
| 579 | if !unavailable.is_empty() { |
| 580 | let detail = unavailable |
| 581 | .into_iter() |
| 582 | .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 583 | .collect::<Vec<_>>() |
| 584 | .join("\n"); |
| 585 | return Some(observation_from_capability_mismatch( |
| 586 | spec, |
| 587 | program, |
| 588 | opt_level, |
| 589 | requested.clone(), |
| 590 | backend_mode, |
| 591 | backend_detail, |
| 592 | detail, |
| 593 | )); |
| 594 | } |
| 595 | |
| 596 | let unsupported = capabilities.unsupported_requests(requested); |
| 597 | if !unsupported.is_empty() { |
| 598 | let detail = format!( |
| 599 | "{} does not support requested artifacts in this adapter: {}", |
| 600 | spec.display_name(), |
| 601 | unsupported.join(", ") |
| 602 | ); |
| 603 | return Some(observation_from_capability_mismatch( |
| 604 | spec, |
| 605 | program, |
| 606 | opt_level, |
| 607 | requested.clone(), |
| 608 | backend_mode, |
| 609 | backend_detail, |
| 610 | detail, |
| 611 | )); |
| 612 | } |
| 613 | |
| 614 | None |
| 615 | } |
| 616 | |
| 617 | fn tool_override(var: &str, default: &str) -> String { |
| 618 | match std::env::var(var) { |
| 619 | Ok(value) if !value.trim().is_empty() => value, |
| 620 | _ => default.to_string(), |
| 621 | } |
| 622 | } |
| 623 | |
| 624 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 625 | enum OutcomeKind { |
| 626 | Pass, |
| 627 | Fail, |
| 628 | Xfail, |
| 629 | Xpass, |
| 630 | Future, |
| 631 | } |
| 632 | |
| 633 | #[derive(Debug, Clone)] |
| 634 | struct Outcome { |
| 635 | suite: String, |
| 636 | case: String, |
| 637 | opt_level: OptLevel, |
| 638 | kind: OutcomeKind, |
| 639 | detail: String, |
| 640 | bundle: Option<PathBuf>, |
| 641 | primary_backend: Option<PrimaryBackendReport>, |
| 642 | consistency_observations: Vec<ConsistencyObservation>, |
| 643 | } |
| 644 | |
| 645 | #[derive(Debug, Default)] |
| 646 | struct Summary { |
| 647 | passed: usize, |
| 648 | failed: usize, |
| 649 | xfailed: usize, |
| 650 | xpassed: usize, |
| 651 | future: usize, |
| 652 | outcomes: Vec<Outcome>, |
| 653 | consistency: BTreeMap<ConsistencyCheck, ConsistencyRollup>, |
| 654 | } |
| 655 | |
| 656 | #[derive(Debug, Clone, Default, PartialEq, Eq)] |
| 657 | struct ConsistencyRollup { |
| 658 | cells: usize, |
| 659 | repeat_counts: BTreeSet<usize>, |
| 660 | unique_variant_counts: BTreeSet<usize>, |
| 661 | varying_components: BTreeSet<String>, |
| 662 | stable_components: BTreeSet<String>, |
| 663 | } |
| 664 | |
| 665 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 666 | struct ConsistencyObservation { |
| 667 | check: ConsistencyCheck, |
| 668 | summary: String, |
| 669 | repeat_count: Option<usize>, |
| 670 | unique_variant_count: Option<usize>, |
| 671 | varying_components: Vec<String>, |
| 672 | stable_components: Vec<String>, |
| 673 | } |
| 674 | |
| 675 | #[derive(Debug, Clone)] |
| 676 | struct ListConfig { |
| 677 | suite_filter: Option<String>, |
| 678 | verbose: bool, |
| 679 | tools: ToolchainConfig, |
| 680 | } |
| 681 | |
| 682 | #[derive(Debug, Clone)] |
| 683 | struct RunConfig { |
| 684 | suite_filter: Option<String>, |
| 685 | case_filter: Option<String>, |
| 686 | opt_filter: Option<BTreeSet<OptLevel>>, |
| 687 | verbose: bool, |
| 688 | fail_fast: bool, |
| 689 | include_future: bool, |
| 690 | all_stages: bool, |
| 691 | json_report: Option<PathBuf>, |
| 692 | markdown_report: Option<PathBuf>, |
| 693 | tools: ToolchainConfig, |
| 694 | } |
| 695 | |
| 696 | #[derive(Debug, Clone)] |
| 697 | struct CompareConfig { |
| 698 | left: CompilerSpec, |
| 699 | right: CompilerSpec, |
| 700 | program: PathBuf, |
| 701 | opt_level: OptLevel, |
| 702 | artifacts: BTreeSet<ArtifactKey>, |
| 703 | json_report: Option<PathBuf>, |
| 704 | markdown_report: Option<PathBuf>, |
| 705 | tools: ToolchainConfig, |
| 706 | } |
| 707 | |
| 708 | #[derive(Debug, Clone)] |
| 709 | struct IntrospectConfig { |
| 710 | compiler: CompilerSpec, |
| 711 | program: PathBuf, |
| 712 | opt_level: OptLevel, |
| 713 | artifacts: BTreeSet<ArtifactKey>, |
| 714 | json_report: Option<PathBuf>, |
| 715 | markdown_report: Option<PathBuf>, |
| 716 | all_artifacts: bool, |
| 717 | summary_only: bool, |
| 718 | max_artifact_lines: Option<usize>, |
| 719 | tools: ToolchainConfig, |
| 720 | } |
| 721 | |
| 722 | #[derive(Debug, Clone)] |
| 723 | struct ExecutionArtifacts { |
| 724 | requested: BTreeSet<Stage>, |
| 725 | armfortas: Option<CaptureResult>, |
| 726 | armfortas_failure: Option<CaptureFailure>, |
| 727 | armfortas_observation: Option<ObservedProgram>, |
| 728 | references: Vec<ReferenceResult>, |
| 729 | reference_observations: Vec<ObservedProgram>, |
| 730 | consistency_issues: Vec<ConsistencyIssue>, |
| 731 | } |
| 732 | |
| 733 | #[derive(Debug, Clone)] |
| 734 | struct ObservedProgram { |
| 735 | observation: CompilerObservation, |
| 736 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 737 | } |
| 738 | |
| 739 | #[derive(Debug, Clone, Copy)] |
| 740 | struct IntrospectionRenderConfig { |
| 741 | summary_only: bool, |
| 742 | max_artifact_lines: Option<usize>, |
| 743 | } |
| 744 | |
| 745 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 746 | enum PrimaryCaptureBackendKind { |
| 747 | Full, |
| 748 | Observable, |
| 749 | } |
| 750 | |
| 751 | impl PrimaryCaptureBackendKind { |
| 752 | fn as_str(&self) -> &'static str { |
| 753 | match self { |
| 754 | Self::Full => "full", |
| 755 | Self::Observable => "observable", |
| 756 | } |
| 757 | } |
| 758 | } |
| 759 | |
| 760 | struct SelectedPrimaryBackend { |
| 761 | kind: PrimaryCaptureBackendKind, |
| 762 | backend: Box<dyn CaptureBackend>, |
| 763 | } |
| 764 | |
| 765 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 766 | struct PrimaryBackendReport { |
| 767 | kind: String, |
| 768 | mode: String, |
| 769 | detail: String, |
| 770 | } |
| 771 | |
| 772 | impl PrimaryBackendReport { |
| 773 | fn from_selected(selected: &SelectedPrimaryBackend) -> Self { |
| 774 | Self { |
| 775 | kind: selected.kind.as_str().to_string(), |
| 776 | mode: selected.backend.mode_name().to_string(), |
| 777 | detail: selected.backend.description().to_string(), |
| 778 | } |
| 779 | } |
| 780 | } |
| 781 | |
| 782 | #[derive(Debug, Clone)] |
| 783 | struct ReferenceResult { |
| 784 | compiler: ReferenceCompiler, |
| 785 | compile_command: String, |
| 786 | compile_exit_code: i32, |
| 787 | compile_stdout: String, |
| 788 | compile_stderr: String, |
| 789 | run: Option<RunCapture>, |
| 790 | run_error: Option<String>, |
| 791 | } |
| 792 | |
| 793 | #[derive(Debug, Clone)] |
| 794 | struct ConsistencyIssue { |
| 795 | check: ConsistencyCheck, |
| 796 | summary: String, |
| 797 | repeat_count: Option<usize>, |
| 798 | unique_variant_count: Option<usize>, |
| 799 | varying_components: Vec<String>, |
| 800 | stable_components: Vec<String>, |
| 801 | detail: String, |
| 802 | temp_root: PathBuf, |
| 803 | } |
| 804 | |
| 805 | impl Summary { |
| 806 | fn record_outcome(&mut self, outcome: &Outcome) { |
| 807 | match outcome.kind { |
| 808 | OutcomeKind::Pass => self.passed += 1, |
| 809 | OutcomeKind::Fail => self.failed += 1, |
| 810 | OutcomeKind::Xfail => self.xfailed += 1, |
| 811 | OutcomeKind::Xpass => self.xpassed += 1, |
| 812 | OutcomeKind::Future => self.future += 1, |
| 813 | } |
| 814 | self.outcomes.push(outcome.clone()); |
| 815 | self.record_consistency(&outcome.consistency_observations); |
| 816 | } |
| 817 | |
| 818 | fn record_consistency(&mut self, observations: &[ConsistencyObservation]) { |
| 819 | for observation in observations { |
| 820 | self.consistency |
| 821 | .entry(observation.check) |
| 822 | .or_default() |
| 823 | .record(observation); |
| 824 | } |
| 825 | } |
| 826 | } |
| 827 | |
| 828 | impl ConsistencyRollup { |
| 829 | fn record(&mut self, observation: &ConsistencyObservation) { |
| 830 | self.cells += 1; |
| 831 | if let Some(repeat_count) = observation.repeat_count { |
| 832 | self.repeat_counts.insert(repeat_count); |
| 833 | } |
| 834 | if let Some(unique_variant_count) = observation.unique_variant_count { |
| 835 | self.unique_variant_counts.insert(unique_variant_count); |
| 836 | } |
| 837 | self.varying_components |
| 838 | .extend(observation.varying_components.iter().cloned()); |
| 839 | self.stable_components |
| 840 | .extend(observation.stable_components.iter().cloned()); |
| 841 | } |
| 842 | } |
| 843 | |
| 844 | impl ConsistencyIssue { |
| 845 | fn observation(&self) -> ConsistencyObservation { |
| 846 | ConsistencyObservation { |
| 847 | check: self.check, |
| 848 | summary: self.summary.clone(), |
| 849 | repeat_count: self.repeat_count, |
| 850 | unique_variant_count: self.unique_variant_count, |
| 851 | varying_components: self.varying_components.clone(), |
| 852 | stable_components: self.stable_components.clone(), |
| 853 | } |
| 854 | } |
| 855 | } |
| 856 | |
| 857 | impl ReferenceResult { |
| 858 | fn infrastructure_error(compiler: ReferenceCompiler, command: String, message: String) -> Self { |
| 859 | Self { |
| 860 | compiler, |
| 861 | compile_command: command, |
| 862 | compile_exit_code: -1, |
| 863 | compile_stdout: String::new(), |
| 864 | compile_stderr: message, |
| 865 | run: None, |
| 866 | run_error: None, |
| 867 | } |
| 868 | } |
| 869 | } |
| 870 | |
| 871 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] |
| 872 | struct RunSignature { |
| 873 | exit_code: i32, |
| 874 | stdout: String, |
| 875 | stderr: String, |
| 876 | } |
| 877 | |
| 878 | pub fn run_cli(args: &[String]) -> i32 { |
| 879 | run_cli_named("afs-tests", args) |
| 880 | } |
| 881 | |
| 882 | pub fn run_cli_named(program_name: &str, args: &[String]) -> i32 { |
| 883 | match parse_cli(args) { |
| 884 | Ok(CommandKind::List(config)) => match discover_suites(default_suite_root()) { |
| 885 | Ok(suites) => { |
| 886 | print_suites( |
| 887 | &filter_suites(&suites, config.suite_filter.as_deref()), |
| 888 | &config, |
| 889 | ); |
| 890 | 0 |
| 891 | } |
| 892 | Err(err) => { |
| 893 | eprintln!("{}: {}", program_name, err); |
| 894 | 1 |
| 895 | } |
| 896 | }, |
| 897 | Ok(CommandKind::Run(config)) => match run_suites(&config) { |
| 898 | Ok(summary) => { |
| 899 | print_summary(&summary); |
| 900 | if let Err(err) = write_requested_reports(&config, &summary) { |
| 901 | eprintln!("afs-tests: {}", err); |
| 902 | return 1; |
| 903 | } |
| 904 | if summary.failed == 0 && summary.xpassed == 0 { |
| 905 | 0 |
| 906 | } else { |
| 907 | 1 |
| 908 | } |
| 909 | } |
| 910 | Err(err) => { |
| 911 | eprintln!("{}: {}", program_name, err); |
| 912 | 1 |
| 913 | } |
| 914 | }, |
| 915 | Ok(CommandKind::Compare(config)) => match run_compare(&config) { |
| 916 | Ok(result) => { |
| 917 | print_compare_result(&result); |
| 918 | if let Err(err) = write_compare_reports(&config, &result) { |
| 919 | eprintln!("{}: {}", program_name, err); |
| 920 | return 1; |
| 921 | } |
| 922 | if result.differences.is_empty() { |
| 923 | 0 |
| 924 | } else { |
| 925 | 1 |
| 926 | } |
| 927 | } |
| 928 | Err(err) => { |
| 929 | eprintln!("{}: {}", program_name, err); |
| 930 | 1 |
| 931 | } |
| 932 | }, |
| 933 | Ok(CommandKind::Introspect(config)) => match run_introspect(&config) { |
| 934 | Ok(observation) => { |
| 935 | print_introspection(&config, &observation); |
| 936 | if let Err(err) = write_introspection_reports(&config, &observation) { |
| 937 | eprintln!("{}: {}", program_name, err); |
| 938 | return 1; |
| 939 | } |
| 940 | if observation.observation.compile_exit_code == 0 { |
| 941 | 0 |
| 942 | } else { |
| 943 | 1 |
| 944 | } |
| 945 | } |
| 946 | Err(err) => { |
| 947 | eprintln!("{}: {}", program_name, err); |
| 948 | 1 |
| 949 | } |
| 950 | }, |
| 951 | Ok(CommandKind::Doctor(config)) => { |
| 952 | println!("{}", render_doctor_report(&config)); |
| 953 | if let Err(err) = write_doctor_reports(&config) { |
| 954 | eprintln!("{}: {}", program_name, err); |
| 955 | return 1; |
| 956 | } |
| 957 | 0 |
| 958 | } |
| 959 | Ok(CommandKind::Help) => { |
| 960 | print_usage(program_name); |
| 961 | 0 |
| 962 | } |
| 963 | Err(err) => { |
| 964 | eprintln!("{}: {}", program_name, err); |
| 965 | print_usage(program_name); |
| 966 | 2 |
| 967 | } |
| 968 | } |
| 969 | } |
| 970 | |
| 971 | enum CommandKind { |
| 972 | List(ListConfig), |
| 973 | Run(RunConfig), |
| 974 | Compare(CompareConfig), |
| 975 | Introspect(IntrospectConfig), |
| 976 | Doctor(DoctorConfig), |
| 977 | Help, |
| 978 | } |
| 979 | |
| 980 | #[derive(Debug, Clone)] |
| 981 | struct DoctorConfig { |
| 982 | tools: ToolchainConfig, |
| 983 | json_report: Option<PathBuf>, |
| 984 | markdown_report: Option<PathBuf>, |
| 985 | } |
| 986 | |
| 987 | fn parse_tool_override_arg( |
| 988 | arg: &str, |
| 989 | queue: &mut VecDeque<&String>, |
| 990 | tools: &mut ToolchainConfig, |
| 991 | ) -> Result<bool, String> { |
| 992 | match arg { |
| 993 | "--armfortas-bin" => { |
| 994 | let value = queue |
| 995 | .pop_front() |
| 996 | .ok_or("--armfortas-bin requires a value")?; |
| 997 | tools.armfortas = ArmfortasCliAdapter::External(value.clone()); |
| 998 | Ok(true) |
| 999 | } |
| 1000 | "--gfortran-bin" => { |
| 1001 | let value = queue.pop_front().ok_or("--gfortran-bin requires a value")?; |
| 1002 | tools.gfortran = value.clone(); |
| 1003 | Ok(true) |
| 1004 | } |
| 1005 | "--flang-bin" => { |
| 1006 | let value = queue.pop_front().ok_or("--flang-bin requires a value")?; |
| 1007 | tools.flang_new = value.clone(); |
| 1008 | Ok(true) |
| 1009 | } |
| 1010 | "--lfortran-bin" => { |
| 1011 | let value = queue.pop_front().ok_or("--lfortran-bin requires a value")?; |
| 1012 | tools.lfortran = value.clone(); |
| 1013 | Ok(true) |
| 1014 | } |
| 1015 | "--ifort-bin" => { |
| 1016 | let value = queue.pop_front().ok_or("--ifort-bin requires a value")?; |
| 1017 | tools.ifort = value.clone(); |
| 1018 | Ok(true) |
| 1019 | } |
| 1020 | "--ifx-bin" => { |
| 1021 | let value = queue.pop_front().ok_or("--ifx-bin requires a value")?; |
| 1022 | tools.ifx = value.clone(); |
| 1023 | Ok(true) |
| 1024 | } |
| 1025 | "--nvfortran-bin" => { |
| 1026 | let value = queue |
| 1027 | .pop_front() |
| 1028 | .ok_or("--nvfortran-bin requires a value")?; |
| 1029 | tools.nvfortran = value.clone(); |
| 1030 | Ok(true) |
| 1031 | } |
| 1032 | "--as-bin" => { |
| 1033 | let value = queue.pop_front().ok_or("--as-bin requires a value")?; |
| 1034 | tools.system_as = value.clone(); |
| 1035 | Ok(true) |
| 1036 | } |
| 1037 | "--otool-bin" => { |
| 1038 | let value = queue.pop_front().ok_or("--otool-bin requires a value")?; |
| 1039 | tools.otool = value.clone(); |
| 1040 | Ok(true) |
| 1041 | } |
| 1042 | "--nm-bin" => { |
| 1043 | let value = queue.pop_front().ok_or("--nm-bin requires a value")?; |
| 1044 | tools.nm = value.clone(); |
| 1045 | Ok(true) |
| 1046 | } |
| 1047 | _ => Ok(false), |
| 1048 | } |
| 1049 | } |
| 1050 | |
| 1051 | fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 1052 | if args.is_empty() { |
| 1053 | return Ok(CommandKind::Help); |
| 1054 | } |
| 1055 | |
| 1056 | match args[0].as_str() { |
| 1057 | "list" => { |
| 1058 | let mut config = ListConfig { |
| 1059 | suite_filter: None, |
| 1060 | verbose: false, |
| 1061 | tools: ToolchainConfig::from_env(), |
| 1062 | }; |
| 1063 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1064 | while let Some(arg) = queue.pop_front() { |
| 1065 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1066 | continue; |
| 1067 | } |
| 1068 | match arg.as_str() { |
| 1069 | "--suite" => { |
| 1070 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
| 1071 | config.suite_filter = Some(value.clone()); |
| 1072 | } |
| 1073 | "--verbose" => config.verbose = true, |
| 1074 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1075 | other => return Err(format!("unknown list option: {}", other)), |
| 1076 | } |
| 1077 | } |
| 1078 | Ok(CommandKind::List(config)) |
| 1079 | } |
| 1080 | "run" => { |
| 1081 | let mut config = RunConfig { |
| 1082 | suite_filter: None, |
| 1083 | case_filter: None, |
| 1084 | opt_filter: None, |
| 1085 | verbose: false, |
| 1086 | fail_fast: false, |
| 1087 | include_future: false, |
| 1088 | all_stages: false, |
| 1089 | json_report: None, |
| 1090 | markdown_report: None, |
| 1091 | tools: ToolchainConfig::from_env(), |
| 1092 | }; |
| 1093 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1094 | while let Some(arg) = queue.pop_front() { |
| 1095 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1096 | continue; |
| 1097 | } |
| 1098 | match arg.as_str() { |
| 1099 | "--suite" => { |
| 1100 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
| 1101 | config.suite_filter = Some(value.clone()); |
| 1102 | } |
| 1103 | "--case" => { |
| 1104 | let value = queue.pop_front().ok_or("--case requires a value")?; |
| 1105 | config.case_filter = Some(value.clone()); |
| 1106 | } |
| 1107 | "--opt" => { |
| 1108 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1109 | let parsed = parse_opt_level_list(value)?; |
| 1110 | let filter = config.opt_filter.get_or_insert_with(BTreeSet::new); |
| 1111 | filter.extend(parsed); |
| 1112 | } |
| 1113 | "--verbose" | "-v" => config.verbose = true, |
| 1114 | "--fail-fast" => config.fail_fast = true, |
| 1115 | "--include-future" => config.include_future = true, |
| 1116 | "--all" => config.all_stages = true, |
| 1117 | "--json-report" => { |
| 1118 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1119 | config.json_report = Some(PathBuf::from(value)); |
| 1120 | } |
| 1121 | "--markdown-report" => { |
| 1122 | let value = queue |
| 1123 | .pop_front() |
| 1124 | .ok_or("--markdown-report requires a value")?; |
| 1125 | config.markdown_report = Some(PathBuf::from(value)); |
| 1126 | } |
| 1127 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1128 | other => return Err(format!("unknown run option: {}", other)), |
| 1129 | } |
| 1130 | } |
| 1131 | Ok(CommandKind::Run(config)) |
| 1132 | } |
| 1133 | "compare" => { |
| 1134 | if args.len() < 3 { |
| 1135 | return Err( |
| 1136 | "compare requires <compiler-a> <compiler-b> and --program <path>".to_string(), |
| 1137 | ); |
| 1138 | } |
| 1139 | let left = CompilerSpec::parse(&args[1]); |
| 1140 | let right = CompilerSpec::parse(&args[2]); |
| 1141 | let mut config = CompareConfig { |
| 1142 | left, |
| 1143 | right, |
| 1144 | program: PathBuf::new(), |
| 1145 | opt_level: OptLevel::O0, |
| 1146 | artifacts: BTreeSet::new(), |
| 1147 | json_report: None, |
| 1148 | markdown_report: None, |
| 1149 | tools: ToolchainConfig::from_env(), |
| 1150 | }; |
| 1151 | let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1152 | while let Some(arg) = queue.pop_front() { |
| 1153 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1154 | continue; |
| 1155 | } |
| 1156 | match arg.as_str() { |
| 1157 | "--program" => { |
| 1158 | let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1159 | config.program = PathBuf::from(value); |
| 1160 | } |
| 1161 | "--opt" => { |
| 1162 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1163 | let parsed = parse_opt_level_list(value)?; |
| 1164 | let opt = parsed |
| 1165 | .into_iter() |
| 1166 | .next() |
| 1167 | .ok_or("--opt requires at least one optimization level")?; |
| 1168 | config.opt_level = opt; |
| 1169 | } |
| 1170 | "--artifact" => { |
| 1171 | let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1172 | config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 1173 | } |
| 1174 | "--json-report" => { |
| 1175 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1176 | config.json_report = Some(PathBuf::from(value)); |
| 1177 | } |
| 1178 | "--markdown-report" => { |
| 1179 | let value = queue |
| 1180 | .pop_front() |
| 1181 | .ok_or("--markdown-report requires a value")?; |
| 1182 | config.markdown_report = Some(PathBuf::from(value)); |
| 1183 | } |
| 1184 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1185 | other => return Err(format!("unknown compare option: {}", other)), |
| 1186 | } |
| 1187 | } |
| 1188 | if config.program.as_os_str().is_empty() { |
| 1189 | return Err("compare requires --program <path>".to_string()); |
| 1190 | } |
| 1191 | Ok(CommandKind::Compare(config)) |
| 1192 | } |
| 1193 | "introspect" => { |
| 1194 | if args.len() < 3 { |
| 1195 | return Err("introspect requires <compiler> <program>".to_string()); |
| 1196 | } |
| 1197 | let compiler = CompilerSpec::parse(&args[1]); |
| 1198 | let mut config = IntrospectConfig { |
| 1199 | compiler, |
| 1200 | program: PathBuf::from(&args[2]), |
| 1201 | opt_level: OptLevel::O0, |
| 1202 | artifacts: BTreeSet::new(), |
| 1203 | json_report: None, |
| 1204 | markdown_report: None, |
| 1205 | all_artifacts: false, |
| 1206 | summary_only: false, |
| 1207 | max_artifact_lines: None, |
| 1208 | tools: ToolchainConfig::from_env(), |
| 1209 | }; |
| 1210 | let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1211 | while let Some(arg) = queue.pop_front() { |
| 1212 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1213 | continue; |
| 1214 | } |
| 1215 | match arg.as_str() { |
| 1216 | "--program" => { |
| 1217 | let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1218 | config.program = PathBuf::from(value); |
| 1219 | } |
| 1220 | "--opt" => { |
| 1221 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1222 | let parsed = parse_opt_level_list(value)?; |
| 1223 | let opt = parsed |
| 1224 | .into_iter() |
| 1225 | .next() |
| 1226 | .ok_or("--opt requires at least one optimization level")?; |
| 1227 | config.opt_level = opt; |
| 1228 | } |
| 1229 | "--artifact" => { |
| 1230 | let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1231 | config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 1232 | } |
| 1233 | "--all" => config.all_artifacts = true, |
| 1234 | "--summary-only" => config.summary_only = true, |
| 1235 | "--max-artifact-lines" => { |
| 1236 | let value = queue |
| 1237 | .pop_front() |
| 1238 | .ok_or("--max-artifact-lines requires a value")?; |
| 1239 | let parsed = value.parse::<usize>().map_err(|_| { |
| 1240 | format!("invalid --max-artifact-lines value '{}'", value) |
| 1241 | })?; |
| 1242 | if parsed == 0 { |
| 1243 | return Err("--max-artifact-lines must be greater than 0".to_string()); |
| 1244 | } |
| 1245 | config.max_artifact_lines = Some(parsed); |
| 1246 | } |
| 1247 | "--json-report" => { |
| 1248 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1249 | config.json_report = Some(PathBuf::from(value)); |
| 1250 | } |
| 1251 | "--markdown-report" => { |
| 1252 | let value = queue |
| 1253 | .pop_front() |
| 1254 | .ok_or("--markdown-report requires a value")?; |
| 1255 | config.markdown_report = Some(PathBuf::from(value)); |
| 1256 | } |
| 1257 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1258 | other => return Err(format!("unknown introspect option: {}", other)), |
| 1259 | } |
| 1260 | } |
| 1261 | Ok(CommandKind::Introspect(config)) |
| 1262 | } |
| 1263 | "doctor" => { |
| 1264 | let mut config = DoctorConfig { |
| 1265 | tools: ToolchainConfig::from_env(), |
| 1266 | json_report: None, |
| 1267 | markdown_report: None, |
| 1268 | }; |
| 1269 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1270 | while let Some(arg) = queue.pop_front() { |
| 1271 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1272 | continue; |
| 1273 | } |
| 1274 | match arg.as_str() { |
| 1275 | "--json-report" => { |
| 1276 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1277 | config.json_report = Some(PathBuf::from(value)); |
| 1278 | } |
| 1279 | "--markdown-report" => { |
| 1280 | let value = queue |
| 1281 | .pop_front() |
| 1282 | .ok_or("--markdown-report requires a value")?; |
| 1283 | config.markdown_report = Some(PathBuf::from(value)); |
| 1284 | } |
| 1285 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1286 | other => return Err(format!("unknown doctor option: {}", other)), |
| 1287 | } |
| 1288 | } |
| 1289 | Ok(CommandKind::Doctor(config)) |
| 1290 | } |
| 1291 | "--help" | "-h" | "help" => Ok(CommandKind::Help), |
| 1292 | other => Err(format!("unknown command: {}", other)), |
| 1293 | } |
| 1294 | } |
| 1295 | |
| 1296 | fn print_usage(program_name: &str) { |
| 1297 | eprintln!( |
| 1298 | "{} — generic compiler bench runner (afs-tests compatibility preserved)", |
| 1299 | program_name |
| 1300 | ); |
| 1301 | eprintln!(); |
| 1302 | eprintln!("usage:"); |
| 1303 | eprintln!( |
| 1304 | " {} list [--suite <filter>] [--verbose] [tool overrides]", |
| 1305 | program_name |
| 1306 | ); |
| 1307 | eprintln!( |
| 1308 | " {} run [--suite <filter>] [--case <filter>] [--opt <O0,O1,...>] [--verbose] [--fail-fast] [--include-future] [--all] [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1309 | program_name |
| 1310 | ); |
| 1311 | eprintln!( |
| 1312 | " {} compare <compiler-a> <compiler-b> --program <path> [--opt <O0>] [--artifact <asm,obj,stdout,stderr,exit-code,executable>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1313 | program_name |
| 1314 | ); |
| 1315 | eprintln!( |
| 1316 | " {} introspect <compiler> <program> [--opt <O0>] [--artifact <list>] [--all] [--summary-only] [--max-artifact-lines <n>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1317 | program_name |
| 1318 | ); |
| 1319 | eprintln!( |
| 1320 | " {} doctor [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--lfortran-bin <path>] [--ifort-bin <path>] [--ifx-bin <path>] [--nvfortran-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1321 | program_name |
| 1322 | ); |
| 1323 | eprintln!(); |
| 1324 | eprintln!("env overrides:"); |
| 1325 | eprintln!(" BENCCH_ARMFORTAS_BIN, BENCCH_GFORTRAN_BIN, BENCCH_FLANG_BIN, BENCCH_LFORTRAN_BIN"); |
| 1326 | eprintln!(" BENCCH_IFORT_BIN, BENCCH_IFX_BIN, BENCCH_NVFORTRAN_BIN"); |
| 1327 | eprintln!(" BENCCH_AS_BIN, BENCCH_OTOOL_BIN, BENCCH_NM_BIN"); |
| 1328 | eprintln!(); |
| 1329 | if linked_capture_available() { |
| 1330 | eprintln!("mode:"); |
| 1331 | eprintln!(" linked armfortas capture is available in this build"); |
| 1332 | } else { |
| 1333 | eprintln!("mode:"); |
| 1334 | eprintln!(" linked armfortas capture is unavailable in this build"); |
| 1335 | eprintln!(" compare, introspect, and generic/observable suite runs still work"); |
| 1336 | eprintln!(" use scripts/bootstrap-linked-armfortas.sh for rich armfortas stages and legacy frontend/module suites"); |
| 1337 | } |
| 1338 | } |
| 1339 | |
| 1340 | fn default_compare_artifacts(extra: &BTreeSet<ArtifactKey>) -> BTreeSet<ArtifactKey> { |
| 1341 | let mut requested = BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]); |
| 1342 | requested.extend(extra.iter().cloned()); |
| 1343 | requested |
| 1344 | } |
| 1345 | |
| 1346 | fn default_differential_artifacts() -> BTreeSet<ArtifactKey> { |
| 1347 | BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]) |
| 1348 | } |
| 1349 | |
| 1350 | fn default_introspection_artifacts( |
| 1351 | compiler: &CompilerSpec, |
| 1352 | all_artifacts: bool, |
| 1353 | ) -> BTreeSet<ArtifactKey> { |
| 1354 | let mut requested = BTreeSet::from([ |
| 1355 | ArtifactKey::Diagnostics, |
| 1356 | ArtifactKey::Runtime, |
| 1357 | ArtifactKey::Asm, |
| 1358 | ArtifactKey::Obj, |
| 1359 | ]); |
| 1360 | if matches!(compiler, CompilerSpec::Named(NamedCompiler::Armfortas)) { |
| 1361 | requested.insert(ArtifactKey::Extra("armfortas.ir".into())); |
| 1362 | if all_artifacts { |
| 1363 | for name in [ |
| 1364 | "armfortas.preprocess", |
| 1365 | "armfortas.tokens", |
| 1366 | "armfortas.ast", |
| 1367 | "armfortas.sema", |
| 1368 | "armfortas.ir", |
| 1369 | "armfortas.optir", |
| 1370 | "armfortas.mir", |
| 1371 | "armfortas.regalloc", |
| 1372 | ] { |
| 1373 | requested.insert(ArtifactKey::Extra(name.to_string())); |
| 1374 | } |
| 1375 | } |
| 1376 | } |
| 1377 | requested |
| 1378 | } |
| 1379 | |
| 1380 | fn run_compare(config: &CompareConfig) -> Result<ComparisonResult, String> { |
| 1381 | let requested = default_compare_artifacts(&config.artifacts); |
| 1382 | preflight_compare_request(config, &requested)?; |
| 1383 | let left = observe_compiler( |
| 1384 | &config.left, |
| 1385 | &config.program, |
| 1386 | config.opt_level, |
| 1387 | &requested, |
| 1388 | &config.tools, |
| 1389 | )?; |
| 1390 | let right = observe_compiler( |
| 1391 | &config.right, |
| 1392 | &config.program, |
| 1393 | config.opt_level, |
| 1394 | &requested, |
| 1395 | &config.tools, |
| 1396 | )?; |
| 1397 | Ok(compare_observations(left, right, &requested)) |
| 1398 | } |
| 1399 | |
| 1400 | fn capability_request_issue( |
| 1401 | spec: &CompilerSpec, |
| 1402 | requested: &BTreeSet<ArtifactKey>, |
| 1403 | tools: &ToolchainConfig, |
| 1404 | ) -> Option<String> { |
| 1405 | let capabilities = compiler_capabilities(spec, tools); |
| 1406 | let unavailable = capabilities.unavailable_requests(requested); |
| 1407 | let unsupported = capabilities.unsupported_requests(requested); |
| 1408 | if unavailable.is_empty() && unsupported.is_empty() { |
| 1409 | return None; |
| 1410 | } |
| 1411 | |
| 1412 | let mut sections = Vec::new(); |
| 1413 | if !unavailable.is_empty() { |
| 1414 | let detail = unavailable |
| 1415 | .into_iter() |
| 1416 | .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 1417 | .collect::<Vec<_>>() |
| 1418 | .join("\n"); |
| 1419 | sections.push(format!( |
| 1420 | "{} unavailable for requested artifacts in this build\n{}", |
| 1421 | spec.display_name(), |
| 1422 | detail |
| 1423 | )); |
| 1424 | } |
| 1425 | if !unsupported.is_empty() { |
| 1426 | sections.push(format!( |
| 1427 | "{} does not support requested artifacts in this adapter: {}", |
| 1428 | spec.display_name(), |
| 1429 | unsupported.join(", ") |
| 1430 | )); |
| 1431 | } |
| 1432 | Some(sections.join("\n")) |
| 1433 | } |
| 1434 | |
| 1435 | fn compare_capability_issue( |
| 1436 | left: &CompilerSpec, |
| 1437 | right: &CompilerSpec, |
| 1438 | requested: &BTreeSet<ArtifactKey>, |
| 1439 | tools: &ToolchainConfig, |
| 1440 | ) -> Option<String> { |
| 1441 | let mut issues = Vec::new(); |
| 1442 | if let Some(issue) = capability_request_issue(left, requested, tools) { |
| 1443 | issues.push(format!("left:\n{}", issue)); |
| 1444 | } |
| 1445 | if let Some(issue) = capability_request_issue(right, requested, tools) { |
| 1446 | issues.push(format!("right:\n{}", issue)); |
| 1447 | } |
| 1448 | if issues.is_empty() { |
| 1449 | None |
| 1450 | } else { |
| 1451 | Some(format!( |
| 1452 | "compare request is not supported for the selected compiler surfaces\n{}", |
| 1453 | issues.join("\n") |
| 1454 | )) |
| 1455 | } |
| 1456 | } |
| 1457 | |
| 1458 | fn preflight_compare_request( |
| 1459 | config: &CompareConfig, |
| 1460 | requested: &BTreeSet<ArtifactKey>, |
| 1461 | ) -> Result<(), String> { |
| 1462 | match compare_capability_issue(&config.left, &config.right, requested, &config.tools) { |
| 1463 | Some(issue) => Err(issue), |
| 1464 | None => Ok(()), |
| 1465 | } |
| 1466 | } |
| 1467 | |
| 1468 | fn run_introspect(config: &IntrospectConfig) -> Result<ObservedProgram, String> { |
| 1469 | let requested = if config.artifacts.is_empty() { |
| 1470 | default_introspection_artifacts(&config.compiler, config.all_artifacts) |
| 1471 | } else { |
| 1472 | let mut requested = config.artifacts.clone(); |
| 1473 | if config.all_artifacts |
| 1474 | && matches!( |
| 1475 | config.compiler, |
| 1476 | CompilerSpec::Named(NamedCompiler::Armfortas) |
| 1477 | ) |
| 1478 | { |
| 1479 | requested.extend(default_introspection_artifacts(&config.compiler, true)); |
| 1480 | } |
| 1481 | requested |
| 1482 | }; |
| 1483 | if let Some(observed) = preflight_introspection_request( |
| 1484 | &config.compiler, |
| 1485 | &config.program, |
| 1486 | config.opt_level, |
| 1487 | &requested, |
| 1488 | &config.tools, |
| 1489 | ) { |
| 1490 | return Ok(observed); |
| 1491 | } |
| 1492 | Ok(ObservedProgram { |
| 1493 | observation: observe_compiler( |
| 1494 | &config.compiler, |
| 1495 | &config.program, |
| 1496 | config.opt_level, |
| 1497 | &requested, |
| 1498 | &config.tools, |
| 1499 | )?, |
| 1500 | requested_artifacts: requested, |
| 1501 | }) |
| 1502 | } |
| 1503 | |
| 1504 | fn requested_linked_armfortas_artifacts(requested: &BTreeSet<ArtifactKey>) -> Vec<String> { |
| 1505 | requested |
| 1506 | .iter() |
| 1507 | .filter_map(|artifact| match artifact { |
| 1508 | ArtifactKey::Extra(name) if name.starts_with("armfortas.") => Some(name.clone()), |
| 1509 | _ => None, |
| 1510 | }) |
| 1511 | .collect() |
| 1512 | } |
| 1513 | |
| 1514 | fn observe_compiler( |
| 1515 | spec: &CompilerSpec, |
| 1516 | program: &Path, |
| 1517 | opt_level: OptLevel, |
| 1518 | requested: &BTreeSet<ArtifactKey>, |
| 1519 | tools: &ToolchainConfig, |
| 1520 | ) -> Result<CompilerObservation, String> { |
| 1521 | match spec { |
| 1522 | CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 1523 | observe_armfortas(program, opt_level, requested, tools) |
| 1524 | } |
| 1525 | CompilerSpec::Named(named) => { |
| 1526 | let binary = tools.named_compiler_binary(*named).ok_or_else(|| { |
| 1527 | format!("named compiler '{}' has no resolved binary", named.as_str()) |
| 1528 | })?; |
| 1529 | observe_external_driver( |
| 1530 | spec, |
| 1531 | &binary, |
| 1532 | program, |
| 1533 | opt_level, |
| 1534 | requested, |
| 1535 | matches!(named, NamedCompiler::Gfortran | NamedCompiler::FlangNew) |
| 1536 | && source_uses_cpp(program), |
| 1537 | "named".to_string(), |
| 1538 | tools.otool_bin(), |
| 1539 | tools.nm_bin(), |
| 1540 | ) |
| 1541 | } |
| 1542 | CompilerSpec::Binary(path) => observe_external_driver( |
| 1543 | spec, |
| 1544 | &path.display().to_string(), |
| 1545 | program, |
| 1546 | opt_level, |
| 1547 | requested, |
| 1548 | false, |
| 1549 | "explicit-path".to_string(), |
| 1550 | tools.otool_bin(), |
| 1551 | tools.nm_bin(), |
| 1552 | ), |
| 1553 | } |
| 1554 | } |
| 1555 | |
| 1556 | fn observe_armfortas( |
| 1557 | program: &Path, |
| 1558 | opt_level: OptLevel, |
| 1559 | requested: &BTreeSet<ArtifactKey>, |
| 1560 | tools: &ToolchainConfig, |
| 1561 | ) -> Result<CompilerObservation, String> { |
| 1562 | let stages = armfortas_requested_stages(requested)?; |
| 1563 | let linked_only_artifacts = requested_linked_armfortas_artifacts(requested); |
| 1564 | let linked_backend = tools.armfortas_adapters(); |
| 1565 | if !linked_only_artifacts.is_empty() && linked_backend.capture_mode_name() == "unavailable" { |
| 1566 | let detail = format!( |
| 1567 | "linked armfortas capture is unavailable in this build; requested {}; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary", |
| 1568 | linked_only_artifacts.join(", ") |
| 1569 | ); |
| 1570 | return Ok(CompilerObservation { |
| 1571 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1572 | program: program.to_path_buf(), |
| 1573 | opt_level, |
| 1574 | compile_exit_code: 1, |
| 1575 | artifacts: BTreeMap::from([( |
| 1576 | ArtifactKey::Diagnostics, |
| 1577 | ArtifactValue::Text(detail.clone()), |
| 1578 | )]), |
| 1579 | provenance: ObservationProvenance { |
| 1580 | compiler_identity: "armfortas".into(), |
| 1581 | adapter_kind: "named".into(), |
| 1582 | backend_mode: linked_backend.capture_mode_name().into(), |
| 1583 | backend_detail: linked_backend.capture_description().into(), |
| 1584 | artifacts_captured: vec!["diagnostics".into()], |
| 1585 | comparison_basis: None, |
| 1586 | failure_stage: None, |
| 1587 | }, |
| 1588 | }); |
| 1589 | } |
| 1590 | let cli_observable_only = requested.iter().all(|artifact| { |
| 1591 | matches!( |
| 1592 | artifact, |
| 1593 | ArtifactKey::Diagnostics |
| 1594 | | ArtifactKey::Runtime |
| 1595 | | ArtifactKey::Stdout |
| 1596 | | ArtifactKey::Stderr |
| 1597 | | ArtifactKey::ExitCode |
| 1598 | | ArtifactKey::Asm |
| 1599 | | ArtifactKey::Obj |
| 1600 | | ArtifactKey::Executable |
| 1601 | ) |
| 1602 | }); |
| 1603 | let (backend_mode, backend_detail, capture) = if cli_observable_only |
| 1604 | && matches!(tools.armfortas, ArmfortasCliAdapter::External(_)) |
| 1605 | { |
| 1606 | let backend = tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level)); |
| 1607 | let detail = backend.description().to_string(); |
| 1608 | let mode = backend.mode_name().to_string(); |
| 1609 | let request = CaptureRequest { |
| 1610 | input: program.to_path_buf(), |
| 1611 | requested: stages.clone(), |
| 1612 | opt_level, |
| 1613 | }; |
| 1614 | (mode, detail, backend.capture(&request)) |
| 1615 | } else { |
| 1616 | let backend = linked_backend; |
| 1617 | let detail = backend.capture_description().to_string(); |
| 1618 | let mode = backend.capture_mode_name().to_string(); |
| 1619 | let request = CaptureRequest { |
| 1620 | input: program.to_path_buf(), |
| 1621 | requested: stages.clone(), |
| 1622 | opt_level, |
| 1623 | }; |
| 1624 | (mode, detail, backend.capture(&request)) |
| 1625 | }; |
| 1626 | |
| 1627 | let mut artifacts = BTreeMap::new(); |
| 1628 | let mut compile_exit_code = 0; |
| 1629 | let mut failure_stage = None; |
| 1630 | match capture { |
| 1631 | Ok(result) => { |
| 1632 | for (stage, captured) in &result.stages { |
| 1633 | match (stage, captured) { |
| 1634 | (Stage::Asm, CapturedStage::Text(text)) |
| 1635 | if requested.contains(&ArtifactKey::Asm) => |
| 1636 | { |
| 1637 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1638 | } |
| 1639 | (Stage::Obj, CapturedStage::Text(text)) |
| 1640 | if requested.contains(&ArtifactKey::Obj) => |
| 1641 | { |
| 1642 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1643 | } |
| 1644 | (Stage::Run, CapturedStage::Run(run)) => { |
| 1645 | insert_run_artifacts(requested, run, &mut artifacts); |
| 1646 | } |
| 1647 | (stage, CapturedStage::Text(text)) => { |
| 1648 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1649 | if requested.contains(&key) { |
| 1650 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1651 | } |
| 1652 | } |
| 1653 | _ => {} |
| 1654 | } |
| 1655 | } |
| 1656 | } |
| 1657 | Err(failure) => { |
| 1658 | compile_exit_code = 1; |
| 1659 | failure_stage = Some(failure.stage.as_str().to_string()); |
| 1660 | artifacts.insert( |
| 1661 | ArtifactKey::Diagnostics, |
| 1662 | ArtifactValue::Text(failure.detail.clone()), |
| 1663 | ); |
| 1664 | for (stage, captured) in &failure.stages { |
| 1665 | match (stage, captured) { |
| 1666 | (Stage::Asm, CapturedStage::Text(text)) |
| 1667 | if requested.contains(&ArtifactKey::Asm) => |
| 1668 | { |
| 1669 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1670 | } |
| 1671 | (Stage::Obj, CapturedStage::Text(text)) |
| 1672 | if requested.contains(&ArtifactKey::Obj) => |
| 1673 | { |
| 1674 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1675 | } |
| 1676 | (Stage::Run, CapturedStage::Run(run)) => { |
| 1677 | insert_run_artifacts(requested, run, &mut artifacts); |
| 1678 | } |
| 1679 | (stage, CapturedStage::Text(text)) => { |
| 1680 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1681 | if requested.contains(&key) { |
| 1682 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1683 | } |
| 1684 | } |
| 1685 | _ => {} |
| 1686 | } |
| 1687 | } |
| 1688 | } |
| 1689 | } |
| 1690 | |
| 1691 | if requested.contains(&ArtifactKey::Executable) && compile_exit_code == 0 { |
| 1692 | let temp_root = next_observation_temp_root("armfortas", opt_level); |
| 1693 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 1694 | format!( |
| 1695 | "cannot create introspection temp dir '{}': {}", |
| 1696 | temp_root.display(), |
| 1697 | e |
| 1698 | ) |
| 1699 | })?; |
| 1700 | let binary = temp_root.join("introspect.out"); |
| 1701 | tools |
| 1702 | .armfortas_adapters() |
| 1703 | .compile_output(program, opt_level, EmitMode::Binary, &binary) |
| 1704 | .map_err(|detail| { |
| 1705 | format!("failed to build armfortas executable artifact:\n{}", detail) |
| 1706 | })?; |
| 1707 | artifacts.insert(ArtifactKey::Executable, ArtifactValue::Path(binary)); |
| 1708 | } |
| 1709 | |
| 1710 | let artifacts_captured = artifacts |
| 1711 | .keys() |
| 1712 | .map(|artifact| artifact.as_str().to_string()) |
| 1713 | .collect::<Vec<_>>(); |
| 1714 | |
| 1715 | Ok(CompilerObservation { |
| 1716 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1717 | program: program.to_path_buf(), |
| 1718 | opt_level, |
| 1719 | compile_exit_code, |
| 1720 | artifacts, |
| 1721 | provenance: ObservationProvenance { |
| 1722 | compiler_identity: "armfortas".into(), |
| 1723 | adapter_kind: "named".into(), |
| 1724 | backend_mode, |
| 1725 | backend_detail, |
| 1726 | artifacts_captured, |
| 1727 | comparison_basis: None, |
| 1728 | failure_stage, |
| 1729 | }, |
| 1730 | }) |
| 1731 | } |
| 1732 | |
| 1733 | #[derive(Debug, Clone)] |
| 1734 | struct DriverCompileResult { |
| 1735 | command: String, |
| 1736 | exit_code: i32, |
| 1737 | stdout: String, |
| 1738 | stderr: String, |
| 1739 | output: PathBuf, |
| 1740 | } |
| 1741 | |
| 1742 | fn observe_external_driver( |
| 1743 | spec: &CompilerSpec, |
| 1744 | binary: &str, |
| 1745 | program: &Path, |
| 1746 | opt_level: OptLevel, |
| 1747 | requested: &BTreeSet<ArtifactKey>, |
| 1748 | uses_cpp: bool, |
| 1749 | adapter_kind: String, |
| 1750 | otool: &str, |
| 1751 | nm: &str, |
| 1752 | ) -> Result<CompilerObservation, String> { |
| 1753 | let temp_root = next_observation_temp_root(&spec.display_name(), opt_level); |
| 1754 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 1755 | format!( |
| 1756 | "cannot create observation temp dir '{}': {}", |
| 1757 | temp_root.display(), |
| 1758 | e |
| 1759 | ) |
| 1760 | })?; |
| 1761 | |
| 1762 | let needs_runtime = requested.contains(&ArtifactKey::Runtime) |
| 1763 | || requested.contains(&ArtifactKey::Stdout) |
| 1764 | || requested.contains(&ArtifactKey::Stderr) |
| 1765 | || requested.contains(&ArtifactKey::ExitCode) |
| 1766 | || requested.contains(&ArtifactKey::Executable); |
| 1767 | let primary_mode = if needs_runtime { |
| 1768 | DriverEmitMode::Binary |
| 1769 | } else if requested.contains(&ArtifactKey::Asm) { |
| 1770 | DriverEmitMode::Asm |
| 1771 | } else if requested.contains(&ArtifactKey::Obj) { |
| 1772 | DriverEmitMode::Obj |
| 1773 | } else { |
| 1774 | DriverEmitMode::Binary |
| 1775 | }; |
| 1776 | let primary_name = match primary_mode { |
| 1777 | DriverEmitMode::Binary => "observe.out", |
| 1778 | DriverEmitMode::Asm => "observe.s", |
| 1779 | DriverEmitMode::Obj => "observe.o", |
| 1780 | }; |
| 1781 | let primary = compile_with_external_driver( |
| 1782 | binary, |
| 1783 | program, |
| 1784 | opt_level, |
| 1785 | primary_mode, |
| 1786 | &temp_root.join(primary_name), |
| 1787 | uses_cpp, |
| 1788 | )?; |
| 1789 | |
| 1790 | let mut artifacts = BTreeMap::new(); |
| 1791 | if !primary.stdout.trim().is_empty() |
| 1792 | || !primary.stderr.trim().is_empty() |
| 1793 | || primary.exit_code != 0 |
| 1794 | { |
| 1795 | let diagnostics = [primary.stdout.trim_end(), primary.stderr.trim_end()] |
| 1796 | .iter() |
| 1797 | .filter(|part| !part.is_empty()) |
| 1798 | .copied() |
| 1799 | .collect::<Vec<_>>() |
| 1800 | .join("\n"); |
| 1801 | artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 1802 | } |
| 1803 | |
| 1804 | let mut compile_exit_code = primary.exit_code; |
| 1805 | if primary.exit_code == 0 { |
| 1806 | match primary_mode { |
| 1807 | DriverEmitMode::Binary => { |
| 1808 | if requested.contains(&ArtifactKey::Executable) { |
| 1809 | artifacts.insert( |
| 1810 | ArtifactKey::Executable, |
| 1811 | ArtifactValue::Path(primary.output.clone()), |
| 1812 | ); |
| 1813 | } |
| 1814 | if needs_runtime { |
| 1815 | let run_command = render_binary_run_command(&primary.output); |
| 1816 | let run = run_binary_capture(&primary.output, &temp_root, &run_command) |
| 1817 | .map_err(|detail| format!("build: {}\n{}", primary.command, detail))?; |
| 1818 | insert_run_artifacts(requested, &run, &mut artifacts); |
| 1819 | } |
| 1820 | } |
| 1821 | DriverEmitMode::Asm if requested.contains(&ArtifactKey::Asm) => { |
| 1822 | artifacts.insert( |
| 1823 | ArtifactKey::Asm, |
| 1824 | ArtifactValue::Text(fs::read_to_string(&primary.output).map_err(|e| { |
| 1825 | format!( |
| 1826 | "cannot read asm artifact '{}': {}", |
| 1827 | primary.output.display(), |
| 1828 | e |
| 1829 | ) |
| 1830 | })?), |
| 1831 | ); |
| 1832 | } |
| 1833 | DriverEmitMode::Obj if requested.contains(&ArtifactKey::Obj) => { |
| 1834 | artifacts.insert( |
| 1835 | ArtifactKey::Obj, |
| 1836 | ArtifactValue::Text( |
| 1837 | object_snapshot_text(&primary.output, otool, nm) |
| 1838 | .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1839 | ), |
| 1840 | ); |
| 1841 | } |
| 1842 | _ => {} |
| 1843 | } |
| 1844 | |
| 1845 | if requested.contains(&ArtifactKey::Asm) && primary_mode != DriverEmitMode::Asm { |
| 1846 | let asm = compile_with_external_driver( |
| 1847 | binary, |
| 1848 | program, |
| 1849 | opt_level, |
| 1850 | DriverEmitMode::Asm, |
| 1851 | &temp_root.join("observe-extra.s"), |
| 1852 | uses_cpp, |
| 1853 | )?; |
| 1854 | if asm.exit_code != 0 { |
| 1855 | compile_exit_code = asm.exit_code; |
| 1856 | artifacts.insert( |
| 1857 | ArtifactKey::Diagnostics, |
| 1858 | ArtifactValue::Text(asm.stderr.trim_end().to_string()), |
| 1859 | ); |
| 1860 | } else { |
| 1861 | artifacts.insert( |
| 1862 | ArtifactKey::Asm, |
| 1863 | ArtifactValue::Text(fs::read_to_string(&asm.output).map_err(|e| { |
| 1864 | format!("cannot read asm artifact '{}': {}", asm.output.display(), e) |
| 1865 | })?), |
| 1866 | ); |
| 1867 | } |
| 1868 | } |
| 1869 | |
| 1870 | if requested.contains(&ArtifactKey::Obj) && primary_mode != DriverEmitMode::Obj { |
| 1871 | let obj = compile_with_external_driver( |
| 1872 | binary, |
| 1873 | program, |
| 1874 | opt_level, |
| 1875 | DriverEmitMode::Obj, |
| 1876 | &temp_root.join("observe-extra.o"), |
| 1877 | uses_cpp, |
| 1878 | )?; |
| 1879 | if obj.exit_code != 0 { |
| 1880 | compile_exit_code = obj.exit_code; |
| 1881 | artifacts.insert( |
| 1882 | ArtifactKey::Diagnostics, |
| 1883 | ArtifactValue::Text(obj.stderr.trim_end().to_string()), |
| 1884 | ); |
| 1885 | } else { |
| 1886 | artifacts.insert( |
| 1887 | ArtifactKey::Obj, |
| 1888 | ArtifactValue::Text( |
| 1889 | object_snapshot_text(&obj.output, otool, nm) |
| 1890 | .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1891 | ), |
| 1892 | ); |
| 1893 | } |
| 1894 | } |
| 1895 | } |
| 1896 | |
| 1897 | let artifacts_captured = artifacts |
| 1898 | .keys() |
| 1899 | .map(|artifact| artifact.as_str().to_string()) |
| 1900 | .collect::<Vec<_>>(); |
| 1901 | |
| 1902 | Ok(CompilerObservation { |
| 1903 | compiler: spec.clone(), |
| 1904 | program: program.to_path_buf(), |
| 1905 | opt_level, |
| 1906 | compile_exit_code, |
| 1907 | artifacts, |
| 1908 | provenance: ObservationProvenance { |
| 1909 | compiler_identity: spec.display_name(), |
| 1910 | adapter_kind, |
| 1911 | backend_mode: "external-driver".into(), |
| 1912 | backend_detail: format!("generic external driver adapter using {}", binary), |
| 1913 | artifacts_captured, |
| 1914 | comparison_basis: None, |
| 1915 | failure_stage: None, |
| 1916 | }, |
| 1917 | }) |
| 1918 | } |
| 1919 | |
| 1920 | fn compile_with_external_driver( |
| 1921 | binary: &str, |
| 1922 | source: &Path, |
| 1923 | opt_level: OptLevel, |
| 1924 | mode: DriverEmitMode, |
| 1925 | output: &Path, |
| 1926 | uses_cpp: bool, |
| 1927 | ) -> Result<DriverCompileResult, String> { |
| 1928 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 1929 | if uses_cpp { |
| 1930 | args.push("-cpp".to_string()); |
| 1931 | } |
| 1932 | match mode { |
| 1933 | DriverEmitMode::Asm => args.push("-S".to_string()), |
| 1934 | DriverEmitMode::Obj => args.push("-c".to_string()), |
| 1935 | DriverEmitMode::Binary => {} |
| 1936 | } |
| 1937 | args.push(source.display().to_string()); |
| 1938 | args.push("-o".to_string()); |
| 1939 | args.push(output.display().to_string()); |
| 1940 | let command = render_command(binary, &args); |
| 1941 | let output_result = Command::new(binary) |
| 1942 | .args(&args) |
| 1943 | .output() |
| 1944 | .map_err(|e| format!("cannot run '{}': {}", binary, e))?; |
| 1945 | Ok(DriverCompileResult { |
| 1946 | command, |
| 1947 | exit_code: output_result.status.code().unwrap_or(-1), |
| 1948 | stdout: String::from_utf8_lossy(&output_result.stdout).into_owned(), |
| 1949 | stderr: String::from_utf8_lossy(&output_result.stderr).into_owned(), |
| 1950 | output: output.to_path_buf(), |
| 1951 | }) |
| 1952 | } |
| 1953 | |
| 1954 | fn next_observation_temp_root(label: &str, opt_level: OptLevel) -> PathBuf { |
| 1955 | default_report_root().join(".tmp").join(format!( |
| 1956 | "observe_{}_{}", |
| 1957 | sanitize_component(label), |
| 1958 | next_report_suffix(opt_level) |
| 1959 | )) |
| 1960 | } |
| 1961 | |
| 1962 | fn armfortas_requested_stages( |
| 1963 | requested: &BTreeSet<ArtifactKey>, |
| 1964 | ) -> Result<BTreeSet<Stage>, String> { |
| 1965 | let mut stages = BTreeSet::new(); |
| 1966 | for artifact in requested { |
| 1967 | match artifact { |
| 1968 | ArtifactKey::Asm => { |
| 1969 | stages.insert(Stage::Asm); |
| 1970 | } |
| 1971 | ArtifactKey::Obj => { |
| 1972 | stages.insert(Stage::Obj); |
| 1973 | } |
| 1974 | ArtifactKey::Runtime |
| 1975 | | ArtifactKey::Stdout |
| 1976 | | ArtifactKey::Stderr |
| 1977 | | ArtifactKey::ExitCode => { |
| 1978 | stages.insert(Stage::Run); |
| 1979 | } |
| 1980 | ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 1981 | ArtifactKey::Extra(name) => { |
| 1982 | let suffix = name |
| 1983 | .strip_prefix("armfortas.") |
| 1984 | .ok_or_else(|| format!("unsupported adapter-specific artifact '{}'", name))?; |
| 1985 | let stage = Stage::parse(suffix) |
| 1986 | .ok_or_else(|| format!("unknown armfortas artifact '{}'", name))?; |
| 1987 | stages.insert(stage); |
| 1988 | } |
| 1989 | } |
| 1990 | } |
| 1991 | if stages.is_empty() { |
| 1992 | stages.insert(Stage::Run); |
| 1993 | } |
| 1994 | Ok(stages) |
| 1995 | } |
| 1996 | |
| 1997 | fn insert_run_artifacts( |
| 1998 | requested: &BTreeSet<ArtifactKey>, |
| 1999 | run: &RunCapture, |
| 2000 | artifacts: &mut BTreeMap<ArtifactKey, ArtifactValue>, |
| 2001 | ) { |
| 2002 | if requested.contains(&ArtifactKey::Runtime) { |
| 2003 | artifacts.insert(ArtifactKey::Runtime, ArtifactValue::Run(run.clone())); |
| 2004 | } |
| 2005 | if requested.contains(&ArtifactKey::Stdout) { |
| 2006 | artifacts.insert(ArtifactKey::Stdout, ArtifactValue::Text(run.stdout.clone())); |
| 2007 | } |
| 2008 | if requested.contains(&ArtifactKey::Stderr) { |
| 2009 | artifacts.insert(ArtifactKey::Stderr, ArtifactValue::Text(run.stderr.clone())); |
| 2010 | } |
| 2011 | if requested.contains(&ArtifactKey::ExitCode) { |
| 2012 | artifacts.insert(ArtifactKey::ExitCode, ArtifactValue::Int(run.exit_code)); |
| 2013 | } |
| 2014 | } |
| 2015 | |
| 2016 | fn compare_observations( |
| 2017 | mut left: CompilerObservation, |
| 2018 | mut right: CompilerObservation, |
| 2019 | requested: &BTreeSet<ArtifactKey>, |
| 2020 | ) -> ComparisonResult { |
| 2021 | let basis = format!( |
| 2022 | "compile-status, diagnostics, runtime{}", |
| 2023 | if requested.is_empty() { |
| 2024 | String::new() |
| 2025 | } else { |
| 2026 | let extras = requested |
| 2027 | .iter() |
| 2028 | .filter(|artifact| { |
| 2029 | !matches!(artifact, ArtifactKey::Diagnostics | ArtifactKey::Runtime) |
| 2030 | }) |
| 2031 | .map(|artifact| artifact.as_str().to_string()) |
| 2032 | .collect::<Vec<_>>(); |
| 2033 | if extras.is_empty() { |
| 2034 | String::new() |
| 2035 | } else { |
| 2036 | format!(", {}", extras.join(", ")) |
| 2037 | } |
| 2038 | } |
| 2039 | ); |
| 2040 | left.provenance.comparison_basis = Some(basis.clone()); |
| 2041 | right.provenance.comparison_basis = Some(basis.clone()); |
| 2042 | |
| 2043 | let mut differences = Vec::new(); |
| 2044 | if left.compile_exit_code != right.compile_exit_code { |
| 2045 | differences.push(ArtifactDifference { |
| 2046 | artifact: "compile-exit-code".into(), |
| 2047 | detail: format!( |
| 2048 | "{}: {}\n{}: {}", |
| 2049 | left.compiler.display_name(), |
| 2050 | left.compile_exit_code, |
| 2051 | right.compiler.display_name(), |
| 2052 | right.compile_exit_code |
| 2053 | ), |
| 2054 | }); |
| 2055 | } |
| 2056 | |
| 2057 | compare_artifact_text( |
| 2058 | &left, |
| 2059 | &right, |
| 2060 | &ArtifactKey::Diagnostics, |
| 2061 | "diagnostics", |
| 2062 | &mut differences, |
| 2063 | ); |
| 2064 | |
| 2065 | if left.compile_exit_code == 0 && right.compile_exit_code == 0 { |
| 2066 | if requested.contains(&ArtifactKey::Runtime) { |
| 2067 | compare_artifact_runtime(&left, &right, &mut differences); |
| 2068 | } |
| 2069 | for artifact in requested { |
| 2070 | match artifact { |
| 2071 | ArtifactKey::Diagnostics | ArtifactKey::Runtime => {} |
| 2072 | ArtifactKey::Stdout | ArtifactKey::Stderr | ArtifactKey::Asm | ArtifactKey::Obj => { |
| 2073 | compare_artifact_text( |
| 2074 | &left, |
| 2075 | &right, |
| 2076 | artifact, |
| 2077 | artifact.as_str(), |
| 2078 | &mut differences, |
| 2079 | ); |
| 2080 | } |
| 2081 | ArtifactKey::ExitCode => { |
| 2082 | compare_artifact_int(&left, &right, artifact, &mut differences) |
| 2083 | } |
| 2084 | ArtifactKey::Executable => { |
| 2085 | compare_artifact_path(&left, &right, artifact, &mut differences) |
| 2086 | } |
| 2087 | ArtifactKey::Extra(name) => { |
| 2088 | compare_artifact_text(&left, &right, artifact, name, &mut differences) |
| 2089 | } |
| 2090 | } |
| 2091 | } |
| 2092 | } |
| 2093 | |
| 2094 | ComparisonResult { |
| 2095 | left, |
| 2096 | right, |
| 2097 | basis, |
| 2098 | differences, |
| 2099 | } |
| 2100 | } |
| 2101 | |
| 2102 | fn compare_artifact_text( |
| 2103 | left: &CompilerObservation, |
| 2104 | right: &CompilerObservation, |
| 2105 | artifact: &ArtifactKey, |
| 2106 | label: &str, |
| 2107 | differences: &mut Vec<ArtifactDifference>, |
| 2108 | ) { |
| 2109 | let left_text = match left.artifacts.get(artifact) { |
| 2110 | Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2111 | _ => "", |
| 2112 | }; |
| 2113 | let right_text = match right.artifacts.get(artifact) { |
| 2114 | Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2115 | _ => "", |
| 2116 | }; |
| 2117 | if left_text != right_text { |
| 2118 | differences.push(ArtifactDifference { |
| 2119 | artifact: label.to_string(), |
| 2120 | detail: describe_text_difference( |
| 2121 | left_text, |
| 2122 | right_text, |
| 2123 | &left.compiler.display_name(), |
| 2124 | &right.compiler.display_name(), |
| 2125 | ), |
| 2126 | }); |
| 2127 | } |
| 2128 | } |
| 2129 | |
| 2130 | fn compare_artifact_runtime( |
| 2131 | left: &CompilerObservation, |
| 2132 | right: &CompilerObservation, |
| 2133 | differences: &mut Vec<ArtifactDifference>, |
| 2134 | ) { |
| 2135 | let left_run = match left.artifacts.get(&ArtifactKey::Runtime) { |
| 2136 | Some(ArtifactValue::Run(run)) => Some(run), |
| 2137 | _ => None, |
| 2138 | }; |
| 2139 | let right_run = match right.artifacts.get(&ArtifactKey::Runtime) { |
| 2140 | Some(ArtifactValue::Run(run)) => Some(run), |
| 2141 | _ => None, |
| 2142 | }; |
| 2143 | match (left_run, right_run) { |
| 2144 | (Some(left_run), Some(right_run)) => { |
| 2145 | if normalize_run_signature(left_run) != normalize_run_signature(right_run) { |
| 2146 | differences.push(ArtifactDifference { |
| 2147 | artifact: "runtime".into(), |
| 2148 | detail: describe_run_difference( |
| 2149 | left_run, |
| 2150 | right_run, |
| 2151 | &left.compiler.display_name(), |
| 2152 | &right.compiler.display_name(), |
| 2153 | ), |
| 2154 | }); |
| 2155 | } |
| 2156 | } |
| 2157 | _ => differences.push(ArtifactDifference { |
| 2158 | artifact: "runtime".into(), |
| 2159 | detail: "one side did not produce a runtime result".into(), |
| 2160 | }), |
| 2161 | } |
| 2162 | } |
| 2163 | |
| 2164 | fn compare_artifact_int( |
| 2165 | left: &CompilerObservation, |
| 2166 | right: &CompilerObservation, |
| 2167 | artifact: &ArtifactKey, |
| 2168 | differences: &mut Vec<ArtifactDifference>, |
| 2169 | ) { |
| 2170 | let left_value = match left.artifacts.get(artifact) { |
| 2171 | Some(ArtifactValue::Int(value)) => Some(*value), |
| 2172 | _ => None, |
| 2173 | }; |
| 2174 | let right_value = match right.artifacts.get(artifact) { |
| 2175 | Some(ArtifactValue::Int(value)) => Some(*value), |
| 2176 | _ => None, |
| 2177 | }; |
| 2178 | if left_value != right_value { |
| 2179 | differences.push(ArtifactDifference { |
| 2180 | artifact: artifact.as_str().to_string(), |
| 2181 | detail: format!( |
| 2182 | "{}: {:?}\n{}: {:?}", |
| 2183 | left.compiler.display_name(), |
| 2184 | left_value, |
| 2185 | right.compiler.display_name(), |
| 2186 | right_value |
| 2187 | ), |
| 2188 | }); |
| 2189 | } |
| 2190 | } |
| 2191 | |
| 2192 | fn compare_artifact_path( |
| 2193 | left: &CompilerObservation, |
| 2194 | right: &CompilerObservation, |
| 2195 | artifact: &ArtifactKey, |
| 2196 | differences: &mut Vec<ArtifactDifference>, |
| 2197 | ) { |
| 2198 | let left_path = match left.artifacts.get(artifact) { |
| 2199 | Some(ArtifactValue::Path(path)) => Some(path), |
| 2200 | _ => None, |
| 2201 | }; |
| 2202 | let right_path = match right.artifacts.get(artifact) { |
| 2203 | Some(ArtifactValue::Path(path)) => Some(path), |
| 2204 | _ => None, |
| 2205 | }; |
| 2206 | |
| 2207 | match (left_path, right_path) { |
| 2208 | (Some(left_path), Some(right_path)) => { |
| 2209 | let left_bytes = fs::read(left_path); |
| 2210 | let right_bytes = fs::read(right_path); |
| 2211 | match (left_bytes, right_bytes) { |
| 2212 | (Ok(left_bytes), Ok(right_bytes)) => { |
| 2213 | if left_bytes != right_bytes { |
| 2214 | differences.push(ArtifactDifference { |
| 2215 | artifact: artifact.as_str().to_string(), |
| 2216 | detail: describe_binary_difference( |
| 2217 | &left_bytes, |
| 2218 | &right_bytes, |
| 2219 | &left.compiler.display_name(), |
| 2220 | &right.compiler.display_name(), |
| 2221 | left_path, |
| 2222 | right_path, |
| 2223 | ), |
| 2224 | }); |
| 2225 | } |
| 2226 | } |
| 2227 | (Err(left_err), Err(right_err)) => { |
| 2228 | differences.push(ArtifactDifference { |
| 2229 | artifact: artifact.as_str().to_string(), |
| 2230 | detail: format!( |
| 2231 | "{}: unable to read '{}': {}\n{}: unable to read '{}': {}", |
| 2232 | left.compiler.display_name(), |
| 2233 | left_path.display(), |
| 2234 | left_err, |
| 2235 | right.compiler.display_name(), |
| 2236 | right_path.display(), |
| 2237 | right_err |
| 2238 | ), |
| 2239 | }); |
| 2240 | } |
| 2241 | (Err(left_err), Ok(_)) => { |
| 2242 | differences.push(ArtifactDifference { |
| 2243 | artifact: artifact.as_str().to_string(), |
| 2244 | detail: format!( |
| 2245 | "{}: unable to read '{}': {}\n{}: readable '{}'", |
| 2246 | left.compiler.display_name(), |
| 2247 | left_path.display(), |
| 2248 | left_err, |
| 2249 | right.compiler.display_name(), |
| 2250 | right_path.display() |
| 2251 | ), |
| 2252 | }); |
| 2253 | } |
| 2254 | (Ok(_), Err(right_err)) => { |
| 2255 | differences.push(ArtifactDifference { |
| 2256 | artifact: artifact.as_str().to_string(), |
| 2257 | detail: format!( |
| 2258 | "{}: readable '{}'\n{}: unable to read '{}': {}", |
| 2259 | left.compiler.display_name(), |
| 2260 | left_path.display(), |
| 2261 | right.compiler.display_name(), |
| 2262 | right_path.display(), |
| 2263 | right_err |
| 2264 | ), |
| 2265 | }); |
| 2266 | } |
| 2267 | } |
| 2268 | } |
| 2269 | _ => { |
| 2270 | let left_value = left_path.map(|path| path.display().to_string()); |
| 2271 | let right_value = right_path.map(|path| path.display().to_string()); |
| 2272 | if left_value != right_value { |
| 2273 | differences.push(ArtifactDifference { |
| 2274 | artifact: artifact.as_str().to_string(), |
| 2275 | detail: format!( |
| 2276 | "{}: {:?}\n{}: {:?}", |
| 2277 | left.compiler.display_name(), |
| 2278 | left_value, |
| 2279 | right.compiler.display_name(), |
| 2280 | right_value |
| 2281 | ), |
| 2282 | }); |
| 2283 | } |
| 2284 | } |
| 2285 | } |
| 2286 | } |
| 2287 | |
| 2288 | fn describe_binary_difference( |
| 2289 | left: &[u8], |
| 2290 | right: &[u8], |
| 2291 | left_label: &str, |
| 2292 | right_label: &str, |
| 2293 | left_path: &Path, |
| 2294 | right_path: &Path, |
| 2295 | ) -> String { |
| 2296 | let shared = left.len().min(right.len()); |
| 2297 | for index in 0..shared { |
| 2298 | if left[index] != right[index] { |
| 2299 | return format!( |
| 2300 | "first differing byte: {}\n{}: {} bytes ({})\n{}: 0x{:02x}\n{}: {} bytes ({})\n{}: 0x{:02x}", |
| 2301 | index, |
| 2302 | left_label, |
| 2303 | left.len(), |
| 2304 | left_path.display(), |
| 2305 | left_label, |
| 2306 | left[index], |
| 2307 | right_label, |
| 2308 | right.len(), |
| 2309 | right_path.display(), |
| 2310 | right_label, |
| 2311 | right[index] |
| 2312 | ); |
| 2313 | } |
| 2314 | } |
| 2315 | |
| 2316 | format!( |
| 2317 | "binary length differs\n{}: {} bytes ({})\n{}: {} bytes ({})", |
| 2318 | left_label, |
| 2319 | left.len(), |
| 2320 | left_path.display(), |
| 2321 | right_label, |
| 2322 | right.len(), |
| 2323 | right_path.display() |
| 2324 | ) |
| 2325 | } |
| 2326 | |
| 2327 | fn compare_status(result: &ComparisonResult) -> &'static str { |
| 2328 | if result.differences.is_empty() { |
| 2329 | "match" |
| 2330 | } else { |
| 2331 | "diff" |
| 2332 | } |
| 2333 | } |
| 2334 | |
| 2335 | fn compare_classification(result: &ComparisonResult) -> &'static str { |
| 2336 | if result.differences.is_empty() { |
| 2337 | return "match"; |
| 2338 | } |
| 2339 | |
| 2340 | let mut has_compile = false; |
| 2341 | let mut has_diagnostics = false; |
| 2342 | let mut has_runtime = false; |
| 2343 | let mut has_artifact = false; |
| 2344 | |
| 2345 | for difference in &result.differences { |
| 2346 | match difference.artifact.as_str() { |
| 2347 | "compile-exit-code" => has_compile = true, |
| 2348 | "diagnostics" => has_diagnostics = true, |
| 2349 | "runtime" => has_runtime = true, |
| 2350 | _ => has_artifact = true, |
| 2351 | } |
| 2352 | } |
| 2353 | |
| 2354 | if has_compile { |
| 2355 | if has_runtime || has_artifact { |
| 2356 | "mixed divergence" |
| 2357 | } else { |
| 2358 | "compile divergence" |
| 2359 | } |
| 2360 | } else if has_runtime && !has_diagnostics && !has_artifact { |
| 2361 | "runtime divergence" |
| 2362 | } else if has_artifact && !has_runtime && !has_diagnostics { |
| 2363 | "artifact divergence" |
| 2364 | } else if has_diagnostics && !has_runtime && !has_artifact { |
| 2365 | "diagnostics divergence" |
| 2366 | } else { |
| 2367 | "mixed divergence" |
| 2368 | } |
| 2369 | } |
| 2370 | |
| 2371 | fn compare_changed_artifacts(result: &ComparisonResult) -> Vec<String> { |
| 2372 | result |
| 2373 | .differences |
| 2374 | .iter() |
| 2375 | .map(|difference| difference.artifact.clone()) |
| 2376 | .collect() |
| 2377 | } |
| 2378 | |
| 2379 | fn render_compare_text(result: &ComparisonResult) -> String { |
| 2380 | let changed_artifacts = compare_changed_artifacts(result); |
| 2381 | let mut lines = vec![ |
| 2382 | "Compare".to_string(), |
| 2383 | format!(" left: {}", result.left.compiler.display_name()), |
| 2384 | format!(" right: {}", result.right.compiler.display_name()), |
| 2385 | format!(" program: {}", result.left.program.display()), |
| 2386 | format!(" opt: {}", result.left.opt_level.as_str()), |
| 2387 | format!(" status: {}", compare_status(result)), |
| 2388 | format!(" classification: {}", compare_classification(result)), |
| 2389 | format!(" basis: {}", result.basis), |
| 2390 | format!(" difference_count: {}", result.differences.len()), |
| 2391 | format!( |
| 2392 | " changed_artifacts: {}", |
| 2393 | if changed_artifacts.is_empty() { |
| 2394 | "none".to_string() |
| 2395 | } else { |
| 2396 | changed_artifacts.join(", ") |
| 2397 | } |
| 2398 | ), |
| 2399 | format!( |
| 2400 | " left_backend: {} ({})", |
| 2401 | result.left.provenance.backend_mode, result.left.provenance.backend_detail |
| 2402 | ), |
| 2403 | format!( |
| 2404 | " right_backend: {} ({})", |
| 2405 | result.right.provenance.backend_mode, result.right.provenance.backend_detail |
| 2406 | ), |
| 2407 | ]; |
| 2408 | |
| 2409 | if result.differences.is_empty() { |
| 2410 | lines.push(String::new()); |
| 2411 | lines.push("No differences detected.".to_string()); |
| 2412 | } else { |
| 2413 | for difference in &result.differences { |
| 2414 | lines.push(String::new()); |
| 2415 | lines.push(format!("== {} ==", difference.artifact)); |
| 2416 | lines.push(difference.detail.clone()); |
| 2417 | } |
| 2418 | } |
| 2419 | |
| 2420 | lines.join("\n") |
| 2421 | } |
| 2422 | |
| 2423 | fn print_compare_result(result: &ComparisonResult) { |
| 2424 | println!("{}", render_compare_text(result)); |
| 2425 | } |
| 2426 | |
| 2427 | fn print_introspection(config: &IntrospectConfig, observed: &ObservedProgram) { |
| 2428 | println!( |
| 2429 | "{}", |
| 2430 | render_introspection_text( |
| 2431 | observed, |
| 2432 | IntrospectionRenderConfig { |
| 2433 | summary_only: config.summary_only, |
| 2434 | max_artifact_lines: config.max_artifact_lines, |
| 2435 | } |
| 2436 | ) |
| 2437 | ); |
| 2438 | } |
| 2439 | |
| 2440 | fn write_compare_reports(config: &CompareConfig, result: &ComparisonResult) -> Result<(), String> { |
| 2441 | if let Some(path) = &config.json_report { |
| 2442 | write_report(path, &render_compare_json(result), "json report")?; |
| 2443 | println!("json report: {}", path.display()); |
| 2444 | } |
| 2445 | if let Some(path) = &config.markdown_report { |
| 2446 | write_report(path, &render_compare_markdown(result), "markdown report")?; |
| 2447 | println!("markdown report: {}", path.display()); |
| 2448 | } |
| 2449 | Ok(()) |
| 2450 | } |
| 2451 | |
| 2452 | fn write_introspection_reports( |
| 2453 | config: &IntrospectConfig, |
| 2454 | observed: &ObservedProgram, |
| 2455 | ) -> Result<(), String> { |
| 2456 | let render_config = IntrospectionRenderConfig { |
| 2457 | summary_only: config.summary_only, |
| 2458 | max_artifact_lines: config.max_artifact_lines, |
| 2459 | }; |
| 2460 | if let Some(path) = &config.json_report { |
| 2461 | write_report(path, &render_introspection_json(observed), "json report")?; |
| 2462 | println!("json report: {}", path.display()); |
| 2463 | } |
| 2464 | if let Some(path) = &config.markdown_report { |
| 2465 | write_report( |
| 2466 | path, |
| 2467 | &render_introspection_markdown(observed, render_config), |
| 2468 | "markdown report", |
| 2469 | )?; |
| 2470 | println!("markdown report: {}", path.display()); |
| 2471 | } |
| 2472 | Ok(()) |
| 2473 | } |
| 2474 | |
| 2475 | fn introspection_status(observation: &CompilerObservation) -> &'static str { |
| 2476 | if observation.compile_exit_code == 0 { |
| 2477 | "compile ok" |
| 2478 | } else { |
| 2479 | "compile failed" |
| 2480 | } |
| 2481 | } |
| 2482 | |
| 2483 | fn diagnostic_excerpt(observation: &CompilerObservation) -> Option<String> { |
| 2484 | match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 2485 | Some(ArtifactValue::Text(text)) => text |
| 2486 | .lines() |
| 2487 | .map(str::trim) |
| 2488 | .find(|line| !line.is_empty()) |
| 2489 | .map(|line| line.to_string()), |
| 2490 | _ => None, |
| 2491 | } |
| 2492 | } |
| 2493 | |
| 2494 | fn failure_stage_summary(observation: &CompilerObservation) -> &str { |
| 2495 | observation |
| 2496 | .provenance |
| 2497 | .failure_stage |
| 2498 | .as_deref() |
| 2499 | .unwrap_or("none") |
| 2500 | } |
| 2501 | |
| 2502 | fn requested_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2503 | observed |
| 2504 | .requested_artifacts |
| 2505 | .iter() |
| 2506 | .map(|artifact| artifact.as_str().to_string()) |
| 2507 | .collect() |
| 2508 | } |
| 2509 | |
| 2510 | fn missing_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2511 | observed |
| 2512 | .requested_artifacts |
| 2513 | .iter() |
| 2514 | .filter(|artifact| { |
| 2515 | if matches!(artifact, ArtifactKey::Diagnostics) |
| 2516 | && observed.observation.compile_exit_code == 0 |
| 2517 | && !observed.observation.artifacts.contains_key(*artifact) |
| 2518 | { |
| 2519 | return false; |
| 2520 | } |
| 2521 | !observed.observation.artifacts.contains_key(*artifact) |
| 2522 | }) |
| 2523 | .map(|artifact| artifact.as_str().to_string()) |
| 2524 | .collect() |
| 2525 | } |
| 2526 | |
| 2527 | fn observation_generic_artifacts<'a>( |
| 2528 | observation: &'a CompilerObservation, |
| 2529 | ) -> Vec<(String, &'a ArtifactValue)> { |
| 2530 | observation |
| 2531 | .artifacts |
| 2532 | .iter() |
| 2533 | .filter(|(artifact, _)| artifact.is_generic()) |
| 2534 | .map(|(artifact, value)| (artifact.as_str().to_string(), value)) |
| 2535 | .collect() |
| 2536 | } |
| 2537 | |
| 2538 | fn observation_adapter_extras<'a>( |
| 2539 | observation: &'a CompilerObservation, |
| 2540 | ) -> BTreeMap<String, Vec<(String, &'a ArtifactValue)>> { |
| 2541 | let mut extras = BTreeMap::new(); |
| 2542 | for (artifact, value) in &observation.artifacts { |
| 2543 | if let ArtifactKey::Extra(name) = artifact { |
| 2544 | let (namespace, local_name) = artifact |
| 2545 | .extra_parts() |
| 2546 | .map(|(namespace, local_name)| (namespace.to_string(), local_name.to_string())) |
| 2547 | .unwrap_or_else(|| ("extra".to_string(), name.clone())); |
| 2548 | extras |
| 2549 | .entry(namespace) |
| 2550 | .or_insert_with(Vec::new) |
| 2551 | .push((local_name, value)); |
| 2552 | } |
| 2553 | } |
| 2554 | extras |
| 2555 | } |
| 2556 | |
| 2557 | fn format_artifact_name_list(names: &[String]) -> String { |
| 2558 | if names.is_empty() { |
| 2559 | "none".to_string() |
| 2560 | } else { |
| 2561 | names.join(", ") |
| 2562 | } |
| 2563 | } |
| 2564 | |
| 2565 | fn format_adapter_extra_summary( |
| 2566 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2567 | ) -> String { |
| 2568 | if extras.is_empty() { |
| 2569 | return "none".to_string(); |
| 2570 | } |
| 2571 | |
| 2572 | extras |
| 2573 | .iter() |
| 2574 | .map(|(namespace, entries)| { |
| 2575 | format!( |
| 2576 | "{}({})", |
| 2577 | namespace, |
| 2578 | entries |
| 2579 | .iter() |
| 2580 | .map(|(name, _)| name.as_str()) |
| 2581 | .collect::<Vec<_>>() |
| 2582 | .join(", ") |
| 2583 | ) |
| 2584 | }) |
| 2585 | .collect::<Vec<_>>() |
| 2586 | .join(", ") |
| 2587 | } |
| 2588 | |
| 2589 | fn render_named_artifact_map_json(entries: &[(String, &ArtifactValue)]) -> String { |
| 2590 | let mut rendered = String::from("{"); |
| 2591 | for (index, (name, value)) in entries.iter().enumerate() { |
| 2592 | if index > 0 { |
| 2593 | rendered.push_str(", "); |
| 2594 | } |
| 2595 | rendered.push_str(&format!( |
| 2596 | "\"{}\": {}", |
| 2597 | json_escape(name), |
| 2598 | render_artifact_value_json(value) |
| 2599 | )); |
| 2600 | } |
| 2601 | rendered.push('}'); |
| 2602 | rendered |
| 2603 | } |
| 2604 | |
| 2605 | fn render_flat_artifacts_json(observation: &CompilerObservation) -> String { |
| 2606 | let mut entries = Vec::new(); |
| 2607 | for (artifact, value) in &observation.artifacts { |
| 2608 | entries.push((artifact.as_str().to_string(), value)); |
| 2609 | } |
| 2610 | render_named_artifact_map_json(&entries) |
| 2611 | } |
| 2612 | |
| 2613 | fn render_artifact_summary_json(value: &ArtifactValue) -> String { |
| 2614 | match value { |
| 2615 | ArtifactValue::Text(text) => format!( |
| 2616 | "{{\"kind\":\"text\",\"summary\":\"{}\",\"line_count\":{},\"char_count\":{}}}", |
| 2617 | json_escape(&artifact_value_summary(value)), |
| 2618 | text_line_count(text), |
| 2619 | text.len() |
| 2620 | ), |
| 2621 | ArtifactValue::Int(number) => format!( |
| 2622 | "{{\"kind\":\"int\",\"summary\":\"{}\",\"value\":{}}}", |
| 2623 | json_escape(&artifact_value_summary(value)), |
| 2624 | number |
| 2625 | ), |
| 2626 | ArtifactValue::Run(run) => format!( |
| 2627 | "{{\"kind\":\"runtime\",\"summary\":\"{}\",\"exit_code\":{},\"stdout_lines\":{},\"stderr_lines\":{}}}", |
| 2628 | json_escape(&artifact_value_summary(value)), |
| 2629 | run.exit_code, |
| 2630 | text_line_count(&run.stdout), |
| 2631 | text_line_count(&run.stderr) |
| 2632 | ), |
| 2633 | ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2634 | Ok(metadata) => format!( |
| 2635 | "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":{}}}", |
| 2636 | json_escape(&artifact_value_summary(value)), |
| 2637 | metadata.len() |
| 2638 | ), |
| 2639 | Err(_) => format!( |
| 2640 | "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":null}}", |
| 2641 | json_escape(&artifact_value_summary(value)) |
| 2642 | ), |
| 2643 | }, |
| 2644 | } |
| 2645 | } |
| 2646 | |
| 2647 | fn render_artifact_summaries_json(observation: &CompilerObservation) -> String { |
| 2648 | let mut rendered = String::from("{"); |
| 2649 | for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 2650 | if index > 0 { |
| 2651 | rendered.push_str(", "); |
| 2652 | } |
| 2653 | rendered.push_str(&format!( |
| 2654 | "\"{}\": {}", |
| 2655 | json_escape(artifact.as_str()), |
| 2656 | render_artifact_summary_json(value) |
| 2657 | )); |
| 2658 | } |
| 2659 | rendered.push('}'); |
| 2660 | rendered |
| 2661 | } |
| 2662 | |
| 2663 | fn render_adapter_extra_summary_json( |
| 2664 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2665 | ) -> String { |
| 2666 | let mut rendered = String::from("{"); |
| 2667 | for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2668 | if index > 0 { |
| 2669 | rendered.push_str(", "); |
| 2670 | } |
| 2671 | let names = entries |
| 2672 | .iter() |
| 2673 | .map(|(name, _)| name.clone()) |
| 2674 | .collect::<Vec<_>>(); |
| 2675 | rendered.push_str(&format!( |
| 2676 | "\"{}\": {}", |
| 2677 | json_escape(namespace), |
| 2678 | json_string_array(&names) |
| 2679 | )); |
| 2680 | } |
| 2681 | rendered.push('}'); |
| 2682 | rendered |
| 2683 | } |
| 2684 | |
| 2685 | fn render_namespaced_artifacts_json( |
| 2686 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2687 | ) -> String { |
| 2688 | let mut rendered = String::from("{"); |
| 2689 | for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2690 | if index > 0 { |
| 2691 | rendered.push_str(", "); |
| 2692 | } |
| 2693 | rendered.push_str(&format!( |
| 2694 | "\"{}\": {}", |
| 2695 | json_escape(namespace), |
| 2696 | render_named_artifact_map_json(entries) |
| 2697 | )); |
| 2698 | } |
| 2699 | rendered.push('}'); |
| 2700 | rendered |
| 2701 | } |
| 2702 | |
| 2703 | fn text_line_count(text: &str) -> usize { |
| 2704 | if text.is_empty() { |
| 2705 | 0 |
| 2706 | } else { |
| 2707 | text.lines().count() |
| 2708 | } |
| 2709 | } |
| 2710 | |
| 2711 | fn render_config_summary(config: IntrospectionRenderConfig) -> String { |
| 2712 | if config.summary_only { |
| 2713 | "summary-only".to_string() |
| 2714 | } else if let Some(limit) = config.max_artifact_lines { |
| 2715 | format!("first {} lines per artifact", limit) |
| 2716 | } else { |
| 2717 | "full artifact bodies".to_string() |
| 2718 | } |
| 2719 | } |
| 2720 | |
| 2721 | fn artifact_value_summary(value: &ArtifactValue) -> String { |
| 2722 | match value { |
| 2723 | ArtifactValue::Text(text) => { |
| 2724 | format!( |
| 2725 | "text, {} lines, {} chars", |
| 2726 | text_line_count(text), |
| 2727 | text.len() |
| 2728 | ) |
| 2729 | } |
| 2730 | ArtifactValue::Int(value) => format!("int, value {}", value), |
| 2731 | ArtifactValue::Run(run) => format!( |
| 2732 | "runtime, exit {}, stdout {} lines, stderr {} lines", |
| 2733 | run.exit_code, |
| 2734 | text_line_count(&run.stdout), |
| 2735 | text_line_count(&run.stderr) |
| 2736 | ), |
| 2737 | ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2738 | Ok(metadata) => format!("path, {} bytes", metadata.len()), |
| 2739 | Err(_) => "path".to_string(), |
| 2740 | }, |
| 2741 | } |
| 2742 | } |
| 2743 | |
| 2744 | fn render_artifact_body_lines( |
| 2745 | value: &ArtifactValue, |
| 2746 | config: IntrospectionRenderConfig, |
| 2747 | ) -> Vec<String> { |
| 2748 | if config.summary_only { |
| 2749 | return vec!["[content omitted by --summary-only]".to_string()]; |
| 2750 | } |
| 2751 | |
| 2752 | let rendered = render_artifact_value_text(value); |
| 2753 | let mut lines = rendered |
| 2754 | .lines() |
| 2755 | .map(|line| line.to_string()) |
| 2756 | .collect::<Vec<_>>(); |
| 2757 | if lines.is_empty() { |
| 2758 | return vec!["<empty>".to_string()]; |
| 2759 | } |
| 2760 | |
| 2761 | if let Some(limit) = config.max_artifact_lines { |
| 2762 | if lines.len() > limit { |
| 2763 | let total = lines.len(); |
| 2764 | lines.truncate(limit); |
| 2765 | lines.push(format!( |
| 2766 | "... (truncated; showing first {} of {} lines)", |
| 2767 | limit, total |
| 2768 | )); |
| 2769 | } |
| 2770 | } |
| 2771 | |
| 2772 | lines |
| 2773 | } |
| 2774 | |
| 2775 | fn render_introspection_text( |
| 2776 | observed: &ObservedProgram, |
| 2777 | render_config: IntrospectionRenderConfig, |
| 2778 | ) -> String { |
| 2779 | let observation = &observed.observation; |
| 2780 | let generic_artifacts = observation_generic_artifacts(observation); |
| 2781 | let generic_names = generic_artifacts |
| 2782 | .iter() |
| 2783 | .map(|(name, _)| name.clone()) |
| 2784 | .collect::<Vec<_>>(); |
| 2785 | let adapter_extras = observation_adapter_extras(observation); |
| 2786 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2787 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2788 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2789 | let mut lines = vec![ |
| 2790 | "Introspect".to_string(), |
| 2791 | format!(" status: {}", introspection_status(observation)), |
| 2792 | format!(" compiler: {}", observation.compiler.display_name()), |
| 2793 | format!(" program: {}", observation.program.display()), |
| 2794 | format!(" opt: {}", observation.opt_level.as_str()), |
| 2795 | format!(" compile_exit_code: {}", observation.compile_exit_code), |
| 2796 | format!(" adapter_kind: {}", observation.provenance.adapter_kind), |
| 2797 | format!(" backend_mode: {}", observation.provenance.backend_mode), |
| 2798 | format!( |
| 2799 | " backend_detail: {}", |
| 2800 | observation.provenance.backend_detail |
| 2801 | ), |
| 2802 | format!(" failure_stage: {}", failure_stage_summary(observation)), |
| 2803 | format!( |
| 2804 | " diagnostic_excerpt: {}", |
| 2805 | diagnostic_excerpt |
| 2806 | .clone() |
| 2807 | .unwrap_or_else(|| "none".to_string()) |
| 2808 | ), |
| 2809 | format!(" content_mode: {}", render_config_summary(render_config)), |
| 2810 | format!(" artifact_count: {}", observation.artifacts.len()), |
| 2811 | format!( |
| 2812 | " requested_artifacts: {}", |
| 2813 | format_artifact_name_list(&requested_artifacts) |
| 2814 | ), |
| 2815 | format!( |
| 2816 | " missing_artifacts: {}", |
| 2817 | format_artifact_name_list(&missing_artifacts) |
| 2818 | ), |
| 2819 | format!( |
| 2820 | " generic_artifacts: {}", |
| 2821 | format_artifact_name_list(&generic_names) |
| 2822 | ), |
| 2823 | format!( |
| 2824 | " adapter_extras: {}", |
| 2825 | format_adapter_extra_summary(&adapter_extras) |
| 2826 | ), |
| 2827 | ]; |
| 2828 | if !observation.provenance.artifacts_captured.is_empty() { |
| 2829 | lines.push(format!( |
| 2830 | " captured_artifacts: {}", |
| 2831 | observation.provenance.artifacts_captured.join(", ") |
| 2832 | )); |
| 2833 | } |
| 2834 | |
| 2835 | if !generic_artifacts.is_empty() { |
| 2836 | lines.push(String::new()); |
| 2837 | lines.push("Generic artifacts".to_string()); |
| 2838 | for (artifact, value) in generic_artifacts { |
| 2839 | lines.push(String::new()); |
| 2840 | lines.push(format!("== {} ==", artifact)); |
| 2841 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2842 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 2843 | } |
| 2844 | } |
| 2845 | |
| 2846 | if !adapter_extras.is_empty() { |
| 2847 | lines.push(String::new()); |
| 2848 | lines.push("Adapter extras".to_string()); |
| 2849 | for (namespace, entries) in adapter_extras { |
| 2850 | lines.push(String::new()); |
| 2851 | lines.push(format!("-- {} --", namespace)); |
| 2852 | for (name, value) in entries { |
| 2853 | lines.push(String::new()); |
| 2854 | lines.push(format!("== {} ==", name)); |
| 2855 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2856 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 2857 | } |
| 2858 | } |
| 2859 | } |
| 2860 | lines.join("\n") |
| 2861 | } |
| 2862 | |
| 2863 | fn render_compare_json(result: &ComparisonResult) -> String { |
| 2864 | let changed_artifacts = compare_changed_artifacts(result); |
| 2865 | format!( |
| 2866 | "{{\n \"status\": \"{}\",\n \"classification\": \"{}\",\n \"difference_count\": {},\n \"changed_artifacts\": {},\n \"basis\": \"{}\",\n \"left\": {},\n \"right\": {},\n \"differences\": {}\n}}\n", |
| 2867 | compare_status(result), |
| 2868 | compare_classification(result), |
| 2869 | result.differences.len(), |
| 2870 | json_string_array(&changed_artifacts), |
| 2871 | json_escape(&result.basis), |
| 2872 | render_observation_json(&result.left), |
| 2873 | render_observation_json(&result.right), |
| 2874 | render_differences_json(&result.differences) |
| 2875 | ) |
| 2876 | } |
| 2877 | |
| 2878 | fn render_compare_markdown(result: &ComparisonResult) -> String { |
| 2879 | let changed_artifacts = compare_changed_artifacts(result); |
| 2880 | let mut lines = vec![ |
| 2881 | "# bencch compare report".to_string(), |
| 2882 | String::new(), |
| 2883 | format!("status: {}", compare_status(result)), |
| 2884 | format!("classification: {}", compare_classification(result)), |
| 2885 | format!( |
| 2886 | "compilers: `{}` vs `{}`", |
| 2887 | result.left.compiler.display_name(), |
| 2888 | result.right.compiler.display_name() |
| 2889 | ), |
| 2890 | format!("basis: {}", result.basis), |
| 2891 | format!("difference_count: {}", result.differences.len()), |
| 2892 | format!( |
| 2893 | "changed_artifacts: {}", |
| 2894 | if changed_artifacts.is_empty() { |
| 2895 | "none".to_string() |
| 2896 | } else { |
| 2897 | changed_artifacts.join(", ") |
| 2898 | } |
| 2899 | ), |
| 2900 | String::new(), |
| 2901 | "## Left".to_string(), |
| 2902 | render_observation_markdown(&result.left), |
| 2903 | String::new(), |
| 2904 | "## Right".to_string(), |
| 2905 | render_observation_markdown(&result.right), |
| 2906 | String::new(), |
| 2907 | "## Differences".to_string(), |
| 2908 | ]; |
| 2909 | if result.differences.is_empty() { |
| 2910 | lines.push("none".to_string()); |
| 2911 | } else { |
| 2912 | for difference in &result.differences { |
| 2913 | lines.push(String::new()); |
| 2914 | lines.push(format!("### `{}`", difference.artifact)); |
| 2915 | lines.push("```text".to_string()); |
| 2916 | lines.extend(difference.detail.lines().map(|line| line.to_string())); |
| 2917 | lines.push("```".to_string()); |
| 2918 | } |
| 2919 | } |
| 2920 | lines.join("\n") + "\n" |
| 2921 | } |
| 2922 | |
| 2923 | fn render_introspection_json(observed: &ObservedProgram) -> String { |
| 2924 | let observation = &observed.observation; |
| 2925 | let generic_artifacts = observation_generic_artifacts(observation); |
| 2926 | let generic_names = generic_artifacts |
| 2927 | .iter() |
| 2928 | .map(|(name, _)| name.clone()) |
| 2929 | .collect::<Vec<_>>(); |
| 2930 | let adapter_extras = observation_adapter_extras(observation); |
| 2931 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2932 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2933 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2934 | format!( |
| 2935 | "{{\n \"status\": \"{}\",\n \"compiler\": \"{}\",\n \"program\": \"{}\",\n \"opt\": \"{}\",\n \"compile_exit_code\": {},\n \"failure\": {{\n \"stage\": {},\n \"diagnostic_excerpt\": {}\n }},\n \"artifact_summary\": {{\n \"artifact_count\": {},\n \"requested_artifacts\": {},\n \"captured_artifacts\": {},\n \"missing_artifacts\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {}\n }},\n \"provenance\": {{\n \"compiler_identity\": \"{}\",\n \"adapter_kind\": \"{}\",\n \"backend_mode\": \"{}\",\n \"backend_detail\": \"{}\",\n \"artifacts_captured\": {},\n \"comparison_basis\": {},\n \"failure_stage\": {}\n }},\n \"artifact_summaries\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {},\n \"artifacts\": {}\n}}\n", |
| 2936 | json_escape(introspection_status(observation)), |
| 2937 | json_escape(&observation.compiler.display_name()), |
| 2938 | json_escape(&observation.program.display().to_string()), |
| 2939 | observation.opt_level.as_str(), |
| 2940 | observation.compile_exit_code, |
| 2941 | match &observation.provenance.failure_stage { |
| 2942 | Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2943 | None => "null".to_string(), |
| 2944 | }, |
| 2945 | match &diagnostic_excerpt { |
| 2946 | Some(line) => format!("\"{}\"", json_escape(line)), |
| 2947 | None => "null".to_string(), |
| 2948 | }, |
| 2949 | observation.artifacts.len(), |
| 2950 | json_string_array(&requested_artifacts), |
| 2951 | json_string_array(&observation.provenance.artifacts_captured), |
| 2952 | json_string_array(&missing_artifacts), |
| 2953 | json_string_array(&generic_names), |
| 2954 | render_adapter_extra_summary_json(&adapter_extras), |
| 2955 | json_escape(&observation.provenance.compiler_identity), |
| 2956 | json_escape(&observation.provenance.adapter_kind), |
| 2957 | json_escape(&observation.provenance.backend_mode), |
| 2958 | json_escape(&observation.provenance.backend_detail), |
| 2959 | json_string_array(&observation.provenance.artifacts_captured), |
| 2960 | match &observation.provenance.comparison_basis { |
| 2961 | Some(basis) => format!("\"{}\"", json_escape(basis)), |
| 2962 | None => "null".to_string(), |
| 2963 | }, |
| 2964 | match &observation.provenance.failure_stage { |
| 2965 | Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2966 | None => "null".to_string(), |
| 2967 | }, |
| 2968 | render_artifact_summaries_json(observation), |
| 2969 | render_named_artifact_map_json(&generic_artifacts), |
| 2970 | render_namespaced_artifacts_json(&adapter_extras), |
| 2971 | render_flat_artifacts_json(observation) |
| 2972 | ) |
| 2973 | } |
| 2974 | |
| 2975 | fn render_introspection_markdown( |
| 2976 | observed: &ObservedProgram, |
| 2977 | render_config: IntrospectionRenderConfig, |
| 2978 | ) -> String { |
| 2979 | let observation = &observed.observation; |
| 2980 | let generic_artifacts = observation_generic_artifacts(observation); |
| 2981 | let generic_names = generic_artifacts |
| 2982 | .iter() |
| 2983 | .map(|(name, _)| name.clone()) |
| 2984 | .collect::<Vec<_>>(); |
| 2985 | let adapter_extras = observation_adapter_extras(observation); |
| 2986 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2987 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2988 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2989 | let mut lines = vec![ |
| 2990 | "# bencch introspect report".to_string(), |
| 2991 | String::new(), |
| 2992 | format!("status: {}", introspection_status(observation)), |
| 2993 | format!("compiler: `{}`", observation.compiler.display_name()), |
| 2994 | format!("program: `{}`", observation.program.display()), |
| 2995 | format!("opt: `{}`", observation.opt_level.as_str()), |
| 2996 | format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 2997 | format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 2998 | format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 2999 | format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3000 | format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3001 | format!( |
| 3002 | "diagnostic_excerpt: {}", |
| 3003 | diagnostic_excerpt |
| 3004 | .map(|line| format!("`{}`", line)) |
| 3005 | .unwrap_or_else(|| "none".to_string()) |
| 3006 | ), |
| 3007 | format!("content_mode: `{}`", render_config_summary(render_config)), |
| 3008 | format!("artifact_count: {}", observation.artifacts.len()), |
| 3009 | format!( |
| 3010 | "requested_artifacts: {}", |
| 3011 | if requested_artifacts.is_empty() { |
| 3012 | "none".to_string() |
| 3013 | } else { |
| 3014 | format!("`{}`", requested_artifacts.join("`, `")) |
| 3015 | } |
| 3016 | ), |
| 3017 | format!( |
| 3018 | "missing_artifacts: {}", |
| 3019 | if missing_artifacts.is_empty() { |
| 3020 | "none".to_string() |
| 3021 | } else { |
| 3022 | format!("`{}`", missing_artifacts.join("`, `")) |
| 3023 | } |
| 3024 | ), |
| 3025 | format!( |
| 3026 | "generic_artifacts: {}", |
| 3027 | if generic_names.is_empty() { |
| 3028 | "none".to_string() |
| 3029 | } else { |
| 3030 | format!("`{}`", generic_names.join("`, `")) |
| 3031 | } |
| 3032 | ), |
| 3033 | format!( |
| 3034 | "adapter_extras: {}", |
| 3035 | format_adapter_extra_summary(&adapter_extras) |
| 3036 | ), |
| 3037 | ]; |
| 3038 | if !observation.provenance.artifacts_captured.is_empty() { |
| 3039 | lines.push(format!( |
| 3040 | "captured_artifacts: `{}`", |
| 3041 | observation.provenance.artifacts_captured.join("`, `") |
| 3042 | )); |
| 3043 | } |
| 3044 | |
| 3045 | if !generic_artifacts.is_empty() { |
| 3046 | lines.push(String::new()); |
| 3047 | lines.push("## Generic artifacts".to_string()); |
| 3048 | for (artifact, value) in generic_artifacts { |
| 3049 | lines.push(String::new()); |
| 3050 | lines.push(format!("### `{}`", artifact)); |
| 3051 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3052 | lines.push("```text".to_string()); |
| 3053 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 3054 | lines.push("```".to_string()); |
| 3055 | } |
| 3056 | } |
| 3057 | |
| 3058 | if !adapter_extras.is_empty() { |
| 3059 | lines.push(String::new()); |
| 3060 | lines.push("## Adapter extras".to_string()); |
| 3061 | for (namespace, entries) in adapter_extras { |
| 3062 | lines.push(String::new()); |
| 3063 | lines.push(format!("### `{}`", namespace)); |
| 3064 | for (name, value) in entries { |
| 3065 | lines.push(String::new()); |
| 3066 | lines.push(format!("#### `{}`", name)); |
| 3067 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3068 | lines.push("```text".to_string()); |
| 3069 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 3070 | lines.push("```".to_string()); |
| 3071 | } |
| 3072 | } |
| 3073 | } |
| 3074 | |
| 3075 | lines.join("\n") + "\n" |
| 3076 | } |
| 3077 | |
| 3078 | fn render_observation_json(observation: &CompilerObservation) -> String { |
| 3079 | let mut lines = vec![ |
| 3080 | "{".to_string(), |
| 3081 | format!( |
| 3082 | " \"compiler\": \"{}\",", |
| 3083 | json_escape(&observation.compiler.display_name()) |
| 3084 | ), |
| 3085 | format!( |
| 3086 | " \"program\": \"{}\",", |
| 3087 | json_escape(&observation.program.display().to_string()) |
| 3088 | ), |
| 3089 | format!(" \"opt\": \"{}\",", observation.opt_level.as_str()), |
| 3090 | format!( |
| 3091 | " \"compile_exit_code\": {},", |
| 3092 | observation.compile_exit_code |
| 3093 | ), |
| 3094 | " \"provenance\": {".to_string(), |
| 3095 | format!( |
| 3096 | " \"compiler_identity\": \"{}\",", |
| 3097 | json_escape(&observation.provenance.compiler_identity) |
| 3098 | ), |
| 3099 | format!( |
| 3100 | " \"adapter_kind\": \"{}\",", |
| 3101 | json_escape(&observation.provenance.adapter_kind) |
| 3102 | ), |
| 3103 | format!( |
| 3104 | " \"backend_mode\": \"{}\",", |
| 3105 | json_escape(&observation.provenance.backend_mode) |
| 3106 | ), |
| 3107 | format!( |
| 3108 | " \"backend_detail\": \"{}\",", |
| 3109 | json_escape(&observation.provenance.backend_detail) |
| 3110 | ), |
| 3111 | format!( |
| 3112 | " \"artifacts_captured\": {},", |
| 3113 | json_string_array(&observation.provenance.artifacts_captured) |
| 3114 | ), |
| 3115 | match &observation.provenance.failure_stage { |
| 3116 | Some(stage) => format!(" \"failure_stage\": \"{}\",", json_escape(stage)), |
| 3117 | None => " \"failure_stage\": null,".to_string(), |
| 3118 | }, |
| 3119 | match &observation.provenance.comparison_basis { |
| 3120 | Some(basis) => format!(" \"comparison_basis\": \"{}\"", json_escape(basis)), |
| 3121 | None => " \"comparison_basis\": null".to_string(), |
| 3122 | }, |
| 3123 | " },".to_string(), |
| 3124 | " \"artifacts\": {".to_string(), |
| 3125 | ]; |
| 3126 | for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 3127 | lines.push(format!( |
| 3128 | " \"{}\": {}{}", |
| 3129 | json_escape(artifact.as_str()), |
| 3130 | render_artifact_value_json(value), |
| 3131 | if index + 1 == observation.artifacts.len() { |
| 3132 | "" |
| 3133 | } else { |
| 3134 | "," |
| 3135 | } |
| 3136 | )); |
| 3137 | } |
| 3138 | lines.push(" }".to_string()); |
| 3139 | lines.push("}".to_string()); |
| 3140 | lines.join("\n") |
| 3141 | } |
| 3142 | |
| 3143 | fn render_observation_markdown(observation: &CompilerObservation) -> String { |
| 3144 | let mut lines = vec![ |
| 3145 | format!("compiler: `{}`", observation.compiler.display_name()), |
| 3146 | format!("program: `{}`", observation.program.display()), |
| 3147 | format!("opt: `{}`", observation.opt_level.as_str()), |
| 3148 | format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 3149 | format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 3150 | format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 3151 | format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3152 | format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3153 | ]; |
| 3154 | if !observation.provenance.artifacts_captured.is_empty() { |
| 3155 | lines.push(format!( |
| 3156 | "artifacts: `{}`", |
| 3157 | observation.provenance.artifacts_captured.join("`, `") |
| 3158 | )); |
| 3159 | } |
| 3160 | for (artifact, value) in &observation.artifacts { |
| 3161 | lines.push(String::new()); |
| 3162 | lines.push(format!("## `{}`", artifact.as_str())); |
| 3163 | lines.push("```text".to_string()); |
| 3164 | lines.extend( |
| 3165 | render_artifact_value_text(value) |
| 3166 | .lines() |
| 3167 | .map(|line| line.to_string()), |
| 3168 | ); |
| 3169 | lines.push("```".to_string()); |
| 3170 | } |
| 3171 | lines.join("\n") |
| 3172 | } |
| 3173 | |
| 3174 | fn render_differences_json(differences: &[ArtifactDifference]) -> String { |
| 3175 | let mut rendered = String::from("["); |
| 3176 | for (index, difference) in differences.iter().enumerate() { |
| 3177 | if index > 0 { |
| 3178 | rendered.push_str(", "); |
| 3179 | } |
| 3180 | rendered.push_str(&format!( |
| 3181 | "{{\"artifact\":\"{}\",\"detail\":\"{}\"}}", |
| 3182 | json_escape(&difference.artifact), |
| 3183 | json_escape(&difference.detail) |
| 3184 | )); |
| 3185 | } |
| 3186 | rendered.push(']'); |
| 3187 | rendered |
| 3188 | } |
| 3189 | |
| 3190 | fn render_artifact_value_json(value: &ArtifactValue) -> String { |
| 3191 | match value { |
| 3192 | ArtifactValue::Text(text) => { |
| 3193 | format!("{{\"kind\":\"text\",\"value\":\"{}\"}}", json_escape(text)) |
| 3194 | } |
| 3195 | ArtifactValue::Int(value) => format!("{{\"kind\":\"int\",\"value\":{}}}", value), |
| 3196 | ArtifactValue::Run(run) => format!( |
| 3197 | "{{\"kind\":\"runtime\",\"exit_code\":{},\"stdout\":\"{}\",\"stderr\":\"{}\"}}", |
| 3198 | run.exit_code, |
| 3199 | json_escape(&run.stdout), |
| 3200 | json_escape(&run.stderr) |
| 3201 | ), |
| 3202 | ArtifactValue::Path(path) => format!( |
| 3203 | "{{\"kind\":\"path\",\"value\":\"{}\"}}", |
| 3204 | json_escape(&path.display().to_string()) |
| 3205 | ), |
| 3206 | } |
| 3207 | } |
| 3208 | |
| 3209 | fn render_artifact_value_text(value: &ArtifactValue) -> String { |
| 3210 | match value { |
| 3211 | ArtifactValue::Text(text) => text.trim_end().to_string(), |
| 3212 | ArtifactValue::Int(value) => value.to_string(), |
| 3213 | ArtifactValue::Run(run) => format_run_capture(run), |
| 3214 | ArtifactValue::Path(path) => path.display().to_string(), |
| 3215 | } |
| 3216 | } |
| 3217 | |
| 3218 | fn default_suite_root() -> PathBuf { |
| 3219 | Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3220 | .join("..") |
| 3221 | .join("suites") |
| 3222 | } |
| 3223 | |
| 3224 | fn default_report_root() -> PathBuf { |
| 3225 | Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3226 | .join("..") |
| 3227 | .join("reports") |
| 3228 | } |
| 3229 | |
| 3230 | fn workspace_root() -> PathBuf { |
| 3231 | Path::new(env!("CARGO_MANIFEST_DIR")).join("..") |
| 3232 | } |
| 3233 | |
| 3234 | fn doctor_report_fields(config: &DoctorConfig) -> Vec<(String, String)> { |
| 3235 | let workspace_root = workspace_root(); |
| 3236 | let suite_root = default_suite_root(); |
| 3237 | let report_root = default_report_root(); |
| 3238 | let armfortas = config.tools.armfortas_adapters(); |
| 3239 | let observable_backend = config |
| 3240 | .tools |
| 3241 | .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3242 | let capture_root = armfortas.capture_root(); |
| 3243 | let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 3244 | |
| 3245 | let mut fields = vec![ |
| 3246 | ("workspace_root".to_string(), display_path(&workspace_root)), |
| 3247 | ("suite_root".to_string(), display_path(&suite_root)), |
| 3248 | ("report_root".to_string(), display_path(&report_root)), |
| 3249 | ( |
| 3250 | "armfortas_cli_adapter".to_string(), |
| 3251 | armfortas.cli_description().to_string(), |
| 3252 | ), |
| 3253 | ( |
| 3254 | "armfortas_capture_adapter".to_string(), |
| 3255 | armfortas.capture_description().to_string(), |
| 3256 | ), |
| 3257 | ( |
| 3258 | "primary_backend_full".to_string(), |
| 3259 | armfortas.capture_description().to_string(), |
| 3260 | ), |
| 3261 | ( |
| 3262 | "primary_backend_observable".to_string(), |
| 3263 | observable_backend.description().to_string(), |
| 3264 | ), |
| 3265 | ( |
| 3266 | "armfortas_capture_root".to_string(), |
| 3267 | capture_root |
| 3268 | .as_ref() |
| 3269 | .map(|root| display_path(root)) |
| 3270 | .unwrap_or_else(|| "unavailable".to_string()), |
| 3271 | ), |
| 3272 | ( |
| 3273 | "armfortas_capture_manifest".to_string(), |
| 3274 | capture_manifest |
| 3275 | .as_ref() |
| 3276 | .map(|manifest| { |
| 3277 | if manifest.exists() { |
| 3278 | display_path(manifest) |
| 3279 | } else { |
| 3280 | "missing".to_string() |
| 3281 | } |
| 3282 | }) |
| 3283 | .unwrap_or_else(|| "unavailable".to_string()), |
| 3284 | ), |
| 3285 | ( |
| 3286 | "armfortas_cli_mode".to_string(), |
| 3287 | armfortas.cli_mode_name().to_string(), |
| 3288 | ), |
| 3289 | ]; |
| 3290 | match armfortas.cli() { |
| 3291 | ArmfortasCliAdapter::Linked => { |
| 3292 | fields.push(( |
| 3293 | "armfortas_cli_status".to_string(), |
| 3294 | capture_root |
| 3295 | .as_ref() |
| 3296 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3297 | .unwrap_or_else(|| { |
| 3298 | "linked adapter requested but unavailable in this build".to_string() |
| 3299 | }), |
| 3300 | )); |
| 3301 | } |
| 3302 | ArmfortasCliAdapter::External(binary) => { |
| 3303 | fields.push(( |
| 3304 | "armfortas_cli_status".to_string(), |
| 3305 | tool_probe_status(binary, false), |
| 3306 | )); |
| 3307 | } |
| 3308 | } |
| 3309 | fields.push(( |
| 3310 | "armfortas_capture_mode".to_string(), |
| 3311 | armfortas.capture_mode_name().to_string(), |
| 3312 | )); |
| 3313 | fields.push(( |
| 3314 | "armfortas_capture_status".to_string(), |
| 3315 | capture_root |
| 3316 | .as_ref() |
| 3317 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3318 | .unwrap_or_else(|| { |
| 3319 | "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh".to_string() |
| 3320 | }), |
| 3321 | )); |
| 3322 | fields.push(( |
| 3323 | "primary_backend_selection".to_string(), |
| 3324 | "observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend".to_string(), |
| 3325 | )); |
| 3326 | for named in NamedCompiler::ALL { |
| 3327 | append_named_compiler_fields(&mut fields, named, &config.tools, capture_root.as_ref()); |
| 3328 | } |
| 3329 | fields.push(( |
| 3330 | "explicit_compiler_path".to_string(), |
| 3331 | "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3332 | .to_string(), |
| 3333 | )); |
| 3334 | let explicit_capabilities = compiler_capabilities( |
| 3335 | &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3336 | &config.tools, |
| 3337 | ); |
| 3338 | fields.push(( |
| 3339 | "explicit_compiler_path.generic_artifacts".to_string(), |
| 3340 | format_artifact_name_list(&explicit_capabilities.generic_artifacts()), |
| 3341 | )); |
| 3342 | fields.push(( |
| 3343 | "explicit_compiler_path.adapter_extras".to_string(), |
| 3344 | capability_extra_summary(&explicit_capabilities.adapter_extras()), |
| 3345 | )); |
| 3346 | fields.push(( |
| 3347 | "gfortran".to_string(), |
| 3348 | tool_probe_status(&config.tools.gfortran, false), |
| 3349 | )); |
| 3350 | fields.push(( |
| 3351 | "flang-new".to_string(), |
| 3352 | tool_probe_status(&config.tools.flang_new, false), |
| 3353 | )); |
| 3354 | fields.push(( |
| 3355 | "lfortran".to_string(), |
| 3356 | tool_probe_status(&config.tools.lfortran, false), |
| 3357 | )); |
| 3358 | fields.push(( |
| 3359 | "ifort".to_string(), |
| 3360 | tool_probe_status(&config.tools.ifort, false), |
| 3361 | )); |
| 3362 | fields.push(( |
| 3363 | "ifx".to_string(), |
| 3364 | tool_probe_status(&config.tools.ifx, false), |
| 3365 | )); |
| 3366 | fields.push(( |
| 3367 | "nvfortran".to_string(), |
| 3368 | tool_probe_status(&config.tools.nvfortran, false), |
| 3369 | )); |
| 3370 | fields.push(( |
| 3371 | "as".to_string(), |
| 3372 | tool_probe_status(&config.tools.system_as, false), |
| 3373 | )); |
| 3374 | fields.push(( |
| 3375 | "otool".to_string(), |
| 3376 | tool_probe_status(&config.tools.otool, false), |
| 3377 | )); |
| 3378 | fields.push(("nm".to_string(), tool_probe_status(&config.tools.nm, false))); |
| 3379 | fields.push(( |
| 3380 | "note".to_string(), |
| 3381 | if capture_root.is_some() { |
| 3382 | "linked capture still depends on the surrounding armfortas checkout".to_string() |
| 3383 | } else { |
| 3384 | "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work".to_string() |
| 3385 | }, |
| 3386 | )); |
| 3387 | fields.push(( |
| 3388 | if capture_root.is_some() { |
| 3389 | "linked_mode_surface".to_string() |
| 3390 | } else { |
| 3391 | "external_only_surface".to_string() |
| 3392 | }, |
| 3393 | if capture_root.is_some() { |
| 3394 | "rich armfortas stages, legacy frontend/module suites, capture consistency".to_string() |
| 3395 | } else { |
| 3396 | "compare, introspect, generic suite-v2, observable-only run cells".to_string() |
| 3397 | }, |
| 3398 | )); |
| 3399 | fields.push(( |
| 3400 | if capture_root.is_some() { |
| 3401 | "external_only_limits".to_string() |
| 3402 | } else { |
| 3403 | "linked_only_surface".to_string() |
| 3404 | }, |
| 3405 | if capture_root.is_some() { |
| 3406 | "none in this build".to_string() |
| 3407 | } else { |
| 3408 | "armfortas.* extras, legacy frontend/module suites, capture consistency".to_string() |
| 3409 | }, |
| 3410 | )); |
| 3411 | |
| 3412 | fields |
| 3413 | } |
| 3414 | |
| 3415 | fn render_doctor_report(config: &DoctorConfig) -> String { |
| 3416 | let mut lines = vec!["Doctor".to_string()]; |
| 3417 | for (field, value) in doctor_report_fields(config) { |
| 3418 | lines.push(format!(" {}: {}", field, value)); |
| 3419 | } |
| 3420 | lines.join("\n") |
| 3421 | } |
| 3422 | |
| 3423 | fn render_doctor_capabilities_json(capabilities: &CompilerCapabilities) -> String { |
| 3424 | let unavailable = capabilities |
| 3425 | .unavailable_artifacts |
| 3426 | .iter() |
| 3427 | .map(|(artifact, reason)| { |
| 3428 | format!( |
| 3429 | "{{\"artifact\":\"{}\",\"reason\":\"{}\"}}", |
| 3430 | json_escape(artifact.as_str()), |
| 3431 | json_escape(reason) |
| 3432 | ) |
| 3433 | }) |
| 3434 | .collect::<Vec<_>>() |
| 3435 | .join(", "); |
| 3436 | format!( |
| 3437 | "{{\"generic_artifacts\":{},\"adapter_extras\":{},\"unavailable_artifacts\":[{}]}}", |
| 3438 | json_string_iter( |
| 3439 | capabilities |
| 3440 | .generic_artifacts() |
| 3441 | .iter() |
| 3442 | .map(|artifact| artifact.as_str()) |
| 3443 | ), |
| 3444 | json_string_vec_map(&capabilities.adapter_extras()), |
| 3445 | unavailable |
| 3446 | ) |
| 3447 | } |
| 3448 | |
| 3449 | fn json_string_vec_map(map: &BTreeMap<String, Vec<String>>) -> String { |
| 3450 | let mut rendered = String::from("{"); |
| 3451 | for (index, (key, values)) in map.iter().enumerate() { |
| 3452 | if index > 0 { |
| 3453 | rendered.push_str(", "); |
| 3454 | } |
| 3455 | rendered.push('"'); |
| 3456 | rendered.push_str(&json_escape(key)); |
| 3457 | rendered.push_str("\": "); |
| 3458 | rendered.push_str(&json_string_iter(values.iter().map(|value| value.as_str()))); |
| 3459 | } |
| 3460 | rendered.push('}'); |
| 3461 | rendered |
| 3462 | } |
| 3463 | |
| 3464 | fn render_named_compiler_entry_json( |
| 3465 | named: NamedCompiler, |
| 3466 | tools: &ToolchainConfig, |
| 3467 | capture_root: Option<&PathBuf>, |
| 3468 | ) -> String { |
| 3469 | let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 3470 | let mut fields = vec![ |
| 3471 | format!( |
| 3472 | "\"accepted_names\": {}", |
| 3473 | json_string_iter(named.accepted_names().iter().copied()) |
| 3474 | ), |
| 3475 | format!( |
| 3476 | "\"candidate_binaries\": {}", |
| 3477 | json_string_iter(named.candidate_binaries().iter().copied()) |
| 3478 | ), |
| 3479 | format!( |
| 3480 | "\"capabilities\": {}", |
| 3481 | render_doctor_capabilities_json(&capabilities) |
| 3482 | ), |
| 3483 | ]; |
| 3484 | if named == NamedCompiler::Armfortas { |
| 3485 | let armfortas = tools.armfortas_adapters(); |
| 3486 | fields.insert( |
| 3487 | 0, |
| 3488 | format!( |
| 3489 | "\"surface\": \"{}\"", |
| 3490 | json_escape(&format!( |
| 3491 | "cli={} capture={}", |
| 3492 | armfortas.cli_mode_name(), |
| 3493 | armfortas.capture_mode_name() |
| 3494 | )) |
| 3495 | ), |
| 3496 | ); |
| 3497 | } else { |
| 3498 | fields.insert( |
| 3499 | 0, |
| 3500 | format!( |
| 3501 | "\"status\": \"{}\"", |
| 3502 | json_escape(&named_compiler_status_value(named, tools, capture_root)) |
| 3503 | ), |
| 3504 | ); |
| 3505 | } |
| 3506 | format!("{{{}}}", fields.join(", ")) |
| 3507 | } |
| 3508 | |
| 3509 | fn render_doctor_json(config: &DoctorConfig) -> String { |
| 3510 | let fields = doctor_report_fields(config); |
| 3511 | let workspace_root = workspace_root(); |
| 3512 | let suite_root = default_suite_root(); |
| 3513 | let report_root = default_report_root(); |
| 3514 | let armfortas = config.tools.armfortas_adapters(); |
| 3515 | let observable_backend = config |
| 3516 | .tools |
| 3517 | .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3518 | let capture_root = armfortas.capture_root(); |
| 3519 | let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 3520 | let explicit_capabilities = compiler_capabilities( |
| 3521 | &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3522 | &config.tools, |
| 3523 | ); |
| 3524 | let named_entries = NamedCompiler::ALL |
| 3525 | .iter() |
| 3526 | .enumerate() |
| 3527 | .map(|(index, named)| { |
| 3528 | format!( |
| 3529 | " \"{}\": {}{}", |
| 3530 | named.as_str(), |
| 3531 | render_named_compiler_entry_json(*named, &config.tools, capture_root.as_ref()), |
| 3532 | if index + 1 == NamedCompiler::ALL.len() { |
| 3533 | "" |
| 3534 | } else { |
| 3535 | "," |
| 3536 | } |
| 3537 | ) |
| 3538 | }) |
| 3539 | .collect::<Vec<_>>(); |
| 3540 | let mut lines = vec![ |
| 3541 | "{".to_string(), |
| 3542 | " \"command\": \"doctor\",".to_string(), |
| 3543 | " \"workspace\": {".to_string(), |
| 3544 | format!( |
| 3545 | " \"workspace_root\": \"{}\",", |
| 3546 | json_escape(&display_path(&workspace_root)) |
| 3547 | ), |
| 3548 | format!( |
| 3549 | " \"suite_root\": \"{}\",", |
| 3550 | json_escape(&display_path(&suite_root)) |
| 3551 | ), |
| 3552 | format!( |
| 3553 | " \"report_root\": \"{}\"", |
| 3554 | json_escape(&display_path(&report_root)) |
| 3555 | ), |
| 3556 | " },".to_string(), |
| 3557 | " \"armfortas\": {".to_string(), |
| 3558 | format!( |
| 3559 | " \"cli_adapter\": \"{}\",", |
| 3560 | json_escape(armfortas.cli_description()) |
| 3561 | ), |
| 3562 | format!( |
| 3563 | " \"capture_adapter\": \"{}\",", |
| 3564 | json_escape(armfortas.capture_description()) |
| 3565 | ), |
| 3566 | format!( |
| 3567 | " \"cli_mode\": \"{}\",", |
| 3568 | json_escape(armfortas.cli_mode_name()) |
| 3569 | ), |
| 3570 | format!( |
| 3571 | " \"cli_status\": \"{}\",", |
| 3572 | json_escape( |
| 3573 | &match armfortas.cli() { |
| 3574 | ArmfortasCliAdapter::Linked => capture_root |
| 3575 | .as_ref() |
| 3576 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3577 | .unwrap_or_else(|| { |
| 3578 | "linked adapter requested but unavailable in this build".to_string() |
| 3579 | }), |
| 3580 | ArmfortasCliAdapter::External(binary) => tool_probe_status(binary, false), |
| 3581 | } |
| 3582 | ) |
| 3583 | ), |
| 3584 | format!( |
| 3585 | " \"capture_mode\": \"{}\",", |
| 3586 | json_escape(armfortas.capture_mode_name()) |
| 3587 | ), |
| 3588 | format!( |
| 3589 | " \"capture_status\": \"{}\",", |
| 3590 | json_escape( |
| 3591 | &capture_root |
| 3592 | .as_ref() |
| 3593 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3594 | .unwrap_or_else(|| { |
| 3595 | "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh" |
| 3596 | .to_string() |
| 3597 | }) |
| 3598 | ) |
| 3599 | ), |
| 3600 | format!( |
| 3601 | " \"capture_root\": {},", |
| 3602 | match capture_root.as_ref() { |
| 3603 | Some(root) => format!("\"{}\"", json_escape(&display_path(root))), |
| 3604 | None => "null".to_string(), |
| 3605 | } |
| 3606 | ), |
| 3607 | format!( |
| 3608 | " \"capture_manifest\": \"{}\"", |
| 3609 | json_escape( |
| 3610 | &capture_manifest |
| 3611 | .as_ref() |
| 3612 | .map(|manifest| { |
| 3613 | if manifest.exists() { |
| 3614 | display_path(manifest) |
| 3615 | } else { |
| 3616 | "missing".to_string() |
| 3617 | } |
| 3618 | }) |
| 3619 | .unwrap_or_else(|| "unavailable".to_string()) |
| 3620 | ) |
| 3621 | ), |
| 3622 | " },".to_string(), |
| 3623 | " \"primary_backends\": {".to_string(), |
| 3624 | format!( |
| 3625 | " \"full\": \"{}\",", |
| 3626 | json_escape(armfortas.capture_description()) |
| 3627 | ), |
| 3628 | format!( |
| 3629 | " \"observable\": \"{}\",", |
| 3630 | json_escape(observable_backend.description()) |
| 3631 | ), |
| 3632 | format!( |
| 3633 | " \"selection\": \"{}\"", |
| 3634 | json_escape("observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend") |
| 3635 | ), |
| 3636 | " },".to_string(), |
| 3637 | " \"named_compilers\": {".to_string(), |
| 3638 | ]; |
| 3639 | lines.extend(named_entries); |
| 3640 | lines.extend([ |
| 3641 | " },".to_string(), |
| 3642 | format!( |
| 3643 | " \"explicit_compiler_path\": {{\"description\":\"{}\",\"capabilities\":{}}},", |
| 3644 | json_escape( |
| 3645 | "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3646 | ), |
| 3647 | render_doctor_capabilities_json(&explicit_capabilities) |
| 3648 | ), |
| 3649 | " \"tools\": {".to_string(), |
| 3650 | format!( |
| 3651 | " \"gfortran\": \"{}\",", |
| 3652 | json_escape(&tool_probe_status(&config.tools.gfortran, false)) |
| 3653 | ), |
| 3654 | format!( |
| 3655 | " \"flang-new\": \"{}\",", |
| 3656 | json_escape(&tool_probe_status(&config.tools.flang_new, false)) |
| 3657 | ), |
| 3658 | format!( |
| 3659 | " \"lfortran\": \"{}\",", |
| 3660 | json_escape(&tool_probe_status(&config.tools.lfortran, false)) |
| 3661 | ), |
| 3662 | format!( |
| 3663 | " \"ifort\": \"{}\",", |
| 3664 | json_escape(&tool_probe_status(&config.tools.ifort, false)) |
| 3665 | ), |
| 3666 | format!( |
| 3667 | " \"ifx\": \"{}\",", |
| 3668 | json_escape(&tool_probe_status(&config.tools.ifx, false)) |
| 3669 | ), |
| 3670 | format!( |
| 3671 | " \"nvfortran\": \"{}\",", |
| 3672 | json_escape(&tool_probe_status(&config.tools.nvfortran, false)) |
| 3673 | ), |
| 3674 | format!( |
| 3675 | " \"as\": \"{}\",", |
| 3676 | json_escape(&tool_probe_status(&config.tools.system_as, false)) |
| 3677 | ), |
| 3678 | format!( |
| 3679 | " \"otool\": \"{}\",", |
| 3680 | json_escape(&tool_probe_status(&config.tools.otool, false)) |
| 3681 | ), |
| 3682 | format!( |
| 3683 | " \"nm\": \"{}\"", |
| 3684 | json_escape(&tool_probe_status(&config.tools.nm, false)) |
| 3685 | ), |
| 3686 | " },".to_string(), |
| 3687 | " \"mode\": {".to_string(), |
| 3688 | format!( |
| 3689 | " \"note\": \"{}\",", |
| 3690 | json_escape( |
| 3691 | if capture_root.is_some() { |
| 3692 | "linked capture still depends on the surrounding armfortas checkout" |
| 3693 | } else { |
| 3694 | "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work" |
| 3695 | } |
| 3696 | ) |
| 3697 | ), |
| 3698 | format!( |
| 3699 | " \"surface_key\": \"{}\",", |
| 3700 | if capture_root.is_some() { |
| 3701 | "linked_mode_surface" |
| 3702 | } else { |
| 3703 | "external_only_surface" |
| 3704 | } |
| 3705 | ), |
| 3706 | format!( |
| 3707 | " \"surface_value\": \"{}\",", |
| 3708 | json_escape( |
| 3709 | if capture_root.is_some() { |
| 3710 | "rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 3711 | } else { |
| 3712 | "compare, introspect, generic suite-v2, observable-only run cells" |
| 3713 | } |
| 3714 | ) |
| 3715 | ), |
| 3716 | format!( |
| 3717 | " \"limits_key\": \"{}\",", |
| 3718 | if capture_root.is_some() { |
| 3719 | "external_only_limits" |
| 3720 | } else { |
| 3721 | "linked_only_surface" |
| 3722 | } |
| 3723 | ), |
| 3724 | format!( |
| 3725 | " \"limits_value\": \"{}\"", |
| 3726 | json_escape( |
| 3727 | if capture_root.is_some() { |
| 3728 | "none in this build" |
| 3729 | } else { |
| 3730 | "armfortas.* extras, legacy frontend/module suites, capture consistency" |
| 3731 | } |
| 3732 | ) |
| 3733 | ), |
| 3734 | " },".to_string(), |
| 3735 | " \"fields\": {".to_string(), |
| 3736 | ]); |
| 3737 | for (index, (field, value)) in fields.iter().enumerate() { |
| 3738 | lines.push(format!( |
| 3739 | " \"{}\": \"{}\"{}", |
| 3740 | json_escape(field), |
| 3741 | json_escape(value), |
| 3742 | if index + 1 == fields.len() { "" } else { "," } |
| 3743 | )); |
| 3744 | } |
| 3745 | lines.push(" }".to_string()); |
| 3746 | lines.push("}".to_string()); |
| 3747 | lines.join("\n") + "\n" |
| 3748 | } |
| 3749 | |
| 3750 | fn render_doctor_markdown(config: &DoctorConfig) -> String { |
| 3751 | let mut lines = vec![ |
| 3752 | "# bencch doctor report".to_string(), |
| 3753 | String::new(), |
| 3754 | "| field | value |".to_string(), |
| 3755 | "| --- | --- |".to_string(), |
| 3756 | ]; |
| 3757 | for (field, value) in doctor_report_fields(config) { |
| 3758 | lines.push(format!( |
| 3759 | "| `{}` | {} |", |
| 3760 | field, |
| 3761 | doctor_markdown_cell(&value) |
| 3762 | )); |
| 3763 | } |
| 3764 | lines.join("\n") + "\n" |
| 3765 | } |
| 3766 | |
| 3767 | fn write_doctor_reports(config: &DoctorConfig) -> Result<(), String> { |
| 3768 | if let Some(path) = &config.json_report { |
| 3769 | write_report(path, &render_doctor_json(config), "json report")?; |
| 3770 | println!("json report: {}", path.display()); |
| 3771 | } |
| 3772 | if let Some(path) = &config.markdown_report { |
| 3773 | write_report(path, &render_doctor_markdown(config), "markdown report")?; |
| 3774 | println!("markdown report: {}", path.display()); |
| 3775 | } |
| 3776 | Ok(()) |
| 3777 | } |
| 3778 | |
| 3779 | fn doctor_markdown_cell(value: &str) -> String { |
| 3780 | value.replace('|', "\\|").replace('\n', "<br>") |
| 3781 | } |
| 3782 | |
| 3783 | fn tool_probe_status(configured: &str, already_resolved_path: bool) -> String { |
| 3784 | let resolved = if already_resolved_path { |
| 3785 | let path = PathBuf::from(configured); |
| 3786 | if path.exists() { |
| 3787 | Some(path) |
| 3788 | } else { |
| 3789 | None |
| 3790 | } |
| 3791 | } else { |
| 3792 | resolve_tool_path(configured) |
| 3793 | }; |
| 3794 | |
| 3795 | match resolved { |
| 3796 | Some(path) => format!("configured={} resolved={}", configured, path.display()), |
| 3797 | None => format!("configured={} resolved=missing", configured), |
| 3798 | } |
| 3799 | } |
| 3800 | |
| 3801 | fn resolve_tool_path(configured: &str) -> Option<PathBuf> { |
| 3802 | let configured_path = Path::new(configured); |
| 3803 | if configured.contains('/') || configured.starts_with('.') { |
| 3804 | return configured_path |
| 3805 | .exists() |
| 3806 | .then(|| configured_path.to_path_buf()); |
| 3807 | } |
| 3808 | |
| 3809 | let path_var = std::env::var_os("PATH")?; |
| 3810 | for entry in std::env::split_paths(&path_var) { |
| 3811 | let candidate = entry.join(configured); |
| 3812 | if candidate.exists() { |
| 3813 | return Some(candidate); |
| 3814 | } |
| 3815 | } |
| 3816 | None |
| 3817 | } |
| 3818 | |
| 3819 | fn display_path(path: &Path) -> String { |
| 3820 | fs::canonicalize(path) |
| 3821 | .unwrap_or_else(|_| path.to_path_buf()) |
| 3822 | .display() |
| 3823 | .to_string() |
| 3824 | } |
| 3825 | |
| 3826 | fn discover_suites(root: PathBuf) -> Result<Vec<SuiteSpec>, String> { |
| 3827 | let mut files = Vec::new(); |
| 3828 | collect_suite_files(&root, &mut files)?; |
| 3829 | files.sort(); |
| 3830 | |
| 3831 | let mut suites = Vec::new(); |
| 3832 | for file in files { |
| 3833 | suites.push(parse_suite_file(&file)?); |
| 3834 | } |
| 3835 | suites.sort_by(|a, b| a.name.cmp(&b.name)); |
| 3836 | Ok(suites) |
| 3837 | } |
| 3838 | |
| 3839 | fn collect_suite_files(root: &Path, files: &mut Vec<PathBuf>) -> Result<(), String> { |
| 3840 | let entries = fs::read_dir(root) |
| 3841 | .map_err(|e| format!("cannot read suite root '{}': {}", root.display(), e))?; |
| 3842 | for entry in entries { |
| 3843 | let entry = |
| 3844 | entry.map_err(|e| format!("cannot read entry in '{}': {}", root.display(), e))?; |
| 3845 | let path = entry.path(); |
| 3846 | if path.is_dir() { |
| 3847 | collect_suite_files(&path, files)?; |
| 3848 | } else if path.extension().and_then(|ext| ext.to_str()) == Some(SUITE_EXTENSION) { |
| 3849 | files.push(path); |
| 3850 | } |
| 3851 | } |
| 3852 | Ok(()) |
| 3853 | } |
| 3854 | |
| 3855 | fn parse_suite_file(path: &Path) -> Result<SuiteSpec, String> { |
| 3856 | let text = fs::read_to_string(path) |
| 3857 | .map_err(|e| format!("cannot read suite '{}': {}", path.display(), e))?; |
| 3858 | |
| 3859 | let mut suite_name = None; |
| 3860 | let mut cases = Vec::new(); |
| 3861 | let mut current = None; |
| 3862 | |
| 3863 | for (index, raw_line) in text.lines().enumerate() { |
| 3864 | let line_no = index + 1; |
| 3865 | let line = raw_line.trim(); |
| 3866 | if line.is_empty() || line.starts_with('#') { |
| 3867 | continue; |
| 3868 | } |
| 3869 | |
| 3870 | if let Some(rest) = line.strip_prefix("suite ") { |
| 3871 | if suite_name.is_some() { |
| 3872 | return Err(format!( |
| 3873 | "{}:{}: duplicate suite declaration", |
| 3874 | path.display(), |
| 3875 | line_no |
| 3876 | )); |
| 3877 | } |
| 3878 | suite_name = Some(parse_quoted(rest, path, line_no)?); |
| 3879 | continue; |
| 3880 | } |
| 3881 | |
| 3882 | if let Some(rest) = line.strip_prefix("case ") { |
| 3883 | if current.is_some() { |
| 3884 | return Err(format!( |
| 3885 | "{}:{}: nested case without end", |
| 3886 | path.display(), |
| 3887 | line_no |
| 3888 | )); |
| 3889 | } |
| 3890 | current = Some(CaseBuilder::new(parse_quoted(rest, path, line_no)?)); |
| 3891 | continue; |
| 3892 | } |
| 3893 | |
| 3894 | if line == "end" { |
| 3895 | let builder = current.take().ok_or_else(|| { |
| 3896 | format!("{}:{}: stray end outside of case", path.display(), line_no) |
| 3897 | })?; |
| 3898 | cases.push(builder.build(path)?); |
| 3899 | continue; |
| 3900 | } |
| 3901 | |
| 3902 | let builder = current.as_mut().ok_or_else(|| { |
| 3903 | format!( |
| 3904 | "{}:{}: expected suite/case declaration first", |
| 3905 | path.display(), |
| 3906 | line_no |
| 3907 | ) |
| 3908 | })?; |
| 3909 | |
| 3910 | if let Some(rest) = line.strip_prefix("source ") { |
| 3911 | builder.source = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 3912 | } else if let Some(rest) = line.strip_prefix("entry ") { |
| 3913 | builder.graph_entry = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 3914 | } else if let Some(rest) = line.strip_prefix("file ") { |
| 3915 | builder |
| 3916 | .graph_files |
| 3917 | .push(resolve_suite_relative_path(rest, path, line_no)?); |
| 3918 | } else if let Some(rest) = line.strip_prefix("compiler ") { |
| 3919 | let (compiler, artifacts) = parse_compiler_artifact_declaration(rest, path, line_no)?; |
| 3920 | builder.generic_compiler = Some(compiler); |
| 3921 | builder.generic_artifacts = artifacts; |
| 3922 | } else if let Some(rest) = line.strip_prefix("compare ") { |
| 3923 | let (left, right, artifacts) = parse_compare_declaration(rest, path, line_no)?; |
| 3924 | builder.generic_compare = Some((left, right)); |
| 3925 | builder.generic_compare_artifacts = artifacts; |
| 3926 | } else if let Some(rest) = line.strip_prefix("armfortas =>") { |
| 3927 | builder.requested = parse_stage_list(rest, path, line_no)?; |
| 3928 | } else if let Some(rest) = line.strip_prefix("repeat =>") { |
| 3929 | builder.repeat_count = parse_repeat_count(rest, path, line_no)?; |
| 3930 | } else if let Some(rest) = line.strip_prefix("opts =>") { |
| 3931 | builder.opt_levels = parse_opt_levels(rest, path, line_no)?; |
| 3932 | } else if let Some(rest) = line.strip_prefix("differential =>") { |
| 3933 | builder.reference_compilers = parse_reference_compilers(rest, path, line_no)?; |
| 3934 | } else if let Some(rest) = line.strip_prefix("consistency =>") { |
| 3935 | builder.consistency_checks = parse_consistency_checks(rest, path, line_no)?; |
| 3936 | } else if let Some(rest) = line.strip_prefix("expect-fail ") { |
| 3937 | builder |
| 3938 | .expectations |
| 3939 | .push(parse_failure_expectation(rest, path, line_no)?); |
| 3940 | } else if let Some(rest) = line.strip_prefix("expect ") { |
| 3941 | builder |
| 3942 | .expectations |
| 3943 | .push(parse_expectation(rest, path, line_no)?); |
| 3944 | } else if let Some(rest) = line.strip_prefix("xfail capability ") { |
| 3945 | if builder.capability_policy.is_some() { |
| 3946 | return Err(format!( |
| 3947 | "{}:{}: duplicate capability policy", |
| 3948 | path.display(), |
| 3949 | line_no |
| 3950 | )); |
| 3951 | } |
| 3952 | builder.capability_policy = Some(parse_capability_policy( |
| 3953 | StatusKind::Xfail, |
| 3954 | rest, |
| 3955 | path, |
| 3956 | line_no, |
| 3957 | )?); |
| 3958 | } else if let Some(rest) = line.strip_prefix("future capability ") { |
| 3959 | if builder.capability_policy.is_some() { |
| 3960 | return Err(format!( |
| 3961 | "{}:{}: duplicate capability policy", |
| 3962 | path.display(), |
| 3963 | line_no |
| 3964 | )); |
| 3965 | } |
| 3966 | builder.capability_policy = Some(parse_capability_policy( |
| 3967 | StatusKind::Future, |
| 3968 | rest, |
| 3969 | path, |
| 3970 | line_no, |
| 3971 | )?); |
| 3972 | } else if let Some(rest) = line.strip_prefix("xfail ") { |
| 3973 | builder |
| 3974 | .status_rules |
| 3975 | .push(parse_status_rule(StatusKind::Xfail, rest, path, line_no)?); |
| 3976 | } else if let Some(rest) = line.strip_prefix("future ") { |
| 3977 | builder |
| 3978 | .status_rules |
| 3979 | .push(parse_status_rule(StatusKind::Future, rest, path, line_no)?); |
| 3980 | } else { |
| 3981 | return Err(format!( |
| 3982 | "{}:{}: unrecognized line '{}'", |
| 3983 | path.display(), |
| 3984 | line_no, |
| 3985 | line |
| 3986 | )); |
| 3987 | } |
| 3988 | } |
| 3989 | |
| 3990 | if current.is_some() { |
| 3991 | return Err(format!("{}: unterminated case block", path.display())); |
| 3992 | } |
| 3993 | |
| 3994 | let suite_name = |
| 3995 | suite_name.ok_or_else(|| format!("{}: missing suite declaration", path.display()))?; |
| 3996 | if cases.is_empty() { |
| 3997 | return Err(format!("{}: suite has no cases", path.display())); |
| 3998 | } |
| 3999 | |
| 4000 | Ok(SuiteSpec { |
| 4001 | name: suite_name, |
| 4002 | path: path.to_path_buf(), |
| 4003 | cases, |
| 4004 | }) |
| 4005 | } |
| 4006 | |
| 4007 | struct CaseBuilder { |
| 4008 | name: String, |
| 4009 | source: Option<PathBuf>, |
| 4010 | graph_entry: Option<PathBuf>, |
| 4011 | graph_files: Vec<PathBuf>, |
| 4012 | requested: BTreeSet<Stage>, |
| 4013 | generic_compiler: Option<CompilerSpec>, |
| 4014 | generic_artifacts: BTreeSet<ArtifactKey>, |
| 4015 | generic_compare: Option<(CompilerSpec, CompilerSpec)>, |
| 4016 | generic_compare_artifacts: BTreeSet<ArtifactKey>, |
| 4017 | opt_levels: Vec<OptLevel>, |
| 4018 | repeat_count: usize, |
| 4019 | reference_compilers: Vec<ReferenceCompiler>, |
| 4020 | consistency_checks: Vec<ConsistencyCheck>, |
| 4021 | expectations: Vec<Expectation>, |
| 4022 | status_rules: Vec<PendingStatusRule>, |
| 4023 | capability_policy: Option<CapabilityPolicy>, |
| 4024 | } |
| 4025 | |
| 4026 | impl CaseBuilder { |
| 4027 | fn new(name: String) -> Self { |
| 4028 | Self { |
| 4029 | name, |
| 4030 | source: None, |
| 4031 | graph_entry: None, |
| 4032 | graph_files: Vec::new(), |
| 4033 | requested: BTreeSet::new(), |
| 4034 | generic_compiler: None, |
| 4035 | generic_artifacts: BTreeSet::new(), |
| 4036 | generic_compare: None, |
| 4037 | generic_compare_artifacts: BTreeSet::new(), |
| 4038 | opt_levels: Vec::new(), |
| 4039 | repeat_count: 2, |
| 4040 | reference_compilers: Vec::new(), |
| 4041 | consistency_checks: Vec::new(), |
| 4042 | expectations: Vec::new(), |
| 4043 | status_rules: Vec::new(), |
| 4044 | capability_policy: None, |
| 4045 | } |
| 4046 | } |
| 4047 | |
| 4048 | fn build(self, suite_path: &Path) -> Result<CaseSpec, String> { |
| 4049 | let generic_mode_count = usize::from(self.generic_compiler.is_some()) |
| 4050 | + usize::from(self.generic_compare.is_some()); |
| 4051 | if generic_mode_count > 1 { |
| 4052 | return Err(format!( |
| 4053 | "{}: case '{}' mixes multiple suite-v2 execution forms", |
| 4054 | suite_path.display(), |
| 4055 | self.name |
| 4056 | )); |
| 4057 | } |
| 4058 | |
| 4059 | if generic_mode_count > 0 && !self.requested.is_empty() { |
| 4060 | return Err(format!( |
| 4061 | "{}: case '{}' mixes generic suite-v2 syntax with legacy 'armfortas => ...' stages", |
| 4062 | suite_path.display(), |
| 4063 | self.name |
| 4064 | )); |
| 4065 | } |
| 4066 | |
| 4067 | if self.source.is_some() && (self.graph_entry.is_some() || !self.graph_files.is_empty()) { |
| 4068 | return Err(format!( |
| 4069 | "{}: case '{}' mixes source with graph entry/file declarations", |
| 4070 | suite_path.display(), |
| 4071 | self.name |
| 4072 | )); |
| 4073 | } |
| 4074 | |
| 4075 | if self.graph_entry.is_some() && self.graph_files.is_empty() { |
| 4076 | return Err(format!( |
| 4077 | "{}: case '{}' declares an entry without any file members", |
| 4078 | suite_path.display(), |
| 4079 | self.name |
| 4080 | )); |
| 4081 | } |
| 4082 | |
| 4083 | if self.graph_entry.is_none() && !self.graph_files.is_empty() { |
| 4084 | return Err(format!( |
| 4085 | "{}: case '{}' declares file members without an entry", |
| 4086 | suite_path.display(), |
| 4087 | self.name |
| 4088 | )); |
| 4089 | } |
| 4090 | |
| 4091 | let (source, graph_files) = if let Some(source) = self.source { |
| 4092 | (source, Vec::new()) |
| 4093 | } else if let Some(entry) = self.graph_entry { |
| 4094 | if !self.graph_files.iter().any(|file| file == &entry) { |
| 4095 | return Err(format!( |
| 4096 | "{}: case '{}' entry '{}' is not listed in file declarations", |
| 4097 | suite_path.display(), |
| 4098 | self.name, |
| 4099 | entry.display() |
| 4100 | )); |
| 4101 | } |
| 4102 | (entry, self.graph_files) |
| 4103 | } else { |
| 4104 | return Err(format!( |
| 4105 | "{}: case '{}' is missing a source path or graph entry", |
| 4106 | suite_path.display(), |
| 4107 | self.name |
| 4108 | )); |
| 4109 | }; |
| 4110 | |
| 4111 | if self.generic_compare.is_some() |
| 4112 | && (!self.reference_compilers.is_empty() || !self.consistency_checks.is_empty()) |
| 4113 | { |
| 4114 | return Err(format!( |
| 4115 | "{}: case '{}' compare suite-v2 cases do not support differential/consistency rules", |
| 4116 | suite_path.display(), |
| 4117 | self.name |
| 4118 | )); |
| 4119 | } |
| 4120 | |
| 4121 | if self.generic_compiler.is_some() { |
| 4122 | let unsupported = self |
| 4123 | .consistency_checks |
| 4124 | .iter() |
| 4125 | .copied() |
| 4126 | .filter(|check| !check.supports_generic_introspect()) |
| 4127 | .collect::<Vec<_>>(); |
| 4128 | if !unsupported.is_empty() { |
| 4129 | return Err(format!( |
| 4130 | "{}: case '{}' generic compiler cases only support cli_asm_reproducible, cli_obj_reproducible, and cli_run_reproducible today (unsupported: {})", |
| 4131 | suite_path.display(), |
| 4132 | self.name, |
| 4133 | unsupported |
| 4134 | .iter() |
| 4135 | .map(ConsistencyCheck::as_str) |
| 4136 | .collect::<Vec<_>>() |
| 4137 | .join(", ") |
| 4138 | )); |
| 4139 | } |
| 4140 | } |
| 4141 | |
| 4142 | let needs_source_comment_resolution = self |
| 4143 | .expectations |
| 4144 | .iter() |
| 4145 | .any(|expectation| matches!(expectation, Expectation::FailSourceComments)) |
| 4146 | || self |
| 4147 | .status_rules |
| 4148 | .iter() |
| 4149 | .any(|rule| matches!(rule, PendingStatusRule::XfailSourceComments)); |
| 4150 | let source_text = if needs_source_comment_resolution { |
| 4151 | Some(fs::read_to_string(&source).map_err(|e| { |
| 4152 | format!( |
| 4153 | "{}: case '{}': cannot read source '{}' for comment-based directives: {}", |
| 4154 | suite_path.display(), |
| 4155 | self.name, |
| 4156 | source.display(), |
| 4157 | e |
| 4158 | ) |
| 4159 | })?) |
| 4160 | } else { |
| 4161 | None |
| 4162 | }; |
| 4163 | |
| 4164 | let generic_introspect = if let Some(compiler) = self.generic_compiler { |
| 4165 | if self.generic_artifacts.is_empty() { |
| 4166 | return Err(format!( |
| 4167 | "{}: case '{}' generic compiler artifact list is empty", |
| 4168 | suite_path.display(), |
| 4169 | self.name |
| 4170 | )); |
| 4171 | } |
| 4172 | Some(GenericIntrospectCase { |
| 4173 | compiler, |
| 4174 | artifacts: self.generic_artifacts, |
| 4175 | }) |
| 4176 | } else { |
| 4177 | None |
| 4178 | }; |
| 4179 | |
| 4180 | let generic_compare = if let Some((left, right)) = self.generic_compare { |
| 4181 | let mut artifacts = self.generic_compare_artifacts; |
| 4182 | if artifacts.is_empty() { |
| 4183 | return Err(format!( |
| 4184 | "{}: case '{}' compare artifact list is empty", |
| 4185 | suite_path.display(), |
| 4186 | self.name |
| 4187 | )); |
| 4188 | } |
| 4189 | artifacts.extend(default_compare_artifacts(&artifacts)); |
| 4190 | Some(GenericCompareCase { |
| 4191 | left, |
| 4192 | right, |
| 4193 | artifacts, |
| 4194 | }) |
| 4195 | } else { |
| 4196 | None |
| 4197 | }; |
| 4198 | |
| 4199 | let mut requested = self.requested; |
| 4200 | if requested.is_empty() && generic_introspect.is_none() && generic_compare.is_none() { |
| 4201 | requested.insert(Stage::Run); |
| 4202 | } |
| 4203 | |
| 4204 | let opt_levels = if self.opt_levels.is_empty() { |
| 4205 | vec![OptLevel::O0] |
| 4206 | } else { |
| 4207 | self.opt_levels |
| 4208 | }; |
| 4209 | let expectations = resolve_source_comment_expectations( |
| 4210 | self.expectations, |
| 4211 | source_text.as_deref(), |
| 4212 | suite_path, |
| 4213 | &self.name, |
| 4214 | &source, |
| 4215 | )?; |
| 4216 | let status_rules = resolve_source_comment_status_rules( |
| 4217 | self.status_rules, |
| 4218 | source_text.as_deref(), |
| 4219 | suite_path, |
| 4220 | &self.name, |
| 4221 | &source, |
| 4222 | )?; |
| 4223 | |
| 4224 | Ok(CaseSpec { |
| 4225 | name: self.name, |
| 4226 | source, |
| 4227 | graph_files, |
| 4228 | requested, |
| 4229 | generic_introspect, |
| 4230 | generic_compare, |
| 4231 | opt_levels, |
| 4232 | repeat_count: self.repeat_count, |
| 4233 | reference_compilers: self.reference_compilers, |
| 4234 | consistency_checks: self.consistency_checks, |
| 4235 | expectations, |
| 4236 | status_rules, |
| 4237 | capability_policy: self.capability_policy, |
| 4238 | }) |
| 4239 | } |
| 4240 | } |
| 4241 | |
| 4242 | fn resolve_suite_relative_path(rest: &str, path: &Path, line_no: usize) -> Result<PathBuf, String> { |
| 4243 | let relative = parse_quoted(rest, path, line_no)?; |
| 4244 | Ok(path |
| 4245 | .parent() |
| 4246 | .unwrap_or_else(|| Path::new(".")) |
| 4247 | .join(relative)) |
| 4248 | } |
| 4249 | |
| 4250 | fn parse_stage_list(rest: &str, path: &Path, line_no: usize) -> Result<BTreeSet<Stage>, String> { |
| 4251 | let mut stages = BTreeSet::new(); |
| 4252 | for raw in rest.split(',') { |
| 4253 | let name = raw.trim(); |
| 4254 | if name.is_empty() { |
| 4255 | continue; |
| 4256 | } |
| 4257 | let stage = Stage::parse(name) |
| 4258 | .ok_or_else(|| format!("{}:{}: unknown stage '{}'", path.display(), line_no, name))?; |
| 4259 | stages.insert(stage); |
| 4260 | } |
| 4261 | if stages.is_empty() { |
| 4262 | return Err(format!( |
| 4263 | "{}:{}: armfortas stage list is empty", |
| 4264 | path.display(), |
| 4265 | line_no |
| 4266 | )); |
| 4267 | } |
| 4268 | Ok(stages) |
| 4269 | } |
| 4270 | |
| 4271 | fn parse_compiler_artifact_declaration( |
| 4272 | rest: &str, |
| 4273 | path: &Path, |
| 4274 | line_no: usize, |
| 4275 | ) -> Result<(CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4276 | let (compiler_raw, artifact_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4277 | format!( |
| 4278 | "{}:{}: compiler declaration must use 'compiler <spec> => <artifacts>'", |
| 4279 | path.display(), |
| 4280 | line_no |
| 4281 | ) |
| 4282 | })?; |
| 4283 | let compiler = parse_compiler_spec_token(compiler_raw.trim(), path, line_no)?; |
| 4284 | let artifacts = ArtifactKey::parse_list(artifact_raw.trim()) |
| 4285 | .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4286 | if artifacts.is_empty() { |
| 4287 | return Err(format!( |
| 4288 | "{}:{}: generic compiler artifact list is empty", |
| 4289 | path.display(), |
| 4290 | line_no |
| 4291 | )); |
| 4292 | } |
| 4293 | Ok((compiler, artifacts)) |
| 4294 | } |
| 4295 | |
| 4296 | fn parse_compare_declaration( |
| 4297 | rest: &str, |
| 4298 | path: &Path, |
| 4299 | line_no: usize, |
| 4300 | ) -> Result<(CompilerSpec, CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4301 | let (compilers_raw, artifacts_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4302 | format!( |
| 4303 | "{}:{}: compare declaration must use 'compare <left> <right> => <artifacts>'", |
| 4304 | path.display(), |
| 4305 | line_no |
| 4306 | ) |
| 4307 | })?; |
| 4308 | let tokens = split_compiler_tokens(compilers_raw.trim(), path, line_no)?; |
| 4309 | if tokens.len() != 2 { |
| 4310 | return Err(format!( |
| 4311 | "{}:{}: compare declaration requires exactly two compiler specs", |
| 4312 | path.display(), |
| 4313 | line_no |
| 4314 | )); |
| 4315 | } |
| 4316 | let left = parse_compiler_spec_token(&tokens[0], path, line_no)?; |
| 4317 | let right = parse_compiler_spec_token(&tokens[1], path, line_no)?; |
| 4318 | let artifacts = ArtifactKey::parse_list(artifacts_raw.trim()) |
| 4319 | .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4320 | Ok((left, right, artifacts)) |
| 4321 | } |
| 4322 | |
| 4323 | fn split_compiler_tokens(raw: &str, path: &Path, line_no: usize) -> Result<Vec<String>, String> { |
| 4324 | let mut tokens = Vec::new(); |
| 4325 | let mut current = String::new(); |
| 4326 | let mut quoted = false; |
| 4327 | |
| 4328 | for ch in raw.chars() { |
| 4329 | match ch { |
| 4330 | '"' => { |
| 4331 | quoted = !quoted; |
| 4332 | current.push(ch); |
| 4333 | } |
| 4334 | c if c.is_whitespace() && !quoted => { |
| 4335 | if !current.trim().is_empty() { |
| 4336 | tokens.push(current.trim().to_string()); |
| 4337 | current.clear(); |
| 4338 | } |
| 4339 | } |
| 4340 | other => current.push(other), |
| 4341 | } |
| 4342 | } |
| 4343 | |
| 4344 | if quoted { |
| 4345 | return Err(format!( |
| 4346 | "{}:{}: unterminated quoted compiler spec '{}'", |
| 4347 | path.display(), |
| 4348 | line_no, |
| 4349 | raw |
| 4350 | )); |
| 4351 | } |
| 4352 | |
| 4353 | if !current.trim().is_empty() { |
| 4354 | tokens.push(current.trim().to_string()); |
| 4355 | } |
| 4356 | |
| 4357 | Ok(tokens) |
| 4358 | } |
| 4359 | |
| 4360 | fn parse_compiler_spec_token( |
| 4361 | raw: &str, |
| 4362 | path: &Path, |
| 4363 | line_no: usize, |
| 4364 | ) -> Result<CompilerSpec, String> { |
| 4365 | let token = if raw.starts_with('"') { |
| 4366 | parse_quoted(raw, path, line_no)? |
| 4367 | } else { |
| 4368 | raw.trim().to_string() |
| 4369 | }; |
| 4370 | if token.is_empty() { |
| 4371 | return Err(format!( |
| 4372 | "{}:{}: compiler declaration is missing a compiler spec", |
| 4373 | path.display(), |
| 4374 | line_no |
| 4375 | )); |
| 4376 | } |
| 4377 | if let Some(named) = NamedCompiler::parse(&token) { |
| 4378 | return Ok(CompilerSpec::Named(named)); |
| 4379 | } |
| 4380 | let parsed = PathBuf::from(&token); |
| 4381 | let resolved = if parsed.is_absolute() { |
| 4382 | parsed |
| 4383 | } else { |
| 4384 | path.parent().unwrap_or_else(|| Path::new(".")).join(parsed) |
| 4385 | }; |
| 4386 | Ok(CompilerSpec::Binary(resolved)) |
| 4387 | } |
| 4388 | |
| 4389 | fn parse_opt_levels(rest: &str, path: &Path, line_no: usize) -> Result<Vec<OptLevel>, String> { |
| 4390 | let mut levels = BTreeSet::new(); |
| 4391 | for raw in rest.split(',') { |
| 4392 | let name = raw.trim(); |
| 4393 | if name.is_empty() { |
| 4394 | continue; |
| 4395 | } |
| 4396 | if name.eq_ignore_ascii_case("all") { |
| 4397 | levels.extend(all_opt_levels()); |
| 4398 | continue; |
| 4399 | } |
| 4400 | let level = parse_opt_level_token(name).ok_or_else(|| { |
| 4401 | format!( |
| 4402 | "{}:{}: unknown opt level '{}'", |
| 4403 | path.display(), |
| 4404 | line_no, |
| 4405 | name |
| 4406 | ) |
| 4407 | })?; |
| 4408 | levels.insert(level); |
| 4409 | } |
| 4410 | if levels.is_empty() { |
| 4411 | return Err(format!( |
| 4412 | "{}:{}: opt level list is empty", |
| 4413 | path.display(), |
| 4414 | line_no |
| 4415 | )); |
| 4416 | } |
| 4417 | Ok(levels.into_iter().collect()) |
| 4418 | } |
| 4419 | |
| 4420 | fn parse_reference_compilers( |
| 4421 | rest: &str, |
| 4422 | path: &Path, |
| 4423 | line_no: usize, |
| 4424 | ) -> Result<Vec<ReferenceCompiler>, String> { |
| 4425 | let mut compilers = BTreeSet::new(); |
| 4426 | for raw in rest.split(',') { |
| 4427 | let name = raw.trim(); |
| 4428 | if name.is_empty() { |
| 4429 | continue; |
| 4430 | } |
| 4431 | let compiler = ReferenceCompiler::parse(name).ok_or_else(|| { |
| 4432 | format!( |
| 4433 | "{}:{}: unknown reference compiler '{}'", |
| 4434 | path.display(), |
| 4435 | line_no, |
| 4436 | name |
| 4437 | ) |
| 4438 | })?; |
| 4439 | compilers.insert(compiler); |
| 4440 | } |
| 4441 | if compilers.is_empty() { |
| 4442 | return Err(format!( |
| 4443 | "{}:{}: differential compiler list is empty", |
| 4444 | path.display(), |
| 4445 | line_no |
| 4446 | )); |
| 4447 | } |
| 4448 | Ok(compilers.into_iter().collect()) |
| 4449 | } |
| 4450 | |
| 4451 | fn parse_repeat_count(rest: &str, path: &Path, line_no: usize) -> Result<usize, String> { |
| 4452 | let count = rest.trim().parse::<usize>().map_err(|_| { |
| 4453 | format!( |
| 4454 | "{}:{}: repeat count must be an integer >= 2", |
| 4455 | path.display(), |
| 4456 | line_no |
| 4457 | ) |
| 4458 | })?; |
| 4459 | if count < 2 { |
| 4460 | return Err(format!( |
| 4461 | "{}:{}: repeat count must be >= 2", |
| 4462 | path.display(), |
| 4463 | line_no |
| 4464 | )); |
| 4465 | } |
| 4466 | Ok(count) |
| 4467 | } |
| 4468 | |
| 4469 | fn parse_consistency_checks( |
| 4470 | rest: &str, |
| 4471 | path: &Path, |
| 4472 | line_no: usize, |
| 4473 | ) -> Result<Vec<ConsistencyCheck>, String> { |
| 4474 | let mut checks = Vec::new(); |
| 4475 | for raw in rest.split(',') { |
| 4476 | let name = raw.trim(); |
| 4477 | if name.is_empty() { |
| 4478 | continue; |
| 4479 | } |
| 4480 | let check = ConsistencyCheck::parse(name).ok_or_else(|| { |
| 4481 | format!( |
| 4482 | "{}:{}: unknown consistency check '{}'", |
| 4483 | path.display(), |
| 4484 | line_no, |
| 4485 | name |
| 4486 | ) |
| 4487 | })?; |
| 4488 | if !checks.contains(&check) { |
| 4489 | checks.push(check); |
| 4490 | } |
| 4491 | } |
| 4492 | if checks.is_empty() { |
| 4493 | return Err(format!( |
| 4494 | "{}:{}: consistency check list is empty", |
| 4495 | path.display(), |
| 4496 | line_no |
| 4497 | )); |
| 4498 | } |
| 4499 | Ok(checks) |
| 4500 | } |
| 4501 | |
| 4502 | fn parse_expectation(rest: &str, path: &Path, line_no: usize) -> Result<Expectation, String> { |
| 4503 | if let Some(prefix) = rest.strip_suffix(" check-comments") { |
| 4504 | return Ok(Expectation::CheckComments(parse_target( |
| 4505 | prefix.trim(), |
| 4506 | path, |
| 4507 | line_no, |
| 4508 | )?)); |
| 4509 | } |
| 4510 | |
| 4511 | if let Some((target, value)) = rest.split_once(" not-contains ") { |
| 4512 | return Ok(Expectation::NotContains { |
| 4513 | target: parse_target(target.trim(), path, line_no)?, |
| 4514 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4515 | }); |
| 4516 | } |
| 4517 | |
| 4518 | if let Some((target, value)) = rest.split_once(" contains ") { |
| 4519 | return Ok(Expectation::Contains { |
| 4520 | target: parse_target(target.trim(), path, line_no)?, |
| 4521 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4522 | }); |
| 4523 | } |
| 4524 | |
| 4525 | if let Some((target, value)) = rest.split_once(" equals ") { |
| 4526 | let target = parse_target(target.trim(), path, line_no)?; |
| 4527 | if matches!( |
| 4528 | target, |
| 4529 | Target::RunExitCode |
| 4530 | | Target::Artifact(ArtifactKey::ExitCode) |
| 4531 | | Target::CompareDifferenceCount |
| 4532 | ) { |
| 4533 | let value = parse_integer(value.trim(), path, line_no)?; |
| 4534 | return Ok(Expectation::IntEquals { target, value }); |
| 4535 | } |
| 4536 | return Ok(Expectation::Equals { |
| 4537 | target, |
| 4538 | value: parse_quoted(value.trim(), path, line_no)?, |
| 4539 | }); |
| 4540 | } |
| 4541 | |
| 4542 | Err(format!( |
| 4543 | "{}:{}: unsupported expectation '{}'", |
| 4544 | path.display(), |
| 4545 | line_no, |
| 4546 | rest |
| 4547 | )) |
| 4548 | } |
| 4549 | |
| 4550 | fn parse_failure_expectation( |
| 4551 | rest: &str, |
| 4552 | path: &Path, |
| 4553 | line_no: usize, |
| 4554 | ) -> Result<Expectation, String> { |
| 4555 | if rest.trim().eq_ignore_ascii_case("comments") { |
| 4556 | return Ok(Expectation::FailSourceComments); |
| 4557 | } |
| 4558 | |
| 4559 | if let Some((target, value)) = rest.split_once(" contains ") { |
| 4560 | return Ok(Expectation::FailContains { |
| 4561 | stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4562 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4563 | }); |
| 4564 | } |
| 4565 | |
| 4566 | if let Some((target, value)) = rest.split_once(" equals ") { |
| 4567 | return Ok(Expectation::FailEquals { |
| 4568 | stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4569 | value: parse_quoted(value.trim(), path, line_no)?, |
| 4570 | }); |
| 4571 | } |
| 4572 | |
| 4573 | Err(format!( |
| 4574 | "{}:{}: unsupported failure expectation '{}'", |
| 4575 | path.display(), |
| 4576 | line_no, |
| 4577 | rest |
| 4578 | )) |
| 4579 | } |
| 4580 | |
| 4581 | fn parse_status_rule( |
| 4582 | kind: StatusKind, |
| 4583 | rest: &str, |
| 4584 | path: &Path, |
| 4585 | line_no: usize, |
| 4586 | ) -> Result<PendingStatusRule, String> { |
| 4587 | let rest = rest.trim(); |
| 4588 | if kind == StatusKind::Xfail && rest.eq_ignore_ascii_case("comments") { |
| 4589 | return Ok(PendingStatusRule::XfailSourceComments); |
| 4590 | } |
| 4591 | if rest.starts_with('"') { |
| 4592 | return Ok(PendingStatusRule::Explicit(StatusRule { |
| 4593 | kind, |
| 4594 | selector: OptSelector::All, |
| 4595 | reason: parse_quoted(rest, path, line_no)?, |
| 4596 | })); |
| 4597 | } |
| 4598 | |
| 4599 | let conditional = rest.strip_prefix("when ").ok_or_else(|| { |
| 4600 | format!( |
| 4601 | "{}:{}: expected quoted reason or 'when <opts> because \"...\"'", |
| 4602 | path.display(), |
| 4603 | line_no |
| 4604 | ) |
| 4605 | })?; |
| 4606 | let (selector, reason) = conditional.split_once(" because ").ok_or_else(|| { |
| 4607 | format!( |
| 4608 | "{}:{}: conditional status must use 'when <opts> because \"...\"'", |
| 4609 | path.display(), |
| 4610 | line_no |
| 4611 | ) |
| 4612 | })?; |
| 4613 | |
| 4614 | Ok(PendingStatusRule::Explicit(StatusRule { |
| 4615 | kind, |
| 4616 | selector: parse_opt_selector(selector.trim(), path, line_no)?, |
| 4617 | reason: parse_quoted(reason.trim(), path, line_no)?, |
| 4618 | })) |
| 4619 | } |
| 4620 | |
| 4621 | fn parse_capability_policy( |
| 4622 | kind: StatusKind, |
| 4623 | rest: &str, |
| 4624 | path: &Path, |
| 4625 | line_no: usize, |
| 4626 | ) -> Result<CapabilityPolicy, String> { |
| 4627 | Ok(CapabilityPolicy { |
| 4628 | kind, |
| 4629 | reason: parse_quoted(rest.trim(), path, line_no)?, |
| 4630 | }) |
| 4631 | } |
| 4632 | |
| 4633 | fn resolve_source_comment_expectations( |
| 4634 | expectations: Vec<Expectation>, |
| 4635 | source_text: Option<&str>, |
| 4636 | suite_path: &Path, |
| 4637 | case_name: &str, |
| 4638 | source_path: &Path, |
| 4639 | ) -> Result<Vec<Expectation>, String> { |
| 4640 | let mut resolved = Vec::with_capacity(expectations.len()); |
| 4641 | for expectation in expectations { |
| 4642 | match expectation { |
| 4643 | Expectation::FailSourceComments => { |
| 4644 | let source_text = source_text.ok_or_else(|| { |
| 4645 | format!( |
| 4646 | "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4647 | suite_path.display(), |
| 4648 | case_name, |
| 4649 | source_path.display() |
| 4650 | ) |
| 4651 | })?; |
| 4652 | let patterns = extract_error_expected_patterns(source_text); |
| 4653 | if patterns.is_empty() { |
| 4654 | return Err(format!( |
| 4655 | "{}: case '{}' requests expect-fail comments but '{}' has no ! ERROR_EXPECTED: lines", |
| 4656 | suite_path.display(), |
| 4657 | case_name, |
| 4658 | source_path.display() |
| 4659 | )); |
| 4660 | } |
| 4661 | resolved.push(Expectation::FailCommentPatterns(patterns)); |
| 4662 | } |
| 4663 | other => resolved.push(other), |
| 4664 | } |
| 4665 | } |
| 4666 | Ok(resolved) |
| 4667 | } |
| 4668 | |
| 4669 | fn resolve_source_comment_status_rules( |
| 4670 | status_rules: Vec<PendingStatusRule>, |
| 4671 | source_text: Option<&str>, |
| 4672 | suite_path: &Path, |
| 4673 | case_name: &str, |
| 4674 | source_path: &Path, |
| 4675 | ) -> Result<Vec<StatusRule>, String> { |
| 4676 | let mut resolved = Vec::with_capacity(status_rules.len()); |
| 4677 | for rule in status_rules { |
| 4678 | match rule { |
| 4679 | PendingStatusRule::Explicit(rule) => resolved.push(rule), |
| 4680 | PendingStatusRule::XfailSourceComments => { |
| 4681 | let source_text = source_text.ok_or_else(|| { |
| 4682 | format!( |
| 4683 | "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4684 | suite_path.display(), |
| 4685 | case_name, |
| 4686 | source_path.display() |
| 4687 | ) |
| 4688 | })?; |
| 4689 | let reason = extract_xfail_reason(source_text).ok_or_else(|| { |
| 4690 | format!( |
| 4691 | "{}: case '{}' requests xfail comments but '{}' has no ! XFAIL: lines", |
| 4692 | suite_path.display(), |
| 4693 | case_name, |
| 4694 | source_path.display() |
| 4695 | ) |
| 4696 | })?; |
| 4697 | resolved.push(StatusRule { |
| 4698 | kind: StatusKind::Xfail, |
| 4699 | selector: OptSelector::All, |
| 4700 | reason, |
| 4701 | }); |
| 4702 | } |
| 4703 | } |
| 4704 | } |
| 4705 | Ok(resolved) |
| 4706 | } |
| 4707 | |
| 4708 | fn parse_opt_selector(raw: &str, path: &Path, line_no: usize) -> Result<OptSelector, String> { |
| 4709 | let raw = raw.trim(); |
| 4710 | if raw.eq_ignore_ascii_case("all") { |
| 4711 | return Ok(OptSelector::All); |
| 4712 | } |
| 4713 | if let Some(rest) = raw.strip_prefix("opts =>") { |
| 4714 | return Ok(OptSelector::Only(parse_opt_levels(rest, path, line_no)?)); |
| 4715 | } |
| 4716 | Ok(OptSelector::Only(parse_opt_levels(raw, path, line_no)?)) |
| 4717 | } |
| 4718 | |
| 4719 | fn parse_target(raw: &str, path: &Path, line_no: usize) -> Result<Target, String> { |
| 4720 | match raw { |
| 4721 | "compare.status" => Ok(Target::CompareStatus), |
| 4722 | "compare.classification" => Ok(Target::CompareClassification), |
| 4723 | "compare.changed_artifacts" => Ok(Target::CompareChangedArtifacts), |
| 4724 | "compare.difference_count" => Ok(Target::CompareDifferenceCount), |
| 4725 | "compare.basis" => Ok(Target::CompareBasis), |
| 4726 | "run.stdout" => Ok(Target::RunStdout), |
| 4727 | "run.stderr" => Ok(Target::RunStderr), |
| 4728 | "run.exit_code" => Ok(Target::RunExitCode), |
| 4729 | _ => { |
| 4730 | if let Some(artifact) = ArtifactKey::parse(raw) { |
| 4731 | return Ok(Target::Artifact(artifact)); |
| 4732 | } |
| 4733 | let stage = Stage::parse(raw).ok_or_else(|| { |
| 4734 | format!( |
| 4735 | "{}:{}: unsupported expectation target '{}'", |
| 4736 | path.display(), |
| 4737 | line_no, |
| 4738 | raw |
| 4739 | ) |
| 4740 | })?; |
| 4741 | Ok(Target::Stage(stage)) |
| 4742 | } |
| 4743 | } |
| 4744 | } |
| 4745 | |
| 4746 | fn parse_failure_stage(raw: &str, path: &Path, line_no: usize) -> Result<FailureStage, String> { |
| 4747 | FailureStage::parse(raw).ok_or_else(|| { |
| 4748 | format!( |
| 4749 | "{}:{}: unsupported failure stage '{}'", |
| 4750 | path.display(), |
| 4751 | line_no, |
| 4752 | raw |
| 4753 | ) |
| 4754 | }) |
| 4755 | } |
| 4756 | |
| 4757 | fn parse_quoted(raw: &str, path: &Path, line_no: usize) -> Result<String, String> { |
| 4758 | let raw = raw.trim(); |
| 4759 | if !(raw.starts_with('"') && raw.ends_with('"')) { |
| 4760 | return Err(format!( |
| 4761 | "{}:{}: expected quoted string, got '{}'", |
| 4762 | path.display(), |
| 4763 | line_no, |
| 4764 | raw |
| 4765 | )); |
| 4766 | } |
| 4767 | let body = &raw[1..raw.len() - 1]; |
| 4768 | Ok(body.replace("\\\"", "\"").replace("\\n", "\n")) |
| 4769 | } |
| 4770 | |
| 4771 | fn parse_integer(raw: &str, path: &Path, line_no: usize) -> Result<i32, String> { |
| 4772 | let value = if raw.starts_with('"') { |
| 4773 | parse_quoted(raw, path, line_no)? |
| 4774 | } else { |
| 4775 | raw.trim().to_string() |
| 4776 | }; |
| 4777 | value.parse::<i32>().map_err(|_| { |
| 4778 | format!( |
| 4779 | "{}:{}: expected integer literal, got '{}'", |
| 4780 | path.display(), |
| 4781 | line_no, |
| 4782 | raw |
| 4783 | ) |
| 4784 | }) |
| 4785 | } |
| 4786 | |
| 4787 | fn parse_opt_level_token(raw: &str) -> Option<OptLevel> { |
| 4788 | let raw = raw.trim(); |
| 4789 | let raw = raw.strip_prefix('-').unwrap_or(raw); |
| 4790 | OptLevel::parse_flag(raw) |
| 4791 | } |
| 4792 | |
| 4793 | fn parse_opt_level_list(raw: &str) -> Result<Vec<OptLevel>, String> { |
| 4794 | let mut levels = BTreeSet::new(); |
| 4795 | for value in raw.split(',') { |
| 4796 | let value = value.trim(); |
| 4797 | if value.is_empty() { |
| 4798 | continue; |
| 4799 | } |
| 4800 | if value.eq_ignore_ascii_case("all") { |
| 4801 | levels.extend(all_opt_levels()); |
| 4802 | continue; |
| 4803 | } |
| 4804 | let level = |
| 4805 | parse_opt_level_token(value).ok_or_else(|| format!("unknown opt level '{}'", value))?; |
| 4806 | levels.insert(level); |
| 4807 | } |
| 4808 | if levels.is_empty() { |
| 4809 | return Err("opt filter is empty".into()); |
| 4810 | } |
| 4811 | Ok(levels.into_iter().collect()) |
| 4812 | } |
| 4813 | |
| 4814 | fn all_opt_levels() -> [OptLevel; 5] { |
| 4815 | [ |
| 4816 | OptLevel::O0, |
| 4817 | OptLevel::O1, |
| 4818 | OptLevel::O2, |
| 4819 | OptLevel::O3, |
| 4820 | OptLevel::Ofast, |
| 4821 | ] |
| 4822 | } |
| 4823 | |
| 4824 | fn filter_suites<'a>(suites: &'a [SuiteSpec], suite_filter: Option<&str>) -> Vec<&'a SuiteSpec> { |
| 4825 | let filter = suite_filter.map(|value| value.to_ascii_lowercase()); |
| 4826 | suites |
| 4827 | .iter() |
| 4828 | .filter(|suite| { |
| 4829 | if let Some(filter) = &filter { |
| 4830 | suite.name.to_ascii_lowercase().contains(filter) |
| 4831 | } else { |
| 4832 | true |
| 4833 | } |
| 4834 | }) |
| 4835 | .collect() |
| 4836 | } |
| 4837 | |
| 4838 | fn print_suites(suites: &[&SuiteSpec], config: &ListConfig) { |
| 4839 | for suite in suites { |
| 4840 | println!("{} ({})", suite.name, suite.cases.len()); |
| 4841 | println!(" {}", suite.path.display()); |
| 4842 | if config.verbose { |
| 4843 | for case in &suite.cases { |
| 4844 | println!(" - {} [{}]", case.name, case_discovery_mode_label(case)); |
| 4845 | for line in case_discovery_lines(case, &config.tools) { |
| 4846 | println!(" {}", line); |
| 4847 | } |
| 4848 | } |
| 4849 | } |
| 4850 | } |
| 4851 | } |
| 4852 | |
| 4853 | fn case_discovery_mode_label(case: &CaseSpec) -> &'static str { |
| 4854 | if case.is_generic_compare() { |
| 4855 | "generic-compare" |
| 4856 | } else if case.is_generic_introspect() { |
| 4857 | "generic-introspect" |
| 4858 | } else { |
| 4859 | "legacy" |
| 4860 | } |
| 4861 | } |
| 4862 | |
| 4863 | fn format_opt_level_list(levels: &[OptLevel]) -> String { |
| 4864 | levels |
| 4865 | .iter() |
| 4866 | .map(OptLevel::as_str) |
| 4867 | .collect::<Vec<_>>() |
| 4868 | .join(", ") |
| 4869 | } |
| 4870 | |
| 4871 | fn case_discovery_lines(case: &CaseSpec, tools: &ToolchainConfig) -> Vec<String> { |
| 4872 | let mut lines = vec![ |
| 4873 | format!("source: {}", case.source_label()), |
| 4874 | format!("opts: {}", format_opt_level_list(&case.opt_levels)), |
| 4875 | ]; |
| 4876 | if let Some(policy) = &case.capability_policy { |
| 4877 | lines.push(format!( |
| 4878 | "capability_policy: {} when blocked ({})", |
| 4879 | match policy.kind { |
| 4880 | StatusKind::Xfail => "xfail", |
| 4881 | StatusKind::Future => "future", |
| 4882 | }, |
| 4883 | policy.reason |
| 4884 | )); |
| 4885 | } |
| 4886 | |
| 4887 | if let Some(generic) = &case.generic_introspect { |
| 4888 | lines.push(format!("compiler: {}", generic.compiler.display_name())); |
| 4889 | lines.push(format!( |
| 4890 | "artifacts: {}", |
| 4891 | format_artifact_name_list( |
| 4892 | &generic |
| 4893 | .artifacts |
| 4894 | .iter() |
| 4895 | .map(|artifact| artifact.as_str().to_string()) |
| 4896 | .collect::<Vec<_>>() |
| 4897 | ) |
| 4898 | )); |
| 4899 | match capability_request_issue(&generic.compiler, &generic.artifacts, tools) { |
| 4900 | Some(issue) => { |
| 4901 | lines.push(if case.capability_policy.is_some() { |
| 4902 | "capability_status: deferred".to_string() |
| 4903 | } else { |
| 4904 | "capability_status: blocked".to_string() |
| 4905 | }); |
| 4906 | lines.extend( |
| 4907 | issue |
| 4908 | .lines() |
| 4909 | .map(|line| format!("capability_detail: {}", line)), |
| 4910 | ); |
| 4911 | } |
| 4912 | None => lines.push("capability_status: ready".to_string()), |
| 4913 | } |
| 4914 | return lines; |
| 4915 | } |
| 4916 | |
| 4917 | if let Some(generic) = &case.generic_compare { |
| 4918 | lines.push(format!( |
| 4919 | "compare: {} vs {}", |
| 4920 | generic.left.display_name(), |
| 4921 | generic.right.display_name() |
| 4922 | )); |
| 4923 | lines.push(format!( |
| 4924 | "artifacts: {}", |
| 4925 | format_artifact_name_list( |
| 4926 | &generic |
| 4927 | .artifacts |
| 4928 | .iter() |
| 4929 | .map(|artifact| artifact.as_str().to_string()) |
| 4930 | .collect::<Vec<_>>() |
| 4931 | ) |
| 4932 | )); |
| 4933 | let mut issues = Vec::new(); |
| 4934 | if let Some(issue) = capability_request_issue(&generic.left, &generic.artifacts, tools) { |
| 4935 | issues.push(format!("left:\n{}", issue)); |
| 4936 | } |
| 4937 | if let Some(issue) = capability_request_issue(&generic.right, &generic.artifacts, tools) { |
| 4938 | issues.push(format!("right:\n{}", issue)); |
| 4939 | } |
| 4940 | if issues.is_empty() { |
| 4941 | lines.push("capability_status: ready".to_string()); |
| 4942 | } else { |
| 4943 | lines.push(if case.capability_policy.is_some() { |
| 4944 | "capability_status: deferred".to_string() |
| 4945 | } else { |
| 4946 | "capability_status: blocked".to_string() |
| 4947 | }); |
| 4948 | lines.extend(issues.into_iter().flat_map(|issue| { |
| 4949 | issue |
| 4950 | .lines() |
| 4951 | .map(|line| format!("capability_detail: {}", line)) |
| 4952 | .collect::<Vec<_>>() |
| 4953 | })); |
| 4954 | } |
| 4955 | return lines; |
| 4956 | } |
| 4957 | |
| 4958 | let needs_linked_capture = primary_backend_kind_for_case(case, &case.requested, tools) |
| 4959 | == PrimaryCaptureBackendKind::Full; |
| 4960 | if case.requested.is_empty() { |
| 4961 | lines.push("stages: run".to_string()); |
| 4962 | } else { |
| 4963 | let stages = case |
| 4964 | .requested |
| 4965 | .iter() |
| 4966 | .map(Stage::as_str) |
| 4967 | .collect::<Vec<_>>() |
| 4968 | .join(", "); |
| 4969 | lines.push(format!("stages: {}", stages)); |
| 4970 | } |
| 4971 | if !case.reference_compilers.is_empty() { |
| 4972 | lines.push(format!( |
| 4973 | "differential: {}", |
| 4974 | case.reference_compilers |
| 4975 | .iter() |
| 4976 | .map(ReferenceCompiler::as_str) |
| 4977 | .collect::<Vec<_>>() |
| 4978 | .join(", ") |
| 4979 | )); |
| 4980 | } |
| 4981 | if !case.consistency_checks.is_empty() { |
| 4982 | lines.push(format!( |
| 4983 | "consistency: {}", |
| 4984 | case.consistency_checks |
| 4985 | .iter() |
| 4986 | .map(ConsistencyCheck::as_str) |
| 4987 | .collect::<Vec<_>>() |
| 4988 | .join(", ") |
| 4989 | )); |
| 4990 | } |
| 4991 | if needs_linked_capture { |
| 4992 | lines.push("surface: linked armfortas capture".to_string()); |
| 4993 | if linked_capture_available() { |
| 4994 | lines.push("capability_status: ready".to_string()); |
| 4995 | } else { |
| 4996 | lines.push(if case.capability_policy.is_some() { |
| 4997 | "capability_status: deferred".to_string() |
| 4998 | } else { |
| 4999 | "capability_status: blocked".to_string() |
| 5000 | }); |
| 5001 | lines.push( |
| 5002 | "capability_detail: linked armfortas capture is unavailable in this build" |
| 5003 | .to_string(), |
| 5004 | ); |
| 5005 | } |
| 5006 | } else { |
| 5007 | lines.push("surface: observable-only legacy path".to_string()); |
| 5008 | lines.push("capability_status: ready".to_string()); |
| 5009 | } |
| 5010 | |
| 5011 | lines |
| 5012 | } |
| 5013 | |
| 5014 | fn run_suites(config: &RunConfig) -> Result<Summary, String> { |
| 5015 | let suites = discover_suites(default_suite_root())?; |
| 5016 | let suites = filter_suites(&suites, config.suite_filter.as_deref()); |
| 5017 | if suites.is_empty() { |
| 5018 | return Err("no suites matched the requested filter".into()); |
| 5019 | } |
| 5020 | |
| 5021 | let case_filter = config |
| 5022 | .case_filter |
| 5023 | .as_ref() |
| 5024 | .map(|value| value.to_ascii_lowercase()); |
| 5025 | let mut summary = Summary::default(); |
| 5026 | let mut matched_cells = 0usize; |
| 5027 | |
| 5028 | for suite in suites { |
| 5029 | println!("=== {} ===", suite.name); |
| 5030 | for case in &suite.cases { |
| 5031 | if let Some(filter) = &case_filter { |
| 5032 | if !case.name.to_ascii_lowercase().contains(filter) { |
| 5033 | continue; |
| 5034 | } |
| 5035 | } |
| 5036 | |
| 5037 | let opt_levels = selected_opt_levels(case, config); |
| 5038 | for opt_level in opt_levels { |
| 5039 | matched_cells += 1; |
| 5040 | let outcome = execute_case_cell(suite, case, opt_level, config)?; |
| 5041 | print_outcome(&outcome); |
| 5042 | summary.record_outcome(&outcome); |
| 5043 | |
| 5044 | if config.fail_fast |
| 5045 | && matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5046 | { |
| 5047 | return Ok(summary); |
| 5048 | } |
| 5049 | } |
| 5050 | } |
| 5051 | } |
| 5052 | |
| 5053 | if matched_cells == 0 { |
| 5054 | return Err("no cases matched the requested filters".into()); |
| 5055 | } |
| 5056 | |
| 5057 | Ok(summary) |
| 5058 | } |
| 5059 | |
| 5060 | fn selected_opt_levels(case: &CaseSpec, config: &RunConfig) -> Vec<OptLevel> { |
| 5061 | case.opt_levels |
| 5062 | .iter() |
| 5063 | .copied() |
| 5064 | .filter(|level| { |
| 5065 | config |
| 5066 | .opt_filter |
| 5067 | .as_ref() |
| 5068 | .map(|filter| filter.contains(level)) |
| 5069 | .unwrap_or(true) |
| 5070 | }) |
| 5071 | .collect() |
| 5072 | } |
| 5073 | |
| 5074 | fn execute_case_cell( |
| 5075 | suite: &SuiteSpec, |
| 5076 | case: &CaseSpec, |
| 5077 | opt_level: OptLevel, |
| 5078 | config: &RunConfig, |
| 5079 | ) -> Result<Outcome, String> { |
| 5080 | if case.is_generic_compare() { |
| 5081 | return execute_generic_compare_case_cell(suite, case, opt_level, config); |
| 5082 | } |
| 5083 | if case.is_generic_introspect() { |
| 5084 | return execute_generic_introspect_case_cell(suite, case, opt_level, config); |
| 5085 | } |
| 5086 | |
| 5087 | let effective_status = status_for_opt(case, opt_level); |
| 5088 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5089 | if !config.include_future { |
| 5090 | return Ok(Outcome { |
| 5091 | suite: suite.name.clone(), |
| 5092 | case: case.name.clone(), |
| 5093 | opt_level, |
| 5094 | kind: OutcomeKind::Future, |
| 5095 | detail: reason.clone(), |
| 5096 | bundle: None, |
| 5097 | primary_backend: None, |
| 5098 | consistency_observations: Vec::new(), |
| 5099 | }); |
| 5100 | } |
| 5101 | } |
| 5102 | |
| 5103 | let mut requested = case.requested.clone(); |
| 5104 | if config.all_stages { |
| 5105 | requested.extend(Stage::ALL); |
| 5106 | } |
| 5107 | for expectation in &case.expectations { |
| 5108 | ensure_target_stage(expectation, &mut requested); |
| 5109 | } |
| 5110 | if !case.reference_compilers.is_empty() { |
| 5111 | requested.insert(Stage::Run); |
| 5112 | } |
| 5113 | for check in &case.consistency_checks { |
| 5114 | ensure_consistency_stage(*check, &mut requested); |
| 5115 | } |
| 5116 | |
| 5117 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5118 | let selected_backend = |
| 5119 | select_primary_capture_backend(case, &requested, opt_level, &config.tools); |
| 5120 | |
| 5121 | if let Some(detail) = legacy_unavailable_backend_detail(case, &selected_backend) { |
| 5122 | cleanup_prepared_input(&prepared); |
| 5123 | return Ok(outcome_from_status_and_execution( |
| 5124 | suite, |
| 5125 | case, |
| 5126 | opt_level, |
| 5127 | capability_effective_status(&effective_status, case), |
| 5128 | Err(detail), |
| 5129 | Some(PrimaryBackendReport::from_selected(&selected_backend)), |
| 5130 | Vec::new(), |
| 5131 | )); |
| 5132 | } |
| 5133 | |
| 5134 | if config.verbose { |
| 5135 | let stage_list = requested |
| 5136 | .iter() |
| 5137 | .map(Stage::as_str) |
| 5138 | .collect::<Vec<_>>() |
| 5139 | .join(", "); |
| 5140 | let refs = if case.reference_compilers.is_empty() { |
| 5141 | "none".to_string() |
| 5142 | } else { |
| 5143 | case.reference_compilers |
| 5144 | .iter() |
| 5145 | .map(ReferenceCompiler::as_str) |
| 5146 | .collect::<Vec<_>>() |
| 5147 | .join(", ") |
| 5148 | }; |
| 5149 | println!(" source: {}", case.source_label()); |
| 5150 | if case.is_graph() { |
| 5151 | for file in &case.graph_files { |
| 5152 | println!(" file: {}", file.display()); |
| 5153 | } |
| 5154 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5155 | } |
| 5156 | println!(" opt: {}", opt_level.as_str()); |
| 5157 | println!(" stages: {}", stage_list); |
| 5158 | println!( |
| 5159 | " primary_backend: {} ({})", |
| 5160 | selected_backend.kind.as_str(), |
| 5161 | selected_backend.backend.mode_name() |
| 5162 | ); |
| 5163 | println!( |
| 5164 | " primary_backend_detail: {}", |
| 5165 | selected_backend.backend.description() |
| 5166 | ); |
| 5167 | println!(" refs: {}", refs); |
| 5168 | if !case.consistency_checks.is_empty() { |
| 5169 | println!(" repeat: {}", case.repeat_count); |
| 5170 | } |
| 5171 | } |
| 5172 | |
| 5173 | let references = run_reference_compilers(&prepared, case, opt_level, &config.tools); |
| 5174 | let mut artifacts = ExecutionArtifacts { |
| 5175 | requested, |
| 5176 | armfortas: None, |
| 5177 | armfortas_failure: None, |
| 5178 | armfortas_observation: None, |
| 5179 | references, |
| 5180 | reference_observations: Vec::new(), |
| 5181 | consistency_issues: Vec::new(), |
| 5182 | }; |
| 5183 | artifacts.reference_observations = artifacts |
| 5184 | .references |
| 5185 | .iter() |
| 5186 | .map(|reference| { |
| 5187 | observed_program_from_reference_result( |
| 5188 | &prepared.compiler_source, |
| 5189 | opt_level, |
| 5190 | default_differential_artifacts(), |
| 5191 | reference, |
| 5192 | ) |
| 5193 | }) |
| 5194 | .collect(); |
| 5195 | |
| 5196 | match execute_primary_armfortas( |
| 5197 | &prepared, |
| 5198 | opt_level, |
| 5199 | &artifacts.requested, |
| 5200 | &selected_backend, |
| 5201 | ) { |
| 5202 | Ok(result) => artifacts.armfortas = Some(result), |
| 5203 | Err(failure) => artifacts.armfortas_failure = Some(failure), |
| 5204 | } |
| 5205 | |
| 5206 | let execution = match (&artifacts.armfortas, &artifacts.armfortas_failure) { |
| 5207 | (Some(result), None) => { |
| 5208 | if has_failure_expectation(case) { |
| 5209 | Err(format!( |
| 5210 | "expected armfortas to fail ({}) but compilation succeeded", |
| 5211 | expected_failure_description(case) |
| 5212 | )) |
| 5213 | } else { |
| 5214 | let observed = legacy_success_observed_program( |
| 5215 | case, |
| 5216 | &prepared.compiler_source, |
| 5217 | opt_level, |
| 5218 | result, |
| 5219 | !artifacts.references.is_empty(), |
| 5220 | &config.tools, |
| 5221 | ); |
| 5222 | artifacts.armfortas_observation = Some(observed.clone()); |
| 5223 | let mut execution = evaluate_observation_expectations(case, &observed); |
| 5224 | if execution.is_ok() && !artifacts.references.is_empty() { |
| 5225 | let references = artifacts |
| 5226 | .reference_observations |
| 5227 | .iter() |
| 5228 | .map(|observed| observed.observation.clone()) |
| 5229 | .collect::<Vec<_>>(); |
| 5230 | execution = compare_differential(&observed.observation, &references); |
| 5231 | } |
| 5232 | if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5233 | artifacts.consistency_issues = |
| 5234 | if legacy_case_uses_generic_consistency_checks(case) { |
| 5235 | run_generic_consistency_checks( |
| 5236 | &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5237 | case, |
| 5238 | &prepared.compiler_source, |
| 5239 | opt_level, |
| 5240 | &config.tools, |
| 5241 | ) |
| 5242 | } else { |
| 5243 | run_consistency_checks( |
| 5244 | case, |
| 5245 | &prepared, |
| 5246 | opt_level, |
| 5247 | result, |
| 5248 | &config.tools, |
| 5249 | ) |
| 5250 | }; |
| 5251 | if !artifacts.consistency_issues.is_empty() { |
| 5252 | execution = Err(format_consistency_issues(&artifacts.consistency_issues)); |
| 5253 | } |
| 5254 | } |
| 5255 | execution |
| 5256 | } |
| 5257 | } |
| 5258 | (None, Some(failure)) => { |
| 5259 | let observed = |
| 5260 | legacy_failure_observed_program(&prepared.compiler_source, case, failure); |
| 5261 | artifacts.armfortas_observation = Some(observed.clone()); |
| 5262 | let mut execution = |
| 5263 | evaluate_failed_armfortas_with_observed(case, &artifacts, &observed); |
| 5264 | if execution.is_ok() && !artifacts.references.is_empty() { |
| 5265 | execution = |
| 5266 | Err("differential comparison requires a successful armfortas run".to_string()); |
| 5267 | } |
| 5268 | execution |
| 5269 | } |
| 5270 | (Some(_), Some(_)) => Err("armfortas produced both a result and a failure".into()), |
| 5271 | (None, None) => Err("armfortas produced neither a result nor a failure".into()), |
| 5272 | }; |
| 5273 | |
| 5274 | let consistency_observations = artifacts |
| 5275 | .consistency_issues |
| 5276 | .iter() |
| 5277 | .map(ConsistencyIssue::observation) |
| 5278 | .collect::<Vec<_>>(); |
| 5279 | let primary_backend = Some(PrimaryBackendReport::from_selected(&selected_backend)); |
| 5280 | |
| 5281 | let mut outcome = outcome_from_status_and_execution( |
| 5282 | suite, |
| 5283 | case, |
| 5284 | opt_level, |
| 5285 | effective_status, |
| 5286 | execution, |
| 5287 | primary_backend, |
| 5288 | consistency_observations, |
| 5289 | ); |
| 5290 | |
| 5291 | let should_bundle = matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5292 | || (matches!(outcome.kind, OutcomeKind::Xfail) && !artifacts.consistency_issues.is_empty()); |
| 5293 | |
| 5294 | if should_bundle { |
| 5295 | match write_failure_bundle(suite, case, &prepared, &outcome, &artifacts) { |
| 5296 | Ok(bundle) => outcome.bundle = Some(bundle), |
| 5297 | Err(err) => { |
| 5298 | if outcome.detail.is_empty() { |
| 5299 | outcome.detail = format!("failed to write failure bundle: {}", err); |
| 5300 | } else { |
| 5301 | outcome.detail.push_str(&format!( |
| 5302 | "\n\nwarning: failed to write failure bundle: {}", |
| 5303 | err |
| 5304 | )); |
| 5305 | } |
| 5306 | } |
| 5307 | } |
| 5308 | } |
| 5309 | |
| 5310 | cleanup_prepared_input(&prepared); |
| 5311 | cleanup_consistency_issues(&artifacts.consistency_issues); |
| 5312 | |
| 5313 | Ok(outcome) |
| 5314 | } |
| 5315 | |
| 5316 | fn legacy_success_observed_program( |
| 5317 | case: &CaseSpec, |
| 5318 | program: &Path, |
| 5319 | opt_level: OptLevel, |
| 5320 | result: &CaptureResult, |
| 5321 | has_references: bool, |
| 5322 | tools: &ToolchainConfig, |
| 5323 | ) -> ObservedProgram { |
| 5324 | let mut requested_artifacts = expected_artifacts_for_legacy_case(case); |
| 5325 | if has_references { |
| 5326 | requested_artifacts.extend(default_differential_artifacts()); |
| 5327 | } |
| 5328 | |
| 5329 | if legacy_case_uses_generic_observation_execution(case, &case.requested) { |
| 5330 | if let Ok(observation) = observe_compiler( |
| 5331 | &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5332 | program, |
| 5333 | opt_level, |
| 5334 | &requested_artifacts, |
| 5335 | tools, |
| 5336 | ) { |
| 5337 | if observation.compile_exit_code == 0 { |
| 5338 | return ObservedProgram { |
| 5339 | observation, |
| 5340 | requested_artifacts, |
| 5341 | }; |
| 5342 | } |
| 5343 | } |
| 5344 | } |
| 5345 | |
| 5346 | observed_program_from_armfortas_capture(program, opt_level, requested_artifacts, result, None) |
| 5347 | } |
| 5348 | |
| 5349 | fn legacy_case_uses_generic_observation_execution( |
| 5350 | case: &CaseSpec, |
| 5351 | requested: &BTreeSet<Stage>, |
| 5352 | ) -> bool { |
| 5353 | !has_failure_expectation(case) |
| 5354 | && !requested.is_empty() |
| 5355 | && requested |
| 5356 | .iter() |
| 5357 | .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)) |
| 5358 | } |
| 5359 | |
| 5360 | fn execute_generic_compare_case_cell( |
| 5361 | suite: &SuiteSpec, |
| 5362 | case: &CaseSpec, |
| 5363 | opt_level: OptLevel, |
| 5364 | config: &RunConfig, |
| 5365 | ) -> Result<Outcome, String> { |
| 5366 | let effective_status = status_for_opt(case, opt_level); |
| 5367 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5368 | if !config.include_future { |
| 5369 | return Ok(Outcome { |
| 5370 | suite: suite.name.clone(), |
| 5371 | case: case.name.clone(), |
| 5372 | opt_level, |
| 5373 | kind: OutcomeKind::Future, |
| 5374 | detail: reason.clone(), |
| 5375 | bundle: None, |
| 5376 | primary_backend: None, |
| 5377 | consistency_observations: Vec::new(), |
| 5378 | }); |
| 5379 | } |
| 5380 | } |
| 5381 | |
| 5382 | let generic = case |
| 5383 | .generic_compare |
| 5384 | .as_ref() |
| 5385 | .ok_or_else(|| "missing generic compare case configuration".to_string())?; |
| 5386 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5387 | |
| 5388 | if let Some(detail) = compare_capability_issue( |
| 5389 | &generic.left, |
| 5390 | &generic.right, |
| 5391 | &generic.artifacts, |
| 5392 | &config.tools, |
| 5393 | ) { |
| 5394 | let mut outcome = outcome_from_status_and_execution( |
| 5395 | suite, |
| 5396 | case, |
| 5397 | opt_level, |
| 5398 | capability_effective_status(&effective_status, case), |
| 5399 | Err(detail), |
| 5400 | None, |
| 5401 | Vec::new(), |
| 5402 | ); |
| 5403 | outcome.detail = outcome.detail.trim().to_string(); |
| 5404 | cleanup_prepared_input(&prepared); |
| 5405 | return Ok(outcome); |
| 5406 | } |
| 5407 | |
| 5408 | if config.verbose { |
| 5409 | let artifacts = generic |
| 5410 | .artifacts |
| 5411 | .iter() |
| 5412 | .map(ArtifactKey::as_str) |
| 5413 | .collect::<Vec<_>>() |
| 5414 | .join(", "); |
| 5415 | println!(" source: {}", case.source_label()); |
| 5416 | if case.is_graph() { |
| 5417 | for file in &case.graph_files { |
| 5418 | println!(" file: {}", file.display()); |
| 5419 | } |
| 5420 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5421 | } |
| 5422 | println!( |
| 5423 | " compare: {} vs {}", |
| 5424 | generic.left.display_name(), |
| 5425 | generic.right.display_name() |
| 5426 | ); |
| 5427 | println!(" opt: {}", opt_level.as_str()); |
| 5428 | println!(" artifacts: {}", artifacts); |
| 5429 | } |
| 5430 | |
| 5431 | let result = run_compare(&CompareConfig { |
| 5432 | left: generic.left.clone(), |
| 5433 | right: generic.right.clone(), |
| 5434 | program: prepared.compiler_source.clone(), |
| 5435 | opt_level, |
| 5436 | artifacts: generic.artifacts.clone(), |
| 5437 | json_report: None, |
| 5438 | markdown_report: None, |
| 5439 | tools: config.tools.clone(), |
| 5440 | }); |
| 5441 | |
| 5442 | let execution = if has_failure_expectation(case) { |
| 5443 | Err("suite-v2 compare cases do not support expect-fail rules".to_string()) |
| 5444 | } else if let Ok(result) = &result { |
| 5445 | evaluate_compare_expectations(case, result) |
| 5446 | } else { |
| 5447 | Err(result.unwrap_err()) |
| 5448 | }; |
| 5449 | |
| 5450 | let mut outcome = match (effective_status, execution) { |
| 5451 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5452 | suite: suite.name.clone(), |
| 5453 | case: case.name.clone(), |
| 5454 | opt_level, |
| 5455 | kind: OutcomeKind::Pass, |
| 5456 | detail: String::new(), |
| 5457 | bundle: None, |
| 5458 | primary_backend: None, |
| 5459 | consistency_observations: Vec::new(), |
| 5460 | }, |
| 5461 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5462 | suite: suite.name.clone(), |
| 5463 | case: case.name.clone(), |
| 5464 | opt_level, |
| 5465 | kind: OutcomeKind::Fail, |
| 5466 | detail, |
| 5467 | bundle: None, |
| 5468 | primary_backend: None, |
| 5469 | consistency_observations: Vec::new(), |
| 5470 | }, |
| 5471 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5472 | suite: suite.name.clone(), |
| 5473 | case: case.name.clone(), |
| 5474 | opt_level, |
| 5475 | kind: OutcomeKind::Xpass, |
| 5476 | detail: reason, |
| 5477 | bundle: None, |
| 5478 | primary_backend: None, |
| 5479 | consistency_observations: Vec::new(), |
| 5480 | }, |
| 5481 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5482 | suite: suite.name.clone(), |
| 5483 | case: case.name.clone(), |
| 5484 | opt_level, |
| 5485 | kind: OutcomeKind::Xfail, |
| 5486 | detail: format!("{}\n{}", reason, detail), |
| 5487 | bundle: None, |
| 5488 | primary_backend: None, |
| 5489 | consistency_observations: Vec::new(), |
| 5490 | }, |
| 5491 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5492 | suite: suite.name.clone(), |
| 5493 | case: case.name.clone(), |
| 5494 | opt_level, |
| 5495 | kind: OutcomeKind::Xpass, |
| 5496 | detail: reason, |
| 5497 | bundle: None, |
| 5498 | primary_backend: None, |
| 5499 | consistency_observations: Vec::new(), |
| 5500 | }, |
| 5501 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5502 | suite: suite.name.clone(), |
| 5503 | case: case.name.clone(), |
| 5504 | opt_level, |
| 5505 | kind: OutcomeKind::Future, |
| 5506 | detail: format!("{}\n{}", reason, detail), |
| 5507 | bundle: None, |
| 5508 | primary_backend: None, |
| 5509 | consistency_observations: Vec::new(), |
| 5510 | }, |
| 5511 | }; |
| 5512 | |
| 5513 | outcome.detail = outcome.detail.trim().to_string(); |
| 5514 | cleanup_prepared_input(&prepared); |
| 5515 | Ok(outcome) |
| 5516 | } |
| 5517 | |
| 5518 | fn execute_generic_introspect_case_cell( |
| 5519 | suite: &SuiteSpec, |
| 5520 | case: &CaseSpec, |
| 5521 | opt_level: OptLevel, |
| 5522 | config: &RunConfig, |
| 5523 | ) -> Result<Outcome, String> { |
| 5524 | let effective_status = status_for_opt(case, opt_level); |
| 5525 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5526 | if !config.include_future { |
| 5527 | return Ok(Outcome { |
| 5528 | suite: suite.name.clone(), |
| 5529 | case: case.name.clone(), |
| 5530 | opt_level, |
| 5531 | kind: OutcomeKind::Future, |
| 5532 | detail: reason.clone(), |
| 5533 | bundle: None, |
| 5534 | primary_backend: None, |
| 5535 | consistency_observations: Vec::new(), |
| 5536 | }); |
| 5537 | } |
| 5538 | } |
| 5539 | |
| 5540 | let generic = case |
| 5541 | .generic_introspect |
| 5542 | .as_ref() |
| 5543 | .ok_or_else(|| "missing generic introspection case configuration".to_string())?; |
| 5544 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5545 | |
| 5546 | if let Some(detail) = |
| 5547 | capability_request_issue(&generic.compiler, &generic.artifacts, &config.tools) |
| 5548 | { |
| 5549 | let mut outcome = outcome_from_status_and_execution( |
| 5550 | suite, |
| 5551 | case, |
| 5552 | opt_level, |
| 5553 | capability_effective_status(&effective_status, case), |
| 5554 | Err(detail), |
| 5555 | None, |
| 5556 | Vec::new(), |
| 5557 | ); |
| 5558 | outcome.detail = outcome.detail.trim().to_string(); |
| 5559 | cleanup_prepared_input(&prepared); |
| 5560 | return Ok(outcome); |
| 5561 | } |
| 5562 | |
| 5563 | if config.verbose { |
| 5564 | let artifacts = generic |
| 5565 | .artifacts |
| 5566 | .iter() |
| 5567 | .map(ArtifactKey::as_str) |
| 5568 | .collect::<Vec<_>>() |
| 5569 | .join(", "); |
| 5570 | println!(" source: {}", case.source_label()); |
| 5571 | if case.is_graph() { |
| 5572 | for file in &case.graph_files { |
| 5573 | println!(" file: {}", file.display()); |
| 5574 | } |
| 5575 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5576 | } |
| 5577 | println!(" compiler: {}", generic.compiler.display_name()); |
| 5578 | println!(" opt: {}", opt_level.as_str()); |
| 5579 | println!(" artifacts: {}", artifacts); |
| 5580 | if !case.reference_compilers.is_empty() { |
| 5581 | println!( |
| 5582 | " refs: {}", |
| 5583 | case.reference_compilers |
| 5584 | .iter() |
| 5585 | .map(ReferenceCompiler::as_str) |
| 5586 | .collect::<Vec<_>>() |
| 5587 | .join(", ") |
| 5588 | ); |
| 5589 | } |
| 5590 | if !case.consistency_checks.is_empty() { |
| 5591 | println!( |
| 5592 | " consistency: {}", |
| 5593 | case.consistency_checks |
| 5594 | .iter() |
| 5595 | .map(ConsistencyCheck::as_str) |
| 5596 | .collect::<Vec<_>>() |
| 5597 | .join(", ") |
| 5598 | ); |
| 5599 | println!(" repeat: {}", case.repeat_count); |
| 5600 | } |
| 5601 | } |
| 5602 | |
| 5603 | let observed = run_introspect(&IntrospectConfig { |
| 5604 | compiler: generic.compiler.clone(), |
| 5605 | program: prepared.compiler_source.clone(), |
| 5606 | opt_level, |
| 5607 | artifacts: generic.artifacts.clone(), |
| 5608 | json_report: None, |
| 5609 | markdown_report: None, |
| 5610 | all_artifacts: false, |
| 5611 | summary_only: false, |
| 5612 | max_artifact_lines: None, |
| 5613 | tools: config.tools.clone(), |
| 5614 | })?; |
| 5615 | |
| 5616 | let mut execution = if observed.observation.compile_exit_code == 0 { |
| 5617 | if has_failure_expectation(case) { |
| 5618 | Err(format!( |
| 5619 | "expected {} to fail ({}) but compilation succeeded", |
| 5620 | generic.compiler.display_name(), |
| 5621 | expected_failure_description(case) |
| 5622 | )) |
| 5623 | } else { |
| 5624 | evaluate_observation_expectations(case, &observed) |
| 5625 | } |
| 5626 | } else if has_failure_expectation(case) { |
| 5627 | evaluate_observation_failure_expectations(case, &observed.observation) |
| 5628 | } else { |
| 5629 | Err(compose_observation_failure_detail(&observed.observation)) |
| 5630 | }; |
| 5631 | |
| 5632 | if execution.is_ok() && !case.reference_compilers.is_empty() { |
| 5633 | execution = run_generic_differential( |
| 5634 | &generic.compiler, |
| 5635 | &prepared.compiler_source, |
| 5636 | opt_level, |
| 5637 | &case.reference_compilers, |
| 5638 | &config.tools, |
| 5639 | ); |
| 5640 | } |
| 5641 | |
| 5642 | let mut consistency_issues = Vec::new(); |
| 5643 | if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5644 | consistency_issues = run_generic_consistency_checks( |
| 5645 | &generic.compiler, |
| 5646 | case, |
| 5647 | &prepared.compiler_source, |
| 5648 | opt_level, |
| 5649 | &config.tools, |
| 5650 | ); |
| 5651 | if !consistency_issues.is_empty() { |
| 5652 | execution = Err(format_consistency_issues(&consistency_issues)); |
| 5653 | } |
| 5654 | } |
| 5655 | |
| 5656 | let consistency_observations = consistency_issues |
| 5657 | .iter() |
| 5658 | .map(ConsistencyIssue::observation) |
| 5659 | .collect::<Vec<_>>(); |
| 5660 | |
| 5661 | let mut outcome = match (effective_status, execution) { |
| 5662 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5663 | suite: suite.name.clone(), |
| 5664 | case: case.name.clone(), |
| 5665 | opt_level, |
| 5666 | kind: OutcomeKind::Pass, |
| 5667 | detail: String::new(), |
| 5668 | bundle: None, |
| 5669 | primary_backend: None, |
| 5670 | consistency_observations: consistency_observations.clone(), |
| 5671 | }, |
| 5672 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5673 | suite: suite.name.clone(), |
| 5674 | case: case.name.clone(), |
| 5675 | opt_level, |
| 5676 | kind: OutcomeKind::Fail, |
| 5677 | detail, |
| 5678 | bundle: None, |
| 5679 | primary_backend: None, |
| 5680 | consistency_observations: consistency_observations.clone(), |
| 5681 | }, |
| 5682 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5683 | suite: suite.name.clone(), |
| 5684 | case: case.name.clone(), |
| 5685 | opt_level, |
| 5686 | kind: OutcomeKind::Xpass, |
| 5687 | detail: reason, |
| 5688 | bundle: None, |
| 5689 | primary_backend: None, |
| 5690 | consistency_observations: consistency_observations.clone(), |
| 5691 | }, |
| 5692 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5693 | suite: suite.name.clone(), |
| 5694 | case: case.name.clone(), |
| 5695 | opt_level, |
| 5696 | kind: OutcomeKind::Xfail, |
| 5697 | detail: format!("{}\n{}", reason, detail), |
| 5698 | bundle: None, |
| 5699 | primary_backend: None, |
| 5700 | consistency_observations: consistency_observations.clone(), |
| 5701 | }, |
| 5702 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5703 | suite: suite.name.clone(), |
| 5704 | case: case.name.clone(), |
| 5705 | opt_level, |
| 5706 | kind: OutcomeKind::Xpass, |
| 5707 | detail: reason, |
| 5708 | bundle: None, |
| 5709 | primary_backend: None, |
| 5710 | consistency_observations: consistency_observations.clone(), |
| 5711 | }, |
| 5712 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5713 | suite: suite.name.clone(), |
| 5714 | case: case.name.clone(), |
| 5715 | opt_level, |
| 5716 | kind: OutcomeKind::Future, |
| 5717 | detail: format!("{}\n{}", reason, detail), |
| 5718 | bundle: None, |
| 5719 | primary_backend: None, |
| 5720 | consistency_observations, |
| 5721 | }, |
| 5722 | }; |
| 5723 | |
| 5724 | outcome.detail = outcome.detail.trim().to_string(); |
| 5725 | cleanup_prepared_input(&prepared); |
| 5726 | cleanup_consistency_issues(&consistency_issues); |
| 5727 | Ok(outcome) |
| 5728 | } |
| 5729 | |
| 5730 | fn prepare_case_input( |
| 5731 | case: &CaseSpec, |
| 5732 | suite: &SuiteSpec, |
| 5733 | opt_level: OptLevel, |
| 5734 | ) -> Result<PreparedInput, String> { |
| 5735 | if case.graph_files.is_empty() { |
| 5736 | return Ok(PreparedInput { |
| 5737 | compiler_source: case.source.clone(), |
| 5738 | generated_source: None, |
| 5739 | temp_root: None, |
| 5740 | }); |
| 5741 | } |
| 5742 | |
| 5743 | let temp_root = default_report_root().join(".tmp").join(format!( |
| 5744 | "graph_{}_{}_{}", |
| 5745 | sanitize_component(&suite.name), |
| 5746 | sanitize_component(&case.name), |
| 5747 | next_report_suffix(opt_level) |
| 5748 | )); |
| 5749 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 5750 | format!( |
| 5751 | "cannot create graph temp dir '{}': {}", |
| 5752 | temp_root.display(), |
| 5753 | e |
| 5754 | ) |
| 5755 | })?; |
| 5756 | |
| 5757 | let extension = case |
| 5758 | .source |
| 5759 | .extension() |
| 5760 | .and_then(|ext| ext.to_str()) |
| 5761 | .filter(|ext| !ext.is_empty()) |
| 5762 | .unwrap_or("f90"); |
| 5763 | let generated_source = temp_root.join(format!( |
| 5764 | "{}_graph.{}", |
| 5765 | sanitize_component(&case.name), |
| 5766 | extension |
| 5767 | )); |
| 5768 | |
| 5769 | let mut combined = String::new(); |
| 5770 | for (index, file) in case.graph_files.iter().enumerate() { |
| 5771 | let text = fs::read_to_string(file) |
| 5772 | .map_err(|e| format!("cannot read graph file '{}': {}", file.display(), e))?; |
| 5773 | if index > 0 { |
| 5774 | combined.push('\n'); |
| 5775 | } |
| 5776 | combined.push_str(&text); |
| 5777 | if !text.ends_with('\n') { |
| 5778 | combined.push('\n'); |
| 5779 | } |
| 5780 | } |
| 5781 | |
| 5782 | fs::write(&generated_source, combined).map_err(|e| { |
| 5783 | format!( |
| 5784 | "cannot write generated graph input '{}': {}", |
| 5785 | generated_source.display(), |
| 5786 | e |
| 5787 | ) |
| 5788 | })?; |
| 5789 | |
| 5790 | Ok(PreparedInput { |
| 5791 | compiler_source: generated_source.clone(), |
| 5792 | generated_source: Some(generated_source), |
| 5793 | temp_root: Some(temp_root), |
| 5794 | }) |
| 5795 | } |
| 5796 | |
| 5797 | fn cleanup_prepared_input(prepared: &PreparedInput) { |
| 5798 | if let Some(temp_root) = &prepared.temp_root { |
| 5799 | let _ = fs::remove_dir_all(temp_root); |
| 5800 | } |
| 5801 | } |
| 5802 | |
| 5803 | fn primary_backend_kind_for_case( |
| 5804 | case: &CaseSpec, |
| 5805 | requested: &BTreeSet<Stage>, |
| 5806 | tools: &ToolchainConfig, |
| 5807 | ) -> PrimaryCaptureBackendKind { |
| 5808 | let cli_observable_only = !requested.is_empty() |
| 5809 | && requested |
| 5810 | .iter() |
| 5811 | .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)); |
| 5812 | let supports_cli_primary = matches!( |
| 5813 | tools.armfortas_adapters().cli(), |
| 5814 | ArmfortasCliAdapter::External(_) |
| 5815 | ); |
| 5816 | let capture_checks_required = case |
| 5817 | .consistency_checks |
| 5818 | .iter() |
| 5819 | .any(ConsistencyCheck::requires_capture_result); |
| 5820 | |
| 5821 | if supports_cli_primary |
| 5822 | && cli_observable_only |
| 5823 | && !has_failure_expectation(case) |
| 5824 | && !capture_checks_required |
| 5825 | { |
| 5826 | PrimaryCaptureBackendKind::Observable |
| 5827 | } else { |
| 5828 | PrimaryCaptureBackendKind::Full |
| 5829 | } |
| 5830 | } |
| 5831 | |
| 5832 | fn select_primary_capture_backend( |
| 5833 | case: &CaseSpec, |
| 5834 | requested: &BTreeSet<Stage>, |
| 5835 | opt_level: OptLevel, |
| 5836 | tools: &ToolchainConfig, |
| 5837 | ) -> SelectedPrimaryBackend { |
| 5838 | let kind = primary_backend_kind_for_case(case, requested, tools); |
| 5839 | let backend: Box<dyn CaptureBackend> = match kind { |
| 5840 | PrimaryCaptureBackendKind::Full => Box::new(tools.armfortas_adapters()), |
| 5841 | PrimaryCaptureBackendKind::Observable => { |
| 5842 | Box::new(tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level))) |
| 5843 | } |
| 5844 | }; |
| 5845 | SelectedPrimaryBackend { kind, backend } |
| 5846 | } |
| 5847 | |
| 5848 | fn execute_primary_armfortas( |
| 5849 | prepared: &PreparedInput, |
| 5850 | opt_level: OptLevel, |
| 5851 | requested: &BTreeSet<Stage>, |
| 5852 | selected: &SelectedPrimaryBackend, |
| 5853 | ) -> Result<CaptureResult, CaptureFailure> { |
| 5854 | let request = CaptureRequest { |
| 5855 | input: prepared.compiler_source.clone(), |
| 5856 | requested: requested.clone(), |
| 5857 | opt_level, |
| 5858 | }; |
| 5859 | selected.backend.capture(&request) |
| 5860 | } |
| 5861 | |
| 5862 | fn status_for_opt(case: &CaseSpec, opt_level: OptLevel) -> EffectiveStatus { |
| 5863 | let mut status = EffectiveStatus::Normal; |
| 5864 | for rule in &case.status_rules { |
| 5865 | if rule.selector.matches(opt_level) { |
| 5866 | status = match rule.kind { |
| 5867 | StatusKind::Xfail => EffectiveStatus::Xfail(rule.reason.clone()), |
| 5868 | StatusKind::Future => EffectiveStatus::Future(rule.reason.clone()), |
| 5869 | }; |
| 5870 | } |
| 5871 | } |
| 5872 | status |
| 5873 | } |
| 5874 | |
| 5875 | fn capability_effective_status(base: &EffectiveStatus, case: &CaseSpec) -> EffectiveStatus { |
| 5876 | match base { |
| 5877 | EffectiveStatus::Normal => match &case.capability_policy { |
| 5878 | Some(policy) => match policy.kind { |
| 5879 | StatusKind::Xfail => EffectiveStatus::Xfail(policy.reason.clone()), |
| 5880 | StatusKind::Future => EffectiveStatus::Future(policy.reason.clone()), |
| 5881 | }, |
| 5882 | None => EffectiveStatus::Normal, |
| 5883 | }, |
| 5884 | other => other.clone(), |
| 5885 | } |
| 5886 | } |
| 5887 | |
| 5888 | fn ensure_target_stage(expectation: &Expectation, requested: &mut BTreeSet<Stage>) { |
| 5889 | match expectation { |
| 5890 | Expectation::CheckComments(target) |
| 5891 | | Expectation::Contains { target, .. } |
| 5892 | | Expectation::NotContains { target, .. } |
| 5893 | | Expectation::Equals { target, .. } |
| 5894 | | Expectation::IntEquals { target, .. } => match target { |
| 5895 | Target::Stage(stage) => { |
| 5896 | requested.insert(*stage); |
| 5897 | } |
| 5898 | Target::Artifact(artifact) => ensure_artifact_stage(artifact, requested), |
| 5899 | Target::CompareStatus |
| 5900 | | Target::CompareClassification |
| 5901 | | Target::CompareChangedArtifacts |
| 5902 | | Target::CompareDifferenceCount |
| 5903 | | Target::CompareBasis => {} |
| 5904 | Target::RunStdout | Target::RunStderr | Target::RunExitCode => { |
| 5905 | requested.insert(Stage::Run); |
| 5906 | } |
| 5907 | }, |
| 5908 | Expectation::FailContains { .. } |
| 5909 | | Expectation::FailEquals { .. } |
| 5910 | | Expectation::FailSourceComments |
| 5911 | | Expectation::FailCommentPatterns(_) => {} |
| 5912 | } |
| 5913 | } |
| 5914 | |
| 5915 | fn ensure_consistency_stage(check: ConsistencyCheck, requested: &mut BTreeSet<Stage>) { |
| 5916 | if let Some(stage) = check.required_stage() { |
| 5917 | requested.insert(stage); |
| 5918 | } |
| 5919 | } |
| 5920 | |
| 5921 | fn ensure_artifact_stage(artifact: &ArtifactKey, requested: &mut BTreeSet<Stage>) { |
| 5922 | match artifact { |
| 5923 | ArtifactKey::Asm => { |
| 5924 | requested.insert(Stage::Asm); |
| 5925 | } |
| 5926 | ArtifactKey::Obj => { |
| 5927 | requested.insert(Stage::Obj); |
| 5928 | } |
| 5929 | ArtifactKey::Runtime |
| 5930 | | ArtifactKey::Stdout |
| 5931 | | ArtifactKey::Stderr |
| 5932 | | ArtifactKey::ExitCode => { |
| 5933 | requested.insert(Stage::Run); |
| 5934 | } |
| 5935 | ArtifactKey::Extra(name) => { |
| 5936 | if let Some(stage) = armfortas_extra_stage(name) { |
| 5937 | requested.insert(stage); |
| 5938 | } |
| 5939 | } |
| 5940 | ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 5941 | } |
| 5942 | } |
| 5943 | |
| 5944 | fn armfortas_extra_stage(name: &str) -> Option<Stage> { |
| 5945 | let (namespace, suffix) = name.split_once('.')?; |
| 5946 | if namespace.eq_ignore_ascii_case("armfortas") { |
| 5947 | Stage::parse(suffix) |
| 5948 | } else { |
| 5949 | None |
| 5950 | } |
| 5951 | } |
| 5952 | |
| 5953 | fn evaluate_observation_expectations( |
| 5954 | case: &CaseSpec, |
| 5955 | observed: &ObservedProgram, |
| 5956 | ) -> Result<(), String> { |
| 5957 | for expectation in &case.expectations { |
| 5958 | match expectation { |
| 5959 | Expectation::CheckComments(target) => { |
| 5960 | let text = observation_target_text(&observed.observation, target)?; |
| 5961 | let source = fs::read_to_string(&case.source) |
| 5962 | .map_err(|e| format!("cannot read '{}': {}", case.source.display(), e))?; |
| 5963 | let checks = if target_uses_ir_comment_checks(target) { |
| 5964 | extract_ir_checks(&source) |
| 5965 | } else { |
| 5966 | extract_checks(&source) |
| 5967 | }; |
| 5968 | if checks.is_empty() { |
| 5969 | let expected_label = if target_uses_ir_comment_checks(target) { |
| 5970 | "! IR_CHECK: / ! IR_NOT:" |
| 5971 | } else { |
| 5972 | "! CHECK:" |
| 5973 | }; |
| 5974 | return Err(format!( |
| 5975 | "case '{}' requested check-comments but '{}' has no {} lines", |
| 5976 | case.name, |
| 5977 | case.source.display(), |
| 5978 | expected_label |
| 5979 | )); |
| 5980 | } |
| 5981 | match_checks(&checks, text, &case.name)?; |
| 5982 | } |
| 5983 | Expectation::Contains { target, needle } => { |
| 5984 | let text = observation_target_text(&observed.observation, target)?; |
| 5985 | if !text.contains(needle) { |
| 5986 | return Err(format!( |
| 5987 | "expected {} to contain {:?}\nactual:\n{}", |
| 5988 | target_name(target), |
| 5989 | needle, |
| 5990 | text |
| 5991 | )); |
| 5992 | } |
| 5993 | } |
| 5994 | Expectation::NotContains { target, needle } => { |
| 5995 | let text = observation_target_text(&observed.observation, target)?; |
| 5996 | if text.contains(needle) { |
| 5997 | return Err(format!( |
| 5998 | "expected {} to not contain {:?}\nactual:\n{}", |
| 5999 | target_name(target), |
| 6000 | needle, |
| 6001 | text |
| 6002 | )); |
| 6003 | } |
| 6004 | } |
| 6005 | Expectation::Equals { target, value } => { |
| 6006 | let text = observation_target_text(&observed.observation, target)?; |
| 6007 | if text.trim_end() != value { |
| 6008 | return Err(format!( |
| 6009 | "expected {} to equal {:?}\nactual:\n{}", |
| 6010 | target_name(target), |
| 6011 | value, |
| 6012 | text |
| 6013 | )); |
| 6014 | } |
| 6015 | } |
| 6016 | Expectation::IntEquals { target, value } => { |
| 6017 | let actual = observation_target_int(&observed.observation, target)?; |
| 6018 | if actual != *value { |
| 6019 | return Err(format!( |
| 6020 | "expected {} to equal {}\nactual: {}", |
| 6021 | target_name(target), |
| 6022 | value, |
| 6023 | actual |
| 6024 | )); |
| 6025 | } |
| 6026 | } |
| 6027 | Expectation::FailContains { .. } |
| 6028 | | Expectation::FailEquals { .. } |
| 6029 | | Expectation::FailSourceComments |
| 6030 | | Expectation::FailCommentPatterns(_) => {} |
| 6031 | } |
| 6032 | } |
| 6033 | Ok(()) |
| 6034 | } |
| 6035 | |
| 6036 | fn evaluate_compare_expectations(case: &CaseSpec, result: &ComparisonResult) -> Result<(), String> { |
| 6037 | for expectation in &case.expectations { |
| 6038 | match expectation { |
| 6039 | Expectation::CheckComments(_) => { |
| 6040 | return Err("compare cases do not support check-comments expectations".into()) |
| 6041 | } |
| 6042 | Expectation::Contains { target, needle } => { |
| 6043 | let text = compare_target_text(result, target)?; |
| 6044 | if !text.contains(needle) { |
| 6045 | return Err(format!( |
| 6046 | "expected {} to contain {:?}\nactual:\n{}", |
| 6047 | target_name(target), |
| 6048 | needle, |
| 6049 | text |
| 6050 | )); |
| 6051 | } |
| 6052 | } |
| 6053 | Expectation::NotContains { target, needle } => { |
| 6054 | let text = compare_target_text(result, target)?; |
| 6055 | if text.contains(needle) { |
| 6056 | return Err(format!( |
| 6057 | "expected {} to not contain {:?}\nactual:\n{}", |
| 6058 | target_name(target), |
| 6059 | needle, |
| 6060 | text |
| 6061 | )); |
| 6062 | } |
| 6063 | } |
| 6064 | Expectation::Equals { target, value } => { |
| 6065 | let text = compare_target_text(result, target)?; |
| 6066 | if text.trim_end() != value { |
| 6067 | return Err(format!( |
| 6068 | "expected {} to equal {:?}\nactual:\n{}", |
| 6069 | target_name(target), |
| 6070 | value, |
| 6071 | text |
| 6072 | )); |
| 6073 | } |
| 6074 | } |
| 6075 | Expectation::IntEquals { target, value } => { |
| 6076 | let actual = compare_target_int(result, target)?; |
| 6077 | if actual != *value { |
| 6078 | return Err(format!( |
| 6079 | "expected {} to equal {}\nactual: {}", |
| 6080 | target_name(target), |
| 6081 | value, |
| 6082 | actual |
| 6083 | )); |
| 6084 | } |
| 6085 | } |
| 6086 | Expectation::FailContains { .. } |
| 6087 | | Expectation::FailEquals { .. } |
| 6088 | | Expectation::FailSourceComments |
| 6089 | | Expectation::FailCommentPatterns(_) => {} |
| 6090 | } |
| 6091 | } |
| 6092 | Ok(()) |
| 6093 | } |
| 6094 | |
| 6095 | fn evaluate_observation_failure_expectations( |
| 6096 | case: &CaseSpec, |
| 6097 | observation: &CompilerObservation, |
| 6098 | ) -> Result<(), String> { |
| 6099 | let mut saw_failure_expectation = false; |
| 6100 | let diagnostics = observation_diagnostics_text(observation).unwrap_or_default(); |
| 6101 | for expectation in &case.expectations { |
| 6102 | match expectation { |
| 6103 | Expectation::FailContains { stage, needle } => { |
| 6104 | saw_failure_expectation = true; |
| 6105 | let actual_stage = observation_failure_stage(observation); |
| 6106 | if actual_stage != Some(*stage) { |
| 6107 | let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6108 | return Err(format!( |
| 6109 | "expected failure stage {} but compiler failed in {}\n{}", |
| 6110 | stage.as_str(), |
| 6111 | actual, |
| 6112 | diagnostics |
| 6113 | )); |
| 6114 | } |
| 6115 | if !diagnostics.contains(needle) { |
| 6116 | return Err(format!( |
| 6117 | "expected failure detail at {} to contain {:?}\nactual:\n{}", |
| 6118 | stage.as_str(), |
| 6119 | needle, |
| 6120 | diagnostics |
| 6121 | )); |
| 6122 | } |
| 6123 | } |
| 6124 | Expectation::FailEquals { stage, value } => { |
| 6125 | saw_failure_expectation = true; |
| 6126 | let actual_stage = observation_failure_stage(observation); |
| 6127 | if actual_stage != Some(*stage) { |
| 6128 | let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6129 | return Err(format!( |
| 6130 | "expected failure stage {} but compiler failed in {}\n{}", |
| 6131 | stage.as_str(), |
| 6132 | actual, |
| 6133 | diagnostics |
| 6134 | )); |
| 6135 | } |
| 6136 | if diagnostics.trim_end() != value { |
| 6137 | return Err(format!( |
| 6138 | "expected failure detail at {} to equal {:?}\nactual:\n{}", |
| 6139 | stage.as_str(), |
| 6140 | value, |
| 6141 | diagnostics |
| 6142 | )); |
| 6143 | } |
| 6144 | } |
| 6145 | Expectation::FailCommentPatterns(patterns) => { |
| 6146 | saw_failure_expectation = true; |
| 6147 | for needle in patterns { |
| 6148 | if !diagnostics.contains(needle) { |
| 6149 | return Err(format!( |
| 6150 | "expected failure detail to contain source comment {:?}\nactual:\n{}", |
| 6151 | needle, diagnostics |
| 6152 | )); |
| 6153 | } |
| 6154 | } |
| 6155 | } |
| 6156 | Expectation::CheckComments(_) |
| 6157 | | Expectation::Contains { .. } |
| 6158 | | Expectation::NotContains { .. } |
| 6159 | | Expectation::Equals { .. } |
| 6160 | | Expectation::IntEquals { .. } |
| 6161 | | Expectation::FailSourceComments => {} |
| 6162 | } |
| 6163 | } |
| 6164 | |
| 6165 | if !saw_failure_expectation { |
| 6166 | return Err(format!( |
| 6167 | "{} failed but the case did not declare an expect-fail rule\n{}", |
| 6168 | observation.compiler.display_name(), |
| 6169 | diagnostics |
| 6170 | )); |
| 6171 | } |
| 6172 | |
| 6173 | Ok(()) |
| 6174 | } |
| 6175 | |
| 6176 | #[cfg(test)] |
| 6177 | fn evaluate_failed_armfortas( |
| 6178 | case: &CaseSpec, |
| 6179 | artifacts: &ExecutionArtifacts, |
| 6180 | failure: &CaptureFailure, |
| 6181 | ) -> Result<(), String> { |
| 6182 | let observed = legacy_failure_observed_program(&case.source, case, failure); |
| 6183 | evaluate_failed_armfortas_with_observed(case, artifacts, &observed) |
| 6184 | } |
| 6185 | |
| 6186 | fn evaluate_failed_armfortas_with_observed( |
| 6187 | case: &CaseSpec, |
| 6188 | artifacts: &ExecutionArtifacts, |
| 6189 | observed: &ObservedProgram, |
| 6190 | ) -> Result<(), String> { |
| 6191 | if has_failure_expectation(case) { |
| 6192 | evaluate_observation_failure_expectations(case, &observed.observation) |
| 6193 | } else { |
| 6194 | match evaluate_observation_expectations(case, observed) { |
| 6195 | Ok(()) => Err(compose_armfortas_failure_detail(artifacts)), |
| 6196 | Err(detail) if is_missing_stage_detail(&detail) => { |
| 6197 | Err(compose_armfortas_failure_detail(artifacts)) |
| 6198 | } |
| 6199 | Err(detail) => Err(detail), |
| 6200 | } |
| 6201 | } |
| 6202 | } |
| 6203 | |
| 6204 | fn legacy_failure_observed_program( |
| 6205 | program: &Path, |
| 6206 | case: &CaseSpec, |
| 6207 | failure: &CaptureFailure, |
| 6208 | ) -> ObservedProgram { |
| 6209 | let partial = failure.partial_result(); |
| 6210 | observed_program_from_armfortas_capture( |
| 6211 | program, |
| 6212 | failure.opt_level, |
| 6213 | expected_artifacts_for_legacy_case(case), |
| 6214 | &partial, |
| 6215 | Some(failure), |
| 6216 | ) |
| 6217 | } |
| 6218 | |
| 6219 | fn has_failure_expectation(case: &CaseSpec) -> bool { |
| 6220 | case.expectations.iter().any(|expectation| { |
| 6221 | matches!( |
| 6222 | expectation, |
| 6223 | Expectation::FailContains { .. } |
| 6224 | | Expectation::FailEquals { .. } |
| 6225 | | Expectation::FailSourceComments |
| 6226 | | Expectation::FailCommentPatterns(_) |
| 6227 | ) |
| 6228 | }) |
| 6229 | } |
| 6230 | |
| 6231 | fn legacy_unavailable_backend_detail( |
| 6232 | case: &CaseSpec, |
| 6233 | selected_backend: &SelectedPrimaryBackend, |
| 6234 | ) -> Option<String> { |
| 6235 | if selected_backend.kind == PrimaryCaptureBackendKind::Full |
| 6236 | && selected_backend.backend.mode_name() == "unavailable" |
| 6237 | { |
| 6238 | Some(format!( |
| 6239 | "case requires linked armfortas capture, but this build only provides the external-driver surface\nsource: {}\nrequired backend: {}\nuse scripts/bootstrap-linked-armfortas.sh for rich stages and legacy frontend/module suites, or run a generic suite-v2 / observable-only case instead", |
| 6240 | case.source_label(), |
| 6241 | selected_backend.backend.description() |
| 6242 | )) |
| 6243 | } else { |
| 6244 | None |
| 6245 | } |
| 6246 | } |
| 6247 | |
| 6248 | fn outcome_from_status_and_execution( |
| 6249 | suite: &SuiteSpec, |
| 6250 | case: &CaseSpec, |
| 6251 | opt_level: OptLevel, |
| 6252 | effective_status: EffectiveStatus, |
| 6253 | execution: Result<(), String>, |
| 6254 | primary_backend: Option<PrimaryBackendReport>, |
| 6255 | consistency_observations: Vec<ConsistencyObservation>, |
| 6256 | ) -> Outcome { |
| 6257 | match (effective_status, execution) { |
| 6258 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 6259 | suite: suite.name.clone(), |
| 6260 | case: case.name.clone(), |
| 6261 | opt_level, |
| 6262 | kind: OutcomeKind::Pass, |
| 6263 | detail: String::new(), |
| 6264 | bundle: None, |
| 6265 | primary_backend, |
| 6266 | consistency_observations, |
| 6267 | }, |
| 6268 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 6269 | suite: suite.name.clone(), |
| 6270 | case: case.name.clone(), |
| 6271 | opt_level, |
| 6272 | kind: OutcomeKind::Fail, |
| 6273 | detail, |
| 6274 | bundle: None, |
| 6275 | primary_backend, |
| 6276 | consistency_observations, |
| 6277 | }, |
| 6278 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 6279 | suite: suite.name.clone(), |
| 6280 | case: case.name.clone(), |
| 6281 | opt_level, |
| 6282 | kind: OutcomeKind::Xpass, |
| 6283 | detail: reason, |
| 6284 | bundle: None, |
| 6285 | primary_backend, |
| 6286 | consistency_observations, |
| 6287 | }, |
| 6288 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 6289 | suite: suite.name.clone(), |
| 6290 | case: case.name.clone(), |
| 6291 | opt_level, |
| 6292 | kind: OutcomeKind::Xfail, |
| 6293 | detail: format!("{}\n{}", reason, detail), |
| 6294 | bundle: None, |
| 6295 | primary_backend, |
| 6296 | consistency_observations, |
| 6297 | }, |
| 6298 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 6299 | suite: suite.name.clone(), |
| 6300 | case: case.name.clone(), |
| 6301 | opt_level, |
| 6302 | kind: OutcomeKind::Xpass, |
| 6303 | detail: reason, |
| 6304 | bundle: None, |
| 6305 | primary_backend, |
| 6306 | consistency_observations, |
| 6307 | }, |
| 6308 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 6309 | suite: suite.name.clone(), |
| 6310 | case: case.name.clone(), |
| 6311 | opt_level, |
| 6312 | kind: OutcomeKind::Future, |
| 6313 | detail: format!("{}\n{}", reason, detail), |
| 6314 | bundle: None, |
| 6315 | primary_backend, |
| 6316 | consistency_observations, |
| 6317 | }, |
| 6318 | } |
| 6319 | } |
| 6320 | |
| 6321 | fn expected_failure_description(case: &CaseSpec) -> String { |
| 6322 | let mut items = Vec::new(); |
| 6323 | for expectation in &case.expectations { |
| 6324 | match expectation { |
| 6325 | Expectation::FailContains { stage, needle } => { |
| 6326 | items.push(format!("{} contains {:?}", stage.as_str(), needle)); |
| 6327 | } |
| 6328 | Expectation::FailEquals { stage, value } => { |
| 6329 | items.push(format!("{} equals {:?}", stage.as_str(), value)); |
| 6330 | } |
| 6331 | Expectation::FailCommentPatterns(patterns) => { |
| 6332 | for needle in patterns { |
| 6333 | items.push(format!("comments contain {:?}", needle)); |
| 6334 | } |
| 6335 | } |
| 6336 | _ => {} |
| 6337 | } |
| 6338 | } |
| 6339 | if items.is_empty() { |
| 6340 | "declared failure".to_string() |
| 6341 | } else { |
| 6342 | items.join(", ") |
| 6343 | } |
| 6344 | } |
| 6345 | |
| 6346 | fn is_missing_stage_detail(detail: &str) -> bool { |
| 6347 | detail.starts_with("missing captured stage '") |
| 6348 | || detail == "missing captured run stage" |
| 6349 | || detail.starts_with("missing artifact '") |
| 6350 | } |
| 6351 | |
| 6352 | fn target_name(target: &Target) -> String { |
| 6353 | match target { |
| 6354 | Target::Stage(stage) => stage.as_str().to_string(), |
| 6355 | Target::Artifact(artifact) => artifact.as_str().to_string(), |
| 6356 | Target::CompareStatus => "compare.status".to_string(), |
| 6357 | Target::CompareClassification => "compare.classification".to_string(), |
| 6358 | Target::CompareChangedArtifacts => "compare.changed_artifacts".to_string(), |
| 6359 | Target::CompareDifferenceCount => "compare.difference_count".to_string(), |
| 6360 | Target::CompareBasis => "compare.basis".to_string(), |
| 6361 | Target::RunStdout => "run.stdout".to_string(), |
| 6362 | Target::RunStderr => "run.stderr".to_string(), |
| 6363 | Target::RunExitCode => "run.exit_code".to_string(), |
| 6364 | } |
| 6365 | } |
| 6366 | |
| 6367 | fn compare_target_text(result: &ComparisonResult, target: &Target) -> Result<String, String> { |
| 6368 | match target { |
| 6369 | Target::CompareStatus => Ok(compare_status(result).to_string()), |
| 6370 | Target::CompareClassification => Ok(compare_classification(result).to_string()), |
| 6371 | Target::CompareChangedArtifacts => { |
| 6372 | let changed = compare_changed_artifacts(result); |
| 6373 | if changed.is_empty() { |
| 6374 | Ok("none".to_string()) |
| 6375 | } else { |
| 6376 | Ok(changed.join(", ")) |
| 6377 | } |
| 6378 | } |
| 6379 | Target::CompareBasis => Ok(result.basis.clone()), |
| 6380 | Target::CompareDifferenceCount => Err( |
| 6381 | "compare.difference_count is numeric; use 'expect compare.difference_count equals <int>'" |
| 6382 | .into(), |
| 6383 | ), |
| 6384 | _ => Err(format!( |
| 6385 | "{} is not a compare text target", |
| 6386 | target_name(target) |
| 6387 | )), |
| 6388 | } |
| 6389 | } |
| 6390 | |
| 6391 | fn compare_target_int(result: &ComparisonResult, target: &Target) -> Result<i32, String> { |
| 6392 | match target { |
| 6393 | Target::CompareDifferenceCount => Ok(result.differences.len() as i32), |
| 6394 | _ => Err(format!( |
| 6395 | "{} is textual; use a string matcher instead", |
| 6396 | target_name(target) |
| 6397 | )), |
| 6398 | } |
| 6399 | } |
| 6400 | |
| 6401 | fn observation_target_text<'a>( |
| 6402 | observation: &'a CompilerObservation, |
| 6403 | target: &Target, |
| 6404 | ) -> Result<&'a str, String> { |
| 6405 | match target { |
| 6406 | Target::Stage(stage) => match stage { |
| 6407 | Stage::Asm => observation_artifact_text(observation, &ArtifactKey::Asm), |
| 6408 | Stage::Obj => observation_artifact_text(observation, &ArtifactKey::Obj), |
| 6409 | Stage::Run => { |
| 6410 | Err("run is structured; use run.stdout, run.stderr, or run.exit_code".into()) |
| 6411 | } |
| 6412 | other => observation_artifact_text( |
| 6413 | observation, |
| 6414 | &ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6415 | ), |
| 6416 | }, |
| 6417 | Target::Artifact(artifact) => observation_artifact_text(observation, artifact), |
| 6418 | Target::CompareStatus |
| 6419 | | Target::CompareClassification |
| 6420 | | Target::CompareChangedArtifacts |
| 6421 | | Target::CompareDifferenceCount |
| 6422 | | Target::CompareBasis => { |
| 6423 | Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6424 | } |
| 6425 | Target::RunStdout => observation_run_stdout(observation), |
| 6426 | Target::RunStderr => observation_run_stderr(observation), |
| 6427 | Target::RunExitCode => { |
| 6428 | Err("run.exit_code is numeric; use 'expect run.exit_code equals <int>'".into()) |
| 6429 | } |
| 6430 | } |
| 6431 | } |
| 6432 | |
| 6433 | fn observation_target_int( |
| 6434 | observation: &CompilerObservation, |
| 6435 | target: &Target, |
| 6436 | ) -> Result<i32, String> { |
| 6437 | match target { |
| 6438 | Target::RunExitCode => observation_run_exit_code(observation), |
| 6439 | Target::Artifact(ArtifactKey::ExitCode) => observation_run_exit_code(observation), |
| 6440 | Target::CompareStatus |
| 6441 | | Target::CompareClassification |
| 6442 | | Target::CompareChangedArtifacts |
| 6443 | | Target::CompareDifferenceCount |
| 6444 | | Target::CompareBasis => { |
| 6445 | Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6446 | } |
| 6447 | _ => Err(format!( |
| 6448 | "{} is textual; use a string matcher instead", |
| 6449 | target_name(target) |
| 6450 | )), |
| 6451 | } |
| 6452 | } |
| 6453 | |
| 6454 | fn observation_artifact_text<'a>( |
| 6455 | observation: &'a CompilerObservation, |
| 6456 | artifact: &ArtifactKey, |
| 6457 | ) -> Result<&'a str, String> { |
| 6458 | match observation.artifacts.get(artifact) { |
| 6459 | Some(ArtifactValue::Text(text)) => Ok(text), |
| 6460 | Some(ArtifactValue::Int(_)) => Err(format!( |
| 6461 | "artifact '{}' is numeric; use an integer matcher instead", |
| 6462 | artifact.as_str() |
| 6463 | )), |
| 6464 | Some(ArtifactValue::Run(_)) => Err(format!( |
| 6465 | "artifact '{}' is structured runtime data; use run.stdout, run.stderr, or run.exit_code", |
| 6466 | artifact.as_str() |
| 6467 | )), |
| 6468 | Some(ArtifactValue::Path(_)) => Err(format!( |
| 6469 | "artifact '{}' is binary/path data, not text", |
| 6470 | artifact.as_str() |
| 6471 | )), |
| 6472 | None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 6473 | } |
| 6474 | } |
| 6475 | |
| 6476 | fn observation_run_stdout(observation: &CompilerObservation) -> Result<&str, String> { |
| 6477 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6478 | Ok(&run.stdout) |
| 6479 | } else { |
| 6480 | observation_artifact_text(observation, &ArtifactKey::Stdout) |
| 6481 | } |
| 6482 | } |
| 6483 | |
| 6484 | fn observation_run_stderr(observation: &CompilerObservation) -> Result<&str, String> { |
| 6485 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6486 | Ok(&run.stderr) |
| 6487 | } else { |
| 6488 | observation_artifact_text(observation, &ArtifactKey::Stderr) |
| 6489 | } |
| 6490 | } |
| 6491 | |
| 6492 | fn observation_run_exit_code(observation: &CompilerObservation) -> Result<i32, String> { |
| 6493 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6494 | Ok(run.exit_code) |
| 6495 | } else { |
| 6496 | match observation.artifacts.get(&ArtifactKey::ExitCode) { |
| 6497 | Some(ArtifactValue::Int(value)) => Ok(*value), |
| 6498 | Some(_) => Err("artifact 'exit-code' is not numeric".into()), |
| 6499 | None => Err("missing artifact 'exit-code'".into()), |
| 6500 | } |
| 6501 | } |
| 6502 | } |
| 6503 | |
| 6504 | fn observation_diagnostics_text(observation: &CompilerObservation) -> Option<&str> { |
| 6505 | match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 6506 | Some(ArtifactValue::Text(text)) => Some(text.as_str()), |
| 6507 | _ => None, |
| 6508 | } |
| 6509 | } |
| 6510 | |
| 6511 | fn observation_failure_stage(observation: &CompilerObservation) -> Option<FailureStage> { |
| 6512 | observation |
| 6513 | .provenance |
| 6514 | .failure_stage |
| 6515 | .as_deref() |
| 6516 | .and_then(FailureStage::parse) |
| 6517 | } |
| 6518 | |
| 6519 | fn compare_differential( |
| 6520 | armfortas: &CompilerObservation, |
| 6521 | references: &[CompilerObservation], |
| 6522 | ) -> Result<(), String> { |
| 6523 | let requested = default_differential_artifacts(); |
| 6524 | let comparisons = references |
| 6525 | .iter() |
| 6526 | .cloned() |
| 6527 | .map(|reference| compare_observations(armfortas.clone(), reference, &requested)) |
| 6528 | .collect::<Vec<_>>(); |
| 6529 | let matching_refs = comparisons |
| 6530 | .iter() |
| 6531 | .filter(|comparison| comparison.differences.is_empty()) |
| 6532 | .count(); |
| 6533 | |
| 6534 | if matching_refs == comparisons.len() { |
| 6535 | return Ok(()); |
| 6536 | } |
| 6537 | |
| 6538 | let reference_disagreement = if references.len() > 1 { |
| 6539 | let baseline = references[0].clone(); |
| 6540 | references[1..].iter().cloned().any(|reference| { |
| 6541 | !compare_observations(baseline.clone(), reference, &requested) |
| 6542 | .differences |
| 6543 | .is_empty() |
| 6544 | }) |
| 6545 | } else { |
| 6546 | false |
| 6547 | }; |
| 6548 | |
| 6549 | let classification = if matching_refs == 0 && !reference_disagreement { |
| 6550 | "classification: armfortas-only divergence" |
| 6551 | } else if reference_disagreement { |
| 6552 | "classification: reference disagreement" |
| 6553 | } else { |
| 6554 | "classification: partial disagreement" |
| 6555 | }; |
| 6556 | |
| 6557 | let detail = comparisons |
| 6558 | .iter() |
| 6559 | .filter(|comparison| !comparison.differences.is_empty()) |
| 6560 | .map(render_compare_text) |
| 6561 | .collect::<Vec<_>>(); |
| 6562 | |
| 6563 | Err(format!( |
| 6564 | "behavior mismatch against reference compilers\n{}\n\n{}", |
| 6565 | classification, |
| 6566 | detail.join("\n\n") |
| 6567 | )) |
| 6568 | } |
| 6569 | |
| 6570 | fn compose_armfortas_failure_detail(artifacts: &ExecutionArtifacts) -> String { |
| 6571 | let mut detail = String::new(); |
| 6572 | if let Some(failure) = &artifacts.armfortas_failure { |
| 6573 | detail.push_str(&format!( |
| 6574 | "armfortas failed in {}\n{}", |
| 6575 | failure.stage.as_str(), |
| 6576 | failure.detail |
| 6577 | )); |
| 6578 | } else { |
| 6579 | detail.push_str("armfortas failed without an error message"); |
| 6580 | } |
| 6581 | |
| 6582 | if !artifacts.references.is_empty() { |
| 6583 | detail.push_str("\n\nreference compilers\n"); |
| 6584 | detail.push_str(&format_reference_summary(&artifacts.references)); |
| 6585 | } |
| 6586 | |
| 6587 | detail |
| 6588 | } |
| 6589 | |
| 6590 | fn compose_observation_failure_detail(observation: &CompilerObservation) -> String { |
| 6591 | if observation.provenance.backend_mode == "unavailable" { |
| 6592 | let mut detail = format!( |
| 6593 | "{} unavailable for requested artifacts in this build", |
| 6594 | observation.compiler.display_name() |
| 6595 | ); |
| 6596 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6597 | detail.push('\n'); |
| 6598 | detail.push_str(diagnostics); |
| 6599 | } |
| 6600 | return detail; |
| 6601 | } |
| 6602 | |
| 6603 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6604 | if diagnostics.contains("does not support requested artifacts in this adapter") { |
| 6605 | return format!( |
| 6606 | "{} does not support requested artifacts in this adapter\n{}", |
| 6607 | observation.compiler.display_name(), |
| 6608 | diagnostics |
| 6609 | ); |
| 6610 | } |
| 6611 | } |
| 6612 | |
| 6613 | let mut detail = String::new(); |
| 6614 | detail.push_str(&format!("{} failed", observation.compiler.display_name())); |
| 6615 | if let Some(stage) = &observation.provenance.failure_stage { |
| 6616 | detail.push_str(&format!(" in {}", stage)); |
| 6617 | } |
| 6618 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6619 | detail.push('\n'); |
| 6620 | detail.push_str(diagnostics); |
| 6621 | } |
| 6622 | detail |
| 6623 | } |
| 6624 | |
| 6625 | fn run_generic_differential( |
| 6626 | compiler: &CompilerSpec, |
| 6627 | program: &Path, |
| 6628 | opt_level: OptLevel, |
| 6629 | references: &[ReferenceCompiler], |
| 6630 | tools: &ToolchainConfig, |
| 6631 | ) -> Result<(), String> { |
| 6632 | let requested = default_differential_artifacts(); |
| 6633 | let primary = observe_compiler(compiler, program, opt_level, &requested, tools)?; |
| 6634 | let references = references |
| 6635 | .iter() |
| 6636 | .copied() |
| 6637 | .map(reference_compiler_spec) |
| 6638 | .map(|reference| observe_compiler(&reference, program, opt_level, &requested, tools)) |
| 6639 | .collect::<Result<Vec<_>, _>>()?; |
| 6640 | compare_differential(&primary, &references) |
| 6641 | } |
| 6642 | |
| 6643 | fn expected_artifacts_for_legacy_case(case: &CaseSpec) -> BTreeSet<ArtifactKey> { |
| 6644 | let mut requested = BTreeSet::new(); |
| 6645 | for stage in &case.requested { |
| 6646 | requested.insert(stage_to_artifact_key(*stage)); |
| 6647 | } |
| 6648 | for expectation in &case.expectations { |
| 6649 | match expectation { |
| 6650 | Expectation::CheckComments(target) |
| 6651 | | Expectation::Contains { target, .. } |
| 6652 | | Expectation::NotContains { target, .. } |
| 6653 | | Expectation::Equals { target, .. } |
| 6654 | | Expectation::IntEquals { target, .. } => match target { |
| 6655 | Target::Stage(stage) => { |
| 6656 | requested.insert(stage_to_artifact_key(*stage)); |
| 6657 | } |
| 6658 | Target::Artifact(artifact) => { |
| 6659 | requested.insert(artifact.clone()); |
| 6660 | } |
| 6661 | Target::RunStdout => { |
| 6662 | requested.insert(ArtifactKey::Stdout); |
| 6663 | } |
| 6664 | Target::RunStderr => { |
| 6665 | requested.insert(ArtifactKey::Stderr); |
| 6666 | } |
| 6667 | Target::RunExitCode => { |
| 6668 | requested.insert(ArtifactKey::ExitCode); |
| 6669 | } |
| 6670 | Target::CompareStatus |
| 6671 | | Target::CompareClassification |
| 6672 | | Target::CompareChangedArtifacts |
| 6673 | | Target::CompareDifferenceCount |
| 6674 | | Target::CompareBasis => {} |
| 6675 | }, |
| 6676 | Expectation::FailContains { .. } |
| 6677 | | Expectation::FailEquals { .. } |
| 6678 | | Expectation::FailSourceComments |
| 6679 | | Expectation::FailCommentPatterns(_) => {} |
| 6680 | } |
| 6681 | } |
| 6682 | requested |
| 6683 | } |
| 6684 | |
| 6685 | fn legacy_case_uses_generic_consistency_checks(case: &CaseSpec) -> bool { |
| 6686 | !case.consistency_checks.is_empty() |
| 6687 | && case |
| 6688 | .consistency_checks |
| 6689 | .iter() |
| 6690 | .copied() |
| 6691 | .all(|check| check.supports_generic_introspect()) |
| 6692 | } |
| 6693 | |
| 6694 | fn stage_to_artifact_key(stage: Stage) -> ArtifactKey { |
| 6695 | match stage { |
| 6696 | Stage::Asm => ArtifactKey::Asm, |
| 6697 | Stage::Obj => ArtifactKey::Obj, |
| 6698 | Stage::Run => ArtifactKey::Runtime, |
| 6699 | other => ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6700 | } |
| 6701 | } |
| 6702 | |
| 6703 | fn observed_program_from_armfortas_capture( |
| 6704 | program: &Path, |
| 6705 | opt_level: OptLevel, |
| 6706 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 6707 | result: &CaptureResult, |
| 6708 | failure: Option<&CaptureFailure>, |
| 6709 | ) -> ObservedProgram { |
| 6710 | let mut artifacts = BTreeMap::new(); |
| 6711 | for (stage, captured) in &result.stages { |
| 6712 | match (stage, captured) { |
| 6713 | (Stage::Asm, CapturedStage::Text(text)) |
| 6714 | if requested_artifacts.contains(&ArtifactKey::Asm) => |
| 6715 | { |
| 6716 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 6717 | } |
| 6718 | (Stage::Obj, CapturedStage::Text(text)) |
| 6719 | if requested_artifacts.contains(&ArtifactKey::Obj) => |
| 6720 | { |
| 6721 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 6722 | } |
| 6723 | (Stage::Run, CapturedStage::Run(run)) => { |
| 6724 | insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 6725 | } |
| 6726 | (stage, CapturedStage::Text(text)) => { |
| 6727 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 6728 | if requested_artifacts.contains(&key) { |
| 6729 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 6730 | } |
| 6731 | } |
| 6732 | _ => {} |
| 6733 | } |
| 6734 | } |
| 6735 | if let Some(failure) = failure { |
| 6736 | if requested_artifacts.contains(&ArtifactKey::Diagnostics) |
| 6737 | || !artifacts.contains_key(&ArtifactKey::Diagnostics) |
| 6738 | { |
| 6739 | artifacts.insert( |
| 6740 | ArtifactKey::Diagnostics, |
| 6741 | ArtifactValue::Text(failure.detail.clone()), |
| 6742 | ); |
| 6743 | } |
| 6744 | } |
| 6745 | let artifacts_captured = artifacts |
| 6746 | .keys() |
| 6747 | .map(|artifact| artifact.as_str().to_string()) |
| 6748 | .collect::<Vec<_>>(); |
| 6749 | ObservedProgram { |
| 6750 | observation: CompilerObservation { |
| 6751 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 6752 | program: program.to_path_buf(), |
| 6753 | opt_level, |
| 6754 | compile_exit_code: if failure.is_some() { 1 } else { 0 }, |
| 6755 | artifacts, |
| 6756 | provenance: ObservationProvenance { |
| 6757 | compiler_identity: "armfortas".into(), |
| 6758 | adapter_kind: "named".into(), |
| 6759 | backend_mode: "suite-legacy-capture".into(), |
| 6760 | backend_detail: "legacy suite cell capture converted into generic observation" |
| 6761 | .into(), |
| 6762 | artifacts_captured, |
| 6763 | comparison_basis: None, |
| 6764 | failure_stage: failure.map(|failure| failure.stage.as_str().to_string()), |
| 6765 | }, |
| 6766 | }, |
| 6767 | requested_artifacts, |
| 6768 | } |
| 6769 | } |
| 6770 | |
| 6771 | fn observed_program_from_reference_result( |
| 6772 | program: &Path, |
| 6773 | opt_level: OptLevel, |
| 6774 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 6775 | reference: &ReferenceResult, |
| 6776 | ) -> ObservedProgram { |
| 6777 | let mut artifacts = BTreeMap::new(); |
| 6778 | let diagnostics = [ |
| 6779 | reference.compile_stdout.trim_end(), |
| 6780 | reference.compile_stderr.trim_end(), |
| 6781 | ] |
| 6782 | .iter() |
| 6783 | .filter(|part| !part.is_empty()) |
| 6784 | .copied() |
| 6785 | .collect::<Vec<_>>() |
| 6786 | .join("\n"); |
| 6787 | |
| 6788 | if requested_artifacts.contains(&ArtifactKey::Diagnostics) && !diagnostics.is_empty() { |
| 6789 | artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 6790 | } |
| 6791 | |
| 6792 | if let Some(run) = &reference.run { |
| 6793 | insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 6794 | } else if let Some(run_error) = &reference.run_error { |
| 6795 | let diagnostics = artifacts |
| 6796 | .entry(ArtifactKey::Diagnostics) |
| 6797 | .or_insert_with(|| ArtifactValue::Text(String::new())); |
| 6798 | if let ArtifactValue::Text(text) = diagnostics { |
| 6799 | if !text.is_empty() { |
| 6800 | text.push('\n'); |
| 6801 | } |
| 6802 | text.push_str(&format!("run error: {}", run_error)); |
| 6803 | } |
| 6804 | } |
| 6805 | |
| 6806 | let artifacts_captured = artifacts |
| 6807 | .keys() |
| 6808 | .map(|artifact| artifact.as_str().to_string()) |
| 6809 | .collect::<Vec<_>>(); |
| 6810 | ObservedProgram { |
| 6811 | observation: CompilerObservation { |
| 6812 | compiler: match reference.compiler { |
| 6813 | ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 6814 | ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 6815 | }, |
| 6816 | program: program.to_path_buf(), |
| 6817 | opt_level, |
| 6818 | compile_exit_code: reference.compile_exit_code, |
| 6819 | artifacts, |
| 6820 | provenance: ObservationProvenance { |
| 6821 | compiler_identity: reference.compiler.as_str().to_string(), |
| 6822 | adapter_kind: "named".into(), |
| 6823 | backend_mode: "legacy-reference".into(), |
| 6824 | backend_detail: format!( |
| 6825 | "legacy differential reference observation via {}", |
| 6826 | reference.compile_command |
| 6827 | ), |
| 6828 | artifacts_captured, |
| 6829 | comparison_basis: None, |
| 6830 | failure_stage: None, |
| 6831 | }, |
| 6832 | }, |
| 6833 | requested_artifacts, |
| 6834 | } |
| 6835 | } |
| 6836 | |
| 6837 | fn reference_compiler_spec(compiler: ReferenceCompiler) -> CompilerSpec { |
| 6838 | match compiler { |
| 6839 | ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 6840 | ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 6841 | } |
| 6842 | } |
| 6843 | |
| 6844 | fn run_generic_consistency_checks( |
| 6845 | compiler: &CompilerSpec, |
| 6846 | case: &CaseSpec, |
| 6847 | source: &Path, |
| 6848 | opt_level: OptLevel, |
| 6849 | tools: &ToolchainConfig, |
| 6850 | ) -> Vec<ConsistencyIssue> { |
| 6851 | let mut failures = Vec::new(); |
| 6852 | for check in &case.consistency_checks { |
| 6853 | let issue = match check { |
| 6854 | ConsistencyCheck::CliAsmReproducible => run_generic_cli_asm_reproducible( |
| 6855 | compiler, |
| 6856 | source, |
| 6857 | opt_level, |
| 6858 | case.repeat_count, |
| 6859 | tools, |
| 6860 | ), |
| 6861 | ConsistencyCheck::CliObjReproducible => run_generic_cli_obj_reproducible( |
| 6862 | compiler, |
| 6863 | source, |
| 6864 | opt_level, |
| 6865 | case.repeat_count, |
| 6866 | tools, |
| 6867 | ), |
| 6868 | ConsistencyCheck::CliRunReproducible => run_generic_cli_run_reproducible( |
| 6869 | compiler, |
| 6870 | source, |
| 6871 | opt_level, |
| 6872 | case.repeat_count, |
| 6873 | tools, |
| 6874 | ), |
| 6875 | _ => Some(ConsistencyIssue { |
| 6876 | check: *check, |
| 6877 | summary: "unsupported generic consistency check".into(), |
| 6878 | repeat_count: None, |
| 6879 | unique_variant_count: None, |
| 6880 | varying_components: Vec::new(), |
| 6881 | stable_components: Vec::new(), |
| 6882 | detail: format!( |
| 6883 | "generic compiler cases do not support '{}' yet", |
| 6884 | check.as_str() |
| 6885 | ), |
| 6886 | temp_root: next_consistency_temp_root(opt_level), |
| 6887 | }), |
| 6888 | }; |
| 6889 | if let Some(issue) = issue { |
| 6890 | failures.push(issue); |
| 6891 | } |
| 6892 | } |
| 6893 | failures |
| 6894 | } |
| 6895 | |
| 6896 | fn run_generic_cli_asm_reproducible( |
| 6897 | compiler: &CompilerSpec, |
| 6898 | source: &Path, |
| 6899 | opt_level: OptLevel, |
| 6900 | repeat_count: usize, |
| 6901 | tools: &ToolchainConfig, |
| 6902 | ) -> Option<ConsistencyIssue> { |
| 6903 | let temp_root = next_consistency_temp_root(opt_level); |
| 6904 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 6905 | return Some(ConsistencyIssue { |
| 6906 | check: ConsistencyCheck::CliAsmReproducible, |
| 6907 | summary: "could not create consistency temp dir".into(), |
| 6908 | repeat_count: None, |
| 6909 | unique_variant_count: None, |
| 6910 | varying_components: Vec::new(), |
| 6911 | stable_components: Vec::new(), |
| 6912 | detail: format!( |
| 6913 | "cannot create consistency temp dir '{}': {}", |
| 6914 | temp_root.display(), |
| 6915 | err |
| 6916 | ), |
| 6917 | temp_root, |
| 6918 | }); |
| 6919 | } |
| 6920 | |
| 6921 | let requested = BTreeSet::from([ArtifactKey::Asm]); |
| 6922 | let mut runs = Vec::new(); |
| 6923 | for index in 0..repeat_count { |
| 6924 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 6925 | Ok(observation) => observation, |
| 6926 | Err(detail) => { |
| 6927 | return Some(ConsistencyIssue { |
| 6928 | check: ConsistencyCheck::CliAsmReproducible, |
| 6929 | summary: "compiler observation failed during consistency check".into(), |
| 6930 | repeat_count: Some(repeat_count), |
| 6931 | unique_variant_count: None, |
| 6932 | varying_components: Vec::new(), |
| 6933 | stable_components: Vec::new(), |
| 6934 | detail, |
| 6935 | temp_root, |
| 6936 | }) |
| 6937 | } |
| 6938 | }; |
| 6939 | let asm = match observation_text_artifact(&observation, &ArtifactKey::Asm) { |
| 6940 | Ok(asm) => asm, |
| 6941 | Err(detail) => { |
| 6942 | return Some(ConsistencyIssue { |
| 6943 | check: ConsistencyCheck::CliAsmReproducible, |
| 6944 | summary: "missing asm artifact during consistency check".into(), |
| 6945 | repeat_count: Some(repeat_count), |
| 6946 | unique_variant_count: None, |
| 6947 | varying_components: Vec::new(), |
| 6948 | stable_components: Vec::new(), |
| 6949 | detail, |
| 6950 | temp_root, |
| 6951 | }) |
| 6952 | } |
| 6953 | }; |
| 6954 | runs.push(TextRun { |
| 6955 | label: format!("run {}", index + 1), |
| 6956 | command: observation_command_hint(&observation), |
| 6957 | normalized: normalize_text_artifact(&asm), |
| 6958 | }); |
| 6959 | } |
| 6960 | |
| 6961 | let unique_variant_count = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 6962 | if unique_variant_count > 1 { |
| 6963 | let (left, right) = first_distinct_text_pair(&runs).unwrap(); |
| 6964 | return Some(ConsistencyIssue { |
| 6965 | check: ConsistencyCheck::CliAsmReproducible, |
| 6966 | summary: format!( |
| 6967 | "repeat_count={} unique_variants={}", |
| 6968 | repeat_count, unique_variant_count |
| 6969 | ), |
| 6970 | repeat_count: Some(repeat_count), |
| 6971 | unique_variant_count: Some(unique_variant_count), |
| 6972 | varying_components: Vec::new(), |
| 6973 | stable_components: Vec::new(), |
| 6974 | detail: format!( |
| 6975 | "asm output was not reproducible for {}\n{}\n{}\n{}", |
| 6976 | compiler.display_name(), |
| 6977 | left.command, |
| 6978 | right.command, |
| 6979 | describe_text_difference( |
| 6980 | &left.normalized, |
| 6981 | &right.normalized, |
| 6982 | &left.label, |
| 6983 | &right.label |
| 6984 | ) |
| 6985 | ), |
| 6986 | temp_root, |
| 6987 | }); |
| 6988 | } |
| 6989 | |
| 6990 | let _ = fs::remove_dir_all(&temp_root); |
| 6991 | None |
| 6992 | } |
| 6993 | |
| 6994 | fn run_generic_cli_obj_reproducible( |
| 6995 | compiler: &CompilerSpec, |
| 6996 | source: &Path, |
| 6997 | opt_level: OptLevel, |
| 6998 | repeat_count: usize, |
| 6999 | tools: &ToolchainConfig, |
| 7000 | ) -> Option<ConsistencyIssue> { |
| 7001 | let temp_root = next_consistency_temp_root(opt_level); |
| 7002 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7003 | return Some(ConsistencyIssue { |
| 7004 | check: ConsistencyCheck::CliObjReproducible, |
| 7005 | summary: "could not create consistency temp dir".into(), |
| 7006 | repeat_count: None, |
| 7007 | unique_variant_count: None, |
| 7008 | varying_components: Vec::new(), |
| 7009 | stable_components: Vec::new(), |
| 7010 | detail: format!( |
| 7011 | "cannot create consistency temp dir '{}': {}", |
| 7012 | temp_root.display(), |
| 7013 | err |
| 7014 | ), |
| 7015 | temp_root, |
| 7016 | }); |
| 7017 | } |
| 7018 | |
| 7019 | let requested = BTreeSet::from([ArtifactKey::Obj]); |
| 7020 | let mut rendered_runs = Vec::new(); |
| 7021 | let mut object_runs = Vec::new(); |
| 7022 | let mut parseable = true; |
| 7023 | for index in 0..repeat_count { |
| 7024 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7025 | Ok(observation) => observation, |
| 7026 | Err(detail) => { |
| 7027 | return Some(ConsistencyIssue { |
| 7028 | check: ConsistencyCheck::CliObjReproducible, |
| 7029 | summary: "compiler observation failed during consistency check".into(), |
| 7030 | repeat_count: Some(repeat_count), |
| 7031 | unique_variant_count: None, |
| 7032 | varying_components: Vec::new(), |
| 7033 | stable_components: Vec::new(), |
| 7034 | detail, |
| 7035 | temp_root, |
| 7036 | }) |
| 7037 | } |
| 7038 | }; |
| 7039 | let obj_text = match observation_text_artifact(&observation, &ArtifactKey::Obj) { |
| 7040 | Ok(text) => text, |
| 7041 | Err(detail) => { |
| 7042 | return Some(ConsistencyIssue { |
| 7043 | check: ConsistencyCheck::CliObjReproducible, |
| 7044 | summary: "missing obj artifact during consistency check".into(), |
| 7045 | repeat_count: Some(repeat_count), |
| 7046 | unique_variant_count: None, |
| 7047 | varying_components: Vec::new(), |
| 7048 | stable_components: Vec::new(), |
| 7049 | detail, |
| 7050 | temp_root, |
| 7051 | }) |
| 7052 | } |
| 7053 | }; |
| 7054 | let label = format!("run {}", index + 1); |
| 7055 | let command = observation_command_hint(&observation); |
| 7056 | rendered_runs.push(TextRun { |
| 7057 | label: label.clone(), |
| 7058 | command: command.clone(), |
| 7059 | normalized: normalize_text_artifact(&obj_text), |
| 7060 | }); |
| 7061 | match parse_object_snapshot_text(&obj_text) { |
| 7062 | Ok(snapshot) => object_runs.push(ObjectRun { |
| 7063 | label, |
| 7064 | command, |
| 7065 | snapshot, |
| 7066 | }), |
| 7067 | Err(_) => parseable = false, |
| 7068 | } |
| 7069 | } |
| 7070 | |
| 7071 | if parseable { |
| 7072 | let rendered = object_runs |
| 7073 | .iter() |
| 7074 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 7075 | .collect::<Vec<_>>(); |
| 7076 | let unique_variant_count = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7077 | if unique_variant_count > 1 { |
| 7078 | let (left, right) = first_distinct_object_pair(&object_runs).unwrap(); |
| 7079 | let snapshots = object_runs |
| 7080 | .iter() |
| 7081 | .map(|run| &run.snapshot) |
| 7082 | .collect::<Vec<_>>(); |
| 7083 | let varying = varying_object_components(&snapshots) |
| 7084 | .into_iter() |
| 7085 | .map(str::to_string) |
| 7086 | .collect::<Vec<_>>(); |
| 7087 | let stable = stable_object_components(&snapshots) |
| 7088 | .into_iter() |
| 7089 | .map(str::to_string) |
| 7090 | .collect::<Vec<_>>(); |
| 7091 | return Some(ConsistencyIssue { |
| 7092 | check: ConsistencyCheck::CliObjReproducible, |
| 7093 | summary: format!( |
| 7094 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7095 | repeat_count, |
| 7096 | unique_variant_count, |
| 7097 | join_or_none_from_strings(&varying), |
| 7098 | join_or_none_from_strings(&stable) |
| 7099 | ), |
| 7100 | repeat_count: Some(repeat_count), |
| 7101 | unique_variant_count: Some(unique_variant_count), |
| 7102 | varying_components: varying, |
| 7103 | stable_components: stable, |
| 7104 | detail: format!( |
| 7105 | "object output was not reproducible for {}\n{}\n{}\n{}", |
| 7106 | compiler.display_name(), |
| 7107 | left.command, |
| 7108 | right.command, |
| 7109 | describe_object_difference( |
| 7110 | &left.snapshot, |
| 7111 | &right.snapshot, |
| 7112 | &left.label, |
| 7113 | &right.label |
| 7114 | ) |
| 7115 | ), |
| 7116 | temp_root, |
| 7117 | }); |
| 7118 | } |
| 7119 | } else { |
| 7120 | let unique_variant_count = |
| 7121 | count_unique_strings(rendered_runs.iter().map(|run| run.normalized.as_str())); |
| 7122 | if unique_variant_count > 1 { |
| 7123 | let (left, right) = first_distinct_text_pair(&rendered_runs).unwrap(); |
| 7124 | return Some(ConsistencyIssue { |
| 7125 | check: ConsistencyCheck::CliObjReproducible, |
| 7126 | summary: format!( |
| 7127 | "repeat_count={} unique_variants={}", |
| 7128 | repeat_count, unique_variant_count |
| 7129 | ), |
| 7130 | repeat_count: Some(repeat_count), |
| 7131 | unique_variant_count: Some(unique_variant_count), |
| 7132 | varying_components: Vec::new(), |
| 7133 | stable_components: Vec::new(), |
| 7134 | detail: format!( |
| 7135 | "object artifact text was not reproducible for {}\n{}\n{}\n{}", |
| 7136 | compiler.display_name(), |
| 7137 | left.command, |
| 7138 | right.command, |
| 7139 | describe_text_difference( |
| 7140 | &left.normalized, |
| 7141 | &right.normalized, |
| 7142 | &left.label, |
| 7143 | &right.label |
| 7144 | ) |
| 7145 | ), |
| 7146 | temp_root, |
| 7147 | }); |
| 7148 | } |
| 7149 | } |
| 7150 | |
| 7151 | let _ = fs::remove_dir_all(&temp_root); |
| 7152 | None |
| 7153 | } |
| 7154 | |
| 7155 | fn run_generic_cli_run_reproducible( |
| 7156 | compiler: &CompilerSpec, |
| 7157 | source: &Path, |
| 7158 | opt_level: OptLevel, |
| 7159 | repeat_count: usize, |
| 7160 | tools: &ToolchainConfig, |
| 7161 | ) -> Option<ConsistencyIssue> { |
| 7162 | let temp_root = next_consistency_temp_root(opt_level); |
| 7163 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7164 | return Some(ConsistencyIssue { |
| 7165 | check: ConsistencyCheck::CliRunReproducible, |
| 7166 | summary: "could not create consistency temp dir".into(), |
| 7167 | repeat_count: None, |
| 7168 | unique_variant_count: None, |
| 7169 | varying_components: Vec::new(), |
| 7170 | stable_components: Vec::new(), |
| 7171 | detail: format!( |
| 7172 | "cannot create consistency temp dir '{}': {}", |
| 7173 | temp_root.display(), |
| 7174 | err |
| 7175 | ), |
| 7176 | temp_root, |
| 7177 | }); |
| 7178 | } |
| 7179 | |
| 7180 | let requested = BTreeSet::from([ArtifactKey::Runtime]); |
| 7181 | let mut runs = Vec::new(); |
| 7182 | for index in 0..repeat_count { |
| 7183 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7184 | Ok(observation) => observation, |
| 7185 | Err(detail) => { |
| 7186 | return Some(ConsistencyIssue { |
| 7187 | check: ConsistencyCheck::CliRunReproducible, |
| 7188 | summary: "compiler observation failed during consistency check".into(), |
| 7189 | repeat_count: Some(repeat_count), |
| 7190 | unique_variant_count: None, |
| 7191 | varying_components: Vec::new(), |
| 7192 | stable_components: Vec::new(), |
| 7193 | detail, |
| 7194 | temp_root, |
| 7195 | }) |
| 7196 | } |
| 7197 | }; |
| 7198 | let run = match observation_run_capture(&observation) { |
| 7199 | Ok(run) => run, |
| 7200 | Err(detail) => { |
| 7201 | return Some(ConsistencyIssue { |
| 7202 | check: ConsistencyCheck::CliRunReproducible, |
| 7203 | summary: "missing runtime artifact during consistency check".into(), |
| 7204 | repeat_count: Some(repeat_count), |
| 7205 | unique_variant_count: None, |
| 7206 | varying_components: Vec::new(), |
| 7207 | stable_components: Vec::new(), |
| 7208 | detail, |
| 7209 | temp_root, |
| 7210 | }) |
| 7211 | } |
| 7212 | }; |
| 7213 | runs.push(BehaviorRun { |
| 7214 | label: format!("run {}", index + 1), |
| 7215 | command: observation_command_hint(&observation), |
| 7216 | signature: normalize_run_signature(&run), |
| 7217 | run, |
| 7218 | }); |
| 7219 | } |
| 7220 | |
| 7221 | let unique_variant_count = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 7222 | if unique_variant_count > 1 { |
| 7223 | let (left, right) = first_distinct_behavior_pair(&runs).unwrap(); |
| 7224 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 7225 | let varying = varying_run_components(&signatures) |
| 7226 | .into_iter() |
| 7227 | .map(str::to_string) |
| 7228 | .collect::<Vec<_>>(); |
| 7229 | let stable = stable_run_components(&signatures) |
| 7230 | .into_iter() |
| 7231 | .map(str::to_string) |
| 7232 | .collect::<Vec<_>>(); |
| 7233 | return Some(ConsistencyIssue { |
| 7234 | check: ConsistencyCheck::CliRunReproducible, |
| 7235 | summary: format!( |
| 7236 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7237 | repeat_count, |
| 7238 | unique_variant_count, |
| 7239 | join_or_none_from_strings(&varying), |
| 7240 | join_or_none_from_strings(&stable) |
| 7241 | ), |
| 7242 | repeat_count: Some(repeat_count), |
| 7243 | unique_variant_count: Some(unique_variant_count), |
| 7244 | varying_components: varying, |
| 7245 | stable_components: stable, |
| 7246 | detail: format!( |
| 7247 | "runtime behavior was not reproducible for {}\n{}\n{}\n{}", |
| 7248 | compiler.display_name(), |
| 7249 | left.command, |
| 7250 | right.command, |
| 7251 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 7252 | ), |
| 7253 | temp_root, |
| 7254 | }); |
| 7255 | } |
| 7256 | |
| 7257 | let _ = fs::remove_dir_all(&temp_root); |
| 7258 | None |
| 7259 | } |
| 7260 | |
| 7261 | fn observation_text_artifact( |
| 7262 | observation: &CompilerObservation, |
| 7263 | artifact: &ArtifactKey, |
| 7264 | ) -> Result<String, String> { |
| 7265 | match observation.artifacts.get(artifact) { |
| 7266 | Some(ArtifactValue::Text(text)) => Ok(text.clone()), |
| 7267 | Some(ArtifactValue::Int(_)) => Err(format!( |
| 7268 | "artifact '{}' is numeric, not text", |
| 7269 | artifact.as_str() |
| 7270 | )), |
| 7271 | Some(ArtifactValue::Run(_)) => Err(format!( |
| 7272 | "artifact '{}' is structured runtime data, not text", |
| 7273 | artifact.as_str() |
| 7274 | )), |
| 7275 | Some(ArtifactValue::Path(path)) => Err(format!( |
| 7276 | "artifact '{}' is path data ('{}'), not text", |
| 7277 | artifact.as_str(), |
| 7278 | path.display() |
| 7279 | )), |
| 7280 | None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 7281 | } |
| 7282 | } |
| 7283 | |
| 7284 | fn observation_run_capture(observation: &CompilerObservation) -> Result<RunCapture, String> { |
| 7285 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 7286 | return Ok(run.clone()); |
| 7287 | } |
| 7288 | |
| 7289 | Ok(RunCapture { |
| 7290 | exit_code: observation_run_exit_code(observation)?, |
| 7291 | stdout: observation_run_stdout(observation)?.to_string(), |
| 7292 | stderr: observation_run_stderr(observation)?.to_string(), |
| 7293 | }) |
| 7294 | } |
| 7295 | |
| 7296 | fn observation_command_hint(observation: &CompilerObservation) -> String { |
| 7297 | format!( |
| 7298 | "{} [{}; {}]", |
| 7299 | observation.compiler.display_name(), |
| 7300 | observation.provenance.backend_mode, |
| 7301 | observation.provenance.backend_detail |
| 7302 | ) |
| 7303 | } |
| 7304 | |
| 7305 | fn run_consistency_checks( |
| 7306 | case: &CaseSpec, |
| 7307 | prepared: &PreparedInput, |
| 7308 | opt_level: OptLevel, |
| 7309 | capture_result: &CaptureResult, |
| 7310 | tools: &ToolchainConfig, |
| 7311 | ) -> Vec<ConsistencyIssue> { |
| 7312 | let mut failures = Vec::new(); |
| 7313 | for check in &case.consistency_checks { |
| 7314 | let issue = match check { |
| 7315 | ConsistencyCheck::CliObjVsSystemAs => { |
| 7316 | run_cli_obj_vs_system_as(&prepared.compiler_source, opt_level, tools) |
| 7317 | } |
| 7318 | ConsistencyCheck::CliAsmReproducible => run_cli_asm_reproducible( |
| 7319 | &prepared.compiler_source, |
| 7320 | opt_level, |
| 7321 | case.repeat_count, |
| 7322 | tools, |
| 7323 | ), |
| 7324 | ConsistencyCheck::CliObjReproducible => run_cli_obj_reproducible( |
| 7325 | &prepared.compiler_source, |
| 7326 | opt_level, |
| 7327 | case.repeat_count, |
| 7328 | tools, |
| 7329 | ), |
| 7330 | ConsistencyCheck::CliRunReproducible => run_cli_run_reproducible( |
| 7331 | &prepared.compiler_source, |
| 7332 | opt_level, |
| 7333 | case.repeat_count, |
| 7334 | tools, |
| 7335 | ), |
| 7336 | ConsistencyCheck::CaptureAsmVsCliAsm => run_capture_asm_vs_cli_asm( |
| 7337 | &prepared.compiler_source, |
| 7338 | opt_level, |
| 7339 | case.repeat_count, |
| 7340 | capture_result, |
| 7341 | tools, |
| 7342 | ), |
| 7343 | ConsistencyCheck::CaptureObjVsCliObj => run_capture_obj_vs_cli_obj( |
| 7344 | &prepared.compiler_source, |
| 7345 | opt_level, |
| 7346 | case.repeat_count, |
| 7347 | capture_result, |
| 7348 | tools, |
| 7349 | ), |
| 7350 | ConsistencyCheck::CaptureRunVsCliRun => run_capture_run_vs_cli_run( |
| 7351 | &prepared.compiler_source, |
| 7352 | opt_level, |
| 7353 | case.repeat_count, |
| 7354 | capture_result, |
| 7355 | tools, |
| 7356 | ), |
| 7357 | ConsistencyCheck::CaptureAsmReproducible => run_capture_asm_reproducible( |
| 7358 | &prepared.compiler_source, |
| 7359 | opt_level, |
| 7360 | case.repeat_count, |
| 7361 | capture_result, |
| 7362 | tools, |
| 7363 | ), |
| 7364 | ConsistencyCheck::CaptureObjReproducible => run_capture_obj_reproducible( |
| 7365 | &prepared.compiler_source, |
| 7366 | opt_level, |
| 7367 | case.repeat_count, |
| 7368 | capture_result, |
| 7369 | tools, |
| 7370 | ), |
| 7371 | ConsistencyCheck::CaptureRunReproducible => run_capture_run_reproducible( |
| 7372 | &prepared.compiler_source, |
| 7373 | opt_level, |
| 7374 | case.repeat_count, |
| 7375 | capture_result, |
| 7376 | tools, |
| 7377 | ), |
| 7378 | }; |
| 7379 | if let Some(issue) = issue { |
| 7380 | failures.push(issue); |
| 7381 | } |
| 7382 | } |
| 7383 | failures |
| 7384 | } |
| 7385 | |
| 7386 | fn format_consistency_issues(issues: &[ConsistencyIssue]) -> String { |
| 7387 | issues |
| 7388 | .iter() |
| 7389 | .map(|issue| { |
| 7390 | format!( |
| 7391 | "consistency check '{}' failed\n{}", |
| 7392 | issue.check.as_str(), |
| 7393 | issue.detail |
| 7394 | ) |
| 7395 | }) |
| 7396 | .collect::<Vec<_>>() |
| 7397 | .join("\n\n") |
| 7398 | } |
| 7399 | |
| 7400 | fn cleanup_consistency_issues(issues: &[ConsistencyIssue]) { |
| 7401 | for issue in issues { |
| 7402 | let _ = fs::remove_dir_all(&issue.temp_root); |
| 7403 | } |
| 7404 | } |
| 7405 | |
| 7406 | fn run_cli_obj_vs_system_as( |
| 7407 | source: &Path, |
| 7408 | opt_level: OptLevel, |
| 7409 | tools: &ToolchainConfig, |
| 7410 | ) -> Option<ConsistencyIssue> { |
| 7411 | let temp_root = next_consistency_temp_root(opt_level); |
| 7412 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7413 | return Some(ConsistencyIssue { |
| 7414 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7415 | summary: "could not create consistency temp dir".into(), |
| 7416 | repeat_count: None, |
| 7417 | unique_variant_count: None, |
| 7418 | varying_components: Vec::new(), |
| 7419 | stable_components: Vec::new(), |
| 7420 | detail: format!( |
| 7421 | "cannot create consistency temp dir '{}': {}", |
| 7422 | temp_root.display(), |
| 7423 | err |
| 7424 | ), |
| 7425 | temp_root, |
| 7426 | }); |
| 7427 | } |
| 7428 | |
| 7429 | let asm_path = temp_root.join("from_cli.s"); |
| 7430 | let asm_obj_path = temp_root.join("from_cli_asm.o"); |
| 7431 | let obj_path = temp_root.join("from_cli_obj.o"); |
| 7432 | |
| 7433 | let asm_command = |
| 7434 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7435 | Ok(command) => command, |
| 7436 | Err(detail) => { |
| 7437 | return Some(ConsistencyIssue { |
| 7438 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7439 | summary: "armfortas -S failed during consistency check".into(), |
| 7440 | repeat_count: None, |
| 7441 | unique_variant_count: None, |
| 7442 | varying_components: Vec::new(), |
| 7443 | stable_components: Vec::new(), |
| 7444 | detail, |
| 7445 | temp_root, |
| 7446 | }) |
| 7447 | } |
| 7448 | }; |
| 7449 | |
| 7450 | let as_args = vec![ |
| 7451 | "-o".to_string(), |
| 7452 | asm_obj_path.display().to_string(), |
| 7453 | asm_path.display().to_string(), |
| 7454 | ]; |
| 7455 | let as_command = render_command(tools.system_as_bin(), &as_args); |
| 7456 | let as_output = match Command::new(tools.system_as_bin()) |
| 7457 | .args([ |
| 7458 | "-o", |
| 7459 | asm_obj_path.to_str().unwrap(), |
| 7460 | asm_path.to_str().unwrap(), |
| 7461 | ]) |
| 7462 | .output() |
| 7463 | { |
| 7464 | Ok(output) => output, |
| 7465 | Err(err) => { |
| 7466 | return Some(ConsistencyIssue { |
| 7467 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7468 | summary: "system assembler invocation failed".into(), |
| 7469 | repeat_count: None, |
| 7470 | unique_variant_count: None, |
| 7471 | varying_components: Vec::new(), |
| 7472 | stable_components: Vec::new(), |
| 7473 | detail: format!("{}\ncannot run assembler: {}", as_command, err), |
| 7474 | temp_root, |
| 7475 | }) |
| 7476 | } |
| 7477 | }; |
| 7478 | if !as_output.status.success() { |
| 7479 | let stderr = String::from_utf8_lossy(&as_output.stderr); |
| 7480 | return Some(ConsistencyIssue { |
| 7481 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7482 | summary: "system assembler rejected armfortas -S output".into(), |
| 7483 | repeat_count: None, |
| 7484 | unique_variant_count: None, |
| 7485 | varying_components: Vec::new(), |
| 7486 | stable_components: Vec::new(), |
| 7487 | detail: format!("{}\nassembler failed:\n{}", as_command, stderr), |
| 7488 | temp_root, |
| 7489 | }); |
| 7490 | } |
| 7491 | |
| 7492 | let obj_command = |
| 7493 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7494 | Ok(command) => command, |
| 7495 | Err(detail) => { |
| 7496 | return Some(ConsistencyIssue { |
| 7497 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7498 | summary: "armfortas -c failed during consistency check".into(), |
| 7499 | repeat_count: None, |
| 7500 | unique_variant_count: None, |
| 7501 | varying_components: Vec::new(), |
| 7502 | stable_components: Vec::new(), |
| 7503 | detail, |
| 7504 | temp_root, |
| 7505 | }) |
| 7506 | } |
| 7507 | }; |
| 7508 | |
| 7509 | let asm_snapshot = match object_snapshot(&asm_obj_path, tools) { |
| 7510 | Ok(snapshot) => snapshot, |
| 7511 | Err(detail) => { |
| 7512 | return Some(ConsistencyIssue { |
| 7513 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7514 | summary: "could not snapshot object assembled from -S output".into(), |
| 7515 | repeat_count: None, |
| 7516 | unique_variant_count: None, |
| 7517 | varying_components: Vec::new(), |
| 7518 | stable_components: Vec::new(), |
| 7519 | detail: format!("{}\n{}", as_command, detail), |
| 7520 | temp_root, |
| 7521 | }) |
| 7522 | } |
| 7523 | }; |
| 7524 | let obj_snapshot = match object_snapshot(&obj_path, tools) { |
| 7525 | Ok(snapshot) => snapshot, |
| 7526 | Err(detail) => { |
| 7527 | return Some(ConsistencyIssue { |
| 7528 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7529 | summary: "could not snapshot object from armfortas -c".into(), |
| 7530 | repeat_count: None, |
| 7531 | unique_variant_count: None, |
| 7532 | varying_components: Vec::new(), |
| 7533 | stable_components: Vec::new(), |
| 7534 | detail: format!("{}\n{}", obj_command, detail), |
| 7535 | temp_root, |
| 7536 | }) |
| 7537 | } |
| 7538 | }; |
| 7539 | |
| 7540 | if asm_snapshot != obj_snapshot { |
| 7541 | let snapshots = [&asm_snapshot, &obj_snapshot]; |
| 7542 | let varying = varying_object_components(&snapshots) |
| 7543 | .into_iter() |
| 7544 | .map(str::to_string) |
| 7545 | .collect::<Vec<_>>(); |
| 7546 | let stable = stable_object_components(&snapshots) |
| 7547 | .into_iter() |
| 7548 | .map(str::to_string) |
| 7549 | .collect::<Vec<_>>(); |
| 7550 | return Some(ConsistencyIssue { |
| 7551 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7552 | summary: format!( |
| 7553 | "varying_components={} stable_components={}", |
| 7554 | join_or_none_from_strings(&varying), |
| 7555 | join_or_none_from_strings(&stable) |
| 7556 | ), |
| 7557 | repeat_count: None, |
| 7558 | unique_variant_count: None, |
| 7559 | varying_components: varying, |
| 7560 | stable_components: stable, |
| 7561 | detail: format!( |
| 7562 | "object snapshot mismatch between armfortas -S | as and armfortas -c\n{}\n{}\n{}\n{}", |
| 7563 | asm_command, |
| 7564 | as_command, |
| 7565 | obj_command, |
| 7566 | describe_object_difference(&asm_snapshot, &obj_snapshot, "-S | as", "-c") |
| 7567 | ), |
| 7568 | temp_root, |
| 7569 | }); |
| 7570 | } |
| 7571 | |
| 7572 | let _ = fs::remove_dir_all(&temp_root); |
| 7573 | None |
| 7574 | } |
| 7575 | |
| 7576 | fn run_cli_asm_reproducible( |
| 7577 | source: &Path, |
| 7578 | opt_level: OptLevel, |
| 7579 | repeat_count: usize, |
| 7580 | tools: &ToolchainConfig, |
| 7581 | ) -> Option<ConsistencyIssue> { |
| 7582 | let temp_root = next_consistency_temp_root(opt_level); |
| 7583 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7584 | return Some(ConsistencyIssue { |
| 7585 | check: ConsistencyCheck::CliAsmReproducible, |
| 7586 | summary: "could not create consistency temp dir".into(), |
| 7587 | repeat_count: None, |
| 7588 | unique_variant_count: None, |
| 7589 | varying_components: Vec::new(), |
| 7590 | stable_components: Vec::new(), |
| 7591 | detail: format!( |
| 7592 | "cannot create consistency temp dir '{}': {}", |
| 7593 | temp_root.display(), |
| 7594 | err |
| 7595 | ), |
| 7596 | temp_root, |
| 7597 | }); |
| 7598 | } |
| 7599 | |
| 7600 | let mut runs = Vec::new(); |
| 7601 | for index in 0..repeat_count { |
| 7602 | let asm_path = temp_root.join(format!("run_{:02}.s", index)); |
| 7603 | let command = |
| 7604 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7605 | Ok(command) => command, |
| 7606 | Err(detail) => { |
| 7607 | return Some(ConsistencyIssue { |
| 7608 | check: ConsistencyCheck::CliAsmReproducible, |
| 7609 | summary: "armfortas -S failed during reproducibility check".into(), |
| 7610 | repeat_count: None, |
| 7611 | unique_variant_count: None, |
| 7612 | varying_components: Vec::new(), |
| 7613 | stable_components: Vec::new(), |
| 7614 | detail, |
| 7615 | temp_root, |
| 7616 | }) |
| 7617 | } |
| 7618 | }; |
| 7619 | let text = match read_text_artifact(&asm_path) { |
| 7620 | Ok(text) => text, |
| 7621 | Err(detail) => { |
| 7622 | return Some(ConsistencyIssue { |
| 7623 | check: ConsistencyCheck::CliAsmReproducible, |
| 7624 | summary: "could not read emitted assembly during reproducibility check".into(), |
| 7625 | repeat_count: None, |
| 7626 | unique_variant_count: None, |
| 7627 | varying_components: Vec::new(), |
| 7628 | stable_components: Vec::new(), |
| 7629 | detail, |
| 7630 | temp_root, |
| 7631 | }) |
| 7632 | } |
| 7633 | }; |
| 7634 | runs.push(TextRun { |
| 7635 | label: format!("run {} (-S)", index + 1), |
| 7636 | command, |
| 7637 | normalized: normalize_text_artifact(&text), |
| 7638 | }); |
| 7639 | } |
| 7640 | |
| 7641 | let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 7642 | if unique_variants > 1 { |
| 7643 | let (left, right) = |
| 7644 | first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7645 | return Some(ConsistencyIssue { |
| 7646 | check: ConsistencyCheck::CliAsmReproducible, |
| 7647 | summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 7648 | repeat_count: Some(repeat_count), |
| 7649 | unique_variant_count: Some(unique_variants), |
| 7650 | varying_components: Vec::new(), |
| 7651 | stable_components: Vec::new(), |
| 7652 | detail: format!( |
| 7653 | "assembly output is not reproducible across repeated armfortas -S runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 7654 | repeat_count, |
| 7655 | unique_variants, |
| 7656 | left.command, |
| 7657 | right.command, |
| 7658 | describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 7659 | ), |
| 7660 | temp_root, |
| 7661 | }); |
| 7662 | } |
| 7663 | |
| 7664 | let _ = fs::remove_dir_all(&temp_root); |
| 7665 | None |
| 7666 | } |
| 7667 | |
| 7668 | fn run_cli_obj_reproducible( |
| 7669 | source: &Path, |
| 7670 | opt_level: OptLevel, |
| 7671 | repeat_count: usize, |
| 7672 | tools: &ToolchainConfig, |
| 7673 | ) -> Option<ConsistencyIssue> { |
| 7674 | let temp_root = next_consistency_temp_root(opt_level); |
| 7675 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7676 | return Some(ConsistencyIssue { |
| 7677 | check: ConsistencyCheck::CliObjReproducible, |
| 7678 | summary: "could not create consistency temp dir".into(), |
| 7679 | repeat_count: None, |
| 7680 | unique_variant_count: None, |
| 7681 | varying_components: Vec::new(), |
| 7682 | stable_components: Vec::new(), |
| 7683 | detail: format!( |
| 7684 | "cannot create consistency temp dir '{}': {}", |
| 7685 | temp_root.display(), |
| 7686 | err |
| 7687 | ), |
| 7688 | temp_root, |
| 7689 | }); |
| 7690 | } |
| 7691 | |
| 7692 | let mut runs = Vec::new(); |
| 7693 | for index in 0..repeat_count { |
| 7694 | let obj_path = temp_root.join(format!("run_{:02}.o", index)); |
| 7695 | let command = |
| 7696 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7697 | Ok(command) => command, |
| 7698 | Err(detail) => { |
| 7699 | return Some(ConsistencyIssue { |
| 7700 | check: ConsistencyCheck::CliObjReproducible, |
| 7701 | summary: "armfortas -c failed during reproducibility check".into(), |
| 7702 | repeat_count: None, |
| 7703 | unique_variant_count: None, |
| 7704 | varying_components: Vec::new(), |
| 7705 | stable_components: Vec::new(), |
| 7706 | detail, |
| 7707 | temp_root, |
| 7708 | }) |
| 7709 | } |
| 7710 | }; |
| 7711 | let snapshot = match object_snapshot(&obj_path, tools) { |
| 7712 | Ok(snapshot) => snapshot, |
| 7713 | Err(detail) => { |
| 7714 | return Some(ConsistencyIssue { |
| 7715 | check: ConsistencyCheck::CliObjReproducible, |
| 7716 | summary: "could not snapshot object during reproducibility check".into(), |
| 7717 | repeat_count: None, |
| 7718 | unique_variant_count: None, |
| 7719 | varying_components: Vec::new(), |
| 7720 | stable_components: Vec::new(), |
| 7721 | detail: format!("{}\n{}", command, detail), |
| 7722 | temp_root, |
| 7723 | }) |
| 7724 | } |
| 7725 | }; |
| 7726 | runs.push(ObjectRun { |
| 7727 | label: format!("run {} (-c)", index + 1), |
| 7728 | command, |
| 7729 | snapshot, |
| 7730 | }); |
| 7731 | } |
| 7732 | |
| 7733 | let rendered = runs |
| 7734 | .iter() |
| 7735 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 7736 | .collect::<Vec<_>>(); |
| 7737 | let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7738 | if unique_variants > 1 { |
| 7739 | let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 7740 | let (left, right) = |
| 7741 | first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7742 | let varying = join_or_none(&varying_object_components(&snapshots)); |
| 7743 | let stable = join_or_none(&stable_object_components(&snapshots)); |
| 7744 | let varying_components = varying_object_components(&snapshots) |
| 7745 | .into_iter() |
| 7746 | .map(str::to_string) |
| 7747 | .collect::<Vec<_>>(); |
| 7748 | let stable_components = stable_object_components(&snapshots) |
| 7749 | .into_iter() |
| 7750 | .map(str::to_string) |
| 7751 | .collect::<Vec<_>>(); |
| 7752 | return Some(ConsistencyIssue { |
| 7753 | check: ConsistencyCheck::CliObjReproducible, |
| 7754 | summary: format!( |
| 7755 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7756 | repeat_count, unique_variants, varying, stable |
| 7757 | ), |
| 7758 | repeat_count: Some(repeat_count), |
| 7759 | unique_variant_count: Some(unique_variants), |
| 7760 | varying_components, |
| 7761 | stable_components, |
| 7762 | detail: format!( |
| 7763 | "object output is not reproducible across repeated armfortas -c runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 7764 | repeat_count, |
| 7765 | unique_variants, |
| 7766 | varying, |
| 7767 | stable, |
| 7768 | left.command, |
| 7769 | right.command, |
| 7770 | describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 7771 | ), |
| 7772 | temp_root, |
| 7773 | }); |
| 7774 | } |
| 7775 | |
| 7776 | let _ = fs::remove_dir_all(&temp_root); |
| 7777 | None |
| 7778 | } |
| 7779 | |
| 7780 | fn run_cli_run_reproducible( |
| 7781 | source: &Path, |
| 7782 | opt_level: OptLevel, |
| 7783 | repeat_count: usize, |
| 7784 | tools: &ToolchainConfig, |
| 7785 | ) -> Option<ConsistencyIssue> { |
| 7786 | let temp_root = next_consistency_temp_root(opt_level); |
| 7787 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7788 | return Some(ConsistencyIssue { |
| 7789 | check: ConsistencyCheck::CliRunReproducible, |
| 7790 | summary: "could not create consistency temp dir".into(), |
| 7791 | repeat_count: None, |
| 7792 | unique_variant_count: None, |
| 7793 | varying_components: Vec::new(), |
| 7794 | stable_components: Vec::new(), |
| 7795 | detail: format!( |
| 7796 | "cannot create consistency temp dir '{}': {}", |
| 7797 | temp_root.display(), |
| 7798 | err |
| 7799 | ), |
| 7800 | temp_root, |
| 7801 | }); |
| 7802 | } |
| 7803 | |
| 7804 | let mut runs = Vec::new(); |
| 7805 | for index in 0..repeat_count { |
| 7806 | let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 7807 | let build_command = match compile_with_driver( |
| 7808 | source, |
| 7809 | opt_level, |
| 7810 | DriverEmitMode::Binary, |
| 7811 | &binary_path, |
| 7812 | tools, |
| 7813 | ) { |
| 7814 | Ok(command) => command, |
| 7815 | Err(detail) => { |
| 7816 | return Some(ConsistencyIssue { |
| 7817 | check: ConsistencyCheck::CliRunReproducible, |
| 7818 | summary: "armfortas binary build failed during runtime reproducibility check" |
| 7819 | .into(), |
| 7820 | repeat_count: None, |
| 7821 | unique_variant_count: None, |
| 7822 | varying_components: Vec::new(), |
| 7823 | stable_components: Vec::new(), |
| 7824 | detail, |
| 7825 | temp_root, |
| 7826 | }) |
| 7827 | } |
| 7828 | }; |
| 7829 | let run_command = render_binary_run_command(&binary_path); |
| 7830 | let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 7831 | Ok(run) => run, |
| 7832 | Err(detail) => { |
| 7833 | return Some(ConsistencyIssue { |
| 7834 | check: ConsistencyCheck::CliRunReproducible, |
| 7835 | summary: "armfortas binary could not run during runtime reproducibility check" |
| 7836 | .into(), |
| 7837 | repeat_count: None, |
| 7838 | unique_variant_count: None, |
| 7839 | varying_components: Vec::new(), |
| 7840 | stable_components: Vec::new(), |
| 7841 | detail, |
| 7842 | temp_root, |
| 7843 | }) |
| 7844 | } |
| 7845 | }; |
| 7846 | let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 7847 | if let Err(err) = write_behavior_run_artifacts( |
| 7848 | &temp_root, |
| 7849 | &format!("cli_run_{:02}", index), |
| 7850 | &command, |
| 7851 | &run, |
| 7852 | ) { |
| 7853 | return Some(ConsistencyIssue { |
| 7854 | check: ConsistencyCheck::CliRunReproducible, |
| 7855 | summary: "could not write cli runtime artifact".into(), |
| 7856 | repeat_count: None, |
| 7857 | unique_variant_count: None, |
| 7858 | varying_components: Vec::new(), |
| 7859 | stable_components: Vec::new(), |
| 7860 | detail: format!("cannot write cli runtime artifact: {}", err), |
| 7861 | temp_root, |
| 7862 | }); |
| 7863 | } |
| 7864 | runs.push(BehaviorRun { |
| 7865 | label: format!("cli run {}", index + 1), |
| 7866 | command, |
| 7867 | signature: normalize_run_signature(&run), |
| 7868 | run, |
| 7869 | }); |
| 7870 | } |
| 7871 | |
| 7872 | let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 7873 | if unique_variants > 1 { |
| 7874 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 7875 | let varying = varying_run_components(&signatures); |
| 7876 | let stable = stable_run_components(&signatures); |
| 7877 | let (left, right) = first_distinct_behavior_pair(&runs) |
| 7878 | .expect("unique variants > 1 implies a distinct pair"); |
| 7879 | return Some(ConsistencyIssue { |
| 7880 | check: ConsistencyCheck::CliRunReproducible, |
| 7881 | summary: format!( |
| 7882 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7883 | repeat_count, |
| 7884 | unique_variants, |
| 7885 | join_or_none(&varying), |
| 7886 | join_or_none(&stable) |
| 7887 | ), |
| 7888 | repeat_count: Some(repeat_count), |
| 7889 | unique_variant_count: Some(unique_variants), |
| 7890 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 7891 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 7892 | detail: format!( |
| 7893 | "armfortas runtime behavior is not reproducible across repeated full CLI builds\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 7894 | repeat_count, |
| 7895 | unique_variants, |
| 7896 | join_or_none(&varying), |
| 7897 | join_or_none(&stable), |
| 7898 | left.command, |
| 7899 | right.command, |
| 7900 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 7901 | ), |
| 7902 | temp_root, |
| 7903 | }); |
| 7904 | } |
| 7905 | |
| 7906 | let _ = fs::remove_dir_all(&temp_root); |
| 7907 | None |
| 7908 | } |
| 7909 | |
| 7910 | fn run_capture_asm_vs_cli_asm( |
| 7911 | source: &Path, |
| 7912 | opt_level: OptLevel, |
| 7913 | repeat_count: usize, |
| 7914 | capture_result: &CaptureResult, |
| 7915 | tools: &ToolchainConfig, |
| 7916 | ) -> Option<ConsistencyIssue> { |
| 7917 | let temp_root = next_consistency_temp_root(opt_level); |
| 7918 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7919 | return Some(ConsistencyIssue { |
| 7920 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 7921 | summary: "could not create consistency temp dir".into(), |
| 7922 | repeat_count: None, |
| 7923 | unique_variant_count: None, |
| 7924 | varying_components: Vec::new(), |
| 7925 | stable_components: Vec::new(), |
| 7926 | detail: format!( |
| 7927 | "cannot create consistency temp dir '{}': {}", |
| 7928 | temp_root.display(), |
| 7929 | err |
| 7930 | ), |
| 7931 | temp_root, |
| 7932 | }); |
| 7933 | } |
| 7934 | |
| 7935 | let capture_command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 7936 | let capture_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 7937 | Ok(text) => text, |
| 7938 | Err(detail) => { |
| 7939 | return Some(ConsistencyIssue { |
| 7940 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 7941 | summary: "capture result did not include assembly text".into(), |
| 7942 | repeat_count: None, |
| 7943 | unique_variant_count: None, |
| 7944 | varying_components: Vec::new(), |
| 7945 | stable_components: Vec::new(), |
| 7946 | detail, |
| 7947 | temp_root, |
| 7948 | }) |
| 7949 | } |
| 7950 | }; |
| 7951 | if let Err(err) = fs::write(temp_root.join("from_capture.s"), capture_text) { |
| 7952 | return Some(ConsistencyIssue { |
| 7953 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 7954 | summary: "could not write captured assembly artifact".into(), |
| 7955 | repeat_count: None, |
| 7956 | unique_variant_count: None, |
| 7957 | varying_components: Vec::new(), |
| 7958 | stable_components: Vec::new(), |
| 7959 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 7960 | temp_root, |
| 7961 | }); |
| 7962 | } |
| 7963 | let capture_normalized = normalize_text_artifact(capture_text); |
| 7964 | |
| 7965 | let mut cli_runs = Vec::new(); |
| 7966 | let mut mismatch_indices = Vec::new(); |
| 7967 | for index in 0..repeat_count { |
| 7968 | let asm_path = temp_root.join(format!("cli_run_{:02}.s", index)); |
| 7969 | let command = |
| 7970 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7971 | Ok(command) => command, |
| 7972 | Err(detail) => { |
| 7973 | return Some(ConsistencyIssue { |
| 7974 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 7975 | summary: "armfortas -S failed during capture-vs-cli consistency check" |
| 7976 | .into(), |
| 7977 | repeat_count: None, |
| 7978 | unique_variant_count: None, |
| 7979 | varying_components: Vec::new(), |
| 7980 | stable_components: Vec::new(), |
| 7981 | detail, |
| 7982 | temp_root, |
| 7983 | }) |
| 7984 | } |
| 7985 | }; |
| 7986 | let text = match read_text_artifact(&asm_path) { |
| 7987 | Ok(text) => text, |
| 7988 | Err(detail) => { |
| 7989 | return Some(ConsistencyIssue { |
| 7990 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 7991 | summary: "could not read cli assembly artifact".into(), |
| 7992 | repeat_count: None, |
| 7993 | unique_variant_count: None, |
| 7994 | varying_components: Vec::new(), |
| 7995 | stable_components: Vec::new(), |
| 7996 | detail, |
| 7997 | temp_root, |
| 7998 | }) |
| 7999 | } |
| 8000 | }; |
| 8001 | let normalized = normalize_text_artifact(&text); |
| 8002 | if normalized != capture_normalized { |
| 8003 | mismatch_indices.push(index); |
| 8004 | } |
| 8005 | cli_runs.push(TextRun { |
| 8006 | label: format!("cli run {} (-S)", index + 1), |
| 8007 | command, |
| 8008 | normalized, |
| 8009 | }); |
| 8010 | } |
| 8011 | |
| 8012 | if !mismatch_indices.is_empty() { |
| 8013 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8014 | let unique_cli_variants = |
| 8015 | count_unique_strings(cli_runs.iter().map(|run| run.normalized.as_str())); |
| 8016 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8017 | return Some(ConsistencyIssue { |
| 8018 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8019 | summary: format!( |
| 8020 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={}", |
| 8021 | repeat_count, |
| 8022 | matching_runs, |
| 8023 | mismatch_indices.len(), |
| 8024 | unique_cli_variants |
| 8025 | ), |
| 8026 | repeat_count: Some(repeat_count), |
| 8027 | unique_variant_count: Some(unique_cli_variants), |
| 8028 | varying_components: Vec::new(), |
| 8029 | stable_components: Vec::new(), |
| 8030 | detail: format!( |
| 8031 | "captured assembly does not match repeated armfortas -S runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8032 | repeat_count, |
| 8033 | matching_runs, |
| 8034 | mismatch_indices.len(), |
| 8035 | unique_cli_variants, |
| 8036 | capture_command, |
| 8037 | first_mismatch.command, |
| 8038 | describe_text_difference( |
| 8039 | &capture_normalized, |
| 8040 | &first_mismatch.normalized, |
| 8041 | "capture asm", |
| 8042 | &first_mismatch.label |
| 8043 | ) |
| 8044 | ), |
| 8045 | temp_root, |
| 8046 | }); |
| 8047 | } |
| 8048 | |
| 8049 | let _ = fs::remove_dir_all(&temp_root); |
| 8050 | None |
| 8051 | } |
| 8052 | |
| 8053 | fn run_capture_obj_vs_cli_obj( |
| 8054 | source: &Path, |
| 8055 | opt_level: OptLevel, |
| 8056 | repeat_count: usize, |
| 8057 | capture_result: &CaptureResult, |
| 8058 | tools: &ToolchainConfig, |
| 8059 | ) -> Option<ConsistencyIssue> { |
| 8060 | let temp_root = next_consistency_temp_root(opt_level); |
| 8061 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8062 | return Some(ConsistencyIssue { |
| 8063 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8064 | summary: "could not create consistency temp dir".into(), |
| 8065 | repeat_count: None, |
| 8066 | unique_variant_count: None, |
| 8067 | varying_components: Vec::new(), |
| 8068 | stable_components: Vec::new(), |
| 8069 | detail: format!( |
| 8070 | "cannot create consistency temp dir '{}': {}", |
| 8071 | temp_root.display(), |
| 8072 | err |
| 8073 | ), |
| 8074 | temp_root, |
| 8075 | }); |
| 8076 | } |
| 8077 | |
| 8078 | let capture_command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8079 | let capture_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8080 | Ok(text) => text, |
| 8081 | Err(detail) => { |
| 8082 | return Some(ConsistencyIssue { |
| 8083 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8084 | summary: "capture result did not include object snapshot text".into(), |
| 8085 | repeat_count: None, |
| 8086 | unique_variant_count: None, |
| 8087 | varying_components: Vec::new(), |
| 8088 | stable_components: Vec::new(), |
| 8089 | detail, |
| 8090 | temp_root, |
| 8091 | }) |
| 8092 | } |
| 8093 | }; |
| 8094 | if let Err(err) = fs::write(temp_root.join("from_capture.obj.txt"), capture_text) { |
| 8095 | return Some(ConsistencyIssue { |
| 8096 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8097 | summary: "could not write captured object snapshot artifact".into(), |
| 8098 | repeat_count: None, |
| 8099 | unique_variant_count: None, |
| 8100 | varying_components: Vec::new(), |
| 8101 | stable_components: Vec::new(), |
| 8102 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8103 | temp_root, |
| 8104 | }); |
| 8105 | } |
| 8106 | let capture_snapshot = match parse_object_snapshot_text(capture_text) { |
| 8107 | Ok(snapshot) => snapshot, |
| 8108 | Err(detail) => { |
| 8109 | return Some(ConsistencyIssue { |
| 8110 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8111 | summary: "captured object snapshot had an unexpected format".into(), |
| 8112 | repeat_count: None, |
| 8113 | unique_variant_count: None, |
| 8114 | varying_components: Vec::new(), |
| 8115 | stable_components: Vec::new(), |
| 8116 | detail, |
| 8117 | temp_root, |
| 8118 | }) |
| 8119 | } |
| 8120 | }; |
| 8121 | |
| 8122 | let mut cli_runs = Vec::new(); |
| 8123 | let mut mismatch_indices = Vec::new(); |
| 8124 | for index in 0..repeat_count { |
| 8125 | let obj_path = temp_root.join(format!("cli_run_{:02}.o", index)); |
| 8126 | let command = |
| 8127 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 8128 | Ok(command) => command, |
| 8129 | Err(detail) => { |
| 8130 | return Some(ConsistencyIssue { |
| 8131 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8132 | summary: "armfortas -c failed during capture-vs-cli consistency check" |
| 8133 | .into(), |
| 8134 | repeat_count: None, |
| 8135 | unique_variant_count: None, |
| 8136 | varying_components: Vec::new(), |
| 8137 | stable_components: Vec::new(), |
| 8138 | detail, |
| 8139 | temp_root, |
| 8140 | }) |
| 8141 | } |
| 8142 | }; |
| 8143 | let snapshot = match object_snapshot(&obj_path, tools) { |
| 8144 | Ok(snapshot) => snapshot, |
| 8145 | Err(detail) => { |
| 8146 | return Some(ConsistencyIssue { |
| 8147 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8148 | summary: "could not snapshot cli object artifact".into(), |
| 8149 | repeat_count: None, |
| 8150 | unique_variant_count: None, |
| 8151 | varying_components: Vec::new(), |
| 8152 | stable_components: Vec::new(), |
| 8153 | detail: format!("{}\n{}", command, detail), |
| 8154 | temp_root, |
| 8155 | }) |
| 8156 | } |
| 8157 | }; |
| 8158 | if let Err(err) = fs::write( |
| 8159 | temp_root.join(format!("cli_run_{:02}.obj.txt", index)), |
| 8160 | render_object_snapshot(&snapshot), |
| 8161 | ) { |
| 8162 | return Some(ConsistencyIssue { |
| 8163 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8164 | summary: "could not write cli object snapshot artifact".into(), |
| 8165 | repeat_count: None, |
| 8166 | unique_variant_count: None, |
| 8167 | varying_components: Vec::new(), |
| 8168 | stable_components: Vec::new(), |
| 8169 | detail: format!("cannot write cli object snapshot artifact: {}", err), |
| 8170 | temp_root, |
| 8171 | }); |
| 8172 | } |
| 8173 | if snapshot != capture_snapshot { |
| 8174 | mismatch_indices.push(index); |
| 8175 | } |
| 8176 | cli_runs.push(ObjectRun { |
| 8177 | label: format!("cli run {} (-c)", index + 1), |
| 8178 | command, |
| 8179 | snapshot, |
| 8180 | }); |
| 8181 | } |
| 8182 | |
| 8183 | if !mismatch_indices.is_empty() { |
| 8184 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8185 | let rendered = cli_runs |
| 8186 | .iter() |
| 8187 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 8188 | .collect::<Vec<_>>(); |
| 8189 | let unique_cli_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8190 | let mismatch_snapshots = mismatch_indices |
| 8191 | .iter() |
| 8192 | .map(|index| &cli_runs[*index].snapshot) |
| 8193 | .collect::<Vec<_>>(); |
| 8194 | let mut summary_snapshots = vec![&capture_snapshot]; |
| 8195 | summary_snapshots.extend(mismatch_snapshots.iter().copied()); |
| 8196 | let varying = varying_object_components(&summary_snapshots) |
| 8197 | .into_iter() |
| 8198 | .map(str::to_string) |
| 8199 | .collect::<Vec<_>>(); |
| 8200 | let stable = stable_object_components(&summary_snapshots) |
| 8201 | .into_iter() |
| 8202 | .map(str::to_string) |
| 8203 | .collect::<Vec<_>>(); |
| 8204 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8205 | return Some(ConsistencyIssue { |
| 8206 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8207 | summary: format!( |
| 8208 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8209 | repeat_count, |
| 8210 | matching_runs, |
| 8211 | mismatch_indices.len(), |
| 8212 | unique_cli_variants, |
| 8213 | join_or_none_from_strings(&varying), |
| 8214 | join_or_none_from_strings(&stable) |
| 8215 | ), |
| 8216 | repeat_count: Some(repeat_count), |
| 8217 | unique_variant_count: Some(unique_cli_variants), |
| 8218 | varying_components: varying, |
| 8219 | stable_components: stable, |
| 8220 | detail: format!( |
| 8221 | "captured object snapshot does not match repeated armfortas -c runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8222 | repeat_count, |
| 8223 | matching_runs, |
| 8224 | mismatch_indices.len(), |
| 8225 | unique_cli_variants, |
| 8226 | capture_command, |
| 8227 | first_mismatch.command, |
| 8228 | describe_object_difference( |
| 8229 | &capture_snapshot, |
| 8230 | &first_mismatch.snapshot, |
| 8231 | "capture obj", |
| 8232 | &first_mismatch.label |
| 8233 | ) |
| 8234 | ), |
| 8235 | temp_root, |
| 8236 | }); |
| 8237 | } |
| 8238 | |
| 8239 | let _ = fs::remove_dir_all(&temp_root); |
| 8240 | None |
| 8241 | } |
| 8242 | |
| 8243 | fn run_capture_run_vs_cli_run( |
| 8244 | source: &Path, |
| 8245 | opt_level: OptLevel, |
| 8246 | repeat_count: usize, |
| 8247 | capture_result: &CaptureResult, |
| 8248 | tools: &ToolchainConfig, |
| 8249 | ) -> Option<ConsistencyIssue> { |
| 8250 | let temp_root = next_consistency_temp_root(opt_level); |
| 8251 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8252 | return Some(ConsistencyIssue { |
| 8253 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8254 | summary: "could not create consistency temp dir".into(), |
| 8255 | repeat_count: None, |
| 8256 | unique_variant_count: None, |
| 8257 | varying_components: Vec::new(), |
| 8258 | stable_components: Vec::new(), |
| 8259 | detail: format!( |
| 8260 | "cannot create consistency temp dir '{}': {}", |
| 8261 | temp_root.display(), |
| 8262 | err |
| 8263 | ), |
| 8264 | temp_root, |
| 8265 | }); |
| 8266 | } |
| 8267 | |
| 8268 | let capture_command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8269 | let capture_run = match capture_run_stage(capture_result) { |
| 8270 | Ok(run) => run.clone(), |
| 8271 | Err(detail) => { |
| 8272 | return Some(ConsistencyIssue { |
| 8273 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8274 | summary: "capture result did not include runtime behavior".into(), |
| 8275 | repeat_count: None, |
| 8276 | unique_variant_count: None, |
| 8277 | varying_components: Vec::new(), |
| 8278 | stable_components: Vec::new(), |
| 8279 | detail, |
| 8280 | temp_root, |
| 8281 | }) |
| 8282 | } |
| 8283 | }; |
| 8284 | if let Err(err) = |
| 8285 | write_behavior_run_artifacts(&temp_root, "from_capture", &capture_command, &capture_run) |
| 8286 | { |
| 8287 | return Some(ConsistencyIssue { |
| 8288 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8289 | summary: "could not write captured runtime artifact".into(), |
| 8290 | repeat_count: None, |
| 8291 | unique_variant_count: None, |
| 8292 | varying_components: Vec::new(), |
| 8293 | stable_components: Vec::new(), |
| 8294 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 8295 | temp_root, |
| 8296 | }); |
| 8297 | } |
| 8298 | let capture_signature = normalize_run_signature(&capture_run); |
| 8299 | |
| 8300 | let mut cli_runs = Vec::new(); |
| 8301 | let mut mismatch_indices = Vec::new(); |
| 8302 | for index in 0..repeat_count { |
| 8303 | let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 8304 | let build_command = match compile_with_driver( |
| 8305 | source, |
| 8306 | opt_level, |
| 8307 | DriverEmitMode::Binary, |
| 8308 | &binary_path, |
| 8309 | tools, |
| 8310 | ) { |
| 8311 | Ok(command) => command, |
| 8312 | Err(detail) => { |
| 8313 | return Some(ConsistencyIssue { |
| 8314 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8315 | summary: "armfortas binary build failed during capture-vs-cli runtime check" |
| 8316 | .into(), |
| 8317 | repeat_count: None, |
| 8318 | unique_variant_count: None, |
| 8319 | varying_components: Vec::new(), |
| 8320 | stable_components: Vec::new(), |
| 8321 | detail, |
| 8322 | temp_root, |
| 8323 | }) |
| 8324 | } |
| 8325 | }; |
| 8326 | let run_command = render_binary_run_command(&binary_path); |
| 8327 | let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 8328 | Ok(run) => run, |
| 8329 | Err(detail) => { |
| 8330 | return Some(ConsistencyIssue { |
| 8331 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8332 | summary: "armfortas binary could not run during capture-vs-cli runtime check" |
| 8333 | .into(), |
| 8334 | repeat_count: None, |
| 8335 | unique_variant_count: None, |
| 8336 | varying_components: Vec::new(), |
| 8337 | stable_components: Vec::new(), |
| 8338 | detail, |
| 8339 | temp_root, |
| 8340 | }) |
| 8341 | } |
| 8342 | }; |
| 8343 | let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 8344 | if let Err(err) = write_behavior_run_artifacts( |
| 8345 | &temp_root, |
| 8346 | &format!("cli_run_{:02}", index), |
| 8347 | &command, |
| 8348 | &run, |
| 8349 | ) { |
| 8350 | return Some(ConsistencyIssue { |
| 8351 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8352 | summary: "could not write cli runtime artifact".into(), |
| 8353 | repeat_count: None, |
| 8354 | unique_variant_count: None, |
| 8355 | varying_components: Vec::new(), |
| 8356 | stable_components: Vec::new(), |
| 8357 | detail: format!("cannot write cli runtime artifact: {}", err), |
| 8358 | temp_root, |
| 8359 | }); |
| 8360 | } |
| 8361 | if normalize_run_signature(&run) != capture_signature { |
| 8362 | mismatch_indices.push(index); |
| 8363 | } |
| 8364 | cli_runs.push(BehaviorRun { |
| 8365 | label: format!("cli run {}", index + 1), |
| 8366 | command, |
| 8367 | signature: normalize_run_signature(&run), |
| 8368 | run, |
| 8369 | }); |
| 8370 | } |
| 8371 | |
| 8372 | if !mismatch_indices.is_empty() { |
| 8373 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8374 | let unique_cli_variants = |
| 8375 | count_unique_run_signatures(cli_runs.iter().map(|run| &run.signature)); |
| 8376 | let mismatch_signatures = mismatch_indices |
| 8377 | .iter() |
| 8378 | .map(|index| &cli_runs[*index].signature) |
| 8379 | .collect::<Vec<_>>(); |
| 8380 | let mut summary_signatures = vec![&capture_signature]; |
| 8381 | summary_signatures.extend(mismatch_signatures.iter().copied()); |
| 8382 | let varying = varying_run_components(&summary_signatures) |
| 8383 | .into_iter() |
| 8384 | .map(str::to_string) |
| 8385 | .collect::<Vec<_>>(); |
| 8386 | let stable = stable_run_components(&summary_signatures) |
| 8387 | .into_iter() |
| 8388 | .map(str::to_string) |
| 8389 | .collect::<Vec<_>>(); |
| 8390 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8391 | return Some(ConsistencyIssue { |
| 8392 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8393 | summary: format!( |
| 8394 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8395 | repeat_count, |
| 8396 | matching_runs, |
| 8397 | mismatch_indices.len(), |
| 8398 | unique_cli_variants, |
| 8399 | join_or_none_from_strings(&varying), |
| 8400 | join_or_none_from_strings(&stable) |
| 8401 | ), |
| 8402 | repeat_count: Some(repeat_count), |
| 8403 | unique_variant_count: Some(unique_cli_variants), |
| 8404 | varying_components: varying, |
| 8405 | stable_components: stable, |
| 8406 | detail: format!( |
| 8407 | "captured runtime behavior does not match repeated full CLI builds\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8408 | repeat_count, |
| 8409 | matching_runs, |
| 8410 | mismatch_indices.len(), |
| 8411 | unique_cli_variants, |
| 8412 | capture_command, |
| 8413 | first_mismatch.command, |
| 8414 | describe_run_difference( |
| 8415 | &capture_run, |
| 8416 | &first_mismatch.run, |
| 8417 | "capture run", |
| 8418 | &first_mismatch.label |
| 8419 | ) |
| 8420 | ), |
| 8421 | temp_root, |
| 8422 | }); |
| 8423 | } |
| 8424 | |
| 8425 | let _ = fs::remove_dir_all(&temp_root); |
| 8426 | None |
| 8427 | } |
| 8428 | |
| 8429 | fn run_capture_asm_reproducible( |
| 8430 | source: &Path, |
| 8431 | opt_level: OptLevel, |
| 8432 | repeat_count: usize, |
| 8433 | capture_result: &CaptureResult, |
| 8434 | tools: &ToolchainConfig, |
| 8435 | ) -> Option<ConsistencyIssue> { |
| 8436 | let temp_root = next_consistency_temp_root(opt_level); |
| 8437 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8438 | return Some(ConsistencyIssue { |
| 8439 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8440 | summary: "could not create consistency temp dir".into(), |
| 8441 | repeat_count: None, |
| 8442 | unique_variant_count: None, |
| 8443 | varying_components: Vec::new(), |
| 8444 | stable_components: Vec::new(), |
| 8445 | detail: format!( |
| 8446 | "cannot create consistency temp dir '{}': {}", |
| 8447 | temp_root.display(), |
| 8448 | err |
| 8449 | ), |
| 8450 | temp_root, |
| 8451 | }); |
| 8452 | } |
| 8453 | |
| 8454 | let mut runs = Vec::new(); |
| 8455 | let command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 8456 | let initial_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 8457 | Ok(text) => text, |
| 8458 | Err(detail) => { |
| 8459 | return Some(ConsistencyIssue { |
| 8460 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8461 | summary: "initial capture result did not include assembly text".into(), |
| 8462 | repeat_count: None, |
| 8463 | unique_variant_count: None, |
| 8464 | varying_components: Vec::new(), |
| 8465 | stable_components: Vec::new(), |
| 8466 | detail, |
| 8467 | temp_root, |
| 8468 | }) |
| 8469 | } |
| 8470 | }; |
| 8471 | if let Err(err) = fs::write(temp_root.join("capture_run_00.s"), initial_text) { |
| 8472 | return Some(ConsistencyIssue { |
| 8473 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8474 | summary: "could not write captured assembly artifact".into(), |
| 8475 | repeat_count: None, |
| 8476 | unique_variant_count: None, |
| 8477 | varying_components: Vec::new(), |
| 8478 | stable_components: Vec::new(), |
| 8479 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 8480 | temp_root, |
| 8481 | }); |
| 8482 | } |
| 8483 | runs.push(TextRun { |
| 8484 | label: "capture run 1".into(), |
| 8485 | command: command.clone(), |
| 8486 | normalized: normalize_text_artifact(initial_text), |
| 8487 | }); |
| 8488 | |
| 8489 | for index in 1..repeat_count { |
| 8490 | let text = match capture_text_from_testing(source, opt_level, Stage::Asm, tools) { |
| 8491 | Ok(text) => text, |
| 8492 | Err(detail) => { |
| 8493 | return Some(ConsistencyIssue { |
| 8494 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8495 | summary: "armfortas::testing capture failed during asm reproducibility check" |
| 8496 | .into(), |
| 8497 | repeat_count: None, |
| 8498 | unique_variant_count: None, |
| 8499 | varying_components: Vec::new(), |
| 8500 | stable_components: Vec::new(), |
| 8501 | detail, |
| 8502 | temp_root, |
| 8503 | }) |
| 8504 | } |
| 8505 | }; |
| 8506 | if let Err(err) = fs::write(temp_root.join(format!("capture_run_{:02}.s", index)), &text) { |
| 8507 | return Some(ConsistencyIssue { |
| 8508 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8509 | summary: "could not write captured assembly artifact".into(), |
| 8510 | repeat_count: None, |
| 8511 | unique_variant_count: None, |
| 8512 | varying_components: Vec::new(), |
| 8513 | stable_components: Vec::new(), |
| 8514 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 8515 | temp_root, |
| 8516 | }); |
| 8517 | } |
| 8518 | runs.push(TextRun { |
| 8519 | label: format!("capture run {}", index + 1), |
| 8520 | command: command.clone(), |
| 8521 | normalized: normalize_text_artifact(&text), |
| 8522 | }); |
| 8523 | } |
| 8524 | |
| 8525 | let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 8526 | if unique_variants > 1 { |
| 8527 | let (left, right) = |
| 8528 | first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8529 | return Some(ConsistencyIssue { |
| 8530 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8531 | summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 8532 | repeat_count: Some(repeat_count), |
| 8533 | unique_variant_count: Some(unique_variants), |
| 8534 | varying_components: Vec::new(), |
| 8535 | stable_components: Vec::new(), |
| 8536 | detail: format!( |
| 8537 | "captured assembly is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 8538 | repeat_count, |
| 8539 | unique_variants, |
| 8540 | left.command, |
| 8541 | right.command, |
| 8542 | describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 8543 | ), |
| 8544 | temp_root, |
| 8545 | }); |
| 8546 | } |
| 8547 | |
| 8548 | let _ = fs::remove_dir_all(&temp_root); |
| 8549 | None |
| 8550 | } |
| 8551 | |
| 8552 | fn run_capture_obj_reproducible( |
| 8553 | source: &Path, |
| 8554 | opt_level: OptLevel, |
| 8555 | repeat_count: usize, |
| 8556 | capture_result: &CaptureResult, |
| 8557 | tools: &ToolchainConfig, |
| 8558 | ) -> Option<ConsistencyIssue> { |
| 8559 | let temp_root = next_consistency_temp_root(opt_level); |
| 8560 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8561 | return Some(ConsistencyIssue { |
| 8562 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8563 | summary: "could not create consistency temp dir".into(), |
| 8564 | repeat_count: None, |
| 8565 | unique_variant_count: None, |
| 8566 | varying_components: Vec::new(), |
| 8567 | stable_components: Vec::new(), |
| 8568 | detail: format!( |
| 8569 | "cannot create consistency temp dir '{}': {}", |
| 8570 | temp_root.display(), |
| 8571 | err |
| 8572 | ), |
| 8573 | temp_root, |
| 8574 | }); |
| 8575 | } |
| 8576 | |
| 8577 | let command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8578 | let initial_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8579 | Ok(text) => text, |
| 8580 | Err(detail) => { |
| 8581 | return Some(ConsistencyIssue { |
| 8582 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8583 | summary: "initial capture result did not include object snapshot text".into(), |
| 8584 | repeat_count: None, |
| 8585 | unique_variant_count: None, |
| 8586 | varying_components: Vec::new(), |
| 8587 | stable_components: Vec::new(), |
| 8588 | detail, |
| 8589 | temp_root, |
| 8590 | }) |
| 8591 | } |
| 8592 | }; |
| 8593 | let initial_snapshot = match parse_object_snapshot_text(initial_text) { |
| 8594 | Ok(snapshot) => snapshot, |
| 8595 | Err(detail) => { |
| 8596 | return Some(ConsistencyIssue { |
| 8597 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8598 | summary: "captured object snapshot had an unexpected format".into(), |
| 8599 | repeat_count: None, |
| 8600 | unique_variant_count: None, |
| 8601 | varying_components: Vec::new(), |
| 8602 | stable_components: Vec::new(), |
| 8603 | detail, |
| 8604 | temp_root, |
| 8605 | }) |
| 8606 | } |
| 8607 | }; |
| 8608 | if let Err(err) = fs::write(temp_root.join("capture_run_00.obj.txt"), initial_text) { |
| 8609 | return Some(ConsistencyIssue { |
| 8610 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8611 | summary: "could not write captured object snapshot artifact".into(), |
| 8612 | repeat_count: None, |
| 8613 | unique_variant_count: None, |
| 8614 | varying_components: Vec::new(), |
| 8615 | stable_components: Vec::new(), |
| 8616 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8617 | temp_root, |
| 8618 | }); |
| 8619 | } |
| 8620 | |
| 8621 | let mut runs = vec![ObjectRun { |
| 8622 | label: "capture run 1".into(), |
| 8623 | command: command.clone(), |
| 8624 | snapshot: initial_snapshot, |
| 8625 | }]; |
| 8626 | |
| 8627 | for index in 1..repeat_count { |
| 8628 | let text = match capture_text_from_testing(source, opt_level, Stage::Obj, tools) { |
| 8629 | Ok(text) => text, |
| 8630 | Err(detail) => { |
| 8631 | return Some(ConsistencyIssue { |
| 8632 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8633 | summary: "armfortas::testing capture failed during obj reproducibility check" |
| 8634 | .into(), |
| 8635 | repeat_count: None, |
| 8636 | unique_variant_count: None, |
| 8637 | varying_components: Vec::new(), |
| 8638 | stable_components: Vec::new(), |
| 8639 | detail, |
| 8640 | temp_root, |
| 8641 | }) |
| 8642 | } |
| 8643 | }; |
| 8644 | if let Err(err) = fs::write( |
| 8645 | temp_root.join(format!("capture_run_{:02}.obj.txt", index)), |
| 8646 | &text, |
| 8647 | ) { |
| 8648 | return Some(ConsistencyIssue { |
| 8649 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8650 | summary: "could not write captured object snapshot artifact".into(), |
| 8651 | repeat_count: None, |
| 8652 | unique_variant_count: None, |
| 8653 | varying_components: Vec::new(), |
| 8654 | stable_components: Vec::new(), |
| 8655 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8656 | temp_root, |
| 8657 | }); |
| 8658 | } |
| 8659 | let snapshot = match parse_object_snapshot_text(&text) { |
| 8660 | Ok(snapshot) => snapshot, |
| 8661 | Err(detail) => { |
| 8662 | return Some(ConsistencyIssue { |
| 8663 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8664 | summary: "captured object snapshot had an unexpected format".into(), |
| 8665 | repeat_count: None, |
| 8666 | unique_variant_count: None, |
| 8667 | varying_components: Vec::new(), |
| 8668 | stable_components: Vec::new(), |
| 8669 | detail, |
| 8670 | temp_root, |
| 8671 | }) |
| 8672 | } |
| 8673 | }; |
| 8674 | runs.push(ObjectRun { |
| 8675 | label: format!("capture run {}", index + 1), |
| 8676 | command: command.clone(), |
| 8677 | snapshot, |
| 8678 | }); |
| 8679 | } |
| 8680 | |
| 8681 | let rendered = runs |
| 8682 | .iter() |
| 8683 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 8684 | .collect::<Vec<_>>(); |
| 8685 | let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8686 | if unique_variants > 1 { |
| 8687 | let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 8688 | let (left, right) = |
| 8689 | first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8690 | let varying = varying_object_components(&snapshots); |
| 8691 | let stable = stable_object_components(&snapshots); |
| 8692 | return Some(ConsistencyIssue { |
| 8693 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8694 | summary: format!( |
| 8695 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8696 | repeat_count, |
| 8697 | unique_variants, |
| 8698 | join_or_none(&varying), |
| 8699 | join_or_none(&stable) |
| 8700 | ), |
| 8701 | repeat_count: Some(repeat_count), |
| 8702 | unique_variant_count: Some(unique_variants), |
| 8703 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8704 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8705 | detail: format!( |
| 8706 | "captured object snapshots are not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8707 | repeat_count, |
| 8708 | unique_variants, |
| 8709 | join_or_none(&varying), |
| 8710 | join_or_none(&stable), |
| 8711 | left.command, |
| 8712 | right.command, |
| 8713 | describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 8714 | ), |
| 8715 | temp_root, |
| 8716 | }); |
| 8717 | } |
| 8718 | |
| 8719 | let _ = fs::remove_dir_all(&temp_root); |
| 8720 | None |
| 8721 | } |
| 8722 | |
| 8723 | fn run_capture_run_reproducible( |
| 8724 | source: &Path, |
| 8725 | opt_level: OptLevel, |
| 8726 | repeat_count: usize, |
| 8727 | capture_result: &CaptureResult, |
| 8728 | tools: &ToolchainConfig, |
| 8729 | ) -> Option<ConsistencyIssue> { |
| 8730 | let temp_root = next_consistency_temp_root(opt_level); |
| 8731 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8732 | return Some(ConsistencyIssue { |
| 8733 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8734 | summary: "could not create consistency temp dir".into(), |
| 8735 | repeat_count: None, |
| 8736 | unique_variant_count: None, |
| 8737 | varying_components: Vec::new(), |
| 8738 | stable_components: Vec::new(), |
| 8739 | detail: format!( |
| 8740 | "cannot create consistency temp dir '{}': {}", |
| 8741 | temp_root.display(), |
| 8742 | err |
| 8743 | ), |
| 8744 | temp_root, |
| 8745 | }); |
| 8746 | } |
| 8747 | |
| 8748 | let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8749 | let initial_run = match capture_run_stage(capture_result) { |
| 8750 | Ok(run) => run.clone(), |
| 8751 | Err(detail) => { |
| 8752 | return Some(ConsistencyIssue { |
| 8753 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8754 | summary: "initial capture result did not include runtime behavior".into(), |
| 8755 | repeat_count: None, |
| 8756 | unique_variant_count: None, |
| 8757 | varying_components: Vec::new(), |
| 8758 | stable_components: Vec::new(), |
| 8759 | detail, |
| 8760 | temp_root, |
| 8761 | }) |
| 8762 | } |
| 8763 | }; |
| 8764 | if let Err(err) = |
| 8765 | write_behavior_run_artifacts(&temp_root, "capture_run_00", &command, &initial_run) |
| 8766 | { |
| 8767 | return Some(ConsistencyIssue { |
| 8768 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8769 | summary: "could not write captured runtime artifact".into(), |
| 8770 | repeat_count: None, |
| 8771 | unique_variant_count: None, |
| 8772 | varying_components: Vec::new(), |
| 8773 | stable_components: Vec::new(), |
| 8774 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 8775 | temp_root, |
| 8776 | }); |
| 8777 | } |
| 8778 | let mut runs = vec![BehaviorRun { |
| 8779 | label: "capture run 1".into(), |
| 8780 | command: command.clone(), |
| 8781 | signature: normalize_run_signature(&initial_run), |
| 8782 | run: initial_run, |
| 8783 | }]; |
| 8784 | |
| 8785 | for index in 1..repeat_count { |
| 8786 | let run = match capture_run_from_testing(source, opt_level, tools) { |
| 8787 | Ok(run) => run, |
| 8788 | Err(detail) => { |
| 8789 | return Some(ConsistencyIssue { |
| 8790 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8791 | summary: |
| 8792 | "armfortas::testing capture failed during runtime reproducibility check" |
| 8793 | .into(), |
| 8794 | repeat_count: None, |
| 8795 | unique_variant_count: None, |
| 8796 | varying_components: Vec::new(), |
| 8797 | stable_components: Vec::new(), |
| 8798 | detail, |
| 8799 | temp_root, |
| 8800 | }) |
| 8801 | } |
| 8802 | }; |
| 8803 | if let Err(err) = write_behavior_run_artifacts( |
| 8804 | &temp_root, |
| 8805 | &format!("capture_run_{:02}", index), |
| 8806 | &command, |
| 8807 | &run, |
| 8808 | ) { |
| 8809 | return Some(ConsistencyIssue { |
| 8810 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8811 | summary: "could not write captured runtime artifact".into(), |
| 8812 | repeat_count: None, |
| 8813 | unique_variant_count: None, |
| 8814 | varying_components: Vec::new(), |
| 8815 | stable_components: Vec::new(), |
| 8816 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 8817 | temp_root, |
| 8818 | }); |
| 8819 | } |
| 8820 | runs.push(BehaviorRun { |
| 8821 | label: format!("capture run {}", index + 1), |
| 8822 | command: command.clone(), |
| 8823 | signature: normalize_run_signature(&run), |
| 8824 | run, |
| 8825 | }); |
| 8826 | } |
| 8827 | |
| 8828 | let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 8829 | if unique_variants > 1 { |
| 8830 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 8831 | let varying = varying_run_components(&signatures); |
| 8832 | let stable = stable_run_components(&signatures); |
| 8833 | let (left, right) = first_distinct_behavior_pair(&runs) |
| 8834 | .expect("unique variants > 1 implies a distinct pair"); |
| 8835 | return Some(ConsistencyIssue { |
| 8836 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8837 | summary: format!( |
| 8838 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8839 | repeat_count, |
| 8840 | unique_variants, |
| 8841 | join_or_none(&varying), |
| 8842 | join_or_none(&stable) |
| 8843 | ), |
| 8844 | repeat_count: Some(repeat_count), |
| 8845 | unique_variant_count: Some(unique_variants), |
| 8846 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8847 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8848 | detail: format!( |
| 8849 | "captured runtime behavior is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8850 | repeat_count, |
| 8851 | unique_variants, |
| 8852 | join_or_none(&varying), |
| 8853 | join_or_none(&stable), |
| 8854 | left.command, |
| 8855 | right.command, |
| 8856 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 8857 | ), |
| 8858 | temp_root, |
| 8859 | }); |
| 8860 | } |
| 8861 | |
| 8862 | let _ = fs::remove_dir_all(&temp_root); |
| 8863 | None |
| 8864 | } |
| 8865 | |
| 8866 | fn run_reference_compilers( |
| 8867 | prepared: &PreparedInput, |
| 8868 | case: &CaseSpec, |
| 8869 | opt_level: OptLevel, |
| 8870 | tools: &ToolchainConfig, |
| 8871 | ) -> Vec<ReferenceResult> { |
| 8872 | case.reference_compilers |
| 8873 | .iter() |
| 8874 | .copied() |
| 8875 | .map(|compiler| run_reference_case(&prepared.compiler_source, opt_level, compiler, tools)) |
| 8876 | .collect() |
| 8877 | } |
| 8878 | |
| 8879 | fn run_reference_case( |
| 8880 | source: &Path, |
| 8881 | opt_level: OptLevel, |
| 8882 | compiler: ReferenceCompiler, |
| 8883 | tools: &ToolchainConfig, |
| 8884 | ) -> ReferenceResult { |
| 8885 | let temp_root = next_report_temp_root(compiler, opt_level); |
| 8886 | let binary = temp_root.join("reference.out"); |
| 8887 | let uses_cpp = source_uses_cpp(source); |
| 8888 | |
| 8889 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 8890 | if uses_cpp { |
| 8891 | args.push("-cpp".to_string()); |
| 8892 | } |
| 8893 | args.push(source.display().to_string()); |
| 8894 | args.push("-o".to_string()); |
| 8895 | args.push(binary.display().to_string()); |
| 8896 | |
| 8897 | let compiler_bin = tools.reference_binary(compiler); |
| 8898 | let command_string = render_command(compiler_bin, &args); |
| 8899 | |
| 8900 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8901 | return ReferenceResult::infrastructure_error( |
| 8902 | compiler, |
| 8903 | command_string, |
| 8904 | format!("cannot create temp dir '{}': {}", temp_root.display(), err), |
| 8905 | ); |
| 8906 | } |
| 8907 | |
| 8908 | let compile = match Command::new(compiler_bin) |
| 8909 | .current_dir(&temp_root) |
| 8910 | .args(&args) |
| 8911 | .output() |
| 8912 | { |
| 8913 | Ok(output) => output, |
| 8914 | Err(err) => { |
| 8915 | return ReferenceResult::infrastructure_error( |
| 8916 | compiler, |
| 8917 | command_string, |
| 8918 | format!("cannot run {}: {}", compiler_bin, err), |
| 8919 | ); |
| 8920 | } |
| 8921 | }; |
| 8922 | |
| 8923 | let mut result = ReferenceResult { |
| 8924 | compiler, |
| 8925 | compile_command: command_string, |
| 8926 | compile_exit_code: compile.status.code().unwrap_or(-1), |
| 8927 | compile_stdout: String::from_utf8_lossy(&compile.stdout).into_owned(), |
| 8928 | compile_stderr: String::from_utf8_lossy(&compile.stderr).into_owned(), |
| 8929 | run: None, |
| 8930 | run_error: None, |
| 8931 | }; |
| 8932 | |
| 8933 | if compile.status.success() { |
| 8934 | match Command::new(&binary).current_dir(&temp_root).output() { |
| 8935 | Ok(output) => { |
| 8936 | result.run = Some(RunCapture { |
| 8937 | exit_code: output.status.code().unwrap_or(-1), |
| 8938 | stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 8939 | stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 8940 | }); |
| 8941 | } |
| 8942 | Err(err) => { |
| 8943 | result.run_error = Some(format!("cannot run '{}': {}", binary.display(), err)); |
| 8944 | } |
| 8945 | } |
| 8946 | } |
| 8947 | |
| 8948 | let _ = fs::remove_dir_all(&temp_root); |
| 8949 | result |
| 8950 | } |
| 8951 | |
| 8952 | fn source_uses_cpp(source: &Path) -> bool { |
| 8953 | fs::read_to_string(source) |
| 8954 | .map(|text| text.lines().any(|line| line.trim_start().starts_with('#'))) |
| 8955 | .unwrap_or(false) |
| 8956 | } |
| 8957 | |
| 8958 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 8959 | enum DriverEmitMode { |
| 8960 | Asm, |
| 8961 | Obj, |
| 8962 | Binary, |
| 8963 | } |
| 8964 | |
| 8965 | fn compile_with_driver( |
| 8966 | source: &Path, |
| 8967 | opt_level: OptLevel, |
| 8968 | mode: DriverEmitMode, |
| 8969 | output: &Path, |
| 8970 | tools: &ToolchainConfig, |
| 8971 | ) -> Result<String, String> { |
| 8972 | let command = render_armfortas_command(source, opt_level, mode, output, tools); |
| 8973 | let emit_mode = match mode { |
| 8974 | DriverEmitMode::Asm => EmitMode::Asm, |
| 8975 | DriverEmitMode::Obj => EmitMode::Obj, |
| 8976 | DriverEmitMode::Binary => EmitMode::Binary, |
| 8977 | }; |
| 8978 | tools |
| 8979 | .armfortas_adapters() |
| 8980 | .compile_output(source, opt_level, emit_mode, output) |
| 8981 | .map_err(|detail| format!("{} failed:\n{}", command, detail))?; |
| 8982 | Ok(command) |
| 8983 | } |
| 8984 | |
| 8985 | fn render_armfortas_command( |
| 8986 | source: &Path, |
| 8987 | opt_level: OptLevel, |
| 8988 | mode: DriverEmitMode, |
| 8989 | output: &Path, |
| 8990 | tools: &ToolchainConfig, |
| 8991 | ) -> String { |
| 8992 | let armfortas = tools.armfortas_adapters(); |
| 8993 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 8994 | match mode { |
| 8995 | DriverEmitMode::Asm => args.push("-S".to_string()), |
| 8996 | DriverEmitMode::Obj => args.push("-c".to_string()), |
| 8997 | DriverEmitMode::Binary => {} |
| 8998 | } |
| 8999 | args.push(source.display().to_string()); |
| 9000 | args.push("-o".to_string()); |
| 9001 | args.push(output.display().to_string()); |
| 9002 | render_command(armfortas.cli_command_name(), &args) |
| 9003 | } |
| 9004 | |
| 9005 | fn render_binary_run_command(binary: &Path) -> String { |
| 9006 | render_command(&binary.display().to_string(), &[]) |
| 9007 | } |
| 9008 | |
| 9009 | fn render_capture_command( |
| 9010 | source: &Path, |
| 9011 | opt_level: OptLevel, |
| 9012 | stage: Stage, |
| 9013 | tools: &ToolchainConfig, |
| 9014 | ) -> String { |
| 9015 | let armfortas = tools.armfortas_adapters(); |
| 9016 | format!( |
| 9017 | "{} {} --stage {} {}", |
| 9018 | armfortas.capture_command_name(), |
| 9019 | opt_level.as_flag(), |
| 9020 | stage.as_str(), |
| 9021 | quote_arg(&source.display().to_string()), |
| 9022 | ) |
| 9023 | } |
| 9024 | |
| 9025 | fn capture_text_from_testing( |
| 9026 | source: &Path, |
| 9027 | opt_level: OptLevel, |
| 9028 | stage: Stage, |
| 9029 | tools: &ToolchainConfig, |
| 9030 | ) -> Result<String, String> { |
| 9031 | let command = render_capture_command(source, opt_level, stage, tools); |
| 9032 | let request = CaptureRequest { |
| 9033 | input: source.to_path_buf(), |
| 9034 | requested: BTreeSet::from([stage]), |
| 9035 | opt_level, |
| 9036 | }; |
| 9037 | let result = tools |
| 9038 | .armfortas_adapters() |
| 9039 | .capture(&request) |
| 9040 | .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9041 | capture_text_stage(&result, stage).map(str::to_string) |
| 9042 | } |
| 9043 | |
| 9044 | fn capture_run_from_testing( |
| 9045 | source: &Path, |
| 9046 | opt_level: OptLevel, |
| 9047 | tools: &ToolchainConfig, |
| 9048 | ) -> Result<RunCapture, String> { |
| 9049 | let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 9050 | let request = CaptureRequest { |
| 9051 | input: source.to_path_buf(), |
| 9052 | requested: BTreeSet::from([Stage::Run]), |
| 9053 | opt_level, |
| 9054 | }; |
| 9055 | let result = tools |
| 9056 | .armfortas_adapters() |
| 9057 | .capture(&request) |
| 9058 | .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9059 | capture_run_stage(&result).cloned() |
| 9060 | } |
| 9061 | |
| 9062 | fn capture_text_stage<'a>(result: &'a CaptureResult, stage: Stage) -> Result<&'a str, String> { |
| 9063 | match result.get(stage) { |
| 9064 | Some(CapturedStage::Text(text)) => Ok(text), |
| 9065 | Some(CapturedStage::Run(_)) => Err(format!( |
| 9066 | "capture result contained non-text data for stage '{}'", |
| 9067 | stage.as_str() |
| 9068 | )), |
| 9069 | None => Err(format!( |
| 9070 | "capture result was missing requested stage '{}'", |
| 9071 | stage.as_str() |
| 9072 | )), |
| 9073 | } |
| 9074 | } |
| 9075 | |
| 9076 | fn capture_run_stage(result: &CaptureResult) -> Result<&RunCapture, String> { |
| 9077 | match result.get(Stage::Run) { |
| 9078 | Some(CapturedStage::Run(run)) => Ok(run), |
| 9079 | Some(CapturedStage::Text(_)) => { |
| 9080 | Err("capture result contained text data for the run stage".into()) |
| 9081 | } |
| 9082 | None => Err("capture result was missing requested stage 'run'".into()), |
| 9083 | } |
| 9084 | } |
| 9085 | |
| 9086 | fn run_binary_capture( |
| 9087 | binary: &Path, |
| 9088 | current_dir: &Path, |
| 9089 | command: &str, |
| 9090 | ) -> Result<RunCapture, String> { |
| 9091 | let output = Command::new(binary) |
| 9092 | .current_dir(current_dir) |
| 9093 | .output() |
| 9094 | .map_err(|err| { |
| 9095 | format!( |
| 9096 | "{} failed:\ncannot run '{}': {}", |
| 9097 | command, |
| 9098 | binary.display(), |
| 9099 | err |
| 9100 | ) |
| 9101 | })?; |
| 9102 | Ok(RunCapture { |
| 9103 | exit_code: output.status.code().unwrap_or(-1), |
| 9104 | stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 9105 | stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 9106 | }) |
| 9107 | } |
| 9108 | |
| 9109 | fn normalize_run_signature(run: &RunCapture) -> RunSignature { |
| 9110 | RunSignature { |
| 9111 | exit_code: run.exit_code, |
| 9112 | stdout: normalize_behavior_text(&run.stdout), |
| 9113 | stderr: normalize_behavior_text(&run.stderr), |
| 9114 | } |
| 9115 | } |
| 9116 | |
| 9117 | fn normalize_behavior_text(text: &str) -> String { |
| 9118 | text.replace("\r\n", "\n") |
| 9119 | .lines() |
| 9120 | .map(normalize_behavior_line) |
| 9121 | .collect::<Vec<_>>() |
| 9122 | .join("\n") |
| 9123 | .trim() |
| 9124 | .to_string() |
| 9125 | } |
| 9126 | |
| 9127 | fn normalize_behavior_line(line: &str) -> String { |
| 9128 | line.split_whitespace() |
| 9129 | .map(normalize_behavior_token) |
| 9130 | .collect::<Vec<_>>() |
| 9131 | .join(" ") |
| 9132 | } |
| 9133 | |
| 9134 | fn normalize_behavior_token(token: &str) -> String { |
| 9135 | if let Some(number) = parse_numeric_token(token) { |
| 9136 | format!("num:{:.6e}", number) |
| 9137 | } else { |
| 9138 | token.to_string() |
| 9139 | } |
| 9140 | } |
| 9141 | |
| 9142 | fn parse_numeric_token(token: &str) -> Option<f64> { |
| 9143 | if token.is_empty() { |
| 9144 | return None; |
| 9145 | } |
| 9146 | |
| 9147 | let normalized = token |
| 9148 | .trim() |
| 9149 | .trim_end_matches(',') |
| 9150 | .trim_end_matches(';') |
| 9151 | .replace('D', "E") |
| 9152 | .replace('d', "e"); |
| 9153 | |
| 9154 | normalized.parse::<f64>().ok() |
| 9155 | } |
| 9156 | |
| 9157 | fn format_reference_summary(references: &[ReferenceResult]) -> String { |
| 9158 | references |
| 9159 | .iter() |
| 9160 | .map(format_reference_result) |
| 9161 | .collect::<Vec<_>>() |
| 9162 | .join("\n\n") |
| 9163 | } |
| 9164 | |
| 9165 | fn format_reference_result(reference: &ReferenceResult) -> String { |
| 9166 | let mut lines = Vec::new(); |
| 9167 | lines.push(reference.compiler.as_str().to_string()); |
| 9168 | lines.push(format!("command: {}", reference.compile_command)); |
| 9169 | lines.push(format!("compile exit: {}", reference.compile_exit_code)); |
| 9170 | if !reference.compile_stdout.trim().is_empty() { |
| 9171 | lines.push(format!( |
| 9172 | "compile stdout:\n{}", |
| 9173 | reference.compile_stdout.trim_end() |
| 9174 | )); |
| 9175 | } |
| 9176 | if !reference.compile_stderr.trim().is_empty() { |
| 9177 | lines.push(format!( |
| 9178 | "compile stderr:\n{}", |
| 9179 | reference.compile_stderr.trim_end() |
| 9180 | )); |
| 9181 | } |
| 9182 | match (&reference.run, &reference.run_error) { |
| 9183 | (Some(run), _) => { |
| 9184 | lines.push(format!("run\n{}", format_run_capture(run))); |
| 9185 | } |
| 9186 | (None, Some(err)) => { |
| 9187 | lines.push(format!("run error: {}", err)); |
| 9188 | } |
| 9189 | (None, None) => {} |
| 9190 | } |
| 9191 | lines.join("\n") |
| 9192 | } |
| 9193 | |
| 9194 | fn format_run_capture(run: &RunCapture) -> String { |
| 9195 | let stdout = if run.stdout.is_empty() { |
| 9196 | "<empty>".to_string() |
| 9197 | } else { |
| 9198 | run.stdout.trim_end().to_string() |
| 9199 | }; |
| 9200 | let stderr = if run.stderr.is_empty() { |
| 9201 | "<empty>".to_string() |
| 9202 | } else { |
| 9203 | run.stderr.trim_end().to_string() |
| 9204 | }; |
| 9205 | format!( |
| 9206 | "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9207 | run.exit_code, stdout, stderr |
| 9208 | ) |
| 9209 | } |
| 9210 | |
| 9211 | fn format_run_signature(signature: &RunSignature) -> String { |
| 9212 | let stdout = if signature.stdout.is_empty() { |
| 9213 | "<empty>".to_string() |
| 9214 | } else { |
| 9215 | signature.stdout.clone() |
| 9216 | }; |
| 9217 | let stderr = if signature.stderr.is_empty() { |
| 9218 | "<empty>".to_string() |
| 9219 | } else { |
| 9220 | signature.stderr.clone() |
| 9221 | }; |
| 9222 | format!( |
| 9223 | "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9224 | signature.exit_code, stdout, stderr |
| 9225 | ) |
| 9226 | } |
| 9227 | |
| 9228 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 9229 | struct ObjectSnapshot { |
| 9230 | text: String, |
| 9231 | load_commands: String, |
| 9232 | relocations: String, |
| 9233 | symbols: String, |
| 9234 | } |
| 9235 | |
| 9236 | #[derive(Debug, Clone)] |
| 9237 | struct TextRun { |
| 9238 | label: String, |
| 9239 | command: String, |
| 9240 | normalized: String, |
| 9241 | } |
| 9242 | |
| 9243 | #[derive(Debug, Clone)] |
| 9244 | struct BehaviorRun { |
| 9245 | label: String, |
| 9246 | command: String, |
| 9247 | signature: RunSignature, |
| 9248 | run: RunCapture, |
| 9249 | } |
| 9250 | |
| 9251 | #[derive(Debug, Clone)] |
| 9252 | struct ObjectRun { |
| 9253 | label: String, |
| 9254 | command: String, |
| 9255 | snapshot: ObjectSnapshot, |
| 9256 | } |
| 9257 | |
| 9258 | fn object_snapshot(path: &Path, tools: &ToolchainConfig) -> Result<ObjectSnapshot, String> { |
| 9259 | let text = normalize_tool_output(&tool_output( |
| 9260 | tools.otool_bin(), |
| 9261 | &["-t", path.to_str().unwrap()], |
| 9262 | )?); |
| 9263 | let load_commands = normalize_tool_output(&tool_output( |
| 9264 | tools.otool_bin(), |
| 9265 | &["-l", path.to_str().unwrap()], |
| 9266 | )?); |
| 9267 | let relocations = normalize_tool_output(&tool_output( |
| 9268 | tools.otool_bin(), |
| 9269 | &["-rv", path.to_str().unwrap()], |
| 9270 | )?); |
| 9271 | let symbols = normalize_tool_output(&tool_output( |
| 9272 | tools.nm_bin(), |
| 9273 | &["-m", path.to_str().unwrap()], |
| 9274 | )?); |
| 9275 | |
| 9276 | Ok(ObjectSnapshot { |
| 9277 | text, |
| 9278 | load_commands, |
| 9279 | relocations, |
| 9280 | symbols, |
| 9281 | }) |
| 9282 | } |
| 9283 | |
| 9284 | fn tool_output(tool: &str, args: &[&str]) -> Result<String, String> { |
| 9285 | let output = Command::new(tool) |
| 9286 | .args(args) |
| 9287 | .output() |
| 9288 | .map_err(|e| format!("cannot run {}: {}", tool, e))?; |
| 9289 | if output.status.success() { |
| 9290 | Ok(String::from_utf8_lossy(&output.stdout).into_owned()) |
| 9291 | } else { |
| 9292 | Err(format!( |
| 9293 | "{} failed:\n{}", |
| 9294 | tool, |
| 9295 | String::from_utf8_lossy(&output.stderr) |
| 9296 | )) |
| 9297 | } |
| 9298 | } |
| 9299 | |
| 9300 | fn normalize_tool_output(text: &str) -> String { |
| 9301 | text.lines() |
| 9302 | .filter(|line| !line.trim_end().ends_with(".o:")) |
| 9303 | .map(str::trim_end) |
| 9304 | .collect::<Vec<_>>() |
| 9305 | .join("\n") |
| 9306 | } |
| 9307 | |
| 9308 | fn read_text_artifact(path: &Path) -> Result<String, String> { |
| 9309 | fs::read_to_string(path).map_err(|e| format!("cannot read '{}': {}", path.display(), e)) |
| 9310 | } |
| 9311 | |
| 9312 | fn normalize_text_artifact(text: &str) -> String { |
| 9313 | text.replace("\r\n", "\n") |
| 9314 | .lines() |
| 9315 | .map(str::trim_end) |
| 9316 | .collect::<Vec<_>>() |
| 9317 | .join("\n") |
| 9318 | } |
| 9319 | |
| 9320 | fn count_unique_strings<'a>(values: impl IntoIterator<Item = &'a str>) -> usize { |
| 9321 | values.into_iter().collect::<BTreeSet<_>>().len() |
| 9322 | } |
| 9323 | |
| 9324 | fn first_distinct_text_pair(runs: &[TextRun]) -> Option<(&TextRun, &TextRun)> { |
| 9325 | for left_index in 0..runs.len() { |
| 9326 | for right_index in (left_index + 1)..runs.len() { |
| 9327 | if runs[left_index].normalized != runs[right_index].normalized { |
| 9328 | return Some((&runs[left_index], &runs[right_index])); |
| 9329 | } |
| 9330 | } |
| 9331 | } |
| 9332 | None |
| 9333 | } |
| 9334 | |
| 9335 | fn count_unique_run_signatures<'a>(values: impl IntoIterator<Item = &'a RunSignature>) -> usize { |
| 9336 | values.into_iter().collect::<BTreeSet<_>>().len() |
| 9337 | } |
| 9338 | |
| 9339 | fn first_distinct_behavior_pair(runs: &[BehaviorRun]) -> Option<(&BehaviorRun, &BehaviorRun)> { |
| 9340 | for left_index in 0..runs.len() { |
| 9341 | for right_index in (left_index + 1)..runs.len() { |
| 9342 | if runs[left_index].signature != runs[right_index].signature { |
| 9343 | return Some((&runs[left_index], &runs[right_index])); |
| 9344 | } |
| 9345 | } |
| 9346 | } |
| 9347 | None |
| 9348 | } |
| 9349 | |
| 9350 | fn first_distinct_object_pair(runs: &[ObjectRun]) -> Option<(&ObjectRun, &ObjectRun)> { |
| 9351 | for left_index in 0..runs.len() { |
| 9352 | for right_index in (left_index + 1)..runs.len() { |
| 9353 | if runs[left_index].snapshot != runs[right_index].snapshot { |
| 9354 | return Some((&runs[left_index], &runs[right_index])); |
| 9355 | } |
| 9356 | } |
| 9357 | } |
| 9358 | None |
| 9359 | } |
| 9360 | |
| 9361 | fn render_object_snapshot(snapshot: &ObjectSnapshot) -> String { |
| 9362 | format!( |
| 9363 | "== text ==\n{}\n\n== load_commands ==\n{}\n\n== relocations ==\n{}\n\n== symbols ==\n{}", |
| 9364 | snapshot.text, snapshot.load_commands, snapshot.relocations, snapshot.symbols |
| 9365 | ) |
| 9366 | } |
| 9367 | |
| 9368 | fn parse_object_snapshot_text(text: &str) -> Result<ObjectSnapshot, String> { |
| 9369 | let text = text |
| 9370 | .strip_prefix("== text ==\n") |
| 9371 | .ok_or_else(|| "object snapshot was missing the '== text ==' header".to_string())?; |
| 9372 | let (text, rest) = text |
| 9373 | .split_once("\n\n== load_commands ==\n") |
| 9374 | .ok_or_else(|| { |
| 9375 | "object snapshot was missing the '== load_commands ==' section".to_string() |
| 9376 | })?; |
| 9377 | let (load_commands, rest) = rest |
| 9378 | .split_once("\n\n== relocations ==\n") |
| 9379 | .ok_or_else(|| "object snapshot was missing the '== relocations ==' section".to_string())?; |
| 9380 | let (relocations, symbols) = rest |
| 9381 | .split_once("\n\n== symbols ==\n") |
| 9382 | .ok_or_else(|| "object snapshot was missing the '== symbols ==' section".to_string())?; |
| 9383 | |
| 9384 | Ok(ObjectSnapshot { |
| 9385 | text: text.to_string(), |
| 9386 | load_commands: load_commands.to_string(), |
| 9387 | relocations: relocations.to_string(), |
| 9388 | symbols: symbols.to_string(), |
| 9389 | }) |
| 9390 | } |
| 9391 | |
| 9392 | fn describe_text_difference( |
| 9393 | expected: &str, |
| 9394 | actual: &str, |
| 9395 | left_label: &str, |
| 9396 | right_label: &str, |
| 9397 | ) -> String { |
| 9398 | let expected_lines: Vec<&str> = expected.lines().collect(); |
| 9399 | let actual_lines: Vec<&str> = actual.lines().collect(); |
| 9400 | let shared = expected_lines.len().min(actual_lines.len()); |
| 9401 | |
| 9402 | for index in 0..shared { |
| 9403 | if expected_lines[index] != actual_lines[index] { |
| 9404 | return format!( |
| 9405 | "first differing line: {}\n{}: {}\n{}: {}", |
| 9406 | index + 1, |
| 9407 | left_label, |
| 9408 | expected_lines[index], |
| 9409 | right_label, |
| 9410 | actual_lines[index] |
| 9411 | ); |
| 9412 | } |
| 9413 | } |
| 9414 | |
| 9415 | let mut detail = format!( |
| 9416 | "snapshot length differs\n{} lines: {}\n{} lines: {}", |
| 9417 | left_label, |
| 9418 | expected_lines.len(), |
| 9419 | right_label, |
| 9420 | actual_lines.len() |
| 9421 | ); |
| 9422 | if let Some(extra) = expected_lines.get(shared) { |
| 9423 | detail.push_str(&format!( |
| 9424 | "\nfirst extra line: {}\n{}: {}", |
| 9425 | shared + 1, |
| 9426 | left_label, |
| 9427 | extra |
| 9428 | )); |
| 9429 | } else if let Some(extra) = actual_lines.get(shared) { |
| 9430 | detail.push_str(&format!( |
| 9431 | "\nfirst extra line: {}\n{}: {}", |
| 9432 | shared + 1, |
| 9433 | right_label, |
| 9434 | extra |
| 9435 | )); |
| 9436 | } |
| 9437 | detail |
| 9438 | } |
| 9439 | |
| 9440 | fn describe_object_difference( |
| 9441 | expected: &ObjectSnapshot, |
| 9442 | actual: &ObjectSnapshot, |
| 9443 | left_label: &str, |
| 9444 | right_label: &str, |
| 9445 | ) -> String { |
| 9446 | let mut differing = Vec::new(); |
| 9447 | if expected.text != actual.text { |
| 9448 | differing.push(("text", &expected.text, &actual.text)); |
| 9449 | } |
| 9450 | if expected.load_commands != actual.load_commands { |
| 9451 | differing.push(( |
| 9452 | "load_commands", |
| 9453 | &expected.load_commands, |
| 9454 | &actual.load_commands, |
| 9455 | )); |
| 9456 | } |
| 9457 | if expected.relocations != actual.relocations { |
| 9458 | differing.push(("relocations", &expected.relocations, &actual.relocations)); |
| 9459 | } |
| 9460 | if expected.symbols != actual.symbols { |
| 9461 | differing.push(("symbols", &expected.symbols, &actual.symbols)); |
| 9462 | } |
| 9463 | |
| 9464 | if differing.is_empty() { |
| 9465 | return "object snapshots matched".to_string(); |
| 9466 | } |
| 9467 | |
| 9468 | let component_list = differing |
| 9469 | .iter() |
| 9470 | .map(|(name, _, _)| *name) |
| 9471 | .collect::<Vec<_>>() |
| 9472 | .join(", "); |
| 9473 | let (first_name, first_expected, first_actual) = differing[0]; |
| 9474 | |
| 9475 | format!( |
| 9476 | "differing object components: {}\n{}\n{}", |
| 9477 | component_list, |
| 9478 | format!("first differing component: {}", first_name), |
| 9479 | describe_text_difference(first_expected, first_actual, left_label, right_label) |
| 9480 | ) |
| 9481 | } |
| 9482 | |
| 9483 | fn describe_run_difference( |
| 9484 | expected: &RunCapture, |
| 9485 | actual: &RunCapture, |
| 9486 | left_label: &str, |
| 9487 | right_label: &str, |
| 9488 | ) -> String { |
| 9489 | let expected = normalize_run_signature(expected); |
| 9490 | let actual = normalize_run_signature(actual); |
| 9491 | let mut differing = Vec::new(); |
| 9492 | if expected.exit_code != actual.exit_code { |
| 9493 | differing.push("exit_code"); |
| 9494 | } |
| 9495 | if expected.stdout != actual.stdout { |
| 9496 | differing.push("stdout"); |
| 9497 | } |
| 9498 | if expected.stderr != actual.stderr { |
| 9499 | differing.push("stderr"); |
| 9500 | } |
| 9501 | |
| 9502 | if differing.is_empty() { |
| 9503 | return "runtime behavior matched".to_string(); |
| 9504 | } |
| 9505 | |
| 9506 | let component_list = differing.join(", "); |
| 9507 | match differing[0] { |
| 9508 | "exit_code" => format!( |
| 9509 | "differing runtime components: {}\nfirst differing component: exit_code\n{}: {}\n{}: {}", |
| 9510 | component_list, |
| 9511 | left_label, |
| 9512 | expected.exit_code, |
| 9513 | right_label, |
| 9514 | actual.exit_code |
| 9515 | ), |
| 9516 | "stdout" => format!( |
| 9517 | "differing runtime components: {}\nfirst differing component: stdout\n{}", |
| 9518 | component_list, |
| 9519 | describe_text_difference(&expected.stdout, &actual.stdout, left_label, right_label) |
| 9520 | ), |
| 9521 | "stderr" => format!( |
| 9522 | "differing runtime components: {}\nfirst differing component: stderr\n{}", |
| 9523 | component_list, |
| 9524 | describe_text_difference(&expected.stderr, &actual.stderr, left_label, right_label) |
| 9525 | ), |
| 9526 | _ => unreachable!("only known runtime components are compared"), |
| 9527 | } |
| 9528 | } |
| 9529 | |
| 9530 | fn varying_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9531 | object_components_by_variation(snapshots, true) |
| 9532 | } |
| 9533 | |
| 9534 | fn stable_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9535 | object_components_by_variation(snapshots, false) |
| 9536 | } |
| 9537 | |
| 9538 | fn varying_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9539 | run_components_by_variation(signatures, true) |
| 9540 | } |
| 9541 | |
| 9542 | fn stable_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9543 | run_components_by_variation(signatures, false) |
| 9544 | } |
| 9545 | |
| 9546 | fn object_components_by_variation( |
| 9547 | snapshots: &[&ObjectSnapshot], |
| 9548 | want_varying: bool, |
| 9549 | ) -> Vec<&'static str> { |
| 9550 | let components = [ |
| 9551 | ( |
| 9552 | "text", |
| 9553 | snapshots |
| 9554 | .iter() |
| 9555 | .map(|snapshot| snapshot.text.as_str()) |
| 9556 | .collect::<Vec<_>>(), |
| 9557 | ), |
| 9558 | ( |
| 9559 | "load_commands", |
| 9560 | snapshots |
| 9561 | .iter() |
| 9562 | .map(|snapshot| snapshot.load_commands.as_str()) |
| 9563 | .collect::<Vec<_>>(), |
| 9564 | ), |
| 9565 | ( |
| 9566 | "relocations", |
| 9567 | snapshots |
| 9568 | .iter() |
| 9569 | .map(|snapshot| snapshot.relocations.as_str()) |
| 9570 | .collect::<Vec<_>>(), |
| 9571 | ), |
| 9572 | ( |
| 9573 | "symbols", |
| 9574 | snapshots |
| 9575 | .iter() |
| 9576 | .map(|snapshot| snapshot.symbols.as_str()) |
| 9577 | .collect::<Vec<_>>(), |
| 9578 | ), |
| 9579 | ]; |
| 9580 | |
| 9581 | components |
| 9582 | .into_iter() |
| 9583 | .filter_map(|(name, values)| { |
| 9584 | let varies = count_unique_strings(values) > 1; |
| 9585 | if varies == want_varying { |
| 9586 | Some(name) |
| 9587 | } else { |
| 9588 | None |
| 9589 | } |
| 9590 | }) |
| 9591 | .collect() |
| 9592 | } |
| 9593 | |
| 9594 | fn run_components_by_variation( |
| 9595 | signatures: &[&RunSignature], |
| 9596 | want_varying: bool, |
| 9597 | ) -> Vec<&'static str> { |
| 9598 | let components = [ |
| 9599 | ( |
| 9600 | "exit_code", |
| 9601 | signatures |
| 9602 | .iter() |
| 9603 | .map(|signature| signature.exit_code.to_string()) |
| 9604 | .collect::<Vec<_>>(), |
| 9605 | ), |
| 9606 | ( |
| 9607 | "stdout", |
| 9608 | signatures |
| 9609 | .iter() |
| 9610 | .map(|signature| signature.stdout.clone()) |
| 9611 | .collect::<Vec<_>>(), |
| 9612 | ), |
| 9613 | ( |
| 9614 | "stderr", |
| 9615 | signatures |
| 9616 | .iter() |
| 9617 | .map(|signature| signature.stderr.clone()) |
| 9618 | .collect::<Vec<_>>(), |
| 9619 | ), |
| 9620 | ]; |
| 9621 | |
| 9622 | components |
| 9623 | .into_iter() |
| 9624 | .filter_map(|(name, values)| { |
| 9625 | let varies = count_unique_strings(values.iter().map(String::as_str)) > 1; |
| 9626 | if varies == want_varying { |
| 9627 | Some(name) |
| 9628 | } else { |
| 9629 | None |
| 9630 | } |
| 9631 | }) |
| 9632 | .collect() |
| 9633 | } |
| 9634 | |
| 9635 | fn join_or_none(values: &[&str]) -> String { |
| 9636 | if values.is_empty() { |
| 9637 | "none".to_string() |
| 9638 | } else { |
| 9639 | values.join(", ") |
| 9640 | } |
| 9641 | } |
| 9642 | |
| 9643 | fn join_or_none_from_strings(values: &[String]) -> String { |
| 9644 | if values.is_empty() { |
| 9645 | "none".to_string() |
| 9646 | } else { |
| 9647 | values.join(", ") |
| 9648 | } |
| 9649 | } |
| 9650 | |
| 9651 | fn join_usize_set(values: &BTreeSet<usize>) -> String { |
| 9652 | if values.is_empty() { |
| 9653 | "n/a".to_string() |
| 9654 | } else { |
| 9655 | values |
| 9656 | .iter() |
| 9657 | .map(|value| value.to_string()) |
| 9658 | .collect::<Vec<_>>() |
| 9659 | .join(", ") |
| 9660 | } |
| 9661 | } |
| 9662 | |
| 9663 | fn join_string_set(values: &BTreeSet<String>) -> String { |
| 9664 | if values.is_empty() { |
| 9665 | "none".to_string() |
| 9666 | } else { |
| 9667 | values.iter().cloned().collect::<Vec<_>>().join(", ") |
| 9668 | } |
| 9669 | } |
| 9670 | |
| 9671 | fn render_consistency_rollup(rollup: &ConsistencyRollup) -> String { |
| 9672 | let mut parts = vec![format!("{} cells", rollup.cells)]; |
| 9673 | if !rollup.repeat_counts.is_empty() { |
| 9674 | parts.push(format!( |
| 9675 | "repeat_count={}", |
| 9676 | join_usize_set(&rollup.repeat_counts) |
| 9677 | )); |
| 9678 | } |
| 9679 | if !rollup.unique_variant_counts.is_empty() { |
| 9680 | parts.push(format!( |
| 9681 | "unique_variants={}", |
| 9682 | join_usize_set(&rollup.unique_variant_counts) |
| 9683 | )); |
| 9684 | } |
| 9685 | if !rollup.varying_components.is_empty() { |
| 9686 | parts.push(format!( |
| 9687 | "varying={}", |
| 9688 | join_string_set(&rollup.varying_components) |
| 9689 | )); |
| 9690 | } |
| 9691 | if !rollup.stable_components.is_empty() { |
| 9692 | parts.push(format!( |
| 9693 | "stable={}", |
| 9694 | join_string_set(&rollup.stable_components) |
| 9695 | )); |
| 9696 | } |
| 9697 | parts.join("; ") |
| 9698 | } |
| 9699 | |
| 9700 | fn write_behavior_run_artifacts( |
| 9701 | root: &Path, |
| 9702 | prefix: &str, |
| 9703 | command: &str, |
| 9704 | run: &RunCapture, |
| 9705 | ) -> Result<(), std::io::Error> { |
| 9706 | let signature = normalize_run_signature(run); |
| 9707 | fs::write(root.join(format!("{}.command.txt", prefix)), command)?; |
| 9708 | fs::write(root.join(format!("{}.stdout.txt", prefix)), &run.stdout)?; |
| 9709 | fs::write(root.join(format!("{}.stderr.txt", prefix)), &run.stderr)?; |
| 9710 | fs::write( |
| 9711 | root.join(format!("{}.exit_code.txt", prefix)), |
| 9712 | format!("{}\n", run.exit_code), |
| 9713 | )?; |
| 9714 | fs::write( |
| 9715 | root.join(format!("{}.normalized.txt", prefix)), |
| 9716 | format_run_signature(&signature), |
| 9717 | )?; |
| 9718 | Ok(()) |
| 9719 | } |
| 9720 | |
| 9721 | fn render_summary(summary: &Summary) -> String { |
| 9722 | let mut lines = vec![ |
| 9723 | "Summary".to_string(), |
| 9724 | format!(" passed: {}", summary.passed), |
| 9725 | format!(" failed: {}", summary.failed), |
| 9726 | format!(" xfailed: {}", summary.xfailed), |
| 9727 | format!(" xpassed: {}", summary.xpassed), |
| 9728 | format!(" future: {}", summary.future), |
| 9729 | ]; |
| 9730 | |
| 9731 | if !summary.consistency.is_empty() { |
| 9732 | lines.push(String::new()); |
| 9733 | lines.push("Consistency".to_string()); |
| 9734 | lines.push(format!(" affected_checks: {}", summary.consistency.len())); |
| 9735 | lines.push(format!( |
| 9736 | " cells_with_issues: {}", |
| 9737 | summary |
| 9738 | .consistency |
| 9739 | .values() |
| 9740 | .map(|rollup| rollup.cells) |
| 9741 | .sum::<usize>() |
| 9742 | )); |
| 9743 | for (check, rollup) in &summary.consistency { |
| 9744 | lines.push(format!( |
| 9745 | " {}: {}", |
| 9746 | check.as_str(), |
| 9747 | render_consistency_rollup(rollup) |
| 9748 | )); |
| 9749 | } |
| 9750 | } |
| 9751 | |
| 9752 | lines.join("\n") |
| 9753 | } |
| 9754 | |
| 9755 | fn write_requested_reports(config: &RunConfig, summary: &Summary) -> Result<(), String> { |
| 9756 | if let Some(path) = &config.json_report { |
| 9757 | write_report(path, &render_json_report(summary), "json report")?; |
| 9758 | println!("json report: {}", path.display()); |
| 9759 | } |
| 9760 | if let Some(path) = &config.markdown_report { |
| 9761 | write_report(path, &render_markdown_report(summary), "markdown report")?; |
| 9762 | println!("markdown report: {}", path.display()); |
| 9763 | } |
| 9764 | Ok(()) |
| 9765 | } |
| 9766 | |
| 9767 | fn write_report(path: &Path, content: &str, label: &str) -> Result<(), String> { |
| 9768 | if let Some(parent) = path.parent() { |
| 9769 | fs::create_dir_all(parent).map_err(|e| { |
| 9770 | format!( |
| 9771 | "cannot create parent directory for {} '{}': {}", |
| 9772 | label, |
| 9773 | path.display(), |
| 9774 | e |
| 9775 | ) |
| 9776 | })?; |
| 9777 | } |
| 9778 | fs::write(path, content) |
| 9779 | .map_err(|e| format!("cannot write {} '{}': {}", label, path.display(), e)) |
| 9780 | } |
| 9781 | |
| 9782 | fn render_json_report(summary: &Summary) -> String { |
| 9783 | let mut lines = vec![ |
| 9784 | "{".to_string(), |
| 9785 | format!(" \"passed\": {},", summary.passed), |
| 9786 | format!(" \"failed\": {},", summary.failed), |
| 9787 | format!(" \"xfailed\": {},", summary.xfailed), |
| 9788 | format!(" \"xpassed\": {},", summary.xpassed), |
| 9789 | format!(" \"future\": {},", summary.future), |
| 9790 | " \"outcomes\": [".to_string(), |
| 9791 | ]; |
| 9792 | |
| 9793 | for (index, outcome) in summary.outcomes.iter().enumerate() { |
| 9794 | lines.push(" {".to_string()); |
| 9795 | lines.push(format!( |
| 9796 | " \"suite\": \"{}\",", |
| 9797 | json_escape(&outcome.suite) |
| 9798 | )); |
| 9799 | lines.push(format!( |
| 9800 | " \"case\": \"{}\",", |
| 9801 | json_escape(&outcome.case) |
| 9802 | )); |
| 9803 | lines.push(format!( |
| 9804 | " \"opt\": \"{}\",", |
| 9805 | outcome.opt_level.as_str() |
| 9806 | )); |
| 9807 | lines.push(format!( |
| 9808 | " \"kind\": \"{}\",", |
| 9809 | outcome_kind_name(outcome.kind) |
| 9810 | )); |
| 9811 | match &outcome.primary_backend { |
| 9812 | Some(backend) => { |
| 9813 | lines.push(" \"primary_backend\": {".to_string()); |
| 9814 | lines.push(format!( |
| 9815 | " \"kind\": \"{}\",", |
| 9816 | json_escape(&backend.kind) |
| 9817 | )); |
| 9818 | lines.push(format!( |
| 9819 | " \"mode\": \"{}\",", |
| 9820 | json_escape(&backend.mode) |
| 9821 | )); |
| 9822 | lines.push(format!( |
| 9823 | " \"detail\": \"{}\"", |
| 9824 | json_escape(&backend.detail) |
| 9825 | )); |
| 9826 | lines.push(" },".to_string()); |
| 9827 | } |
| 9828 | None => lines.push(" \"primary_backend\": null,".to_string()), |
| 9829 | } |
| 9830 | lines.push(format!( |
| 9831 | " \"detail\": \"{}\",", |
| 9832 | json_escape(&outcome.detail) |
| 9833 | )); |
| 9834 | match &outcome.bundle { |
| 9835 | Some(bundle) => lines.push(format!( |
| 9836 | " \"bundle\": \"{}\",", |
| 9837 | json_escape(&bundle.display().to_string()) |
| 9838 | )), |
| 9839 | None => lines.push(" \"bundle\": null,".to_string()), |
| 9840 | } |
| 9841 | lines.push(format!( |
| 9842 | " \"consistency\": {}", |
| 9843 | render_json_consistency_observations(&outcome.consistency_observations) |
| 9844 | )); |
| 9845 | lines.push(if index + 1 == summary.outcomes.len() { |
| 9846 | " }".to_string() |
| 9847 | } else { |
| 9848 | " },".to_string() |
| 9849 | }); |
| 9850 | } |
| 9851 | |
| 9852 | lines.push(" ],".to_string()); |
| 9853 | lines.push(" \"consistency\": {".to_string()); |
| 9854 | for (index, (check, rollup)) in summary.consistency.iter().enumerate() { |
| 9855 | lines.push(format!(" \"{}\": {{", check.as_str())); |
| 9856 | lines.push(format!(" \"cells\": {},", rollup.cells)); |
| 9857 | lines.push(format!( |
| 9858 | " \"repeat_counts\": {},", |
| 9859 | json_usize_set(&rollup.repeat_counts) |
| 9860 | )); |
| 9861 | lines.push(format!( |
| 9862 | " \"unique_variant_counts\": {},", |
| 9863 | json_usize_set(&rollup.unique_variant_counts) |
| 9864 | )); |
| 9865 | lines.push(format!( |
| 9866 | " \"varying_components\": {},", |
| 9867 | json_string_iter(rollup.varying_components.iter().map(|value| value.as_str())) |
| 9868 | )); |
| 9869 | lines.push(format!( |
| 9870 | " \"stable_components\": {}", |
| 9871 | json_string_iter(rollup.stable_components.iter().map(|value| value.as_str())) |
| 9872 | )); |
| 9873 | lines.push(if index + 1 == summary.consistency.len() { |
| 9874 | " }".to_string() |
| 9875 | } else { |
| 9876 | " },".to_string() |
| 9877 | }); |
| 9878 | } |
| 9879 | lines.push(" }".to_string()); |
| 9880 | lines.push("}".to_string()); |
| 9881 | lines.join("\n") + "\n" |
| 9882 | } |
| 9883 | |
| 9884 | fn render_json_consistency_observations(observations: &[ConsistencyObservation]) -> String { |
| 9885 | let mut rendered = String::from("["); |
| 9886 | for (index, observation) in observations.iter().enumerate() { |
| 9887 | if index > 0 { |
| 9888 | rendered.push_str(", "); |
| 9889 | } |
| 9890 | rendered.push('{'); |
| 9891 | rendered.push_str(&format!( |
| 9892 | "\"check\":\"{}\",\"summary\":\"{}\",", |
| 9893 | observation.check.as_str(), |
| 9894 | json_escape(&observation.summary) |
| 9895 | )); |
| 9896 | match observation.repeat_count { |
| 9897 | Some(count) => rendered.push_str(&format!("\"repeat_count\":{},", count)), |
| 9898 | None => rendered.push_str("\"repeat_count\":null,"), |
| 9899 | } |
| 9900 | match observation.unique_variant_count { |
| 9901 | Some(count) => rendered.push_str(&format!("\"unique_variant_count\":{},", count)), |
| 9902 | None => rendered.push_str("\"unique_variant_count\":null,"), |
| 9903 | } |
| 9904 | rendered.push_str(&format!( |
| 9905 | "\"varying_components\":{},\"stable_components\":{}", |
| 9906 | json_string_array(&observation.varying_components), |
| 9907 | json_string_array(&observation.stable_components) |
| 9908 | )); |
| 9909 | rendered.push('}'); |
| 9910 | } |
| 9911 | rendered.push(']'); |
| 9912 | rendered |
| 9913 | } |
| 9914 | |
| 9915 | fn render_markdown_report(summary: &Summary) -> String { |
| 9916 | let mut lines = vec![ |
| 9917 | "# afs-tests report".to_string(), |
| 9918 | String::new(), |
| 9919 | "## Summary".to_string(), |
| 9920 | String::new(), |
| 9921 | "| kind | count |".to_string(), |
| 9922 | "| --- | ---: |".to_string(), |
| 9923 | format!("| passed | {} |", summary.passed), |
| 9924 | format!("| failed | {} |", summary.failed), |
| 9925 | format!("| xfailed | {} |", summary.xfailed), |
| 9926 | format!("| xpassed | {} |", summary.xpassed), |
| 9927 | format!("| future | {} |", summary.future), |
| 9928 | ]; |
| 9929 | |
| 9930 | if !summary.consistency.is_empty() { |
| 9931 | lines.push(String::new()); |
| 9932 | lines.push("## Consistency".to_string()); |
| 9933 | lines.push(String::new()); |
| 9934 | lines.push("| check | cells | repeats | unique variants | varying | stable |".to_string()); |
| 9935 | lines.push("| --- | ---: | --- | --- | --- | --- |".to_string()); |
| 9936 | for (check, rollup) in &summary.consistency { |
| 9937 | lines.push(format!( |
| 9938 | "| `{}` | {} | {} | {} | {} | {} |", |
| 9939 | check.as_str(), |
| 9940 | rollup.cells, |
| 9941 | join_usize_set(&rollup.repeat_counts), |
| 9942 | join_usize_set(&rollup.unique_variant_counts), |
| 9943 | join_string_set(&rollup.varying_components), |
| 9944 | join_string_set(&rollup.stable_components), |
| 9945 | )); |
| 9946 | } |
| 9947 | } |
| 9948 | |
| 9949 | lines.push(String::new()); |
| 9950 | lines.push("## Outcomes".to_string()); |
| 9951 | for outcome in &summary.outcomes { |
| 9952 | lines.push(String::new()); |
| 9953 | lines.push(format!( |
| 9954 | "### `{}` / `{}` / `{}` / `{}`", |
| 9955 | outcome.suite, |
| 9956 | outcome.case, |
| 9957 | outcome.opt_level.as_str(), |
| 9958 | outcome_kind_name(outcome.kind) |
| 9959 | )); |
| 9960 | if let Some(backend) = &outcome.primary_backend { |
| 9961 | lines.push(format!( |
| 9962 | "primary_backend: `{}` (`{}`)", |
| 9963 | backend.kind, backend.mode |
| 9964 | )); |
| 9965 | lines.push(format!("primary_backend_detail: {}", backend.detail)); |
| 9966 | } |
| 9967 | if let Some(bundle) = &outcome.bundle { |
| 9968 | lines.push(format!("bundle: `{}`", bundle.display())); |
| 9969 | } |
| 9970 | if !outcome.detail.trim().is_empty() { |
| 9971 | lines.push(String::new()); |
| 9972 | lines.push("```text".to_string()); |
| 9973 | lines.extend( |
| 9974 | outcome |
| 9975 | .detail |
| 9976 | .trim_end() |
| 9977 | .lines() |
| 9978 | .map(|line| line.to_string()), |
| 9979 | ); |
| 9980 | lines.push("```".to_string()); |
| 9981 | } |
| 9982 | } |
| 9983 | |
| 9984 | lines.join("\n") + "\n" |
| 9985 | } |
| 9986 | |
| 9987 | fn outcome_kind_name(kind: OutcomeKind) -> &'static str { |
| 9988 | match kind { |
| 9989 | OutcomeKind::Pass => "pass", |
| 9990 | OutcomeKind::Fail => "fail", |
| 9991 | OutcomeKind::Xfail => "xfail", |
| 9992 | OutcomeKind::Xpass => "xpass", |
| 9993 | OutcomeKind::Future => "future", |
| 9994 | } |
| 9995 | } |
| 9996 | |
| 9997 | fn json_escape(text: &str) -> String { |
| 9998 | let mut escaped = String::new(); |
| 9999 | for ch in text.chars() { |
| 10000 | match ch { |
| 10001 | '\\' => escaped.push_str("\\\\"), |
| 10002 | '"' => escaped.push_str("\\\""), |
| 10003 | '\n' => escaped.push_str("\\n"), |
| 10004 | '\r' => escaped.push_str("\\r"), |
| 10005 | '\t' => escaped.push_str("\\t"), |
| 10006 | c if c.is_control() => escaped.push_str(&format!("\\u{:04x}", c as u32)), |
| 10007 | c => escaped.push(c), |
| 10008 | } |
| 10009 | } |
| 10010 | escaped |
| 10011 | } |
| 10012 | |
| 10013 | fn json_string_array(items: &[String]) -> String { |
| 10014 | json_string_iter(items.iter().map(|item| item.as_str())) |
| 10015 | } |
| 10016 | |
| 10017 | fn json_string_iter<'a>(items: impl Iterator<Item = &'a str>) -> String { |
| 10018 | let mut rendered = String::from("["); |
| 10019 | for (index, item) in items.enumerate() { |
| 10020 | if index > 0 { |
| 10021 | rendered.push_str(", "); |
| 10022 | } |
| 10023 | rendered.push('"'); |
| 10024 | rendered.push_str(&json_escape(item)); |
| 10025 | rendered.push('"'); |
| 10026 | } |
| 10027 | rendered.push(']'); |
| 10028 | rendered |
| 10029 | } |
| 10030 | |
| 10031 | fn json_usize_set(items: &BTreeSet<usize>) -> String { |
| 10032 | let mut rendered = String::from("["); |
| 10033 | for (index, item) in items.iter().enumerate() { |
| 10034 | if index > 0 { |
| 10035 | rendered.push_str(", "); |
| 10036 | } |
| 10037 | rendered.push_str(&item.to_string()); |
| 10038 | } |
| 10039 | rendered.push(']'); |
| 10040 | rendered |
| 10041 | } |
| 10042 | |
| 10043 | fn write_failure_bundle( |
| 10044 | suite: &SuiteSpec, |
| 10045 | case: &CaseSpec, |
| 10046 | prepared: &PreparedInput, |
| 10047 | outcome: &Outcome, |
| 10048 | artifacts: &ExecutionArtifacts, |
| 10049 | ) -> Result<PathBuf, String> { |
| 10050 | let bundle_root = default_report_root() |
| 10051 | .join(sanitize_component(&suite.name)) |
| 10052 | .join(sanitize_component(&case.name)) |
| 10053 | .join(next_report_suffix(outcome.opt_level)); |
| 10054 | fs::create_dir_all(&bundle_root).map_err(|e| { |
| 10055 | format!( |
| 10056 | "cannot create report bundle '{}': {}", |
| 10057 | bundle_root.display(), |
| 10058 | e |
| 10059 | ) |
| 10060 | })?; |
| 10061 | |
| 10062 | let stage_list = artifacts |
| 10063 | .requested |
| 10064 | .iter() |
| 10065 | .map(Stage::as_str) |
| 10066 | .collect::<Vec<_>>() |
| 10067 | .join(", "); |
| 10068 | let refs = if case.reference_compilers.is_empty() { |
| 10069 | "none".to_string() |
| 10070 | } else { |
| 10071 | case.reference_compilers |
| 10072 | .iter() |
| 10073 | .map(ReferenceCompiler::as_str) |
| 10074 | .collect::<Vec<_>>() |
| 10075 | .join(", ") |
| 10076 | }; |
| 10077 | let consistency = if case.consistency_checks.is_empty() { |
| 10078 | "none".to_string() |
| 10079 | } else { |
| 10080 | case.consistency_checks |
| 10081 | .iter() |
| 10082 | .map(ConsistencyCheck::as_str) |
| 10083 | .collect::<Vec<_>>() |
| 10084 | .join(", ") |
| 10085 | }; |
| 10086 | let primary_backend_kind = outcome |
| 10087 | .primary_backend |
| 10088 | .as_ref() |
| 10089 | .map(|backend| backend.kind.as_str()) |
| 10090 | .unwrap_or("none"); |
| 10091 | let primary_backend_mode = outcome |
| 10092 | .primary_backend |
| 10093 | .as_ref() |
| 10094 | .map(|backend| backend.mode.as_str()) |
| 10095 | .unwrap_or("none"); |
| 10096 | let primary_backend_detail = outcome |
| 10097 | .primary_backend |
| 10098 | .as_ref() |
| 10099 | .map(|backend| backend.detail.as_str()) |
| 10100 | .unwrap_or("none"); |
| 10101 | let metadata = format!( |
| 10102 | "suite: {}\ncase: {}\noutcome: {:?}\nopt: {}\nsource: {}\nrequested_stages: {}\nrepeat_count: {}\nreference_compilers: {}\nconsistency_checks: {}\nprimary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\n", |
| 10103 | suite.name, |
| 10104 | case.name, |
| 10105 | outcome.kind, |
| 10106 | outcome.opt_level.as_str(), |
| 10107 | case.source_label(), |
| 10108 | stage_list, |
| 10109 | case.repeat_count, |
| 10110 | refs, |
| 10111 | consistency, |
| 10112 | primary_backend_kind, |
| 10113 | primary_backend_mode, |
| 10114 | primary_backend_detail |
| 10115 | ); |
| 10116 | fs::write(bundle_root.join("metadata.txt"), metadata) |
| 10117 | .map_err(|e| format!("cannot write bundle metadata: {}", e))?; |
| 10118 | fs::write(bundle_root.join("detail.txt"), &outcome.detail) |
| 10119 | .map_err(|e| format!("cannot write bundle detail: {}", e))?; |
| 10120 | |
| 10121 | write_case_sources_bundle(&bundle_root, case, prepared)?; |
| 10122 | |
| 10123 | let armfortas_root = bundle_root.join("armfortas"); |
| 10124 | fs::create_dir_all(&armfortas_root) |
| 10125 | .map_err(|e| format!("cannot create armfortas bundle dir: {}", e))?; |
| 10126 | write_armfortas_bundle_metadata(&armfortas_root, outcome, artifacts)?; |
| 10127 | if let Some(result) = &artifacts.armfortas { |
| 10128 | write_capture_result(&armfortas_root, result)?; |
| 10129 | } |
| 10130 | if let Some(failure) = &artifacts.armfortas_failure { |
| 10131 | write_capture_result(&armfortas_root, &failure.partial_result())?; |
| 10132 | fs::write( |
| 10133 | armfortas_root.join("error.txt"), |
| 10134 | format!("stage: {}\n{}\n", failure.stage.as_str(), failure.detail), |
| 10135 | ) |
| 10136 | .map_err(|e| format!("cannot write armfortas error bundle: {}", e))?; |
| 10137 | } |
| 10138 | write_armfortas_observation_bundle(&armfortas_root, prepared, artifacts)?; |
| 10139 | |
| 10140 | if !artifacts.references.is_empty() { |
| 10141 | let refs_root = bundle_root.join("references"); |
| 10142 | fs::create_dir_all(&refs_root) |
| 10143 | .map_err(|e| format!("cannot create references bundle dir: {}", e))?; |
| 10144 | let reference_observations = reference_observations_for_bundle( |
| 10145 | &prepared.compiler_source, |
| 10146 | outcome.opt_level, |
| 10147 | artifacts, |
| 10148 | ); |
| 10149 | write_reference_summary_bundle(&refs_root, &artifacts.references, &reference_observations)?; |
| 10150 | for (index, reference) in artifacts.references.iter().enumerate() { |
| 10151 | write_reference_bundle( |
| 10152 | &refs_root, |
| 10153 | &prepared.compiler_source, |
| 10154 | outcome.opt_level, |
| 10155 | reference, |
| 10156 | reference_observations.get(index), |
| 10157 | )?; |
| 10158 | } |
| 10159 | } |
| 10160 | |
| 10161 | if !artifacts.consistency_issues.is_empty() { |
| 10162 | write_consistency_bundle(&bundle_root, &artifacts.consistency_issues)?; |
| 10163 | } |
| 10164 | |
| 10165 | Ok(bundle_root) |
| 10166 | } |
| 10167 | |
| 10168 | fn write_armfortas_bundle_metadata( |
| 10169 | armfortas_root: &Path, |
| 10170 | outcome: &Outcome, |
| 10171 | artifacts: &ExecutionArtifacts, |
| 10172 | ) -> Result<(), String> { |
| 10173 | let primary_backend_kind = outcome |
| 10174 | .primary_backend |
| 10175 | .as_ref() |
| 10176 | .map(|backend| backend.kind.as_str()) |
| 10177 | .unwrap_or("none"); |
| 10178 | let primary_backend_mode = outcome |
| 10179 | .primary_backend |
| 10180 | .as_ref() |
| 10181 | .map(|backend| backend.mode.as_str()) |
| 10182 | .unwrap_or("none"); |
| 10183 | let primary_backend_detail = outcome |
| 10184 | .primary_backend |
| 10185 | .as_ref() |
| 10186 | .map(|backend| backend.detail.as_str()) |
| 10187 | .unwrap_or("none"); |
| 10188 | let captured_stages = if let Some(result) = &artifacts.armfortas { |
| 10189 | join_or_none(&result.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10190 | } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10191 | join_or_none(&failure.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10192 | } else { |
| 10193 | "none".to_string() |
| 10194 | }; |
| 10195 | let error_stage = artifacts |
| 10196 | .armfortas_failure |
| 10197 | .as_ref() |
| 10198 | .map(|failure| failure.stage.as_str()) |
| 10199 | .unwrap_or("none"); |
| 10200 | let metadata = format!( |
| 10201 | "primary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\ncaptured_stages: {}\nerror_stage: {}\n", |
| 10202 | primary_backend_kind, |
| 10203 | primary_backend_mode, |
| 10204 | primary_backend_detail, |
| 10205 | captured_stages, |
| 10206 | error_stage |
| 10207 | ); |
| 10208 | fs::write(armfortas_root.join("metadata.txt"), metadata) |
| 10209 | .map_err(|e| format!("cannot write armfortas bundle metadata: {}", e)) |
| 10210 | } |
| 10211 | |
| 10212 | fn write_case_sources_bundle( |
| 10213 | bundle_root: &Path, |
| 10214 | case: &CaseSpec, |
| 10215 | prepared: &PreparedInput, |
| 10216 | ) -> Result<(), String> { |
| 10217 | if case.graph_files.is_empty() { |
| 10218 | let source_text = fs::read_to_string(&case.source) |
| 10219 | .map_err(|e| format!("cannot read case source '{}': {}", case.source.display(), e))?; |
| 10220 | fs::write(bundle_root.join("source.f90"), source_text) |
| 10221 | .map_err(|e| format!("cannot write bundle source copy: {}", e))?; |
| 10222 | return Ok(()); |
| 10223 | } |
| 10224 | |
| 10225 | let generated_source = prepared.generated_source.as_ref().ok_or_else(|| { |
| 10226 | format!( |
| 10227 | "graph case '{}' was missing a generated compiler source", |
| 10228 | case.name |
| 10229 | ) |
| 10230 | })?; |
| 10231 | let generated_text = fs::read_to_string(generated_source).map_err(|e| { |
| 10232 | format!( |
| 10233 | "cannot read generated graph source '{}': {}", |
| 10234 | generated_source.display(), |
| 10235 | e |
| 10236 | ) |
| 10237 | })?; |
| 10238 | fs::write(bundle_root.join("source.f90"), generated_text) |
| 10239 | .map_err(|e| format!("cannot write generated bundle source copy: {}", e))?; |
| 10240 | |
| 10241 | let sources_root = bundle_root.join("sources"); |
| 10242 | fs::create_dir_all(&sources_root) |
| 10243 | .map_err(|e| format!("cannot create bundle sources dir: {}", e))?; |
| 10244 | for (index, file) in case.graph_files.iter().enumerate() { |
| 10245 | let text = fs::read_to_string(file) |
| 10246 | .map_err(|e| format!("cannot read graph source '{}': {}", file.display(), e))?; |
| 10247 | let file_name = file |
| 10248 | .file_name() |
| 10249 | .and_then(|name| name.to_str()) |
| 10250 | .unwrap_or("source.f90"); |
| 10251 | let target = sources_root.join(format!("{:02}_{}", index, file_name)); |
| 10252 | fs::write(target, text).map_err(|e| format!("cannot write bundle graph source: {}", e))?; |
| 10253 | } |
| 10254 | |
| 10255 | Ok(()) |
| 10256 | } |
| 10257 | |
| 10258 | fn write_capture_result(root: &Path, result: &CaptureResult) -> Result<(), String> { |
| 10259 | for (stage, captured) in &result.stages { |
| 10260 | match captured { |
| 10261 | CapturedStage::Text(text) => { |
| 10262 | fs::write(root.join(format!("{}.txt", stage.as_str())), text).map_err(|e| { |
| 10263 | format!("cannot write '{}' stage bundle: {}", stage.as_str(), e) |
| 10264 | })?; |
| 10265 | } |
| 10266 | CapturedStage::Run(run) => { |
| 10267 | fs::write(root.join("run.stdout.txt"), &run.stdout) |
| 10268 | .map_err(|e| format!("cannot write run stdout bundle: {}", e))?; |
| 10269 | fs::write(root.join("run.stderr.txt"), &run.stderr) |
| 10270 | .map_err(|e| format!("cannot write run stderr bundle: {}", e))?; |
| 10271 | fs::write( |
| 10272 | root.join("run.exit_code.txt"), |
| 10273 | format!("{}\n", run.exit_code), |
| 10274 | ) |
| 10275 | .map_err(|e| format!("cannot write run exit-code bundle: {}", e))?; |
| 10276 | } |
| 10277 | } |
| 10278 | } |
| 10279 | Ok(()) |
| 10280 | } |
| 10281 | |
| 10282 | fn write_armfortas_observation_bundle( |
| 10283 | armfortas_root: &Path, |
| 10284 | prepared: &PreparedInput, |
| 10285 | artifacts: &ExecutionArtifacts, |
| 10286 | ) -> Result<(), String> { |
| 10287 | let observed = match observed_program_for_armfortas_bundle(prepared, artifacts) { |
| 10288 | Some(observed) => observed, |
| 10289 | None => return Ok(()), |
| 10290 | }; |
| 10291 | let render_config = IntrospectionRenderConfig { |
| 10292 | summary_only: false, |
| 10293 | max_artifact_lines: None, |
| 10294 | }; |
| 10295 | fs::write( |
| 10296 | armfortas_root.join("observation.txt"), |
| 10297 | render_introspection_text(&observed, render_config), |
| 10298 | ) |
| 10299 | .map_err(|e| format!("cannot write armfortas observation text bundle: {}", e))?; |
| 10300 | fs::write( |
| 10301 | armfortas_root.join("observation.json"), |
| 10302 | render_introspection_json(&observed), |
| 10303 | ) |
| 10304 | .map_err(|e| format!("cannot write armfortas observation json bundle: {}", e))?; |
| 10305 | fs::write( |
| 10306 | armfortas_root.join("observation.md"), |
| 10307 | render_introspection_markdown(&observed, render_config), |
| 10308 | ) |
| 10309 | .map_err(|e| format!("cannot write armfortas observation markdown bundle: {}", e))?; |
| 10310 | Ok(()) |
| 10311 | } |
| 10312 | |
| 10313 | fn observed_program_for_armfortas_bundle( |
| 10314 | prepared: &PreparedInput, |
| 10315 | artifacts: &ExecutionArtifacts, |
| 10316 | ) -> Option<ObservedProgram> { |
| 10317 | if let Some(observed) = &artifacts.armfortas_observation { |
| 10318 | Some(observed.clone()) |
| 10319 | } else if let Some(result) = &artifacts.armfortas { |
| 10320 | Some(observed_program_from_armfortas_capture( |
| 10321 | &prepared.compiler_source, |
| 10322 | result.opt_level, |
| 10323 | bundle_artifacts_for_capture_result(result), |
| 10324 | result, |
| 10325 | None, |
| 10326 | )) |
| 10327 | } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10328 | let partial = failure.partial_result(); |
| 10329 | Some(observed_program_from_armfortas_capture( |
| 10330 | &prepared.compiler_source, |
| 10331 | failure.opt_level, |
| 10332 | bundle_artifacts_for_capture_failure(failure), |
| 10333 | &partial, |
| 10334 | Some(failure), |
| 10335 | )) |
| 10336 | } else { |
| 10337 | None |
| 10338 | } |
| 10339 | } |
| 10340 | |
| 10341 | fn bundle_artifacts_for_capture_result(result: &CaptureResult) -> BTreeSet<ArtifactKey> { |
| 10342 | bundle_artifacts_for_stages(&result.stages) |
| 10343 | } |
| 10344 | |
| 10345 | fn bundle_artifacts_for_capture_failure(failure: &CaptureFailure) -> BTreeSet<ArtifactKey> { |
| 10346 | let mut requested = bundle_artifacts_for_stages(&failure.stages); |
| 10347 | requested.insert(ArtifactKey::Diagnostics); |
| 10348 | requested |
| 10349 | } |
| 10350 | |
| 10351 | fn bundle_artifacts_for_stages(stages: &BTreeMap<Stage, CapturedStage>) -> BTreeSet<ArtifactKey> { |
| 10352 | let mut requested = BTreeSet::new(); |
| 10353 | for (stage, captured) in stages { |
| 10354 | match (stage, captured) { |
| 10355 | (Stage::Asm, CapturedStage::Text(_)) => { |
| 10356 | requested.insert(ArtifactKey::Asm); |
| 10357 | } |
| 10358 | (Stage::Obj, CapturedStage::Text(_)) => { |
| 10359 | requested.insert(ArtifactKey::Obj); |
| 10360 | } |
| 10361 | (Stage::Run, CapturedStage::Run(_)) => { |
| 10362 | requested.insert(ArtifactKey::Runtime); |
| 10363 | } |
| 10364 | (stage, CapturedStage::Text(_)) => { |
| 10365 | requested.insert(ArtifactKey::Extra(format!("armfortas.{}", stage.as_str()))); |
| 10366 | } |
| 10367 | _ => {} |
| 10368 | } |
| 10369 | } |
| 10370 | requested |
| 10371 | } |
| 10372 | |
| 10373 | fn reference_observations_for_bundle( |
| 10374 | program: &Path, |
| 10375 | opt_level: OptLevel, |
| 10376 | artifacts: &ExecutionArtifacts, |
| 10377 | ) -> Vec<ObservedProgram> { |
| 10378 | if artifacts.reference_observations.len() == artifacts.references.len() { |
| 10379 | artifacts.reference_observations.clone() |
| 10380 | } else { |
| 10381 | artifacts |
| 10382 | .references |
| 10383 | .iter() |
| 10384 | .map(|reference| { |
| 10385 | observed_program_from_reference_result( |
| 10386 | program, |
| 10387 | opt_level, |
| 10388 | default_differential_artifacts(), |
| 10389 | reference, |
| 10390 | ) |
| 10391 | }) |
| 10392 | .collect() |
| 10393 | } |
| 10394 | } |
| 10395 | |
| 10396 | fn write_reference_summary_bundle( |
| 10397 | refs_root: &Path, |
| 10398 | references: &[ReferenceResult], |
| 10399 | observations: &[ObservedProgram], |
| 10400 | ) -> Result<(), String> { |
| 10401 | let summary = render_reference_bundle_summary(references, observations); |
| 10402 | fs::write(refs_root.join("summary.txt"), summary) |
| 10403 | .map_err(|e| format!("cannot write reference summary bundle: {}", e)) |
| 10404 | } |
| 10405 | |
| 10406 | fn render_reference_bundle_summary( |
| 10407 | references: &[ReferenceResult], |
| 10408 | observations: &[ObservedProgram], |
| 10409 | ) -> String { |
| 10410 | let mut lines = vec![ |
| 10411 | format!("reference_count: {}", references.len()), |
| 10412 | format!( |
| 10413 | "compilers: {}", |
| 10414 | if references.is_empty() { |
| 10415 | "none".to_string() |
| 10416 | } else { |
| 10417 | references |
| 10418 | .iter() |
| 10419 | .map(|reference| reference.compiler.as_str()) |
| 10420 | .collect::<Vec<_>>() |
| 10421 | .join(", ") |
| 10422 | } |
| 10423 | ), |
| 10424 | ]; |
| 10425 | |
| 10426 | for (reference, observed) in references.iter().zip(observations.iter()) { |
| 10427 | let observation = &observed.observation; |
| 10428 | lines.push(String::new()); |
| 10429 | lines.push(format!("compiler: {}", reference.compiler.as_str())); |
| 10430 | lines.push(format!("status: {}", introspection_status(observation))); |
| 10431 | lines.push(format!( |
| 10432 | "compile_exit_code: {}", |
| 10433 | observation.compile_exit_code |
| 10434 | )); |
| 10435 | lines.push(format!("command: {}", reference.compile_command)); |
| 10436 | lines.push(format!( |
| 10437 | "generic_artifacts: {}", |
| 10438 | join_or_none_from_strings( |
| 10439 | &observation_generic_artifacts(observation) |
| 10440 | .into_iter() |
| 10441 | .map(|(name, _)| name) |
| 10442 | .collect::<Vec<_>>() |
| 10443 | ) |
| 10444 | )); |
| 10445 | lines.push(format!( |
| 10446 | "adapter_extras: {}", |
| 10447 | format_adapter_extra_summary(&observation_adapter_extras(observation)) |
| 10448 | )); |
| 10449 | } |
| 10450 | |
| 10451 | lines.join("\n") + "\n" |
| 10452 | } |
| 10453 | |
| 10454 | fn write_reference_bundle( |
| 10455 | root: &Path, |
| 10456 | program: &Path, |
| 10457 | opt_level: OptLevel, |
| 10458 | reference: &ReferenceResult, |
| 10459 | observed: Option<&ObservedProgram>, |
| 10460 | ) -> Result<(), String> { |
| 10461 | let ref_root = root.join(sanitize_component(reference.compiler.as_str())); |
| 10462 | fs::create_dir_all(&ref_root) |
| 10463 | .map_err(|e| format!("cannot create reference bundle dir: {}", e))?; |
| 10464 | fs::write(ref_root.join("command.txt"), &reference.compile_command) |
| 10465 | .map_err(|e| format!("cannot write reference command bundle: {}", e))?; |
| 10466 | fs::write( |
| 10467 | ref_root.join("compile.exit_code.txt"), |
| 10468 | format!("{}\n", reference.compile_exit_code), |
| 10469 | ) |
| 10470 | .map_err(|e| format!("cannot write reference compile exit-code bundle: {}", e))?; |
| 10471 | fs::write( |
| 10472 | ref_root.join("compile.stdout.txt"), |
| 10473 | &reference.compile_stdout, |
| 10474 | ) |
| 10475 | .map_err(|e| format!("cannot write reference compile stdout bundle: {}", e))?; |
| 10476 | fs::write( |
| 10477 | ref_root.join("compile.stderr.txt"), |
| 10478 | &reference.compile_stderr, |
| 10479 | ) |
| 10480 | .map_err(|e| format!("cannot write reference compile stderr bundle: {}", e))?; |
| 10481 | if let Some(run) = &reference.run { |
| 10482 | fs::write(ref_root.join("run.stdout.txt"), &run.stdout) |
| 10483 | .map_err(|e| format!("cannot write reference run stdout bundle: {}", e))?; |
| 10484 | fs::write(ref_root.join("run.stderr.txt"), &run.stderr) |
| 10485 | .map_err(|e| format!("cannot write reference run stderr bundle: {}", e))?; |
| 10486 | fs::write( |
| 10487 | ref_root.join("run.exit_code.txt"), |
| 10488 | format!("{}\n", run.exit_code), |
| 10489 | ) |
| 10490 | .map_err(|e| format!("cannot write reference run exit-code bundle: {}", e))?; |
| 10491 | } |
| 10492 | if let Some(err) = &reference.run_error { |
| 10493 | fs::write(ref_root.join("run.error.txt"), err) |
| 10494 | .map_err(|e| format!("cannot write reference run error bundle: {}", e))?; |
| 10495 | } |
| 10496 | write_reference_observation_bundle(&ref_root, program, opt_level, reference, observed)?; |
| 10497 | Ok(()) |
| 10498 | } |
| 10499 | |
| 10500 | fn write_reference_observation_bundle( |
| 10501 | ref_root: &Path, |
| 10502 | program: &Path, |
| 10503 | opt_level: OptLevel, |
| 10504 | reference: &ReferenceResult, |
| 10505 | observed: Option<&ObservedProgram>, |
| 10506 | ) -> Result<(), String> { |
| 10507 | let observed = observed.cloned().unwrap_or_else(|| { |
| 10508 | observed_program_from_reference_result( |
| 10509 | program, |
| 10510 | opt_level, |
| 10511 | default_differential_artifacts(), |
| 10512 | reference, |
| 10513 | ) |
| 10514 | }); |
| 10515 | let render_config = IntrospectionRenderConfig { |
| 10516 | summary_only: false, |
| 10517 | max_artifact_lines: None, |
| 10518 | }; |
| 10519 | fs::write( |
| 10520 | ref_root.join("observation.txt"), |
| 10521 | render_introspection_text(&observed, render_config), |
| 10522 | ) |
| 10523 | .map_err(|e| format!("cannot write reference observation text bundle: {}", e))?; |
| 10524 | fs::write( |
| 10525 | ref_root.join("observation.json"), |
| 10526 | render_introspection_json(&observed), |
| 10527 | ) |
| 10528 | .map_err(|e| format!("cannot write reference observation json bundle: {}", e))?; |
| 10529 | fs::write( |
| 10530 | ref_root.join("observation.md"), |
| 10531 | render_introspection_markdown(&observed, render_config), |
| 10532 | ) |
| 10533 | .map_err(|e| format!("cannot write reference observation markdown bundle: {}", e))?; |
| 10534 | Ok(()) |
| 10535 | } |
| 10536 | |
| 10537 | fn render_consistency_bundle_summary(issues: &[ConsistencyIssue]) -> String { |
| 10538 | let mut rollups = BTreeMap::new(); |
| 10539 | for issue in issues { |
| 10540 | rollups |
| 10541 | .entry(issue.check) |
| 10542 | .or_insert_with(ConsistencyRollup::default) |
| 10543 | .record(&issue.observation()); |
| 10544 | } |
| 10545 | |
| 10546 | let mut aggregate = ConsistencyRollup::default(); |
| 10547 | for issue in issues { |
| 10548 | aggregate.record(&issue.observation()); |
| 10549 | } |
| 10550 | |
| 10551 | let checks = if rollups.is_empty() { |
| 10552 | "none".to_string() |
| 10553 | } else { |
| 10554 | rollups |
| 10555 | .keys() |
| 10556 | .map(ConsistencyCheck::as_str) |
| 10557 | .collect::<Vec<_>>() |
| 10558 | .join(", ") |
| 10559 | }; |
| 10560 | |
| 10561 | let mut lines = vec![ |
| 10562 | format!("issue_count: {}", issues.len()), |
| 10563 | format!("checks: {}", checks), |
| 10564 | ]; |
| 10565 | |
| 10566 | if !aggregate.repeat_counts.is_empty() { |
| 10567 | lines.push(format!( |
| 10568 | "repeat_counts: {}", |
| 10569 | join_usize_set(&aggregate.repeat_counts) |
| 10570 | )); |
| 10571 | } |
| 10572 | if !aggregate.unique_variant_counts.is_empty() { |
| 10573 | lines.push(format!( |
| 10574 | "unique_variants: {}", |
| 10575 | join_usize_set(&aggregate.unique_variant_counts) |
| 10576 | )); |
| 10577 | } |
| 10578 | if !aggregate.varying_components.is_empty() { |
| 10579 | lines.push(format!( |
| 10580 | "varying_components: {}", |
| 10581 | join_string_set(&aggregate.varying_components) |
| 10582 | )); |
| 10583 | } |
| 10584 | if !aggregate.stable_components.is_empty() { |
| 10585 | lines.push(format!( |
| 10586 | "stable_components: {}", |
| 10587 | join_string_set(&aggregate.stable_components) |
| 10588 | )); |
| 10589 | } |
| 10590 | |
| 10591 | if !rollups.is_empty() { |
| 10592 | lines.push(String::new()); |
| 10593 | lines.push("per_check:".to_string()); |
| 10594 | for (check, rollup) in rollups { |
| 10595 | lines.push(format!( |
| 10596 | " {}: {}", |
| 10597 | check.as_str(), |
| 10598 | render_consistency_rollup(&rollup) |
| 10599 | )); |
| 10600 | } |
| 10601 | } |
| 10602 | |
| 10603 | lines.push(String::new()); |
| 10604 | for issue in issues { |
| 10605 | lines.push(format!("check: {}", issue.check.as_str())); |
| 10606 | lines.push(format!("summary: {}", issue.summary)); |
| 10607 | if let Some(repeat_count) = issue.repeat_count { |
| 10608 | lines.push(format!("repeat_count: {}", repeat_count)); |
| 10609 | } |
| 10610 | if let Some(unique_variant_count) = issue.unique_variant_count { |
| 10611 | lines.push(format!("unique_variants: {}", unique_variant_count)); |
| 10612 | } |
| 10613 | if !issue.varying_components.is_empty() { |
| 10614 | lines.push(format!( |
| 10615 | "varying_components: {}", |
| 10616 | join_or_none_from_strings(&issue.varying_components) |
| 10617 | )); |
| 10618 | } |
| 10619 | if !issue.stable_components.is_empty() { |
| 10620 | lines.push(format!( |
| 10621 | "stable_components: {}", |
| 10622 | join_or_none_from_strings(&issue.stable_components) |
| 10623 | )); |
| 10624 | } |
| 10625 | lines.push(format!( |
| 10626 | "artifacts: {}", |
| 10627 | sanitize_component(issue.check.as_str()) |
| 10628 | )); |
| 10629 | lines.push(String::new()); |
| 10630 | } |
| 10631 | |
| 10632 | lines.join("\n") |
| 10633 | } |
| 10634 | |
| 10635 | fn write_consistency_bundle(root: &Path, issues: &[ConsistencyIssue]) -> Result<(), String> { |
| 10636 | let consistency_root = root.join("consistency"); |
| 10637 | fs::create_dir_all(&consistency_root) |
| 10638 | .map_err(|e| format!("cannot create consistency bundle dir: {}", e))?; |
| 10639 | |
| 10640 | let summary = render_consistency_bundle_summary(issues); |
| 10641 | fs::write(consistency_root.join("summary.txt"), summary) |
| 10642 | .map_err(|e| format!("cannot write consistency summary bundle: {}", e))?; |
| 10643 | |
| 10644 | for issue in issues { |
| 10645 | let issue_root = consistency_root.join(sanitize_component(issue.check.as_str())); |
| 10646 | fs::create_dir_all(&issue_root) |
| 10647 | .map_err(|e| format!("cannot create consistency issue bundle dir: {}", e))?; |
| 10648 | fs::write(issue_root.join("summary.txt"), &issue.summary) |
| 10649 | .map_err(|e| format!("cannot write consistency issue summary bundle: {}", e))?; |
| 10650 | fs::write(issue_root.join("detail.txt"), &issue.detail) |
| 10651 | .map_err(|e| format!("cannot write consistency issue detail bundle: {}", e))?; |
| 10652 | let runs_root = issue_root.join("artifacts"); |
| 10653 | copy_directory_recursive(&issue.temp_root, &runs_root)?; |
| 10654 | } |
| 10655 | |
| 10656 | Ok(()) |
| 10657 | } |
| 10658 | |
| 10659 | fn copy_directory_recursive(source: &Path, destination: &Path) -> Result<(), String> { |
| 10660 | fs::create_dir_all(destination).map_err(|e| { |
| 10661 | format!( |
| 10662 | "cannot create copied artifact dir '{}': {}", |
| 10663 | destination.display(), |
| 10664 | e |
| 10665 | ) |
| 10666 | })?; |
| 10667 | |
| 10668 | for entry in fs::read_dir(source) |
| 10669 | .map_err(|e| format!("cannot read artifact dir '{}': {}", source.display(), e))? |
| 10670 | { |
| 10671 | let entry = entry |
| 10672 | .map_err(|e| format!("cannot read artifact entry '{}': {}", source.display(), e))?; |
| 10673 | let source_path = entry.path(); |
| 10674 | let destination_path = destination.join(entry.file_name()); |
| 10675 | let file_type = entry.file_type().map_err(|e| { |
| 10676 | format!( |
| 10677 | "cannot read artifact type '{}': {}", |
| 10678 | source_path.display(), |
| 10679 | e |
| 10680 | ) |
| 10681 | })?; |
| 10682 | if file_type.is_dir() { |
| 10683 | copy_directory_recursive(&source_path, &destination_path)?; |
| 10684 | } else { |
| 10685 | fs::copy(&source_path, &destination_path).map_err(|e| { |
| 10686 | format!( |
| 10687 | "cannot copy artifact '{}' to '{}': {}", |
| 10688 | source_path.display(), |
| 10689 | destination_path.display(), |
| 10690 | e |
| 10691 | ) |
| 10692 | })?; |
| 10693 | } |
| 10694 | } |
| 10695 | |
| 10696 | Ok(()) |
| 10697 | } |
| 10698 | |
| 10699 | fn render_command(binary: &str, args: &[String]) -> String { |
| 10700 | let mut rendered = vec![quote_arg(binary)]; |
| 10701 | rendered.extend(args.iter().map(|arg| quote_arg(arg))); |
| 10702 | rendered.join(" ") |
| 10703 | } |
| 10704 | |
| 10705 | fn quote_arg(arg: &str) -> String { |
| 10706 | if arg |
| 10707 | .chars() |
| 10708 | .all(|ch| ch.is_ascii_alphanumeric() || "-_./".contains(ch)) |
| 10709 | { |
| 10710 | arg.to_string() |
| 10711 | } else { |
| 10712 | format!("{:?}", arg) |
| 10713 | } |
| 10714 | } |
| 10715 | |
| 10716 | fn sanitize_component(value: &str) -> String { |
| 10717 | let mut out = String::new(); |
| 10718 | for ch in value.chars() { |
| 10719 | if ch.is_ascii_alphanumeric() { |
| 10720 | out.push(ch.to_ascii_lowercase()); |
| 10721 | } else { |
| 10722 | out.push('_'); |
| 10723 | } |
| 10724 | } |
| 10725 | while out.contains("__") { |
| 10726 | out = out.replace("__", "_"); |
| 10727 | } |
| 10728 | out.trim_matches('_').to_string() |
| 10729 | } |
| 10730 | |
| 10731 | fn next_report_temp_root(compiler: ReferenceCompiler, opt_level: OptLevel) -> PathBuf { |
| 10732 | default_report_root().join(".tmp").join(format!( |
| 10733 | "{}_{}_{}", |
| 10734 | sanitize_component(compiler.as_str()), |
| 10735 | opt_level.as_str().to_ascii_lowercase(), |
| 10736 | next_report_suffix(opt_level) |
| 10737 | )) |
| 10738 | } |
| 10739 | |
| 10740 | fn next_primary_cli_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10741 | default_report_root().join(".tmp").join(format!( |
| 10742 | "primary_cli_{}_{}", |
| 10743 | opt_level.as_str().to_ascii_lowercase(), |
| 10744 | next_report_suffix(opt_level) |
| 10745 | )) |
| 10746 | } |
| 10747 | |
| 10748 | fn next_consistency_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10749 | default_report_root().join(".tmp").join(format!( |
| 10750 | "consistency_{}_{}", |
| 10751 | opt_level.as_str().to_ascii_lowercase(), |
| 10752 | next_report_suffix(opt_level) |
| 10753 | )) |
| 10754 | } |
| 10755 | |
| 10756 | fn next_report_suffix(opt_level: OptLevel) -> String { |
| 10757 | format!( |
| 10758 | "{}-{}-{:04}", |
| 10759 | opt_level.as_str().to_ascii_lowercase(), |
| 10760 | std::process::id(), |
| 10761 | REPORT_COUNTER.fetch_add(1, Ordering::Relaxed) |
| 10762 | ) |
| 10763 | } |
| 10764 | |
| 10765 | fn print_outcome(outcome: &Outcome) { |
| 10766 | let label = format!( |
| 10767 | "{}::{}[{}]", |
| 10768 | outcome.suite, |
| 10769 | outcome.case, |
| 10770 | outcome.opt_level.as_str() |
| 10771 | ); |
| 10772 | match outcome.kind { |
| 10773 | OutcomeKind::Pass => println!("PASS {}", label), |
| 10774 | OutcomeKind::Fail => { |
| 10775 | println!("FAIL {}", label); |
| 10776 | if !outcome.detail.is_empty() { |
| 10777 | println!("{}", outcome.detail); |
| 10778 | } |
| 10779 | } |
| 10780 | OutcomeKind::Xfail => { |
| 10781 | println!("XFAIL {}", label); |
| 10782 | if !outcome.detail.is_empty() { |
| 10783 | println!("{}", outcome.detail); |
| 10784 | } |
| 10785 | } |
| 10786 | OutcomeKind::Xpass => { |
| 10787 | println!("XPASS {}", label); |
| 10788 | if !outcome.detail.is_empty() { |
| 10789 | println!("{}", outcome.detail); |
| 10790 | } |
| 10791 | } |
| 10792 | OutcomeKind::Future => { |
| 10793 | println!("FUTURE {}", label); |
| 10794 | if !outcome.detail.is_empty() { |
| 10795 | println!("{}", outcome.detail); |
| 10796 | } |
| 10797 | } |
| 10798 | } |
| 10799 | if let Some(bundle) = &outcome.bundle { |
| 10800 | println!("bundle: {}", bundle.display()); |
| 10801 | } |
| 10802 | } |
| 10803 | |
| 10804 | fn print_summary(summary: &Summary) { |
| 10805 | println!(); |
| 10806 | println!("{}", render_summary(summary)); |
| 10807 | } |
| 10808 | |
| 10809 | #[derive(Debug, Clone)] |
| 10810 | struct Check { |
| 10811 | line_num: usize, |
| 10812 | pattern: String, |
| 10813 | negative: bool, |
| 10814 | kind: &'static str, |
| 10815 | } |
| 10816 | |
| 10817 | fn extract_checks(source: &str) -> Vec<Check> { |
| 10818 | source |
| 10819 | .lines() |
| 10820 | .enumerate() |
| 10821 | .filter_map(|(i, line)| { |
| 10822 | let trimmed = line.trim(); |
| 10823 | trimmed.strip_prefix("! CHECK:").map(|rest| Check { |
| 10824 | line_num: i + 1, |
| 10825 | pattern: rest.trim().to_string(), |
| 10826 | negative: false, |
| 10827 | kind: "CHECK", |
| 10828 | }) |
| 10829 | }) |
| 10830 | .collect() |
| 10831 | } |
| 10832 | |
| 10833 | fn extract_xfail_reason(source: &str) -> Option<String> { |
| 10834 | source.lines().find_map(|line| { |
| 10835 | line.trim() |
| 10836 | .strip_prefix("! XFAIL:") |
| 10837 | .map(|rest| rest.trim().to_string()) |
| 10838 | }) |
| 10839 | } |
| 10840 | |
| 10841 | fn extract_error_expected_patterns(source: &str) -> Vec<String> { |
| 10842 | source |
| 10843 | .lines() |
| 10844 | .filter_map(|line| { |
| 10845 | line.trim() |
| 10846 | .strip_prefix("! ERROR_EXPECTED:") |
| 10847 | .map(|rest| rest.trim().to_string()) |
| 10848 | }) |
| 10849 | .collect() |
| 10850 | } |
| 10851 | |
| 10852 | fn extract_ir_checks(source: &str) -> Vec<Check> { |
| 10853 | source |
| 10854 | .lines() |
| 10855 | .enumerate() |
| 10856 | .filter_map(|(i, line)| { |
| 10857 | let trimmed = line.trim(); |
| 10858 | if let Some(rest) = trimmed.strip_prefix("! IR_CHECK:") { |
| 10859 | Some(Check { |
| 10860 | line_num: i + 1, |
| 10861 | pattern: rest.trim().to_string(), |
| 10862 | negative: false, |
| 10863 | kind: "IR_CHECK", |
| 10864 | }) |
| 10865 | } else { |
| 10866 | trimmed.strip_prefix("! IR_NOT:").map(|rest| Check { |
| 10867 | line_num: i + 1, |
| 10868 | pattern: rest.trim().to_string(), |
| 10869 | negative: true, |
| 10870 | kind: "IR_NOT", |
| 10871 | }) |
| 10872 | } |
| 10873 | }) |
| 10874 | .collect() |
| 10875 | } |
| 10876 | |
| 10877 | fn match_checks(checks: &[Check], output: &str, case_name: &str) -> Result<(), String> { |
| 10878 | let output_lines: Vec<&str> = output.lines().collect(); |
| 10879 | let mut output_idx = 0; |
| 10880 | |
| 10881 | for check in checks { |
| 10882 | if check.negative { |
| 10883 | if output.contains(&check.pattern) { |
| 10884 | return Err(format!( |
| 10885 | "{}:{}: {} failed: substring '{}' appears in output\nfull output:\n{}", |
| 10886 | case_name, check.line_num, check.kind, check.pattern, output |
| 10887 | )); |
| 10888 | } |
| 10889 | continue; |
| 10890 | } |
| 10891 | |
| 10892 | let mut found = false; |
| 10893 | while output_idx < output_lines.len() { |
| 10894 | if output_lines[output_idx].trim().contains(&check.pattern) { |
| 10895 | found = true; |
| 10896 | output_idx += 1; |
| 10897 | break; |
| 10898 | } |
| 10899 | output_idx += 1; |
| 10900 | } |
| 10901 | if !found { |
| 10902 | return Err(format!( |
| 10903 | "{}:{}: {} failed: expected '{}' not found in remaining output\nfull output:\n{}", |
| 10904 | case_name, check.line_num, check.kind, check.pattern, output |
| 10905 | )); |
| 10906 | } |
| 10907 | } |
| 10908 | |
| 10909 | Ok(()) |
| 10910 | } |
| 10911 | |
| 10912 | fn target_uses_ir_comment_checks(target: &Target) -> bool { |
| 10913 | match target { |
| 10914 | Target::Stage(Stage::Ir) => true, |
| 10915 | Target::Artifact(ArtifactKey::Extra(name)) => name == "armfortas.ir", |
| 10916 | _ => false, |
| 10917 | } |
| 10918 | } |
| 10919 | |
| 10920 | #[cfg(test)] |
| 10921 | mod tests { |
| 10922 | use super::*; |
| 10923 | |
| 10924 | struct DummyBackend { |
| 10925 | mode: &'static str, |
| 10926 | detail: &'static str, |
| 10927 | } |
| 10928 | |
| 10929 | impl CaptureBackend for DummyBackend { |
| 10930 | fn mode_name(&self) -> &'static str { |
| 10931 | self.mode |
| 10932 | } |
| 10933 | |
| 10934 | fn description(&self) -> &'static str { |
| 10935 | self.detail |
| 10936 | } |
| 10937 | |
| 10938 | fn capture(&self, request: &CaptureRequest) -> Result<CaptureResult, CaptureFailure> { |
| 10939 | Err(CaptureFailure { |
| 10940 | input: request.input.clone(), |
| 10941 | opt_level: request.opt_level, |
| 10942 | stage: FailureStage::Ir, |
| 10943 | detail: self.detail.to_string(), |
| 10944 | stages: BTreeMap::new(), |
| 10945 | }) |
| 10946 | } |
| 10947 | } |
| 10948 | |
| 10949 | #[cfg(unix)] |
| 10950 | fn bencch_repo_root() -> PathBuf { |
| 10951 | PathBuf::from(env!("CARGO_MANIFEST_DIR")) |
| 10952 | .parent() |
| 10953 | .unwrap() |
| 10954 | .to_path_buf() |
| 10955 | } |
| 10956 | |
| 10957 | #[cfg(unix)] |
| 10958 | fn fake_compiler_fixture(name: &str) -> PathBuf { |
| 10959 | bencch_repo_root() |
| 10960 | .join("fixtures") |
| 10961 | .join("fake_compilers") |
| 10962 | .join(name) |
| 10963 | } |
| 10964 | |
| 10965 | #[cfg(unix)] |
| 10966 | fn runtime_fixture(name: &str) -> PathBuf { |
| 10967 | bencch_repo_root() |
| 10968 | .join("fixtures") |
| 10969 | .join("runtime") |
| 10970 | .join(name) |
| 10971 | } |
| 10972 | |
| 10973 | fn full_introspection_render_config() -> IntrospectionRenderConfig { |
| 10974 | IntrospectionRenderConfig { |
| 10975 | summary_only: false, |
| 10976 | max_artifact_lines: None, |
| 10977 | } |
| 10978 | } |
| 10979 | |
| 10980 | #[cfg(unix)] |
| 10981 | fn invalid_fixture(name: &str) -> PathBuf { |
| 10982 | bencch_repo_root() |
| 10983 | .join("fixtures") |
| 10984 | .join("invalid") |
| 10985 | .join(name) |
| 10986 | } |
| 10987 | |
| 10988 | #[cfg(unix)] |
| 10989 | fn ensure_fixture_executable(path: &Path) { |
| 10990 | let mut perms = fs::metadata(path).unwrap().permissions(); |
| 10991 | perms.set_mode(0o755); |
| 10992 | fs::set_permissions(path, perms).unwrap(); |
| 10993 | } |
| 10994 | |
| 10995 | #[cfg(unix)] |
| 10996 | fn command_is_available(name: &str) -> bool { |
| 10997 | Command::new("which") |
| 10998 | .arg(name) |
| 10999 | .output() |
| 11000 | .map(|output| output.status.success()) |
| 11001 | .unwrap_or(false) |
| 11002 | } |
| 11003 | |
| 11004 | #[cfg(unix)] |
| 11005 | fn armfortas_smoke_binary() -> Option<PathBuf> { |
| 11006 | if let Some(path) = std::env::var_os("BENCCH_ARMFORTAS_SMOKE_BIN") { |
| 11007 | let path = PathBuf::from(path); |
| 11008 | if path.is_file() { |
| 11009 | return Some(path); |
| 11010 | } |
| 11011 | } |
| 11012 | |
| 11013 | let candidate = bencch_repo_root() |
| 11014 | .parent()? |
| 11015 | .join("target") |
| 11016 | .join("debug") |
| 11017 | .join("armfortas"); |
| 11018 | if candidate.is_file() { |
| 11019 | Some(candidate) |
| 11020 | } else { |
| 11021 | None |
| 11022 | } |
| 11023 | } |
| 11024 | |
| 11025 | #[cfg(unix)] |
| 11026 | fn stable_runtime_compare_corpus() -> Vec<PathBuf> { |
| 11027 | [ |
| 11028 | "allocatable.f90", |
| 11029 | "do_while.f90", |
| 11030 | "exit_cycle.f90", |
| 11031 | "nested_loops.f90", |
| 11032 | "subroutine_call.f90", |
| 11033 | "string_fixed.f90", |
| 11034 | "if_else.f90", |
| 11035 | "mixed_types.f90", |
| 11036 | "select_case.f90", |
| 11037 | "function_call.f90", |
| 11038 | "real_function.f90", |
| 11039 | "where_construct.f90", |
| 11040 | ] |
| 11041 | .into_iter() |
| 11042 | .map(runtime_fixture) |
| 11043 | .collect() |
| 11044 | } |
| 11045 | |
| 11046 | fn stable_runtime_compare_opt_levels() -> Vec<OptLevel> { |
| 11047 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 11048 | } |
| 11049 | use crate::compiler::test_support::{ |
| 11050 | verify_module, BlockParam, FloatWidth, Function, Inst, InstKind, IntWidth, IrType, Module, |
| 11051 | Position, Span, Terminator, ValueId, |
| 11052 | }; |
| 11053 | #[cfg(unix)] |
| 11054 | use std::os::unix::fs::PermissionsExt; |
| 11055 | |
| 11056 | fn dummy_span() -> Span { |
| 11057 | Span { |
| 11058 | file_id: 0, |
| 11059 | start: Position { line: 1, col: 1 }, |
| 11060 | end: Position { line: 1, col: 1 }, |
| 11061 | } |
| 11062 | } |
| 11063 | |
| 11064 | #[test] |
| 11065 | fn primary_backend_selection_uses_observable_backend_for_external_cases() { |
| 11066 | let case = CaseSpec { |
| 11067 | name: "runtime_case".into(), |
| 11068 | source: PathBuf::from("demo.f90"), |
| 11069 | graph_files: Vec::new(), |
| 11070 | requested: BTreeSet::from([Stage::Run]), |
| 11071 | generic_introspect: None, |
| 11072 | generic_compare: None, |
| 11073 | opt_levels: vec![OptLevel::O0], |
| 11074 | repeat_count: 3, |
| 11075 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 11076 | consistency_checks: vec![ConsistencyCheck::CliRunReproducible], |
| 11077 | expectations: vec![Expectation::Contains { |
| 11078 | target: Target::RunStdout, |
| 11079 | needle: "42".into(), |
| 11080 | }], |
| 11081 | status_rules: Vec::new(), |
| 11082 | capability_policy: None, |
| 11083 | }; |
| 11084 | let requested = BTreeSet::from([Stage::Run]); |
| 11085 | let external_tools = ToolchainConfig { |
| 11086 | armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 11087 | gfortran: "gfortran".into(), |
| 11088 | flang_new: "flang-new".into(), |
| 11089 | lfortran: "lfortran".into(), |
| 11090 | ifort: "ifort".into(), |
| 11091 | ifx: "ifx".into(), |
| 11092 | nvfortran: "nvfortran".into(), |
| 11093 | system_as: "as".into(), |
| 11094 | otool: "otool".into(), |
| 11095 | nm: "nm".into(), |
| 11096 | }; |
| 11097 | |
| 11098 | assert_eq!( |
| 11099 | primary_backend_kind_for_case(&case, &requested, &external_tools), |
| 11100 | PrimaryCaptureBackendKind::Observable |
| 11101 | ); |
| 11102 | let selected = |
| 11103 | select_primary_capture_backend(&case, &requested, OptLevel::O0, &external_tools); |
| 11104 | assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11105 | |
| 11106 | let linked_tools = ToolchainConfig { |
| 11107 | armfortas: ArmfortasCliAdapter::Linked, |
| 11108 | ..external_tools.clone() |
| 11109 | }; |
| 11110 | assert_eq!( |
| 11111 | primary_backend_kind_for_case(&case, &requested, &linked_tools), |
| 11112 | PrimaryCaptureBackendKind::Full |
| 11113 | ); |
| 11114 | |
| 11115 | let mut capture_check_case = case.clone(); |
| 11116 | capture_check_case.consistency_checks = vec![ConsistencyCheck::CaptureRunVsCliRun]; |
| 11117 | assert_eq!( |
| 11118 | primary_backend_kind_for_case(&capture_check_case, &requested, &external_tools), |
| 11119 | PrimaryCaptureBackendKind::Full |
| 11120 | ); |
| 11121 | |
| 11122 | let mut failure_case = case.clone(); |
| 11123 | failure_case.expectations.push(Expectation::FailContains { |
| 11124 | stage: FailureStage::Run, |
| 11125 | needle: "broken".into(), |
| 11126 | }); |
| 11127 | assert_eq!( |
| 11128 | primary_backend_kind_for_case(&failure_case, &requested, &external_tools), |
| 11129 | PrimaryCaptureBackendKind::Full |
| 11130 | ); |
| 11131 | |
| 11132 | let richer_request = BTreeSet::from([Stage::Run, Stage::Asm]); |
| 11133 | assert_eq!( |
| 11134 | primary_backend_kind_for_case(&case, &richer_request, &external_tools), |
| 11135 | PrimaryCaptureBackendKind::Observable |
| 11136 | ); |
| 11137 | |
| 11138 | let asm_only_request = BTreeSet::from([Stage::Asm]); |
| 11139 | assert_eq!( |
| 11140 | primary_backend_kind_for_case(&case, &asm_only_request, &external_tools), |
| 11141 | PrimaryCaptureBackendKind::Observable |
| 11142 | ); |
| 11143 | } |
| 11144 | |
| 11145 | #[test] |
| 11146 | fn legacy_unavailable_backend_detail_is_explicit() { |
| 11147 | let case = CaseSpec { |
| 11148 | name: "frontend_case".into(), |
| 11149 | source: PathBuf::from("frontend.f90"), |
| 11150 | graph_files: Vec::new(), |
| 11151 | requested: BTreeSet::from([Stage::Tokens]), |
| 11152 | generic_introspect: None, |
| 11153 | generic_compare: None, |
| 11154 | opt_levels: vec![OptLevel::O0], |
| 11155 | repeat_count: 1, |
| 11156 | reference_compilers: Vec::new(), |
| 11157 | consistency_checks: Vec::new(), |
| 11158 | expectations: vec![Expectation::Contains { |
| 11159 | target: Target::Stage(Stage::Tokens), |
| 11160 | needle: "program".into(), |
| 11161 | }], |
| 11162 | status_rules: Vec::new(), |
| 11163 | capability_policy: None, |
| 11164 | }; |
| 11165 | let backend = SelectedPrimaryBackend { |
| 11166 | kind: PrimaryCaptureBackendKind::Full, |
| 11167 | backend: Box::new(DummyBackend { |
| 11168 | mode: "unavailable", |
| 11169 | detail: "unavailable without linked-armfortas feature", |
| 11170 | }), |
| 11171 | }; |
| 11172 | |
| 11173 | let detail = legacy_unavailable_backend_detail(&case, &backend).unwrap(); |
| 11174 | assert!(detail.contains("case requires linked armfortas capture")); |
| 11175 | assert!(detail.contains("scripts/bootstrap-linked-armfortas.sh")); |
| 11176 | } |
| 11177 | |
| 11178 | #[test] |
| 11179 | fn legacy_cli_consistency_cases_use_generic_observation_path() { |
| 11180 | let cli_only_case = CaseSpec { |
| 11181 | name: "cli-consistency".into(), |
| 11182 | source: PathBuf::from("demo.f90"), |
| 11183 | graph_files: Vec::new(), |
| 11184 | requested: BTreeSet::from([Stage::Run]), |
| 11185 | generic_introspect: None, |
| 11186 | generic_compare: None, |
| 11187 | opt_levels: vec![OptLevel::O0], |
| 11188 | repeat_count: 3, |
| 11189 | reference_compilers: Vec::new(), |
| 11190 | consistency_checks: vec![ |
| 11191 | ConsistencyCheck::CliAsmReproducible, |
| 11192 | ConsistencyCheck::CliRunReproducible, |
| 11193 | ], |
| 11194 | expectations: vec![Expectation::Contains { |
| 11195 | target: Target::RunStdout, |
| 11196 | needle: "42".into(), |
| 11197 | }], |
| 11198 | status_rules: Vec::new(), |
| 11199 | capability_policy: None, |
| 11200 | }; |
| 11201 | assert!(legacy_case_uses_generic_consistency_checks(&cli_only_case)); |
| 11202 | |
| 11203 | let mixed_case = CaseSpec { |
| 11204 | consistency_checks: vec![ |
| 11205 | ConsistencyCheck::CliRunReproducible, |
| 11206 | ConsistencyCheck::CaptureRunReproducible, |
| 11207 | ], |
| 11208 | ..cli_only_case.clone() |
| 11209 | }; |
| 11210 | assert!(!legacy_case_uses_generic_consistency_checks(&mixed_case)); |
| 11211 | } |
| 11212 | |
| 11213 | #[test] |
| 11214 | fn legacy_observable_cases_use_generic_observation_execution() { |
| 11215 | let observable_case = CaseSpec { |
| 11216 | name: "observable".into(), |
| 11217 | source: PathBuf::from("demo.f90"), |
| 11218 | graph_files: Vec::new(), |
| 11219 | requested: BTreeSet::from([Stage::Run]), |
| 11220 | generic_introspect: None, |
| 11221 | generic_compare: None, |
| 11222 | opt_levels: vec![OptLevel::O0], |
| 11223 | repeat_count: 3, |
| 11224 | reference_compilers: Vec::new(), |
| 11225 | consistency_checks: Vec::new(), |
| 11226 | expectations: vec![Expectation::Contains { |
| 11227 | target: Target::RunStdout, |
| 11228 | needle: "42".into(), |
| 11229 | }], |
| 11230 | status_rules: Vec::new(), |
| 11231 | capability_policy: None, |
| 11232 | }; |
| 11233 | assert!(legacy_case_uses_generic_observation_execution( |
| 11234 | &observable_case, |
| 11235 | &observable_case.requested |
| 11236 | )); |
| 11237 | |
| 11238 | let richer_case = CaseSpec { |
| 11239 | requested: BTreeSet::from([Stage::Run, Stage::Ir]), |
| 11240 | ..observable_case.clone() |
| 11241 | }; |
| 11242 | assert!(!legacy_case_uses_generic_observation_execution( |
| 11243 | &richer_case, |
| 11244 | &richer_case.requested |
| 11245 | )); |
| 11246 | |
| 11247 | let failure_case = CaseSpec { |
| 11248 | expectations: vec![Expectation::FailContains { |
| 11249 | stage: FailureStage::Run, |
| 11250 | needle: "boom".into(), |
| 11251 | }], |
| 11252 | ..observable_case |
| 11253 | }; |
| 11254 | assert!(!legacy_case_uses_generic_observation_execution( |
| 11255 | &failure_case, |
| 11256 | &failure_case.requested |
| 11257 | )); |
| 11258 | } |
| 11259 | |
| 11260 | #[cfg(unix)] |
| 11261 | #[test] |
| 11262 | fn external_cli_primary_execution_returns_observable_stages() { |
| 11263 | let root = std::env::temp_dir().join("afs_tests_external_cli_primary"); |
| 11264 | let _ = fs::remove_dir_all(&root); |
| 11265 | fs::create_dir_all(&root).unwrap(); |
| 11266 | |
| 11267 | let source = root.join("demo.f90"); |
| 11268 | fs::write(&source, "program demo\nprint *, 42\nend program\n").unwrap(); |
| 11269 | |
| 11270 | let compiler = root.join("fake-armfortas"); |
| 11271 | fs::write( |
| 11272 | &compiler, |
| 11273 | "#!/bin/sh\nmode=bin\nout=\"\"\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n -S)\n mode=asm\n shift\n ;;\n -c)\n mode=obj\n shift\n ;;\n -o)\n out=\"$2\"\n shift 2\n ;;\n *)\n shift\n ;;\n esac\ndone\nif [ \"$mode\" = asm ]; then\n cat > \"$out\" <<'EOF'\n.globl _main\n_main:\n ret\nEOF\nelif [ \"$mode\" = obj ]; then\n printf 'fake object\\n' > \"$out\"\nelse\n cat > \"$out\" <<'EOF'\n#!/bin/sh\nprintf '42\\n'\nEOF\n chmod +x \"$out\"\nfi\n", |
| 11274 | ) |
| 11275 | .unwrap(); |
| 11276 | let mut perms = fs::metadata(&compiler).unwrap().permissions(); |
| 11277 | perms.set_mode(0o755); |
| 11278 | fs::set_permissions(&compiler, perms).unwrap(); |
| 11279 | |
| 11280 | let case = CaseSpec { |
| 11281 | name: "runtime_case".into(), |
| 11282 | source: source.clone(), |
| 11283 | graph_files: Vec::new(), |
| 11284 | requested: BTreeSet::from([Stage::Asm, Stage::Run]), |
| 11285 | generic_introspect: None, |
| 11286 | generic_compare: None, |
| 11287 | opt_levels: vec![OptLevel::O0], |
| 11288 | repeat_count: 3, |
| 11289 | reference_compilers: Vec::new(), |
| 11290 | consistency_checks: Vec::new(), |
| 11291 | expectations: vec![ |
| 11292 | Expectation::Contains { |
| 11293 | target: Target::Stage(Stage::Asm), |
| 11294 | needle: ".globl _main".into(), |
| 11295 | }, |
| 11296 | Expectation::Contains { |
| 11297 | target: Target::RunStdout, |
| 11298 | needle: "42".into(), |
| 11299 | }, |
| 11300 | ], |
| 11301 | status_rules: Vec::new(), |
| 11302 | capability_policy: None, |
| 11303 | }; |
| 11304 | let prepared = PreparedInput { |
| 11305 | compiler_source: source.clone(), |
| 11306 | generated_source: None, |
| 11307 | temp_root: None, |
| 11308 | }; |
| 11309 | let tools = ToolchainConfig { |
| 11310 | armfortas: ArmfortasCliAdapter::External(compiler.display().to_string()), |
| 11311 | gfortran: "gfortran".into(), |
| 11312 | flang_new: "flang-new".into(), |
| 11313 | lfortran: "lfortran".into(), |
| 11314 | ifort: "ifort".into(), |
| 11315 | ifx: "ifx".into(), |
| 11316 | nvfortran: "nvfortran".into(), |
| 11317 | system_as: "as".into(), |
| 11318 | otool: "otool".into(), |
| 11319 | nm: "nm".into(), |
| 11320 | }; |
| 11321 | let requested = BTreeSet::from([Stage::Asm, Stage::Run]); |
| 11322 | let selected = select_primary_capture_backend(&case, &requested, OptLevel::O0, &tools); |
| 11323 | assert_eq!(selected.kind, PrimaryCaptureBackendKind::Observable); |
| 11324 | assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11325 | |
| 11326 | let result = |
| 11327 | execute_primary_armfortas(&prepared, OptLevel::O0, &requested, &selected).unwrap(); |
| 11328 | let asm = capture_text_stage(&result, Stage::Asm).unwrap(); |
| 11329 | let run = capture_run_stage(&result).unwrap(); |
| 11330 | assert!(asm.contains(".globl _main")); |
| 11331 | assert_eq!(run.exit_code, 0); |
| 11332 | assert_eq!(run.stdout, "42\n"); |
| 11333 | assert!(run.stderr.is_empty()); |
| 11334 | assert_eq!(result.stages.len(), 2); |
| 11335 | |
| 11336 | let _ = fs::remove_dir_all(&root); |
| 11337 | } |
| 11338 | |
| 11339 | #[cfg(unix)] |
| 11340 | #[test] |
| 11341 | fn compare_uses_generic_external_driver_observations() { |
| 11342 | let compiler_a = fake_compiler_fixture("match_42_a.sh"); |
| 11343 | let compiler_b = fake_compiler_fixture("runtime_41.sh"); |
| 11344 | let source = runtime_fixture("mixed_types.f90"); |
| 11345 | ensure_fixture_executable(&compiler_a); |
| 11346 | ensure_fixture_executable(&compiler_b); |
| 11347 | |
| 11348 | let config = CompareConfig { |
| 11349 | left: CompilerSpec::Binary(compiler_a.clone()), |
| 11350 | right: CompilerSpec::Binary(compiler_b.clone()), |
| 11351 | program: source.clone(), |
| 11352 | opt_level: OptLevel::O0, |
| 11353 | artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 11354 | json_report: None, |
| 11355 | markdown_report: None, |
| 11356 | tools: ToolchainConfig::from_env(), |
| 11357 | }; |
| 11358 | |
| 11359 | let result = run_compare(&config).unwrap(); |
| 11360 | assert_eq!(result.left.provenance.backend_mode, "external-driver"); |
| 11361 | assert_eq!(result.right.provenance.backend_mode, "external-driver"); |
| 11362 | assert!(result |
| 11363 | .differences |
| 11364 | .iter() |
| 11365 | .any(|difference| difference.artifact == "runtime")); |
| 11366 | assert!(result |
| 11367 | .differences |
| 11368 | .iter() |
| 11369 | .any(|difference| difference.artifact == "asm")); |
| 11370 | let rendered = render_compare_text(&result); |
| 11371 | assert!(rendered.contains("status: diff")); |
| 11372 | assert!(rendered.contains("classification: mixed divergence")); |
| 11373 | assert!(rendered.contains("difference_count: 2")); |
| 11374 | } |
| 11375 | |
| 11376 | #[cfg(unix)] |
| 11377 | #[test] |
| 11378 | fn compare_fixture_compilers_report_compile_failures() { |
| 11379 | let source = runtime_fixture("mixed_types.f90"); |
| 11380 | let compiler_fail = fake_compiler_fixture("compile_fail.sh"); |
| 11381 | let compiler_ok = fake_compiler_fixture("match_42_a.sh"); |
| 11382 | ensure_fixture_executable(&compiler_fail); |
| 11383 | ensure_fixture_executable(&compiler_ok); |
| 11384 | |
| 11385 | let config = CompareConfig { |
| 11386 | left: CompilerSpec::Binary(compiler_fail), |
| 11387 | right: CompilerSpec::Binary(compiler_ok), |
| 11388 | program: source, |
| 11389 | opt_level: OptLevel::O0, |
| 11390 | artifacts: BTreeSet::new(), |
| 11391 | json_report: None, |
| 11392 | markdown_report: None, |
| 11393 | tools: ToolchainConfig::from_env(), |
| 11394 | }; |
| 11395 | |
| 11396 | let result = run_compare(&config).unwrap(); |
| 11397 | assert_eq!(compare_status(&result), "diff"); |
| 11398 | assert_eq!(compare_classification(&result), "compile divergence"); |
| 11399 | assert!(result |
| 11400 | .differences |
| 11401 | .iter() |
| 11402 | .any(|difference| difference.artifact == "compile-exit-code")); |
| 11403 | let diagnostics = result |
| 11404 | .differences |
| 11405 | .iter() |
| 11406 | .find(|difference| difference.artifact == "diagnostics") |
| 11407 | .unwrap(); |
| 11408 | assert!(diagnostics |
| 11409 | .detail |
| 11410 | .contains("fake compiler failure: missing lowering pass")); |
| 11411 | } |
| 11412 | |
| 11413 | #[test] |
| 11414 | fn compare_rejects_capability_mismatch_for_namespaced_artifacts() { |
| 11415 | let config = CompareConfig { |
| 11416 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11417 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11418 | program: runtime_fixture("if_else.f90"), |
| 11419 | opt_level: OptLevel::O0, |
| 11420 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 11421 | json_report: None, |
| 11422 | markdown_report: None, |
| 11423 | tools: ToolchainConfig::from_env(), |
| 11424 | }; |
| 11425 | |
| 11426 | let err = run_compare(&config).unwrap_err(); |
| 11427 | assert!(err.contains("compare request is not supported")); |
| 11428 | assert!(err.contains("right:")); |
| 11429 | assert!(err.contains("gfortran does not support requested artifacts")); |
| 11430 | assert!(err.contains("armfortas.ir")); |
| 11431 | } |
| 11432 | |
| 11433 | #[test] |
| 11434 | fn compare_executable_artifact_uses_file_contents_not_paths() { |
| 11435 | let root = std::env::temp_dir().join("bencch_compare_executable_paths"); |
| 11436 | let _ = fs::remove_dir_all(&root); |
| 11437 | fs::create_dir_all(&root).unwrap(); |
| 11438 | |
| 11439 | let left_exe = root.join("left.out"); |
| 11440 | let right_exe = root.join("right.out"); |
| 11441 | fs::write(&left_exe, b"same executable bytes").unwrap(); |
| 11442 | fs::write(&right_exe, b"same executable bytes").unwrap(); |
| 11443 | |
| 11444 | let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11445 | let left = CompilerObservation { |
| 11446 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11447 | program: PathBuf::from("demo.f90"), |
| 11448 | opt_level: OptLevel::O0, |
| 11449 | compile_exit_code: 0, |
| 11450 | artifacts: BTreeMap::from([( |
| 11451 | ArtifactKey::Executable, |
| 11452 | ArtifactValue::Path(left_exe.clone()), |
| 11453 | )]), |
| 11454 | provenance: ObservationProvenance { |
| 11455 | compiler_identity: "left".into(), |
| 11456 | adapter_kind: "explicit-path".into(), |
| 11457 | backend_mode: "external-driver".into(), |
| 11458 | backend_detail: "left detail".into(), |
| 11459 | artifacts_captured: vec!["executable".into()], |
| 11460 | comparison_basis: None, |
| 11461 | failure_stage: None, |
| 11462 | }, |
| 11463 | }; |
| 11464 | let right = CompilerObservation { |
| 11465 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11466 | program: PathBuf::from("demo.f90"), |
| 11467 | opt_level: OptLevel::O0, |
| 11468 | compile_exit_code: 0, |
| 11469 | artifacts: BTreeMap::from([( |
| 11470 | ArtifactKey::Executable, |
| 11471 | ArtifactValue::Path(right_exe.clone()), |
| 11472 | )]), |
| 11473 | provenance: ObservationProvenance { |
| 11474 | compiler_identity: "right".into(), |
| 11475 | adapter_kind: "explicit-path".into(), |
| 11476 | backend_mode: "external-driver".into(), |
| 11477 | backend_detail: "right detail".into(), |
| 11478 | artifacts_captured: vec!["executable".into()], |
| 11479 | comparison_basis: None, |
| 11480 | failure_stage: None, |
| 11481 | }, |
| 11482 | }; |
| 11483 | |
| 11484 | let result = compare_observations(left, right, &requested); |
| 11485 | assert!(result.differences.is_empty()); |
| 11486 | |
| 11487 | let _ = fs::remove_dir_all(&root); |
| 11488 | } |
| 11489 | |
| 11490 | #[test] |
| 11491 | fn compare_executable_artifact_reports_binary_difference() { |
| 11492 | let root = std::env::temp_dir().join("bencch_compare_executable_bytes"); |
| 11493 | let _ = fs::remove_dir_all(&root); |
| 11494 | fs::create_dir_all(&root).unwrap(); |
| 11495 | |
| 11496 | let left_exe = root.join("left.out"); |
| 11497 | let right_exe = root.join("right.out"); |
| 11498 | fs::write(&left_exe, b"abc").unwrap(); |
| 11499 | fs::write(&right_exe, b"axc").unwrap(); |
| 11500 | |
| 11501 | let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11502 | let left = CompilerObservation { |
| 11503 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11504 | program: PathBuf::from("demo.f90"), |
| 11505 | opt_level: OptLevel::O0, |
| 11506 | compile_exit_code: 0, |
| 11507 | artifacts: BTreeMap::from([( |
| 11508 | ArtifactKey::Executable, |
| 11509 | ArtifactValue::Path(left_exe.clone()), |
| 11510 | )]), |
| 11511 | provenance: ObservationProvenance { |
| 11512 | compiler_identity: "left".into(), |
| 11513 | adapter_kind: "explicit-path".into(), |
| 11514 | backend_mode: "external-driver".into(), |
| 11515 | backend_detail: "left detail".into(), |
| 11516 | artifacts_captured: vec!["executable".into()], |
| 11517 | comparison_basis: None, |
| 11518 | failure_stage: None, |
| 11519 | }, |
| 11520 | }; |
| 11521 | let right = CompilerObservation { |
| 11522 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11523 | program: PathBuf::from("demo.f90"), |
| 11524 | opt_level: OptLevel::O0, |
| 11525 | compile_exit_code: 0, |
| 11526 | artifacts: BTreeMap::from([( |
| 11527 | ArtifactKey::Executable, |
| 11528 | ArtifactValue::Path(right_exe.clone()), |
| 11529 | )]), |
| 11530 | provenance: ObservationProvenance { |
| 11531 | compiler_identity: "right".into(), |
| 11532 | adapter_kind: "explicit-path".into(), |
| 11533 | backend_mode: "external-driver".into(), |
| 11534 | backend_detail: "right detail".into(), |
| 11535 | artifacts_captured: vec!["executable".into()], |
| 11536 | comparison_basis: None, |
| 11537 | failure_stage: None, |
| 11538 | }, |
| 11539 | }; |
| 11540 | |
| 11541 | let result = compare_observations(left, right, &requested); |
| 11542 | assert_eq!(result.differences.len(), 1); |
| 11543 | assert_eq!(result.differences[0].artifact, "executable"); |
| 11544 | assert!(result.differences[0] |
| 11545 | .detail |
| 11546 | .contains("first differing byte: 1")); |
| 11547 | |
| 11548 | let _ = fs::remove_dir_all(&root); |
| 11549 | } |
| 11550 | |
| 11551 | #[cfg(unix)] |
| 11552 | #[test] |
| 11553 | fn compare_cli_with_fixture_compilers_writes_match_reports() { |
| 11554 | let left = fake_compiler_fixture("match_42_a.sh"); |
| 11555 | let right = fake_compiler_fixture("match_42_b.sh"); |
| 11556 | let source = runtime_fixture("mixed_types.f90"); |
| 11557 | ensure_fixture_executable(&left); |
| 11558 | ensure_fixture_executable(&right); |
| 11559 | |
| 11560 | let root = std::env::temp_dir().join("bencch_compare_fixture_reports"); |
| 11561 | let _ = fs::remove_dir_all(&root); |
| 11562 | fs::create_dir_all(&root).unwrap(); |
| 11563 | let json_report = root.join("compare.json"); |
| 11564 | let markdown_report = root.join("compare.md"); |
| 11565 | |
| 11566 | let args = vec![ |
| 11567 | "compare".to_string(), |
| 11568 | left.display().to_string(), |
| 11569 | right.display().to_string(), |
| 11570 | "--program".to_string(), |
| 11571 | source.display().to_string(), |
| 11572 | "--artifact".to_string(), |
| 11573 | "asm,obj".to_string(), |
| 11574 | "--json-report".to_string(), |
| 11575 | json_report.display().to_string(), |
| 11576 | "--markdown-report".to_string(), |
| 11577 | markdown_report.display().to_string(), |
| 11578 | ]; |
| 11579 | |
| 11580 | let exit = run_cli_named("bencch", &args); |
| 11581 | assert_eq!(exit, 0); |
| 11582 | |
| 11583 | let json = fs::read_to_string(&json_report).unwrap(); |
| 11584 | assert!(json.contains("\"status\": \"match\"")); |
| 11585 | assert!(json.contains("\"classification\": \"match\"")); |
| 11586 | assert!(json.contains("\"difference_count\": 0")); |
| 11587 | assert!(json.contains("\"changed_artifacts\": []")); |
| 11588 | |
| 11589 | let markdown = fs::read_to_string(&markdown_report).unwrap(); |
| 11590 | assert!(markdown.contains("status: match")); |
| 11591 | assert!(markdown.contains("classification: match")); |
| 11592 | assert!(markdown.contains("difference_count: 0")); |
| 11593 | assert!(markdown.contains("changed_artifacts: none")); |
| 11594 | |
| 11595 | let _ = fs::remove_dir_all(&root); |
| 11596 | } |
| 11597 | |
| 11598 | #[cfg(unix)] |
| 11599 | #[test] |
| 11600 | fn compare_named_real_compilers_match_on_runtime_corpus() { |
| 11601 | if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 11602 | return; |
| 11603 | } |
| 11604 | |
| 11605 | for opt_level in stable_runtime_compare_opt_levels() { |
| 11606 | for program in stable_runtime_compare_corpus() { |
| 11607 | let config = CompareConfig { |
| 11608 | left: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11609 | right: CompilerSpec::Named(NamedCompiler::FlangNew), |
| 11610 | program, |
| 11611 | opt_level, |
| 11612 | artifacts: BTreeSet::new(), |
| 11613 | json_report: None, |
| 11614 | markdown_report: None, |
| 11615 | tools: ToolchainConfig::from_env(), |
| 11616 | }; |
| 11617 | |
| 11618 | let result = run_compare(&config).unwrap(); |
| 11619 | assert_eq!(compare_status(&result), "match"); |
| 11620 | assert_eq!(compare_classification(&result), "match"); |
| 11621 | assert!(result.differences.is_empty()); |
| 11622 | assert_eq!(result.left.provenance.adapter_kind, "named"); |
| 11623 | assert_eq!(result.right.provenance.adapter_kind, "named"); |
| 11624 | assert_eq!(result.left.opt_level, opt_level); |
| 11625 | assert_eq!(result.right.opt_level, opt_level); |
| 11626 | } |
| 11627 | } |
| 11628 | } |
| 11629 | |
| 11630 | #[cfg(unix)] |
| 11631 | #[test] |
| 11632 | fn compare_armfortas_and_gfortran_match_on_runtime_corpus_when_available() { |
| 11633 | if !command_is_available("gfortran") { |
| 11634 | return; |
| 11635 | } |
| 11636 | let Some(armfortas_bin) = armfortas_smoke_binary() else { |
| 11637 | return; |
| 11638 | }; |
| 11639 | |
| 11640 | let mut tools = ToolchainConfig::from_env(); |
| 11641 | tools.armfortas = ArmfortasCliAdapter::External(armfortas_bin.display().to_string()); |
| 11642 | |
| 11643 | for opt_level in stable_runtime_compare_opt_levels() { |
| 11644 | for program in stable_runtime_compare_corpus() { |
| 11645 | let config = CompareConfig { |
| 11646 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11647 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11648 | program, |
| 11649 | opt_level, |
| 11650 | artifacts: BTreeSet::new(), |
| 11651 | json_report: None, |
| 11652 | markdown_report: None, |
| 11653 | tools: tools.clone(), |
| 11654 | }; |
| 11655 | |
| 11656 | let result = run_compare(&config).unwrap(); |
| 11657 | assert_eq!(compare_status(&result), "match"); |
| 11658 | assert_eq!(compare_classification(&result), "match"); |
| 11659 | assert!(result.differences.is_empty()); |
| 11660 | assert_eq!(result.left.provenance.backend_mode, "cli-observable"); |
| 11661 | assert_eq!(result.left.opt_level, opt_level); |
| 11662 | assert_eq!(result.right.opt_level, opt_level); |
| 11663 | } |
| 11664 | } |
| 11665 | } |
| 11666 | |
| 11667 | #[cfg(unix)] |
| 11668 | #[test] |
| 11669 | fn introspect_armfortas_rich_artifacts_on_runtime_fixture() { |
| 11670 | let config = IntrospectConfig { |
| 11671 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11672 | program: runtime_fixture("if_else.f90"), |
| 11673 | opt_level: OptLevel::O0, |
| 11674 | artifacts: BTreeSet::from([ |
| 11675 | ArtifactKey::Asm, |
| 11676 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11677 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11678 | ]), |
| 11679 | json_report: None, |
| 11680 | markdown_report: None, |
| 11681 | all_artifacts: false, |
| 11682 | summary_only: false, |
| 11683 | max_artifact_lines: None, |
| 11684 | tools: ToolchainConfig::from_env(), |
| 11685 | }; |
| 11686 | |
| 11687 | let observed = run_introspect(&config).unwrap(); |
| 11688 | let observation = &observed.observation; |
| 11689 | assert_eq!(observation.compile_exit_code, 0); |
| 11690 | assert_eq!(observation.provenance.backend_mode, "linked"); |
| 11691 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 11692 | assert!(observation |
| 11693 | .artifacts |
| 11694 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 11695 | assert!(observation |
| 11696 | .artifacts |
| 11697 | .contains_key(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 11698 | |
| 11699 | let ir = match observation |
| 11700 | .artifacts |
| 11701 | .get(&ArtifactKey::Extra("armfortas.ir".into())) |
| 11702 | .unwrap() |
| 11703 | { |
| 11704 | ArtifactValue::Text(text) => text, |
| 11705 | other => panic!("expected text ir artifact, got {:?}", other), |
| 11706 | }; |
| 11707 | assert!(ir.contains("func") || ir.contains("module")); |
| 11708 | |
| 11709 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 11710 | assert!(rendered.contains("Generic artifacts")); |
| 11711 | assert!(rendered.contains("Adapter extras")); |
| 11712 | assert!(rendered.contains("-- armfortas --")); |
| 11713 | assert!(rendered.contains("== ir ==")); |
| 11714 | } |
| 11715 | |
| 11716 | #[cfg(unix)] |
| 11717 | #[test] |
| 11718 | fn introspect_armfortas_all_artifacts_includes_stage_extras() { |
| 11719 | let config = IntrospectConfig { |
| 11720 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11721 | program: runtime_fixture("if_else.f90"), |
| 11722 | opt_level: OptLevel::O0, |
| 11723 | artifacts: BTreeSet::new(), |
| 11724 | json_report: None, |
| 11725 | markdown_report: None, |
| 11726 | all_artifacts: true, |
| 11727 | summary_only: false, |
| 11728 | max_artifact_lines: None, |
| 11729 | tools: ToolchainConfig::from_env(), |
| 11730 | }; |
| 11731 | |
| 11732 | let observed = run_introspect(&config).unwrap(); |
| 11733 | let observation = &observed.observation; |
| 11734 | for artifact in [ |
| 11735 | ArtifactKey::Asm, |
| 11736 | ArtifactKey::Obj, |
| 11737 | ArtifactKey::Runtime, |
| 11738 | ArtifactKey::Extra("armfortas.preprocess".into()), |
| 11739 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11740 | ArtifactKey::Extra("armfortas.ast".into()), |
| 11741 | ArtifactKey::Extra("armfortas.sema".into()), |
| 11742 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11743 | ArtifactKey::Extra("armfortas.optir".into()), |
| 11744 | ArtifactKey::Extra("armfortas.mir".into()), |
| 11745 | ArtifactKey::Extra("armfortas.regalloc".into()), |
| 11746 | ] { |
| 11747 | assert!( |
| 11748 | observation.artifacts.contains_key(&artifact), |
| 11749 | "missing artifact {}", |
| 11750 | artifact.as_str() |
| 11751 | ); |
| 11752 | } |
| 11753 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 11754 | } |
| 11755 | |
| 11756 | #[cfg(unix)] |
| 11757 | #[test] |
| 11758 | fn introspect_armfortas_failure_reports_stage_and_partial_capture() { |
| 11759 | let config = IntrospectConfig { |
| 11760 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11761 | program: invalid_fixture("parse_error.f90"), |
| 11762 | opt_level: OptLevel::O0, |
| 11763 | artifacts: BTreeSet::from([ |
| 11764 | ArtifactKey::Asm, |
| 11765 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11766 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11767 | ]), |
| 11768 | json_report: None, |
| 11769 | markdown_report: None, |
| 11770 | all_artifacts: false, |
| 11771 | summary_only: false, |
| 11772 | max_artifact_lines: None, |
| 11773 | tools: ToolchainConfig::from_env(), |
| 11774 | }; |
| 11775 | |
| 11776 | let observed = run_introspect(&config).unwrap(); |
| 11777 | let observation = &observed.observation; |
| 11778 | assert_eq!(observation.compile_exit_code, 1); |
| 11779 | assert_eq!( |
| 11780 | observation.provenance.failure_stage.as_deref(), |
| 11781 | Some("parser") |
| 11782 | ); |
| 11783 | assert!(observation |
| 11784 | .artifacts |
| 11785 | .contains_key(&ArtifactKey::Diagnostics)); |
| 11786 | assert!(observation |
| 11787 | .artifacts |
| 11788 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 11789 | assert!(missing_introspection_artifact_names(&observed).contains(&"asm".to_string())); |
| 11790 | assert!( |
| 11791 | missing_introspection_artifact_names(&observed).contains(&"armfortas.ir".to_string()) |
| 11792 | ); |
| 11793 | |
| 11794 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 11795 | assert!(rendered.contains("status: compile failed")); |
| 11796 | assert!(rendered.contains("failure_stage: parser")); |
| 11797 | assert!(rendered.contains("diagnostic_excerpt:")); |
| 11798 | } |
| 11799 | |
| 11800 | #[cfg(unix)] |
| 11801 | #[test] |
| 11802 | fn introspect_named_external_compiler_reports_generic_artifacts_when_available() { |
| 11803 | if !command_is_available("gfortran") { |
| 11804 | return; |
| 11805 | } |
| 11806 | |
| 11807 | let config = IntrospectConfig { |
| 11808 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11809 | program: runtime_fixture("if_else.f90"), |
| 11810 | opt_level: OptLevel::O0, |
| 11811 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 11812 | json_report: None, |
| 11813 | markdown_report: None, |
| 11814 | all_artifacts: false, |
| 11815 | summary_only: false, |
| 11816 | max_artifact_lines: None, |
| 11817 | tools: ToolchainConfig::from_env(), |
| 11818 | }; |
| 11819 | |
| 11820 | let observed = run_introspect(&config).unwrap(); |
| 11821 | let observation = &observed.observation; |
| 11822 | assert_eq!(observation.compile_exit_code, 0); |
| 11823 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 11824 | assert_eq!(observation.provenance.adapter_kind, "named"); |
| 11825 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 11826 | assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 11827 | assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 11828 | assert!(observation_adapter_extras(observation).is_empty()); |
| 11829 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 11830 | } |
| 11831 | |
| 11832 | #[cfg(unix)] |
| 11833 | #[test] |
| 11834 | fn introspect_explicit_path_compiler_reports_generic_artifacts_when_available() { |
| 11835 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 11836 | ensure_fixture_executable(&compiler); |
| 11837 | |
| 11838 | let config = IntrospectConfig { |
| 11839 | compiler: CompilerSpec::Binary(compiler.clone()), |
| 11840 | program: runtime_fixture("if_else.f90"), |
| 11841 | opt_level: OptLevel::O0, |
| 11842 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 11843 | json_report: None, |
| 11844 | markdown_report: None, |
| 11845 | all_artifacts: false, |
| 11846 | summary_only: false, |
| 11847 | max_artifact_lines: None, |
| 11848 | tools: ToolchainConfig::from_env(), |
| 11849 | }; |
| 11850 | |
| 11851 | let observed = run_introspect(&config).unwrap(); |
| 11852 | let observation = &observed.observation; |
| 11853 | assert_eq!(observation.compile_exit_code, 0); |
| 11854 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 11855 | assert_eq!(observation.provenance.adapter_kind, "explicit-path"); |
| 11856 | assert!(observation |
| 11857 | .provenance |
| 11858 | .backend_detail |
| 11859 | .contains("match_42_a.sh")); |
| 11860 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 11861 | assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 11862 | assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 11863 | assert!(observation_adapter_extras(observation).is_empty()); |
| 11864 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 11865 | } |
| 11866 | |
| 11867 | #[cfg(unix)] |
| 11868 | #[test] |
| 11869 | fn introspect_external_failure_reports_missing_requested_artifacts() { |
| 11870 | let compiler = fake_compiler_fixture("compile_fail.sh"); |
| 11871 | ensure_fixture_executable(&compiler); |
| 11872 | |
| 11873 | let config = IntrospectConfig { |
| 11874 | compiler: CompilerSpec::Binary(compiler), |
| 11875 | program: runtime_fixture("if_else.f90"), |
| 11876 | opt_level: OptLevel::O0, |
| 11877 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 11878 | json_report: None, |
| 11879 | markdown_report: None, |
| 11880 | all_artifacts: false, |
| 11881 | summary_only: false, |
| 11882 | max_artifact_lines: None, |
| 11883 | tools: ToolchainConfig::from_env(), |
| 11884 | }; |
| 11885 | |
| 11886 | let observed = run_introspect(&config).unwrap(); |
| 11887 | let observation = &observed.observation; |
| 11888 | assert_eq!(observation.compile_exit_code, 1); |
| 11889 | assert_eq!(observation.provenance.failure_stage, None); |
| 11890 | assert!(observation |
| 11891 | .artifacts |
| 11892 | .contains_key(&ArtifactKey::Diagnostics)); |
| 11893 | assert_eq!( |
| 11894 | missing_introspection_artifact_names(&observed), |
| 11895 | vec!["asm".to_string(), "obj".to_string(), "runtime".to_string()] |
| 11896 | ); |
| 11897 | |
| 11898 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 11899 | assert!(rendered.contains("status: compile failed")); |
| 11900 | assert!(rendered.contains("failure_stage: none")); |
| 11901 | } |
| 11902 | |
| 11903 | #[test] |
| 11904 | fn introspect_named_external_compiler_rejects_namespaced_artifacts() { |
| 11905 | let config = IntrospectConfig { |
| 11906 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11907 | program: runtime_fixture("if_else.f90"), |
| 11908 | opt_level: OptLevel::O0, |
| 11909 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 11910 | json_report: None, |
| 11911 | markdown_report: None, |
| 11912 | all_artifacts: false, |
| 11913 | summary_only: false, |
| 11914 | max_artifact_lines: None, |
| 11915 | tools: ToolchainConfig::from_env(), |
| 11916 | }; |
| 11917 | |
| 11918 | let observed = run_introspect(&config).unwrap(); |
| 11919 | let observation = &observed.observation; |
| 11920 | assert_eq!(observation.compile_exit_code, 1); |
| 11921 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 11922 | assert_eq!(observation.provenance.failure_stage, None); |
| 11923 | let diagnostics = match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 11924 | Some(ArtifactValue::Text(text)) => text, |
| 11925 | other => panic!("expected text diagnostics, got {:?}", other), |
| 11926 | }; |
| 11927 | assert!(diagnostics.contains("does not support requested artifacts")); |
| 11928 | assert!(diagnostics.contains("armfortas.ir")); |
| 11929 | } |
| 11930 | |
| 11931 | #[test] |
| 11932 | fn compose_observation_failure_detail_uses_unavailable_wording() { |
| 11933 | let observation = CompilerObservation { |
| 11934 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11935 | program: PathBuf::from("demo.f90"), |
| 11936 | opt_level: OptLevel::O0, |
| 11937 | compile_exit_code: 1, |
| 11938 | artifacts: BTreeMap::from([( |
| 11939 | ArtifactKey::Diagnostics, |
| 11940 | ArtifactValue::Text("linked armfortas capture is unavailable in this build".into()), |
| 11941 | )]), |
| 11942 | provenance: ObservationProvenance { |
| 11943 | compiler_identity: "armfortas".into(), |
| 11944 | adapter_kind: "named".into(), |
| 11945 | backend_mode: "unavailable".into(), |
| 11946 | backend_detail: "unavailable without linked-armfortas feature".into(), |
| 11947 | artifacts_captured: vec!["diagnostics".into()], |
| 11948 | comparison_basis: None, |
| 11949 | failure_stage: None, |
| 11950 | }, |
| 11951 | }; |
| 11952 | |
| 11953 | let detail = compose_observation_failure_detail(&observation); |
| 11954 | assert!(detail.contains("armfortas unavailable for requested artifacts in this build")); |
| 11955 | assert!(!detail.contains("failed in")); |
| 11956 | } |
| 11957 | |
| 11958 | #[test] |
| 11959 | fn compose_observation_failure_detail_uses_unsupported_wording() { |
| 11960 | let observation = CompilerObservation { |
| 11961 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11962 | program: PathBuf::from("demo.f90"), |
| 11963 | opt_level: OptLevel::O0, |
| 11964 | compile_exit_code: 1, |
| 11965 | artifacts: BTreeMap::from([( |
| 11966 | ArtifactKey::Diagnostics, |
| 11967 | ArtifactValue::Text( |
| 11968 | "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 11969 | .into(), |
| 11970 | ), |
| 11971 | )]), |
| 11972 | provenance: ObservationProvenance { |
| 11973 | compiler_identity: "gfortran".into(), |
| 11974 | adapter_kind: "named".into(), |
| 11975 | backend_mode: "external-driver".into(), |
| 11976 | backend_detail: "generic external driver adapter using gfortran".into(), |
| 11977 | artifacts_captured: vec!["diagnostics".into()], |
| 11978 | comparison_basis: None, |
| 11979 | failure_stage: None, |
| 11980 | }, |
| 11981 | }; |
| 11982 | |
| 11983 | let detail = compose_observation_failure_detail(&observation); |
| 11984 | assert!(detail.contains("gfortran does not support requested artifacts in this adapter")); |
| 11985 | assert!(!detail.contains("gfortran failed")); |
| 11986 | } |
| 11987 | |
| 11988 | #[test] |
| 11989 | fn execute_generic_introspect_case_reports_capability_mismatch_clearly() { |
| 11990 | let suite = SuiteSpec { |
| 11991 | name: "v2/generic-introspect".into(), |
| 11992 | path: PathBuf::from("suite.afs"), |
| 11993 | cases: Vec::new(), |
| 11994 | }; |
| 11995 | let case = CaseSpec { |
| 11996 | name: "gfortran-armfortas-ir".into(), |
| 11997 | source: runtime_fixture("if_else.f90"), |
| 11998 | graph_files: Vec::new(), |
| 11999 | requested: BTreeSet::new(), |
| 12000 | generic_introspect: Some(GenericIntrospectCase { |
| 12001 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12002 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12003 | }), |
| 12004 | generic_compare: None, |
| 12005 | opt_levels: vec![OptLevel::O0], |
| 12006 | repeat_count: 2, |
| 12007 | reference_compilers: Vec::new(), |
| 12008 | consistency_checks: Vec::new(), |
| 12009 | expectations: vec![Expectation::Contains { |
| 12010 | target: Target::Artifact(ArtifactKey::Extra("armfortas.ir".into())), |
| 12011 | needle: "func".into(), |
| 12012 | }], |
| 12013 | status_rules: Vec::new(), |
| 12014 | capability_policy: None, |
| 12015 | }; |
| 12016 | let config = RunConfig { |
| 12017 | suite_filter: None, |
| 12018 | case_filter: None, |
| 12019 | opt_filter: None, |
| 12020 | verbose: false, |
| 12021 | fail_fast: false, |
| 12022 | include_future: false, |
| 12023 | all_stages: false, |
| 12024 | json_report: None, |
| 12025 | markdown_report: None, |
| 12026 | tools: ToolchainConfig::from_env(), |
| 12027 | }; |
| 12028 | |
| 12029 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12030 | assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12031 | assert!(outcome |
| 12032 | .detail |
| 12033 | .contains("gfortran does not support requested artifacts in this adapter")); |
| 12034 | assert!(!outcome.detail.contains("gfortran failed")); |
| 12035 | } |
| 12036 | |
| 12037 | #[test] |
| 12038 | fn execute_generic_introspect_case_applies_future_capability_policy() { |
| 12039 | let suite = SuiteSpec { |
| 12040 | name: "v2/capability-policy".into(), |
| 12041 | path: PathBuf::from("suite.afs"), |
| 12042 | cases: Vec::new(), |
| 12043 | }; |
| 12044 | let case = CaseSpec { |
| 12045 | name: "gfortran-armfortas-ir".into(), |
| 12046 | source: runtime_fixture("if_else.f90"), |
| 12047 | graph_files: Vec::new(), |
| 12048 | requested: BTreeSet::new(), |
| 12049 | generic_introspect: Some(GenericIntrospectCase { |
| 12050 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12051 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12052 | }), |
| 12053 | generic_compare: None, |
| 12054 | opt_levels: vec![OptLevel::O0], |
| 12055 | repeat_count: 2, |
| 12056 | reference_compilers: Vec::new(), |
| 12057 | consistency_checks: Vec::new(), |
| 12058 | expectations: Vec::new(), |
| 12059 | status_rules: Vec::new(), |
| 12060 | capability_policy: Some(CapabilityPolicy { |
| 12061 | kind: StatusKind::Future, |
| 12062 | reason: "generic gfortran surface has no armfortas extras".into(), |
| 12063 | }), |
| 12064 | }; |
| 12065 | let config = RunConfig { |
| 12066 | suite_filter: None, |
| 12067 | case_filter: None, |
| 12068 | opt_filter: None, |
| 12069 | verbose: false, |
| 12070 | fail_fast: false, |
| 12071 | include_future: false, |
| 12072 | all_stages: false, |
| 12073 | json_report: None, |
| 12074 | markdown_report: None, |
| 12075 | tools: ToolchainConfig::from_env(), |
| 12076 | }; |
| 12077 | |
| 12078 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12079 | assert_eq!(outcome.kind, OutcomeKind::Future); |
| 12080 | assert!(outcome |
| 12081 | .detail |
| 12082 | .contains("generic gfortran surface has no armfortas extras")); |
| 12083 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12084 | } |
| 12085 | |
| 12086 | #[cfg(unix)] |
| 12087 | #[test] |
| 12088 | fn execute_generic_suite_case_uses_introspect_engine() { |
| 12089 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12090 | ensure_fixture_executable(&compiler); |
| 12091 | |
| 12092 | let suite = SuiteSpec { |
| 12093 | name: "v2/generic-introspect".into(), |
| 12094 | path: PathBuf::from("suite.afs"), |
| 12095 | cases: Vec::new(), |
| 12096 | }; |
| 12097 | let case = CaseSpec { |
| 12098 | name: "fake-runtime".into(), |
| 12099 | source: runtime_fixture("if_else.f90"), |
| 12100 | graph_files: Vec::new(), |
| 12101 | requested: BTreeSet::new(), |
| 12102 | generic_introspect: Some(GenericIntrospectCase { |
| 12103 | compiler: CompilerSpec::Binary(compiler), |
| 12104 | artifacts: BTreeSet::from([ |
| 12105 | ArtifactKey::Asm, |
| 12106 | ArtifactKey::Obj, |
| 12107 | ArtifactKey::Runtime, |
| 12108 | ]), |
| 12109 | }), |
| 12110 | generic_compare: None, |
| 12111 | opt_levels: vec![OptLevel::O0], |
| 12112 | repeat_count: 2, |
| 12113 | reference_compilers: Vec::new(), |
| 12114 | consistency_checks: Vec::new(), |
| 12115 | expectations: vec![ |
| 12116 | Expectation::Contains { |
| 12117 | target: Target::Artifact(ArtifactKey::Asm), |
| 12118 | needle: ".globl _main".into(), |
| 12119 | }, |
| 12120 | Expectation::Contains { |
| 12121 | target: Target::RunStdout, |
| 12122 | needle: "42".into(), |
| 12123 | }, |
| 12124 | ], |
| 12125 | status_rules: Vec::new(), |
| 12126 | capability_policy: None, |
| 12127 | }; |
| 12128 | let config = RunConfig { |
| 12129 | suite_filter: None, |
| 12130 | case_filter: None, |
| 12131 | opt_filter: None, |
| 12132 | verbose: false, |
| 12133 | fail_fast: false, |
| 12134 | include_future: false, |
| 12135 | all_stages: false, |
| 12136 | json_report: None, |
| 12137 | markdown_report: None, |
| 12138 | tools: ToolchainConfig::from_env(), |
| 12139 | }; |
| 12140 | |
| 12141 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12142 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12143 | assert!(outcome.detail.is_empty()); |
| 12144 | assert!(outcome.bundle.is_none()); |
| 12145 | } |
| 12146 | |
| 12147 | #[cfg(unix)] |
| 12148 | #[test] |
| 12149 | fn execute_generic_suite_case_supports_cli_consistency() { |
| 12150 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12151 | ensure_fixture_executable(&compiler); |
| 12152 | |
| 12153 | let suite = SuiteSpec { |
| 12154 | name: "v2/generic-consistency".into(), |
| 12155 | path: PathBuf::from("suite.afs"), |
| 12156 | cases: Vec::new(), |
| 12157 | }; |
| 12158 | let case = CaseSpec { |
| 12159 | name: "fake-runtime-consistency".into(), |
| 12160 | source: runtime_fixture("if_else.f90"), |
| 12161 | graph_files: Vec::new(), |
| 12162 | requested: BTreeSet::new(), |
| 12163 | generic_introspect: Some(GenericIntrospectCase { |
| 12164 | compiler: CompilerSpec::Binary(compiler), |
| 12165 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Runtime]), |
| 12166 | }), |
| 12167 | generic_compare: None, |
| 12168 | opt_levels: vec![OptLevel::O0], |
| 12169 | repeat_count: 3, |
| 12170 | reference_compilers: Vec::new(), |
| 12171 | consistency_checks: vec![ |
| 12172 | ConsistencyCheck::CliAsmReproducible, |
| 12173 | ConsistencyCheck::CliRunReproducible, |
| 12174 | ], |
| 12175 | expectations: vec![ |
| 12176 | Expectation::Contains { |
| 12177 | target: Target::Artifact(ArtifactKey::Asm), |
| 12178 | needle: ".globl _main".into(), |
| 12179 | }, |
| 12180 | Expectation::Contains { |
| 12181 | target: Target::RunStdout, |
| 12182 | needle: "42".into(), |
| 12183 | }, |
| 12184 | ], |
| 12185 | status_rules: Vec::new(), |
| 12186 | capability_policy: None, |
| 12187 | }; |
| 12188 | let config = RunConfig { |
| 12189 | suite_filter: None, |
| 12190 | case_filter: None, |
| 12191 | opt_filter: None, |
| 12192 | verbose: false, |
| 12193 | fail_fast: false, |
| 12194 | include_future: false, |
| 12195 | all_stages: false, |
| 12196 | json_report: None, |
| 12197 | markdown_report: None, |
| 12198 | tools: ToolchainConfig::from_env(), |
| 12199 | }; |
| 12200 | |
| 12201 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12202 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12203 | assert!(outcome.detail.is_empty()); |
| 12204 | assert!(outcome.consistency_observations.is_empty()); |
| 12205 | } |
| 12206 | |
| 12207 | #[cfg(unix)] |
| 12208 | #[test] |
| 12209 | fn execute_generic_suite_case_supports_differential_when_available() { |
| 12210 | if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 12211 | return; |
| 12212 | } |
| 12213 | |
| 12214 | let suite = SuiteSpec { |
| 12215 | name: "v2/generic-differential".into(), |
| 12216 | path: PathBuf::from("suite.afs"), |
| 12217 | cases: Vec::new(), |
| 12218 | }; |
| 12219 | let case = CaseSpec { |
| 12220 | name: "gfortran-vs-flang".into(), |
| 12221 | source: runtime_fixture("if_else.f90"), |
| 12222 | graph_files: Vec::new(), |
| 12223 | requested: BTreeSet::new(), |
| 12224 | generic_introspect: Some(GenericIntrospectCase { |
| 12225 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12226 | artifacts: BTreeSet::from([ArtifactKey::Runtime]), |
| 12227 | }), |
| 12228 | generic_compare: None, |
| 12229 | opt_levels: vec![OptLevel::O0], |
| 12230 | repeat_count: 2, |
| 12231 | reference_compilers: vec![ReferenceCompiler::FlangNew], |
| 12232 | consistency_checks: Vec::new(), |
| 12233 | expectations: vec![ |
| 12234 | Expectation::Contains { |
| 12235 | target: Target::RunStdout, |
| 12236 | needle: "positive".into(), |
| 12237 | }, |
| 12238 | Expectation::IntEquals { |
| 12239 | target: Target::RunExitCode, |
| 12240 | value: 0, |
| 12241 | }, |
| 12242 | ], |
| 12243 | status_rules: Vec::new(), |
| 12244 | capability_policy: None, |
| 12245 | }; |
| 12246 | let config = RunConfig { |
| 12247 | suite_filter: None, |
| 12248 | case_filter: None, |
| 12249 | opt_filter: None, |
| 12250 | verbose: false, |
| 12251 | fail_fast: false, |
| 12252 | include_future: false, |
| 12253 | all_stages: false, |
| 12254 | json_report: None, |
| 12255 | markdown_report: None, |
| 12256 | tools: ToolchainConfig::from_env(), |
| 12257 | }; |
| 12258 | |
| 12259 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12260 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12261 | assert!(outcome.detail.is_empty()); |
| 12262 | } |
| 12263 | |
| 12264 | #[cfg(unix)] |
| 12265 | #[test] |
| 12266 | fn execute_generic_compare_suite_case_uses_compare_engine() { |
| 12267 | let left = fake_compiler_fixture("match_42_a.sh"); |
| 12268 | let right = fake_compiler_fixture("runtime_41.sh"); |
| 12269 | ensure_fixture_executable(&left); |
| 12270 | ensure_fixture_executable(&right); |
| 12271 | |
| 12272 | let suite = SuiteSpec { |
| 12273 | name: "v2/generic-compare".into(), |
| 12274 | path: PathBuf::from("suite.afs"), |
| 12275 | cases: Vec::new(), |
| 12276 | }; |
| 12277 | let case = CaseSpec { |
| 12278 | name: "fake-divergence".into(), |
| 12279 | source: runtime_fixture("if_else.f90"), |
| 12280 | graph_files: Vec::new(), |
| 12281 | requested: BTreeSet::new(), |
| 12282 | generic_introspect: None, |
| 12283 | generic_compare: Some(GenericCompareCase { |
| 12284 | left: CompilerSpec::Binary(left), |
| 12285 | right: CompilerSpec::Binary(right), |
| 12286 | artifacts: BTreeSet::from([ |
| 12287 | ArtifactKey::Diagnostics, |
| 12288 | ArtifactKey::Runtime, |
| 12289 | ArtifactKey::Asm, |
| 12290 | ]), |
| 12291 | }), |
| 12292 | opt_levels: vec![OptLevel::O0], |
| 12293 | repeat_count: 2, |
| 12294 | reference_compilers: Vec::new(), |
| 12295 | consistency_checks: Vec::new(), |
| 12296 | expectations: vec![ |
| 12297 | Expectation::Equals { |
| 12298 | target: Target::CompareStatus, |
| 12299 | value: "diff".into(), |
| 12300 | }, |
| 12301 | Expectation::Equals { |
| 12302 | target: Target::CompareClassification, |
| 12303 | value: "mixed divergence".into(), |
| 12304 | }, |
| 12305 | Expectation::Contains { |
| 12306 | target: Target::CompareChangedArtifacts, |
| 12307 | needle: "asm".into(), |
| 12308 | }, |
| 12309 | Expectation::Contains { |
| 12310 | target: Target::CompareChangedArtifacts, |
| 12311 | needle: "runtime".into(), |
| 12312 | }, |
| 12313 | Expectation::IntEquals { |
| 12314 | target: Target::CompareDifferenceCount, |
| 12315 | value: 2, |
| 12316 | }, |
| 12317 | ], |
| 12318 | status_rules: Vec::new(), |
| 12319 | capability_policy: None, |
| 12320 | }; |
| 12321 | let config = RunConfig { |
| 12322 | suite_filter: None, |
| 12323 | case_filter: None, |
| 12324 | opt_filter: None, |
| 12325 | verbose: false, |
| 12326 | fail_fast: false, |
| 12327 | include_future: false, |
| 12328 | all_stages: false, |
| 12329 | json_report: None, |
| 12330 | markdown_report: None, |
| 12331 | tools: ToolchainConfig::from_env(), |
| 12332 | }; |
| 12333 | |
| 12334 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12335 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12336 | } |
| 12337 | |
| 12338 | #[test] |
| 12339 | fn execute_generic_compare_suite_case_reports_capability_mismatch() { |
| 12340 | let suite = SuiteSpec { |
| 12341 | name: "v2/generic-compare".into(), |
| 12342 | path: PathBuf::from("suite.afs"), |
| 12343 | cases: Vec::new(), |
| 12344 | }; |
| 12345 | let case = CaseSpec { |
| 12346 | name: "armfortas-ir-vs-gfortran".into(), |
| 12347 | source: runtime_fixture("if_else.f90"), |
| 12348 | graph_files: Vec::new(), |
| 12349 | requested: BTreeSet::new(), |
| 12350 | generic_introspect: None, |
| 12351 | generic_compare: Some(GenericCompareCase { |
| 12352 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12353 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12354 | artifacts: BTreeSet::from([ |
| 12355 | ArtifactKey::Diagnostics, |
| 12356 | ArtifactKey::Runtime, |
| 12357 | ArtifactKey::Extra("armfortas.ir".into()), |
| 12358 | ]), |
| 12359 | }), |
| 12360 | opt_levels: vec![OptLevel::O0], |
| 12361 | repeat_count: 2, |
| 12362 | reference_compilers: Vec::new(), |
| 12363 | consistency_checks: Vec::new(), |
| 12364 | expectations: vec![Expectation::Equals { |
| 12365 | target: Target::CompareStatus, |
| 12366 | value: "match".into(), |
| 12367 | }], |
| 12368 | status_rules: Vec::new(), |
| 12369 | capability_policy: None, |
| 12370 | }; |
| 12371 | let config = RunConfig { |
| 12372 | suite_filter: None, |
| 12373 | case_filter: None, |
| 12374 | opt_filter: None, |
| 12375 | verbose: false, |
| 12376 | fail_fast: false, |
| 12377 | include_future: false, |
| 12378 | all_stages: false, |
| 12379 | json_report: None, |
| 12380 | markdown_report: None, |
| 12381 | tools: ToolchainConfig::from_env(), |
| 12382 | }; |
| 12383 | |
| 12384 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12385 | assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12386 | assert!(outcome.detail.contains("compare request is not supported")); |
| 12387 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12388 | } |
| 12389 | |
| 12390 | #[test] |
| 12391 | fn execute_generic_compare_suite_case_applies_xfail_capability_policy() { |
| 12392 | let suite = SuiteSpec { |
| 12393 | name: "v2/capability-policy".into(), |
| 12394 | path: PathBuf::from("suite.afs"), |
| 12395 | cases: Vec::new(), |
| 12396 | }; |
| 12397 | let case = CaseSpec { |
| 12398 | name: "armfortas-ir-vs-gfortran".into(), |
| 12399 | source: runtime_fixture("if_else.f90"), |
| 12400 | graph_files: Vec::new(), |
| 12401 | requested: BTreeSet::new(), |
| 12402 | generic_introspect: None, |
| 12403 | generic_compare: Some(GenericCompareCase { |
| 12404 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12405 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12406 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12407 | }), |
| 12408 | opt_levels: vec![OptLevel::O0], |
| 12409 | repeat_count: 2, |
| 12410 | reference_compilers: Vec::new(), |
| 12411 | consistency_checks: Vec::new(), |
| 12412 | expectations: Vec::new(), |
| 12413 | status_rules: Vec::new(), |
| 12414 | capability_policy: Some(CapabilityPolicy { |
| 12415 | kind: StatusKind::Xfail, |
| 12416 | reason: "mixed-surface namespaced compare stays soft for now".into(), |
| 12417 | }), |
| 12418 | }; |
| 12419 | let config = RunConfig { |
| 12420 | suite_filter: None, |
| 12421 | case_filter: None, |
| 12422 | opt_filter: None, |
| 12423 | verbose: false, |
| 12424 | fail_fast: false, |
| 12425 | include_future: false, |
| 12426 | all_stages: false, |
| 12427 | json_report: None, |
| 12428 | markdown_report: None, |
| 12429 | tools: ToolchainConfig::from_env(), |
| 12430 | }; |
| 12431 | |
| 12432 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12433 | assert_eq!(outcome.kind, OutcomeKind::Xfail); |
| 12434 | assert!(outcome |
| 12435 | .detail |
| 12436 | .contains("mixed-surface namespaced compare stays soft for now")); |
| 12437 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12438 | } |
| 12439 | |
| 12440 | #[test] |
| 12441 | fn parses_suite_and_case() { |
| 12442 | let root = std::env::temp_dir().join("afs_tests_parser_spec.afs"); |
| 12443 | fs::write( |
| 12444 | &root, |
| 12445 | r#"suite "runtime/smoke" |
| 12446 | |
| 12447 | case "hello" |
| 12448 | source "../../../test_programs/hello.f90" |
| 12449 | armfortas => run, ir |
| 12450 | expect run.stdout check-comments |
| 12451 | expect ir contains "module main" |
| 12452 | expect asm not-contains "x18" |
| 12453 | end |
| 12454 | "#, |
| 12455 | ) |
| 12456 | .unwrap(); |
| 12457 | |
| 12458 | let suite = parse_suite_file(&root).unwrap(); |
| 12459 | assert_eq!(suite.name, "runtime/smoke"); |
| 12460 | assert_eq!(suite.cases.len(), 1); |
| 12461 | assert!(suite.cases[0].requested.contains(&Stage::Run)); |
| 12462 | assert!(suite.cases[0].requested.contains(&Stage::Ir)); |
| 12463 | assert!(suite.cases[0].generic_introspect.is_none()); |
| 12464 | assert!(matches!( |
| 12465 | suite.cases[0].expectations[2], |
| 12466 | Expectation::NotContains { |
| 12467 | target: Target::Artifact(ArtifactKey::Asm), |
| 12468 | .. |
| 12469 | } |
| 12470 | )); |
| 12471 | assert_eq!(suite.cases[0].opt_levels, vec![OptLevel::O0]); |
| 12472 | let _ = fs::remove_file(&root); |
| 12473 | } |
| 12474 | |
| 12475 | #[test] |
| 12476 | fn parses_generic_compiler_case() { |
| 12477 | let root = std::env::temp_dir().join("bencch_generic_parser_spec.afs"); |
| 12478 | fs::write( |
| 12479 | &root, |
| 12480 | r#"suite "v2/generic-introspect" |
| 12481 | |
| 12482 | case "fake-runtime" |
| 12483 | source "../../fixtures/runtime/if_else.f90" |
| 12484 | compiler gfortran => asm, obj, runtime |
| 12485 | expect asm contains ".globl _main" |
| 12486 | expect run.stdout contains "42" |
| 12487 | end |
| 12488 | "#, |
| 12489 | ) |
| 12490 | .unwrap(); |
| 12491 | |
| 12492 | let suite = parse_suite_file(&root).unwrap(); |
| 12493 | let case = &suite.cases[0]; |
| 12494 | let generic = case.generic_introspect.as_ref().unwrap(); |
| 12495 | assert_eq!( |
| 12496 | generic.compiler, |
| 12497 | CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12498 | ); |
| 12499 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12500 | assert!(generic.artifacts.contains(&ArtifactKey::Obj)); |
| 12501 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12502 | assert!(case.requested.is_empty()); |
| 12503 | let _ = fs::remove_file(&root); |
| 12504 | } |
| 12505 | |
| 12506 | #[test] |
| 12507 | fn parses_generic_compiler_case_with_differential_and_cli_consistency() { |
| 12508 | let root = std::env::temp_dir().join("bencch_generic_differential_parser_spec.afs"); |
| 12509 | fs::write( |
| 12510 | &root, |
| 12511 | r#"suite "v2/generic-differential" |
| 12512 | |
| 12513 | case "gfortran_runtime_matrix" |
| 12514 | source "../../fixtures/runtime/if_else.f90" |
| 12515 | opts => O0, O1, O2 |
| 12516 | repeat => 3 |
| 12517 | compiler gfortran => runtime, asm |
| 12518 | differential => flang-new |
| 12519 | consistency => cli_asm_reproducible, cli_run_reproducible |
| 12520 | expect run.stdout check-comments |
| 12521 | expect run.exit_code equals 0 |
| 12522 | end |
| 12523 | "#, |
| 12524 | ) |
| 12525 | .unwrap(); |
| 12526 | |
| 12527 | let suite = parse_suite_file(&root).unwrap(); |
| 12528 | let case = &suite.cases[0]; |
| 12529 | let generic = case.generic_introspect.as_ref().unwrap(); |
| 12530 | assert_eq!( |
| 12531 | generic.compiler, |
| 12532 | CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12533 | ); |
| 12534 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12535 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12536 | assert_eq!(case.reference_compilers, vec![ReferenceCompiler::FlangNew]); |
| 12537 | assert_eq!( |
| 12538 | case.consistency_checks, |
| 12539 | vec![ |
| 12540 | ConsistencyCheck::CliAsmReproducible, |
| 12541 | ConsistencyCheck::CliRunReproducible, |
| 12542 | ] |
| 12543 | ); |
| 12544 | let _ = fs::remove_file(&root); |
| 12545 | } |
| 12546 | |
| 12547 | #[test] |
| 12548 | fn rejects_capture_consistency_on_generic_compiler_case() { |
| 12549 | let root = std::env::temp_dir().join("bencch_generic_capture_consistency_parser_spec.afs"); |
| 12550 | fs::write( |
| 12551 | &root, |
| 12552 | r#"suite "v2/generic-consistency" |
| 12553 | |
| 12554 | case "armfortas_capture_run" |
| 12555 | source "../../fixtures/runtime/if_else.f90" |
| 12556 | compiler armfortas => runtime |
| 12557 | consistency => capture_run_reproducible |
| 12558 | expect run.exit_code equals 0 |
| 12559 | end |
| 12560 | "#, |
| 12561 | ) |
| 12562 | .unwrap(); |
| 12563 | |
| 12564 | let err = parse_suite_file(&root).unwrap_err(); |
| 12565 | assert!(err.contains("generic compiler cases only support")); |
| 12566 | assert!(err.contains("capture_run_reproducible")); |
| 12567 | let _ = fs::remove_file(&root); |
| 12568 | } |
| 12569 | |
| 12570 | #[test] |
| 12571 | fn parses_generic_compare_case() { |
| 12572 | let root = std::env::temp_dir().join("bencch_generic_compare_parser_spec.afs"); |
| 12573 | fs::write( |
| 12574 | &root, |
| 12575 | r#"suite "v2/generic-compare" |
| 12576 | |
| 12577 | case "fake-match" |
| 12578 | source "../../fixtures/runtime/if_else.f90" |
| 12579 | opts => O0, O1, O2 |
| 12580 | compare gfortran flang-new => asm |
| 12581 | expect compare.status equals "match" |
| 12582 | expect compare.difference_count equals 0 |
| 12583 | end |
| 12584 | "#, |
| 12585 | ) |
| 12586 | .unwrap(); |
| 12587 | |
| 12588 | let suite = parse_suite_file(&root).unwrap(); |
| 12589 | let case = &suite.cases[0]; |
| 12590 | let generic = case.generic_compare.as_ref().unwrap(); |
| 12591 | assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Gfortran)); |
| 12592 | assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::FlangNew)); |
| 12593 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12594 | assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12595 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12596 | assert_eq!( |
| 12597 | case.opt_levels, |
| 12598 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12599 | ); |
| 12600 | let _ = fs::remove_file(&root); |
| 12601 | } |
| 12602 | |
| 12603 | #[test] |
| 12604 | fn parses_generic_compare_case_with_namespaced_artifact() { |
| 12605 | let root = std::env::temp_dir().join("bencch_generic_compare_namespaced_spec.afs"); |
| 12606 | fs::write( |
| 12607 | &root, |
| 12608 | r#"suite "v2/generic-compare" |
| 12609 | |
| 12610 | case "armfortas-ir" |
| 12611 | source "../../fixtures/runtime/if_else.f90" |
| 12612 | compare armfortas armfortas => armfortas.ir |
| 12613 | expect compare.status equals "match" |
| 12614 | end |
| 12615 | "#, |
| 12616 | ) |
| 12617 | .unwrap(); |
| 12618 | |
| 12619 | let suite = parse_suite_file(&root).unwrap(); |
| 12620 | let case = &suite.cases[0]; |
| 12621 | let generic = case.generic_compare.as_ref().unwrap(); |
| 12622 | assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12623 | assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12624 | assert!(generic |
| 12625 | .artifacts |
| 12626 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 12627 | assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12628 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12629 | |
| 12630 | let _ = fs::remove_file(&root); |
| 12631 | } |
| 12632 | |
| 12633 | #[test] |
| 12634 | fn parses_capability_policy_for_generic_case() { |
| 12635 | let root = std::env::temp_dir().join("bencch_generic_capability_policy_spec.afs"); |
| 12636 | fs::write( |
| 12637 | &root, |
| 12638 | r#"suite "v2/capability-policy" |
| 12639 | |
| 12640 | case "gfortran_armfortas_ir" |
| 12641 | source "../../fixtures/runtime/if_else.f90" |
| 12642 | compiler gfortran => armfortas.ir |
| 12643 | future capability "generic gfortran surface has no armfortas extras" |
| 12644 | end |
| 12645 | "#, |
| 12646 | ) |
| 12647 | .unwrap(); |
| 12648 | |
| 12649 | let suite = parse_suite_file(&root).unwrap(); |
| 12650 | let case = &suite.cases[0]; |
| 12651 | let policy = case.capability_policy.as_ref().unwrap(); |
| 12652 | assert!(matches!(policy.kind, StatusKind::Future)); |
| 12653 | assert_eq!( |
| 12654 | policy.reason, |
| 12655 | "generic gfortran surface has no armfortas extras" |
| 12656 | ); |
| 12657 | let _ = fs::remove_file(&root); |
| 12658 | } |
| 12659 | |
| 12660 | #[test] |
| 12661 | fn parses_matrix_status_and_differential() { |
| 12662 | let root = std::env::temp_dir().join("afs_tests_matrix_spec.afs"); |
| 12663 | fs::write( |
| 12664 | &root, |
| 12665 | r#"suite "runtime/matrix" |
| 12666 | |
| 12667 | case "hello" |
| 12668 | source "../../../test_programs/hello.f90" |
| 12669 | opts => O0, O1, O2 |
| 12670 | armfortas => run |
| 12671 | differential => gfortran, flang-new |
| 12672 | expect run.exit_code equals 0 |
| 12673 | xfail when O1, O2 because "known issue" |
| 12674 | end |
| 12675 | "#, |
| 12676 | ) |
| 12677 | .unwrap(); |
| 12678 | |
| 12679 | let suite = parse_suite_file(&root).unwrap(); |
| 12680 | let case = &suite.cases[0]; |
| 12681 | assert_eq!( |
| 12682 | case.opt_levels, |
| 12683 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12684 | ); |
| 12685 | assert_eq!( |
| 12686 | case.reference_compilers, |
| 12687 | vec![ReferenceCompiler::Gfortran, ReferenceCompiler::FlangNew] |
| 12688 | ); |
| 12689 | assert!(matches!( |
| 12690 | status_for_opt(case, OptLevel::O0), |
| 12691 | EffectiveStatus::Normal |
| 12692 | )); |
| 12693 | assert!(matches!( |
| 12694 | status_for_opt(case, OptLevel::O1), |
| 12695 | EffectiveStatus::Xfail(_) |
| 12696 | )); |
| 12697 | let _ = fs::remove_file(&root); |
| 12698 | } |
| 12699 | |
| 12700 | #[test] |
| 12701 | fn parses_consistency_checks() { |
| 12702 | let root = std::env::temp_dir().join("afs_tests_consistency_spec.afs"); |
| 12703 | fs::write( |
| 12704 | &root, |
| 12705 | r#"suite "consistency/object" |
| 12706 | |
| 12707 | case "driver_paths" |
| 12708 | source "../../fixtures/backend/runtime_calls.f90" |
| 12709 | armfortas => asm, obj |
| 12710 | repeat => 5 |
| 12711 | consistency => cli_obj_vs_system_as, cli-obj-vs-system-as, cli_asm_reproducible, cli-obj-reproducible, cli_run_reproducible, capture_asm_vs_cli_asm, capture-obj-vs-cli-obj, capture_run_vs_cli_run, capture_asm_reproducible, capture-obj-reproducible, capture_run_reproducible |
| 12712 | expect obj contains "_main" |
| 12713 | end |
| 12714 | "#, |
| 12715 | ) |
| 12716 | .unwrap(); |
| 12717 | |
| 12718 | let suite = parse_suite_file(&root).unwrap(); |
| 12719 | let case = &suite.cases[0]; |
| 12720 | assert_eq!( |
| 12721 | case.consistency_checks, |
| 12722 | vec![ |
| 12723 | ConsistencyCheck::CliObjVsSystemAs, |
| 12724 | ConsistencyCheck::CliAsmReproducible, |
| 12725 | ConsistencyCheck::CliObjReproducible, |
| 12726 | ConsistencyCheck::CliRunReproducible, |
| 12727 | ConsistencyCheck::CaptureAsmVsCliAsm, |
| 12728 | ConsistencyCheck::CaptureObjVsCliObj, |
| 12729 | ConsistencyCheck::CaptureRunVsCliRun, |
| 12730 | ConsistencyCheck::CaptureAsmReproducible, |
| 12731 | ConsistencyCheck::CaptureObjReproducible, |
| 12732 | ConsistencyCheck::CaptureRunReproducible, |
| 12733 | ] |
| 12734 | ); |
| 12735 | assert_eq!(case.repeat_count, 5); |
| 12736 | let _ = fs::remove_file(&root); |
| 12737 | } |
| 12738 | |
| 12739 | #[test] |
| 12740 | fn parses_graph_case() { |
| 12741 | let root = std::env::temp_dir().join("afs_tests_graph_spec"); |
| 12742 | let _ = fs::remove_dir_all(&root); |
| 12743 | fs::create_dir_all(&root).unwrap(); |
| 12744 | fs::write( |
| 12745 | root.join("math_values.f90"), |
| 12746 | "module math_values\nend module\n", |
| 12747 | ) |
| 12748 | .unwrap(); |
| 12749 | fs::write(root.join("main.f90"), "program main\nend program\n").unwrap(); |
| 12750 | fs::write( |
| 12751 | root.join("graph.afs"), |
| 12752 | r#"suite "modules/graph" |
| 12753 | |
| 12754 | case "basic_use" |
| 12755 | entry "main.f90" |
| 12756 | file "math_values.f90" |
| 12757 | file "main.f90" |
| 12758 | armfortas => run |
| 12759 | expect run.exit_code equals 0 |
| 12760 | end |
| 12761 | "#, |
| 12762 | ) |
| 12763 | .unwrap(); |
| 12764 | |
| 12765 | let suite = parse_suite_file(&root.join("graph.afs")).unwrap(); |
| 12766 | let case = &suite.cases[0]; |
| 12767 | assert_eq!(case.source, root.join("main.f90")); |
| 12768 | assert_eq!( |
| 12769 | case.graph_files, |
| 12770 | vec![root.join("math_values.f90"), root.join("main.f90")] |
| 12771 | ); |
| 12772 | |
| 12773 | let _ = fs::remove_dir_all(&root); |
| 12774 | } |
| 12775 | |
| 12776 | #[test] |
| 12777 | fn parse_cli_collects_tool_overrides() { |
| 12778 | let args = vec![ |
| 12779 | "run".to_string(), |
| 12780 | "--suite".to_string(), |
| 12781 | "consistency/runtime".to_string(), |
| 12782 | "--json-report".to_string(), |
| 12783 | "/tmp/report.json".to_string(), |
| 12784 | "--markdown-report".to_string(), |
| 12785 | "/tmp/report.md".to_string(), |
| 12786 | "--armfortas-bin".to_string(), |
| 12787 | "/tmp/armfortas".to_string(), |
| 12788 | "--gfortran-bin".to_string(), |
| 12789 | "/tmp/gfortran".to_string(), |
| 12790 | "--flang-bin".to_string(), |
| 12791 | "/tmp/flang-new".to_string(), |
| 12792 | "--lfortran-bin".to_string(), |
| 12793 | "/tmp/lfortran".to_string(), |
| 12794 | "--ifort-bin".to_string(), |
| 12795 | "/tmp/ifort".to_string(), |
| 12796 | "--ifx-bin".to_string(), |
| 12797 | "/tmp/ifx".to_string(), |
| 12798 | "--nvfortran-bin".to_string(), |
| 12799 | "/tmp/nvfortran".to_string(), |
| 12800 | "--as-bin".to_string(), |
| 12801 | "/tmp/as".to_string(), |
| 12802 | "--otool-bin".to_string(), |
| 12803 | "/tmp/otool".to_string(), |
| 12804 | "--nm-bin".to_string(), |
| 12805 | "/tmp/nm".to_string(), |
| 12806 | ]; |
| 12807 | |
| 12808 | let command = parse_cli(&args).unwrap(); |
| 12809 | let config = match command { |
| 12810 | CommandKind::Run(config) => config, |
| 12811 | other => panic!( |
| 12812 | "expected run command, got {:?}", |
| 12813 | std::mem::discriminant(&other) |
| 12814 | ), |
| 12815 | }; |
| 12816 | |
| 12817 | assert_eq!(config.suite_filter.as_deref(), Some("consistency/runtime")); |
| 12818 | assert_eq!( |
| 12819 | config.json_report.as_deref(), |
| 12820 | Some(Path::new("/tmp/report.json")) |
| 12821 | ); |
| 12822 | assert_eq!( |
| 12823 | config.markdown_report.as_deref(), |
| 12824 | Some(Path::new("/tmp/report.md")) |
| 12825 | ); |
| 12826 | assert_eq!( |
| 12827 | config.tools.armfortas, |
| 12828 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 12829 | ); |
| 12830 | assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 12831 | assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 12832 | assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 12833 | assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 12834 | assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 12835 | assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 12836 | assert_eq!(config.tools.system_as, "/tmp/as"); |
| 12837 | assert_eq!(config.tools.otool, "/tmp/otool"); |
| 12838 | assert_eq!(config.tools.nm, "/tmp/nm"); |
| 12839 | } |
| 12840 | |
| 12841 | #[test] |
| 12842 | fn parse_cli_collects_list_config() { |
| 12843 | let args = vec![ |
| 12844 | "list".to_string(), |
| 12845 | "--suite".to_string(), |
| 12846 | "v2/generic".to_string(), |
| 12847 | "--verbose".to_string(), |
| 12848 | "--armfortas-bin".to_string(), |
| 12849 | "/tmp/armfortas".to_string(), |
| 12850 | ]; |
| 12851 | |
| 12852 | let command = parse_cli(&args).unwrap(); |
| 12853 | let config = match command { |
| 12854 | CommandKind::List(config) => config, |
| 12855 | other => panic!( |
| 12856 | "expected list command, got {:?}", |
| 12857 | std::mem::discriminant(&other) |
| 12858 | ), |
| 12859 | }; |
| 12860 | |
| 12861 | assert_eq!(config.suite_filter.as_deref(), Some("v2/generic")); |
| 12862 | assert!(config.verbose); |
| 12863 | assert_eq!( |
| 12864 | config.tools.armfortas, |
| 12865 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 12866 | ); |
| 12867 | } |
| 12868 | |
| 12869 | #[test] |
| 12870 | fn parse_cli_collects_doctor_tool_overrides() { |
| 12871 | let args = vec![ |
| 12872 | "doctor".to_string(), |
| 12873 | "--json-report".to_string(), |
| 12874 | "/tmp/doctor.json".to_string(), |
| 12875 | "--markdown-report".to_string(), |
| 12876 | "/tmp/doctor.md".to_string(), |
| 12877 | "--armfortas-bin".to_string(), |
| 12878 | "/tmp/armfortas".to_string(), |
| 12879 | "--gfortran-bin".to_string(), |
| 12880 | "/tmp/gfortran".to_string(), |
| 12881 | "--flang-bin".to_string(), |
| 12882 | "/tmp/flang-new".to_string(), |
| 12883 | "--lfortran-bin".to_string(), |
| 12884 | "/tmp/lfortran".to_string(), |
| 12885 | "--ifort-bin".to_string(), |
| 12886 | "/tmp/ifort".to_string(), |
| 12887 | "--ifx-bin".to_string(), |
| 12888 | "/tmp/ifx".to_string(), |
| 12889 | "--nvfortran-bin".to_string(), |
| 12890 | "/tmp/nvfortran".to_string(), |
| 12891 | "--as-bin".to_string(), |
| 12892 | "/tmp/as".to_string(), |
| 12893 | "--otool-bin".to_string(), |
| 12894 | "/tmp/otool".to_string(), |
| 12895 | "--nm-bin".to_string(), |
| 12896 | "/tmp/nm".to_string(), |
| 12897 | ]; |
| 12898 | |
| 12899 | let command = parse_cli(&args).unwrap(); |
| 12900 | let config = match command { |
| 12901 | CommandKind::Doctor(config) => config, |
| 12902 | other => panic!( |
| 12903 | "expected doctor command, got {:?}", |
| 12904 | std::mem::discriminant(&other) |
| 12905 | ), |
| 12906 | }; |
| 12907 | |
| 12908 | assert_eq!( |
| 12909 | config.tools.armfortas, |
| 12910 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 12911 | ); |
| 12912 | assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 12913 | assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 12914 | assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 12915 | assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 12916 | assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 12917 | assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 12918 | assert_eq!(config.tools.system_as, "/tmp/as"); |
| 12919 | assert_eq!(config.tools.otool, "/tmp/otool"); |
| 12920 | assert_eq!(config.tools.nm, "/tmp/nm"); |
| 12921 | assert_eq!( |
| 12922 | config.json_report.as_deref(), |
| 12923 | Some(Path::new("/tmp/doctor.json")) |
| 12924 | ); |
| 12925 | assert_eq!( |
| 12926 | config.markdown_report.as_deref(), |
| 12927 | Some(Path::new("/tmp/doctor.md")) |
| 12928 | ); |
| 12929 | } |
| 12930 | |
| 12931 | #[test] |
| 12932 | fn parse_cli_collects_compare_config() { |
| 12933 | let args = vec![ |
| 12934 | "compare".to_string(), |
| 12935 | "armfortas".to_string(), |
| 12936 | "/tmp/other-compiler".to_string(), |
| 12937 | "--program".to_string(), |
| 12938 | "/tmp/demo.f90".to_string(), |
| 12939 | "--opt".to_string(), |
| 12940 | "O2".to_string(), |
| 12941 | "--artifact".to_string(), |
| 12942 | "asm,obj,armfortas.ir".to_string(), |
| 12943 | "--json-report".to_string(), |
| 12944 | "/tmp/compare.json".to_string(), |
| 12945 | "--markdown-report".to_string(), |
| 12946 | "/tmp/compare.md".to_string(), |
| 12947 | ]; |
| 12948 | |
| 12949 | let command = parse_cli(&args).unwrap(); |
| 12950 | let config = match command { |
| 12951 | CommandKind::Compare(config) => config, |
| 12952 | other => panic!( |
| 12953 | "expected compare command, got {:?}", |
| 12954 | std::mem::discriminant(&other) |
| 12955 | ), |
| 12956 | }; |
| 12957 | |
| 12958 | assert_eq!(config.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12959 | assert_eq!( |
| 12960 | config.right, |
| 12961 | CompilerSpec::Binary(PathBuf::from("/tmp/other-compiler")) |
| 12962 | ); |
| 12963 | assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 12964 | assert_eq!(config.opt_level, OptLevel::O2); |
| 12965 | assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 12966 | assert!(config.artifacts.contains(&ArtifactKey::Obj)); |
| 12967 | assert!(config |
| 12968 | .artifacts |
| 12969 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 12970 | assert_eq!( |
| 12971 | config.json_report.as_deref(), |
| 12972 | Some(Path::new("/tmp/compare.json")) |
| 12973 | ); |
| 12974 | assert_eq!( |
| 12975 | config.markdown_report.as_deref(), |
| 12976 | Some(Path::new("/tmp/compare.md")) |
| 12977 | ); |
| 12978 | } |
| 12979 | |
| 12980 | #[test] |
| 12981 | fn parse_cli_collects_introspect_config() { |
| 12982 | let args = vec![ |
| 12983 | "introspect".to_string(), |
| 12984 | "armfortas".to_string(), |
| 12985 | "/tmp/demo.f90".to_string(), |
| 12986 | "--artifact".to_string(), |
| 12987 | "armfortas.ir,asm".to_string(), |
| 12988 | "--all".to_string(), |
| 12989 | "--summary-only".to_string(), |
| 12990 | "--max-artifact-lines".to_string(), |
| 12991 | "12".to_string(), |
| 12992 | "--json-report".to_string(), |
| 12993 | "/tmp/introspect.json".to_string(), |
| 12994 | ]; |
| 12995 | |
| 12996 | let command = parse_cli(&args).unwrap(); |
| 12997 | let config = match command { |
| 12998 | CommandKind::Introspect(config) => config, |
| 12999 | other => panic!( |
| 13000 | "expected introspect command, got {:?}", |
| 13001 | std::mem::discriminant(&other) |
| 13002 | ), |
| 13003 | }; |
| 13004 | |
| 13005 | assert_eq!( |
| 13006 | config.compiler, |
| 13007 | CompilerSpec::Named(NamedCompiler::Armfortas) |
| 13008 | ); |
| 13009 | assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 13010 | assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 13011 | assert!(config |
| 13012 | .artifacts |
| 13013 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 13014 | assert!(config.all_artifacts); |
| 13015 | assert!(config.summary_only); |
| 13016 | assert_eq!(config.max_artifact_lines, Some(12)); |
| 13017 | assert_eq!( |
| 13018 | config.json_report.as_deref(), |
| 13019 | Some(Path::new("/tmp/introspect.json")) |
| 13020 | ); |
| 13021 | } |
| 13022 | |
| 13023 | #[test] |
| 13024 | fn parses_failure_expectation() { |
| 13025 | let root = std::env::temp_dir().join("afs_tests_failure_spec.afs"); |
| 13026 | fs::write( |
| 13027 | &root, |
| 13028 | r#"suite "frontend/parser" |
| 13029 | |
| 13030 | case "missing_then" |
| 13031 | source "../../fixtures/frontend/parser/missing_then.f90" |
| 13032 | armfortas => tokens |
| 13033 | expect tokens contains "if" |
| 13034 | expect-fail parser contains "expected" |
| 13035 | end |
| 13036 | "#, |
| 13037 | ) |
| 13038 | .unwrap(); |
| 13039 | |
| 13040 | let suite = parse_suite_file(&root).unwrap(); |
| 13041 | assert_eq!(suite.cases.len(), 1); |
| 13042 | assert!(has_failure_expectation(&suite.cases[0])); |
| 13043 | let _ = fs::remove_file(&root); |
| 13044 | } |
| 13045 | |
| 13046 | #[test] |
| 13047 | fn parses_failure_expectation_from_source_comments() { |
| 13048 | let root = std::env::temp_dir().join("afs_tests_failure_comment_spec"); |
| 13049 | let _ = fs::remove_dir_all(&root); |
| 13050 | fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13051 | fs::create_dir_all(root.join("suites")).unwrap(); |
| 13052 | |
| 13053 | let source = root.join("fixtures/error_expected.f90"); |
| 13054 | fs::write( |
| 13055 | &source, |
| 13056 | "! ERROR_EXPECTED: hidden\nprogram error_expected\n print *, hidden\nend program\n", |
| 13057 | ) |
| 13058 | .unwrap(); |
| 13059 | |
| 13060 | let suite_path = root.join("suites/spec.afs"); |
| 13061 | fs::write( |
| 13062 | &suite_path, |
| 13063 | r#"suite "v2/comment-failure" |
| 13064 | |
| 13065 | case "error_expected_comments" |
| 13066 | source "../fixtures/error_expected.f90" |
| 13067 | compiler armfortas => diagnostics |
| 13068 | expect-fail comments |
| 13069 | end |
| 13070 | "#, |
| 13071 | ) |
| 13072 | .unwrap(); |
| 13073 | |
| 13074 | let suite = parse_suite_file(&suite_path).unwrap(); |
| 13075 | match &suite.cases[0].expectations[0] { |
| 13076 | Expectation::FailCommentPatterns(patterns) => { |
| 13077 | assert_eq!(patterns, &vec!["hidden".to_string()]); |
| 13078 | } |
| 13079 | other => panic!("expected source-comment failure expectation, got {other:?}"), |
| 13080 | } |
| 13081 | |
| 13082 | let _ = fs::remove_dir_all(&root); |
| 13083 | } |
| 13084 | |
| 13085 | #[test] |
| 13086 | fn resolves_xfail_comments_from_source() { |
| 13087 | let root = std::env::temp_dir().join("afs_tests_xfail_comment_spec"); |
| 13088 | let _ = fs::remove_dir_all(&root); |
| 13089 | fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13090 | fs::create_dir_all(root.join("suites")).unwrap(); |
| 13091 | |
| 13092 | let source = root.join("fixtures/xfail_case.f90"); |
| 13093 | fs::write( |
| 13094 | &source, |
| 13095 | "! XFAIL: audit BLOCKING-1 (demo)\nprogram xfail_case\nend program\n", |
| 13096 | ) |
| 13097 | .unwrap(); |
| 13098 | |
| 13099 | let suite_path = root.join("suites/spec.afs"); |
| 13100 | fs::write( |
| 13101 | &suite_path, |
| 13102 | r#"suite "v2/comment-xfail" |
| 13103 | |
| 13104 | case "xfail_comments" |
| 13105 | source "../fixtures/xfail_case.f90" |
| 13106 | compiler armfortas => runtime |
| 13107 | xfail comments |
| 13108 | end |
| 13109 | "#, |
| 13110 | ) |
| 13111 | .unwrap(); |
| 13112 | |
| 13113 | let suite = parse_suite_file(&suite_path).unwrap(); |
| 13114 | match status_for_opt(&suite.cases[0], OptLevel::O0) { |
| 13115 | EffectiveStatus::Xfail(reason) => { |
| 13116 | assert_eq!(reason, "audit BLOCKING-1 (demo)"); |
| 13117 | } |
| 13118 | other => panic!("expected xfail status, got {other:?}"), |
| 13119 | } |
| 13120 | |
| 13121 | let _ = fs::remove_dir_all(&root); |
| 13122 | } |
| 13123 | |
| 13124 | #[test] |
| 13125 | fn check_matching_preserves_order() { |
| 13126 | let checks = vec![ |
| 13127 | Check { |
| 13128 | line_num: 1, |
| 13129 | pattern: "alpha".into(), |
| 13130 | negative: false, |
| 13131 | kind: "CHECK", |
| 13132 | }, |
| 13133 | Check { |
| 13134 | line_num: 2, |
| 13135 | pattern: "omega".into(), |
| 13136 | negative: false, |
| 13137 | kind: "CHECK", |
| 13138 | }, |
| 13139 | ]; |
| 13140 | assert!(match_checks(&checks, "alpha\nmiddle\nomega\n", "demo").is_ok()); |
| 13141 | assert!(match_checks(&checks, "omega\nalpha\n", "demo").is_err()); |
| 13142 | } |
| 13143 | |
| 13144 | #[test] |
| 13145 | fn ir_check_matching_supports_negative_patterns() { |
| 13146 | let checks = vec![ |
| 13147 | Check { |
| 13148 | line_num: 1, |
| 13149 | pattern: "func @demo".into(), |
| 13150 | negative: false, |
| 13151 | kind: "IR_CHECK", |
| 13152 | }, |
| 13153 | Check { |
| 13154 | line_num: 2, |
| 13155 | pattern: "zeroinit".into(), |
| 13156 | negative: true, |
| 13157 | kind: "IR_NOT", |
| 13158 | }, |
| 13159 | ]; |
| 13160 | let ok_ir = "module main\n\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13161 | assert!(match_checks(&checks, ok_ir, "demo").is_ok()); |
| 13162 | |
| 13163 | let bad_ir = "module main\n global @value: i32 = zeroinit\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13164 | let err = match_checks(&checks, bad_ir, "demo").unwrap_err(); |
| 13165 | assert!(err.contains("IR_NOT failed")); |
| 13166 | } |
| 13167 | |
| 13168 | fn run_only_result(stdout: &str, stderr: &str, exit_code: i32) -> CaptureResult { |
| 13169 | CaptureResult { |
| 13170 | input: PathBuf::from("demo.f90"), |
| 13171 | opt_level: OptLevel::O0, |
| 13172 | stages: std::collections::BTreeMap::from([( |
| 13173 | Stage::Run, |
| 13174 | CapturedStage::Run(RunCapture { |
| 13175 | exit_code, |
| 13176 | stdout: stdout.into(), |
| 13177 | stderr: stderr.into(), |
| 13178 | }), |
| 13179 | )]), |
| 13180 | } |
| 13181 | } |
| 13182 | |
| 13183 | fn reference_run( |
| 13184 | compiler: ReferenceCompiler, |
| 13185 | stdout: &str, |
| 13186 | stderr: &str, |
| 13187 | exit_code: i32, |
| 13188 | ) -> ReferenceResult { |
| 13189 | ReferenceResult { |
| 13190 | compiler, |
| 13191 | compile_command: format!("{} demo.f90 -o demo", compiler.as_str()), |
| 13192 | compile_exit_code: 0, |
| 13193 | compile_stdout: String::new(), |
| 13194 | compile_stderr: String::new(), |
| 13195 | run: Some(RunCapture { |
| 13196 | exit_code, |
| 13197 | stdout: stdout.into(), |
| 13198 | stderr: stderr.into(), |
| 13199 | }), |
| 13200 | run_error: None, |
| 13201 | } |
| 13202 | } |
| 13203 | |
| 13204 | fn differential_armfortas_observation( |
| 13205 | stdout: &str, |
| 13206 | stderr: &str, |
| 13207 | exit_code: i32, |
| 13208 | ) -> CompilerObservation { |
| 13209 | observed_program_from_armfortas_capture( |
| 13210 | Path::new("demo.f90"), |
| 13211 | OptLevel::O0, |
| 13212 | default_differential_artifacts(), |
| 13213 | &run_only_result(stdout, stderr, exit_code), |
| 13214 | None, |
| 13215 | ) |
| 13216 | .observation |
| 13217 | } |
| 13218 | |
| 13219 | fn differential_reference_observation( |
| 13220 | compiler: ReferenceCompiler, |
| 13221 | stdout: &str, |
| 13222 | stderr: &str, |
| 13223 | exit_code: i32, |
| 13224 | ) -> CompilerObservation { |
| 13225 | observed_program_from_reference_result( |
| 13226 | Path::new("demo.f90"), |
| 13227 | OptLevel::O0, |
| 13228 | default_differential_artifacts(), |
| 13229 | &reference_run(compiler, stdout, stderr, exit_code), |
| 13230 | ) |
| 13231 | .observation |
| 13232 | } |
| 13233 | |
| 13234 | #[test] |
| 13235 | fn not_contains_expectation_checks_text_absence() { |
| 13236 | let case = CaseSpec { |
| 13237 | name: "no_reserved_register".into(), |
| 13238 | source: PathBuf::from("demo.f90"), |
| 13239 | graph_files: Vec::new(), |
| 13240 | requested: BTreeSet::from([Stage::Asm]), |
| 13241 | generic_introspect: None, |
| 13242 | generic_compare: None, |
| 13243 | opt_levels: vec![OptLevel::O0], |
| 13244 | repeat_count: 2, |
| 13245 | reference_compilers: Vec::new(), |
| 13246 | consistency_checks: Vec::new(), |
| 13247 | expectations: vec![Expectation::NotContains { |
| 13248 | target: Target::Stage(Stage::Asm), |
| 13249 | needle: "x18".into(), |
| 13250 | }], |
| 13251 | status_rules: Vec::new(), |
| 13252 | capability_policy: None, |
| 13253 | }; |
| 13254 | let result = CaptureResult { |
| 13255 | input: PathBuf::from("demo.f90"), |
| 13256 | opt_level: OptLevel::O0, |
| 13257 | stages: std::collections::BTreeMap::from([( |
| 13258 | Stage::Asm, |
| 13259 | CapturedStage::Text("mov x19, x0\nret\n".into()), |
| 13260 | )]), |
| 13261 | }; |
| 13262 | let observed = observed_program_from_armfortas_capture( |
| 13263 | Path::new("demo.f90"), |
| 13264 | OptLevel::O0, |
| 13265 | expected_artifacts_for_legacy_case(&case), |
| 13266 | &result, |
| 13267 | None, |
| 13268 | ); |
| 13269 | assert!(evaluate_observation_expectations(&case, &observed).is_ok()); |
| 13270 | |
| 13271 | let bad = CaptureResult { |
| 13272 | input: PathBuf::from("demo.f90"), |
| 13273 | opt_level: OptLevel::O0, |
| 13274 | stages: std::collections::BTreeMap::from([( |
| 13275 | Stage::Asm, |
| 13276 | CapturedStage::Text("mov x18, x0\nret\n".into()), |
| 13277 | )]), |
| 13278 | }; |
| 13279 | let observed = observed_program_from_armfortas_capture( |
| 13280 | Path::new("demo.f90"), |
| 13281 | OptLevel::O0, |
| 13282 | expected_artifacts_for_legacy_case(&case), |
| 13283 | &bad, |
| 13284 | None, |
| 13285 | ); |
| 13286 | let err = evaluate_observation_expectations(&case, &observed).unwrap_err(); |
| 13287 | assert!(err.contains("expected asm to not contain")); |
| 13288 | } |
| 13289 | |
| 13290 | #[test] |
| 13291 | fn writes_failure_bundle_with_artifacts() { |
| 13292 | let source = std::env::temp_dir().join("afs_tests_bundle_source.f90"); |
| 13293 | fs::write(&source, "program hello\nprint *, 'hello'\nend program\n").unwrap(); |
| 13294 | |
| 13295 | let suite = SuiteSpec { |
| 13296 | name: "runtime/bundles".into(), |
| 13297 | path: PathBuf::from("/tmp/runtime/bundles.afs"), |
| 13298 | cases: Vec::new(), |
| 13299 | }; |
| 13300 | let case = CaseSpec { |
| 13301 | name: "hello_bundle".into(), |
| 13302 | source: source.clone(), |
| 13303 | graph_files: Vec::new(), |
| 13304 | requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13305 | generic_introspect: None, |
| 13306 | generic_compare: None, |
| 13307 | opt_levels: vec![OptLevel::O0], |
| 13308 | repeat_count: 3, |
| 13309 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 13310 | consistency_checks: vec![ConsistencyCheck::CliObjVsSystemAs], |
| 13311 | expectations: Vec::new(), |
| 13312 | status_rules: Vec::new(), |
| 13313 | capability_policy: None, |
| 13314 | }; |
| 13315 | let mut stages = std::collections::BTreeMap::new(); |
| 13316 | stages.insert(Stage::Ir, CapturedStage::Text("module main".into())); |
| 13317 | stages.insert( |
| 13318 | Stage::Run, |
| 13319 | CapturedStage::Run(RunCapture { |
| 13320 | exit_code: 1, |
| 13321 | stdout: "oops\n".into(), |
| 13322 | stderr: "broken\n".into(), |
| 13323 | }), |
| 13324 | ); |
| 13325 | let artifacts = ExecutionArtifacts { |
| 13326 | requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13327 | armfortas: None, |
| 13328 | armfortas_failure: Some(CaptureFailure { |
| 13329 | input: source.clone(), |
| 13330 | opt_level: OptLevel::O0, |
| 13331 | stage: FailureStage::Sema, |
| 13332 | detail: "compiler failed".into(), |
| 13333 | stages, |
| 13334 | }), |
| 13335 | armfortas_observation: Some(ObservedProgram { |
| 13336 | observation: CompilerObservation { |
| 13337 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13338 | program: source.clone(), |
| 13339 | opt_level: OptLevel::O0, |
| 13340 | compile_exit_code: 1, |
| 13341 | artifacts: BTreeMap::from([ |
| 13342 | ( |
| 13343 | ArtifactKey::Diagnostics, |
| 13344 | ArtifactValue::Text("cached observation failure".into()), |
| 13345 | ), |
| 13346 | ( |
| 13347 | ArtifactKey::Extra("armfortas.ast".into()), |
| 13348 | ArtifactValue::Text("program hello".into()), |
| 13349 | ), |
| 13350 | ]), |
| 13351 | provenance: ObservationProvenance { |
| 13352 | compiler_identity: "armfortas".into(), |
| 13353 | adapter_kind: "named".into(), |
| 13354 | backend_mode: "linked".into(), |
| 13355 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13356 | artifacts_captured: vec!["diagnostics".into(), "armfortas.ast".into()], |
| 13357 | comparison_basis: None, |
| 13358 | failure_stage: Some("sema".into()), |
| 13359 | }, |
| 13360 | }, |
| 13361 | requested_artifacts: BTreeSet::from([ |
| 13362 | ArtifactKey::Diagnostics, |
| 13363 | ArtifactKey::Extra("armfortas.ast".into()), |
| 13364 | ]), |
| 13365 | }), |
| 13366 | references: vec![ReferenceResult { |
| 13367 | compiler: ReferenceCompiler::Gfortran, |
| 13368 | compile_command: "gfortran hello.f90 -o hello".into(), |
| 13369 | compile_exit_code: 0, |
| 13370 | compile_stdout: String::new(), |
| 13371 | compile_stderr: String::new(), |
| 13372 | run: Some(RunCapture { |
| 13373 | exit_code: 0, |
| 13374 | stdout: "hello\n".into(), |
| 13375 | stderr: String::new(), |
| 13376 | }), |
| 13377 | run_error: None, |
| 13378 | }], |
| 13379 | reference_observations: vec![ObservedProgram { |
| 13380 | observation: CompilerObservation { |
| 13381 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 13382 | program: source.clone(), |
| 13383 | opt_level: OptLevel::O0, |
| 13384 | compile_exit_code: 0, |
| 13385 | artifacts: BTreeMap::from([( |
| 13386 | ArtifactKey::Asm, |
| 13387 | ArtifactValue::Text(".globl _main".into()), |
| 13388 | )]), |
| 13389 | provenance: ObservationProvenance { |
| 13390 | compiler_identity: "gfortran".into(), |
| 13391 | adapter_kind: "named".into(), |
| 13392 | backend_mode: "legacy-reference".into(), |
| 13393 | backend_detail: "cached reference observation".into(), |
| 13394 | artifacts_captured: vec!["asm".into()], |
| 13395 | comparison_basis: None, |
| 13396 | failure_stage: None, |
| 13397 | }, |
| 13398 | }, |
| 13399 | requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 13400 | }], |
| 13401 | consistency_issues: { |
| 13402 | let asm_temp_root = |
| 13403 | std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm"); |
| 13404 | fs::create_dir_all(&asm_temp_root).unwrap(); |
| 13405 | fs::write(asm_temp_root.join("run_00.s"), "mov x19, x0\n").unwrap(); |
| 13406 | |
| 13407 | let obj_temp_root = |
| 13408 | std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj"); |
| 13409 | fs::create_dir_all(&obj_temp_root).unwrap(); |
| 13410 | fs::write(obj_temp_root.join("run_00.o"), "fake object bytes\n").unwrap(); |
| 13411 | |
| 13412 | vec![ |
| 13413 | ConsistencyIssue { |
| 13414 | check: ConsistencyCheck::CliAsmReproducible, |
| 13415 | summary: "repeat_count=3 unique_variants=3".into(), |
| 13416 | repeat_count: Some(3), |
| 13417 | unique_variant_count: Some(3), |
| 13418 | varying_components: Vec::new(), |
| 13419 | stable_components: Vec::new(), |
| 13420 | detail: "assembly output is not reproducible".into(), |
| 13421 | temp_root: asm_temp_root, |
| 13422 | }, |
| 13423 | ConsistencyIssue { |
| 13424 | check: ConsistencyCheck::CliObjReproducible, |
| 13425 | summary: "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols".into(), |
| 13426 | repeat_count: Some(3), |
| 13427 | unique_variant_count: Some(2), |
| 13428 | varying_components: vec!["text".into()], |
| 13429 | stable_components: vec![ |
| 13430 | "load_commands".into(), |
| 13431 | "relocations".into(), |
| 13432 | "symbols".into(), |
| 13433 | ], |
| 13434 | detail: "object output is not reproducible".into(), |
| 13435 | temp_root: obj_temp_root, |
| 13436 | }, |
| 13437 | ] |
| 13438 | }, |
| 13439 | }; |
| 13440 | let outcome = Outcome { |
| 13441 | suite: suite.name.clone(), |
| 13442 | case: case.name.clone(), |
| 13443 | opt_level: OptLevel::O0, |
| 13444 | kind: OutcomeKind::Fail, |
| 13445 | detail: "boom".into(), |
| 13446 | bundle: None, |
| 13447 | primary_backend: Some(PrimaryBackendReport { |
| 13448 | kind: "full".into(), |
| 13449 | mode: "linked".into(), |
| 13450 | detail: "linked armfortas::testing capture adapter".into(), |
| 13451 | }), |
| 13452 | consistency_observations: Vec::new(), |
| 13453 | }; |
| 13454 | let prepared = PreparedInput { |
| 13455 | compiler_source: source.clone(), |
| 13456 | generated_source: None, |
| 13457 | temp_root: None, |
| 13458 | }; |
| 13459 | |
| 13460 | let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13461 | assert!(bundle.join("metadata.txt").exists()); |
| 13462 | assert!(bundle.join("detail.txt").exists()); |
| 13463 | assert!(bundle.join("source.f90").exists()); |
| 13464 | assert!(bundle.join("armfortas").join("ir.txt").exists()); |
| 13465 | assert!(bundle.join("armfortas").join("metadata.txt").exists()); |
| 13466 | assert!(bundle.join("armfortas").join("observation.txt").exists()); |
| 13467 | assert!(bundle.join("armfortas").join("observation.json").exists()); |
| 13468 | assert!(bundle.join("armfortas").join("observation.md").exists()); |
| 13469 | assert!(bundle.join("armfortas").join("run.stdout.txt").exists()); |
| 13470 | assert!(bundle.join("armfortas").join("error.txt").exists()); |
| 13471 | assert!(bundle |
| 13472 | .join("references") |
| 13473 | .join("gfortran") |
| 13474 | .join("observation.txt") |
| 13475 | .exists()); |
| 13476 | assert!(bundle |
| 13477 | .join("references") |
| 13478 | .join("gfortran") |
| 13479 | .join("observation.json") |
| 13480 | .exists()); |
| 13481 | assert!(bundle |
| 13482 | .join("references") |
| 13483 | .join("gfortran") |
| 13484 | .join("observation.md") |
| 13485 | .exists()); |
| 13486 | assert!(bundle.join("references").join("summary.txt").exists()); |
| 13487 | assert!(bundle |
| 13488 | .join("references") |
| 13489 | .join("gfortran") |
| 13490 | .join("run.stdout.txt") |
| 13491 | .exists()); |
| 13492 | assert!(bundle.join("consistency").join("summary.txt").exists()); |
| 13493 | let metadata = fs::read_to_string(bundle.join("metadata.txt")).unwrap(); |
| 13494 | assert!(metadata.contains("primary_backend_kind: full")); |
| 13495 | assert!(metadata.contains("primary_backend_mode: linked")); |
| 13496 | assert!( |
| 13497 | metadata.contains("primary_backend_detail: linked armfortas::testing capture adapter") |
| 13498 | ); |
| 13499 | let armfortas_metadata = |
| 13500 | fs::read_to_string(bundle.join("armfortas").join("metadata.txt")).unwrap(); |
| 13501 | assert!(armfortas_metadata.contains("primary_backend_kind: full")); |
| 13502 | assert!(armfortas_metadata.contains("primary_backend_mode: linked")); |
| 13503 | assert!(armfortas_metadata |
| 13504 | .contains("primary_backend_detail: linked armfortas::testing capture adapter")); |
| 13505 | assert!(armfortas_metadata.contains("captured_stages: ir, run")); |
| 13506 | assert!(armfortas_metadata.contains("error_stage: sema")); |
| 13507 | let observation = |
| 13508 | fs::read_to_string(bundle.join("armfortas").join("observation.txt")).unwrap(); |
| 13509 | assert!(observation.contains("Introspect")); |
| 13510 | assert!(observation.contains("compiler: armfortas")); |
| 13511 | assert!(observation.contains("failure_stage: sema")); |
| 13512 | assert!(observation.contains("generic_artifacts: diagnostics")); |
| 13513 | assert!(observation.contains("adapter_extras: armfortas(ast)")); |
| 13514 | assert!(observation.contains("cached observation failure")); |
| 13515 | let reference_observation = fs::read_to_string( |
| 13516 | bundle |
| 13517 | .join("references") |
| 13518 | .join("gfortran") |
| 13519 | .join("observation.txt"), |
| 13520 | ) |
| 13521 | .unwrap(); |
| 13522 | assert!(reference_observation.contains("Introspect")); |
| 13523 | assert!(reference_observation.contains("compiler: gfortran")); |
| 13524 | assert!(reference_observation.contains("status: compile ok")); |
| 13525 | assert!(reference_observation.contains("requested_artifacts: asm")); |
| 13526 | assert!(reference_observation.contains("generic_artifacts: asm")); |
| 13527 | assert!(reference_observation.contains("cached reference observation")); |
| 13528 | let reference_summary = |
| 13529 | fs::read_to_string(bundle.join("references").join("summary.txt")).unwrap(); |
| 13530 | assert!(reference_summary.contains("reference_count: 1")); |
| 13531 | assert!(reference_summary.contains("compilers: gfortran")); |
| 13532 | assert!(reference_summary.contains("compiler: gfortran")); |
| 13533 | assert!(reference_summary.contains("status: compile ok")); |
| 13534 | assert!(reference_summary.contains("compile_exit_code: 0")); |
| 13535 | assert!(reference_summary.contains("command: gfortran hello.f90 -o hello")); |
| 13536 | assert!(reference_summary.contains("generic_artifacts: asm")); |
| 13537 | assert!(reference_summary.contains("adapter_extras: none")); |
| 13538 | let consistency_summary = |
| 13539 | fs::read_to_string(bundle.join("consistency").join("summary.txt")).unwrap(); |
| 13540 | assert!(consistency_summary.contains("issue_count: 2")); |
| 13541 | assert!(consistency_summary.contains("checks: cli_asm_reproducible, cli_obj_reproducible")); |
| 13542 | assert!(consistency_summary.contains("repeat_counts: 3")); |
| 13543 | assert!(consistency_summary.contains("unique_variants: 2, 3")); |
| 13544 | assert!(consistency_summary.contains("varying_components: text")); |
| 13545 | assert!( |
| 13546 | consistency_summary.contains("stable_components: load_commands, relocations, symbols") |
| 13547 | ); |
| 13548 | assert!(bundle |
| 13549 | .join("consistency") |
| 13550 | .join("cli_asm_reproducible") |
| 13551 | .join("summary.txt") |
| 13552 | .exists()); |
| 13553 | assert!(bundle |
| 13554 | .join("consistency") |
| 13555 | .join("cli_asm_reproducible") |
| 13556 | .join("detail.txt") |
| 13557 | .exists()); |
| 13558 | assert!(bundle |
| 13559 | .join("consistency") |
| 13560 | .join("cli_asm_reproducible") |
| 13561 | .join("artifacts") |
| 13562 | .join("run_00.s") |
| 13563 | .exists()); |
| 13564 | assert!(bundle |
| 13565 | .join("consistency") |
| 13566 | .join("cli_obj_reproducible") |
| 13567 | .join("artifacts") |
| 13568 | .join("run_00.o") |
| 13569 | .exists()); |
| 13570 | |
| 13571 | let _ = fs::remove_dir_all(bundle); |
| 13572 | let _ = |
| 13573 | fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm")); |
| 13574 | let _ = |
| 13575 | fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj")); |
| 13576 | let _ = fs::remove_file(source); |
| 13577 | } |
| 13578 | |
| 13579 | #[test] |
| 13580 | fn materializes_graph_input_in_declared_file_order() { |
| 13581 | let root = std::env::temp_dir().join("afs_tests_graph_materialize"); |
| 13582 | let _ = fs::remove_dir_all(&root); |
| 13583 | fs::create_dir_all(&root).unwrap(); |
| 13584 | let module = root.join("math_values.f90"); |
| 13585 | let main = root.join("main.f90"); |
| 13586 | fs::write(&module, "module math_values\ncontains\nend module\n").unwrap(); |
| 13587 | fs::write(&main, "program main\nuse math_values\nend program\n").unwrap(); |
| 13588 | |
| 13589 | let suite = SuiteSpec { |
| 13590 | name: "modules/graph".into(), |
| 13591 | path: root.join("graph.afs"), |
| 13592 | cases: Vec::new(), |
| 13593 | }; |
| 13594 | let case = CaseSpec { |
| 13595 | name: "basic_use".into(), |
| 13596 | source: main.clone(), |
| 13597 | graph_files: vec![module.clone(), main.clone()], |
| 13598 | requested: BTreeSet::from([Stage::Run]), |
| 13599 | generic_introspect: None, |
| 13600 | generic_compare: None, |
| 13601 | opt_levels: vec![OptLevel::O0], |
| 13602 | repeat_count: 2, |
| 13603 | reference_compilers: Vec::new(), |
| 13604 | consistency_checks: Vec::new(), |
| 13605 | expectations: Vec::new(), |
| 13606 | status_rules: Vec::new(), |
| 13607 | capability_policy: None, |
| 13608 | }; |
| 13609 | |
| 13610 | let prepared = prepare_case_input(&case, &suite, OptLevel::O0).unwrap(); |
| 13611 | let generated = fs::read_to_string(&prepared.compiler_source).unwrap(); |
| 13612 | assert!(generated.contains("module math_values")); |
| 13613 | assert!(generated.contains("program main")); |
| 13614 | assert!( |
| 13615 | generated.find("module math_values").unwrap() < generated.find("program main").unwrap() |
| 13616 | ); |
| 13617 | |
| 13618 | cleanup_prepared_input(&prepared); |
| 13619 | let _ = fs::remove_dir_all(&root); |
| 13620 | } |
| 13621 | |
| 13622 | #[test] |
| 13623 | fn graph_failure_bundle_writes_authored_sources() { |
| 13624 | let root = std::env::temp_dir().join("afs_tests_graph_bundle"); |
| 13625 | let _ = fs::remove_dir_all(&root); |
| 13626 | fs::create_dir_all(&root).unwrap(); |
| 13627 | let module = root.join("math_values.f90"); |
| 13628 | let main = root.join("main.f90"); |
| 13629 | let generated = root.join("generated.f90"); |
| 13630 | fs::write( |
| 13631 | &module, |
| 13632 | "module math_values\n integer :: answer = 42\nend module\n", |
| 13633 | ) |
| 13634 | .unwrap(); |
| 13635 | fs::write( |
| 13636 | &main, |
| 13637 | "program main\n use math_values\n print *, answer\nend program\n", |
| 13638 | ) |
| 13639 | .unwrap(); |
| 13640 | fs::write(&generated, "module math_values\n integer :: answer = 42\nend module\n\nprogram main\n use math_values\n print *, answer\nend program\n").unwrap(); |
| 13641 | |
| 13642 | let suite = SuiteSpec { |
| 13643 | name: "modules/bundles".into(), |
| 13644 | path: root.join("bundle.afs"), |
| 13645 | cases: Vec::new(), |
| 13646 | }; |
| 13647 | let case = CaseSpec { |
| 13648 | name: "graph_bundle".into(), |
| 13649 | source: main.clone(), |
| 13650 | graph_files: vec![module.clone(), main.clone()], |
| 13651 | requested: BTreeSet::from([Stage::Run]), |
| 13652 | generic_introspect: None, |
| 13653 | generic_compare: None, |
| 13654 | opt_levels: vec![OptLevel::O0], |
| 13655 | repeat_count: 2, |
| 13656 | reference_compilers: Vec::new(), |
| 13657 | consistency_checks: Vec::new(), |
| 13658 | expectations: Vec::new(), |
| 13659 | status_rules: Vec::new(), |
| 13660 | capability_policy: None, |
| 13661 | }; |
| 13662 | let outcome = Outcome { |
| 13663 | suite: suite.name.clone(), |
| 13664 | case: case.name.clone(), |
| 13665 | opt_level: OptLevel::O0, |
| 13666 | kind: OutcomeKind::Fail, |
| 13667 | detail: "boom".into(), |
| 13668 | bundle: None, |
| 13669 | primary_backend: Some(PrimaryBackendReport { |
| 13670 | kind: "full".into(), |
| 13671 | mode: "linked".into(), |
| 13672 | detail: "linked armfortas::testing capture adapter".into(), |
| 13673 | }), |
| 13674 | consistency_observations: Vec::new(), |
| 13675 | }; |
| 13676 | let artifacts = ExecutionArtifacts { |
| 13677 | requested: BTreeSet::from([Stage::Run]), |
| 13678 | armfortas: Some(run_only_result("42\n", "", 0)), |
| 13679 | armfortas_failure: None, |
| 13680 | armfortas_observation: None, |
| 13681 | references: Vec::new(), |
| 13682 | reference_observations: Vec::new(), |
| 13683 | consistency_issues: Vec::new(), |
| 13684 | }; |
| 13685 | let prepared = PreparedInput { |
| 13686 | compiler_source: generated.clone(), |
| 13687 | generated_source: Some(generated.clone()), |
| 13688 | temp_root: None, |
| 13689 | }; |
| 13690 | |
| 13691 | let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13692 | assert!(bundle.join("source.f90").exists()); |
| 13693 | assert!(bundle.join("sources").join("00_math_values.f90").exists()); |
| 13694 | assert!(bundle.join("sources").join("01_main.f90").exists()); |
| 13695 | |
| 13696 | let _ = fs::remove_dir_all(bundle); |
| 13697 | let _ = fs::remove_dir_all(&root); |
| 13698 | } |
| 13699 | |
| 13700 | #[test] |
| 13701 | fn armfortas_bundle_observation_prefers_cached_observation() { |
| 13702 | let prepared = PreparedInput { |
| 13703 | compiler_source: PathBuf::from("demo.f90"), |
| 13704 | generated_source: None, |
| 13705 | temp_root: None, |
| 13706 | }; |
| 13707 | let artifacts = ExecutionArtifacts { |
| 13708 | requested: BTreeSet::from([Stage::Run]), |
| 13709 | armfortas: Some(run_only_result("42\n", "", 0)), |
| 13710 | armfortas_failure: None, |
| 13711 | armfortas_observation: Some(ObservedProgram { |
| 13712 | observation: CompilerObservation { |
| 13713 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13714 | program: PathBuf::from("demo.f90"), |
| 13715 | opt_level: OptLevel::O0, |
| 13716 | compile_exit_code: 0, |
| 13717 | artifacts: BTreeMap::from([( |
| 13718 | ArtifactKey::Extra("armfortas.sema".into()), |
| 13719 | ArtifactValue::Text("ok".into()), |
| 13720 | )]), |
| 13721 | provenance: ObservationProvenance { |
| 13722 | compiler_identity: "armfortas".into(), |
| 13723 | adapter_kind: "named".into(), |
| 13724 | backend_mode: "linked".into(), |
| 13725 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13726 | artifacts_captured: vec!["armfortas.sema".into()], |
| 13727 | comparison_basis: None, |
| 13728 | failure_stage: None, |
| 13729 | }, |
| 13730 | }, |
| 13731 | requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.sema".into())]), |
| 13732 | }), |
| 13733 | references: Vec::new(), |
| 13734 | reference_observations: Vec::new(), |
| 13735 | consistency_issues: Vec::new(), |
| 13736 | }; |
| 13737 | |
| 13738 | let observed = observed_program_for_armfortas_bundle(&prepared, &artifacts).unwrap(); |
| 13739 | assert!(observed |
| 13740 | .observation |
| 13741 | .artifacts |
| 13742 | .contains_key(&ArtifactKey::Extra("armfortas.sema".into()))); |
| 13743 | assert!(!observed |
| 13744 | .observation |
| 13745 | .artifacts |
| 13746 | .contains_key(&ArtifactKey::Runtime)); |
| 13747 | } |
| 13748 | |
| 13749 | #[test] |
| 13750 | fn render_summary_includes_consistency_rollups() { |
| 13751 | let mut summary = Summary::default(); |
| 13752 | summary.record_consistency(&[ |
| 13753 | ConsistencyObservation { |
| 13754 | check: ConsistencyCheck::CliAsmReproducible, |
| 13755 | summary: "repeat_count=3 unique_variants=3".into(), |
| 13756 | repeat_count: Some(3), |
| 13757 | unique_variant_count: Some(3), |
| 13758 | varying_components: Vec::new(), |
| 13759 | stable_components: Vec::new(), |
| 13760 | }, |
| 13761 | ConsistencyObservation { |
| 13762 | check: ConsistencyCheck::CliObjReproducible, |
| 13763 | summary: |
| 13764 | "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols" |
| 13765 | .into(), |
| 13766 | repeat_count: Some(3), |
| 13767 | unique_variant_count: Some(2), |
| 13768 | varying_components: vec!["text".into()], |
| 13769 | stable_components: vec![ |
| 13770 | "load_commands".into(), |
| 13771 | "relocations".into(), |
| 13772 | "symbols".into(), |
| 13773 | ], |
| 13774 | }, |
| 13775 | ]); |
| 13776 | |
| 13777 | let rendered = render_summary(&summary); |
| 13778 | assert!(rendered.contains("Consistency")); |
| 13779 | assert!(rendered.contains("affected_checks: 2")); |
| 13780 | assert!(rendered.contains("cells_with_issues: 2")); |
| 13781 | assert!( |
| 13782 | rendered.contains("cli_asm_reproducible: 1 cells; repeat_count=3; unique_variants=3") |
| 13783 | ); |
| 13784 | assert!(rendered.contains( |
| 13785 | "cli_obj_reproducible: 1 cells; repeat_count=3; unique_variants=2; varying=text; stable=load_commands, relocations, symbols" |
| 13786 | )); |
| 13787 | } |
| 13788 | |
| 13789 | #[test] |
| 13790 | fn render_reports_include_outcomes() { |
| 13791 | let mut summary = Summary::default(); |
| 13792 | summary.record_outcome(&Outcome { |
| 13793 | suite: "modules/runtime-graphs".into(), |
| 13794 | case: "module_chain_runtime".into(), |
| 13795 | opt_level: OptLevel::O0, |
| 13796 | kind: OutcomeKind::Xfail, |
| 13797 | detail: "expected 42, got 0".into(), |
| 13798 | bundle: Some(PathBuf::from("/tmp/bundle")), |
| 13799 | primary_backend: Some(PrimaryBackendReport { |
| 13800 | kind: "observable".into(), |
| 13801 | mode: "cli-observable".into(), |
| 13802 | detail: "cli-observable armfortas driver capture adapter".into(), |
| 13803 | }), |
| 13804 | consistency_observations: vec![ConsistencyObservation { |
| 13805 | check: ConsistencyCheck::CliRunReproducible, |
| 13806 | summary: "repeat_count=3 unique_variants=1".into(), |
| 13807 | repeat_count: Some(3), |
| 13808 | unique_variant_count: Some(1), |
| 13809 | varying_components: Vec::new(), |
| 13810 | stable_components: vec!["exit_code".into(), "stdout".into(), "stderr".into()], |
| 13811 | }], |
| 13812 | }); |
| 13813 | |
| 13814 | let json = render_json_report(&summary); |
| 13815 | assert!(json.contains("\"outcomes\": [")); |
| 13816 | assert!(json.contains("\"suite\": \"modules/runtime-graphs\"")); |
| 13817 | assert!(json.contains("\"bundle\": \"/tmp/bundle\"")); |
| 13818 | assert!(json.contains("\"primary_backend\": {")); |
| 13819 | assert!(json.contains("\"mode\": \"cli-observable\"")); |
| 13820 | |
| 13821 | let markdown = render_markdown_report(&summary); |
| 13822 | assert!(markdown.contains("# afs-tests report")); |
| 13823 | assert!(markdown |
| 13824 | .contains("### `modules/runtime-graphs` / `module_chain_runtime` / `O0` / `xfail`")); |
| 13825 | assert!(markdown.contains("primary_backend: `observable` (`cli-observable`)")); |
| 13826 | assert!(markdown.contains("bundle: `/tmp/bundle`")); |
| 13827 | assert!(markdown.contains("expected 42, got 0")); |
| 13828 | } |
| 13829 | |
| 13830 | #[test] |
| 13831 | fn render_generic_reports_include_provenance() { |
| 13832 | let observation = CompilerObservation { |
| 13833 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13834 | program: PathBuf::from("demo.f90"), |
| 13835 | opt_level: OptLevel::O0, |
| 13836 | compile_exit_code: 0, |
| 13837 | artifacts: BTreeMap::from([ |
| 13838 | ( |
| 13839 | ArtifactKey::Asm, |
| 13840 | ArtifactValue::Text(".globl _main\n".into()), |
| 13841 | ), |
| 13842 | ( |
| 13843 | ArtifactKey::Extra("armfortas.ir".into()), |
| 13844 | ArtifactValue::Text("module main".into()), |
| 13845 | ), |
| 13846 | ]), |
| 13847 | provenance: ObservationProvenance { |
| 13848 | compiler_identity: "armfortas".into(), |
| 13849 | adapter_kind: "named".into(), |
| 13850 | backend_mode: "linked".into(), |
| 13851 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13852 | artifacts_captured: vec!["asm".into(), "armfortas.ir".into()], |
| 13853 | comparison_basis: None, |
| 13854 | failure_stage: None, |
| 13855 | }, |
| 13856 | }; |
| 13857 | let compare = ComparisonResult { |
| 13858 | left: observation.clone(), |
| 13859 | right: CompilerObservation { |
| 13860 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 13861 | program: PathBuf::from("demo.f90"), |
| 13862 | opt_level: OptLevel::O0, |
| 13863 | compile_exit_code: 0, |
| 13864 | artifacts: BTreeMap::from([( |
| 13865 | ArtifactKey::Asm, |
| 13866 | ArtifactValue::Text(".arch armv8.5-a".into()), |
| 13867 | )]), |
| 13868 | provenance: ObservationProvenance { |
| 13869 | compiler_identity: "gfortran".into(), |
| 13870 | adapter_kind: "named".into(), |
| 13871 | backend_mode: "external-driver".into(), |
| 13872 | backend_detail: "generic external driver adapter using gfortran".into(), |
| 13873 | artifacts_captured: vec!["asm".into()], |
| 13874 | comparison_basis: Some("compile-status, diagnostics, runtime, asm".into()), |
| 13875 | failure_stage: None, |
| 13876 | }, |
| 13877 | }, |
| 13878 | basis: "compile-status, diagnostics, runtime, asm".into(), |
| 13879 | differences: vec![ArtifactDifference { |
| 13880 | artifact: "asm".into(), |
| 13881 | detail: "first differing line: 1".into(), |
| 13882 | }], |
| 13883 | }; |
| 13884 | |
| 13885 | let observed = ObservedProgram { |
| 13886 | observation: observation.clone(), |
| 13887 | requested_artifacts: BTreeSet::from([ |
| 13888 | ArtifactKey::Asm, |
| 13889 | ArtifactKey::Extra("armfortas.ir".into()), |
| 13890 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 13891 | ]), |
| 13892 | }; |
| 13893 | |
| 13894 | let introspection_text = |
| 13895 | render_introspection_text(&observed, full_introspection_render_config()); |
| 13896 | assert!(introspection_text.contains("status: compile ok")); |
| 13897 | assert!(introspection_text.contains("artifact_count: 2")); |
| 13898 | assert!( |
| 13899 | introspection_text.contains("requested_artifacts: asm, armfortas.ir, armfortas.tokens") |
| 13900 | ); |
| 13901 | assert!(introspection_text.contains("missing_artifacts: armfortas.tokens")); |
| 13902 | assert!(introspection_text.contains("generic_artifacts: asm")); |
| 13903 | assert!(introspection_text.contains("adapter_extras: armfortas(ir)")); |
| 13904 | assert!(introspection_text.contains("Generic artifacts")); |
| 13905 | assert!(introspection_text.contains("Adapter extras")); |
| 13906 | |
| 13907 | let introspection_json = render_introspection_json(&observed); |
| 13908 | assert!(introspection_json.contains("\"status\": \"compile ok\"")); |
| 13909 | assert!(introspection_json.contains("\"artifact_count\": 2")); |
| 13910 | assert!(introspection_json.contains( |
| 13911 | "\"requested_artifacts\": [\"asm\", \"armfortas.ir\", \"armfortas.tokens\"]" |
| 13912 | )); |
| 13913 | assert!(introspection_json.contains("\"missing_artifacts\": [\"armfortas.tokens\"]")); |
| 13914 | assert!(introspection_json.contains("\"artifact_summaries\":")); |
| 13915 | assert!(introspection_json.contains("\"asm\": {\"kind\":\"text\"")); |
| 13916 | assert!(introspection_json.contains("\"line_count\":1")); |
| 13917 | assert!(introspection_json.contains("\"generic_artifacts\": [\"asm\"]")); |
| 13918 | assert!(introspection_json.contains("\"adapter_extras\": {\"armfortas\": [\"ir\"]}")); |
| 13919 | assert!(introspection_json.contains("\"backend_mode\": \"linked\"")); |
| 13920 | assert!(introspection_json.contains("\"armfortas.ir\"")); |
| 13921 | |
| 13922 | let introspection_markdown = |
| 13923 | render_introspection_markdown(&observed, full_introspection_render_config()); |
| 13924 | assert!(introspection_markdown.contains("# bencch introspect report")); |
| 13925 | assert!(introspection_markdown.contains("status: compile ok")); |
| 13926 | assert!(introspection_markdown.contains("failure_stage: `none`")); |
| 13927 | assert!(introspection_markdown.contains("artifact_count: 2")); |
| 13928 | assert!(introspection_markdown |
| 13929 | .contains("requested_artifacts: `asm`, `armfortas.ir`, `armfortas.tokens`")); |
| 13930 | assert!(introspection_markdown.contains("missing_artifacts: `armfortas.tokens`")); |
| 13931 | assert!(introspection_markdown.contains("## Generic artifacts")); |
| 13932 | assert!(introspection_markdown.contains("## Adapter extras")); |
| 13933 | assert!(introspection_markdown.contains("### `armfortas`")); |
| 13934 | assert!(introspection_markdown.contains("#### `ir`")); |
| 13935 | |
| 13936 | let compare_markdown = render_compare_markdown(&compare); |
| 13937 | assert!(compare_markdown.contains("# bencch compare report")); |
| 13938 | assert!(compare_markdown.contains("status: diff")); |
| 13939 | assert!(compare_markdown.contains("classification: artifact divergence")); |
| 13940 | assert!(compare_markdown.contains("difference_count: 1")); |
| 13941 | assert!(compare_markdown.contains("changed_artifacts: asm")); |
| 13942 | assert!(compare_markdown.contains("backend_mode: `external-driver`")); |
| 13943 | assert!(compare_markdown.contains("### `asm`")); |
| 13944 | |
| 13945 | let compare_json = render_compare_json(&compare); |
| 13946 | assert!(compare_json.contains("\"status\": \"diff\"")); |
| 13947 | assert!(compare_json.contains("\"classification\": \"artifact divergence\"")); |
| 13948 | assert!(compare_json.contains("\"difference_count\": 1")); |
| 13949 | assert!(compare_json.contains("\"changed_artifacts\": [\"asm\"]")); |
| 13950 | } |
| 13951 | |
| 13952 | #[test] |
| 13953 | fn render_failure_introspection_reports_stage_and_excerpt() { |
| 13954 | let observed = ObservedProgram { |
| 13955 | observation: CompilerObservation { |
| 13956 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13957 | program: PathBuf::from("broken.f90"), |
| 13958 | opt_level: OptLevel::O0, |
| 13959 | compile_exit_code: 1, |
| 13960 | artifacts: BTreeMap::from([ |
| 13961 | ( |
| 13962 | ArtifactKey::Diagnostics, |
| 13963 | ArtifactValue::Text("undefined symbol: missing_value\nmore detail".into()), |
| 13964 | ), |
| 13965 | ( |
| 13966 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 13967 | ArtifactValue::Text("token stream".into()), |
| 13968 | ), |
| 13969 | ]), |
| 13970 | provenance: ObservationProvenance { |
| 13971 | compiler_identity: "armfortas".into(), |
| 13972 | adapter_kind: "named".into(), |
| 13973 | backend_mode: "linked".into(), |
| 13974 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13975 | artifacts_captured: vec!["diagnostics".into(), "armfortas.tokens".into()], |
| 13976 | comparison_basis: None, |
| 13977 | failure_stage: Some("sema".into()), |
| 13978 | }, |
| 13979 | }, |
| 13980 | requested_artifacts: BTreeSet::from([ |
| 13981 | ArtifactKey::Asm, |
| 13982 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 13983 | ]), |
| 13984 | }; |
| 13985 | |
| 13986 | let text = render_introspection_text(&observed, full_introspection_render_config()); |
| 13987 | assert!(text.contains("status: compile failed")); |
| 13988 | assert!(text.contains("failure_stage: sema")); |
| 13989 | assert!(text.contains("diagnostic_excerpt: undefined symbol: missing_value")); |
| 13990 | assert!(text.contains("missing_artifacts: asm")); |
| 13991 | |
| 13992 | let json = render_introspection_json(&observed); |
| 13993 | assert!(json.contains("\"stage\": \"sema\"")); |
| 13994 | assert!(json.contains("\"diagnostic_excerpt\": \"undefined symbol: missing_value\"")); |
| 13995 | assert!(json.contains("\"failure_stage\": \"sema\"")); |
| 13996 | assert!(json.contains("\"diagnostics\": {\"kind\":\"text\"")); |
| 13997 | assert!(json.contains("\"summary\":\"text, 2 lines, ")); |
| 13998 | assert!(json.contains("\"line_count\":2")); |
| 13999 | |
| 14000 | let markdown = render_introspection_markdown(&observed, full_introspection_render_config()); |
| 14001 | assert!(markdown.contains("status: compile failed")); |
| 14002 | assert!(markdown.contains("failure_stage: `sema`")); |
| 14003 | assert!(markdown.contains("diagnostic_excerpt: `undefined symbol: missing_value`")); |
| 14004 | } |
| 14005 | |
| 14006 | #[test] |
| 14007 | fn render_introspection_summary_only_omits_artifact_bodies() { |
| 14008 | let observed = ObservedProgram { |
| 14009 | observation: CompilerObservation { |
| 14010 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14011 | program: PathBuf::from("demo.f90"), |
| 14012 | opt_level: OptLevel::O0, |
| 14013 | compile_exit_code: 0, |
| 14014 | artifacts: BTreeMap::from([( |
| 14015 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 14016 | ArtifactValue::Text("line1\nline2\nline3".into()), |
| 14017 | )]), |
| 14018 | provenance: ObservationProvenance { |
| 14019 | compiler_identity: "armfortas".into(), |
| 14020 | adapter_kind: "named".into(), |
| 14021 | backend_mode: "linked".into(), |
| 14022 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14023 | artifacts_captured: vec!["armfortas.tokens".into()], |
| 14024 | comparison_basis: None, |
| 14025 | failure_stage: None, |
| 14026 | }, |
| 14027 | }, |
| 14028 | requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.tokens".into())]), |
| 14029 | }; |
| 14030 | |
| 14031 | let config = IntrospectionRenderConfig { |
| 14032 | summary_only: true, |
| 14033 | max_artifact_lines: Some(1), |
| 14034 | }; |
| 14035 | let text = render_introspection_text(&observed, config); |
| 14036 | assert!(text.contains("content_mode: summary-only")); |
| 14037 | assert!(text.contains("summary: text, 3 lines, 17 chars")); |
| 14038 | assert!(text.contains("[content omitted by --summary-only]")); |
| 14039 | assert!(!text.contains("line2")); |
| 14040 | |
| 14041 | let markdown = render_introspection_markdown(&observed, config); |
| 14042 | assert!(markdown.contains("content_mode: `summary-only`")); |
| 14043 | assert!(markdown.contains("[content omitted by --summary-only]")); |
| 14044 | } |
| 14045 | |
| 14046 | #[test] |
| 14047 | fn render_introspection_truncates_large_artifacts() { |
| 14048 | let observed = ObservedProgram { |
| 14049 | observation: CompilerObservation { |
| 14050 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14051 | program: PathBuf::from("demo.f90"), |
| 14052 | opt_level: OptLevel::O0, |
| 14053 | compile_exit_code: 0, |
| 14054 | artifacts: BTreeMap::from([( |
| 14055 | ArtifactKey::Asm, |
| 14056 | ArtifactValue::Text("a\nb\nc\nd".into()), |
| 14057 | )]), |
| 14058 | provenance: ObservationProvenance { |
| 14059 | compiler_identity: "armfortas".into(), |
| 14060 | adapter_kind: "named".into(), |
| 14061 | backend_mode: "linked".into(), |
| 14062 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14063 | artifacts_captured: vec!["asm".into()], |
| 14064 | comparison_basis: None, |
| 14065 | failure_stage: None, |
| 14066 | }, |
| 14067 | }, |
| 14068 | requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 14069 | }; |
| 14070 | |
| 14071 | let config = IntrospectionRenderConfig { |
| 14072 | summary_only: false, |
| 14073 | max_artifact_lines: Some(2), |
| 14074 | }; |
| 14075 | let text = render_introspection_text(&observed, config); |
| 14076 | assert!(text.contains("content_mode: first 2 lines per artifact")); |
| 14077 | assert!(text.contains("a\nb\n... (truncated; showing first 2 of 4 lines)")); |
| 14078 | assert!(!text.contains("\nc\nd")); |
| 14079 | |
| 14080 | let markdown = render_introspection_markdown(&observed, config); |
| 14081 | assert!(markdown.contains("content_mode: `first 2 lines per artifact`")); |
| 14082 | assert!(markdown.contains("... (truncated; showing first 2 of 4 lines)")); |
| 14083 | } |
| 14084 | |
| 14085 | #[test] |
| 14086 | fn write_requested_reports_emits_files() { |
| 14087 | let root = std::env::temp_dir().join("afs_tests_report_output"); |
| 14088 | let _ = fs::remove_dir_all(&root); |
| 14089 | let json_path = root.join("result.json"); |
| 14090 | let markdown_path = root.join("result.md"); |
| 14091 | let config = RunConfig { |
| 14092 | suite_filter: None, |
| 14093 | case_filter: None, |
| 14094 | opt_filter: None, |
| 14095 | verbose: false, |
| 14096 | fail_fast: false, |
| 14097 | include_future: false, |
| 14098 | all_stages: false, |
| 14099 | json_report: Some(json_path.clone()), |
| 14100 | markdown_report: Some(markdown_path.clone()), |
| 14101 | tools: ToolchainConfig::from_env(), |
| 14102 | }; |
| 14103 | let mut summary = Summary::default(); |
| 14104 | summary.record_outcome(&Outcome { |
| 14105 | suite: "frontend/parser".into(), |
| 14106 | case: "where_construct".into(), |
| 14107 | opt_level: OptLevel::O0, |
| 14108 | kind: OutcomeKind::Pass, |
| 14109 | detail: String::new(), |
| 14110 | bundle: None, |
| 14111 | primary_backend: Some(PrimaryBackendReport { |
| 14112 | kind: "full".into(), |
| 14113 | mode: "linked".into(), |
| 14114 | detail: "linked armfortas::testing capture adapter".into(), |
| 14115 | }), |
| 14116 | consistency_observations: Vec::new(), |
| 14117 | }); |
| 14118 | |
| 14119 | write_requested_reports(&config, &summary).unwrap(); |
| 14120 | |
| 14121 | let json = fs::read_to_string(&json_path).unwrap(); |
| 14122 | let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14123 | assert!(json.contains("\"passed\": 1")); |
| 14124 | assert!(markdown.contains("| passed | 1 |")); |
| 14125 | |
| 14126 | let _ = fs::remove_dir_all(&root); |
| 14127 | } |
| 14128 | |
| 14129 | #[test] |
| 14130 | fn render_doctor_report_includes_tool_status() { |
| 14131 | let root = std::env::temp_dir().join("afs_tests_doctor_paths"); |
| 14132 | let _ = fs::remove_dir_all(&root); |
| 14133 | fs::create_dir_all(&root).unwrap(); |
| 14134 | let armfortas_bin = root.join("armfortas"); |
| 14135 | let gfortran_bin = root.join("gfortran"); |
| 14136 | fs::write(&armfortas_bin, "").unwrap(); |
| 14137 | fs::write(&gfortran_bin, "").unwrap(); |
| 14138 | |
| 14139 | let config = DoctorConfig { |
| 14140 | tools: ToolchainConfig { |
| 14141 | armfortas: ArmfortasCliAdapter::External(armfortas_bin.display().to_string()), |
| 14142 | gfortran: gfortran_bin.display().to_string(), |
| 14143 | flang_new: "/tmp/does-not-exist-flang".into(), |
| 14144 | lfortran: "/tmp/does-not-exist-lfortran".into(), |
| 14145 | ifort: "/tmp/does-not-exist-ifort".into(), |
| 14146 | ifx: "/tmp/does-not-exist-ifx".into(), |
| 14147 | nvfortran: "/tmp/does-not-exist-nvfortran".into(), |
| 14148 | system_as: "/tmp/does-not-exist-as".into(), |
| 14149 | otool: "/tmp/does-not-exist-otool".into(), |
| 14150 | nm: "/tmp/does-not-exist-nm".into(), |
| 14151 | }, |
| 14152 | json_report: None, |
| 14153 | markdown_report: None, |
| 14154 | }; |
| 14155 | |
| 14156 | let rendered = render_doctor_report(&config); |
| 14157 | assert!(rendered.contains("Doctor")); |
| 14158 | assert!(rendered.contains("armfortas_cli_adapter: external armfortas binary adapter")); |
| 14159 | assert!(rendered.contains("armfortas_cli_mode: external")); |
| 14160 | assert!(rendered |
| 14161 | .contains("armfortas_capture_adapter: linked armfortas::testing capture adapter")); |
| 14162 | assert!(rendered.contains("armfortas_capture_mode: linked")); |
| 14163 | assert!(rendered.contains("armfortas_capture_manifest:")); |
| 14164 | assert!( |
| 14165 | rendered.contains("primary_backend_full: linked armfortas::testing capture adapter") |
| 14166 | ); |
| 14167 | assert!(rendered.contains( |
| 14168 | "linked_mode_surface: rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 14169 | )); |
| 14170 | assert!(rendered.contains( |
| 14171 | "primary_backend_observable: cli-observable armfortas driver capture adapter" |
| 14172 | )); |
| 14173 | assert!(rendered.contains( |
| 14174 | "primary_backend_selection: observable backend is selected for asm/obj/run-only cells" |
| 14175 | )); |
| 14176 | assert!(rendered.contains("named_compiler.armfortas: cli=external capture=linked")); |
| 14177 | assert!(rendered.contains( |
| 14178 | "named_compiler.armfortas.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14179 | )); |
| 14180 | assert!(rendered.contains("named_compiler.armfortas.adapter_extras: armfortas(")); |
| 14181 | assert!(rendered.contains("named_compiler.armfortas.unavailable_artifacts: none")); |
| 14182 | assert!(rendered.contains("named_compiler.gfortran:")); |
| 14183 | assert!(rendered.contains( |
| 14184 | "named_compiler.gfortran.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14185 | )); |
| 14186 | assert!(rendered.contains("named_compiler.gfortran.adapter_extras: none")); |
| 14187 | assert!(rendered.contains("named_compiler.lfortran:")); |
| 14188 | assert!(rendered.contains("named_compiler.lfortran.accepted_names: lfortran")); |
| 14189 | assert!(rendered.contains("named_compiler.lfortran.candidate_binaries: lfortran")); |
| 14190 | assert!(rendered.contains("named_compiler.ifx.accepted_names: ifx")); |
| 14191 | assert!(rendered.contains("named_compiler.nvfortran.accepted_names: nvfortran, pgfortran")); |
| 14192 | assert!(rendered.contains( |
| 14193 | "explicit_compiler_path: any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 14194 | )); |
| 14195 | assert!(rendered.contains( |
| 14196 | "explicit_compiler_path.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14197 | )); |
| 14198 | assert!(rendered.contains(&format!( |
| 14199 | "configured={} resolved={}", |
| 14200 | armfortas_bin.display(), |
| 14201 | armfortas_bin.display() |
| 14202 | ))); |
| 14203 | assert!(rendered.contains(&format!( |
| 14204 | "configured={} resolved={}", |
| 14205 | gfortran_bin.display(), |
| 14206 | gfortran_bin.display() |
| 14207 | ))); |
| 14208 | assert!(rendered.contains("configured=/tmp/does-not-exist-flang resolved=missing")); |
| 14209 | let rendered_json = render_doctor_json(&config); |
| 14210 | let rendered_markdown = render_doctor_markdown(&config); |
| 14211 | assert!(rendered_json.contains("\"command\": \"doctor\"")); |
| 14212 | assert!(rendered_json.contains("\"workspace\": {")); |
| 14213 | assert!(rendered_json.contains("\"named_compilers\": {")); |
| 14214 | assert!(rendered_json.contains("\"tools\": {")); |
| 14215 | assert!(rendered_json.contains("\"lfortran\": {")); |
| 14216 | assert!(rendered_json.contains("\"named_compiler.armfortas.adapter_extras\"")); |
| 14217 | assert!(rendered_markdown.contains("# bencch doctor report")); |
| 14218 | assert!(rendered_markdown.contains("| `named_compiler.armfortas` |")); |
| 14219 | |
| 14220 | let _ = fs::remove_dir_all(&root); |
| 14221 | } |
| 14222 | |
| 14223 | #[test] |
| 14224 | fn case_discovery_lines_report_capability_block_for_generic_introspect() { |
| 14225 | let case = CaseSpec { |
| 14226 | name: "unsupported_extra".into(), |
| 14227 | source: PathBuf::from("demo.f90"), |
| 14228 | graph_files: Vec::new(), |
| 14229 | requested: BTreeSet::new(), |
| 14230 | generic_introspect: Some(GenericIntrospectCase { |
| 14231 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14232 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 14233 | }), |
| 14234 | generic_compare: None, |
| 14235 | opt_levels: vec![OptLevel::O0], |
| 14236 | repeat_count: 2, |
| 14237 | reference_compilers: Vec::new(), |
| 14238 | consistency_checks: Vec::new(), |
| 14239 | expectations: Vec::new(), |
| 14240 | status_rules: Vec::new(), |
| 14241 | capability_policy: None, |
| 14242 | }; |
| 14243 | |
| 14244 | let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14245 | assert!(lines.contains(&"capability_status: blocked".to_string())); |
| 14246 | assert!(lines.iter().any(|line| line.contains( |
| 14247 | "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 14248 | ))); |
| 14249 | } |
| 14250 | |
| 14251 | #[test] |
| 14252 | fn case_discovery_lines_report_capability_policy_as_deferred() { |
| 14253 | let case = CaseSpec { |
| 14254 | name: "unsupported_extra".into(), |
| 14255 | source: PathBuf::from("demo.f90"), |
| 14256 | graph_files: Vec::new(), |
| 14257 | requested: BTreeSet::new(), |
| 14258 | generic_introspect: Some(GenericIntrospectCase { |
| 14259 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14260 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 14261 | }), |
| 14262 | generic_compare: None, |
| 14263 | opt_levels: vec![OptLevel::O0], |
| 14264 | repeat_count: 2, |
| 14265 | reference_compilers: Vec::new(), |
| 14266 | consistency_checks: Vec::new(), |
| 14267 | expectations: Vec::new(), |
| 14268 | status_rules: Vec::new(), |
| 14269 | capability_policy: Some(CapabilityPolicy { |
| 14270 | kind: StatusKind::Future, |
| 14271 | reason: "generic gfortran surface has no armfortas extras".into(), |
| 14272 | }), |
| 14273 | }; |
| 14274 | |
| 14275 | let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14276 | assert!(lines.contains(&"capability_status: deferred".to_string())); |
| 14277 | assert!(lines.contains( |
| 14278 | &"capability_policy: future when blocked (generic gfortran surface has no armfortas extras)" |
| 14279 | .to_string() |
| 14280 | )); |
| 14281 | } |
| 14282 | |
| 14283 | #[test] |
| 14284 | fn case_discovery_lines_distinguish_legacy_surfaces() { |
| 14285 | let observable_case = CaseSpec { |
| 14286 | name: "observable".into(), |
| 14287 | source: PathBuf::from("demo.f90"), |
| 14288 | graph_files: Vec::new(), |
| 14289 | requested: BTreeSet::from([Stage::Run]), |
| 14290 | generic_introspect: None, |
| 14291 | generic_compare: None, |
| 14292 | opt_levels: vec![OptLevel::O0], |
| 14293 | repeat_count: 2, |
| 14294 | reference_compilers: Vec::new(), |
| 14295 | consistency_checks: Vec::new(), |
| 14296 | expectations: Vec::new(), |
| 14297 | status_rules: Vec::new(), |
| 14298 | capability_policy: None, |
| 14299 | }; |
| 14300 | let linked_case = CaseSpec { |
| 14301 | name: "linked".into(), |
| 14302 | source: PathBuf::from("demo.f90"), |
| 14303 | graph_files: Vec::new(), |
| 14304 | requested: BTreeSet::from([Stage::Tokens]), |
| 14305 | generic_introspect: None, |
| 14306 | generic_compare: None, |
| 14307 | opt_levels: vec![OptLevel::O0, OptLevel::O1], |
| 14308 | repeat_count: 2, |
| 14309 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 14310 | consistency_checks: vec![ConsistencyCheck::CaptureAsmReproducible], |
| 14311 | expectations: Vec::new(), |
| 14312 | status_rules: Vec::new(), |
| 14313 | capability_policy: None, |
| 14314 | }; |
| 14315 | |
| 14316 | let tools = ToolchainConfig { |
| 14317 | armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 14318 | ..ToolchainConfig::from_env() |
| 14319 | }; |
| 14320 | let observable_lines = case_discovery_lines(&observable_case, &tools); |
| 14321 | let linked_lines = case_discovery_lines(&linked_case, &tools); |
| 14322 | |
| 14323 | assert!(observable_lines.contains(&"surface: observable-only legacy path".to_string())); |
| 14324 | assert!(observable_lines.contains(&"capability_status: ready".to_string())); |
| 14325 | assert!(linked_lines.contains(&"surface: linked armfortas capture".to_string())); |
| 14326 | assert!(linked_lines |
| 14327 | .iter() |
| 14328 | .any(|line| line.contains("differential: gfortran"))); |
| 14329 | assert!(linked_lines |
| 14330 | .iter() |
| 14331 | .any(|line| line.contains("consistency: capture_asm_reproducible"))); |
| 14332 | } |
| 14333 | |
| 14334 | #[test] |
| 14335 | fn write_doctor_reports_emits_files() { |
| 14336 | let root = std::env::temp_dir().join("afs_tests_doctor_report_output"); |
| 14337 | let _ = fs::remove_dir_all(&root); |
| 14338 | fs::create_dir_all(&root).unwrap(); |
| 14339 | let json_path = root.join("doctor.json"); |
| 14340 | let markdown_path = root.join("doctor.md"); |
| 14341 | let config = DoctorConfig { |
| 14342 | tools: ToolchainConfig::from_env(), |
| 14343 | json_report: Some(json_path.clone()), |
| 14344 | markdown_report: Some(markdown_path.clone()), |
| 14345 | }; |
| 14346 | |
| 14347 | write_doctor_reports(&config).unwrap(); |
| 14348 | |
| 14349 | let json = fs::read_to_string(&json_path).unwrap(); |
| 14350 | let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14351 | assert!(json.contains("\"command\": \"doctor\"")); |
| 14352 | assert!(json.contains("\"workspace_root\"")); |
| 14353 | assert!(markdown.contains("# bencch doctor report")); |
| 14354 | assert!(markdown.contains("| `workspace_root` |")); |
| 14355 | |
| 14356 | let _ = fs::remove_dir_all(&root); |
| 14357 | } |
| 14358 | |
| 14359 | #[test] |
| 14360 | fn differential_reports_armfortas_only_divergence() { |
| 14361 | let armfortas = differential_armfortas_observation("0\n", "", 0); |
| 14362 | let refs = vec![ |
| 14363 | differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14364 | differential_reference_observation(ReferenceCompiler::FlangNew, "42\n", "", 0), |
| 14365 | ]; |
| 14366 | |
| 14367 | let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 14368 | assert!(err.contains("classification: armfortas-only divergence")); |
| 14369 | assert!(err.contains("basis: compile-status, diagnostics, runtime")); |
| 14370 | } |
| 14371 | |
| 14372 | #[test] |
| 14373 | fn differential_reports_reference_disagreement() { |
| 14374 | let armfortas = differential_armfortas_observation("42\n", "", 0); |
| 14375 | let refs = vec![ |
| 14376 | differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14377 | differential_reference_observation(ReferenceCompiler::FlangNew, "99\n", "", 0), |
| 14378 | ]; |
| 14379 | |
| 14380 | let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 14381 | assert!(err.contains("classification: reference disagreement")); |
| 14382 | } |
| 14383 | |
| 14384 | #[test] |
| 14385 | fn differential_tolerates_numeric_formatting_differences() { |
| 14386 | let armfortas = differential_armfortas_observation(" 5.5000000E0\n", "", 0); |
| 14387 | let refs = vec![ |
| 14388 | differential_reference_observation( |
| 14389 | ReferenceCompiler::Gfortran, |
| 14390 | " 5.50000000\n", |
| 14391 | "", |
| 14392 | 0, |
| 14393 | ), |
| 14394 | differential_reference_observation(ReferenceCompiler::FlangNew, " 5.5\n", "", 0), |
| 14395 | ]; |
| 14396 | |
| 14397 | assert!(compare_differential(&armfortas, &refs).is_ok()); |
| 14398 | } |
| 14399 | |
| 14400 | #[test] |
| 14401 | fn consistency_diff_reports_first_mismatch() { |
| 14402 | let detail = describe_text_difference("alpha\nbeta\n", "alpha\ngamma\n", "left", "right"); |
| 14403 | assert!(detail.contains("first differing line: 2")); |
| 14404 | assert!(detail.contains("left: beta")); |
| 14405 | assert!(detail.contains("right: gamma")); |
| 14406 | } |
| 14407 | |
| 14408 | #[test] |
| 14409 | fn consistency_diff_reports_first_extra_line() { |
| 14410 | let detail = describe_text_difference("alpha\nbeta\n", "", "left", "right"); |
| 14411 | assert!(detail.contains("snapshot length differs")); |
| 14412 | assert!(detail.contains("first extra line: 1")); |
| 14413 | assert!(detail.contains("left: alpha")); |
| 14414 | } |
| 14415 | |
| 14416 | #[test] |
| 14417 | fn object_diff_reports_changed_components() { |
| 14418 | let expected = ObjectSnapshot { |
| 14419 | text: "alpha\nbeta\n".into(), |
| 14420 | load_commands: "same".into(), |
| 14421 | relocations: "same".into(), |
| 14422 | symbols: "same".into(), |
| 14423 | }; |
| 14424 | let actual = ObjectSnapshot { |
| 14425 | text: "alpha\ngamma\n".into(), |
| 14426 | load_commands: "same".into(), |
| 14427 | relocations: "same".into(), |
| 14428 | symbols: "same".into(), |
| 14429 | }; |
| 14430 | |
| 14431 | let detail = describe_object_difference(&expected, &actual, "first -c", "second -c"); |
| 14432 | assert!(detail.contains("differing object components: text")); |
| 14433 | assert!(detail.contains("first differing component: text")); |
| 14434 | assert!(detail.contains("first -c: beta")); |
| 14435 | assert!(detail.contains("second -c: gamma")); |
| 14436 | } |
| 14437 | |
| 14438 | #[test] |
| 14439 | fn object_component_variation_classifies_text_only_instability() { |
| 14440 | let first = ObjectSnapshot { |
| 14441 | text: "alpha".into(), |
| 14442 | load_commands: "load".into(), |
| 14443 | relocations: "reloc".into(), |
| 14444 | symbols: "symbols".into(), |
| 14445 | }; |
| 14446 | let second = ObjectSnapshot { |
| 14447 | text: "beta".into(), |
| 14448 | load_commands: "load".into(), |
| 14449 | relocations: "reloc".into(), |
| 14450 | symbols: "symbols".into(), |
| 14451 | }; |
| 14452 | let snapshots = vec![&first, &second]; |
| 14453 | |
| 14454 | assert_eq!(varying_object_components(&snapshots), vec!["text"]); |
| 14455 | assert_eq!( |
| 14456 | stable_object_components(&snapshots), |
| 14457 | vec!["load_commands", "relocations", "symbols"] |
| 14458 | ); |
| 14459 | } |
| 14460 | |
| 14461 | #[test] |
| 14462 | fn run_diff_reports_changed_components() { |
| 14463 | let left = RunCapture { |
| 14464 | exit_code: 0, |
| 14465 | stdout: "alpha\nbeta\n".into(), |
| 14466 | stderr: String::new(), |
| 14467 | }; |
| 14468 | let right = RunCapture { |
| 14469 | exit_code: 0, |
| 14470 | stdout: "alpha\ngamma\n".into(), |
| 14471 | stderr: String::new(), |
| 14472 | }; |
| 14473 | |
| 14474 | let detail = describe_run_difference(&left, &right, "capture run", "cli run 2"); |
| 14475 | assert!(detail.contains("differing runtime components: stdout")); |
| 14476 | assert!(detail.contains("first differing component: stdout")); |
| 14477 | assert!(detail.contains("capture run: beta")); |
| 14478 | assert!(detail.contains("cli run 2: gamma")); |
| 14479 | } |
| 14480 | |
| 14481 | #[test] |
| 14482 | fn run_component_variation_classifies_stdout_only_instability() { |
| 14483 | let first = RunSignature { |
| 14484 | exit_code: 0, |
| 14485 | stdout: "alpha".into(), |
| 14486 | stderr: String::new(), |
| 14487 | }; |
| 14488 | let second = RunSignature { |
| 14489 | exit_code: 0, |
| 14490 | stdout: "beta".into(), |
| 14491 | stderr: String::new(), |
| 14492 | }; |
| 14493 | let signatures = vec![&first, &second]; |
| 14494 | |
| 14495 | assert_eq!(varying_run_components(&signatures), vec!["stdout"]); |
| 14496 | assert_eq!( |
| 14497 | stable_run_components(&signatures), |
| 14498 | vec!["exit_code", "stderr"] |
| 14499 | ); |
| 14500 | } |
| 14501 | |
| 14502 | #[test] |
| 14503 | fn parse_object_snapshot_text_round_trips_rendered_snapshot() { |
| 14504 | let snapshot = ObjectSnapshot { |
| 14505 | text: "text bytes".into(), |
| 14506 | load_commands: "load commands".into(), |
| 14507 | relocations: "relocations".into(), |
| 14508 | symbols: "symbols".into(), |
| 14509 | }; |
| 14510 | |
| 14511 | let rendered = render_object_snapshot(&snapshot); |
| 14512 | let parsed = parse_object_snapshot_text(&rendered).unwrap(); |
| 14513 | assert_eq!(parsed, snapshot); |
| 14514 | } |
| 14515 | |
| 14516 | #[test] |
| 14517 | fn verifier_regression_detects_integer_op_on_float_values() { |
| 14518 | let mut module = Module::new("verify".into()); |
| 14519 | let mut func = Function::new("broken".into(), vec![], IrType::Void); |
| 14520 | func.blocks[0].insts.push(Inst { |
| 14521 | id: ValueId(0), |
| 14522 | kind: InstKind::ConstFloat(1.0, FloatWidth::F32), |
| 14523 | ty: IrType::Float(FloatWidth::F32), |
| 14524 | span: dummy_span(), |
| 14525 | }); |
| 14526 | func.blocks[0].insts.push(Inst { |
| 14527 | id: ValueId(1), |
| 14528 | kind: InstKind::ConstFloat(2.0, FloatWidth::F32), |
| 14529 | ty: IrType::Float(FloatWidth::F32), |
| 14530 | span: dummy_span(), |
| 14531 | }); |
| 14532 | func.blocks[0].insts.push(Inst { |
| 14533 | id: ValueId(2), |
| 14534 | kind: InstKind::IAdd(ValueId(0), ValueId(1)), |
| 14535 | ty: IrType::Int(IntWidth::I32), |
| 14536 | span: dummy_span(), |
| 14537 | }); |
| 14538 | func.blocks[0].terminator = Some(Terminator::Return(None)); |
| 14539 | module.add_function(func); |
| 14540 | |
| 14541 | let errors = verify_module(&module); |
| 14542 | assert!( |
| 14543 | errors |
| 14544 | .iter() |
| 14545 | .any(|error| error.msg.contains("non-integer operand")), |
| 14546 | "expected verifier error, got: {:?}", |
| 14547 | errors |
| 14548 | ); |
| 14549 | } |
| 14550 | |
| 14551 | #[test] |
| 14552 | fn verifier_regression_detects_branch_argument_mismatch() { |
| 14553 | let mut module = Module::new("verify".into()); |
| 14554 | let mut func = Function::new("broken_branch".into(), vec![], IrType::Void); |
| 14555 | let target = func.create_block("target"); |
| 14556 | func.block_mut(target).params.push(BlockParam { |
| 14557 | id: ValueId(0), |
| 14558 | ty: IrType::Int(IntWidth::I32), |
| 14559 | }); |
| 14560 | func.blocks[0].terminator = Some(Terminator::Branch(target, vec![])); |
| 14561 | func.block_mut(target).terminator = Some(Terminator::Return(None)); |
| 14562 | module.add_function(func); |
| 14563 | |
| 14564 | let errors = verify_module(&module); |
| 14565 | assert!( |
| 14566 | errors |
| 14567 | .iter() |
| 14568 | .any(|error| error.msg.contains("expected 1 args, got 0")), |
| 14569 | "expected verifier error, got: {:?}", |
| 14570 | errors |
| 14571 | ); |
| 14572 | } |
| 14573 | |
| 14574 | #[test] |
| 14575 | fn verifier_regression_detects_store_to_non_pointer() { |
| 14576 | let mut module = Module::new("verify".into()); |
| 14577 | let mut func = Function::new("broken_store".into(), vec![], IrType::Void); |
| 14578 | func.blocks[0].insts.push(Inst { |
| 14579 | id: ValueId(0), |
| 14580 | kind: InstKind::ConstInt(42, IntWidth::I32), |
| 14581 | ty: IrType::Int(IntWidth::I32), |
| 14582 | span: dummy_span(), |
| 14583 | }); |
| 14584 | func.blocks[0].insts.push(Inst { |
| 14585 | id: ValueId(1), |
| 14586 | kind: InstKind::ConstInt(0, IntWidth::I32), |
| 14587 | ty: IrType::Int(IntWidth::I32), |
| 14588 | span: dummy_span(), |
| 14589 | }); |
| 14590 | func.blocks[0].insts.push(Inst { |
| 14591 | id: ValueId(2), |
| 14592 | kind: InstKind::Store(ValueId(0), ValueId(1)), |
| 14593 | ty: IrType::Void, |
| 14594 | span: dummy_span(), |
| 14595 | }); |
| 14596 | func.blocks[0].terminator = Some(Terminator::Return(None)); |
| 14597 | module.add_function(func); |
| 14598 | |
| 14599 | let errors = verify_module(&module); |
| 14600 | assert!( |
| 14601 | errors.iter().any(|error| error.msg.contains("non-pointer")), |
| 14602 | "expected verifier error, got: {:?}", |
| 14603 | errors |
| 14604 | ); |
| 14605 | } |
| 14606 | |
| 14607 | #[test] |
| 14608 | fn failure_expectation_precedes_partial_stage_checks() { |
| 14609 | let case = CaseSpec { |
| 14610 | name: "missing_then".into(), |
| 14611 | source: PathBuf::from("demo.f90"), |
| 14612 | graph_files: Vec::new(), |
| 14613 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14614 | generic_introspect: None, |
| 14615 | generic_compare: None, |
| 14616 | opt_levels: vec![OptLevel::O0], |
| 14617 | repeat_count: 3, |
| 14618 | reference_compilers: Vec::new(), |
| 14619 | consistency_checks: Vec::new(), |
| 14620 | expectations: vec![ |
| 14621 | Expectation::Contains { |
| 14622 | target: Target::RunStdout, |
| 14623 | needle: "42".into(), |
| 14624 | }, |
| 14625 | Expectation::FailContains { |
| 14626 | stage: FailureStage::Parser, |
| 14627 | needle: "expected 'then'".into(), |
| 14628 | }, |
| 14629 | ], |
| 14630 | status_rules: Vec::new(), |
| 14631 | capability_policy: None, |
| 14632 | }; |
| 14633 | let artifacts = ExecutionArtifacts { |
| 14634 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14635 | armfortas: None, |
| 14636 | armfortas_failure: None, |
| 14637 | armfortas_observation: None, |
| 14638 | references: Vec::new(), |
| 14639 | reference_observations: Vec::new(), |
| 14640 | consistency_issues: Vec::new(), |
| 14641 | }; |
| 14642 | let failure = CaptureFailure { |
| 14643 | input: PathBuf::from("demo.f90"), |
| 14644 | opt_level: OptLevel::O0, |
| 14645 | stage: FailureStage::Parser, |
| 14646 | detail: "expected 'then'".into(), |
| 14647 | stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14648 | }; |
| 14649 | |
| 14650 | assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14651 | } |
| 14652 | |
| 14653 | #[test] |
| 14654 | fn legacy_capture_failures_use_observation_failure_semantics() { |
| 14655 | let case = CaseSpec { |
| 14656 | name: "hidden_use_only".into(), |
| 14657 | source: PathBuf::from("demo.f90"), |
| 14658 | graph_files: Vec::new(), |
| 14659 | requested: BTreeSet::from([Stage::Run]), |
| 14660 | generic_introspect: None, |
| 14661 | generic_compare: None, |
| 14662 | opt_levels: vec![OptLevel::O0], |
| 14663 | repeat_count: 3, |
| 14664 | reference_compilers: Vec::new(), |
| 14665 | consistency_checks: Vec::new(), |
| 14666 | expectations: vec![Expectation::FailCommentPatterns(vec!["hidden".into()])], |
| 14667 | status_rules: Vec::new(), |
| 14668 | capability_policy: None, |
| 14669 | }; |
| 14670 | let artifacts = ExecutionArtifacts { |
| 14671 | requested: BTreeSet::from([Stage::Run]), |
| 14672 | armfortas: None, |
| 14673 | armfortas_failure: None, |
| 14674 | armfortas_observation: None, |
| 14675 | references: Vec::new(), |
| 14676 | reference_observations: Vec::new(), |
| 14677 | consistency_issues: Vec::new(), |
| 14678 | }; |
| 14679 | let failure = CaptureFailure { |
| 14680 | input: PathBuf::from("demo.f90"), |
| 14681 | opt_level: OptLevel::O0, |
| 14682 | stage: FailureStage::Sema, |
| 14683 | detail: "demo.f90:4:3: semantic error: hidden".into(), |
| 14684 | stages: BTreeMap::new(), |
| 14685 | }; |
| 14686 | |
| 14687 | assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14688 | } |
| 14689 | |
| 14690 | #[test] |
| 14691 | fn legacy_failure_observed_program_uses_prepared_source_and_partial_stages() { |
| 14692 | let case = CaseSpec { |
| 14693 | name: "missing_then".into(), |
| 14694 | source: PathBuf::from("authored.f90"), |
| 14695 | graph_files: Vec::new(), |
| 14696 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14697 | generic_introspect: None, |
| 14698 | generic_compare: None, |
| 14699 | opt_levels: vec![OptLevel::O0], |
| 14700 | repeat_count: 3, |
| 14701 | reference_compilers: Vec::new(), |
| 14702 | consistency_checks: Vec::new(), |
| 14703 | expectations: vec![Expectation::FailContains { |
| 14704 | stage: FailureStage::Parser, |
| 14705 | needle: "expected 'then'".into(), |
| 14706 | }], |
| 14707 | status_rules: Vec::new(), |
| 14708 | capability_policy: None, |
| 14709 | }; |
| 14710 | let failure = CaptureFailure { |
| 14711 | input: PathBuf::from("generated.f90"), |
| 14712 | opt_level: OptLevel::O0, |
| 14713 | stage: FailureStage::Parser, |
| 14714 | detail: "expected 'then'".into(), |
| 14715 | stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14716 | }; |
| 14717 | |
| 14718 | let observed = legacy_failure_observed_program(Path::new("prepared.f90"), &case, &failure); |
| 14719 | assert_eq!(observed.observation.program, PathBuf::from("prepared.f90")); |
| 14720 | assert_eq!(observed.observation.compile_exit_code, 1); |
| 14721 | assert_eq!( |
| 14722 | observed.observation.provenance.failure_stage.as_deref(), |
| 14723 | Some("parser") |
| 14724 | ); |
| 14725 | assert!(observed |
| 14726 | .observation |
| 14727 | .artifacts |
| 14728 | .contains_key(&ArtifactKey::Diagnostics)); |
| 14729 | assert!(observed |
| 14730 | .observation |
| 14731 | .artifacts |
| 14732 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 14733 | } |
| 14734 | |
| 14735 | #[test] |
| 14736 | fn unexpected_capture_failure_reports_compiler_failure_detail() { |
| 14737 | let case = CaseSpec { |
| 14738 | name: "module_procedure_runtime".into(), |
| 14739 | source: PathBuf::from("graph.f90"), |
| 14740 | graph_files: Vec::new(), |
| 14741 | requested: BTreeSet::from([Stage::Run]), |
| 14742 | generic_introspect: None, |
| 14743 | generic_compare: None, |
| 14744 | opt_levels: vec![OptLevel::O0], |
| 14745 | repeat_count: 3, |
| 14746 | reference_compilers: Vec::new(), |
| 14747 | consistency_checks: Vec::new(), |
| 14748 | expectations: vec![Expectation::Contains { |
| 14749 | target: Target::RunStdout, |
| 14750 | needle: "42".into(), |
| 14751 | }], |
| 14752 | status_rules: Vec::new(), |
| 14753 | capability_policy: None, |
| 14754 | }; |
| 14755 | let failure = CaptureFailure { |
| 14756 | input: PathBuf::from("graph.f90"), |
| 14757 | opt_level: OptLevel::O0, |
| 14758 | stage: FailureStage::Run, |
| 14759 | detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 14760 | stages: BTreeMap::new(), |
| 14761 | }; |
| 14762 | let artifacts = ExecutionArtifacts { |
| 14763 | requested: BTreeSet::from([Stage::Run]), |
| 14764 | armfortas: None, |
| 14765 | armfortas_failure: Some(failure.clone()), |
| 14766 | armfortas_observation: None, |
| 14767 | references: Vec::new(), |
| 14768 | reference_observations: Vec::new(), |
| 14769 | consistency_issues: Vec::new(), |
| 14770 | }; |
| 14771 | |
| 14772 | let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 14773 | assert!(err.contains("armfortas failed in run")); |
| 14774 | assert!(err.contains("_add_one")); |
| 14775 | assert!(!err.contains("missing captured run stage")); |
| 14776 | } |
| 14777 | |
| 14778 | #[test] |
| 14779 | fn partial_stage_expectation_failure_is_preserved_on_capture_failure() { |
| 14780 | let case = CaseSpec { |
| 14781 | name: "module_procedure_backend".into(), |
| 14782 | source: PathBuf::from("graph.f90"), |
| 14783 | graph_files: Vec::new(), |
| 14784 | requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 14785 | generic_introspect: None, |
| 14786 | generic_compare: None, |
| 14787 | opt_levels: vec![OptLevel::O0], |
| 14788 | repeat_count: 3, |
| 14789 | reference_compilers: Vec::new(), |
| 14790 | consistency_checks: Vec::new(), |
| 14791 | expectations: vec![Expectation::Contains { |
| 14792 | target: Target::Stage(Stage::Asm), |
| 14793 | needle: ".globl _add_one".into(), |
| 14794 | }], |
| 14795 | status_rules: Vec::new(), |
| 14796 | capability_policy: None, |
| 14797 | }; |
| 14798 | let failure = CaptureFailure { |
| 14799 | input: PathBuf::from("graph.f90"), |
| 14800 | opt_level: OptLevel::O0, |
| 14801 | stage: FailureStage::Run, |
| 14802 | detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 14803 | stages: BTreeMap::from([(Stage::Asm, CapturedStage::Text(".globl _main\n".into()))]), |
| 14804 | }; |
| 14805 | let artifacts = ExecutionArtifacts { |
| 14806 | requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 14807 | armfortas: None, |
| 14808 | armfortas_failure: Some(failure.clone()), |
| 14809 | armfortas_observation: None, |
| 14810 | references: Vec::new(), |
| 14811 | reference_observations: Vec::new(), |
| 14812 | consistency_issues: Vec::new(), |
| 14813 | }; |
| 14814 | |
| 14815 | let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 14816 | assert!(err.contains("expected asm to contain")); |
| 14817 | assert!(!err.contains("armfortas failed in run")); |
| 14818 | } |
| 14819 | } |
| 14820 |