| 1 | mod compiler; |
| 2 | |
| 3 | use std::collections::{BTreeMap, BTreeSet, VecDeque}; |
| 4 | use std::fs; |
| 5 | use std::path::{Path, PathBuf}; |
| 6 | use std::process::Command; |
| 7 | use std::sync::atomic::{AtomicU64, Ordering}; |
| 8 | |
| 9 | use crate::compiler::{ |
| 10 | linked_capture_available, object_snapshot_text, ArmfortasAdapters, ArmfortasCliAdapter, |
| 11 | CaptureBackend, CaptureFailure, CaptureRequest, CaptureResult, CapturedStage, |
| 12 | CliObservableCaptureBackend, EmitMode, FailureStage, OptLevel, RunCapture, Stage, |
| 13 | }; |
| 14 | use bencch_core::{ |
| 15 | ArtifactDifference, ArtifactKey, ArtifactValue, ComparisonResult, CompilerCapabilities, |
| 16 | CompilerObservation, CompilerSpec, NamedCompiler, ObservationProvenance, |
| 17 | }; |
| 18 | |
| 19 | const SUITE_EXTENSION: &str = "afs"; |
| 20 | |
| 21 | static REPORT_COUNTER: AtomicU64 = AtomicU64::new(0); |
| 22 | |
| 23 | #[derive(Debug, Clone)] |
| 24 | struct SuiteSpec { |
| 25 | name: String, |
| 26 | path: PathBuf, |
| 27 | cases: Vec<CaseSpec>, |
| 28 | } |
| 29 | |
| 30 | #[derive(Debug, Clone)] |
| 31 | struct CaseSpec { |
| 32 | name: String, |
| 33 | source: PathBuf, |
| 34 | graph_files: Vec<PathBuf>, |
| 35 | requested: BTreeSet<Stage>, |
| 36 | generic_introspect: Option<GenericIntrospectCase>, |
| 37 | generic_compare: Option<GenericCompareCase>, |
| 38 | opt_levels: Vec<OptLevel>, |
| 39 | repeat_count: usize, |
| 40 | reference_compilers: Vec<ReferenceCompiler>, |
| 41 | consistency_checks: Vec<ConsistencyCheck>, |
| 42 | expectations: Vec<Expectation>, |
| 43 | status_rules: Vec<StatusRule>, |
| 44 | capability_policy: Option<CapabilityPolicy>, |
| 45 | } |
| 46 | |
| 47 | impl CaseSpec { |
| 48 | fn is_graph(&self) -> bool { |
| 49 | !self.graph_files.is_empty() |
| 50 | } |
| 51 | |
| 52 | fn is_generic_introspect(&self) -> bool { |
| 53 | self.generic_introspect.is_some() |
| 54 | } |
| 55 | |
| 56 | fn is_generic_compare(&self) -> bool { |
| 57 | self.generic_compare.is_some() |
| 58 | } |
| 59 | |
| 60 | fn source_label(&self) -> String { |
| 61 | if self.is_graph() { |
| 62 | format!( |
| 63 | "graph entry {} ({} files)", |
| 64 | self.source.display(), |
| 65 | self.graph_files.len() |
| 66 | ) |
| 67 | } else { |
| 68 | self.source.display().to_string() |
| 69 | } |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | #[derive(Debug, Clone)] |
| 74 | struct GenericIntrospectCase { |
| 75 | compiler: CompilerSpec, |
| 76 | artifacts: BTreeSet<ArtifactKey>, |
| 77 | } |
| 78 | |
| 79 | #[derive(Debug, Clone)] |
| 80 | struct GenericCompareCase { |
| 81 | left: CompilerSpec, |
| 82 | right: CompilerSpec, |
| 83 | artifacts: BTreeSet<ArtifactKey>, |
| 84 | } |
| 85 | |
| 86 | #[derive(Debug, Clone)] |
| 87 | struct PreparedInput { |
| 88 | compiler_source: PathBuf, |
| 89 | generated_source: Option<PathBuf>, |
| 90 | temp_root: Option<PathBuf>, |
| 91 | } |
| 92 | |
| 93 | #[derive(Debug, Clone)] |
| 94 | struct StatusRule { |
| 95 | kind: StatusKind, |
| 96 | selector: OptSelector, |
| 97 | reason: String, |
| 98 | } |
| 99 | |
| 100 | #[derive(Debug, Clone)] |
| 101 | struct CapabilityPolicy { |
| 102 | kind: StatusKind, |
| 103 | reason: String, |
| 104 | } |
| 105 | |
| 106 | #[derive(Debug, Clone)] |
| 107 | enum PendingStatusRule { |
| 108 | Explicit(StatusRule), |
| 109 | XfailSourceComments, |
| 110 | } |
| 111 | |
| 112 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 113 | enum StatusKind { |
| 114 | Xfail, |
| 115 | Future, |
| 116 | } |
| 117 | |
| 118 | #[derive(Debug, Clone)] |
| 119 | enum OptSelector { |
| 120 | All, |
| 121 | Only(Vec<OptLevel>), |
| 122 | } |
| 123 | |
| 124 | impl OptSelector { |
| 125 | fn matches(&self, opt_level: OptLevel) -> bool { |
| 126 | match self { |
| 127 | Self::All => true, |
| 128 | Self::Only(levels) => levels.contains(&opt_level), |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | #[derive(Debug, Clone)] |
| 134 | enum EffectiveStatus { |
| 135 | Normal, |
| 136 | Xfail(String), |
| 137 | Future(String), |
| 138 | } |
| 139 | |
| 140 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] |
| 141 | enum ConsistencyCheck { |
| 142 | CliObjVsSystemAs, |
| 143 | CliAsmReproducible, |
| 144 | CliObjReproducible, |
| 145 | CliRunReproducible, |
| 146 | CaptureAsmVsCliAsm, |
| 147 | CaptureObjVsCliObj, |
| 148 | CaptureRunVsCliRun, |
| 149 | CaptureAsmReproducible, |
| 150 | CaptureObjReproducible, |
| 151 | CaptureRunReproducible, |
| 152 | } |
| 153 | |
| 154 | impl ConsistencyCheck { |
| 155 | fn parse(name: &str) -> Option<Self> { |
| 156 | match name.trim().to_ascii_lowercase().as_str() { |
| 157 | "cli_obj_vs_system_as" | "cli-obj-vs-system-as" => Some(Self::CliObjVsSystemAs), |
| 158 | "cli_asm_reproducible" | "cli-asm-reproducible" => Some(Self::CliAsmReproducible), |
| 159 | "cli_obj_reproducible" | "cli-obj-reproducible" => Some(Self::CliObjReproducible), |
| 160 | "cli_run_reproducible" | "cli-run-reproducible" => Some(Self::CliRunReproducible), |
| 161 | "capture_asm_vs_cli_asm" | "capture-asm-vs-cli-asm" => Some(Self::CaptureAsmVsCliAsm), |
| 162 | "capture_obj_vs_cli_obj" | "capture-obj-vs-cli-obj" => Some(Self::CaptureObjVsCliObj), |
| 163 | "capture_run_vs_cli_run" | "capture-run-vs-cli-run" => Some(Self::CaptureRunVsCliRun), |
| 164 | "capture_asm_reproducible" | "capture-asm-reproducible" => { |
| 165 | Some(Self::CaptureAsmReproducible) |
| 166 | } |
| 167 | "capture_obj_reproducible" | "capture-obj-reproducible" => { |
| 168 | Some(Self::CaptureObjReproducible) |
| 169 | } |
| 170 | "capture_run_reproducible" | "capture-run-reproducible" => { |
| 171 | Some(Self::CaptureRunReproducible) |
| 172 | } |
| 173 | _ => None, |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | fn as_str(&self) -> &'static str { |
| 178 | match self { |
| 179 | Self::CliObjVsSystemAs => "cli_obj_vs_system_as", |
| 180 | Self::CliAsmReproducible => "cli_asm_reproducible", |
| 181 | Self::CliObjReproducible => "cli_obj_reproducible", |
| 182 | Self::CliRunReproducible => "cli_run_reproducible", |
| 183 | Self::CaptureAsmVsCliAsm => "capture_asm_vs_cli_asm", |
| 184 | Self::CaptureObjVsCliObj => "capture_obj_vs_cli_obj", |
| 185 | Self::CaptureRunVsCliRun => "capture_run_vs_cli_run", |
| 186 | Self::CaptureAsmReproducible => "capture_asm_reproducible", |
| 187 | Self::CaptureObjReproducible => "capture_obj_reproducible", |
| 188 | Self::CaptureRunReproducible => "capture_run_reproducible", |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | fn required_stage(&self) -> Option<Stage> { |
| 193 | match self { |
| 194 | Self::CliObjVsSystemAs |
| 195 | | Self::CliAsmReproducible |
| 196 | | Self::CliObjReproducible |
| 197 | | Self::CliRunReproducible => None, |
| 198 | Self::CaptureAsmVsCliAsm | Self::CaptureAsmReproducible => Some(Stage::Asm), |
| 199 | Self::CaptureObjVsCliObj | Self::CaptureObjReproducible => Some(Stage::Obj), |
| 200 | Self::CaptureRunVsCliRun | Self::CaptureRunReproducible => Some(Stage::Run), |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | fn requires_capture_result(&self) -> bool { |
| 205 | matches!( |
| 206 | self, |
| 207 | Self::CaptureAsmVsCliAsm |
| 208 | | Self::CaptureObjVsCliObj |
| 209 | | Self::CaptureRunVsCliRun |
| 210 | | Self::CaptureAsmReproducible |
| 211 | | Self::CaptureObjReproducible |
| 212 | | Self::CaptureRunReproducible |
| 213 | ) |
| 214 | } |
| 215 | |
| 216 | fn supports_generic_introspect(&self) -> bool { |
| 217 | matches!( |
| 218 | self, |
| 219 | Self::CliAsmReproducible | Self::CliObjReproducible | Self::CliRunReproducible |
| 220 | ) |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | #[derive(Debug, Clone)] |
| 225 | enum Expectation { |
| 226 | CheckComments(Target), |
| 227 | Contains { target: Target, needle: String }, |
| 228 | NotContains { target: Target, needle: String }, |
| 229 | Equals { target: Target, value: String }, |
| 230 | IntEquals { target: Target, value: i32 }, |
| 231 | FailContains { stage: FailureStage, needle: String }, |
| 232 | FailEquals { stage: FailureStage, value: String }, |
| 233 | FailSourceComments, |
| 234 | FailCommentPatterns(Vec<String>), |
| 235 | } |
| 236 | |
| 237 | #[derive(Debug, Clone)] |
| 238 | enum Target { |
| 239 | Stage(Stage), |
| 240 | Artifact(ArtifactKey), |
| 241 | CompareStatus, |
| 242 | CompareClassification, |
| 243 | CompareChangedArtifacts, |
| 244 | CompareDifferenceCount, |
| 245 | CompareBasis, |
| 246 | RunStdout, |
| 247 | RunStderr, |
| 248 | RunExitCode, |
| 249 | } |
| 250 | |
| 251 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] |
| 252 | enum ReferenceCompiler { |
| 253 | Gfortran, |
| 254 | FlangNew, |
| 255 | } |
| 256 | |
| 257 | impl ReferenceCompiler { |
| 258 | fn parse(name: &str) -> Option<Self> { |
| 259 | match name.trim().to_ascii_lowercase().as_str() { |
| 260 | "gfortran" => Some(Self::Gfortran), |
| 261 | "flang-new" | "flang_new" | "flang" => Some(Self::FlangNew), |
| 262 | _ => None, |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | fn binary_name(&self) -> &'static str { |
| 267 | match self { |
| 268 | Self::Gfortran => "gfortran", |
| 269 | Self::FlangNew => "flang-new", |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | fn as_str(&self) -> &'static str { |
| 274 | self.binary_name() |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 279 | struct ToolchainConfig { |
| 280 | armfortas: ArmfortasCliAdapter, |
| 281 | gfortran: String, |
| 282 | flang_new: String, |
| 283 | lfortran: String, |
| 284 | ifort: String, |
| 285 | ifx: String, |
| 286 | nvfortran: String, |
| 287 | system_as: String, |
| 288 | otool: String, |
| 289 | nm: String, |
| 290 | } |
| 291 | |
| 292 | impl ToolchainConfig { |
| 293 | fn from_env() -> Self { |
| 294 | Self { |
| 295 | armfortas: match std::env::var("BENCCH_ARMFORTAS_BIN") { |
| 296 | Ok(value) if !value.trim().is_empty() => ArmfortasCliAdapter::External(value), |
| 297 | _ if linked_capture_available() => ArmfortasCliAdapter::Linked, |
| 298 | _ => ArmfortasCliAdapter::External("armfortas".into()), |
| 299 | }, |
| 300 | gfortran: tool_override("BENCCH_GFORTRAN_BIN", "gfortran"), |
| 301 | flang_new: tool_override("BENCCH_FLANG_BIN", "flang-new"), |
| 302 | lfortran: tool_override("BENCCH_LFORTRAN_BIN", "lfortran"), |
| 303 | ifort: tool_override("BENCCH_IFORT_BIN", "ifort"), |
| 304 | ifx: tool_override("BENCCH_IFX_BIN", "ifx"), |
| 305 | nvfortran: tool_override("BENCCH_NVFORTRAN_BIN", "nvfortran"), |
| 306 | system_as: tool_override("BENCCH_AS_BIN", "as"), |
| 307 | otool: tool_override("BENCCH_OTOOL_BIN", "otool"), |
| 308 | nm: tool_override("BENCCH_NM_BIN", "nm"), |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | fn armfortas_adapters(&self) -> ArmfortasAdapters { |
| 313 | ArmfortasAdapters::new(self.armfortas.clone()) |
| 314 | } |
| 315 | |
| 316 | fn cli_observable_capture_backend(&self, work_root: PathBuf) -> CliObservableCaptureBackend { |
| 317 | CliObservableCaptureBackend::new( |
| 318 | self.armfortas.clone(), |
| 319 | work_root, |
| 320 | self.otool.clone(), |
| 321 | self.nm.clone(), |
| 322 | ) |
| 323 | } |
| 324 | |
| 325 | fn reference_binary(&self, compiler: ReferenceCompiler) -> &str { |
| 326 | match compiler { |
| 327 | ReferenceCompiler::Gfortran => &self.gfortran, |
| 328 | ReferenceCompiler::FlangNew => &self.flang_new, |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | fn system_as_bin(&self) -> &str { |
| 333 | &self.system_as |
| 334 | } |
| 335 | |
| 336 | fn otool_bin(&self) -> &str { |
| 337 | &self.otool |
| 338 | } |
| 339 | |
| 340 | fn nm_bin(&self) -> &str { |
| 341 | &self.nm |
| 342 | } |
| 343 | |
| 344 | fn named_compiler_binary(&self, compiler: NamedCompiler) -> Option<String> { |
| 345 | match compiler { |
| 346 | NamedCompiler::Armfortas => match &self.armfortas { |
| 347 | ArmfortasCliAdapter::Linked => None, |
| 348 | ArmfortasCliAdapter::External(binary) => Some(binary.clone()), |
| 349 | }, |
| 350 | NamedCompiler::Gfortran => Some(self.gfortran.clone()), |
| 351 | NamedCompiler::FlangNew => Some(self.flang_new.clone()), |
| 352 | NamedCompiler::LFortran => Some(self.lfortran.clone()), |
| 353 | NamedCompiler::Ifort => Some(self.ifort.clone()), |
| 354 | NamedCompiler::Ifx => Some(self.ifx.clone()), |
| 355 | NamedCompiler::Nvfortran => Some(self.nvfortran.clone()), |
| 356 | } |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | fn generic_external_capabilities(spec: CompilerSpec) -> CompilerCapabilities { |
| 361 | CompilerCapabilities::new(spec).support_all([ |
| 362 | ArtifactKey::Diagnostics, |
| 363 | ArtifactKey::ExitCode, |
| 364 | ArtifactKey::Stdout, |
| 365 | ArtifactKey::Stderr, |
| 366 | ArtifactKey::Asm, |
| 367 | ArtifactKey::Obj, |
| 368 | ArtifactKey::Executable, |
| 369 | ArtifactKey::Runtime, |
| 370 | ]) |
| 371 | } |
| 372 | |
| 373 | fn armfortas_capabilities(tools: &ToolchainConfig) -> CompilerCapabilities { |
| 374 | let mut capabilities = |
| 375 | generic_external_capabilities(CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 376 | let linked_reason = "linked armfortas capture is unavailable in this build; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary".to_string(); |
| 377 | let capture_available = tools.armfortas_adapters().capture_mode_name() != "unavailable"; |
| 378 | for stage in Stage::ALL { |
| 379 | if matches!(stage, Stage::Asm | Stage::Obj | Stage::Run) { |
| 380 | continue; |
| 381 | } |
| 382 | let artifact = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 383 | capabilities = if capture_available { |
| 384 | capabilities.support(artifact) |
| 385 | } else { |
| 386 | capabilities.mark_unavailable(artifact, linked_reason.clone()) |
| 387 | }; |
| 388 | } |
| 389 | capabilities |
| 390 | } |
| 391 | |
| 392 | fn compiler_capabilities(spec: &CompilerSpec, tools: &ToolchainConfig) -> CompilerCapabilities { |
| 393 | match spec { |
| 394 | CompilerSpec::Named(NamedCompiler::Armfortas) => armfortas_capabilities(tools), |
| 395 | CompilerSpec::Named(named) => generic_external_capabilities(CompilerSpec::Named(*named)), |
| 396 | CompilerSpec::Binary(path) => { |
| 397 | generic_external_capabilities(CompilerSpec::Binary(path.clone())) |
| 398 | } |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | fn capability_extra_summary(extras: &BTreeMap<String, Vec<String>>) -> String { |
| 403 | if extras.is_empty() { |
| 404 | return "none".to_string(); |
| 405 | } |
| 406 | |
| 407 | extras |
| 408 | .iter() |
| 409 | .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 410 | .collect::<Vec<_>>() |
| 411 | .join(", ") |
| 412 | } |
| 413 | |
| 414 | fn capability_unavailable_summary(capabilities: &CompilerCapabilities) -> String { |
| 415 | if capabilities.unavailable_artifacts.is_empty() { |
| 416 | return "none".to_string(); |
| 417 | } |
| 418 | |
| 419 | let mut grouped = BTreeMap::<String, Vec<String>>::new(); |
| 420 | for artifact in capabilities.unavailable_artifacts.keys() { |
| 421 | if let Some((namespace, local_name)) = artifact.extra_parts() { |
| 422 | grouped |
| 423 | .entry(namespace.to_string()) |
| 424 | .or_insert_with(Vec::new) |
| 425 | .push(local_name.to_string()); |
| 426 | } else { |
| 427 | grouped |
| 428 | .entry("generic".to_string()) |
| 429 | .or_insert_with(Vec::new) |
| 430 | .push(artifact.as_str().to_string()); |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | grouped |
| 435 | .iter() |
| 436 | .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 437 | .collect::<Vec<_>>() |
| 438 | .join(", ") |
| 439 | } |
| 440 | |
| 441 | fn named_compiler_status_value( |
| 442 | named: NamedCompiler, |
| 443 | tools: &ToolchainConfig, |
| 444 | capture_root: Option<&PathBuf>, |
| 445 | ) -> String { |
| 446 | let probe = named_compiler_probe(named, tools, capture_root); |
| 447 | match probe.resolved_path { |
| 448 | Some(path) => format!( |
| 449 | "configured={} resolved={}", |
| 450 | probe.configured, |
| 451 | path.display() |
| 452 | ), |
| 453 | None => { |
| 454 | if probe.status == "linked" { |
| 455 | probe |
| 456 | .detail |
| 457 | .unwrap_or_else(|| "linked via Cargo".to_string()) |
| 458 | } else { |
| 459 | format!("configured={} resolved=missing", probe.configured) |
| 460 | } |
| 461 | } |
| 462 | } |
| 463 | } |
| 464 | |
| 465 | fn append_named_compiler_fields( |
| 466 | fields: &mut Vec<(String, String)>, |
| 467 | named: NamedCompiler, |
| 468 | tools: &ToolchainConfig, |
| 469 | capture_root: Option<&PathBuf>, |
| 470 | ) { |
| 471 | let prefix = format!("named_compiler.{}", named.as_str()); |
| 472 | let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 473 | let probe = named_compiler_probe(named, tools, capture_root); |
| 474 | if named == NamedCompiler::Armfortas { |
| 475 | let armfortas = tools.armfortas_adapters(); |
| 476 | fields.push(( |
| 477 | prefix.clone(), |
| 478 | format!( |
| 479 | "cli={} capture={}", |
| 480 | armfortas.cli_mode_name(), |
| 481 | armfortas.capture_mode_name() |
| 482 | ), |
| 483 | )); |
| 484 | } else { |
| 485 | fields.push(( |
| 486 | prefix.clone(), |
| 487 | named_compiler_status_value(named, tools, capture_root), |
| 488 | )); |
| 489 | } |
| 490 | fields.push(( |
| 491 | format!("{}.accepted_names", prefix), |
| 492 | named.accepted_names().join(", "), |
| 493 | )); |
| 494 | fields.push(( |
| 495 | format!("{}.candidate_binaries", prefix), |
| 496 | named.candidate_binaries().join(", "), |
| 497 | )); |
| 498 | fields.push(( |
| 499 | format!("{}.generic_artifacts", prefix), |
| 500 | format_artifact_name_list(&capabilities.generic_artifacts()), |
| 501 | )); |
| 502 | fields.push(( |
| 503 | format!("{}.adapter_extras", prefix), |
| 504 | capability_extra_summary(&capabilities.adapter_extras()), |
| 505 | )); |
| 506 | fields.push(( |
| 507 | format!("{}.unavailable_artifacts", prefix), |
| 508 | capability_unavailable_summary(&capabilities), |
| 509 | )); |
| 510 | fields.push((format!("{}.probe_status", prefix), probe.status.clone())); |
| 511 | fields.push(( |
| 512 | format!("{}.probe_resolved_path", prefix), |
| 513 | probe |
| 514 | .resolved_path |
| 515 | .as_ref() |
| 516 | .map(|path| display_path(path)) |
| 517 | .unwrap_or_else(|| "none".to_string()), |
| 518 | )); |
| 519 | fields.push(( |
| 520 | format!("{}.probe_banner", prefix), |
| 521 | probe.banner.clone().unwrap_or_else(|| "none".to_string()), |
| 522 | )); |
| 523 | fields.push(( |
| 524 | format!("{}.probe_detail", prefix), |
| 525 | probe.detail.clone().unwrap_or_else(|| "none".to_string()), |
| 526 | )); |
| 527 | } |
| 528 | |
| 529 | fn compiler_capability_backend(spec: &CompilerSpec, tools: &ToolchainConfig) -> (String, String) { |
| 530 | match spec { |
| 531 | CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 532 | let adapters = tools.armfortas_adapters(); |
| 533 | ( |
| 534 | adapters.capture_mode_name().to_string(), |
| 535 | adapters.capture_description().to_string(), |
| 536 | ) |
| 537 | } |
| 538 | CompilerSpec::Named(named) => { |
| 539 | let binary = tools |
| 540 | .named_compiler_binary(*named) |
| 541 | .unwrap_or_else(|| named.as_str().to_string()); |
| 542 | ( |
| 543 | "external-driver".to_string(), |
| 544 | format!("generic external driver adapter using {}", binary), |
| 545 | ) |
| 546 | } |
| 547 | CompilerSpec::Binary(path) => ( |
| 548 | "external-driver".to_string(), |
| 549 | format!("generic external driver adapter using {}", path.display()), |
| 550 | ), |
| 551 | } |
| 552 | } |
| 553 | |
| 554 | fn observation_from_capability_mismatch( |
| 555 | spec: &CompilerSpec, |
| 556 | program: &Path, |
| 557 | opt_level: OptLevel, |
| 558 | requested: BTreeSet<ArtifactKey>, |
| 559 | backend_mode: String, |
| 560 | backend_detail: String, |
| 561 | detail: String, |
| 562 | ) -> ObservedProgram { |
| 563 | ObservedProgram { |
| 564 | observation: CompilerObservation { |
| 565 | compiler: spec.clone(), |
| 566 | program: program.to_path_buf(), |
| 567 | opt_level, |
| 568 | compile_exit_code: 1, |
| 569 | artifacts: BTreeMap::from([(ArtifactKey::Diagnostics, ArtifactValue::Text(detail))]), |
| 570 | provenance: ObservationProvenance { |
| 571 | compiler_identity: spec.display_name(), |
| 572 | adapter_kind: match spec { |
| 573 | CompilerSpec::Named(_) => "named".into(), |
| 574 | CompilerSpec::Binary(_) => "explicit-path".into(), |
| 575 | }, |
| 576 | backend_mode, |
| 577 | backend_detail, |
| 578 | artifacts_captured: vec!["diagnostics".into()], |
| 579 | comparison_basis: None, |
| 580 | failure_stage: None, |
| 581 | }, |
| 582 | }, |
| 583 | requested_artifacts: requested, |
| 584 | } |
| 585 | } |
| 586 | |
| 587 | fn preflight_introspection_request( |
| 588 | spec: &CompilerSpec, |
| 589 | program: &Path, |
| 590 | opt_level: OptLevel, |
| 591 | requested: &BTreeSet<ArtifactKey>, |
| 592 | tools: &ToolchainConfig, |
| 593 | ) -> Option<ObservedProgram> { |
| 594 | let capabilities = compiler_capabilities(spec, tools); |
| 595 | let (backend_mode, backend_detail) = compiler_capability_backend(spec, tools); |
| 596 | |
| 597 | let unavailable = capabilities.unavailable_requests(requested); |
| 598 | if !unavailable.is_empty() { |
| 599 | let detail = unavailable |
| 600 | .into_iter() |
| 601 | .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 602 | .collect::<Vec<_>>() |
| 603 | .join("\n"); |
| 604 | return Some(observation_from_capability_mismatch( |
| 605 | spec, |
| 606 | program, |
| 607 | opt_level, |
| 608 | requested.clone(), |
| 609 | backend_mode, |
| 610 | backend_detail, |
| 611 | detail, |
| 612 | )); |
| 613 | } |
| 614 | |
| 615 | let unsupported = capabilities.unsupported_requests(requested); |
| 616 | if !unsupported.is_empty() { |
| 617 | let detail = format!( |
| 618 | "{} does not support requested artifacts in this adapter: {}", |
| 619 | spec.display_name(), |
| 620 | unsupported.join(", ") |
| 621 | ); |
| 622 | return Some(observation_from_capability_mismatch( |
| 623 | spec, |
| 624 | program, |
| 625 | opt_level, |
| 626 | requested.clone(), |
| 627 | backend_mode, |
| 628 | backend_detail, |
| 629 | detail, |
| 630 | )); |
| 631 | } |
| 632 | |
| 633 | None |
| 634 | } |
| 635 | |
| 636 | fn tool_override(var: &str, default: &str) -> String { |
| 637 | match std::env::var(var) { |
| 638 | Ok(value) if !value.trim().is_empty() => value, |
| 639 | _ => default.to_string(), |
| 640 | } |
| 641 | } |
| 642 | |
| 643 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 644 | enum OutcomeKind { |
| 645 | Pass, |
| 646 | Fail, |
| 647 | Xfail, |
| 648 | Xpass, |
| 649 | Future, |
| 650 | } |
| 651 | |
| 652 | #[derive(Debug, Clone)] |
| 653 | struct Outcome { |
| 654 | suite: String, |
| 655 | case: String, |
| 656 | opt_level: OptLevel, |
| 657 | kind: OutcomeKind, |
| 658 | detail: String, |
| 659 | bundle: Option<PathBuf>, |
| 660 | primary_backend: Option<PrimaryBackendReport>, |
| 661 | consistency_observations: Vec<ConsistencyObservation>, |
| 662 | } |
| 663 | |
| 664 | #[derive(Debug, Default)] |
| 665 | struct Summary { |
| 666 | passed: usize, |
| 667 | failed: usize, |
| 668 | xfailed: usize, |
| 669 | xpassed: usize, |
| 670 | future: usize, |
| 671 | outcomes: Vec<Outcome>, |
| 672 | consistency: BTreeMap<ConsistencyCheck, ConsistencyRollup>, |
| 673 | } |
| 674 | |
| 675 | #[derive(Debug, Clone, Default, PartialEq, Eq)] |
| 676 | struct ConsistencyRollup { |
| 677 | cells: usize, |
| 678 | repeat_counts: BTreeSet<usize>, |
| 679 | unique_variant_counts: BTreeSet<usize>, |
| 680 | varying_components: BTreeSet<String>, |
| 681 | stable_components: BTreeSet<String>, |
| 682 | } |
| 683 | |
| 684 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 685 | struct ConsistencyObservation { |
| 686 | check: ConsistencyCheck, |
| 687 | summary: String, |
| 688 | repeat_count: Option<usize>, |
| 689 | unique_variant_count: Option<usize>, |
| 690 | varying_components: Vec<String>, |
| 691 | stable_components: Vec<String>, |
| 692 | } |
| 693 | |
| 694 | #[derive(Debug, Clone)] |
| 695 | struct ListConfig { |
| 696 | suite_filter: Option<String>, |
| 697 | verbose: bool, |
| 698 | tools: ToolchainConfig, |
| 699 | } |
| 700 | |
| 701 | #[derive(Debug, Clone)] |
| 702 | struct RunConfig { |
| 703 | suite_filter: Option<String>, |
| 704 | case_filter: Option<String>, |
| 705 | opt_filter: Option<BTreeSet<OptLevel>>, |
| 706 | verbose: bool, |
| 707 | fail_fast: bool, |
| 708 | include_future: bool, |
| 709 | all_stages: bool, |
| 710 | json_report: Option<PathBuf>, |
| 711 | markdown_report: Option<PathBuf>, |
| 712 | tools: ToolchainConfig, |
| 713 | } |
| 714 | |
| 715 | #[derive(Debug, Clone)] |
| 716 | struct CompareConfig { |
| 717 | left: CompilerSpec, |
| 718 | right: CompilerSpec, |
| 719 | program: PathBuf, |
| 720 | opt_level: OptLevel, |
| 721 | artifacts: BTreeSet<ArtifactKey>, |
| 722 | json_report: Option<PathBuf>, |
| 723 | markdown_report: Option<PathBuf>, |
| 724 | tools: ToolchainConfig, |
| 725 | } |
| 726 | |
| 727 | #[derive(Debug, Clone)] |
| 728 | struct IntrospectConfig { |
| 729 | compiler: CompilerSpec, |
| 730 | program: PathBuf, |
| 731 | opt_level: OptLevel, |
| 732 | artifacts: BTreeSet<ArtifactKey>, |
| 733 | json_report: Option<PathBuf>, |
| 734 | markdown_report: Option<PathBuf>, |
| 735 | all_artifacts: bool, |
| 736 | summary_only: bool, |
| 737 | max_artifact_lines: Option<usize>, |
| 738 | tools: ToolchainConfig, |
| 739 | } |
| 740 | |
| 741 | #[derive(Debug, Clone)] |
| 742 | struct ExecutionArtifacts { |
| 743 | requested: BTreeSet<Stage>, |
| 744 | armfortas: Option<CaptureResult>, |
| 745 | armfortas_failure: Option<CaptureFailure>, |
| 746 | armfortas_observation: Option<ObservedProgram>, |
| 747 | references: Vec<ReferenceResult>, |
| 748 | reference_observations: Vec<ObservedProgram>, |
| 749 | consistency_issues: Vec<ConsistencyIssue>, |
| 750 | } |
| 751 | |
| 752 | #[derive(Debug, Clone)] |
| 753 | struct ObservedProgram { |
| 754 | observation: CompilerObservation, |
| 755 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 756 | } |
| 757 | |
| 758 | #[derive(Debug, Clone, Copy)] |
| 759 | struct IntrospectionRenderConfig { |
| 760 | summary_only: bool, |
| 761 | max_artifact_lines: Option<usize>, |
| 762 | } |
| 763 | |
| 764 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 765 | enum PrimaryCaptureBackendKind { |
| 766 | Full, |
| 767 | Observable, |
| 768 | } |
| 769 | |
| 770 | impl PrimaryCaptureBackendKind { |
| 771 | fn as_str(&self) -> &'static str { |
| 772 | match self { |
| 773 | Self::Full => "full", |
| 774 | Self::Observable => "observable", |
| 775 | } |
| 776 | } |
| 777 | } |
| 778 | |
| 779 | struct SelectedPrimaryBackend { |
| 780 | kind: PrimaryCaptureBackendKind, |
| 781 | backend: Box<dyn CaptureBackend>, |
| 782 | } |
| 783 | |
| 784 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 785 | struct PrimaryBackendReport { |
| 786 | kind: String, |
| 787 | mode: String, |
| 788 | detail: String, |
| 789 | } |
| 790 | |
| 791 | impl PrimaryBackendReport { |
| 792 | fn from_selected(selected: &SelectedPrimaryBackend) -> Self { |
| 793 | Self { |
| 794 | kind: selected.kind.as_str().to_string(), |
| 795 | mode: selected.backend.mode_name().to_string(), |
| 796 | detail: selected.backend.description().to_string(), |
| 797 | } |
| 798 | } |
| 799 | } |
| 800 | |
| 801 | #[derive(Debug, Clone)] |
| 802 | struct ReferenceResult { |
| 803 | compiler: ReferenceCompiler, |
| 804 | compile_command: String, |
| 805 | compile_exit_code: i32, |
| 806 | compile_stdout: String, |
| 807 | compile_stderr: String, |
| 808 | run: Option<RunCapture>, |
| 809 | run_error: Option<String>, |
| 810 | } |
| 811 | |
| 812 | #[derive(Debug, Clone)] |
| 813 | struct ConsistencyIssue { |
| 814 | check: ConsistencyCheck, |
| 815 | summary: String, |
| 816 | repeat_count: Option<usize>, |
| 817 | unique_variant_count: Option<usize>, |
| 818 | varying_components: Vec<String>, |
| 819 | stable_components: Vec<String>, |
| 820 | detail: String, |
| 821 | temp_root: PathBuf, |
| 822 | } |
| 823 | |
| 824 | impl Summary { |
| 825 | fn record_outcome(&mut self, outcome: &Outcome) { |
| 826 | match outcome.kind { |
| 827 | OutcomeKind::Pass => self.passed += 1, |
| 828 | OutcomeKind::Fail => self.failed += 1, |
| 829 | OutcomeKind::Xfail => self.xfailed += 1, |
| 830 | OutcomeKind::Xpass => self.xpassed += 1, |
| 831 | OutcomeKind::Future => self.future += 1, |
| 832 | } |
| 833 | self.outcomes.push(outcome.clone()); |
| 834 | self.record_consistency(&outcome.consistency_observations); |
| 835 | } |
| 836 | |
| 837 | fn record_consistency(&mut self, observations: &[ConsistencyObservation]) { |
| 838 | for observation in observations { |
| 839 | self.consistency |
| 840 | .entry(observation.check) |
| 841 | .or_default() |
| 842 | .record(observation); |
| 843 | } |
| 844 | } |
| 845 | } |
| 846 | |
| 847 | impl ConsistencyRollup { |
| 848 | fn record(&mut self, observation: &ConsistencyObservation) { |
| 849 | self.cells += 1; |
| 850 | if let Some(repeat_count) = observation.repeat_count { |
| 851 | self.repeat_counts.insert(repeat_count); |
| 852 | } |
| 853 | if let Some(unique_variant_count) = observation.unique_variant_count { |
| 854 | self.unique_variant_counts.insert(unique_variant_count); |
| 855 | } |
| 856 | self.varying_components |
| 857 | .extend(observation.varying_components.iter().cloned()); |
| 858 | self.stable_components |
| 859 | .extend(observation.stable_components.iter().cloned()); |
| 860 | } |
| 861 | } |
| 862 | |
| 863 | impl ConsistencyIssue { |
| 864 | fn observation(&self) -> ConsistencyObservation { |
| 865 | ConsistencyObservation { |
| 866 | check: self.check, |
| 867 | summary: self.summary.clone(), |
| 868 | repeat_count: self.repeat_count, |
| 869 | unique_variant_count: self.unique_variant_count, |
| 870 | varying_components: self.varying_components.clone(), |
| 871 | stable_components: self.stable_components.clone(), |
| 872 | } |
| 873 | } |
| 874 | } |
| 875 | |
| 876 | impl ReferenceResult { |
| 877 | fn infrastructure_error(compiler: ReferenceCompiler, command: String, message: String) -> Self { |
| 878 | Self { |
| 879 | compiler, |
| 880 | compile_command: command, |
| 881 | compile_exit_code: -1, |
| 882 | compile_stdout: String::new(), |
| 883 | compile_stderr: message, |
| 884 | run: None, |
| 885 | run_error: None, |
| 886 | } |
| 887 | } |
| 888 | } |
| 889 | |
| 890 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] |
| 891 | struct RunSignature { |
| 892 | exit_code: i32, |
| 893 | stdout: String, |
| 894 | stderr: String, |
| 895 | } |
| 896 | |
| 897 | pub fn run_cli(args: &[String]) -> i32 { |
| 898 | run_cli_named("afs-tests", args) |
| 899 | } |
| 900 | |
| 901 | pub fn run_cli_named(program_name: &str, args: &[String]) -> i32 { |
| 902 | match parse_cli(args) { |
| 903 | Ok(CommandKind::List(config)) => match discover_suites(default_suite_root()) { |
| 904 | Ok(suites) => { |
| 905 | print_suites( |
| 906 | &filter_suites(&suites, config.suite_filter.as_deref()), |
| 907 | &config, |
| 908 | ); |
| 909 | 0 |
| 910 | } |
| 911 | Err(err) => { |
| 912 | eprintln!("{}: {}", program_name, err); |
| 913 | 1 |
| 914 | } |
| 915 | }, |
| 916 | Ok(CommandKind::Run(config)) => match run_suites(&config) { |
| 917 | Ok(summary) => { |
| 918 | print_summary(&summary); |
| 919 | if let Err(err) = write_requested_reports(&config, &summary) { |
| 920 | eprintln!("afs-tests: {}", err); |
| 921 | return 1; |
| 922 | } |
| 923 | if summary.failed == 0 && summary.xpassed == 0 { |
| 924 | 0 |
| 925 | } else { |
| 926 | 1 |
| 927 | } |
| 928 | } |
| 929 | Err(err) => { |
| 930 | eprintln!("{}: {}", program_name, err); |
| 931 | 1 |
| 932 | } |
| 933 | }, |
| 934 | Ok(CommandKind::Compare(config)) => match run_compare(&config) { |
| 935 | Ok(result) => { |
| 936 | print_compare_result(&result); |
| 937 | if let Err(err) = write_compare_reports(&config, &result) { |
| 938 | eprintln!("{}: {}", program_name, err); |
| 939 | return 1; |
| 940 | } |
| 941 | if result.differences.is_empty() { |
| 942 | 0 |
| 943 | } else { |
| 944 | 1 |
| 945 | } |
| 946 | } |
| 947 | Err(err) => { |
| 948 | eprintln!("{}: {}", program_name, err); |
| 949 | 1 |
| 950 | } |
| 951 | }, |
| 952 | Ok(CommandKind::Introspect(config)) => match run_introspect(&config) { |
| 953 | Ok(observation) => { |
| 954 | print_introspection(&config, &observation); |
| 955 | if let Err(err) = write_introspection_reports(&config, &observation) { |
| 956 | eprintln!("{}: {}", program_name, err); |
| 957 | return 1; |
| 958 | } |
| 959 | if observation.observation.compile_exit_code == 0 { |
| 960 | 0 |
| 961 | } else { |
| 962 | 1 |
| 963 | } |
| 964 | } |
| 965 | Err(err) => { |
| 966 | eprintln!("{}: {}", program_name, err); |
| 967 | 1 |
| 968 | } |
| 969 | }, |
| 970 | Ok(CommandKind::Doctor(config)) => { |
| 971 | println!("{}", render_doctor_report(&config)); |
| 972 | if let Err(err) = write_doctor_reports(&config) { |
| 973 | eprintln!("{}: {}", program_name, err); |
| 974 | return 1; |
| 975 | } |
| 976 | 0 |
| 977 | } |
| 978 | Ok(CommandKind::Help) => { |
| 979 | print_usage(program_name); |
| 980 | 0 |
| 981 | } |
| 982 | Err(err) => { |
| 983 | eprintln!("{}: {}", program_name, err); |
| 984 | print_usage(program_name); |
| 985 | 2 |
| 986 | } |
| 987 | } |
| 988 | } |
| 989 | |
| 990 | enum CommandKind { |
| 991 | List(ListConfig), |
| 992 | Run(RunConfig), |
| 993 | Compare(CompareConfig), |
| 994 | Introspect(IntrospectConfig), |
| 995 | Doctor(DoctorConfig), |
| 996 | Help, |
| 997 | } |
| 998 | |
| 999 | #[derive(Debug, Clone)] |
| 1000 | struct DoctorConfig { |
| 1001 | tools: ToolchainConfig, |
| 1002 | json_report: Option<PathBuf>, |
| 1003 | markdown_report: Option<PathBuf>, |
| 1004 | } |
| 1005 | |
| 1006 | fn parse_tool_override_arg( |
| 1007 | arg: &str, |
| 1008 | queue: &mut VecDeque<&String>, |
| 1009 | tools: &mut ToolchainConfig, |
| 1010 | ) -> Result<bool, String> { |
| 1011 | match arg { |
| 1012 | "--armfortas-bin" => { |
| 1013 | let value = queue |
| 1014 | .pop_front() |
| 1015 | .ok_or("--armfortas-bin requires a value")?; |
| 1016 | tools.armfortas = ArmfortasCliAdapter::External(value.clone()); |
| 1017 | Ok(true) |
| 1018 | } |
| 1019 | "--gfortran-bin" => { |
| 1020 | let value = queue.pop_front().ok_or("--gfortran-bin requires a value")?; |
| 1021 | tools.gfortran = value.clone(); |
| 1022 | Ok(true) |
| 1023 | } |
| 1024 | "--flang-bin" => { |
| 1025 | let value = queue.pop_front().ok_or("--flang-bin requires a value")?; |
| 1026 | tools.flang_new = value.clone(); |
| 1027 | Ok(true) |
| 1028 | } |
| 1029 | "--lfortran-bin" => { |
| 1030 | let value = queue.pop_front().ok_or("--lfortran-bin requires a value")?; |
| 1031 | tools.lfortran = value.clone(); |
| 1032 | Ok(true) |
| 1033 | } |
| 1034 | "--ifort-bin" => { |
| 1035 | let value = queue.pop_front().ok_or("--ifort-bin requires a value")?; |
| 1036 | tools.ifort = value.clone(); |
| 1037 | Ok(true) |
| 1038 | } |
| 1039 | "--ifx-bin" => { |
| 1040 | let value = queue.pop_front().ok_or("--ifx-bin requires a value")?; |
| 1041 | tools.ifx = value.clone(); |
| 1042 | Ok(true) |
| 1043 | } |
| 1044 | "--nvfortran-bin" => { |
| 1045 | let value = queue |
| 1046 | .pop_front() |
| 1047 | .ok_or("--nvfortran-bin requires a value")?; |
| 1048 | tools.nvfortran = value.clone(); |
| 1049 | Ok(true) |
| 1050 | } |
| 1051 | "--as-bin" => { |
| 1052 | let value = queue.pop_front().ok_or("--as-bin requires a value")?; |
| 1053 | tools.system_as = value.clone(); |
| 1054 | Ok(true) |
| 1055 | } |
| 1056 | "--otool-bin" => { |
| 1057 | let value = queue.pop_front().ok_or("--otool-bin requires a value")?; |
| 1058 | tools.otool = value.clone(); |
| 1059 | Ok(true) |
| 1060 | } |
| 1061 | "--nm-bin" => { |
| 1062 | let value = queue.pop_front().ok_or("--nm-bin requires a value")?; |
| 1063 | tools.nm = value.clone(); |
| 1064 | Ok(true) |
| 1065 | } |
| 1066 | _ => Ok(false), |
| 1067 | } |
| 1068 | } |
| 1069 | |
| 1070 | fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 1071 | if args.is_empty() { |
| 1072 | return Ok(CommandKind::Help); |
| 1073 | } |
| 1074 | |
| 1075 | match args[0].as_str() { |
| 1076 | "list" => { |
| 1077 | let mut config = ListConfig { |
| 1078 | suite_filter: None, |
| 1079 | verbose: false, |
| 1080 | tools: ToolchainConfig::from_env(), |
| 1081 | }; |
| 1082 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1083 | while let Some(arg) = queue.pop_front() { |
| 1084 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1085 | continue; |
| 1086 | } |
| 1087 | match arg.as_str() { |
| 1088 | "--suite" => { |
| 1089 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
| 1090 | config.suite_filter = Some(value.clone()); |
| 1091 | } |
| 1092 | "--verbose" => config.verbose = true, |
| 1093 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1094 | other => return Err(format!("unknown list option: {}", other)), |
| 1095 | } |
| 1096 | } |
| 1097 | Ok(CommandKind::List(config)) |
| 1098 | } |
| 1099 | "run" => { |
| 1100 | let mut config = RunConfig { |
| 1101 | suite_filter: None, |
| 1102 | case_filter: None, |
| 1103 | opt_filter: None, |
| 1104 | verbose: false, |
| 1105 | fail_fast: false, |
| 1106 | include_future: false, |
| 1107 | all_stages: false, |
| 1108 | json_report: None, |
| 1109 | markdown_report: None, |
| 1110 | tools: ToolchainConfig::from_env(), |
| 1111 | }; |
| 1112 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1113 | while let Some(arg) = queue.pop_front() { |
| 1114 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1115 | continue; |
| 1116 | } |
| 1117 | match arg.as_str() { |
| 1118 | "--suite" => { |
| 1119 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
| 1120 | config.suite_filter = Some(value.clone()); |
| 1121 | } |
| 1122 | "--case" => { |
| 1123 | let value = queue.pop_front().ok_or("--case requires a value")?; |
| 1124 | config.case_filter = Some(value.clone()); |
| 1125 | } |
| 1126 | "--opt" => { |
| 1127 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1128 | let parsed = parse_opt_level_list(value)?; |
| 1129 | let filter = config.opt_filter.get_or_insert_with(BTreeSet::new); |
| 1130 | filter.extend(parsed); |
| 1131 | } |
| 1132 | "--verbose" | "-v" => config.verbose = true, |
| 1133 | "--fail-fast" => config.fail_fast = true, |
| 1134 | "--include-future" => config.include_future = true, |
| 1135 | "--all" => config.all_stages = true, |
| 1136 | "--json-report" => { |
| 1137 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1138 | config.json_report = Some(PathBuf::from(value)); |
| 1139 | } |
| 1140 | "--markdown-report" => { |
| 1141 | let value = queue |
| 1142 | .pop_front() |
| 1143 | .ok_or("--markdown-report requires a value")?; |
| 1144 | config.markdown_report = Some(PathBuf::from(value)); |
| 1145 | } |
| 1146 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1147 | other => return Err(format!("unknown run option: {}", other)), |
| 1148 | } |
| 1149 | } |
| 1150 | Ok(CommandKind::Run(config)) |
| 1151 | } |
| 1152 | "compare" => { |
| 1153 | if args.len() < 3 { |
| 1154 | return Err( |
| 1155 | "compare requires <compiler-a> <compiler-b> and --program <path>".to_string(), |
| 1156 | ); |
| 1157 | } |
| 1158 | let left = CompilerSpec::parse(&args[1]); |
| 1159 | let right = CompilerSpec::parse(&args[2]); |
| 1160 | let mut config = CompareConfig { |
| 1161 | left, |
| 1162 | right, |
| 1163 | program: PathBuf::new(), |
| 1164 | opt_level: OptLevel::O0, |
| 1165 | artifacts: BTreeSet::new(), |
| 1166 | json_report: None, |
| 1167 | markdown_report: None, |
| 1168 | tools: ToolchainConfig::from_env(), |
| 1169 | }; |
| 1170 | let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1171 | while let Some(arg) = queue.pop_front() { |
| 1172 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1173 | continue; |
| 1174 | } |
| 1175 | match arg.as_str() { |
| 1176 | "--program" => { |
| 1177 | let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1178 | config.program = PathBuf::from(value); |
| 1179 | } |
| 1180 | "--opt" => { |
| 1181 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1182 | let parsed = parse_opt_level_list(value)?; |
| 1183 | let opt = parsed |
| 1184 | .into_iter() |
| 1185 | .next() |
| 1186 | .ok_or("--opt requires at least one optimization level")?; |
| 1187 | config.opt_level = opt; |
| 1188 | } |
| 1189 | "--artifact" => { |
| 1190 | let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1191 | config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 1192 | } |
| 1193 | "--json-report" => { |
| 1194 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1195 | config.json_report = Some(PathBuf::from(value)); |
| 1196 | } |
| 1197 | "--markdown-report" => { |
| 1198 | let value = queue |
| 1199 | .pop_front() |
| 1200 | .ok_or("--markdown-report requires a value")?; |
| 1201 | config.markdown_report = Some(PathBuf::from(value)); |
| 1202 | } |
| 1203 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1204 | other => return Err(format!("unknown compare option: {}", other)), |
| 1205 | } |
| 1206 | } |
| 1207 | if config.program.as_os_str().is_empty() { |
| 1208 | return Err("compare requires --program <path>".to_string()); |
| 1209 | } |
| 1210 | Ok(CommandKind::Compare(config)) |
| 1211 | } |
| 1212 | "introspect" => { |
| 1213 | if args.len() < 3 { |
| 1214 | return Err("introspect requires <compiler> <program>".to_string()); |
| 1215 | } |
| 1216 | let compiler = CompilerSpec::parse(&args[1]); |
| 1217 | let mut config = IntrospectConfig { |
| 1218 | compiler, |
| 1219 | program: PathBuf::from(&args[2]), |
| 1220 | opt_level: OptLevel::O0, |
| 1221 | artifacts: BTreeSet::new(), |
| 1222 | json_report: None, |
| 1223 | markdown_report: None, |
| 1224 | all_artifacts: false, |
| 1225 | summary_only: false, |
| 1226 | max_artifact_lines: None, |
| 1227 | tools: ToolchainConfig::from_env(), |
| 1228 | }; |
| 1229 | let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1230 | while let Some(arg) = queue.pop_front() { |
| 1231 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1232 | continue; |
| 1233 | } |
| 1234 | match arg.as_str() { |
| 1235 | "--program" => { |
| 1236 | let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1237 | config.program = PathBuf::from(value); |
| 1238 | } |
| 1239 | "--opt" => { |
| 1240 | let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1241 | let parsed = parse_opt_level_list(value)?; |
| 1242 | let opt = parsed |
| 1243 | .into_iter() |
| 1244 | .next() |
| 1245 | .ok_or("--opt requires at least one optimization level")?; |
| 1246 | config.opt_level = opt; |
| 1247 | } |
| 1248 | "--artifact" => { |
| 1249 | let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1250 | config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 1251 | } |
| 1252 | "--all" => config.all_artifacts = true, |
| 1253 | "--summary-only" => config.summary_only = true, |
| 1254 | "--max-artifact-lines" => { |
| 1255 | let value = queue |
| 1256 | .pop_front() |
| 1257 | .ok_or("--max-artifact-lines requires a value")?; |
| 1258 | let parsed = value.parse::<usize>().map_err(|_| { |
| 1259 | format!("invalid --max-artifact-lines value '{}'", value) |
| 1260 | })?; |
| 1261 | if parsed == 0 { |
| 1262 | return Err("--max-artifact-lines must be greater than 0".to_string()); |
| 1263 | } |
| 1264 | config.max_artifact_lines = Some(parsed); |
| 1265 | } |
| 1266 | "--json-report" => { |
| 1267 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1268 | config.json_report = Some(PathBuf::from(value)); |
| 1269 | } |
| 1270 | "--markdown-report" => { |
| 1271 | let value = queue |
| 1272 | .pop_front() |
| 1273 | .ok_or("--markdown-report requires a value")?; |
| 1274 | config.markdown_report = Some(PathBuf::from(value)); |
| 1275 | } |
| 1276 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1277 | other => return Err(format!("unknown introspect option: {}", other)), |
| 1278 | } |
| 1279 | } |
| 1280 | Ok(CommandKind::Introspect(config)) |
| 1281 | } |
| 1282 | "doctor" => { |
| 1283 | let mut config = DoctorConfig { |
| 1284 | tools: ToolchainConfig::from_env(), |
| 1285 | json_report: None, |
| 1286 | markdown_report: None, |
| 1287 | }; |
| 1288 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1289 | while let Some(arg) = queue.pop_front() { |
| 1290 | if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1291 | continue; |
| 1292 | } |
| 1293 | match arg.as_str() { |
| 1294 | "--json-report" => { |
| 1295 | let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1296 | config.json_report = Some(PathBuf::from(value)); |
| 1297 | } |
| 1298 | "--markdown-report" => { |
| 1299 | let value = queue |
| 1300 | .pop_front() |
| 1301 | .ok_or("--markdown-report requires a value")?; |
| 1302 | config.markdown_report = Some(PathBuf::from(value)); |
| 1303 | } |
| 1304 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 1305 | other => return Err(format!("unknown doctor option: {}", other)), |
| 1306 | } |
| 1307 | } |
| 1308 | Ok(CommandKind::Doctor(config)) |
| 1309 | } |
| 1310 | "--help" | "-h" | "help" => Ok(CommandKind::Help), |
| 1311 | other => Err(format!("unknown command: {}", other)), |
| 1312 | } |
| 1313 | } |
| 1314 | |
| 1315 | fn print_usage(program_name: &str) { |
| 1316 | eprintln!( |
| 1317 | "{} — generic compiler bench runner (afs-tests compatibility preserved)", |
| 1318 | program_name |
| 1319 | ); |
| 1320 | eprintln!(); |
| 1321 | eprintln!("usage:"); |
| 1322 | eprintln!( |
| 1323 | " {} list [--suite <filter>] [--verbose] [tool overrides]", |
| 1324 | program_name |
| 1325 | ); |
| 1326 | eprintln!( |
| 1327 | " {} run [--suite <filter>] [--case <filter>] [--opt <O0,O1,...>] [--verbose] [--fail-fast] [--include-future] [--all] [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1328 | program_name |
| 1329 | ); |
| 1330 | eprintln!( |
| 1331 | " {} compare <compiler-a> <compiler-b> --program <path> [--opt <O0>] [--artifact <asm,obj,stdout,stderr,exit-code,executable>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1332 | program_name |
| 1333 | ); |
| 1334 | eprintln!( |
| 1335 | " {} introspect <compiler> <program> [--opt <O0>] [--artifact <list>] [--all] [--summary-only] [--max-artifact-lines <n>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1336 | program_name |
| 1337 | ); |
| 1338 | eprintln!( |
| 1339 | " {} doctor [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--lfortran-bin <path>] [--ifort-bin <path>] [--ifx-bin <path>] [--nvfortran-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1340 | program_name |
| 1341 | ); |
| 1342 | eprintln!(); |
| 1343 | eprintln!("env overrides:"); |
| 1344 | eprintln!(" BENCCH_ARMFORTAS_BIN, BENCCH_GFORTRAN_BIN, BENCCH_FLANG_BIN, BENCCH_LFORTRAN_BIN"); |
| 1345 | eprintln!(" BENCCH_IFORT_BIN, BENCCH_IFX_BIN, BENCCH_NVFORTRAN_BIN"); |
| 1346 | eprintln!(" BENCCH_AS_BIN, BENCCH_OTOOL_BIN, BENCCH_NM_BIN"); |
| 1347 | eprintln!(); |
| 1348 | if linked_capture_available() { |
| 1349 | eprintln!("mode:"); |
| 1350 | eprintln!(" linked armfortas capture is available in this build"); |
| 1351 | } else { |
| 1352 | eprintln!("mode:"); |
| 1353 | eprintln!(" linked armfortas capture is unavailable in this build"); |
| 1354 | eprintln!(" compare, introspect, and generic/observable suite runs still work"); |
| 1355 | eprintln!(" use scripts/bootstrap-linked-armfortas.sh for rich armfortas stages and legacy frontend/module suites"); |
| 1356 | } |
| 1357 | } |
| 1358 | |
| 1359 | fn default_compare_artifacts(extra: &BTreeSet<ArtifactKey>) -> BTreeSet<ArtifactKey> { |
| 1360 | let mut requested = BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]); |
| 1361 | requested.extend(extra.iter().cloned()); |
| 1362 | requested |
| 1363 | } |
| 1364 | |
| 1365 | fn default_differential_artifacts() -> BTreeSet<ArtifactKey> { |
| 1366 | BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]) |
| 1367 | } |
| 1368 | |
| 1369 | fn default_introspection_artifacts( |
| 1370 | compiler: &CompilerSpec, |
| 1371 | all_artifacts: bool, |
| 1372 | ) -> BTreeSet<ArtifactKey> { |
| 1373 | let mut requested = BTreeSet::from([ |
| 1374 | ArtifactKey::Diagnostics, |
| 1375 | ArtifactKey::Runtime, |
| 1376 | ArtifactKey::Asm, |
| 1377 | ArtifactKey::Obj, |
| 1378 | ]); |
| 1379 | if matches!(compiler, CompilerSpec::Named(NamedCompiler::Armfortas)) { |
| 1380 | requested.insert(ArtifactKey::Extra("armfortas.ir".into())); |
| 1381 | if all_artifacts { |
| 1382 | for name in [ |
| 1383 | "armfortas.preprocess", |
| 1384 | "armfortas.tokens", |
| 1385 | "armfortas.ast", |
| 1386 | "armfortas.sema", |
| 1387 | "armfortas.ir", |
| 1388 | "armfortas.optir", |
| 1389 | "armfortas.mir", |
| 1390 | "armfortas.regalloc", |
| 1391 | ] { |
| 1392 | requested.insert(ArtifactKey::Extra(name.to_string())); |
| 1393 | } |
| 1394 | } |
| 1395 | } |
| 1396 | requested |
| 1397 | } |
| 1398 | |
| 1399 | fn run_compare(config: &CompareConfig) -> Result<ComparisonResult, String> { |
| 1400 | let requested = default_compare_artifacts(&config.artifacts); |
| 1401 | preflight_compare_request(config, &requested)?; |
| 1402 | let left = observe_compiler( |
| 1403 | &config.left, |
| 1404 | &config.program, |
| 1405 | config.opt_level, |
| 1406 | &requested, |
| 1407 | &config.tools, |
| 1408 | )?; |
| 1409 | let right = observe_compiler( |
| 1410 | &config.right, |
| 1411 | &config.program, |
| 1412 | config.opt_level, |
| 1413 | &requested, |
| 1414 | &config.tools, |
| 1415 | )?; |
| 1416 | Ok(compare_observations(left, right, &requested)) |
| 1417 | } |
| 1418 | |
| 1419 | fn capability_request_issue( |
| 1420 | spec: &CompilerSpec, |
| 1421 | requested: &BTreeSet<ArtifactKey>, |
| 1422 | tools: &ToolchainConfig, |
| 1423 | ) -> Option<String> { |
| 1424 | let capabilities = compiler_capabilities(spec, tools); |
| 1425 | let unavailable = capabilities.unavailable_requests(requested); |
| 1426 | let unsupported = capabilities.unsupported_requests(requested); |
| 1427 | if unavailable.is_empty() && unsupported.is_empty() { |
| 1428 | return None; |
| 1429 | } |
| 1430 | |
| 1431 | let mut sections = Vec::new(); |
| 1432 | if !unavailable.is_empty() { |
| 1433 | let detail = unavailable |
| 1434 | .into_iter() |
| 1435 | .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 1436 | .collect::<Vec<_>>() |
| 1437 | .join("\n"); |
| 1438 | sections.push(format!( |
| 1439 | "{} unavailable for requested artifacts in this build\n{}", |
| 1440 | spec.display_name(), |
| 1441 | detail |
| 1442 | )); |
| 1443 | } |
| 1444 | if !unsupported.is_empty() { |
| 1445 | sections.push(format!( |
| 1446 | "{} does not support requested artifacts in this adapter: {}", |
| 1447 | spec.display_name(), |
| 1448 | unsupported.join(", ") |
| 1449 | )); |
| 1450 | } |
| 1451 | Some(sections.join("\n")) |
| 1452 | } |
| 1453 | |
| 1454 | fn compare_capability_issue( |
| 1455 | left: &CompilerSpec, |
| 1456 | right: &CompilerSpec, |
| 1457 | requested: &BTreeSet<ArtifactKey>, |
| 1458 | tools: &ToolchainConfig, |
| 1459 | ) -> Option<String> { |
| 1460 | let mut issues = Vec::new(); |
| 1461 | if let Some(issue) = capability_request_issue(left, requested, tools) { |
| 1462 | issues.push(format!("left:\n{}", issue)); |
| 1463 | } |
| 1464 | if let Some(issue) = capability_request_issue(right, requested, tools) { |
| 1465 | issues.push(format!("right:\n{}", issue)); |
| 1466 | } |
| 1467 | if issues.is_empty() { |
| 1468 | None |
| 1469 | } else { |
| 1470 | Some(format!( |
| 1471 | "compare request is not supported for the selected compiler surfaces\n{}", |
| 1472 | issues.join("\n") |
| 1473 | )) |
| 1474 | } |
| 1475 | } |
| 1476 | |
| 1477 | fn preflight_compare_request( |
| 1478 | config: &CompareConfig, |
| 1479 | requested: &BTreeSet<ArtifactKey>, |
| 1480 | ) -> Result<(), String> { |
| 1481 | match compare_capability_issue(&config.left, &config.right, requested, &config.tools) { |
| 1482 | Some(issue) => Err(issue), |
| 1483 | None => Ok(()), |
| 1484 | } |
| 1485 | } |
| 1486 | |
| 1487 | fn run_introspect(config: &IntrospectConfig) -> Result<ObservedProgram, String> { |
| 1488 | let requested = if config.artifacts.is_empty() { |
| 1489 | default_introspection_artifacts(&config.compiler, config.all_artifacts) |
| 1490 | } else { |
| 1491 | let mut requested = config.artifacts.clone(); |
| 1492 | if config.all_artifacts |
| 1493 | && matches!( |
| 1494 | config.compiler, |
| 1495 | CompilerSpec::Named(NamedCompiler::Armfortas) |
| 1496 | ) |
| 1497 | { |
| 1498 | requested.extend(default_introspection_artifacts(&config.compiler, true)); |
| 1499 | } |
| 1500 | requested |
| 1501 | }; |
| 1502 | if let Some(observed) = preflight_introspection_request( |
| 1503 | &config.compiler, |
| 1504 | &config.program, |
| 1505 | config.opt_level, |
| 1506 | &requested, |
| 1507 | &config.tools, |
| 1508 | ) { |
| 1509 | return Ok(observed); |
| 1510 | } |
| 1511 | Ok(ObservedProgram { |
| 1512 | observation: observe_compiler( |
| 1513 | &config.compiler, |
| 1514 | &config.program, |
| 1515 | config.opt_level, |
| 1516 | &requested, |
| 1517 | &config.tools, |
| 1518 | )?, |
| 1519 | requested_artifacts: requested, |
| 1520 | }) |
| 1521 | } |
| 1522 | |
| 1523 | fn requested_linked_armfortas_artifacts(requested: &BTreeSet<ArtifactKey>) -> Vec<String> { |
| 1524 | requested |
| 1525 | .iter() |
| 1526 | .filter_map(|artifact| match artifact { |
| 1527 | ArtifactKey::Extra(name) if name.starts_with("armfortas.") => Some(name.clone()), |
| 1528 | _ => None, |
| 1529 | }) |
| 1530 | .collect() |
| 1531 | } |
| 1532 | |
| 1533 | fn observe_compiler( |
| 1534 | spec: &CompilerSpec, |
| 1535 | program: &Path, |
| 1536 | opt_level: OptLevel, |
| 1537 | requested: &BTreeSet<ArtifactKey>, |
| 1538 | tools: &ToolchainConfig, |
| 1539 | ) -> Result<CompilerObservation, String> { |
| 1540 | match spec { |
| 1541 | CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 1542 | observe_armfortas(program, opt_level, requested, tools) |
| 1543 | } |
| 1544 | CompilerSpec::Named(named) => { |
| 1545 | let binary = tools.named_compiler_binary(*named).ok_or_else(|| { |
| 1546 | format!("named compiler '{}' has no resolved binary", named.as_str()) |
| 1547 | })?; |
| 1548 | observe_external_driver( |
| 1549 | spec, |
| 1550 | &binary, |
| 1551 | program, |
| 1552 | opt_level, |
| 1553 | requested, |
| 1554 | matches!(named, NamedCompiler::Gfortran | NamedCompiler::FlangNew) |
| 1555 | && source_uses_cpp(program), |
| 1556 | "named".to_string(), |
| 1557 | tools.otool_bin(), |
| 1558 | tools.nm_bin(), |
| 1559 | ) |
| 1560 | } |
| 1561 | CompilerSpec::Binary(path) => observe_external_driver( |
| 1562 | spec, |
| 1563 | &path.display().to_string(), |
| 1564 | program, |
| 1565 | opt_level, |
| 1566 | requested, |
| 1567 | false, |
| 1568 | "explicit-path".to_string(), |
| 1569 | tools.otool_bin(), |
| 1570 | tools.nm_bin(), |
| 1571 | ), |
| 1572 | } |
| 1573 | } |
| 1574 | |
| 1575 | fn observe_armfortas( |
| 1576 | program: &Path, |
| 1577 | opt_level: OptLevel, |
| 1578 | requested: &BTreeSet<ArtifactKey>, |
| 1579 | tools: &ToolchainConfig, |
| 1580 | ) -> Result<CompilerObservation, String> { |
| 1581 | let stages = armfortas_requested_stages(requested)?; |
| 1582 | let linked_only_artifacts = requested_linked_armfortas_artifacts(requested); |
| 1583 | let linked_backend = tools.armfortas_adapters(); |
| 1584 | if !linked_only_artifacts.is_empty() && linked_backend.capture_mode_name() == "unavailable" { |
| 1585 | let detail = format!( |
| 1586 | "linked armfortas capture is unavailable in this build; requested {}; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary", |
| 1587 | linked_only_artifacts.join(", ") |
| 1588 | ); |
| 1589 | return Ok(CompilerObservation { |
| 1590 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1591 | program: program.to_path_buf(), |
| 1592 | opt_level, |
| 1593 | compile_exit_code: 1, |
| 1594 | artifacts: BTreeMap::from([( |
| 1595 | ArtifactKey::Diagnostics, |
| 1596 | ArtifactValue::Text(detail.clone()), |
| 1597 | )]), |
| 1598 | provenance: ObservationProvenance { |
| 1599 | compiler_identity: "armfortas".into(), |
| 1600 | adapter_kind: "named".into(), |
| 1601 | backend_mode: linked_backend.capture_mode_name().into(), |
| 1602 | backend_detail: linked_backend.capture_description().into(), |
| 1603 | artifacts_captured: vec!["diagnostics".into()], |
| 1604 | comparison_basis: None, |
| 1605 | failure_stage: None, |
| 1606 | }, |
| 1607 | }); |
| 1608 | } |
| 1609 | let cli_observable_only = requested.iter().all(|artifact| { |
| 1610 | matches!( |
| 1611 | artifact, |
| 1612 | ArtifactKey::Diagnostics |
| 1613 | | ArtifactKey::Runtime |
| 1614 | | ArtifactKey::Stdout |
| 1615 | | ArtifactKey::Stderr |
| 1616 | | ArtifactKey::ExitCode |
| 1617 | | ArtifactKey::Asm |
| 1618 | | ArtifactKey::Obj |
| 1619 | | ArtifactKey::Executable |
| 1620 | ) |
| 1621 | }); |
| 1622 | let (backend_mode, backend_detail, capture) = if cli_observable_only |
| 1623 | && matches!(tools.armfortas, ArmfortasCliAdapter::External(_)) |
| 1624 | { |
| 1625 | let backend = tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level)); |
| 1626 | let detail = backend.description().to_string(); |
| 1627 | let mode = backend.mode_name().to_string(); |
| 1628 | let request = CaptureRequest { |
| 1629 | input: program.to_path_buf(), |
| 1630 | requested: stages.clone(), |
| 1631 | opt_level, |
| 1632 | }; |
| 1633 | (mode, detail, backend.capture(&request)) |
| 1634 | } else { |
| 1635 | let backend = linked_backend; |
| 1636 | let detail = backend.capture_description().to_string(); |
| 1637 | let mode = backend.capture_mode_name().to_string(); |
| 1638 | let request = CaptureRequest { |
| 1639 | input: program.to_path_buf(), |
| 1640 | requested: stages.clone(), |
| 1641 | opt_level, |
| 1642 | }; |
| 1643 | (mode, detail, backend.capture(&request)) |
| 1644 | }; |
| 1645 | |
| 1646 | let mut artifacts = BTreeMap::new(); |
| 1647 | let mut compile_exit_code = 0; |
| 1648 | let mut failure_stage = None; |
| 1649 | match capture { |
| 1650 | Ok(result) => { |
| 1651 | for (stage, captured) in &result.stages { |
| 1652 | match (stage, captured) { |
| 1653 | (Stage::Asm, CapturedStage::Text(text)) |
| 1654 | if requested.contains(&ArtifactKey::Asm) => |
| 1655 | { |
| 1656 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1657 | } |
| 1658 | (Stage::Obj, CapturedStage::Text(text)) |
| 1659 | if requested.contains(&ArtifactKey::Obj) => |
| 1660 | { |
| 1661 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1662 | } |
| 1663 | (Stage::Run, CapturedStage::Run(run)) => { |
| 1664 | insert_run_artifacts(requested, run, &mut artifacts); |
| 1665 | } |
| 1666 | (stage, CapturedStage::Text(text)) => { |
| 1667 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1668 | if requested.contains(&key) { |
| 1669 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1670 | } |
| 1671 | } |
| 1672 | _ => {} |
| 1673 | } |
| 1674 | } |
| 1675 | } |
| 1676 | Err(failure) => { |
| 1677 | compile_exit_code = 1; |
| 1678 | failure_stage = Some(failure.stage.as_str().to_string()); |
| 1679 | artifacts.insert( |
| 1680 | ArtifactKey::Diagnostics, |
| 1681 | ArtifactValue::Text(failure.detail.clone()), |
| 1682 | ); |
| 1683 | for (stage, captured) in &failure.stages { |
| 1684 | match (stage, captured) { |
| 1685 | (Stage::Asm, CapturedStage::Text(text)) |
| 1686 | if requested.contains(&ArtifactKey::Asm) => |
| 1687 | { |
| 1688 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1689 | } |
| 1690 | (Stage::Obj, CapturedStage::Text(text)) |
| 1691 | if requested.contains(&ArtifactKey::Obj) => |
| 1692 | { |
| 1693 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1694 | } |
| 1695 | (Stage::Run, CapturedStage::Run(run)) => { |
| 1696 | insert_run_artifacts(requested, run, &mut artifacts); |
| 1697 | } |
| 1698 | (stage, CapturedStage::Text(text)) => { |
| 1699 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1700 | if requested.contains(&key) { |
| 1701 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1702 | } |
| 1703 | } |
| 1704 | _ => {} |
| 1705 | } |
| 1706 | } |
| 1707 | } |
| 1708 | } |
| 1709 | |
| 1710 | if requested.contains(&ArtifactKey::Executable) && compile_exit_code == 0 { |
| 1711 | let temp_root = next_observation_temp_root("armfortas", opt_level); |
| 1712 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 1713 | format!( |
| 1714 | "cannot create introspection temp dir '{}': {}", |
| 1715 | temp_root.display(), |
| 1716 | e |
| 1717 | ) |
| 1718 | })?; |
| 1719 | let binary = temp_root.join("introspect.out"); |
| 1720 | tools |
| 1721 | .armfortas_adapters() |
| 1722 | .compile_output(program, opt_level, EmitMode::Binary, &binary) |
| 1723 | .map_err(|detail| { |
| 1724 | format!("failed to build armfortas executable artifact:\n{}", detail) |
| 1725 | })?; |
| 1726 | artifacts.insert(ArtifactKey::Executable, ArtifactValue::Path(binary)); |
| 1727 | } |
| 1728 | |
| 1729 | let artifacts_captured = artifacts |
| 1730 | .keys() |
| 1731 | .map(|artifact| artifact.as_str().to_string()) |
| 1732 | .collect::<Vec<_>>(); |
| 1733 | |
| 1734 | Ok(CompilerObservation { |
| 1735 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1736 | program: program.to_path_buf(), |
| 1737 | opt_level, |
| 1738 | compile_exit_code, |
| 1739 | artifacts, |
| 1740 | provenance: ObservationProvenance { |
| 1741 | compiler_identity: "armfortas".into(), |
| 1742 | adapter_kind: "named".into(), |
| 1743 | backend_mode, |
| 1744 | backend_detail, |
| 1745 | artifacts_captured, |
| 1746 | comparison_basis: None, |
| 1747 | failure_stage, |
| 1748 | }, |
| 1749 | }) |
| 1750 | } |
| 1751 | |
| 1752 | #[derive(Debug, Clone)] |
| 1753 | struct DriverCompileResult { |
| 1754 | command: String, |
| 1755 | exit_code: i32, |
| 1756 | stdout: String, |
| 1757 | stderr: String, |
| 1758 | output: PathBuf, |
| 1759 | } |
| 1760 | |
| 1761 | fn observe_external_driver( |
| 1762 | spec: &CompilerSpec, |
| 1763 | binary: &str, |
| 1764 | program: &Path, |
| 1765 | opt_level: OptLevel, |
| 1766 | requested: &BTreeSet<ArtifactKey>, |
| 1767 | uses_cpp: bool, |
| 1768 | adapter_kind: String, |
| 1769 | otool: &str, |
| 1770 | nm: &str, |
| 1771 | ) -> Result<CompilerObservation, String> { |
| 1772 | let temp_root = next_observation_temp_root(&spec.display_name(), opt_level); |
| 1773 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 1774 | format!( |
| 1775 | "cannot create observation temp dir '{}': {}", |
| 1776 | temp_root.display(), |
| 1777 | e |
| 1778 | ) |
| 1779 | })?; |
| 1780 | |
| 1781 | let needs_runtime = requested.contains(&ArtifactKey::Runtime) |
| 1782 | || requested.contains(&ArtifactKey::Stdout) |
| 1783 | || requested.contains(&ArtifactKey::Stderr) |
| 1784 | || requested.contains(&ArtifactKey::ExitCode) |
| 1785 | || requested.contains(&ArtifactKey::Executable); |
| 1786 | let primary_mode = if needs_runtime { |
| 1787 | DriverEmitMode::Binary |
| 1788 | } else if requested.contains(&ArtifactKey::Asm) { |
| 1789 | DriverEmitMode::Asm |
| 1790 | } else if requested.contains(&ArtifactKey::Obj) { |
| 1791 | DriverEmitMode::Obj |
| 1792 | } else { |
| 1793 | DriverEmitMode::Binary |
| 1794 | }; |
| 1795 | let primary_name = match primary_mode { |
| 1796 | DriverEmitMode::Binary => "observe.out", |
| 1797 | DriverEmitMode::Asm => "observe.s", |
| 1798 | DriverEmitMode::Obj => "observe.o", |
| 1799 | }; |
| 1800 | let primary = compile_with_external_driver( |
| 1801 | binary, |
| 1802 | program, |
| 1803 | opt_level, |
| 1804 | primary_mode, |
| 1805 | &temp_root.join(primary_name), |
| 1806 | uses_cpp, |
| 1807 | )?; |
| 1808 | |
| 1809 | let mut artifacts = BTreeMap::new(); |
| 1810 | if !primary.stdout.trim().is_empty() |
| 1811 | || !primary.stderr.trim().is_empty() |
| 1812 | || primary.exit_code != 0 |
| 1813 | { |
| 1814 | let diagnostics = [primary.stdout.trim_end(), primary.stderr.trim_end()] |
| 1815 | .iter() |
| 1816 | .filter(|part| !part.is_empty()) |
| 1817 | .copied() |
| 1818 | .collect::<Vec<_>>() |
| 1819 | .join("\n"); |
| 1820 | artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 1821 | } |
| 1822 | |
| 1823 | let mut compile_exit_code = primary.exit_code; |
| 1824 | if primary.exit_code == 0 { |
| 1825 | match primary_mode { |
| 1826 | DriverEmitMode::Binary => { |
| 1827 | if requested.contains(&ArtifactKey::Executable) { |
| 1828 | artifacts.insert( |
| 1829 | ArtifactKey::Executable, |
| 1830 | ArtifactValue::Path(primary.output.clone()), |
| 1831 | ); |
| 1832 | } |
| 1833 | if needs_runtime { |
| 1834 | let run_command = render_binary_run_command(&primary.output); |
| 1835 | let run = run_binary_capture(&primary.output, &temp_root, &run_command) |
| 1836 | .map_err(|detail| format!("build: {}\n{}", primary.command, detail))?; |
| 1837 | insert_run_artifacts(requested, &run, &mut artifacts); |
| 1838 | } |
| 1839 | } |
| 1840 | DriverEmitMode::Asm if requested.contains(&ArtifactKey::Asm) => { |
| 1841 | artifacts.insert( |
| 1842 | ArtifactKey::Asm, |
| 1843 | ArtifactValue::Text(fs::read_to_string(&primary.output).map_err(|e| { |
| 1844 | format!( |
| 1845 | "cannot read asm artifact '{}': {}", |
| 1846 | primary.output.display(), |
| 1847 | e |
| 1848 | ) |
| 1849 | })?), |
| 1850 | ); |
| 1851 | } |
| 1852 | DriverEmitMode::Obj if requested.contains(&ArtifactKey::Obj) => { |
| 1853 | artifacts.insert( |
| 1854 | ArtifactKey::Obj, |
| 1855 | ArtifactValue::Text( |
| 1856 | object_snapshot_text(&primary.output, otool, nm) |
| 1857 | .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1858 | ), |
| 1859 | ); |
| 1860 | } |
| 1861 | _ => {} |
| 1862 | } |
| 1863 | |
| 1864 | if requested.contains(&ArtifactKey::Asm) && primary_mode != DriverEmitMode::Asm { |
| 1865 | let asm = compile_with_external_driver( |
| 1866 | binary, |
| 1867 | program, |
| 1868 | opt_level, |
| 1869 | DriverEmitMode::Asm, |
| 1870 | &temp_root.join("observe-extra.s"), |
| 1871 | uses_cpp, |
| 1872 | )?; |
| 1873 | if asm.exit_code != 0 { |
| 1874 | compile_exit_code = asm.exit_code; |
| 1875 | artifacts.insert( |
| 1876 | ArtifactKey::Diagnostics, |
| 1877 | ArtifactValue::Text(asm.stderr.trim_end().to_string()), |
| 1878 | ); |
| 1879 | } else { |
| 1880 | artifacts.insert( |
| 1881 | ArtifactKey::Asm, |
| 1882 | ArtifactValue::Text(fs::read_to_string(&asm.output).map_err(|e| { |
| 1883 | format!("cannot read asm artifact '{}': {}", asm.output.display(), e) |
| 1884 | })?), |
| 1885 | ); |
| 1886 | } |
| 1887 | } |
| 1888 | |
| 1889 | if requested.contains(&ArtifactKey::Obj) && primary_mode != DriverEmitMode::Obj { |
| 1890 | let obj = compile_with_external_driver( |
| 1891 | binary, |
| 1892 | program, |
| 1893 | opt_level, |
| 1894 | DriverEmitMode::Obj, |
| 1895 | &temp_root.join("observe-extra.o"), |
| 1896 | uses_cpp, |
| 1897 | )?; |
| 1898 | if obj.exit_code != 0 { |
| 1899 | compile_exit_code = obj.exit_code; |
| 1900 | artifacts.insert( |
| 1901 | ArtifactKey::Diagnostics, |
| 1902 | ArtifactValue::Text(obj.stderr.trim_end().to_string()), |
| 1903 | ); |
| 1904 | } else { |
| 1905 | artifacts.insert( |
| 1906 | ArtifactKey::Obj, |
| 1907 | ArtifactValue::Text( |
| 1908 | object_snapshot_text(&obj.output, otool, nm) |
| 1909 | .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1910 | ), |
| 1911 | ); |
| 1912 | } |
| 1913 | } |
| 1914 | } |
| 1915 | |
| 1916 | let artifacts_captured = artifacts |
| 1917 | .keys() |
| 1918 | .map(|artifact| artifact.as_str().to_string()) |
| 1919 | .collect::<Vec<_>>(); |
| 1920 | |
| 1921 | Ok(CompilerObservation { |
| 1922 | compiler: spec.clone(), |
| 1923 | program: program.to_path_buf(), |
| 1924 | opt_level, |
| 1925 | compile_exit_code, |
| 1926 | artifacts, |
| 1927 | provenance: ObservationProvenance { |
| 1928 | compiler_identity: spec.display_name(), |
| 1929 | adapter_kind, |
| 1930 | backend_mode: "external-driver".into(), |
| 1931 | backend_detail: format!("generic external driver adapter using {}", binary), |
| 1932 | artifacts_captured, |
| 1933 | comparison_basis: None, |
| 1934 | failure_stage: None, |
| 1935 | }, |
| 1936 | }) |
| 1937 | } |
| 1938 | |
| 1939 | fn compile_with_external_driver( |
| 1940 | binary: &str, |
| 1941 | source: &Path, |
| 1942 | opt_level: OptLevel, |
| 1943 | mode: DriverEmitMode, |
| 1944 | output: &Path, |
| 1945 | uses_cpp: bool, |
| 1946 | ) -> Result<DriverCompileResult, String> { |
| 1947 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 1948 | if uses_cpp { |
| 1949 | args.push("-cpp".to_string()); |
| 1950 | } |
| 1951 | match mode { |
| 1952 | DriverEmitMode::Asm => args.push("-S".to_string()), |
| 1953 | DriverEmitMode::Obj => args.push("-c".to_string()), |
| 1954 | DriverEmitMode::Binary => {} |
| 1955 | } |
| 1956 | args.push(source.display().to_string()); |
| 1957 | args.push("-o".to_string()); |
| 1958 | args.push(output.display().to_string()); |
| 1959 | let command = render_command(binary, &args); |
| 1960 | let output_result = Command::new(binary) |
| 1961 | .args(&args) |
| 1962 | .output() |
| 1963 | .map_err(|e| format!("cannot run '{}': {}", binary, e))?; |
| 1964 | Ok(DriverCompileResult { |
| 1965 | command, |
| 1966 | exit_code: output_result.status.code().unwrap_or(-1), |
| 1967 | stdout: String::from_utf8_lossy(&output_result.stdout).into_owned(), |
| 1968 | stderr: String::from_utf8_lossy(&output_result.stderr).into_owned(), |
| 1969 | output: output.to_path_buf(), |
| 1970 | }) |
| 1971 | } |
| 1972 | |
| 1973 | fn next_observation_temp_root(label: &str, opt_level: OptLevel) -> PathBuf { |
| 1974 | default_report_root().join(".tmp").join(format!( |
| 1975 | "observe_{}_{}", |
| 1976 | sanitize_component(label), |
| 1977 | next_report_suffix(opt_level) |
| 1978 | )) |
| 1979 | } |
| 1980 | |
| 1981 | fn armfortas_requested_stages( |
| 1982 | requested: &BTreeSet<ArtifactKey>, |
| 1983 | ) -> Result<BTreeSet<Stage>, String> { |
| 1984 | let mut stages = BTreeSet::new(); |
| 1985 | for artifact in requested { |
| 1986 | match artifact { |
| 1987 | ArtifactKey::Asm => { |
| 1988 | stages.insert(Stage::Asm); |
| 1989 | } |
| 1990 | ArtifactKey::Obj => { |
| 1991 | stages.insert(Stage::Obj); |
| 1992 | } |
| 1993 | ArtifactKey::Runtime |
| 1994 | | ArtifactKey::Stdout |
| 1995 | | ArtifactKey::Stderr |
| 1996 | | ArtifactKey::ExitCode => { |
| 1997 | stages.insert(Stage::Run); |
| 1998 | } |
| 1999 | ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 2000 | ArtifactKey::Extra(name) => { |
| 2001 | let suffix = name |
| 2002 | .strip_prefix("armfortas.") |
| 2003 | .ok_or_else(|| format!("unsupported adapter-specific artifact '{}'", name))?; |
| 2004 | let stage = Stage::parse(suffix) |
| 2005 | .ok_or_else(|| format!("unknown armfortas artifact '{}'", name))?; |
| 2006 | stages.insert(stage); |
| 2007 | } |
| 2008 | } |
| 2009 | } |
| 2010 | if stages.is_empty() { |
| 2011 | stages.insert(Stage::Run); |
| 2012 | } |
| 2013 | Ok(stages) |
| 2014 | } |
| 2015 | |
| 2016 | fn insert_run_artifacts( |
| 2017 | requested: &BTreeSet<ArtifactKey>, |
| 2018 | run: &RunCapture, |
| 2019 | artifacts: &mut BTreeMap<ArtifactKey, ArtifactValue>, |
| 2020 | ) { |
| 2021 | if requested.contains(&ArtifactKey::Runtime) { |
| 2022 | artifacts.insert(ArtifactKey::Runtime, ArtifactValue::Run(run.clone())); |
| 2023 | } |
| 2024 | if requested.contains(&ArtifactKey::Stdout) { |
| 2025 | artifacts.insert(ArtifactKey::Stdout, ArtifactValue::Text(run.stdout.clone())); |
| 2026 | } |
| 2027 | if requested.contains(&ArtifactKey::Stderr) { |
| 2028 | artifacts.insert(ArtifactKey::Stderr, ArtifactValue::Text(run.stderr.clone())); |
| 2029 | } |
| 2030 | if requested.contains(&ArtifactKey::ExitCode) { |
| 2031 | artifacts.insert(ArtifactKey::ExitCode, ArtifactValue::Int(run.exit_code)); |
| 2032 | } |
| 2033 | } |
| 2034 | |
| 2035 | fn compare_observations( |
| 2036 | mut left: CompilerObservation, |
| 2037 | mut right: CompilerObservation, |
| 2038 | requested: &BTreeSet<ArtifactKey>, |
| 2039 | ) -> ComparisonResult { |
| 2040 | let basis = format!( |
| 2041 | "compile-status, diagnostics, runtime{}", |
| 2042 | if requested.is_empty() { |
| 2043 | String::new() |
| 2044 | } else { |
| 2045 | let extras = requested |
| 2046 | .iter() |
| 2047 | .filter(|artifact| { |
| 2048 | !matches!(artifact, ArtifactKey::Diagnostics | ArtifactKey::Runtime) |
| 2049 | }) |
| 2050 | .map(|artifact| artifact.as_str().to_string()) |
| 2051 | .collect::<Vec<_>>(); |
| 2052 | if extras.is_empty() { |
| 2053 | String::new() |
| 2054 | } else { |
| 2055 | format!(", {}", extras.join(", ")) |
| 2056 | } |
| 2057 | } |
| 2058 | ); |
| 2059 | left.provenance.comparison_basis = Some(basis.clone()); |
| 2060 | right.provenance.comparison_basis = Some(basis.clone()); |
| 2061 | |
| 2062 | let mut differences = Vec::new(); |
| 2063 | if left.compile_exit_code != right.compile_exit_code { |
| 2064 | differences.push(ArtifactDifference { |
| 2065 | artifact: "compile-exit-code".into(), |
| 2066 | detail: format!( |
| 2067 | "{}: {}\n{}: {}", |
| 2068 | left.compiler.display_name(), |
| 2069 | left.compile_exit_code, |
| 2070 | right.compiler.display_name(), |
| 2071 | right.compile_exit_code |
| 2072 | ), |
| 2073 | }); |
| 2074 | } |
| 2075 | |
| 2076 | compare_artifact_text( |
| 2077 | &left, |
| 2078 | &right, |
| 2079 | &ArtifactKey::Diagnostics, |
| 2080 | "diagnostics", |
| 2081 | &mut differences, |
| 2082 | ); |
| 2083 | |
| 2084 | if left.compile_exit_code == 0 && right.compile_exit_code == 0 { |
| 2085 | if requested.contains(&ArtifactKey::Runtime) { |
| 2086 | compare_artifact_runtime(&left, &right, &mut differences); |
| 2087 | } |
| 2088 | for artifact in requested { |
| 2089 | match artifact { |
| 2090 | ArtifactKey::Diagnostics | ArtifactKey::Runtime => {} |
| 2091 | ArtifactKey::Stdout | ArtifactKey::Stderr | ArtifactKey::Asm | ArtifactKey::Obj => { |
| 2092 | compare_artifact_text( |
| 2093 | &left, |
| 2094 | &right, |
| 2095 | artifact, |
| 2096 | artifact.as_str(), |
| 2097 | &mut differences, |
| 2098 | ); |
| 2099 | } |
| 2100 | ArtifactKey::ExitCode => { |
| 2101 | compare_artifact_int(&left, &right, artifact, &mut differences) |
| 2102 | } |
| 2103 | ArtifactKey::Executable => { |
| 2104 | compare_artifact_path(&left, &right, artifact, &mut differences) |
| 2105 | } |
| 2106 | ArtifactKey::Extra(name) => { |
| 2107 | compare_artifact_text(&left, &right, artifact, name, &mut differences) |
| 2108 | } |
| 2109 | } |
| 2110 | } |
| 2111 | } |
| 2112 | |
| 2113 | ComparisonResult { |
| 2114 | left, |
| 2115 | right, |
| 2116 | basis, |
| 2117 | differences, |
| 2118 | } |
| 2119 | } |
| 2120 | |
| 2121 | fn compare_artifact_text( |
| 2122 | left: &CompilerObservation, |
| 2123 | right: &CompilerObservation, |
| 2124 | artifact: &ArtifactKey, |
| 2125 | label: &str, |
| 2126 | differences: &mut Vec<ArtifactDifference>, |
| 2127 | ) { |
| 2128 | let left_text = match left.artifacts.get(artifact) { |
| 2129 | Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2130 | _ => "", |
| 2131 | }; |
| 2132 | let right_text = match right.artifacts.get(artifact) { |
| 2133 | Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2134 | _ => "", |
| 2135 | }; |
| 2136 | if left_text != right_text { |
| 2137 | differences.push(ArtifactDifference { |
| 2138 | artifact: label.to_string(), |
| 2139 | detail: describe_text_difference( |
| 2140 | left_text, |
| 2141 | right_text, |
| 2142 | &left.compiler.display_name(), |
| 2143 | &right.compiler.display_name(), |
| 2144 | ), |
| 2145 | }); |
| 2146 | } |
| 2147 | } |
| 2148 | |
| 2149 | fn compare_artifact_runtime( |
| 2150 | left: &CompilerObservation, |
| 2151 | right: &CompilerObservation, |
| 2152 | differences: &mut Vec<ArtifactDifference>, |
| 2153 | ) { |
| 2154 | let left_run = match left.artifacts.get(&ArtifactKey::Runtime) { |
| 2155 | Some(ArtifactValue::Run(run)) => Some(run), |
| 2156 | _ => None, |
| 2157 | }; |
| 2158 | let right_run = match right.artifacts.get(&ArtifactKey::Runtime) { |
| 2159 | Some(ArtifactValue::Run(run)) => Some(run), |
| 2160 | _ => None, |
| 2161 | }; |
| 2162 | match (left_run, right_run) { |
| 2163 | (Some(left_run), Some(right_run)) => { |
| 2164 | if normalize_run_signature(left_run) != normalize_run_signature(right_run) { |
| 2165 | differences.push(ArtifactDifference { |
| 2166 | artifact: "runtime".into(), |
| 2167 | detail: describe_run_difference( |
| 2168 | left_run, |
| 2169 | right_run, |
| 2170 | &left.compiler.display_name(), |
| 2171 | &right.compiler.display_name(), |
| 2172 | ), |
| 2173 | }); |
| 2174 | } |
| 2175 | } |
| 2176 | _ => differences.push(ArtifactDifference { |
| 2177 | artifact: "runtime".into(), |
| 2178 | detail: "one side did not produce a runtime result".into(), |
| 2179 | }), |
| 2180 | } |
| 2181 | } |
| 2182 | |
| 2183 | fn compare_artifact_int( |
| 2184 | left: &CompilerObservation, |
| 2185 | right: &CompilerObservation, |
| 2186 | artifact: &ArtifactKey, |
| 2187 | differences: &mut Vec<ArtifactDifference>, |
| 2188 | ) { |
| 2189 | let left_value = match left.artifacts.get(artifact) { |
| 2190 | Some(ArtifactValue::Int(value)) => Some(*value), |
| 2191 | _ => None, |
| 2192 | }; |
| 2193 | let right_value = match right.artifacts.get(artifact) { |
| 2194 | Some(ArtifactValue::Int(value)) => Some(*value), |
| 2195 | _ => None, |
| 2196 | }; |
| 2197 | if left_value != right_value { |
| 2198 | differences.push(ArtifactDifference { |
| 2199 | artifact: artifact.as_str().to_string(), |
| 2200 | detail: format!( |
| 2201 | "{}: {:?}\n{}: {:?}", |
| 2202 | left.compiler.display_name(), |
| 2203 | left_value, |
| 2204 | right.compiler.display_name(), |
| 2205 | right_value |
| 2206 | ), |
| 2207 | }); |
| 2208 | } |
| 2209 | } |
| 2210 | |
| 2211 | fn compare_artifact_path( |
| 2212 | left: &CompilerObservation, |
| 2213 | right: &CompilerObservation, |
| 2214 | artifact: &ArtifactKey, |
| 2215 | differences: &mut Vec<ArtifactDifference>, |
| 2216 | ) { |
| 2217 | let left_path = match left.artifacts.get(artifact) { |
| 2218 | Some(ArtifactValue::Path(path)) => Some(path), |
| 2219 | _ => None, |
| 2220 | }; |
| 2221 | let right_path = match right.artifacts.get(artifact) { |
| 2222 | Some(ArtifactValue::Path(path)) => Some(path), |
| 2223 | _ => None, |
| 2224 | }; |
| 2225 | |
| 2226 | match (left_path, right_path) { |
| 2227 | (Some(left_path), Some(right_path)) => { |
| 2228 | let left_bytes = fs::read(left_path); |
| 2229 | let right_bytes = fs::read(right_path); |
| 2230 | match (left_bytes, right_bytes) { |
| 2231 | (Ok(left_bytes), Ok(right_bytes)) => { |
| 2232 | if left_bytes != right_bytes { |
| 2233 | differences.push(ArtifactDifference { |
| 2234 | artifact: artifact.as_str().to_string(), |
| 2235 | detail: describe_binary_difference( |
| 2236 | &left_bytes, |
| 2237 | &right_bytes, |
| 2238 | &left.compiler.display_name(), |
| 2239 | &right.compiler.display_name(), |
| 2240 | left_path, |
| 2241 | right_path, |
| 2242 | ), |
| 2243 | }); |
| 2244 | } |
| 2245 | } |
| 2246 | (Err(left_err), Err(right_err)) => { |
| 2247 | differences.push(ArtifactDifference { |
| 2248 | artifact: artifact.as_str().to_string(), |
| 2249 | detail: format!( |
| 2250 | "{}: unable to read '{}': {}\n{}: unable to read '{}': {}", |
| 2251 | left.compiler.display_name(), |
| 2252 | left_path.display(), |
| 2253 | left_err, |
| 2254 | right.compiler.display_name(), |
| 2255 | right_path.display(), |
| 2256 | right_err |
| 2257 | ), |
| 2258 | }); |
| 2259 | } |
| 2260 | (Err(left_err), Ok(_)) => { |
| 2261 | differences.push(ArtifactDifference { |
| 2262 | artifact: artifact.as_str().to_string(), |
| 2263 | detail: format!( |
| 2264 | "{}: unable to read '{}': {}\n{}: readable '{}'", |
| 2265 | left.compiler.display_name(), |
| 2266 | left_path.display(), |
| 2267 | left_err, |
| 2268 | right.compiler.display_name(), |
| 2269 | right_path.display() |
| 2270 | ), |
| 2271 | }); |
| 2272 | } |
| 2273 | (Ok(_), Err(right_err)) => { |
| 2274 | differences.push(ArtifactDifference { |
| 2275 | artifact: artifact.as_str().to_string(), |
| 2276 | detail: format!( |
| 2277 | "{}: readable '{}'\n{}: unable to read '{}': {}", |
| 2278 | left.compiler.display_name(), |
| 2279 | left_path.display(), |
| 2280 | right.compiler.display_name(), |
| 2281 | right_path.display(), |
| 2282 | right_err |
| 2283 | ), |
| 2284 | }); |
| 2285 | } |
| 2286 | } |
| 2287 | } |
| 2288 | _ => { |
| 2289 | let left_value = left_path.map(|path| path.display().to_string()); |
| 2290 | let right_value = right_path.map(|path| path.display().to_string()); |
| 2291 | if left_value != right_value { |
| 2292 | differences.push(ArtifactDifference { |
| 2293 | artifact: artifact.as_str().to_string(), |
| 2294 | detail: format!( |
| 2295 | "{}: {:?}\n{}: {:?}", |
| 2296 | left.compiler.display_name(), |
| 2297 | left_value, |
| 2298 | right.compiler.display_name(), |
| 2299 | right_value |
| 2300 | ), |
| 2301 | }); |
| 2302 | } |
| 2303 | } |
| 2304 | } |
| 2305 | } |
| 2306 | |
| 2307 | fn describe_binary_difference( |
| 2308 | left: &[u8], |
| 2309 | right: &[u8], |
| 2310 | left_label: &str, |
| 2311 | right_label: &str, |
| 2312 | left_path: &Path, |
| 2313 | right_path: &Path, |
| 2314 | ) -> String { |
| 2315 | let shared = left.len().min(right.len()); |
| 2316 | for index in 0..shared { |
| 2317 | if left[index] != right[index] { |
| 2318 | return format!( |
| 2319 | "first differing byte: {}\n{}: {} bytes ({})\n{}: 0x{:02x}\n{}: {} bytes ({})\n{}: 0x{:02x}", |
| 2320 | index, |
| 2321 | left_label, |
| 2322 | left.len(), |
| 2323 | left_path.display(), |
| 2324 | left_label, |
| 2325 | left[index], |
| 2326 | right_label, |
| 2327 | right.len(), |
| 2328 | right_path.display(), |
| 2329 | right_label, |
| 2330 | right[index] |
| 2331 | ); |
| 2332 | } |
| 2333 | } |
| 2334 | |
| 2335 | format!( |
| 2336 | "binary length differs\n{}: {} bytes ({})\n{}: {} bytes ({})", |
| 2337 | left_label, |
| 2338 | left.len(), |
| 2339 | left_path.display(), |
| 2340 | right_label, |
| 2341 | right.len(), |
| 2342 | right_path.display() |
| 2343 | ) |
| 2344 | } |
| 2345 | |
| 2346 | fn compare_status(result: &ComparisonResult) -> &'static str { |
| 2347 | if result.differences.is_empty() { |
| 2348 | "match" |
| 2349 | } else { |
| 2350 | "diff" |
| 2351 | } |
| 2352 | } |
| 2353 | |
| 2354 | fn compare_classification(result: &ComparisonResult) -> &'static str { |
| 2355 | if result.differences.is_empty() { |
| 2356 | return "match"; |
| 2357 | } |
| 2358 | |
| 2359 | let mut has_compile = false; |
| 2360 | let mut has_diagnostics = false; |
| 2361 | let mut has_runtime = false; |
| 2362 | let mut has_artifact = false; |
| 2363 | |
| 2364 | for difference in &result.differences { |
| 2365 | match difference.artifact.as_str() { |
| 2366 | "compile-exit-code" => has_compile = true, |
| 2367 | "diagnostics" => has_diagnostics = true, |
| 2368 | "runtime" => has_runtime = true, |
| 2369 | _ => has_artifact = true, |
| 2370 | } |
| 2371 | } |
| 2372 | |
| 2373 | if has_compile { |
| 2374 | if has_runtime || has_artifact { |
| 2375 | "mixed divergence" |
| 2376 | } else { |
| 2377 | "compile divergence" |
| 2378 | } |
| 2379 | } else if has_runtime && !has_diagnostics && !has_artifact { |
| 2380 | "runtime divergence" |
| 2381 | } else if has_artifact && !has_runtime && !has_diagnostics { |
| 2382 | "artifact divergence" |
| 2383 | } else if has_diagnostics && !has_runtime && !has_artifact { |
| 2384 | "diagnostics divergence" |
| 2385 | } else { |
| 2386 | "mixed divergence" |
| 2387 | } |
| 2388 | } |
| 2389 | |
| 2390 | fn compare_changed_artifacts(result: &ComparisonResult) -> Vec<String> { |
| 2391 | result |
| 2392 | .differences |
| 2393 | .iter() |
| 2394 | .map(|difference| difference.artifact.clone()) |
| 2395 | .collect() |
| 2396 | } |
| 2397 | |
| 2398 | fn render_compare_text(result: &ComparisonResult) -> String { |
| 2399 | let changed_artifacts = compare_changed_artifacts(result); |
| 2400 | let mut lines = vec![ |
| 2401 | "Compare".to_string(), |
| 2402 | format!(" left: {}", result.left.compiler.display_name()), |
| 2403 | format!(" right: {}", result.right.compiler.display_name()), |
| 2404 | format!(" program: {}", result.left.program.display()), |
| 2405 | format!(" opt: {}", result.left.opt_level.as_str()), |
| 2406 | format!(" status: {}", compare_status(result)), |
| 2407 | format!(" classification: {}", compare_classification(result)), |
| 2408 | format!(" basis: {}", result.basis), |
| 2409 | format!(" difference_count: {}", result.differences.len()), |
| 2410 | format!( |
| 2411 | " changed_artifacts: {}", |
| 2412 | if changed_artifacts.is_empty() { |
| 2413 | "none".to_string() |
| 2414 | } else { |
| 2415 | changed_artifacts.join(", ") |
| 2416 | } |
| 2417 | ), |
| 2418 | format!( |
| 2419 | " left_backend: {} ({})", |
| 2420 | result.left.provenance.backend_mode, result.left.provenance.backend_detail |
| 2421 | ), |
| 2422 | format!( |
| 2423 | " right_backend: {} ({})", |
| 2424 | result.right.provenance.backend_mode, result.right.provenance.backend_detail |
| 2425 | ), |
| 2426 | ]; |
| 2427 | |
| 2428 | if result.differences.is_empty() { |
| 2429 | lines.push(String::new()); |
| 2430 | lines.push("No differences detected.".to_string()); |
| 2431 | } else { |
| 2432 | for difference in &result.differences { |
| 2433 | lines.push(String::new()); |
| 2434 | lines.push(format!("== {} ==", difference.artifact)); |
| 2435 | lines.push(difference.detail.clone()); |
| 2436 | } |
| 2437 | } |
| 2438 | |
| 2439 | lines.join("\n") |
| 2440 | } |
| 2441 | |
| 2442 | fn print_compare_result(result: &ComparisonResult) { |
| 2443 | println!("{}", render_compare_text(result)); |
| 2444 | } |
| 2445 | |
| 2446 | fn print_introspection(config: &IntrospectConfig, observed: &ObservedProgram) { |
| 2447 | println!( |
| 2448 | "{}", |
| 2449 | render_introspection_text( |
| 2450 | observed, |
| 2451 | IntrospectionRenderConfig { |
| 2452 | summary_only: config.summary_only, |
| 2453 | max_artifact_lines: config.max_artifact_lines, |
| 2454 | } |
| 2455 | ) |
| 2456 | ); |
| 2457 | } |
| 2458 | |
| 2459 | fn write_compare_reports(config: &CompareConfig, result: &ComparisonResult) -> Result<(), String> { |
| 2460 | if let Some(path) = &config.json_report { |
| 2461 | write_report(path, &render_compare_json(result), "json report")?; |
| 2462 | println!("json report: {}", path.display()); |
| 2463 | } |
| 2464 | if let Some(path) = &config.markdown_report { |
| 2465 | write_report(path, &render_compare_markdown(result), "markdown report")?; |
| 2466 | println!("markdown report: {}", path.display()); |
| 2467 | } |
| 2468 | Ok(()) |
| 2469 | } |
| 2470 | |
| 2471 | fn write_introspection_reports( |
| 2472 | config: &IntrospectConfig, |
| 2473 | observed: &ObservedProgram, |
| 2474 | ) -> Result<(), String> { |
| 2475 | let render_config = IntrospectionRenderConfig { |
| 2476 | summary_only: config.summary_only, |
| 2477 | max_artifact_lines: config.max_artifact_lines, |
| 2478 | }; |
| 2479 | if let Some(path) = &config.json_report { |
| 2480 | write_report(path, &render_introspection_json(observed), "json report")?; |
| 2481 | println!("json report: {}", path.display()); |
| 2482 | } |
| 2483 | if let Some(path) = &config.markdown_report { |
| 2484 | write_report( |
| 2485 | path, |
| 2486 | &render_introspection_markdown(observed, render_config), |
| 2487 | "markdown report", |
| 2488 | )?; |
| 2489 | println!("markdown report: {}", path.display()); |
| 2490 | } |
| 2491 | Ok(()) |
| 2492 | } |
| 2493 | |
| 2494 | fn introspection_status(observation: &CompilerObservation) -> &'static str { |
| 2495 | if observation.compile_exit_code == 0 { |
| 2496 | "compile ok" |
| 2497 | } else { |
| 2498 | "compile failed" |
| 2499 | } |
| 2500 | } |
| 2501 | |
| 2502 | fn diagnostic_excerpt(observation: &CompilerObservation) -> Option<String> { |
| 2503 | match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 2504 | Some(ArtifactValue::Text(text)) => text |
| 2505 | .lines() |
| 2506 | .map(str::trim) |
| 2507 | .find(|line| !line.is_empty()) |
| 2508 | .map(|line| line.to_string()), |
| 2509 | _ => None, |
| 2510 | } |
| 2511 | } |
| 2512 | |
| 2513 | fn failure_stage_summary(observation: &CompilerObservation) -> &str { |
| 2514 | observation |
| 2515 | .provenance |
| 2516 | .failure_stage |
| 2517 | .as_deref() |
| 2518 | .unwrap_or("none") |
| 2519 | } |
| 2520 | |
| 2521 | fn requested_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2522 | observed |
| 2523 | .requested_artifacts |
| 2524 | .iter() |
| 2525 | .map(|artifact| artifact.as_str().to_string()) |
| 2526 | .collect() |
| 2527 | } |
| 2528 | |
| 2529 | fn missing_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2530 | observed |
| 2531 | .requested_artifacts |
| 2532 | .iter() |
| 2533 | .filter(|artifact| { |
| 2534 | if matches!(artifact, ArtifactKey::Diagnostics) |
| 2535 | && observed.observation.compile_exit_code == 0 |
| 2536 | && !observed.observation.artifacts.contains_key(*artifact) |
| 2537 | { |
| 2538 | return false; |
| 2539 | } |
| 2540 | !observed.observation.artifacts.contains_key(*artifact) |
| 2541 | }) |
| 2542 | .map(|artifact| artifact.as_str().to_string()) |
| 2543 | .collect() |
| 2544 | } |
| 2545 | |
| 2546 | fn observation_generic_artifacts<'a>( |
| 2547 | observation: &'a CompilerObservation, |
| 2548 | ) -> Vec<(String, &'a ArtifactValue)> { |
| 2549 | observation |
| 2550 | .artifacts |
| 2551 | .iter() |
| 2552 | .filter(|(artifact, _)| artifact.is_generic()) |
| 2553 | .map(|(artifact, value)| (artifact.as_str().to_string(), value)) |
| 2554 | .collect() |
| 2555 | } |
| 2556 | |
| 2557 | fn observation_adapter_extras<'a>( |
| 2558 | observation: &'a CompilerObservation, |
| 2559 | ) -> BTreeMap<String, Vec<(String, &'a ArtifactValue)>> { |
| 2560 | let mut extras = BTreeMap::new(); |
| 2561 | for (artifact, value) in &observation.artifacts { |
| 2562 | if let ArtifactKey::Extra(name) = artifact { |
| 2563 | let (namespace, local_name) = artifact |
| 2564 | .extra_parts() |
| 2565 | .map(|(namespace, local_name)| (namespace.to_string(), local_name.to_string())) |
| 2566 | .unwrap_or_else(|| ("extra".to_string(), name.clone())); |
| 2567 | extras |
| 2568 | .entry(namespace) |
| 2569 | .or_insert_with(Vec::new) |
| 2570 | .push((local_name, value)); |
| 2571 | } |
| 2572 | } |
| 2573 | extras |
| 2574 | } |
| 2575 | |
| 2576 | fn format_artifact_name_list(names: &[String]) -> String { |
| 2577 | if names.is_empty() { |
| 2578 | "none".to_string() |
| 2579 | } else { |
| 2580 | names.join(", ") |
| 2581 | } |
| 2582 | } |
| 2583 | |
| 2584 | fn format_adapter_extra_summary( |
| 2585 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2586 | ) -> String { |
| 2587 | if extras.is_empty() { |
| 2588 | return "none".to_string(); |
| 2589 | } |
| 2590 | |
| 2591 | extras |
| 2592 | .iter() |
| 2593 | .map(|(namespace, entries)| { |
| 2594 | format!( |
| 2595 | "{}({})", |
| 2596 | namespace, |
| 2597 | entries |
| 2598 | .iter() |
| 2599 | .map(|(name, _)| name.as_str()) |
| 2600 | .collect::<Vec<_>>() |
| 2601 | .join(", ") |
| 2602 | ) |
| 2603 | }) |
| 2604 | .collect::<Vec<_>>() |
| 2605 | .join(", ") |
| 2606 | } |
| 2607 | |
| 2608 | fn render_named_artifact_map_json(entries: &[(String, &ArtifactValue)]) -> String { |
| 2609 | let mut rendered = String::from("{"); |
| 2610 | for (index, (name, value)) in entries.iter().enumerate() { |
| 2611 | if index > 0 { |
| 2612 | rendered.push_str(", "); |
| 2613 | } |
| 2614 | rendered.push_str(&format!( |
| 2615 | "\"{}\": {}", |
| 2616 | json_escape(name), |
| 2617 | render_artifact_value_json(value) |
| 2618 | )); |
| 2619 | } |
| 2620 | rendered.push('}'); |
| 2621 | rendered |
| 2622 | } |
| 2623 | |
| 2624 | fn render_flat_artifacts_json(observation: &CompilerObservation) -> String { |
| 2625 | let mut entries = Vec::new(); |
| 2626 | for (artifact, value) in &observation.artifacts { |
| 2627 | entries.push((artifact.as_str().to_string(), value)); |
| 2628 | } |
| 2629 | render_named_artifact_map_json(&entries) |
| 2630 | } |
| 2631 | |
| 2632 | fn render_artifact_summary_json(value: &ArtifactValue) -> String { |
| 2633 | match value { |
| 2634 | ArtifactValue::Text(text) => format!( |
| 2635 | "{{\"kind\":\"text\",\"summary\":\"{}\",\"line_count\":{},\"char_count\":{}}}", |
| 2636 | json_escape(&artifact_value_summary(value)), |
| 2637 | text_line_count(text), |
| 2638 | text.len() |
| 2639 | ), |
| 2640 | ArtifactValue::Int(number) => format!( |
| 2641 | "{{\"kind\":\"int\",\"summary\":\"{}\",\"value\":{}}}", |
| 2642 | json_escape(&artifact_value_summary(value)), |
| 2643 | number |
| 2644 | ), |
| 2645 | ArtifactValue::Run(run) => format!( |
| 2646 | "{{\"kind\":\"runtime\",\"summary\":\"{}\",\"exit_code\":{},\"stdout_lines\":{},\"stderr_lines\":{}}}", |
| 2647 | json_escape(&artifact_value_summary(value)), |
| 2648 | run.exit_code, |
| 2649 | text_line_count(&run.stdout), |
| 2650 | text_line_count(&run.stderr) |
| 2651 | ), |
| 2652 | ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2653 | Ok(metadata) => format!( |
| 2654 | "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":{}}}", |
| 2655 | json_escape(&artifact_value_summary(value)), |
| 2656 | metadata.len() |
| 2657 | ), |
| 2658 | Err(_) => format!( |
| 2659 | "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":null}}", |
| 2660 | json_escape(&artifact_value_summary(value)) |
| 2661 | ), |
| 2662 | }, |
| 2663 | } |
| 2664 | } |
| 2665 | |
| 2666 | fn render_artifact_summaries_json(observation: &CompilerObservation) -> String { |
| 2667 | let mut rendered = String::from("{"); |
| 2668 | for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 2669 | if index > 0 { |
| 2670 | rendered.push_str(", "); |
| 2671 | } |
| 2672 | rendered.push_str(&format!( |
| 2673 | "\"{}\": {}", |
| 2674 | json_escape(artifact.as_str()), |
| 2675 | render_artifact_summary_json(value) |
| 2676 | )); |
| 2677 | } |
| 2678 | rendered.push('}'); |
| 2679 | rendered |
| 2680 | } |
| 2681 | |
| 2682 | fn render_adapter_extra_summary_json( |
| 2683 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2684 | ) -> String { |
| 2685 | let mut rendered = String::from("{"); |
| 2686 | for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2687 | if index > 0 { |
| 2688 | rendered.push_str(", "); |
| 2689 | } |
| 2690 | let names = entries |
| 2691 | .iter() |
| 2692 | .map(|(name, _)| name.clone()) |
| 2693 | .collect::<Vec<_>>(); |
| 2694 | rendered.push_str(&format!( |
| 2695 | "\"{}\": {}", |
| 2696 | json_escape(namespace), |
| 2697 | json_string_array(&names) |
| 2698 | )); |
| 2699 | } |
| 2700 | rendered.push('}'); |
| 2701 | rendered |
| 2702 | } |
| 2703 | |
| 2704 | fn render_namespaced_artifacts_json( |
| 2705 | extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2706 | ) -> String { |
| 2707 | let mut rendered = String::from("{"); |
| 2708 | for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2709 | if index > 0 { |
| 2710 | rendered.push_str(", "); |
| 2711 | } |
| 2712 | rendered.push_str(&format!( |
| 2713 | "\"{}\": {}", |
| 2714 | json_escape(namespace), |
| 2715 | render_named_artifact_map_json(entries) |
| 2716 | )); |
| 2717 | } |
| 2718 | rendered.push('}'); |
| 2719 | rendered |
| 2720 | } |
| 2721 | |
| 2722 | fn text_line_count(text: &str) -> usize { |
| 2723 | if text.is_empty() { |
| 2724 | 0 |
| 2725 | } else { |
| 2726 | text.lines().count() |
| 2727 | } |
| 2728 | } |
| 2729 | |
| 2730 | fn render_config_summary(config: IntrospectionRenderConfig) -> String { |
| 2731 | if config.summary_only { |
| 2732 | "summary-only".to_string() |
| 2733 | } else if let Some(limit) = config.max_artifact_lines { |
| 2734 | format!("first {} lines per artifact", limit) |
| 2735 | } else { |
| 2736 | "full artifact bodies".to_string() |
| 2737 | } |
| 2738 | } |
| 2739 | |
| 2740 | fn artifact_value_summary(value: &ArtifactValue) -> String { |
| 2741 | match value { |
| 2742 | ArtifactValue::Text(text) => { |
| 2743 | format!( |
| 2744 | "text, {} lines, {} chars", |
| 2745 | text_line_count(text), |
| 2746 | text.len() |
| 2747 | ) |
| 2748 | } |
| 2749 | ArtifactValue::Int(value) => format!("int, value {}", value), |
| 2750 | ArtifactValue::Run(run) => format!( |
| 2751 | "runtime, exit {}, stdout {} lines, stderr {} lines", |
| 2752 | run.exit_code, |
| 2753 | text_line_count(&run.stdout), |
| 2754 | text_line_count(&run.stderr) |
| 2755 | ), |
| 2756 | ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2757 | Ok(metadata) => format!("path, {} bytes", metadata.len()), |
| 2758 | Err(_) => "path".to_string(), |
| 2759 | }, |
| 2760 | } |
| 2761 | } |
| 2762 | |
| 2763 | fn render_artifact_body_lines( |
| 2764 | value: &ArtifactValue, |
| 2765 | config: IntrospectionRenderConfig, |
| 2766 | ) -> Vec<String> { |
| 2767 | if config.summary_only { |
| 2768 | return vec!["[content omitted by --summary-only]".to_string()]; |
| 2769 | } |
| 2770 | |
| 2771 | let rendered = render_artifact_value_text(value); |
| 2772 | let mut lines = rendered |
| 2773 | .lines() |
| 2774 | .map(|line| line.to_string()) |
| 2775 | .collect::<Vec<_>>(); |
| 2776 | if lines.is_empty() { |
| 2777 | return vec!["<empty>".to_string()]; |
| 2778 | } |
| 2779 | |
| 2780 | if let Some(limit) = config.max_artifact_lines { |
| 2781 | if lines.len() > limit { |
| 2782 | let total = lines.len(); |
| 2783 | lines.truncate(limit); |
| 2784 | lines.push(format!( |
| 2785 | "... (truncated; showing first {} of {} lines)", |
| 2786 | limit, total |
| 2787 | )); |
| 2788 | } |
| 2789 | } |
| 2790 | |
| 2791 | lines |
| 2792 | } |
| 2793 | |
| 2794 | fn render_introspection_text( |
| 2795 | observed: &ObservedProgram, |
| 2796 | render_config: IntrospectionRenderConfig, |
| 2797 | ) -> String { |
| 2798 | let observation = &observed.observation; |
| 2799 | let generic_artifacts = observation_generic_artifacts(observation); |
| 2800 | let generic_names = generic_artifacts |
| 2801 | .iter() |
| 2802 | .map(|(name, _)| name.clone()) |
| 2803 | .collect::<Vec<_>>(); |
| 2804 | let adapter_extras = observation_adapter_extras(observation); |
| 2805 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2806 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2807 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2808 | let mut lines = vec![ |
| 2809 | "Introspect".to_string(), |
| 2810 | format!(" status: {}", introspection_status(observation)), |
| 2811 | format!(" compiler: {}", observation.compiler.display_name()), |
| 2812 | format!(" program: {}", observation.program.display()), |
| 2813 | format!(" opt: {}", observation.opt_level.as_str()), |
| 2814 | format!(" compile_exit_code: {}", observation.compile_exit_code), |
| 2815 | format!(" adapter_kind: {}", observation.provenance.adapter_kind), |
| 2816 | format!(" backend_mode: {}", observation.provenance.backend_mode), |
| 2817 | format!( |
| 2818 | " backend_detail: {}", |
| 2819 | observation.provenance.backend_detail |
| 2820 | ), |
| 2821 | format!(" failure_stage: {}", failure_stage_summary(observation)), |
| 2822 | format!( |
| 2823 | " diagnostic_excerpt: {}", |
| 2824 | diagnostic_excerpt |
| 2825 | .clone() |
| 2826 | .unwrap_or_else(|| "none".to_string()) |
| 2827 | ), |
| 2828 | format!(" content_mode: {}", render_config_summary(render_config)), |
| 2829 | format!(" artifact_count: {}", observation.artifacts.len()), |
| 2830 | format!( |
| 2831 | " requested_artifacts: {}", |
| 2832 | format_artifact_name_list(&requested_artifacts) |
| 2833 | ), |
| 2834 | format!( |
| 2835 | " missing_artifacts: {}", |
| 2836 | format_artifact_name_list(&missing_artifacts) |
| 2837 | ), |
| 2838 | format!( |
| 2839 | " generic_artifacts: {}", |
| 2840 | format_artifact_name_list(&generic_names) |
| 2841 | ), |
| 2842 | format!( |
| 2843 | " adapter_extras: {}", |
| 2844 | format_adapter_extra_summary(&adapter_extras) |
| 2845 | ), |
| 2846 | ]; |
| 2847 | if !observation.provenance.artifacts_captured.is_empty() { |
| 2848 | lines.push(format!( |
| 2849 | " captured_artifacts: {}", |
| 2850 | observation.provenance.artifacts_captured.join(", ") |
| 2851 | )); |
| 2852 | } |
| 2853 | |
| 2854 | if !generic_artifacts.is_empty() { |
| 2855 | lines.push(String::new()); |
| 2856 | lines.push("Generic artifacts".to_string()); |
| 2857 | for (artifact, value) in generic_artifacts { |
| 2858 | lines.push(String::new()); |
| 2859 | lines.push(format!("== {} ==", artifact)); |
| 2860 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2861 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 2862 | } |
| 2863 | } |
| 2864 | |
| 2865 | if !adapter_extras.is_empty() { |
| 2866 | lines.push(String::new()); |
| 2867 | lines.push("Adapter extras".to_string()); |
| 2868 | for (namespace, entries) in adapter_extras { |
| 2869 | lines.push(String::new()); |
| 2870 | lines.push(format!("-- {} --", namespace)); |
| 2871 | for (name, value) in entries { |
| 2872 | lines.push(String::new()); |
| 2873 | lines.push(format!("== {} ==", name)); |
| 2874 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2875 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 2876 | } |
| 2877 | } |
| 2878 | } |
| 2879 | lines.join("\n") |
| 2880 | } |
| 2881 | |
| 2882 | fn render_compare_json(result: &ComparisonResult) -> String { |
| 2883 | let changed_artifacts = compare_changed_artifacts(result); |
| 2884 | format!( |
| 2885 | "{{\n \"status\": \"{}\",\n \"classification\": \"{}\",\n \"difference_count\": {},\n \"changed_artifacts\": {},\n \"basis\": \"{}\",\n \"left\": {},\n \"right\": {},\n \"differences\": {}\n}}\n", |
| 2886 | compare_status(result), |
| 2887 | compare_classification(result), |
| 2888 | result.differences.len(), |
| 2889 | json_string_array(&changed_artifacts), |
| 2890 | json_escape(&result.basis), |
| 2891 | render_observation_json(&result.left), |
| 2892 | render_observation_json(&result.right), |
| 2893 | render_differences_json(&result.differences) |
| 2894 | ) |
| 2895 | } |
| 2896 | |
| 2897 | fn render_compare_markdown(result: &ComparisonResult) -> String { |
| 2898 | let changed_artifacts = compare_changed_artifacts(result); |
| 2899 | let mut lines = vec![ |
| 2900 | "# bencch compare report".to_string(), |
| 2901 | String::new(), |
| 2902 | format!("status: {}", compare_status(result)), |
| 2903 | format!("classification: {}", compare_classification(result)), |
| 2904 | format!( |
| 2905 | "compilers: `{}` vs `{}`", |
| 2906 | result.left.compiler.display_name(), |
| 2907 | result.right.compiler.display_name() |
| 2908 | ), |
| 2909 | format!("basis: {}", result.basis), |
| 2910 | format!("difference_count: {}", result.differences.len()), |
| 2911 | format!( |
| 2912 | "changed_artifacts: {}", |
| 2913 | if changed_artifacts.is_empty() { |
| 2914 | "none".to_string() |
| 2915 | } else { |
| 2916 | changed_artifacts.join(", ") |
| 2917 | } |
| 2918 | ), |
| 2919 | String::new(), |
| 2920 | "## Left".to_string(), |
| 2921 | render_observation_markdown(&result.left), |
| 2922 | String::new(), |
| 2923 | "## Right".to_string(), |
| 2924 | render_observation_markdown(&result.right), |
| 2925 | String::new(), |
| 2926 | "## Differences".to_string(), |
| 2927 | ]; |
| 2928 | if result.differences.is_empty() { |
| 2929 | lines.push("none".to_string()); |
| 2930 | } else { |
| 2931 | for difference in &result.differences { |
| 2932 | lines.push(String::new()); |
| 2933 | lines.push(format!("### `{}`", difference.artifact)); |
| 2934 | lines.push("```text".to_string()); |
| 2935 | lines.extend(difference.detail.lines().map(|line| line.to_string())); |
| 2936 | lines.push("```".to_string()); |
| 2937 | } |
| 2938 | } |
| 2939 | lines.join("\n") + "\n" |
| 2940 | } |
| 2941 | |
| 2942 | fn render_introspection_json(observed: &ObservedProgram) -> String { |
| 2943 | let observation = &observed.observation; |
| 2944 | let generic_artifacts = observation_generic_artifacts(observation); |
| 2945 | let generic_names = generic_artifacts |
| 2946 | .iter() |
| 2947 | .map(|(name, _)| name.clone()) |
| 2948 | .collect::<Vec<_>>(); |
| 2949 | let adapter_extras = observation_adapter_extras(observation); |
| 2950 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2951 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2952 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2953 | format!( |
| 2954 | "{{\n \"status\": \"{}\",\n \"compiler\": \"{}\",\n \"program\": \"{}\",\n \"opt\": \"{}\",\n \"compile_exit_code\": {},\n \"failure\": {{\n \"stage\": {},\n \"diagnostic_excerpt\": {}\n }},\n \"artifact_summary\": {{\n \"artifact_count\": {},\n \"requested_artifacts\": {},\n \"captured_artifacts\": {},\n \"missing_artifacts\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {}\n }},\n \"provenance\": {{\n \"compiler_identity\": \"{}\",\n \"adapter_kind\": \"{}\",\n \"backend_mode\": \"{}\",\n \"backend_detail\": \"{}\",\n \"artifacts_captured\": {},\n \"comparison_basis\": {},\n \"failure_stage\": {}\n }},\n \"artifact_summaries\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {},\n \"artifacts\": {}\n}}\n", |
| 2955 | json_escape(introspection_status(observation)), |
| 2956 | json_escape(&observation.compiler.display_name()), |
| 2957 | json_escape(&observation.program.display().to_string()), |
| 2958 | observation.opt_level.as_str(), |
| 2959 | observation.compile_exit_code, |
| 2960 | match &observation.provenance.failure_stage { |
| 2961 | Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2962 | None => "null".to_string(), |
| 2963 | }, |
| 2964 | match &diagnostic_excerpt { |
| 2965 | Some(line) => format!("\"{}\"", json_escape(line)), |
| 2966 | None => "null".to_string(), |
| 2967 | }, |
| 2968 | observation.artifacts.len(), |
| 2969 | json_string_array(&requested_artifacts), |
| 2970 | json_string_array(&observation.provenance.artifacts_captured), |
| 2971 | json_string_array(&missing_artifacts), |
| 2972 | json_string_array(&generic_names), |
| 2973 | render_adapter_extra_summary_json(&adapter_extras), |
| 2974 | json_escape(&observation.provenance.compiler_identity), |
| 2975 | json_escape(&observation.provenance.adapter_kind), |
| 2976 | json_escape(&observation.provenance.backend_mode), |
| 2977 | json_escape(&observation.provenance.backend_detail), |
| 2978 | json_string_array(&observation.provenance.artifacts_captured), |
| 2979 | match &observation.provenance.comparison_basis { |
| 2980 | Some(basis) => format!("\"{}\"", json_escape(basis)), |
| 2981 | None => "null".to_string(), |
| 2982 | }, |
| 2983 | match &observation.provenance.failure_stage { |
| 2984 | Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2985 | None => "null".to_string(), |
| 2986 | }, |
| 2987 | render_artifact_summaries_json(observation), |
| 2988 | render_named_artifact_map_json(&generic_artifacts), |
| 2989 | render_namespaced_artifacts_json(&adapter_extras), |
| 2990 | render_flat_artifacts_json(observation) |
| 2991 | ) |
| 2992 | } |
| 2993 | |
| 2994 | fn render_introspection_markdown( |
| 2995 | observed: &ObservedProgram, |
| 2996 | render_config: IntrospectionRenderConfig, |
| 2997 | ) -> String { |
| 2998 | let observation = &observed.observation; |
| 2999 | let generic_artifacts = observation_generic_artifacts(observation); |
| 3000 | let generic_names = generic_artifacts |
| 3001 | .iter() |
| 3002 | .map(|(name, _)| name.clone()) |
| 3003 | .collect::<Vec<_>>(); |
| 3004 | let adapter_extras = observation_adapter_extras(observation); |
| 3005 | let requested_artifacts = requested_introspection_artifact_names(observed); |
| 3006 | let missing_artifacts = missing_introspection_artifact_names(observed); |
| 3007 | let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 3008 | let mut lines = vec![ |
| 3009 | "# bencch introspect report".to_string(), |
| 3010 | String::new(), |
| 3011 | format!("status: {}", introspection_status(observation)), |
| 3012 | format!("compiler: `{}`", observation.compiler.display_name()), |
| 3013 | format!("program: `{}`", observation.program.display()), |
| 3014 | format!("opt: `{}`", observation.opt_level.as_str()), |
| 3015 | format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 3016 | format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 3017 | format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 3018 | format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3019 | format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3020 | format!( |
| 3021 | "diagnostic_excerpt: {}", |
| 3022 | diagnostic_excerpt |
| 3023 | .map(|line| format!("`{}`", line)) |
| 3024 | .unwrap_or_else(|| "none".to_string()) |
| 3025 | ), |
| 3026 | format!("content_mode: `{}`", render_config_summary(render_config)), |
| 3027 | format!("artifact_count: {}", observation.artifacts.len()), |
| 3028 | format!( |
| 3029 | "requested_artifacts: {}", |
| 3030 | if requested_artifacts.is_empty() { |
| 3031 | "none".to_string() |
| 3032 | } else { |
| 3033 | format!("`{}`", requested_artifacts.join("`, `")) |
| 3034 | } |
| 3035 | ), |
| 3036 | format!( |
| 3037 | "missing_artifacts: {}", |
| 3038 | if missing_artifacts.is_empty() { |
| 3039 | "none".to_string() |
| 3040 | } else { |
| 3041 | format!("`{}`", missing_artifacts.join("`, `")) |
| 3042 | } |
| 3043 | ), |
| 3044 | format!( |
| 3045 | "generic_artifacts: {}", |
| 3046 | if generic_names.is_empty() { |
| 3047 | "none".to_string() |
| 3048 | } else { |
| 3049 | format!("`{}`", generic_names.join("`, `")) |
| 3050 | } |
| 3051 | ), |
| 3052 | format!( |
| 3053 | "adapter_extras: {}", |
| 3054 | format_adapter_extra_summary(&adapter_extras) |
| 3055 | ), |
| 3056 | ]; |
| 3057 | if !observation.provenance.artifacts_captured.is_empty() { |
| 3058 | lines.push(format!( |
| 3059 | "captured_artifacts: `{}`", |
| 3060 | observation.provenance.artifacts_captured.join("`, `") |
| 3061 | )); |
| 3062 | } |
| 3063 | |
| 3064 | if !generic_artifacts.is_empty() { |
| 3065 | lines.push(String::new()); |
| 3066 | lines.push("## Generic artifacts".to_string()); |
| 3067 | for (artifact, value) in generic_artifacts { |
| 3068 | lines.push(String::new()); |
| 3069 | lines.push(format!("### `{}`", artifact)); |
| 3070 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3071 | lines.push("```text".to_string()); |
| 3072 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 3073 | lines.push("```".to_string()); |
| 3074 | } |
| 3075 | } |
| 3076 | |
| 3077 | if !adapter_extras.is_empty() { |
| 3078 | lines.push(String::new()); |
| 3079 | lines.push("## Adapter extras".to_string()); |
| 3080 | for (namespace, entries) in adapter_extras { |
| 3081 | lines.push(String::new()); |
| 3082 | lines.push(format!("### `{}`", namespace)); |
| 3083 | for (name, value) in entries { |
| 3084 | lines.push(String::new()); |
| 3085 | lines.push(format!("#### `{}`", name)); |
| 3086 | lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3087 | lines.push("```text".to_string()); |
| 3088 | lines.extend(render_artifact_body_lines(value, render_config)); |
| 3089 | lines.push("```".to_string()); |
| 3090 | } |
| 3091 | } |
| 3092 | } |
| 3093 | |
| 3094 | lines.join("\n") + "\n" |
| 3095 | } |
| 3096 | |
| 3097 | fn render_observation_json(observation: &CompilerObservation) -> String { |
| 3098 | let mut lines = vec![ |
| 3099 | "{".to_string(), |
| 3100 | format!( |
| 3101 | " \"compiler\": \"{}\",", |
| 3102 | json_escape(&observation.compiler.display_name()) |
| 3103 | ), |
| 3104 | format!( |
| 3105 | " \"program\": \"{}\",", |
| 3106 | json_escape(&observation.program.display().to_string()) |
| 3107 | ), |
| 3108 | format!(" \"opt\": \"{}\",", observation.opt_level.as_str()), |
| 3109 | format!( |
| 3110 | " \"compile_exit_code\": {},", |
| 3111 | observation.compile_exit_code |
| 3112 | ), |
| 3113 | " \"provenance\": {".to_string(), |
| 3114 | format!( |
| 3115 | " \"compiler_identity\": \"{}\",", |
| 3116 | json_escape(&observation.provenance.compiler_identity) |
| 3117 | ), |
| 3118 | format!( |
| 3119 | " \"adapter_kind\": \"{}\",", |
| 3120 | json_escape(&observation.provenance.adapter_kind) |
| 3121 | ), |
| 3122 | format!( |
| 3123 | " \"backend_mode\": \"{}\",", |
| 3124 | json_escape(&observation.provenance.backend_mode) |
| 3125 | ), |
| 3126 | format!( |
| 3127 | " \"backend_detail\": \"{}\",", |
| 3128 | json_escape(&observation.provenance.backend_detail) |
| 3129 | ), |
| 3130 | format!( |
| 3131 | " \"artifacts_captured\": {},", |
| 3132 | json_string_array(&observation.provenance.artifacts_captured) |
| 3133 | ), |
| 3134 | match &observation.provenance.failure_stage { |
| 3135 | Some(stage) => format!(" \"failure_stage\": \"{}\",", json_escape(stage)), |
| 3136 | None => " \"failure_stage\": null,".to_string(), |
| 3137 | }, |
| 3138 | match &observation.provenance.comparison_basis { |
| 3139 | Some(basis) => format!(" \"comparison_basis\": \"{}\"", json_escape(basis)), |
| 3140 | None => " \"comparison_basis\": null".to_string(), |
| 3141 | }, |
| 3142 | " },".to_string(), |
| 3143 | " \"artifacts\": {".to_string(), |
| 3144 | ]; |
| 3145 | for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 3146 | lines.push(format!( |
| 3147 | " \"{}\": {}{}", |
| 3148 | json_escape(artifact.as_str()), |
| 3149 | render_artifact_value_json(value), |
| 3150 | if index + 1 == observation.artifacts.len() { |
| 3151 | "" |
| 3152 | } else { |
| 3153 | "," |
| 3154 | } |
| 3155 | )); |
| 3156 | } |
| 3157 | lines.push(" }".to_string()); |
| 3158 | lines.push("}".to_string()); |
| 3159 | lines.join("\n") |
| 3160 | } |
| 3161 | |
| 3162 | fn render_observation_markdown(observation: &CompilerObservation) -> String { |
| 3163 | let mut lines = vec![ |
| 3164 | format!("compiler: `{}`", observation.compiler.display_name()), |
| 3165 | format!("program: `{}`", observation.program.display()), |
| 3166 | format!("opt: `{}`", observation.opt_level.as_str()), |
| 3167 | format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 3168 | format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 3169 | format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 3170 | format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3171 | format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3172 | ]; |
| 3173 | if !observation.provenance.artifacts_captured.is_empty() { |
| 3174 | lines.push(format!( |
| 3175 | "artifacts: `{}`", |
| 3176 | observation.provenance.artifacts_captured.join("`, `") |
| 3177 | )); |
| 3178 | } |
| 3179 | for (artifact, value) in &observation.artifacts { |
| 3180 | lines.push(String::new()); |
| 3181 | lines.push(format!("## `{}`", artifact.as_str())); |
| 3182 | lines.push("```text".to_string()); |
| 3183 | lines.extend( |
| 3184 | render_artifact_value_text(value) |
| 3185 | .lines() |
| 3186 | .map(|line| line.to_string()), |
| 3187 | ); |
| 3188 | lines.push("```".to_string()); |
| 3189 | } |
| 3190 | lines.join("\n") |
| 3191 | } |
| 3192 | |
| 3193 | fn render_differences_json(differences: &[ArtifactDifference]) -> String { |
| 3194 | let mut rendered = String::from("["); |
| 3195 | for (index, difference) in differences.iter().enumerate() { |
| 3196 | if index > 0 { |
| 3197 | rendered.push_str(", "); |
| 3198 | } |
| 3199 | rendered.push_str(&format!( |
| 3200 | "{{\"artifact\":\"{}\",\"detail\":\"{}\"}}", |
| 3201 | json_escape(&difference.artifact), |
| 3202 | json_escape(&difference.detail) |
| 3203 | )); |
| 3204 | } |
| 3205 | rendered.push(']'); |
| 3206 | rendered |
| 3207 | } |
| 3208 | |
| 3209 | fn render_artifact_value_json(value: &ArtifactValue) -> String { |
| 3210 | match value { |
| 3211 | ArtifactValue::Text(text) => { |
| 3212 | format!("{{\"kind\":\"text\",\"value\":\"{}\"}}", json_escape(text)) |
| 3213 | } |
| 3214 | ArtifactValue::Int(value) => format!("{{\"kind\":\"int\",\"value\":{}}}", value), |
| 3215 | ArtifactValue::Run(run) => format!( |
| 3216 | "{{\"kind\":\"runtime\",\"exit_code\":{},\"stdout\":\"{}\",\"stderr\":\"{}\"}}", |
| 3217 | run.exit_code, |
| 3218 | json_escape(&run.stdout), |
| 3219 | json_escape(&run.stderr) |
| 3220 | ), |
| 3221 | ArtifactValue::Path(path) => format!( |
| 3222 | "{{\"kind\":\"path\",\"value\":\"{}\"}}", |
| 3223 | json_escape(&path.display().to_string()) |
| 3224 | ), |
| 3225 | } |
| 3226 | } |
| 3227 | |
| 3228 | fn render_artifact_value_text(value: &ArtifactValue) -> String { |
| 3229 | match value { |
| 3230 | ArtifactValue::Text(text) => text.trim_end().to_string(), |
| 3231 | ArtifactValue::Int(value) => value.to_string(), |
| 3232 | ArtifactValue::Run(run) => format_run_capture(run), |
| 3233 | ArtifactValue::Path(path) => path.display().to_string(), |
| 3234 | } |
| 3235 | } |
| 3236 | |
| 3237 | fn default_suite_root() -> PathBuf { |
| 3238 | Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3239 | .join("..") |
| 3240 | .join("suites") |
| 3241 | } |
| 3242 | |
| 3243 | fn default_report_root() -> PathBuf { |
| 3244 | Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3245 | .join("..") |
| 3246 | .join("reports") |
| 3247 | } |
| 3248 | |
| 3249 | fn workspace_root() -> PathBuf { |
| 3250 | Path::new(env!("CARGO_MANIFEST_DIR")).join("..") |
| 3251 | } |
| 3252 | |
| 3253 | fn doctor_report_fields(config: &DoctorConfig) -> Vec<(String, String)> { |
| 3254 | let workspace_root = workspace_root(); |
| 3255 | let suite_root = default_suite_root(); |
| 3256 | let report_root = default_report_root(); |
| 3257 | let armfortas = config.tools.armfortas_adapters(); |
| 3258 | let observable_backend = config |
| 3259 | .tools |
| 3260 | .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3261 | let capture_root = armfortas.capture_root(); |
| 3262 | let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 3263 | |
| 3264 | let mut fields = vec![ |
| 3265 | ("workspace_root".to_string(), display_path(&workspace_root)), |
| 3266 | ("suite_root".to_string(), display_path(&suite_root)), |
| 3267 | ("report_root".to_string(), display_path(&report_root)), |
| 3268 | ( |
| 3269 | "armfortas_cli_adapter".to_string(), |
| 3270 | armfortas.cli_description().to_string(), |
| 3271 | ), |
| 3272 | ( |
| 3273 | "armfortas_capture_adapter".to_string(), |
| 3274 | armfortas.capture_description().to_string(), |
| 3275 | ), |
| 3276 | ( |
| 3277 | "primary_backend_full".to_string(), |
| 3278 | armfortas.capture_description().to_string(), |
| 3279 | ), |
| 3280 | ( |
| 3281 | "primary_backend_observable".to_string(), |
| 3282 | observable_backend.description().to_string(), |
| 3283 | ), |
| 3284 | ( |
| 3285 | "armfortas_capture_root".to_string(), |
| 3286 | capture_root |
| 3287 | .as_ref() |
| 3288 | .map(|root| display_path(root)) |
| 3289 | .unwrap_or_else(|| "unavailable".to_string()), |
| 3290 | ), |
| 3291 | ( |
| 3292 | "armfortas_capture_manifest".to_string(), |
| 3293 | capture_manifest |
| 3294 | .as_ref() |
| 3295 | .map(|manifest| { |
| 3296 | if manifest.exists() { |
| 3297 | display_path(manifest) |
| 3298 | } else { |
| 3299 | "missing".to_string() |
| 3300 | } |
| 3301 | }) |
| 3302 | .unwrap_or_else(|| "unavailable".to_string()), |
| 3303 | ), |
| 3304 | ( |
| 3305 | "armfortas_cli_mode".to_string(), |
| 3306 | armfortas.cli_mode_name().to_string(), |
| 3307 | ), |
| 3308 | ]; |
| 3309 | match armfortas.cli() { |
| 3310 | ArmfortasCliAdapter::Linked => { |
| 3311 | fields.push(( |
| 3312 | "armfortas_cli_status".to_string(), |
| 3313 | capture_root |
| 3314 | .as_ref() |
| 3315 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3316 | .unwrap_or_else(|| { |
| 3317 | "linked adapter requested but unavailable in this build".to_string() |
| 3318 | }), |
| 3319 | )); |
| 3320 | } |
| 3321 | ArmfortasCliAdapter::External(binary) => { |
| 3322 | fields.push(( |
| 3323 | "armfortas_cli_status".to_string(), |
| 3324 | tool_probe_status(binary, false), |
| 3325 | )); |
| 3326 | } |
| 3327 | } |
| 3328 | fields.push(( |
| 3329 | "armfortas_capture_mode".to_string(), |
| 3330 | armfortas.capture_mode_name().to_string(), |
| 3331 | )); |
| 3332 | fields.push(( |
| 3333 | "armfortas_capture_status".to_string(), |
| 3334 | capture_root |
| 3335 | .as_ref() |
| 3336 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3337 | .unwrap_or_else(|| { |
| 3338 | "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh".to_string() |
| 3339 | }), |
| 3340 | )); |
| 3341 | fields.push(( |
| 3342 | "primary_backend_selection".to_string(), |
| 3343 | "observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend".to_string(), |
| 3344 | )); |
| 3345 | for named in NamedCompiler::ALL { |
| 3346 | append_named_compiler_fields(&mut fields, named, &config.tools, capture_root.as_ref()); |
| 3347 | } |
| 3348 | fields.push(( |
| 3349 | "explicit_compiler_path".to_string(), |
| 3350 | "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3351 | .to_string(), |
| 3352 | )); |
| 3353 | let explicit_capabilities = compiler_capabilities( |
| 3354 | &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3355 | &config.tools, |
| 3356 | ); |
| 3357 | fields.push(( |
| 3358 | "explicit_compiler_path.generic_artifacts".to_string(), |
| 3359 | format_artifact_name_list(&explicit_capabilities.generic_artifacts()), |
| 3360 | )); |
| 3361 | fields.push(( |
| 3362 | "explicit_compiler_path.adapter_extras".to_string(), |
| 3363 | capability_extra_summary(&explicit_capabilities.adapter_extras()), |
| 3364 | )); |
| 3365 | fields.push(( |
| 3366 | "gfortran".to_string(), |
| 3367 | tool_probe_status(&config.tools.gfortran, false), |
| 3368 | )); |
| 3369 | fields.push(( |
| 3370 | "flang-new".to_string(), |
| 3371 | tool_probe_status(&config.tools.flang_new, false), |
| 3372 | )); |
| 3373 | fields.push(( |
| 3374 | "lfortran".to_string(), |
| 3375 | tool_probe_status(&config.tools.lfortran, false), |
| 3376 | )); |
| 3377 | fields.push(( |
| 3378 | "ifort".to_string(), |
| 3379 | tool_probe_status(&config.tools.ifort, false), |
| 3380 | )); |
| 3381 | fields.push(( |
| 3382 | "ifx".to_string(), |
| 3383 | tool_probe_status(&config.tools.ifx, false), |
| 3384 | )); |
| 3385 | fields.push(( |
| 3386 | "nvfortran".to_string(), |
| 3387 | tool_probe_status(&config.tools.nvfortran, false), |
| 3388 | )); |
| 3389 | fields.push(( |
| 3390 | "as".to_string(), |
| 3391 | tool_probe_status(&config.tools.system_as, false), |
| 3392 | )); |
| 3393 | fields.push(( |
| 3394 | "otool".to_string(), |
| 3395 | tool_probe_status(&config.tools.otool, false), |
| 3396 | )); |
| 3397 | fields.push(("nm".to_string(), tool_probe_status(&config.tools.nm, false))); |
| 3398 | fields.push(( |
| 3399 | "note".to_string(), |
| 3400 | if capture_root.is_some() { |
| 3401 | "linked capture still depends on the surrounding armfortas checkout".to_string() |
| 3402 | } else { |
| 3403 | "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work".to_string() |
| 3404 | }, |
| 3405 | )); |
| 3406 | fields.push(( |
| 3407 | if capture_root.is_some() { |
| 3408 | "linked_mode_surface".to_string() |
| 3409 | } else { |
| 3410 | "external_only_surface".to_string() |
| 3411 | }, |
| 3412 | if capture_root.is_some() { |
| 3413 | "rich armfortas stages, legacy frontend/module suites, capture consistency".to_string() |
| 3414 | } else { |
| 3415 | "compare, introspect, generic suite-v2, observable-only run cells".to_string() |
| 3416 | }, |
| 3417 | )); |
| 3418 | fields.push(( |
| 3419 | if capture_root.is_some() { |
| 3420 | "external_only_limits".to_string() |
| 3421 | } else { |
| 3422 | "linked_only_surface".to_string() |
| 3423 | }, |
| 3424 | if capture_root.is_some() { |
| 3425 | "none in this build".to_string() |
| 3426 | } else { |
| 3427 | "armfortas.* extras, legacy frontend/module suites, capture consistency".to_string() |
| 3428 | }, |
| 3429 | )); |
| 3430 | |
| 3431 | fields |
| 3432 | } |
| 3433 | |
| 3434 | fn render_doctor_report(config: &DoctorConfig) -> String { |
| 3435 | let mut lines = vec!["Doctor".to_string()]; |
| 3436 | for (field, value) in doctor_report_fields(config) { |
| 3437 | lines.push(format!(" {}: {}", field, value)); |
| 3438 | } |
| 3439 | lines.join("\n") |
| 3440 | } |
| 3441 | |
| 3442 | fn render_doctor_capabilities_json(capabilities: &CompilerCapabilities) -> String { |
| 3443 | let unavailable = capabilities |
| 3444 | .unavailable_artifacts |
| 3445 | .iter() |
| 3446 | .map(|(artifact, reason)| { |
| 3447 | format!( |
| 3448 | "{{\"artifact\":\"{}\",\"reason\":\"{}\"}}", |
| 3449 | json_escape(artifact.as_str()), |
| 3450 | json_escape(reason) |
| 3451 | ) |
| 3452 | }) |
| 3453 | .collect::<Vec<_>>() |
| 3454 | .join(", "); |
| 3455 | format!( |
| 3456 | "{{\"generic_artifacts\":{},\"adapter_extras\":{},\"unavailable_artifacts\":[{}]}}", |
| 3457 | json_string_iter( |
| 3458 | capabilities |
| 3459 | .generic_artifacts() |
| 3460 | .iter() |
| 3461 | .map(|artifact| artifact.as_str()) |
| 3462 | ), |
| 3463 | json_string_vec_map(&capabilities.adapter_extras()), |
| 3464 | unavailable |
| 3465 | ) |
| 3466 | } |
| 3467 | |
| 3468 | fn render_tool_probe_json(probe: &ToolProbe) -> String { |
| 3469 | format!( |
| 3470 | "{{\"configured\":\"{}\",\"status\":\"{}\",\"resolved_path\":{},\"banner\":{},\"detail\":{}}}", |
| 3471 | json_escape(&probe.configured), |
| 3472 | json_escape(&probe.status), |
| 3473 | match probe.resolved_path.as_ref() { |
| 3474 | Some(path) => format!("\"{}\"", json_escape(&display_path(path))), |
| 3475 | None => "null".to_string(), |
| 3476 | }, |
| 3477 | match probe.banner.as_ref() { |
| 3478 | Some(banner) => format!("\"{}\"", json_escape(banner)), |
| 3479 | None => "null".to_string(), |
| 3480 | }, |
| 3481 | match probe.detail.as_ref() { |
| 3482 | Some(detail) => format!("\"{}\"", json_escape(detail)), |
| 3483 | None => "null".to_string(), |
| 3484 | }, |
| 3485 | ) |
| 3486 | } |
| 3487 | |
| 3488 | fn json_string_vec_map(map: &BTreeMap<String, Vec<String>>) -> String { |
| 3489 | let mut rendered = String::from("{"); |
| 3490 | for (index, (key, values)) in map.iter().enumerate() { |
| 3491 | if index > 0 { |
| 3492 | rendered.push_str(", "); |
| 3493 | } |
| 3494 | rendered.push('"'); |
| 3495 | rendered.push_str(&json_escape(key)); |
| 3496 | rendered.push_str("\": "); |
| 3497 | rendered.push_str(&json_string_iter(values.iter().map(|value| value.as_str()))); |
| 3498 | } |
| 3499 | rendered.push('}'); |
| 3500 | rendered |
| 3501 | } |
| 3502 | |
| 3503 | fn render_named_compiler_entry_json( |
| 3504 | named: NamedCompiler, |
| 3505 | tools: &ToolchainConfig, |
| 3506 | capture_root: Option<&PathBuf>, |
| 3507 | ) -> String { |
| 3508 | let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 3509 | let probe = named_compiler_probe(named, tools, capture_root); |
| 3510 | let mut fields = vec![ |
| 3511 | format!( |
| 3512 | "\"accepted_names\": {}", |
| 3513 | json_string_iter(named.accepted_names().iter().copied()) |
| 3514 | ), |
| 3515 | format!( |
| 3516 | "\"candidate_binaries\": {}", |
| 3517 | json_string_iter(named.candidate_binaries().iter().copied()) |
| 3518 | ), |
| 3519 | format!( |
| 3520 | "\"capabilities\": {}", |
| 3521 | render_doctor_capabilities_json(&capabilities) |
| 3522 | ), |
| 3523 | format!("\"probe\": {}", render_tool_probe_json(&probe)), |
| 3524 | ]; |
| 3525 | if named == NamedCompiler::Armfortas { |
| 3526 | let armfortas = tools.armfortas_adapters(); |
| 3527 | fields.insert( |
| 3528 | 0, |
| 3529 | format!( |
| 3530 | "\"surface\": \"{}\"", |
| 3531 | json_escape(&format!( |
| 3532 | "cli={} capture={}", |
| 3533 | armfortas.cli_mode_name(), |
| 3534 | armfortas.capture_mode_name() |
| 3535 | )) |
| 3536 | ), |
| 3537 | ); |
| 3538 | } else { |
| 3539 | fields.insert( |
| 3540 | 0, |
| 3541 | format!( |
| 3542 | "\"status\": \"{}\"", |
| 3543 | json_escape(&named_compiler_status_value(named, tools, capture_root)) |
| 3544 | ), |
| 3545 | ); |
| 3546 | } |
| 3547 | format!("{{{}}}", fields.join(", ")) |
| 3548 | } |
| 3549 | |
| 3550 | fn render_doctor_json(config: &DoctorConfig) -> String { |
| 3551 | let fields = doctor_report_fields(config); |
| 3552 | let workspace_root = workspace_root(); |
| 3553 | let suite_root = default_suite_root(); |
| 3554 | let report_root = default_report_root(); |
| 3555 | let armfortas = config.tools.armfortas_adapters(); |
| 3556 | let observable_backend = config |
| 3557 | .tools |
| 3558 | .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3559 | let capture_root = armfortas.capture_root(); |
| 3560 | let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 3561 | let explicit_capabilities = compiler_capabilities( |
| 3562 | &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3563 | &config.tools, |
| 3564 | ); |
| 3565 | let named_entries = NamedCompiler::ALL |
| 3566 | .iter() |
| 3567 | .enumerate() |
| 3568 | .map(|(index, named)| { |
| 3569 | format!( |
| 3570 | " \"{}\": {}{}", |
| 3571 | named.as_str(), |
| 3572 | render_named_compiler_entry_json(*named, &config.tools, capture_root.as_ref()), |
| 3573 | if index + 1 == NamedCompiler::ALL.len() { |
| 3574 | "" |
| 3575 | } else { |
| 3576 | "," |
| 3577 | } |
| 3578 | ) |
| 3579 | }) |
| 3580 | .collect::<Vec<_>>(); |
| 3581 | let mut lines = vec![ |
| 3582 | "{".to_string(), |
| 3583 | " \"command\": \"doctor\",".to_string(), |
| 3584 | " \"workspace\": {".to_string(), |
| 3585 | format!( |
| 3586 | " \"workspace_root\": \"{}\",", |
| 3587 | json_escape(&display_path(&workspace_root)) |
| 3588 | ), |
| 3589 | format!( |
| 3590 | " \"suite_root\": \"{}\",", |
| 3591 | json_escape(&display_path(&suite_root)) |
| 3592 | ), |
| 3593 | format!( |
| 3594 | " \"report_root\": \"{}\"", |
| 3595 | json_escape(&display_path(&report_root)) |
| 3596 | ), |
| 3597 | " },".to_string(), |
| 3598 | " \"armfortas\": {".to_string(), |
| 3599 | format!( |
| 3600 | " \"cli_adapter\": \"{}\",", |
| 3601 | json_escape(armfortas.cli_description()) |
| 3602 | ), |
| 3603 | format!( |
| 3604 | " \"capture_adapter\": \"{}\",", |
| 3605 | json_escape(armfortas.capture_description()) |
| 3606 | ), |
| 3607 | format!( |
| 3608 | " \"cli_mode\": \"{}\",", |
| 3609 | json_escape(armfortas.cli_mode_name()) |
| 3610 | ), |
| 3611 | format!( |
| 3612 | " \"cli_status\": \"{}\",", |
| 3613 | json_escape( |
| 3614 | &match armfortas.cli() { |
| 3615 | ArmfortasCliAdapter::Linked => capture_root |
| 3616 | .as_ref() |
| 3617 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3618 | .unwrap_or_else(|| { |
| 3619 | "linked adapter requested but unavailable in this build".to_string() |
| 3620 | }), |
| 3621 | ArmfortasCliAdapter::External(binary) => tool_probe_status(binary, false), |
| 3622 | } |
| 3623 | ) |
| 3624 | ), |
| 3625 | format!( |
| 3626 | " \"capture_mode\": \"{}\",", |
| 3627 | json_escape(armfortas.capture_mode_name()) |
| 3628 | ), |
| 3629 | format!( |
| 3630 | " \"capture_status\": \"{}\",", |
| 3631 | json_escape( |
| 3632 | &capture_root |
| 3633 | .as_ref() |
| 3634 | .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3635 | .unwrap_or_else(|| { |
| 3636 | "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh" |
| 3637 | .to_string() |
| 3638 | }) |
| 3639 | ) |
| 3640 | ), |
| 3641 | format!( |
| 3642 | " \"capture_root\": {},", |
| 3643 | match capture_root.as_ref() { |
| 3644 | Some(root) => format!("\"{}\"", json_escape(&display_path(root))), |
| 3645 | None => "null".to_string(), |
| 3646 | } |
| 3647 | ), |
| 3648 | format!( |
| 3649 | " \"capture_manifest\": \"{}\"", |
| 3650 | json_escape( |
| 3651 | &capture_manifest |
| 3652 | .as_ref() |
| 3653 | .map(|manifest| { |
| 3654 | if manifest.exists() { |
| 3655 | display_path(manifest) |
| 3656 | } else { |
| 3657 | "missing".to_string() |
| 3658 | } |
| 3659 | }) |
| 3660 | .unwrap_or_else(|| "unavailable".to_string()) |
| 3661 | ) |
| 3662 | ), |
| 3663 | " },".to_string(), |
| 3664 | " \"primary_backends\": {".to_string(), |
| 3665 | format!( |
| 3666 | " \"full\": \"{}\",", |
| 3667 | json_escape(armfortas.capture_description()) |
| 3668 | ), |
| 3669 | format!( |
| 3670 | " \"observable\": \"{}\",", |
| 3671 | json_escape(observable_backend.description()) |
| 3672 | ), |
| 3673 | format!( |
| 3674 | " \"selection\": \"{}\"", |
| 3675 | json_escape("observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend") |
| 3676 | ), |
| 3677 | " },".to_string(), |
| 3678 | " \"named_compilers\": {".to_string(), |
| 3679 | ]; |
| 3680 | lines.extend(named_entries); |
| 3681 | lines.extend([ |
| 3682 | " },".to_string(), |
| 3683 | format!( |
| 3684 | " \"explicit_compiler_path\": {{\"description\":\"{}\",\"capabilities\":{}}},", |
| 3685 | json_escape( |
| 3686 | "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3687 | ), |
| 3688 | render_doctor_capabilities_json(&explicit_capabilities) |
| 3689 | ), |
| 3690 | " \"tools\": {".to_string(), |
| 3691 | format!( |
| 3692 | " \"gfortran\": \"{}\",", |
| 3693 | json_escape(&tool_probe_status(&config.tools.gfortran, false)) |
| 3694 | ), |
| 3695 | format!( |
| 3696 | " \"flang-new\": \"{}\",", |
| 3697 | json_escape(&tool_probe_status(&config.tools.flang_new, false)) |
| 3698 | ), |
| 3699 | format!( |
| 3700 | " \"lfortran\": \"{}\",", |
| 3701 | json_escape(&tool_probe_status(&config.tools.lfortran, false)) |
| 3702 | ), |
| 3703 | format!( |
| 3704 | " \"ifort\": \"{}\",", |
| 3705 | json_escape(&tool_probe_status(&config.tools.ifort, false)) |
| 3706 | ), |
| 3707 | format!( |
| 3708 | " \"ifx\": \"{}\",", |
| 3709 | json_escape(&tool_probe_status(&config.tools.ifx, false)) |
| 3710 | ), |
| 3711 | format!( |
| 3712 | " \"nvfortran\": \"{}\",", |
| 3713 | json_escape(&tool_probe_status(&config.tools.nvfortran, false)) |
| 3714 | ), |
| 3715 | format!( |
| 3716 | " \"as\": \"{}\",", |
| 3717 | json_escape(&tool_probe_status(&config.tools.system_as, false)) |
| 3718 | ), |
| 3719 | format!( |
| 3720 | " \"otool\": \"{}\",", |
| 3721 | json_escape(&tool_probe_status(&config.tools.otool, false)) |
| 3722 | ), |
| 3723 | format!( |
| 3724 | " \"nm\": \"{}\"", |
| 3725 | json_escape(&tool_probe_status(&config.tools.nm, false)) |
| 3726 | ), |
| 3727 | " },".to_string(), |
| 3728 | " \"mode\": {".to_string(), |
| 3729 | format!( |
| 3730 | " \"note\": \"{}\",", |
| 3731 | json_escape( |
| 3732 | if capture_root.is_some() { |
| 3733 | "linked capture still depends on the surrounding armfortas checkout" |
| 3734 | } else { |
| 3735 | "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work" |
| 3736 | } |
| 3737 | ) |
| 3738 | ), |
| 3739 | format!( |
| 3740 | " \"surface_key\": \"{}\",", |
| 3741 | if capture_root.is_some() { |
| 3742 | "linked_mode_surface" |
| 3743 | } else { |
| 3744 | "external_only_surface" |
| 3745 | } |
| 3746 | ), |
| 3747 | format!( |
| 3748 | " \"surface_value\": \"{}\",", |
| 3749 | json_escape( |
| 3750 | if capture_root.is_some() { |
| 3751 | "rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 3752 | } else { |
| 3753 | "compare, introspect, generic suite-v2, observable-only run cells" |
| 3754 | } |
| 3755 | ) |
| 3756 | ), |
| 3757 | format!( |
| 3758 | " \"limits_key\": \"{}\",", |
| 3759 | if capture_root.is_some() { |
| 3760 | "external_only_limits" |
| 3761 | } else { |
| 3762 | "linked_only_surface" |
| 3763 | } |
| 3764 | ), |
| 3765 | format!( |
| 3766 | " \"limits_value\": \"{}\"", |
| 3767 | json_escape( |
| 3768 | if capture_root.is_some() { |
| 3769 | "none in this build" |
| 3770 | } else { |
| 3771 | "armfortas.* extras, legacy frontend/module suites, capture consistency" |
| 3772 | } |
| 3773 | ) |
| 3774 | ), |
| 3775 | " },".to_string(), |
| 3776 | " \"fields\": {".to_string(), |
| 3777 | ]); |
| 3778 | for (index, (field, value)) in fields.iter().enumerate() { |
| 3779 | lines.push(format!( |
| 3780 | " \"{}\": \"{}\"{}", |
| 3781 | json_escape(field), |
| 3782 | json_escape(value), |
| 3783 | if index + 1 == fields.len() { "" } else { "," } |
| 3784 | )); |
| 3785 | } |
| 3786 | lines.push(" }".to_string()); |
| 3787 | lines.push("}".to_string()); |
| 3788 | lines.join("\n") + "\n" |
| 3789 | } |
| 3790 | |
| 3791 | fn render_doctor_markdown(config: &DoctorConfig) -> String { |
| 3792 | let mut lines = vec![ |
| 3793 | "# bencch doctor report".to_string(), |
| 3794 | String::new(), |
| 3795 | "| field | value |".to_string(), |
| 3796 | "| --- | --- |".to_string(), |
| 3797 | ]; |
| 3798 | for (field, value) in doctor_report_fields(config) { |
| 3799 | lines.push(format!( |
| 3800 | "| `{}` | {} |", |
| 3801 | field, |
| 3802 | doctor_markdown_cell(&value) |
| 3803 | )); |
| 3804 | } |
| 3805 | lines.join("\n") + "\n" |
| 3806 | } |
| 3807 | |
| 3808 | fn write_doctor_reports(config: &DoctorConfig) -> Result<(), String> { |
| 3809 | if let Some(path) = &config.json_report { |
| 3810 | write_report(path, &render_doctor_json(config), "json report")?; |
| 3811 | println!("json report: {}", path.display()); |
| 3812 | } |
| 3813 | if let Some(path) = &config.markdown_report { |
| 3814 | write_report(path, &render_doctor_markdown(config), "markdown report")?; |
| 3815 | println!("markdown report: {}", path.display()); |
| 3816 | } |
| 3817 | Ok(()) |
| 3818 | } |
| 3819 | |
| 3820 | fn doctor_markdown_cell(value: &str) -> String { |
| 3821 | value.replace('|', "\\|").replace('\n', "<br>") |
| 3822 | } |
| 3823 | |
| 3824 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 3825 | struct ToolProbe { |
| 3826 | configured: String, |
| 3827 | resolved_path: Option<PathBuf>, |
| 3828 | status: String, |
| 3829 | banner: Option<String>, |
| 3830 | detail: Option<String>, |
| 3831 | } |
| 3832 | |
| 3833 | fn tool_probe(configured: &str, already_resolved_path: bool) -> ToolProbe { |
| 3834 | let resolved_path = if already_resolved_path { |
| 3835 | let path = PathBuf::from(configured); |
| 3836 | if path.exists() { |
| 3837 | Some(path) |
| 3838 | } else { |
| 3839 | None |
| 3840 | } |
| 3841 | } else { |
| 3842 | resolve_tool_path(configured) |
| 3843 | }; |
| 3844 | |
| 3845 | match resolved_path { |
| 3846 | Some(path) => match probe_tool_banner(&path) { |
| 3847 | Ok((arg, banner)) => ToolProbe { |
| 3848 | configured: configured.to_string(), |
| 3849 | resolved_path: Some(path), |
| 3850 | status: "invokable".into(), |
| 3851 | banner: Some(banner), |
| 3852 | detail: Some(format!("probe succeeded with {}", arg)), |
| 3853 | }, |
| 3854 | Err(detail) => ToolProbe { |
| 3855 | configured: configured.to_string(), |
| 3856 | resolved_path: Some(path), |
| 3857 | status: "resolved".into(), |
| 3858 | banner: None, |
| 3859 | detail: Some(detail), |
| 3860 | }, |
| 3861 | }, |
| 3862 | None => ToolProbe { |
| 3863 | configured: configured.to_string(), |
| 3864 | resolved_path: None, |
| 3865 | status: "missing".into(), |
| 3866 | banner: None, |
| 3867 | detail: Some("binary not found on disk or PATH".into()), |
| 3868 | }, |
| 3869 | } |
| 3870 | } |
| 3871 | |
| 3872 | fn linked_tool_probe(capture_root: Option<&PathBuf>) -> ToolProbe { |
| 3873 | match capture_root { |
| 3874 | Some(root) => ToolProbe { |
| 3875 | configured: "linked".into(), |
| 3876 | resolved_path: Some(root.clone()), |
| 3877 | status: "linked".into(), |
| 3878 | banner: None, |
| 3879 | detail: Some(format!("linked via Cargo to {}", display_path(root))), |
| 3880 | }, |
| 3881 | None => ToolProbe { |
| 3882 | configured: "linked".into(), |
| 3883 | resolved_path: None, |
| 3884 | status: "unavailable".into(), |
| 3885 | banner: None, |
| 3886 | detail: Some("linked adapter requested but unavailable in this build".into()), |
| 3887 | }, |
| 3888 | } |
| 3889 | } |
| 3890 | |
| 3891 | fn named_compiler_probe( |
| 3892 | named: NamedCompiler, |
| 3893 | tools: &ToolchainConfig, |
| 3894 | capture_root: Option<&PathBuf>, |
| 3895 | ) -> ToolProbe { |
| 3896 | match named { |
| 3897 | NamedCompiler::Armfortas => match &tools.armfortas { |
| 3898 | ArmfortasCliAdapter::Linked => linked_tool_probe(capture_root), |
| 3899 | ArmfortasCliAdapter::External(binary) => tool_probe(binary, false), |
| 3900 | }, |
| 3901 | _ => tool_probe( |
| 3902 | &tools |
| 3903 | .named_compiler_binary(named) |
| 3904 | .unwrap_or_else(|| named.as_str().to_string()), |
| 3905 | false, |
| 3906 | ), |
| 3907 | } |
| 3908 | } |
| 3909 | |
| 3910 | fn compiler_spec_probe( |
| 3911 | spec: &CompilerSpec, |
| 3912 | tools: &ToolchainConfig, |
| 3913 | capture_root: Option<&PathBuf>, |
| 3914 | ) -> ToolProbe { |
| 3915 | match spec { |
| 3916 | CompilerSpec::Named(named) => named_compiler_probe(*named, tools, capture_root), |
| 3917 | CompilerSpec::Binary(path) => tool_probe(&path.display().to_string(), true), |
| 3918 | } |
| 3919 | } |
| 3920 | |
| 3921 | fn probe_tool_banner(path: &Path) -> Result<(String, String), String> { |
| 3922 | let mut last_detail = None; |
| 3923 | for arg in ["--version", "-V", "-v"] { |
| 3924 | match Command::new(path).arg(arg).output() { |
| 3925 | Ok(output) => { |
| 3926 | let stdout = String::from_utf8_lossy(&output.stdout); |
| 3927 | let stderr = String::from_utf8_lossy(&output.stderr); |
| 3928 | let combined = stdout |
| 3929 | .lines() |
| 3930 | .chain(stderr.lines()) |
| 3931 | .map(str::trim) |
| 3932 | .find(|line| !line.is_empty()) |
| 3933 | .map(|line| compact_probe_banner(line)); |
| 3934 | if let Some(line) = combined { |
| 3935 | return Ok((arg.to_string(), line)); |
| 3936 | } |
| 3937 | last_detail = Some(format!( |
| 3938 | "{} returned no banner output (exit={})", |
| 3939 | arg, |
| 3940 | output.status.code().unwrap_or(-1) |
| 3941 | )); |
| 3942 | } |
| 3943 | Err(err) => { |
| 3944 | last_detail = Some(format!("{} failed: {}", arg, err)); |
| 3945 | } |
| 3946 | } |
| 3947 | } |
| 3948 | |
| 3949 | Err(last_detail.unwrap_or_else(|| "probe failed".to_string())) |
| 3950 | } |
| 3951 | |
| 3952 | fn compact_probe_banner(line: &str) -> String { |
| 3953 | let compact = line.split_whitespace().collect::<Vec<_>>().join(" "); |
| 3954 | let chars = compact.chars().collect::<Vec<_>>(); |
| 3955 | if chars.len() > 120 { |
| 3956 | chars.into_iter().take(117).collect::<String>() + "..." |
| 3957 | } else { |
| 3958 | compact |
| 3959 | } |
| 3960 | } |
| 3961 | |
| 3962 | fn tool_probe_status(configured: &str, already_resolved_path: bool) -> String { |
| 3963 | let probe = tool_probe(configured, already_resolved_path); |
| 3964 | match probe.resolved_path { |
| 3965 | Some(path) => format!("configured={} resolved={}", configured, path.display()), |
| 3966 | None => format!("configured={} resolved=missing", configured), |
| 3967 | } |
| 3968 | } |
| 3969 | |
| 3970 | fn resolve_tool_path(configured: &str) -> Option<PathBuf> { |
| 3971 | let configured_path = Path::new(configured); |
| 3972 | if configured.contains('/') || configured.starts_with('.') { |
| 3973 | return configured_path |
| 3974 | .exists() |
| 3975 | .then(|| configured_path.to_path_buf()); |
| 3976 | } |
| 3977 | |
| 3978 | let path_var = std::env::var_os("PATH")?; |
| 3979 | for entry in std::env::split_paths(&path_var) { |
| 3980 | let candidate = entry.join(configured); |
| 3981 | if candidate.exists() { |
| 3982 | return Some(candidate); |
| 3983 | } |
| 3984 | } |
| 3985 | None |
| 3986 | } |
| 3987 | |
| 3988 | fn display_path(path: &Path) -> String { |
| 3989 | fs::canonicalize(path) |
| 3990 | .unwrap_or_else(|_| path.to_path_buf()) |
| 3991 | .display() |
| 3992 | .to_string() |
| 3993 | } |
| 3994 | |
| 3995 | fn discover_suites(root: PathBuf) -> Result<Vec<SuiteSpec>, String> { |
| 3996 | let mut files = Vec::new(); |
| 3997 | collect_suite_files(&root, &mut files)?; |
| 3998 | files.sort(); |
| 3999 | |
| 4000 | let mut suites = Vec::new(); |
| 4001 | for file in files { |
| 4002 | suites.push(parse_suite_file(&file)?); |
| 4003 | } |
| 4004 | suites.sort_by(|a, b| a.name.cmp(&b.name)); |
| 4005 | Ok(suites) |
| 4006 | } |
| 4007 | |
| 4008 | fn collect_suite_files(root: &Path, files: &mut Vec<PathBuf>) -> Result<(), String> { |
| 4009 | let entries = fs::read_dir(root) |
| 4010 | .map_err(|e| format!("cannot read suite root '{}': {}", root.display(), e))?; |
| 4011 | for entry in entries { |
| 4012 | let entry = |
| 4013 | entry.map_err(|e| format!("cannot read entry in '{}': {}", root.display(), e))?; |
| 4014 | let path = entry.path(); |
| 4015 | if path.is_dir() { |
| 4016 | collect_suite_files(&path, files)?; |
| 4017 | } else if path.extension().and_then(|ext| ext.to_str()) == Some(SUITE_EXTENSION) { |
| 4018 | files.push(path); |
| 4019 | } |
| 4020 | } |
| 4021 | Ok(()) |
| 4022 | } |
| 4023 | |
| 4024 | fn parse_suite_file(path: &Path) -> Result<SuiteSpec, String> { |
| 4025 | let text = fs::read_to_string(path) |
| 4026 | .map_err(|e| format!("cannot read suite '{}': {}", path.display(), e))?; |
| 4027 | |
| 4028 | let mut suite_name = None; |
| 4029 | let mut cases = Vec::new(); |
| 4030 | let mut current = None; |
| 4031 | |
| 4032 | for (index, raw_line) in text.lines().enumerate() { |
| 4033 | let line_no = index + 1; |
| 4034 | let line = raw_line.trim(); |
| 4035 | if line.is_empty() || line.starts_with('#') { |
| 4036 | continue; |
| 4037 | } |
| 4038 | |
| 4039 | if let Some(rest) = line.strip_prefix("suite ") { |
| 4040 | if suite_name.is_some() { |
| 4041 | return Err(format!( |
| 4042 | "{}:{}: duplicate suite declaration", |
| 4043 | path.display(), |
| 4044 | line_no |
| 4045 | )); |
| 4046 | } |
| 4047 | suite_name = Some(parse_quoted(rest, path, line_no)?); |
| 4048 | continue; |
| 4049 | } |
| 4050 | |
| 4051 | if let Some(rest) = line.strip_prefix("case ") { |
| 4052 | if current.is_some() { |
| 4053 | return Err(format!( |
| 4054 | "{}:{}: nested case without end", |
| 4055 | path.display(), |
| 4056 | line_no |
| 4057 | )); |
| 4058 | } |
| 4059 | current = Some(CaseBuilder::new(parse_quoted(rest, path, line_no)?)); |
| 4060 | continue; |
| 4061 | } |
| 4062 | |
| 4063 | if line == "end" { |
| 4064 | let builder = current.take().ok_or_else(|| { |
| 4065 | format!("{}:{}: stray end outside of case", path.display(), line_no) |
| 4066 | })?; |
| 4067 | cases.push(builder.build(path)?); |
| 4068 | continue; |
| 4069 | } |
| 4070 | |
| 4071 | let builder = current.as_mut().ok_or_else(|| { |
| 4072 | format!( |
| 4073 | "{}:{}: expected suite/case declaration first", |
| 4074 | path.display(), |
| 4075 | line_no |
| 4076 | ) |
| 4077 | })?; |
| 4078 | |
| 4079 | if let Some(rest) = line.strip_prefix("source ") { |
| 4080 | builder.source = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 4081 | } else if let Some(rest) = line.strip_prefix("entry ") { |
| 4082 | builder.graph_entry = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 4083 | } else if let Some(rest) = line.strip_prefix("file ") { |
| 4084 | builder |
| 4085 | .graph_files |
| 4086 | .push(resolve_suite_relative_path(rest, path, line_no)?); |
| 4087 | } else if let Some(rest) = line.strip_prefix("compiler ") { |
| 4088 | let (compiler, artifacts) = parse_compiler_artifact_declaration(rest, path, line_no)?; |
| 4089 | builder.generic_compiler = Some(compiler); |
| 4090 | builder.generic_artifacts = artifacts; |
| 4091 | } else if let Some(rest) = line.strip_prefix("compare ") { |
| 4092 | let (left, right, artifacts) = parse_compare_declaration(rest, path, line_no)?; |
| 4093 | builder.generic_compare = Some((left, right)); |
| 4094 | builder.generic_compare_artifacts = artifacts; |
| 4095 | } else if let Some(rest) = line.strip_prefix("armfortas =>") { |
| 4096 | builder.requested = parse_stage_list(rest, path, line_no)?; |
| 4097 | } else if let Some(rest) = line.strip_prefix("repeat =>") { |
| 4098 | builder.repeat_count = parse_repeat_count(rest, path, line_no)?; |
| 4099 | } else if let Some(rest) = line.strip_prefix("opts =>") { |
| 4100 | builder.opt_levels = parse_opt_levels(rest, path, line_no)?; |
| 4101 | } else if let Some(rest) = line.strip_prefix("differential =>") { |
| 4102 | builder.reference_compilers = parse_reference_compilers(rest, path, line_no)?; |
| 4103 | } else if let Some(rest) = line.strip_prefix("consistency =>") { |
| 4104 | builder.consistency_checks = parse_consistency_checks(rest, path, line_no)?; |
| 4105 | } else if let Some(rest) = line.strip_prefix("expect-fail ") { |
| 4106 | builder |
| 4107 | .expectations |
| 4108 | .push(parse_failure_expectation(rest, path, line_no)?); |
| 4109 | } else if let Some(rest) = line.strip_prefix("expect ") { |
| 4110 | builder |
| 4111 | .expectations |
| 4112 | .push(parse_expectation(rest, path, line_no)?); |
| 4113 | } else if let Some(rest) = line.strip_prefix("xfail capability ") { |
| 4114 | if builder.capability_policy.is_some() { |
| 4115 | return Err(format!( |
| 4116 | "{}:{}: duplicate capability policy", |
| 4117 | path.display(), |
| 4118 | line_no |
| 4119 | )); |
| 4120 | } |
| 4121 | builder.capability_policy = Some(parse_capability_policy( |
| 4122 | StatusKind::Xfail, |
| 4123 | rest, |
| 4124 | path, |
| 4125 | line_no, |
| 4126 | )?); |
| 4127 | } else if let Some(rest) = line.strip_prefix("future capability ") { |
| 4128 | if builder.capability_policy.is_some() { |
| 4129 | return Err(format!( |
| 4130 | "{}:{}: duplicate capability policy", |
| 4131 | path.display(), |
| 4132 | line_no |
| 4133 | )); |
| 4134 | } |
| 4135 | builder.capability_policy = Some(parse_capability_policy( |
| 4136 | StatusKind::Future, |
| 4137 | rest, |
| 4138 | path, |
| 4139 | line_no, |
| 4140 | )?); |
| 4141 | } else if let Some(rest) = line.strip_prefix("xfail ") { |
| 4142 | builder |
| 4143 | .status_rules |
| 4144 | .push(parse_status_rule(StatusKind::Xfail, rest, path, line_no)?); |
| 4145 | } else if let Some(rest) = line.strip_prefix("future ") { |
| 4146 | builder |
| 4147 | .status_rules |
| 4148 | .push(parse_status_rule(StatusKind::Future, rest, path, line_no)?); |
| 4149 | } else { |
| 4150 | return Err(format!( |
| 4151 | "{}:{}: unrecognized line '{}'", |
| 4152 | path.display(), |
| 4153 | line_no, |
| 4154 | line |
| 4155 | )); |
| 4156 | } |
| 4157 | } |
| 4158 | |
| 4159 | if current.is_some() { |
| 4160 | return Err(format!("{}: unterminated case block", path.display())); |
| 4161 | } |
| 4162 | |
| 4163 | let suite_name = |
| 4164 | suite_name.ok_or_else(|| format!("{}: missing suite declaration", path.display()))?; |
| 4165 | if cases.is_empty() { |
| 4166 | return Err(format!("{}: suite has no cases", path.display())); |
| 4167 | } |
| 4168 | |
| 4169 | Ok(SuiteSpec { |
| 4170 | name: suite_name, |
| 4171 | path: path.to_path_buf(), |
| 4172 | cases, |
| 4173 | }) |
| 4174 | } |
| 4175 | |
| 4176 | struct CaseBuilder { |
| 4177 | name: String, |
| 4178 | source: Option<PathBuf>, |
| 4179 | graph_entry: Option<PathBuf>, |
| 4180 | graph_files: Vec<PathBuf>, |
| 4181 | requested: BTreeSet<Stage>, |
| 4182 | generic_compiler: Option<CompilerSpec>, |
| 4183 | generic_artifacts: BTreeSet<ArtifactKey>, |
| 4184 | generic_compare: Option<(CompilerSpec, CompilerSpec)>, |
| 4185 | generic_compare_artifacts: BTreeSet<ArtifactKey>, |
| 4186 | opt_levels: Vec<OptLevel>, |
| 4187 | repeat_count: usize, |
| 4188 | reference_compilers: Vec<ReferenceCompiler>, |
| 4189 | consistency_checks: Vec<ConsistencyCheck>, |
| 4190 | expectations: Vec<Expectation>, |
| 4191 | status_rules: Vec<PendingStatusRule>, |
| 4192 | capability_policy: Option<CapabilityPolicy>, |
| 4193 | } |
| 4194 | |
| 4195 | impl CaseBuilder { |
| 4196 | fn new(name: String) -> Self { |
| 4197 | Self { |
| 4198 | name, |
| 4199 | source: None, |
| 4200 | graph_entry: None, |
| 4201 | graph_files: Vec::new(), |
| 4202 | requested: BTreeSet::new(), |
| 4203 | generic_compiler: None, |
| 4204 | generic_artifacts: BTreeSet::new(), |
| 4205 | generic_compare: None, |
| 4206 | generic_compare_artifacts: BTreeSet::new(), |
| 4207 | opt_levels: Vec::new(), |
| 4208 | repeat_count: 2, |
| 4209 | reference_compilers: Vec::new(), |
| 4210 | consistency_checks: Vec::new(), |
| 4211 | expectations: Vec::new(), |
| 4212 | status_rules: Vec::new(), |
| 4213 | capability_policy: None, |
| 4214 | } |
| 4215 | } |
| 4216 | |
| 4217 | fn build(self, suite_path: &Path) -> Result<CaseSpec, String> { |
| 4218 | let generic_mode_count = usize::from(self.generic_compiler.is_some()) |
| 4219 | + usize::from(self.generic_compare.is_some()); |
| 4220 | if generic_mode_count > 1 { |
| 4221 | return Err(format!( |
| 4222 | "{}: case '{}' mixes multiple suite-v2 execution forms", |
| 4223 | suite_path.display(), |
| 4224 | self.name |
| 4225 | )); |
| 4226 | } |
| 4227 | |
| 4228 | if generic_mode_count > 0 && !self.requested.is_empty() { |
| 4229 | return Err(format!( |
| 4230 | "{}: case '{}' mixes generic suite-v2 syntax with legacy 'armfortas => ...' stages", |
| 4231 | suite_path.display(), |
| 4232 | self.name |
| 4233 | )); |
| 4234 | } |
| 4235 | |
| 4236 | if self.source.is_some() && (self.graph_entry.is_some() || !self.graph_files.is_empty()) { |
| 4237 | return Err(format!( |
| 4238 | "{}: case '{}' mixes source with graph entry/file declarations", |
| 4239 | suite_path.display(), |
| 4240 | self.name |
| 4241 | )); |
| 4242 | } |
| 4243 | |
| 4244 | if self.graph_entry.is_some() && self.graph_files.is_empty() { |
| 4245 | return Err(format!( |
| 4246 | "{}: case '{}' declares an entry without any file members", |
| 4247 | suite_path.display(), |
| 4248 | self.name |
| 4249 | )); |
| 4250 | } |
| 4251 | |
| 4252 | if self.graph_entry.is_none() && !self.graph_files.is_empty() { |
| 4253 | return Err(format!( |
| 4254 | "{}: case '{}' declares file members without an entry", |
| 4255 | suite_path.display(), |
| 4256 | self.name |
| 4257 | )); |
| 4258 | } |
| 4259 | |
| 4260 | let (source, graph_files) = if let Some(source) = self.source { |
| 4261 | (source, Vec::new()) |
| 4262 | } else if let Some(entry) = self.graph_entry { |
| 4263 | if !self.graph_files.iter().any(|file| file == &entry) { |
| 4264 | return Err(format!( |
| 4265 | "{}: case '{}' entry '{}' is not listed in file declarations", |
| 4266 | suite_path.display(), |
| 4267 | self.name, |
| 4268 | entry.display() |
| 4269 | )); |
| 4270 | } |
| 4271 | (entry, self.graph_files) |
| 4272 | } else { |
| 4273 | return Err(format!( |
| 4274 | "{}: case '{}' is missing a source path or graph entry", |
| 4275 | suite_path.display(), |
| 4276 | self.name |
| 4277 | )); |
| 4278 | }; |
| 4279 | |
| 4280 | if self.generic_compare.is_some() |
| 4281 | && (!self.reference_compilers.is_empty() || !self.consistency_checks.is_empty()) |
| 4282 | { |
| 4283 | return Err(format!( |
| 4284 | "{}: case '{}' compare suite-v2 cases do not support differential/consistency rules", |
| 4285 | suite_path.display(), |
| 4286 | self.name |
| 4287 | )); |
| 4288 | } |
| 4289 | |
| 4290 | if self.generic_compiler.is_some() { |
| 4291 | let unsupported = self |
| 4292 | .consistency_checks |
| 4293 | .iter() |
| 4294 | .copied() |
| 4295 | .filter(|check| !check.supports_generic_introspect()) |
| 4296 | .collect::<Vec<_>>(); |
| 4297 | if !unsupported.is_empty() { |
| 4298 | return Err(format!( |
| 4299 | "{}: case '{}' generic compiler cases only support cli_asm_reproducible, cli_obj_reproducible, and cli_run_reproducible today (unsupported: {})", |
| 4300 | suite_path.display(), |
| 4301 | self.name, |
| 4302 | unsupported |
| 4303 | .iter() |
| 4304 | .map(ConsistencyCheck::as_str) |
| 4305 | .collect::<Vec<_>>() |
| 4306 | .join(", ") |
| 4307 | )); |
| 4308 | } |
| 4309 | } |
| 4310 | |
| 4311 | let needs_source_comment_resolution = self |
| 4312 | .expectations |
| 4313 | .iter() |
| 4314 | .any(|expectation| matches!(expectation, Expectation::FailSourceComments)) |
| 4315 | || self |
| 4316 | .status_rules |
| 4317 | .iter() |
| 4318 | .any(|rule| matches!(rule, PendingStatusRule::XfailSourceComments)); |
| 4319 | let source_text = if needs_source_comment_resolution { |
| 4320 | Some(fs::read_to_string(&source).map_err(|e| { |
| 4321 | format!( |
| 4322 | "{}: case '{}': cannot read source '{}' for comment-based directives: {}", |
| 4323 | suite_path.display(), |
| 4324 | self.name, |
| 4325 | source.display(), |
| 4326 | e |
| 4327 | ) |
| 4328 | })?) |
| 4329 | } else { |
| 4330 | None |
| 4331 | }; |
| 4332 | |
| 4333 | let generic_introspect = if let Some(compiler) = self.generic_compiler { |
| 4334 | if self.generic_artifacts.is_empty() { |
| 4335 | return Err(format!( |
| 4336 | "{}: case '{}' generic compiler artifact list is empty", |
| 4337 | suite_path.display(), |
| 4338 | self.name |
| 4339 | )); |
| 4340 | } |
| 4341 | Some(GenericIntrospectCase { |
| 4342 | compiler, |
| 4343 | artifacts: self.generic_artifacts, |
| 4344 | }) |
| 4345 | } else { |
| 4346 | None |
| 4347 | }; |
| 4348 | |
| 4349 | let generic_compare = if let Some((left, right)) = self.generic_compare { |
| 4350 | let mut artifacts = self.generic_compare_artifacts; |
| 4351 | if artifacts.is_empty() { |
| 4352 | return Err(format!( |
| 4353 | "{}: case '{}' compare artifact list is empty", |
| 4354 | suite_path.display(), |
| 4355 | self.name |
| 4356 | )); |
| 4357 | } |
| 4358 | artifacts.extend(default_compare_artifacts(&artifacts)); |
| 4359 | Some(GenericCompareCase { |
| 4360 | left, |
| 4361 | right, |
| 4362 | artifacts, |
| 4363 | }) |
| 4364 | } else { |
| 4365 | None |
| 4366 | }; |
| 4367 | |
| 4368 | let mut requested = self.requested; |
| 4369 | if requested.is_empty() && generic_introspect.is_none() && generic_compare.is_none() { |
| 4370 | requested.insert(Stage::Run); |
| 4371 | } |
| 4372 | |
| 4373 | let opt_levels = if self.opt_levels.is_empty() { |
| 4374 | vec![OptLevel::O0] |
| 4375 | } else { |
| 4376 | self.opt_levels |
| 4377 | }; |
| 4378 | let expectations = resolve_source_comment_expectations( |
| 4379 | self.expectations, |
| 4380 | source_text.as_deref(), |
| 4381 | suite_path, |
| 4382 | &self.name, |
| 4383 | &source, |
| 4384 | )?; |
| 4385 | let status_rules = resolve_source_comment_status_rules( |
| 4386 | self.status_rules, |
| 4387 | source_text.as_deref(), |
| 4388 | suite_path, |
| 4389 | &self.name, |
| 4390 | &source, |
| 4391 | )?; |
| 4392 | |
| 4393 | Ok(CaseSpec { |
| 4394 | name: self.name, |
| 4395 | source, |
| 4396 | graph_files, |
| 4397 | requested, |
| 4398 | generic_introspect, |
| 4399 | generic_compare, |
| 4400 | opt_levels, |
| 4401 | repeat_count: self.repeat_count, |
| 4402 | reference_compilers: self.reference_compilers, |
| 4403 | consistency_checks: self.consistency_checks, |
| 4404 | expectations, |
| 4405 | status_rules, |
| 4406 | capability_policy: self.capability_policy, |
| 4407 | }) |
| 4408 | } |
| 4409 | } |
| 4410 | |
| 4411 | fn resolve_suite_relative_path(rest: &str, path: &Path, line_no: usize) -> Result<PathBuf, String> { |
| 4412 | let relative = parse_quoted(rest, path, line_no)?; |
| 4413 | Ok(path |
| 4414 | .parent() |
| 4415 | .unwrap_or_else(|| Path::new(".")) |
| 4416 | .join(relative)) |
| 4417 | } |
| 4418 | |
| 4419 | fn parse_stage_list(rest: &str, path: &Path, line_no: usize) -> Result<BTreeSet<Stage>, String> { |
| 4420 | let mut stages = BTreeSet::new(); |
| 4421 | for raw in rest.split(',') { |
| 4422 | let name = raw.trim(); |
| 4423 | if name.is_empty() { |
| 4424 | continue; |
| 4425 | } |
| 4426 | let stage = Stage::parse(name) |
| 4427 | .ok_or_else(|| format!("{}:{}: unknown stage '{}'", path.display(), line_no, name))?; |
| 4428 | stages.insert(stage); |
| 4429 | } |
| 4430 | if stages.is_empty() { |
| 4431 | return Err(format!( |
| 4432 | "{}:{}: armfortas stage list is empty", |
| 4433 | path.display(), |
| 4434 | line_no |
| 4435 | )); |
| 4436 | } |
| 4437 | Ok(stages) |
| 4438 | } |
| 4439 | |
| 4440 | fn parse_compiler_artifact_declaration( |
| 4441 | rest: &str, |
| 4442 | path: &Path, |
| 4443 | line_no: usize, |
| 4444 | ) -> Result<(CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4445 | let (compiler_raw, artifact_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4446 | format!( |
| 4447 | "{}:{}: compiler declaration must use 'compiler <spec> => <artifacts>'", |
| 4448 | path.display(), |
| 4449 | line_no |
| 4450 | ) |
| 4451 | })?; |
| 4452 | let compiler = parse_compiler_spec_token(compiler_raw.trim(), path, line_no)?; |
| 4453 | let artifacts = ArtifactKey::parse_list(artifact_raw.trim()) |
| 4454 | .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4455 | if artifacts.is_empty() { |
| 4456 | return Err(format!( |
| 4457 | "{}:{}: generic compiler artifact list is empty", |
| 4458 | path.display(), |
| 4459 | line_no |
| 4460 | )); |
| 4461 | } |
| 4462 | Ok((compiler, artifacts)) |
| 4463 | } |
| 4464 | |
| 4465 | fn parse_compare_declaration( |
| 4466 | rest: &str, |
| 4467 | path: &Path, |
| 4468 | line_no: usize, |
| 4469 | ) -> Result<(CompilerSpec, CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4470 | let (compilers_raw, artifacts_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4471 | format!( |
| 4472 | "{}:{}: compare declaration must use 'compare <left> <right> => <artifacts>'", |
| 4473 | path.display(), |
| 4474 | line_no |
| 4475 | ) |
| 4476 | })?; |
| 4477 | let tokens = split_compiler_tokens(compilers_raw.trim(), path, line_no)?; |
| 4478 | if tokens.len() != 2 { |
| 4479 | return Err(format!( |
| 4480 | "{}:{}: compare declaration requires exactly two compiler specs", |
| 4481 | path.display(), |
| 4482 | line_no |
| 4483 | )); |
| 4484 | } |
| 4485 | let left = parse_compiler_spec_token(&tokens[0], path, line_no)?; |
| 4486 | let right = parse_compiler_spec_token(&tokens[1], path, line_no)?; |
| 4487 | let artifacts = ArtifactKey::parse_list(artifacts_raw.trim()) |
| 4488 | .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4489 | Ok((left, right, artifacts)) |
| 4490 | } |
| 4491 | |
| 4492 | fn split_compiler_tokens(raw: &str, path: &Path, line_no: usize) -> Result<Vec<String>, String> { |
| 4493 | let mut tokens = Vec::new(); |
| 4494 | let mut current = String::new(); |
| 4495 | let mut quoted = false; |
| 4496 | |
| 4497 | for ch in raw.chars() { |
| 4498 | match ch { |
| 4499 | '"' => { |
| 4500 | quoted = !quoted; |
| 4501 | current.push(ch); |
| 4502 | } |
| 4503 | c if c.is_whitespace() && !quoted => { |
| 4504 | if !current.trim().is_empty() { |
| 4505 | tokens.push(current.trim().to_string()); |
| 4506 | current.clear(); |
| 4507 | } |
| 4508 | } |
| 4509 | other => current.push(other), |
| 4510 | } |
| 4511 | } |
| 4512 | |
| 4513 | if quoted { |
| 4514 | return Err(format!( |
| 4515 | "{}:{}: unterminated quoted compiler spec '{}'", |
| 4516 | path.display(), |
| 4517 | line_no, |
| 4518 | raw |
| 4519 | )); |
| 4520 | } |
| 4521 | |
| 4522 | if !current.trim().is_empty() { |
| 4523 | tokens.push(current.trim().to_string()); |
| 4524 | } |
| 4525 | |
| 4526 | Ok(tokens) |
| 4527 | } |
| 4528 | |
| 4529 | fn parse_compiler_spec_token( |
| 4530 | raw: &str, |
| 4531 | path: &Path, |
| 4532 | line_no: usize, |
| 4533 | ) -> Result<CompilerSpec, String> { |
| 4534 | let token = if raw.starts_with('"') { |
| 4535 | parse_quoted(raw, path, line_no)? |
| 4536 | } else { |
| 4537 | raw.trim().to_string() |
| 4538 | }; |
| 4539 | if token.is_empty() { |
| 4540 | return Err(format!( |
| 4541 | "{}:{}: compiler declaration is missing a compiler spec", |
| 4542 | path.display(), |
| 4543 | line_no |
| 4544 | )); |
| 4545 | } |
| 4546 | if let Some(named) = NamedCompiler::parse(&token) { |
| 4547 | return Ok(CompilerSpec::Named(named)); |
| 4548 | } |
| 4549 | let parsed = PathBuf::from(&token); |
| 4550 | let resolved = if parsed.is_absolute() { |
| 4551 | parsed |
| 4552 | } else { |
| 4553 | path.parent().unwrap_or_else(|| Path::new(".")).join(parsed) |
| 4554 | }; |
| 4555 | Ok(CompilerSpec::Binary(resolved)) |
| 4556 | } |
| 4557 | |
| 4558 | fn parse_opt_levels(rest: &str, path: &Path, line_no: usize) -> Result<Vec<OptLevel>, String> { |
| 4559 | let mut levels = BTreeSet::new(); |
| 4560 | for raw in rest.split(',') { |
| 4561 | let name = raw.trim(); |
| 4562 | if name.is_empty() { |
| 4563 | continue; |
| 4564 | } |
| 4565 | if name.eq_ignore_ascii_case("all") { |
| 4566 | levels.extend(all_opt_levels()); |
| 4567 | continue; |
| 4568 | } |
| 4569 | let level = parse_opt_level_token(name).ok_or_else(|| { |
| 4570 | format!( |
| 4571 | "{}:{}: unknown opt level '{}'", |
| 4572 | path.display(), |
| 4573 | line_no, |
| 4574 | name |
| 4575 | ) |
| 4576 | })?; |
| 4577 | levels.insert(level); |
| 4578 | } |
| 4579 | if levels.is_empty() { |
| 4580 | return Err(format!( |
| 4581 | "{}:{}: opt level list is empty", |
| 4582 | path.display(), |
| 4583 | line_no |
| 4584 | )); |
| 4585 | } |
| 4586 | Ok(levels.into_iter().collect()) |
| 4587 | } |
| 4588 | |
| 4589 | fn parse_reference_compilers( |
| 4590 | rest: &str, |
| 4591 | path: &Path, |
| 4592 | line_no: usize, |
| 4593 | ) -> Result<Vec<ReferenceCompiler>, String> { |
| 4594 | let mut compilers = BTreeSet::new(); |
| 4595 | for raw in rest.split(',') { |
| 4596 | let name = raw.trim(); |
| 4597 | if name.is_empty() { |
| 4598 | continue; |
| 4599 | } |
| 4600 | let compiler = ReferenceCompiler::parse(name).ok_or_else(|| { |
| 4601 | format!( |
| 4602 | "{}:{}: unknown reference compiler '{}'", |
| 4603 | path.display(), |
| 4604 | line_no, |
| 4605 | name |
| 4606 | ) |
| 4607 | })?; |
| 4608 | compilers.insert(compiler); |
| 4609 | } |
| 4610 | if compilers.is_empty() { |
| 4611 | return Err(format!( |
| 4612 | "{}:{}: differential compiler list is empty", |
| 4613 | path.display(), |
| 4614 | line_no |
| 4615 | )); |
| 4616 | } |
| 4617 | Ok(compilers.into_iter().collect()) |
| 4618 | } |
| 4619 | |
| 4620 | fn parse_repeat_count(rest: &str, path: &Path, line_no: usize) -> Result<usize, String> { |
| 4621 | let count = rest.trim().parse::<usize>().map_err(|_| { |
| 4622 | format!( |
| 4623 | "{}:{}: repeat count must be an integer >= 2", |
| 4624 | path.display(), |
| 4625 | line_no |
| 4626 | ) |
| 4627 | })?; |
| 4628 | if count < 2 { |
| 4629 | return Err(format!( |
| 4630 | "{}:{}: repeat count must be >= 2", |
| 4631 | path.display(), |
| 4632 | line_no |
| 4633 | )); |
| 4634 | } |
| 4635 | Ok(count) |
| 4636 | } |
| 4637 | |
| 4638 | fn parse_consistency_checks( |
| 4639 | rest: &str, |
| 4640 | path: &Path, |
| 4641 | line_no: usize, |
| 4642 | ) -> Result<Vec<ConsistencyCheck>, String> { |
| 4643 | let mut checks = Vec::new(); |
| 4644 | for raw in rest.split(',') { |
| 4645 | let name = raw.trim(); |
| 4646 | if name.is_empty() { |
| 4647 | continue; |
| 4648 | } |
| 4649 | let check = ConsistencyCheck::parse(name).ok_or_else(|| { |
| 4650 | format!( |
| 4651 | "{}:{}: unknown consistency check '{}'", |
| 4652 | path.display(), |
| 4653 | line_no, |
| 4654 | name |
| 4655 | ) |
| 4656 | })?; |
| 4657 | if !checks.contains(&check) { |
| 4658 | checks.push(check); |
| 4659 | } |
| 4660 | } |
| 4661 | if checks.is_empty() { |
| 4662 | return Err(format!( |
| 4663 | "{}:{}: consistency check list is empty", |
| 4664 | path.display(), |
| 4665 | line_no |
| 4666 | )); |
| 4667 | } |
| 4668 | Ok(checks) |
| 4669 | } |
| 4670 | |
| 4671 | fn parse_expectation(rest: &str, path: &Path, line_no: usize) -> Result<Expectation, String> { |
| 4672 | if let Some(prefix) = rest.strip_suffix(" check-comments") { |
| 4673 | return Ok(Expectation::CheckComments(parse_target( |
| 4674 | prefix.trim(), |
| 4675 | path, |
| 4676 | line_no, |
| 4677 | )?)); |
| 4678 | } |
| 4679 | |
| 4680 | if let Some((target, value)) = rest.split_once(" not-contains ") { |
| 4681 | return Ok(Expectation::NotContains { |
| 4682 | target: parse_target(target.trim(), path, line_no)?, |
| 4683 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4684 | }); |
| 4685 | } |
| 4686 | |
| 4687 | if let Some((target, value)) = rest.split_once(" contains ") { |
| 4688 | return Ok(Expectation::Contains { |
| 4689 | target: parse_target(target.trim(), path, line_no)?, |
| 4690 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4691 | }); |
| 4692 | } |
| 4693 | |
| 4694 | if let Some((target, value)) = rest.split_once(" equals ") { |
| 4695 | let target = parse_target(target.trim(), path, line_no)?; |
| 4696 | if matches!( |
| 4697 | target, |
| 4698 | Target::RunExitCode |
| 4699 | | Target::Artifact(ArtifactKey::ExitCode) |
| 4700 | | Target::CompareDifferenceCount |
| 4701 | ) { |
| 4702 | let value = parse_integer(value.trim(), path, line_no)?; |
| 4703 | return Ok(Expectation::IntEquals { target, value }); |
| 4704 | } |
| 4705 | return Ok(Expectation::Equals { |
| 4706 | target, |
| 4707 | value: parse_quoted(value.trim(), path, line_no)?, |
| 4708 | }); |
| 4709 | } |
| 4710 | |
| 4711 | Err(format!( |
| 4712 | "{}:{}: unsupported expectation '{}'", |
| 4713 | path.display(), |
| 4714 | line_no, |
| 4715 | rest |
| 4716 | )) |
| 4717 | } |
| 4718 | |
| 4719 | fn parse_failure_expectation( |
| 4720 | rest: &str, |
| 4721 | path: &Path, |
| 4722 | line_no: usize, |
| 4723 | ) -> Result<Expectation, String> { |
| 4724 | if rest.trim().eq_ignore_ascii_case("comments") { |
| 4725 | return Ok(Expectation::FailSourceComments); |
| 4726 | } |
| 4727 | |
| 4728 | if let Some((target, value)) = rest.split_once(" contains ") { |
| 4729 | return Ok(Expectation::FailContains { |
| 4730 | stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4731 | needle: parse_quoted(value.trim(), path, line_no)?, |
| 4732 | }); |
| 4733 | } |
| 4734 | |
| 4735 | if let Some((target, value)) = rest.split_once(" equals ") { |
| 4736 | return Ok(Expectation::FailEquals { |
| 4737 | stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4738 | value: parse_quoted(value.trim(), path, line_no)?, |
| 4739 | }); |
| 4740 | } |
| 4741 | |
| 4742 | Err(format!( |
| 4743 | "{}:{}: unsupported failure expectation '{}'", |
| 4744 | path.display(), |
| 4745 | line_no, |
| 4746 | rest |
| 4747 | )) |
| 4748 | } |
| 4749 | |
| 4750 | fn parse_status_rule( |
| 4751 | kind: StatusKind, |
| 4752 | rest: &str, |
| 4753 | path: &Path, |
| 4754 | line_no: usize, |
| 4755 | ) -> Result<PendingStatusRule, String> { |
| 4756 | let rest = rest.trim(); |
| 4757 | if kind == StatusKind::Xfail && rest.eq_ignore_ascii_case("comments") { |
| 4758 | return Ok(PendingStatusRule::XfailSourceComments); |
| 4759 | } |
| 4760 | if rest.starts_with('"') { |
| 4761 | return Ok(PendingStatusRule::Explicit(StatusRule { |
| 4762 | kind, |
| 4763 | selector: OptSelector::All, |
| 4764 | reason: parse_quoted(rest, path, line_no)?, |
| 4765 | })); |
| 4766 | } |
| 4767 | |
| 4768 | let conditional = rest.strip_prefix("when ").ok_or_else(|| { |
| 4769 | format!( |
| 4770 | "{}:{}: expected quoted reason or 'when <opts> because \"...\"'", |
| 4771 | path.display(), |
| 4772 | line_no |
| 4773 | ) |
| 4774 | })?; |
| 4775 | let (selector, reason) = conditional.split_once(" because ").ok_or_else(|| { |
| 4776 | format!( |
| 4777 | "{}:{}: conditional status must use 'when <opts> because \"...\"'", |
| 4778 | path.display(), |
| 4779 | line_no |
| 4780 | ) |
| 4781 | })?; |
| 4782 | |
| 4783 | Ok(PendingStatusRule::Explicit(StatusRule { |
| 4784 | kind, |
| 4785 | selector: parse_opt_selector(selector.trim(), path, line_no)?, |
| 4786 | reason: parse_quoted(reason.trim(), path, line_no)?, |
| 4787 | })) |
| 4788 | } |
| 4789 | |
| 4790 | fn parse_capability_policy( |
| 4791 | kind: StatusKind, |
| 4792 | rest: &str, |
| 4793 | path: &Path, |
| 4794 | line_no: usize, |
| 4795 | ) -> Result<CapabilityPolicy, String> { |
| 4796 | Ok(CapabilityPolicy { |
| 4797 | kind, |
| 4798 | reason: parse_quoted(rest.trim(), path, line_no)?, |
| 4799 | }) |
| 4800 | } |
| 4801 | |
| 4802 | fn resolve_source_comment_expectations( |
| 4803 | expectations: Vec<Expectation>, |
| 4804 | source_text: Option<&str>, |
| 4805 | suite_path: &Path, |
| 4806 | case_name: &str, |
| 4807 | source_path: &Path, |
| 4808 | ) -> Result<Vec<Expectation>, String> { |
| 4809 | let mut resolved = Vec::with_capacity(expectations.len()); |
| 4810 | for expectation in expectations { |
| 4811 | match expectation { |
| 4812 | Expectation::FailSourceComments => { |
| 4813 | let source_text = source_text.ok_or_else(|| { |
| 4814 | format!( |
| 4815 | "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4816 | suite_path.display(), |
| 4817 | case_name, |
| 4818 | source_path.display() |
| 4819 | ) |
| 4820 | })?; |
| 4821 | let patterns = extract_error_expected_patterns(source_text); |
| 4822 | if patterns.is_empty() { |
| 4823 | return Err(format!( |
| 4824 | "{}: case '{}' requests expect-fail comments but '{}' has no ! ERROR_EXPECTED: lines", |
| 4825 | suite_path.display(), |
| 4826 | case_name, |
| 4827 | source_path.display() |
| 4828 | )); |
| 4829 | } |
| 4830 | resolved.push(Expectation::FailCommentPatterns(patterns)); |
| 4831 | } |
| 4832 | other => resolved.push(other), |
| 4833 | } |
| 4834 | } |
| 4835 | Ok(resolved) |
| 4836 | } |
| 4837 | |
| 4838 | fn resolve_source_comment_status_rules( |
| 4839 | status_rules: Vec<PendingStatusRule>, |
| 4840 | source_text: Option<&str>, |
| 4841 | suite_path: &Path, |
| 4842 | case_name: &str, |
| 4843 | source_path: &Path, |
| 4844 | ) -> Result<Vec<StatusRule>, String> { |
| 4845 | let mut resolved = Vec::with_capacity(status_rules.len()); |
| 4846 | for rule in status_rules { |
| 4847 | match rule { |
| 4848 | PendingStatusRule::Explicit(rule) => resolved.push(rule), |
| 4849 | PendingStatusRule::XfailSourceComments => { |
| 4850 | let source_text = source_text.ok_or_else(|| { |
| 4851 | format!( |
| 4852 | "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4853 | suite_path.display(), |
| 4854 | case_name, |
| 4855 | source_path.display() |
| 4856 | ) |
| 4857 | })?; |
| 4858 | let reason = extract_xfail_reason(source_text).ok_or_else(|| { |
| 4859 | format!( |
| 4860 | "{}: case '{}' requests xfail comments but '{}' has no ! XFAIL: lines", |
| 4861 | suite_path.display(), |
| 4862 | case_name, |
| 4863 | source_path.display() |
| 4864 | ) |
| 4865 | })?; |
| 4866 | resolved.push(StatusRule { |
| 4867 | kind: StatusKind::Xfail, |
| 4868 | selector: OptSelector::All, |
| 4869 | reason, |
| 4870 | }); |
| 4871 | } |
| 4872 | } |
| 4873 | } |
| 4874 | Ok(resolved) |
| 4875 | } |
| 4876 | |
| 4877 | fn parse_opt_selector(raw: &str, path: &Path, line_no: usize) -> Result<OptSelector, String> { |
| 4878 | let raw = raw.trim(); |
| 4879 | if raw.eq_ignore_ascii_case("all") { |
| 4880 | return Ok(OptSelector::All); |
| 4881 | } |
| 4882 | if let Some(rest) = raw.strip_prefix("opts =>") { |
| 4883 | return Ok(OptSelector::Only(parse_opt_levels(rest, path, line_no)?)); |
| 4884 | } |
| 4885 | Ok(OptSelector::Only(parse_opt_levels(raw, path, line_no)?)) |
| 4886 | } |
| 4887 | |
| 4888 | fn parse_target(raw: &str, path: &Path, line_no: usize) -> Result<Target, String> { |
| 4889 | match raw { |
| 4890 | "compare.status" => Ok(Target::CompareStatus), |
| 4891 | "compare.classification" => Ok(Target::CompareClassification), |
| 4892 | "compare.changed_artifacts" => Ok(Target::CompareChangedArtifacts), |
| 4893 | "compare.difference_count" => Ok(Target::CompareDifferenceCount), |
| 4894 | "compare.basis" => Ok(Target::CompareBasis), |
| 4895 | "run.stdout" => Ok(Target::RunStdout), |
| 4896 | "run.stderr" => Ok(Target::RunStderr), |
| 4897 | "run.exit_code" => Ok(Target::RunExitCode), |
| 4898 | _ => { |
| 4899 | if let Some(artifact) = ArtifactKey::parse(raw) { |
| 4900 | return Ok(Target::Artifact(artifact)); |
| 4901 | } |
| 4902 | let stage = Stage::parse(raw).ok_or_else(|| { |
| 4903 | format!( |
| 4904 | "{}:{}: unsupported expectation target '{}'", |
| 4905 | path.display(), |
| 4906 | line_no, |
| 4907 | raw |
| 4908 | ) |
| 4909 | })?; |
| 4910 | Ok(Target::Stage(stage)) |
| 4911 | } |
| 4912 | } |
| 4913 | } |
| 4914 | |
| 4915 | fn parse_failure_stage(raw: &str, path: &Path, line_no: usize) -> Result<FailureStage, String> { |
| 4916 | FailureStage::parse(raw).ok_or_else(|| { |
| 4917 | format!( |
| 4918 | "{}:{}: unsupported failure stage '{}'", |
| 4919 | path.display(), |
| 4920 | line_no, |
| 4921 | raw |
| 4922 | ) |
| 4923 | }) |
| 4924 | } |
| 4925 | |
| 4926 | fn parse_quoted(raw: &str, path: &Path, line_no: usize) -> Result<String, String> { |
| 4927 | let raw = raw.trim(); |
| 4928 | if !(raw.starts_with('"') && raw.ends_with('"')) { |
| 4929 | return Err(format!( |
| 4930 | "{}:{}: expected quoted string, got '{}'", |
| 4931 | path.display(), |
| 4932 | line_no, |
| 4933 | raw |
| 4934 | )); |
| 4935 | } |
| 4936 | let body = &raw[1..raw.len() - 1]; |
| 4937 | Ok(body.replace("\\\"", "\"").replace("\\n", "\n")) |
| 4938 | } |
| 4939 | |
| 4940 | fn parse_integer(raw: &str, path: &Path, line_no: usize) -> Result<i32, String> { |
| 4941 | let value = if raw.starts_with('"') { |
| 4942 | parse_quoted(raw, path, line_no)? |
| 4943 | } else { |
| 4944 | raw.trim().to_string() |
| 4945 | }; |
| 4946 | value.parse::<i32>().map_err(|_| { |
| 4947 | format!( |
| 4948 | "{}:{}: expected integer literal, got '{}'", |
| 4949 | path.display(), |
| 4950 | line_no, |
| 4951 | raw |
| 4952 | ) |
| 4953 | }) |
| 4954 | } |
| 4955 | |
| 4956 | fn parse_opt_level_token(raw: &str) -> Option<OptLevel> { |
| 4957 | let raw = raw.trim(); |
| 4958 | let raw = raw.strip_prefix('-').unwrap_or(raw); |
| 4959 | OptLevel::parse_flag(raw) |
| 4960 | } |
| 4961 | |
| 4962 | fn parse_opt_level_list(raw: &str) -> Result<Vec<OptLevel>, String> { |
| 4963 | let mut levels = BTreeSet::new(); |
| 4964 | for value in raw.split(',') { |
| 4965 | let value = value.trim(); |
| 4966 | if value.is_empty() { |
| 4967 | continue; |
| 4968 | } |
| 4969 | if value.eq_ignore_ascii_case("all") { |
| 4970 | levels.extend(all_opt_levels()); |
| 4971 | continue; |
| 4972 | } |
| 4973 | let level = |
| 4974 | parse_opt_level_token(value).ok_or_else(|| format!("unknown opt level '{}'", value))?; |
| 4975 | levels.insert(level); |
| 4976 | } |
| 4977 | if levels.is_empty() { |
| 4978 | return Err("opt filter is empty".into()); |
| 4979 | } |
| 4980 | Ok(levels.into_iter().collect()) |
| 4981 | } |
| 4982 | |
| 4983 | fn all_opt_levels() -> [OptLevel; 5] { |
| 4984 | [ |
| 4985 | OptLevel::O0, |
| 4986 | OptLevel::O1, |
| 4987 | OptLevel::O2, |
| 4988 | OptLevel::O3, |
| 4989 | OptLevel::Ofast, |
| 4990 | ] |
| 4991 | } |
| 4992 | |
| 4993 | fn filter_suites<'a>(suites: &'a [SuiteSpec], suite_filter: Option<&str>) -> Vec<&'a SuiteSpec> { |
| 4994 | let filter = suite_filter.map(|value| value.to_ascii_lowercase()); |
| 4995 | suites |
| 4996 | .iter() |
| 4997 | .filter(|suite| { |
| 4998 | if let Some(filter) = &filter { |
| 4999 | suite.name.to_ascii_lowercase().contains(filter) |
| 5000 | } else { |
| 5001 | true |
| 5002 | } |
| 5003 | }) |
| 5004 | .collect() |
| 5005 | } |
| 5006 | |
| 5007 | fn print_suites(suites: &[&SuiteSpec], config: &ListConfig) { |
| 5008 | for suite in suites { |
| 5009 | println!("{} ({})", suite.name, suite.cases.len()); |
| 5010 | println!(" {}", suite.path.display()); |
| 5011 | if config.verbose { |
| 5012 | for case in &suite.cases { |
| 5013 | println!(" - {} [{}]", case.name, case_discovery_mode_label(case)); |
| 5014 | for line in case_discovery_lines(case, &config.tools) { |
| 5015 | println!(" {}", line); |
| 5016 | } |
| 5017 | } |
| 5018 | } |
| 5019 | } |
| 5020 | } |
| 5021 | |
| 5022 | fn case_discovery_mode_label(case: &CaseSpec) -> &'static str { |
| 5023 | if case.is_generic_compare() { |
| 5024 | "generic-compare" |
| 5025 | } else if case.is_generic_introspect() { |
| 5026 | "generic-introspect" |
| 5027 | } else { |
| 5028 | "legacy" |
| 5029 | } |
| 5030 | } |
| 5031 | |
| 5032 | fn format_opt_level_list(levels: &[OptLevel]) -> String { |
| 5033 | levels |
| 5034 | .iter() |
| 5035 | .map(OptLevel::as_str) |
| 5036 | .collect::<Vec<_>>() |
| 5037 | .join(", ") |
| 5038 | } |
| 5039 | |
| 5040 | fn case_discovery_lines(case: &CaseSpec, tools: &ToolchainConfig) -> Vec<String> { |
| 5041 | let mut lines = vec![ |
| 5042 | format!("source: {}", case.source_label()), |
| 5043 | format!("opts: {}", format_opt_level_list(&case.opt_levels)), |
| 5044 | ]; |
| 5045 | if let Some(policy) = &case.capability_policy { |
| 5046 | lines.push(format!( |
| 5047 | "capability_policy: {} when blocked ({})", |
| 5048 | match policy.kind { |
| 5049 | StatusKind::Xfail => "xfail", |
| 5050 | StatusKind::Future => "future", |
| 5051 | }, |
| 5052 | policy.reason |
| 5053 | )); |
| 5054 | } |
| 5055 | |
| 5056 | if let Some(generic) = &case.generic_introspect { |
| 5057 | let capture_root = tools.armfortas_adapters().capture_root(); |
| 5058 | let probe = compiler_spec_probe(&generic.compiler, tools, capture_root.as_ref()); |
| 5059 | lines.push(format!("compiler: {}", generic.compiler.display_name())); |
| 5060 | lines.push(format!("compiler_probe_status: {}", probe.status)); |
| 5061 | lines.push(format!( |
| 5062 | "compiler_probe_resolved_path: {}", |
| 5063 | probe |
| 5064 | .resolved_path |
| 5065 | .as_ref() |
| 5066 | .map(|path| display_path(path)) |
| 5067 | .unwrap_or_else(|| "none".to_string()) |
| 5068 | )); |
| 5069 | if let Some(banner) = &probe.banner { |
| 5070 | lines.push(format!("compiler_probe_banner: {}", banner)); |
| 5071 | } |
| 5072 | lines.push(format!( |
| 5073 | "artifacts: {}", |
| 5074 | format_artifact_name_list( |
| 5075 | &generic |
| 5076 | .artifacts |
| 5077 | .iter() |
| 5078 | .map(|artifact| artifact.as_str().to_string()) |
| 5079 | .collect::<Vec<_>>() |
| 5080 | ) |
| 5081 | )); |
| 5082 | match capability_request_issue(&generic.compiler, &generic.artifacts, tools) { |
| 5083 | Some(issue) => { |
| 5084 | lines.push(if case.capability_policy.is_some() { |
| 5085 | "capability_status: deferred".to_string() |
| 5086 | } else { |
| 5087 | "capability_status: blocked".to_string() |
| 5088 | }); |
| 5089 | lines.extend( |
| 5090 | issue |
| 5091 | .lines() |
| 5092 | .map(|line| format!("capability_detail: {}", line)), |
| 5093 | ); |
| 5094 | } |
| 5095 | None => lines.push("capability_status: ready".to_string()), |
| 5096 | } |
| 5097 | return lines; |
| 5098 | } |
| 5099 | |
| 5100 | if let Some(generic) = &case.generic_compare { |
| 5101 | let capture_root = tools.armfortas_adapters().capture_root(); |
| 5102 | let left_probe = compiler_spec_probe(&generic.left, tools, capture_root.as_ref()); |
| 5103 | let right_probe = compiler_spec_probe(&generic.right, tools, capture_root.as_ref()); |
| 5104 | lines.push(format!( |
| 5105 | "compare: {} vs {}", |
| 5106 | generic.left.display_name(), |
| 5107 | generic.right.display_name() |
| 5108 | )); |
| 5109 | lines.push(format!("left_probe_status: {}", left_probe.status)); |
| 5110 | lines.push(format!( |
| 5111 | "left_probe_resolved_path: {}", |
| 5112 | left_probe |
| 5113 | .resolved_path |
| 5114 | .as_ref() |
| 5115 | .map(|path| display_path(path)) |
| 5116 | .unwrap_or_else(|| "none".to_string()) |
| 5117 | )); |
| 5118 | if let Some(banner) = &left_probe.banner { |
| 5119 | lines.push(format!("left_probe_banner: {}", banner)); |
| 5120 | } |
| 5121 | lines.push(format!("right_probe_status: {}", right_probe.status)); |
| 5122 | lines.push(format!( |
| 5123 | "right_probe_resolved_path: {}", |
| 5124 | right_probe |
| 5125 | .resolved_path |
| 5126 | .as_ref() |
| 5127 | .map(|path| display_path(path)) |
| 5128 | .unwrap_or_else(|| "none".to_string()) |
| 5129 | )); |
| 5130 | if let Some(banner) = &right_probe.banner { |
| 5131 | lines.push(format!("right_probe_banner: {}", banner)); |
| 5132 | } |
| 5133 | lines.push(format!( |
| 5134 | "artifacts: {}", |
| 5135 | format_artifact_name_list( |
| 5136 | &generic |
| 5137 | .artifacts |
| 5138 | .iter() |
| 5139 | .map(|artifact| artifact.as_str().to_string()) |
| 5140 | .collect::<Vec<_>>() |
| 5141 | ) |
| 5142 | )); |
| 5143 | let mut issues = Vec::new(); |
| 5144 | if let Some(issue) = capability_request_issue(&generic.left, &generic.artifacts, tools) { |
| 5145 | issues.push(format!("left:\n{}", issue)); |
| 5146 | } |
| 5147 | if let Some(issue) = capability_request_issue(&generic.right, &generic.artifacts, tools) { |
| 5148 | issues.push(format!("right:\n{}", issue)); |
| 5149 | } |
| 5150 | if issues.is_empty() { |
| 5151 | lines.push("capability_status: ready".to_string()); |
| 5152 | } else { |
| 5153 | lines.push(if case.capability_policy.is_some() { |
| 5154 | "capability_status: deferred".to_string() |
| 5155 | } else { |
| 5156 | "capability_status: blocked".to_string() |
| 5157 | }); |
| 5158 | lines.extend(issues.into_iter().flat_map(|issue| { |
| 5159 | issue |
| 5160 | .lines() |
| 5161 | .map(|line| format!("capability_detail: {}", line)) |
| 5162 | .collect::<Vec<_>>() |
| 5163 | })); |
| 5164 | } |
| 5165 | return lines; |
| 5166 | } |
| 5167 | |
| 5168 | let needs_linked_capture = primary_backend_kind_for_case(case, &case.requested, tools) |
| 5169 | == PrimaryCaptureBackendKind::Full; |
| 5170 | if case.requested.is_empty() { |
| 5171 | lines.push("stages: run".to_string()); |
| 5172 | } else { |
| 5173 | let stages = case |
| 5174 | .requested |
| 5175 | .iter() |
| 5176 | .map(Stage::as_str) |
| 5177 | .collect::<Vec<_>>() |
| 5178 | .join(", "); |
| 5179 | lines.push(format!("stages: {}", stages)); |
| 5180 | } |
| 5181 | if !case.reference_compilers.is_empty() { |
| 5182 | lines.push(format!( |
| 5183 | "differential: {}", |
| 5184 | case.reference_compilers |
| 5185 | .iter() |
| 5186 | .map(ReferenceCompiler::as_str) |
| 5187 | .collect::<Vec<_>>() |
| 5188 | .join(", ") |
| 5189 | )); |
| 5190 | } |
| 5191 | if !case.consistency_checks.is_empty() { |
| 5192 | lines.push(format!( |
| 5193 | "consistency: {}", |
| 5194 | case.consistency_checks |
| 5195 | .iter() |
| 5196 | .map(ConsistencyCheck::as_str) |
| 5197 | .collect::<Vec<_>>() |
| 5198 | .join(", ") |
| 5199 | )); |
| 5200 | } |
| 5201 | if needs_linked_capture { |
| 5202 | lines.push("surface: linked armfortas capture".to_string()); |
| 5203 | if linked_capture_available() { |
| 5204 | lines.push("capability_status: ready".to_string()); |
| 5205 | } else { |
| 5206 | lines.push(if case.capability_policy.is_some() { |
| 5207 | "capability_status: deferred".to_string() |
| 5208 | } else { |
| 5209 | "capability_status: blocked".to_string() |
| 5210 | }); |
| 5211 | lines.push( |
| 5212 | "capability_detail: linked armfortas capture is unavailable in this build" |
| 5213 | .to_string(), |
| 5214 | ); |
| 5215 | } |
| 5216 | } else { |
| 5217 | lines.push("surface: observable-only legacy path".to_string()); |
| 5218 | lines.push("capability_status: ready".to_string()); |
| 5219 | } |
| 5220 | |
| 5221 | lines |
| 5222 | } |
| 5223 | |
| 5224 | fn run_suites(config: &RunConfig) -> Result<Summary, String> { |
| 5225 | let suites = discover_suites(default_suite_root())?; |
| 5226 | let suites = filter_suites(&suites, config.suite_filter.as_deref()); |
| 5227 | if suites.is_empty() { |
| 5228 | return Err("no suites matched the requested filter".into()); |
| 5229 | } |
| 5230 | |
| 5231 | let case_filter = config |
| 5232 | .case_filter |
| 5233 | .as_ref() |
| 5234 | .map(|value| value.to_ascii_lowercase()); |
| 5235 | let mut summary = Summary::default(); |
| 5236 | let mut matched_cells = 0usize; |
| 5237 | |
| 5238 | for suite in suites { |
| 5239 | println!("=== {} ===", suite.name); |
| 5240 | for case in &suite.cases { |
| 5241 | if let Some(filter) = &case_filter { |
| 5242 | if !case.name.to_ascii_lowercase().contains(filter) { |
| 5243 | continue; |
| 5244 | } |
| 5245 | } |
| 5246 | |
| 5247 | let opt_levels = selected_opt_levels(case, config); |
| 5248 | for opt_level in opt_levels { |
| 5249 | matched_cells += 1; |
| 5250 | let outcome = execute_case_cell(suite, case, opt_level, config)?; |
| 5251 | print_outcome(&outcome); |
| 5252 | summary.record_outcome(&outcome); |
| 5253 | |
| 5254 | if config.fail_fast |
| 5255 | && matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5256 | { |
| 5257 | return Ok(summary); |
| 5258 | } |
| 5259 | } |
| 5260 | } |
| 5261 | } |
| 5262 | |
| 5263 | if matched_cells == 0 { |
| 5264 | return Err("no cases matched the requested filters".into()); |
| 5265 | } |
| 5266 | |
| 5267 | Ok(summary) |
| 5268 | } |
| 5269 | |
| 5270 | fn selected_opt_levels(case: &CaseSpec, config: &RunConfig) -> Vec<OptLevel> { |
| 5271 | case.opt_levels |
| 5272 | .iter() |
| 5273 | .copied() |
| 5274 | .filter(|level| { |
| 5275 | config |
| 5276 | .opt_filter |
| 5277 | .as_ref() |
| 5278 | .map(|filter| filter.contains(level)) |
| 5279 | .unwrap_or(true) |
| 5280 | }) |
| 5281 | .collect() |
| 5282 | } |
| 5283 | |
| 5284 | fn execute_case_cell( |
| 5285 | suite: &SuiteSpec, |
| 5286 | case: &CaseSpec, |
| 5287 | opt_level: OptLevel, |
| 5288 | config: &RunConfig, |
| 5289 | ) -> Result<Outcome, String> { |
| 5290 | if case.is_generic_compare() { |
| 5291 | return execute_generic_compare_case_cell(suite, case, opt_level, config); |
| 5292 | } |
| 5293 | if case.is_generic_introspect() { |
| 5294 | return execute_generic_introspect_case_cell(suite, case, opt_level, config); |
| 5295 | } |
| 5296 | |
| 5297 | let effective_status = status_for_opt(case, opt_level); |
| 5298 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5299 | if !config.include_future { |
| 5300 | return Ok(Outcome { |
| 5301 | suite: suite.name.clone(), |
| 5302 | case: case.name.clone(), |
| 5303 | opt_level, |
| 5304 | kind: OutcomeKind::Future, |
| 5305 | detail: reason.clone(), |
| 5306 | bundle: None, |
| 5307 | primary_backend: None, |
| 5308 | consistency_observations: Vec::new(), |
| 5309 | }); |
| 5310 | } |
| 5311 | } |
| 5312 | |
| 5313 | let mut requested = case.requested.clone(); |
| 5314 | if config.all_stages { |
| 5315 | requested.extend(Stage::ALL); |
| 5316 | } |
| 5317 | for expectation in &case.expectations { |
| 5318 | ensure_target_stage(expectation, &mut requested); |
| 5319 | } |
| 5320 | if !case.reference_compilers.is_empty() { |
| 5321 | requested.insert(Stage::Run); |
| 5322 | } |
| 5323 | for check in &case.consistency_checks { |
| 5324 | ensure_consistency_stage(*check, &mut requested); |
| 5325 | } |
| 5326 | |
| 5327 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5328 | let selected_backend = |
| 5329 | select_primary_capture_backend(case, &requested, opt_level, &config.tools); |
| 5330 | |
| 5331 | if let Some(detail) = legacy_unavailable_backend_detail(case, &selected_backend) { |
| 5332 | cleanup_prepared_input(&prepared); |
| 5333 | return Ok(outcome_from_status_and_execution( |
| 5334 | suite, |
| 5335 | case, |
| 5336 | opt_level, |
| 5337 | capability_effective_status(&effective_status, case), |
| 5338 | Err(detail), |
| 5339 | Some(PrimaryBackendReport::from_selected(&selected_backend)), |
| 5340 | Vec::new(), |
| 5341 | )); |
| 5342 | } |
| 5343 | |
| 5344 | if config.verbose { |
| 5345 | let stage_list = requested |
| 5346 | .iter() |
| 5347 | .map(Stage::as_str) |
| 5348 | .collect::<Vec<_>>() |
| 5349 | .join(", "); |
| 5350 | let refs = if case.reference_compilers.is_empty() { |
| 5351 | "none".to_string() |
| 5352 | } else { |
| 5353 | case.reference_compilers |
| 5354 | .iter() |
| 5355 | .map(ReferenceCompiler::as_str) |
| 5356 | .collect::<Vec<_>>() |
| 5357 | .join(", ") |
| 5358 | }; |
| 5359 | println!(" source: {}", case.source_label()); |
| 5360 | if case.is_graph() { |
| 5361 | for file in &case.graph_files { |
| 5362 | println!(" file: {}", file.display()); |
| 5363 | } |
| 5364 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5365 | } |
| 5366 | println!(" opt: {}", opt_level.as_str()); |
| 5367 | println!(" stages: {}", stage_list); |
| 5368 | println!( |
| 5369 | " primary_backend: {} ({})", |
| 5370 | selected_backend.kind.as_str(), |
| 5371 | selected_backend.backend.mode_name() |
| 5372 | ); |
| 5373 | println!( |
| 5374 | " primary_backend_detail: {}", |
| 5375 | selected_backend.backend.description() |
| 5376 | ); |
| 5377 | println!(" refs: {}", refs); |
| 5378 | if !case.consistency_checks.is_empty() { |
| 5379 | println!(" repeat: {}", case.repeat_count); |
| 5380 | } |
| 5381 | } |
| 5382 | |
| 5383 | let references = run_reference_compilers(&prepared, case, opt_level, &config.tools); |
| 5384 | let mut artifacts = ExecutionArtifacts { |
| 5385 | requested, |
| 5386 | armfortas: None, |
| 5387 | armfortas_failure: None, |
| 5388 | armfortas_observation: None, |
| 5389 | references, |
| 5390 | reference_observations: Vec::new(), |
| 5391 | consistency_issues: Vec::new(), |
| 5392 | }; |
| 5393 | artifacts.reference_observations = artifacts |
| 5394 | .references |
| 5395 | .iter() |
| 5396 | .map(|reference| { |
| 5397 | observed_program_from_reference_result( |
| 5398 | &prepared.compiler_source, |
| 5399 | opt_level, |
| 5400 | default_differential_artifacts(), |
| 5401 | reference, |
| 5402 | ) |
| 5403 | }) |
| 5404 | .collect(); |
| 5405 | |
| 5406 | match execute_primary_armfortas( |
| 5407 | &prepared, |
| 5408 | opt_level, |
| 5409 | &artifacts.requested, |
| 5410 | &selected_backend, |
| 5411 | ) { |
| 5412 | Ok(result) => artifacts.armfortas = Some(result), |
| 5413 | Err(failure) => artifacts.armfortas_failure = Some(failure), |
| 5414 | } |
| 5415 | |
| 5416 | let execution = match (&artifacts.armfortas, &artifacts.armfortas_failure) { |
| 5417 | (Some(result), None) => { |
| 5418 | if has_failure_expectation(case) { |
| 5419 | Err(format!( |
| 5420 | "expected armfortas to fail ({}) but compilation succeeded", |
| 5421 | expected_failure_description(case) |
| 5422 | )) |
| 5423 | } else { |
| 5424 | let observed = legacy_success_observed_program( |
| 5425 | case, |
| 5426 | &prepared.compiler_source, |
| 5427 | opt_level, |
| 5428 | result, |
| 5429 | !artifacts.references.is_empty(), |
| 5430 | &config.tools, |
| 5431 | ); |
| 5432 | artifacts.armfortas_observation = Some(observed.clone()); |
| 5433 | let mut execution = evaluate_observation_expectations(case, &observed); |
| 5434 | if execution.is_ok() && !artifacts.references.is_empty() { |
| 5435 | let references = artifacts |
| 5436 | .reference_observations |
| 5437 | .iter() |
| 5438 | .map(|observed| observed.observation.clone()) |
| 5439 | .collect::<Vec<_>>(); |
| 5440 | execution = compare_differential(&observed.observation, &references); |
| 5441 | } |
| 5442 | if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5443 | artifacts.consistency_issues = |
| 5444 | if legacy_case_uses_generic_consistency_checks(case) { |
| 5445 | run_generic_consistency_checks( |
| 5446 | &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5447 | case, |
| 5448 | &prepared.compiler_source, |
| 5449 | opt_level, |
| 5450 | &config.tools, |
| 5451 | ) |
| 5452 | } else { |
| 5453 | run_consistency_checks( |
| 5454 | case, |
| 5455 | &prepared, |
| 5456 | opt_level, |
| 5457 | result, |
| 5458 | &config.tools, |
| 5459 | ) |
| 5460 | }; |
| 5461 | if !artifacts.consistency_issues.is_empty() { |
| 5462 | execution = Err(format_consistency_issues(&artifacts.consistency_issues)); |
| 5463 | } |
| 5464 | } |
| 5465 | execution |
| 5466 | } |
| 5467 | } |
| 5468 | (None, Some(failure)) => { |
| 5469 | let observed = |
| 5470 | legacy_failure_observed_program(&prepared.compiler_source, case, failure); |
| 5471 | artifacts.armfortas_observation = Some(observed.clone()); |
| 5472 | let mut execution = |
| 5473 | evaluate_failed_armfortas_with_observed(case, &artifacts, &observed); |
| 5474 | if execution.is_ok() && !artifacts.references.is_empty() { |
| 5475 | execution = |
| 5476 | Err("differential comparison requires a successful armfortas run".to_string()); |
| 5477 | } |
| 5478 | execution |
| 5479 | } |
| 5480 | (Some(_), Some(_)) => Err("armfortas produced both a result and a failure".into()), |
| 5481 | (None, None) => Err("armfortas produced neither a result nor a failure".into()), |
| 5482 | }; |
| 5483 | |
| 5484 | let consistency_observations = artifacts |
| 5485 | .consistency_issues |
| 5486 | .iter() |
| 5487 | .map(ConsistencyIssue::observation) |
| 5488 | .collect::<Vec<_>>(); |
| 5489 | let primary_backend = Some(PrimaryBackendReport::from_selected(&selected_backend)); |
| 5490 | |
| 5491 | let mut outcome = outcome_from_status_and_execution( |
| 5492 | suite, |
| 5493 | case, |
| 5494 | opt_level, |
| 5495 | effective_status, |
| 5496 | execution, |
| 5497 | primary_backend, |
| 5498 | consistency_observations, |
| 5499 | ); |
| 5500 | |
| 5501 | let should_bundle = matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5502 | || (matches!(outcome.kind, OutcomeKind::Xfail) && !artifacts.consistency_issues.is_empty()); |
| 5503 | |
| 5504 | if should_bundle { |
| 5505 | match write_failure_bundle(suite, case, &prepared, &outcome, &artifacts) { |
| 5506 | Ok(bundle) => outcome.bundle = Some(bundle), |
| 5507 | Err(err) => { |
| 5508 | if outcome.detail.is_empty() { |
| 5509 | outcome.detail = format!("failed to write failure bundle: {}", err); |
| 5510 | } else { |
| 5511 | outcome.detail.push_str(&format!( |
| 5512 | "\n\nwarning: failed to write failure bundle: {}", |
| 5513 | err |
| 5514 | )); |
| 5515 | } |
| 5516 | } |
| 5517 | } |
| 5518 | } |
| 5519 | |
| 5520 | cleanup_prepared_input(&prepared); |
| 5521 | cleanup_consistency_issues(&artifacts.consistency_issues); |
| 5522 | |
| 5523 | Ok(outcome) |
| 5524 | } |
| 5525 | |
| 5526 | fn legacy_success_observed_program( |
| 5527 | case: &CaseSpec, |
| 5528 | program: &Path, |
| 5529 | opt_level: OptLevel, |
| 5530 | result: &CaptureResult, |
| 5531 | has_references: bool, |
| 5532 | tools: &ToolchainConfig, |
| 5533 | ) -> ObservedProgram { |
| 5534 | let mut requested_artifacts = expected_artifacts_for_legacy_case(case); |
| 5535 | if has_references { |
| 5536 | requested_artifacts.extend(default_differential_artifacts()); |
| 5537 | } |
| 5538 | |
| 5539 | if legacy_case_uses_generic_observation_execution(case, &case.requested) { |
| 5540 | if let Ok(observation) = observe_compiler( |
| 5541 | &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5542 | program, |
| 5543 | opt_level, |
| 5544 | &requested_artifacts, |
| 5545 | tools, |
| 5546 | ) { |
| 5547 | if observation.compile_exit_code == 0 { |
| 5548 | return ObservedProgram { |
| 5549 | observation, |
| 5550 | requested_artifacts, |
| 5551 | }; |
| 5552 | } |
| 5553 | } |
| 5554 | } |
| 5555 | |
| 5556 | observed_program_from_armfortas_capture(program, opt_level, requested_artifacts, result, None) |
| 5557 | } |
| 5558 | |
| 5559 | fn legacy_case_uses_generic_observation_execution( |
| 5560 | case: &CaseSpec, |
| 5561 | requested: &BTreeSet<Stage>, |
| 5562 | ) -> bool { |
| 5563 | !has_failure_expectation(case) |
| 5564 | && !requested.is_empty() |
| 5565 | && requested |
| 5566 | .iter() |
| 5567 | .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)) |
| 5568 | } |
| 5569 | |
| 5570 | fn execute_generic_compare_case_cell( |
| 5571 | suite: &SuiteSpec, |
| 5572 | case: &CaseSpec, |
| 5573 | opt_level: OptLevel, |
| 5574 | config: &RunConfig, |
| 5575 | ) -> Result<Outcome, String> { |
| 5576 | let effective_status = status_for_opt(case, opt_level); |
| 5577 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5578 | if !config.include_future { |
| 5579 | return Ok(Outcome { |
| 5580 | suite: suite.name.clone(), |
| 5581 | case: case.name.clone(), |
| 5582 | opt_level, |
| 5583 | kind: OutcomeKind::Future, |
| 5584 | detail: reason.clone(), |
| 5585 | bundle: None, |
| 5586 | primary_backend: None, |
| 5587 | consistency_observations: Vec::new(), |
| 5588 | }); |
| 5589 | } |
| 5590 | } |
| 5591 | |
| 5592 | let generic = case |
| 5593 | .generic_compare |
| 5594 | .as_ref() |
| 5595 | .ok_or_else(|| "missing generic compare case configuration".to_string())?; |
| 5596 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5597 | |
| 5598 | if let Some(detail) = compare_capability_issue( |
| 5599 | &generic.left, |
| 5600 | &generic.right, |
| 5601 | &generic.artifacts, |
| 5602 | &config.tools, |
| 5603 | ) { |
| 5604 | let mut outcome = outcome_from_status_and_execution( |
| 5605 | suite, |
| 5606 | case, |
| 5607 | opt_level, |
| 5608 | capability_effective_status(&effective_status, case), |
| 5609 | Err(detail), |
| 5610 | None, |
| 5611 | Vec::new(), |
| 5612 | ); |
| 5613 | outcome.detail = outcome.detail.trim().to_string(); |
| 5614 | cleanup_prepared_input(&prepared); |
| 5615 | return Ok(outcome); |
| 5616 | } |
| 5617 | |
| 5618 | if config.verbose { |
| 5619 | let artifacts = generic |
| 5620 | .artifacts |
| 5621 | .iter() |
| 5622 | .map(ArtifactKey::as_str) |
| 5623 | .collect::<Vec<_>>() |
| 5624 | .join(", "); |
| 5625 | println!(" source: {}", case.source_label()); |
| 5626 | if case.is_graph() { |
| 5627 | for file in &case.graph_files { |
| 5628 | println!(" file: {}", file.display()); |
| 5629 | } |
| 5630 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5631 | } |
| 5632 | println!( |
| 5633 | " compare: {} vs {}", |
| 5634 | generic.left.display_name(), |
| 5635 | generic.right.display_name() |
| 5636 | ); |
| 5637 | println!(" opt: {}", opt_level.as_str()); |
| 5638 | println!(" artifacts: {}", artifacts); |
| 5639 | } |
| 5640 | |
| 5641 | let result = run_compare(&CompareConfig { |
| 5642 | left: generic.left.clone(), |
| 5643 | right: generic.right.clone(), |
| 5644 | program: prepared.compiler_source.clone(), |
| 5645 | opt_level, |
| 5646 | artifacts: generic.artifacts.clone(), |
| 5647 | json_report: None, |
| 5648 | markdown_report: None, |
| 5649 | tools: config.tools.clone(), |
| 5650 | }); |
| 5651 | |
| 5652 | let execution = if has_failure_expectation(case) { |
| 5653 | Err("suite-v2 compare cases do not support expect-fail rules".to_string()) |
| 5654 | } else if let Ok(result) = &result { |
| 5655 | evaluate_compare_expectations(case, result) |
| 5656 | } else { |
| 5657 | Err(result.unwrap_err()) |
| 5658 | }; |
| 5659 | |
| 5660 | let mut outcome = match (effective_status, execution) { |
| 5661 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5662 | suite: suite.name.clone(), |
| 5663 | case: case.name.clone(), |
| 5664 | opt_level, |
| 5665 | kind: OutcomeKind::Pass, |
| 5666 | detail: String::new(), |
| 5667 | bundle: None, |
| 5668 | primary_backend: None, |
| 5669 | consistency_observations: Vec::new(), |
| 5670 | }, |
| 5671 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5672 | suite: suite.name.clone(), |
| 5673 | case: case.name.clone(), |
| 5674 | opt_level, |
| 5675 | kind: OutcomeKind::Fail, |
| 5676 | detail, |
| 5677 | bundle: None, |
| 5678 | primary_backend: None, |
| 5679 | consistency_observations: Vec::new(), |
| 5680 | }, |
| 5681 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5682 | suite: suite.name.clone(), |
| 5683 | case: case.name.clone(), |
| 5684 | opt_level, |
| 5685 | kind: OutcomeKind::Xpass, |
| 5686 | detail: reason, |
| 5687 | bundle: None, |
| 5688 | primary_backend: None, |
| 5689 | consistency_observations: Vec::new(), |
| 5690 | }, |
| 5691 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5692 | suite: suite.name.clone(), |
| 5693 | case: case.name.clone(), |
| 5694 | opt_level, |
| 5695 | kind: OutcomeKind::Xfail, |
| 5696 | detail: format!("{}\n{}", reason, detail), |
| 5697 | bundle: None, |
| 5698 | primary_backend: None, |
| 5699 | consistency_observations: Vec::new(), |
| 5700 | }, |
| 5701 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5702 | suite: suite.name.clone(), |
| 5703 | case: case.name.clone(), |
| 5704 | opt_level, |
| 5705 | kind: OutcomeKind::Xpass, |
| 5706 | detail: reason, |
| 5707 | bundle: None, |
| 5708 | primary_backend: None, |
| 5709 | consistency_observations: Vec::new(), |
| 5710 | }, |
| 5711 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5712 | suite: suite.name.clone(), |
| 5713 | case: case.name.clone(), |
| 5714 | opt_level, |
| 5715 | kind: OutcomeKind::Future, |
| 5716 | detail: format!("{}\n{}", reason, detail), |
| 5717 | bundle: None, |
| 5718 | primary_backend: None, |
| 5719 | consistency_observations: Vec::new(), |
| 5720 | }, |
| 5721 | }; |
| 5722 | |
| 5723 | outcome.detail = outcome.detail.trim().to_string(); |
| 5724 | cleanup_prepared_input(&prepared); |
| 5725 | Ok(outcome) |
| 5726 | } |
| 5727 | |
| 5728 | fn execute_generic_introspect_case_cell( |
| 5729 | suite: &SuiteSpec, |
| 5730 | case: &CaseSpec, |
| 5731 | opt_level: OptLevel, |
| 5732 | config: &RunConfig, |
| 5733 | ) -> Result<Outcome, String> { |
| 5734 | let effective_status = status_for_opt(case, opt_level); |
| 5735 | if let EffectiveStatus::Future(reason) = &effective_status { |
| 5736 | if !config.include_future { |
| 5737 | return Ok(Outcome { |
| 5738 | suite: suite.name.clone(), |
| 5739 | case: case.name.clone(), |
| 5740 | opt_level, |
| 5741 | kind: OutcomeKind::Future, |
| 5742 | detail: reason.clone(), |
| 5743 | bundle: None, |
| 5744 | primary_backend: None, |
| 5745 | consistency_observations: Vec::new(), |
| 5746 | }); |
| 5747 | } |
| 5748 | } |
| 5749 | |
| 5750 | let generic = case |
| 5751 | .generic_introspect |
| 5752 | .as_ref() |
| 5753 | .ok_or_else(|| "missing generic introspection case configuration".to_string())?; |
| 5754 | let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5755 | |
| 5756 | if let Some(detail) = |
| 5757 | capability_request_issue(&generic.compiler, &generic.artifacts, &config.tools) |
| 5758 | { |
| 5759 | let mut outcome = outcome_from_status_and_execution( |
| 5760 | suite, |
| 5761 | case, |
| 5762 | opt_level, |
| 5763 | capability_effective_status(&effective_status, case), |
| 5764 | Err(detail), |
| 5765 | None, |
| 5766 | Vec::new(), |
| 5767 | ); |
| 5768 | outcome.detail = outcome.detail.trim().to_string(); |
| 5769 | cleanup_prepared_input(&prepared); |
| 5770 | return Ok(outcome); |
| 5771 | } |
| 5772 | |
| 5773 | if config.verbose { |
| 5774 | let artifacts = generic |
| 5775 | .artifacts |
| 5776 | .iter() |
| 5777 | .map(ArtifactKey::as_str) |
| 5778 | .collect::<Vec<_>>() |
| 5779 | .join(", "); |
| 5780 | println!(" source: {}", case.source_label()); |
| 5781 | if case.is_graph() { |
| 5782 | for file in &case.graph_files { |
| 5783 | println!(" file: {}", file.display()); |
| 5784 | } |
| 5785 | println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5786 | } |
| 5787 | println!(" compiler: {}", generic.compiler.display_name()); |
| 5788 | println!(" opt: {}", opt_level.as_str()); |
| 5789 | println!(" artifacts: {}", artifacts); |
| 5790 | if !case.reference_compilers.is_empty() { |
| 5791 | println!( |
| 5792 | " refs: {}", |
| 5793 | case.reference_compilers |
| 5794 | .iter() |
| 5795 | .map(ReferenceCompiler::as_str) |
| 5796 | .collect::<Vec<_>>() |
| 5797 | .join(", ") |
| 5798 | ); |
| 5799 | } |
| 5800 | if !case.consistency_checks.is_empty() { |
| 5801 | println!( |
| 5802 | " consistency: {}", |
| 5803 | case.consistency_checks |
| 5804 | .iter() |
| 5805 | .map(ConsistencyCheck::as_str) |
| 5806 | .collect::<Vec<_>>() |
| 5807 | .join(", ") |
| 5808 | ); |
| 5809 | println!(" repeat: {}", case.repeat_count); |
| 5810 | } |
| 5811 | } |
| 5812 | |
| 5813 | let observed = run_introspect(&IntrospectConfig { |
| 5814 | compiler: generic.compiler.clone(), |
| 5815 | program: prepared.compiler_source.clone(), |
| 5816 | opt_level, |
| 5817 | artifacts: generic.artifacts.clone(), |
| 5818 | json_report: None, |
| 5819 | markdown_report: None, |
| 5820 | all_artifacts: false, |
| 5821 | summary_only: false, |
| 5822 | max_artifact_lines: None, |
| 5823 | tools: config.tools.clone(), |
| 5824 | })?; |
| 5825 | |
| 5826 | let mut execution = if observed.observation.compile_exit_code == 0 { |
| 5827 | if has_failure_expectation(case) { |
| 5828 | Err(format!( |
| 5829 | "expected {} to fail ({}) but compilation succeeded", |
| 5830 | generic.compiler.display_name(), |
| 5831 | expected_failure_description(case) |
| 5832 | )) |
| 5833 | } else { |
| 5834 | evaluate_observation_expectations(case, &observed) |
| 5835 | } |
| 5836 | } else if has_failure_expectation(case) { |
| 5837 | evaluate_observation_failure_expectations(case, &observed.observation) |
| 5838 | } else { |
| 5839 | Err(compose_observation_failure_detail(&observed.observation)) |
| 5840 | }; |
| 5841 | |
| 5842 | if execution.is_ok() && !case.reference_compilers.is_empty() { |
| 5843 | execution = run_generic_differential( |
| 5844 | &generic.compiler, |
| 5845 | &prepared.compiler_source, |
| 5846 | opt_level, |
| 5847 | &case.reference_compilers, |
| 5848 | &config.tools, |
| 5849 | ); |
| 5850 | } |
| 5851 | |
| 5852 | let mut consistency_issues = Vec::new(); |
| 5853 | if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5854 | consistency_issues = run_generic_consistency_checks( |
| 5855 | &generic.compiler, |
| 5856 | case, |
| 5857 | &prepared.compiler_source, |
| 5858 | opt_level, |
| 5859 | &config.tools, |
| 5860 | ); |
| 5861 | if !consistency_issues.is_empty() { |
| 5862 | execution = Err(format_consistency_issues(&consistency_issues)); |
| 5863 | } |
| 5864 | } |
| 5865 | |
| 5866 | let consistency_observations = consistency_issues |
| 5867 | .iter() |
| 5868 | .map(ConsistencyIssue::observation) |
| 5869 | .collect::<Vec<_>>(); |
| 5870 | |
| 5871 | let mut outcome = match (effective_status, execution) { |
| 5872 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5873 | suite: suite.name.clone(), |
| 5874 | case: case.name.clone(), |
| 5875 | opt_level, |
| 5876 | kind: OutcomeKind::Pass, |
| 5877 | detail: String::new(), |
| 5878 | bundle: None, |
| 5879 | primary_backend: None, |
| 5880 | consistency_observations: consistency_observations.clone(), |
| 5881 | }, |
| 5882 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5883 | suite: suite.name.clone(), |
| 5884 | case: case.name.clone(), |
| 5885 | opt_level, |
| 5886 | kind: OutcomeKind::Fail, |
| 5887 | detail, |
| 5888 | bundle: None, |
| 5889 | primary_backend: None, |
| 5890 | consistency_observations: consistency_observations.clone(), |
| 5891 | }, |
| 5892 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5893 | suite: suite.name.clone(), |
| 5894 | case: case.name.clone(), |
| 5895 | opt_level, |
| 5896 | kind: OutcomeKind::Xpass, |
| 5897 | detail: reason, |
| 5898 | bundle: None, |
| 5899 | primary_backend: None, |
| 5900 | consistency_observations: consistency_observations.clone(), |
| 5901 | }, |
| 5902 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5903 | suite: suite.name.clone(), |
| 5904 | case: case.name.clone(), |
| 5905 | opt_level, |
| 5906 | kind: OutcomeKind::Xfail, |
| 5907 | detail: format!("{}\n{}", reason, detail), |
| 5908 | bundle: None, |
| 5909 | primary_backend: None, |
| 5910 | consistency_observations: consistency_observations.clone(), |
| 5911 | }, |
| 5912 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5913 | suite: suite.name.clone(), |
| 5914 | case: case.name.clone(), |
| 5915 | opt_level, |
| 5916 | kind: OutcomeKind::Xpass, |
| 5917 | detail: reason, |
| 5918 | bundle: None, |
| 5919 | primary_backend: None, |
| 5920 | consistency_observations: consistency_observations.clone(), |
| 5921 | }, |
| 5922 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5923 | suite: suite.name.clone(), |
| 5924 | case: case.name.clone(), |
| 5925 | opt_level, |
| 5926 | kind: OutcomeKind::Future, |
| 5927 | detail: format!("{}\n{}", reason, detail), |
| 5928 | bundle: None, |
| 5929 | primary_backend: None, |
| 5930 | consistency_observations, |
| 5931 | }, |
| 5932 | }; |
| 5933 | |
| 5934 | outcome.detail = outcome.detail.trim().to_string(); |
| 5935 | cleanup_prepared_input(&prepared); |
| 5936 | cleanup_consistency_issues(&consistency_issues); |
| 5937 | Ok(outcome) |
| 5938 | } |
| 5939 | |
| 5940 | fn prepare_case_input( |
| 5941 | case: &CaseSpec, |
| 5942 | suite: &SuiteSpec, |
| 5943 | opt_level: OptLevel, |
| 5944 | ) -> Result<PreparedInput, String> { |
| 5945 | if case.graph_files.is_empty() { |
| 5946 | return Ok(PreparedInput { |
| 5947 | compiler_source: case.source.clone(), |
| 5948 | generated_source: None, |
| 5949 | temp_root: None, |
| 5950 | }); |
| 5951 | } |
| 5952 | |
| 5953 | let temp_root = default_report_root().join(".tmp").join(format!( |
| 5954 | "graph_{}_{}_{}", |
| 5955 | sanitize_component(&suite.name), |
| 5956 | sanitize_component(&case.name), |
| 5957 | next_report_suffix(opt_level) |
| 5958 | )); |
| 5959 | fs::create_dir_all(&temp_root).map_err(|e| { |
| 5960 | format!( |
| 5961 | "cannot create graph temp dir '{}': {}", |
| 5962 | temp_root.display(), |
| 5963 | e |
| 5964 | ) |
| 5965 | })?; |
| 5966 | |
| 5967 | let extension = case |
| 5968 | .source |
| 5969 | .extension() |
| 5970 | .and_then(|ext| ext.to_str()) |
| 5971 | .filter(|ext| !ext.is_empty()) |
| 5972 | .unwrap_or("f90"); |
| 5973 | let generated_source = temp_root.join(format!( |
| 5974 | "{}_graph.{}", |
| 5975 | sanitize_component(&case.name), |
| 5976 | extension |
| 5977 | )); |
| 5978 | |
| 5979 | let mut combined = String::new(); |
| 5980 | for (index, file) in case.graph_files.iter().enumerate() { |
| 5981 | let text = fs::read_to_string(file) |
| 5982 | .map_err(|e| format!("cannot read graph file '{}': {}", file.display(), e))?; |
| 5983 | if index > 0 { |
| 5984 | combined.push('\n'); |
| 5985 | } |
| 5986 | combined.push_str(&text); |
| 5987 | if !text.ends_with('\n') { |
| 5988 | combined.push('\n'); |
| 5989 | } |
| 5990 | } |
| 5991 | |
| 5992 | fs::write(&generated_source, combined).map_err(|e| { |
| 5993 | format!( |
| 5994 | "cannot write generated graph input '{}': {}", |
| 5995 | generated_source.display(), |
| 5996 | e |
| 5997 | ) |
| 5998 | })?; |
| 5999 | |
| 6000 | Ok(PreparedInput { |
| 6001 | compiler_source: generated_source.clone(), |
| 6002 | generated_source: Some(generated_source), |
| 6003 | temp_root: Some(temp_root), |
| 6004 | }) |
| 6005 | } |
| 6006 | |
| 6007 | fn cleanup_prepared_input(prepared: &PreparedInput) { |
| 6008 | if let Some(temp_root) = &prepared.temp_root { |
| 6009 | let _ = fs::remove_dir_all(temp_root); |
| 6010 | } |
| 6011 | } |
| 6012 | |
| 6013 | fn primary_backend_kind_for_case( |
| 6014 | case: &CaseSpec, |
| 6015 | requested: &BTreeSet<Stage>, |
| 6016 | tools: &ToolchainConfig, |
| 6017 | ) -> PrimaryCaptureBackendKind { |
| 6018 | let cli_observable_only = !requested.is_empty() |
| 6019 | && requested |
| 6020 | .iter() |
| 6021 | .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)); |
| 6022 | let supports_cli_primary = matches!( |
| 6023 | tools.armfortas_adapters().cli(), |
| 6024 | ArmfortasCliAdapter::External(_) |
| 6025 | ); |
| 6026 | let capture_checks_required = case |
| 6027 | .consistency_checks |
| 6028 | .iter() |
| 6029 | .any(ConsistencyCheck::requires_capture_result); |
| 6030 | |
| 6031 | if supports_cli_primary |
| 6032 | && cli_observable_only |
| 6033 | && !has_failure_expectation(case) |
| 6034 | && !capture_checks_required |
| 6035 | { |
| 6036 | PrimaryCaptureBackendKind::Observable |
| 6037 | } else { |
| 6038 | PrimaryCaptureBackendKind::Full |
| 6039 | } |
| 6040 | } |
| 6041 | |
| 6042 | fn select_primary_capture_backend( |
| 6043 | case: &CaseSpec, |
| 6044 | requested: &BTreeSet<Stage>, |
| 6045 | opt_level: OptLevel, |
| 6046 | tools: &ToolchainConfig, |
| 6047 | ) -> SelectedPrimaryBackend { |
| 6048 | let kind = primary_backend_kind_for_case(case, requested, tools); |
| 6049 | let backend: Box<dyn CaptureBackend> = match kind { |
| 6050 | PrimaryCaptureBackendKind::Full => Box::new(tools.armfortas_adapters()), |
| 6051 | PrimaryCaptureBackendKind::Observable => { |
| 6052 | Box::new(tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level))) |
| 6053 | } |
| 6054 | }; |
| 6055 | SelectedPrimaryBackend { kind, backend } |
| 6056 | } |
| 6057 | |
| 6058 | fn execute_primary_armfortas( |
| 6059 | prepared: &PreparedInput, |
| 6060 | opt_level: OptLevel, |
| 6061 | requested: &BTreeSet<Stage>, |
| 6062 | selected: &SelectedPrimaryBackend, |
| 6063 | ) -> Result<CaptureResult, CaptureFailure> { |
| 6064 | let request = CaptureRequest { |
| 6065 | input: prepared.compiler_source.clone(), |
| 6066 | requested: requested.clone(), |
| 6067 | opt_level, |
| 6068 | }; |
| 6069 | selected.backend.capture(&request) |
| 6070 | } |
| 6071 | |
| 6072 | fn status_for_opt(case: &CaseSpec, opt_level: OptLevel) -> EffectiveStatus { |
| 6073 | let mut status = EffectiveStatus::Normal; |
| 6074 | for rule in &case.status_rules { |
| 6075 | if rule.selector.matches(opt_level) { |
| 6076 | status = match rule.kind { |
| 6077 | StatusKind::Xfail => EffectiveStatus::Xfail(rule.reason.clone()), |
| 6078 | StatusKind::Future => EffectiveStatus::Future(rule.reason.clone()), |
| 6079 | }; |
| 6080 | } |
| 6081 | } |
| 6082 | status |
| 6083 | } |
| 6084 | |
| 6085 | fn capability_effective_status(base: &EffectiveStatus, case: &CaseSpec) -> EffectiveStatus { |
| 6086 | match base { |
| 6087 | EffectiveStatus::Normal => match &case.capability_policy { |
| 6088 | Some(policy) => match policy.kind { |
| 6089 | StatusKind::Xfail => EffectiveStatus::Xfail(policy.reason.clone()), |
| 6090 | StatusKind::Future => EffectiveStatus::Future(policy.reason.clone()), |
| 6091 | }, |
| 6092 | None => EffectiveStatus::Normal, |
| 6093 | }, |
| 6094 | other => other.clone(), |
| 6095 | } |
| 6096 | } |
| 6097 | |
| 6098 | fn ensure_target_stage(expectation: &Expectation, requested: &mut BTreeSet<Stage>) { |
| 6099 | match expectation { |
| 6100 | Expectation::CheckComments(target) |
| 6101 | | Expectation::Contains { target, .. } |
| 6102 | | Expectation::NotContains { target, .. } |
| 6103 | | Expectation::Equals { target, .. } |
| 6104 | | Expectation::IntEquals { target, .. } => match target { |
| 6105 | Target::Stage(stage) => { |
| 6106 | requested.insert(*stage); |
| 6107 | } |
| 6108 | Target::Artifact(artifact) => ensure_artifact_stage(artifact, requested), |
| 6109 | Target::CompareStatus |
| 6110 | | Target::CompareClassification |
| 6111 | | Target::CompareChangedArtifacts |
| 6112 | | Target::CompareDifferenceCount |
| 6113 | | Target::CompareBasis => {} |
| 6114 | Target::RunStdout | Target::RunStderr | Target::RunExitCode => { |
| 6115 | requested.insert(Stage::Run); |
| 6116 | } |
| 6117 | }, |
| 6118 | Expectation::FailContains { .. } |
| 6119 | | Expectation::FailEquals { .. } |
| 6120 | | Expectation::FailSourceComments |
| 6121 | | Expectation::FailCommentPatterns(_) => {} |
| 6122 | } |
| 6123 | } |
| 6124 | |
| 6125 | fn ensure_consistency_stage(check: ConsistencyCheck, requested: &mut BTreeSet<Stage>) { |
| 6126 | if let Some(stage) = check.required_stage() { |
| 6127 | requested.insert(stage); |
| 6128 | } |
| 6129 | } |
| 6130 | |
| 6131 | fn ensure_artifact_stage(artifact: &ArtifactKey, requested: &mut BTreeSet<Stage>) { |
| 6132 | match artifact { |
| 6133 | ArtifactKey::Asm => { |
| 6134 | requested.insert(Stage::Asm); |
| 6135 | } |
| 6136 | ArtifactKey::Obj => { |
| 6137 | requested.insert(Stage::Obj); |
| 6138 | } |
| 6139 | ArtifactKey::Runtime |
| 6140 | | ArtifactKey::Stdout |
| 6141 | | ArtifactKey::Stderr |
| 6142 | | ArtifactKey::ExitCode => { |
| 6143 | requested.insert(Stage::Run); |
| 6144 | } |
| 6145 | ArtifactKey::Extra(name) => { |
| 6146 | if let Some(stage) = armfortas_extra_stage(name) { |
| 6147 | requested.insert(stage); |
| 6148 | } |
| 6149 | } |
| 6150 | ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 6151 | } |
| 6152 | } |
| 6153 | |
| 6154 | fn armfortas_extra_stage(name: &str) -> Option<Stage> { |
| 6155 | let (namespace, suffix) = name.split_once('.')?; |
| 6156 | if namespace.eq_ignore_ascii_case("armfortas") { |
| 6157 | Stage::parse(suffix) |
| 6158 | } else { |
| 6159 | None |
| 6160 | } |
| 6161 | } |
| 6162 | |
| 6163 | fn evaluate_observation_expectations( |
| 6164 | case: &CaseSpec, |
| 6165 | observed: &ObservedProgram, |
| 6166 | ) -> Result<(), String> { |
| 6167 | for expectation in &case.expectations { |
| 6168 | match expectation { |
| 6169 | Expectation::CheckComments(target) => { |
| 6170 | let text = observation_target_text(&observed.observation, target)?; |
| 6171 | let source = fs::read_to_string(&case.source) |
| 6172 | .map_err(|e| format!("cannot read '{}': {}", case.source.display(), e))?; |
| 6173 | let checks = if target_uses_ir_comment_checks(target) { |
| 6174 | extract_ir_checks(&source) |
| 6175 | } else { |
| 6176 | extract_checks(&source) |
| 6177 | }; |
| 6178 | if checks.is_empty() { |
| 6179 | let expected_label = if target_uses_ir_comment_checks(target) { |
| 6180 | "! IR_CHECK: / ! IR_NOT:" |
| 6181 | } else { |
| 6182 | "! CHECK:" |
| 6183 | }; |
| 6184 | return Err(format!( |
| 6185 | "case '{}' requested check-comments but '{}' has no {} lines", |
| 6186 | case.name, |
| 6187 | case.source.display(), |
| 6188 | expected_label |
| 6189 | )); |
| 6190 | } |
| 6191 | match_checks(&checks, text, &case.name)?; |
| 6192 | } |
| 6193 | Expectation::Contains { target, needle } => { |
| 6194 | let text = observation_target_text(&observed.observation, target)?; |
| 6195 | if !text.contains(needle) { |
| 6196 | return Err(format!( |
| 6197 | "expected {} to contain {:?}\nactual:\n{}", |
| 6198 | target_name(target), |
| 6199 | needle, |
| 6200 | text |
| 6201 | )); |
| 6202 | } |
| 6203 | } |
| 6204 | Expectation::NotContains { target, needle } => { |
| 6205 | let text = observation_target_text(&observed.observation, target)?; |
| 6206 | if text.contains(needle) { |
| 6207 | return Err(format!( |
| 6208 | "expected {} to not contain {:?}\nactual:\n{}", |
| 6209 | target_name(target), |
| 6210 | needle, |
| 6211 | text |
| 6212 | )); |
| 6213 | } |
| 6214 | } |
| 6215 | Expectation::Equals { target, value } => { |
| 6216 | let text = observation_target_text(&observed.observation, target)?; |
| 6217 | if text.trim_end() != value { |
| 6218 | return Err(format!( |
| 6219 | "expected {} to equal {:?}\nactual:\n{}", |
| 6220 | target_name(target), |
| 6221 | value, |
| 6222 | text |
| 6223 | )); |
| 6224 | } |
| 6225 | } |
| 6226 | Expectation::IntEquals { target, value } => { |
| 6227 | let actual = observation_target_int(&observed.observation, target)?; |
| 6228 | if actual != *value { |
| 6229 | return Err(format!( |
| 6230 | "expected {} to equal {}\nactual: {}", |
| 6231 | target_name(target), |
| 6232 | value, |
| 6233 | actual |
| 6234 | )); |
| 6235 | } |
| 6236 | } |
| 6237 | Expectation::FailContains { .. } |
| 6238 | | Expectation::FailEquals { .. } |
| 6239 | | Expectation::FailSourceComments |
| 6240 | | Expectation::FailCommentPatterns(_) => {} |
| 6241 | } |
| 6242 | } |
| 6243 | Ok(()) |
| 6244 | } |
| 6245 | |
| 6246 | fn evaluate_compare_expectations(case: &CaseSpec, result: &ComparisonResult) -> Result<(), String> { |
| 6247 | for expectation in &case.expectations { |
| 6248 | match expectation { |
| 6249 | Expectation::CheckComments(_) => { |
| 6250 | return Err("compare cases do not support check-comments expectations".into()) |
| 6251 | } |
| 6252 | Expectation::Contains { target, needle } => { |
| 6253 | let text = compare_target_text(result, target)?; |
| 6254 | if !text.contains(needle) { |
| 6255 | return Err(format!( |
| 6256 | "expected {} to contain {:?}\nactual:\n{}", |
| 6257 | target_name(target), |
| 6258 | needle, |
| 6259 | text |
| 6260 | )); |
| 6261 | } |
| 6262 | } |
| 6263 | Expectation::NotContains { target, needle } => { |
| 6264 | let text = compare_target_text(result, target)?; |
| 6265 | if text.contains(needle) { |
| 6266 | return Err(format!( |
| 6267 | "expected {} to not contain {:?}\nactual:\n{}", |
| 6268 | target_name(target), |
| 6269 | needle, |
| 6270 | text |
| 6271 | )); |
| 6272 | } |
| 6273 | } |
| 6274 | Expectation::Equals { target, value } => { |
| 6275 | let text = compare_target_text(result, target)?; |
| 6276 | if text.trim_end() != value { |
| 6277 | return Err(format!( |
| 6278 | "expected {} to equal {:?}\nactual:\n{}", |
| 6279 | target_name(target), |
| 6280 | value, |
| 6281 | text |
| 6282 | )); |
| 6283 | } |
| 6284 | } |
| 6285 | Expectation::IntEquals { target, value } => { |
| 6286 | let actual = compare_target_int(result, target)?; |
| 6287 | if actual != *value { |
| 6288 | return Err(format!( |
| 6289 | "expected {} to equal {}\nactual: {}", |
| 6290 | target_name(target), |
| 6291 | value, |
| 6292 | actual |
| 6293 | )); |
| 6294 | } |
| 6295 | } |
| 6296 | Expectation::FailContains { .. } |
| 6297 | | Expectation::FailEquals { .. } |
| 6298 | | Expectation::FailSourceComments |
| 6299 | | Expectation::FailCommentPatterns(_) => {} |
| 6300 | } |
| 6301 | } |
| 6302 | Ok(()) |
| 6303 | } |
| 6304 | |
| 6305 | fn evaluate_observation_failure_expectations( |
| 6306 | case: &CaseSpec, |
| 6307 | observation: &CompilerObservation, |
| 6308 | ) -> Result<(), String> { |
| 6309 | let mut saw_failure_expectation = false; |
| 6310 | let diagnostics = observation_diagnostics_text(observation).unwrap_or_default(); |
| 6311 | for expectation in &case.expectations { |
| 6312 | match expectation { |
| 6313 | Expectation::FailContains { stage, needle } => { |
| 6314 | saw_failure_expectation = true; |
| 6315 | let actual_stage = observation_failure_stage(observation); |
| 6316 | if actual_stage != Some(*stage) { |
| 6317 | let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6318 | return Err(format!( |
| 6319 | "expected failure stage {} but compiler failed in {}\n{}", |
| 6320 | stage.as_str(), |
| 6321 | actual, |
| 6322 | diagnostics |
| 6323 | )); |
| 6324 | } |
| 6325 | if !diagnostics.contains(needle) { |
| 6326 | return Err(format!( |
| 6327 | "expected failure detail at {} to contain {:?}\nactual:\n{}", |
| 6328 | stage.as_str(), |
| 6329 | needle, |
| 6330 | diagnostics |
| 6331 | )); |
| 6332 | } |
| 6333 | } |
| 6334 | Expectation::FailEquals { stage, value } => { |
| 6335 | saw_failure_expectation = true; |
| 6336 | let actual_stage = observation_failure_stage(observation); |
| 6337 | if actual_stage != Some(*stage) { |
| 6338 | let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6339 | return Err(format!( |
| 6340 | "expected failure stage {} but compiler failed in {}\n{}", |
| 6341 | stage.as_str(), |
| 6342 | actual, |
| 6343 | diagnostics |
| 6344 | )); |
| 6345 | } |
| 6346 | if diagnostics.trim_end() != value { |
| 6347 | return Err(format!( |
| 6348 | "expected failure detail at {} to equal {:?}\nactual:\n{}", |
| 6349 | stage.as_str(), |
| 6350 | value, |
| 6351 | diagnostics |
| 6352 | )); |
| 6353 | } |
| 6354 | } |
| 6355 | Expectation::FailCommentPatterns(patterns) => { |
| 6356 | saw_failure_expectation = true; |
| 6357 | for needle in patterns { |
| 6358 | if !diagnostics.contains(needle) { |
| 6359 | return Err(format!( |
| 6360 | "expected failure detail to contain source comment {:?}\nactual:\n{}", |
| 6361 | needle, diagnostics |
| 6362 | )); |
| 6363 | } |
| 6364 | } |
| 6365 | } |
| 6366 | Expectation::CheckComments(_) |
| 6367 | | Expectation::Contains { .. } |
| 6368 | | Expectation::NotContains { .. } |
| 6369 | | Expectation::Equals { .. } |
| 6370 | | Expectation::IntEquals { .. } |
| 6371 | | Expectation::FailSourceComments => {} |
| 6372 | } |
| 6373 | } |
| 6374 | |
| 6375 | if !saw_failure_expectation { |
| 6376 | return Err(format!( |
| 6377 | "{} failed but the case did not declare an expect-fail rule\n{}", |
| 6378 | observation.compiler.display_name(), |
| 6379 | diagnostics |
| 6380 | )); |
| 6381 | } |
| 6382 | |
| 6383 | Ok(()) |
| 6384 | } |
| 6385 | |
| 6386 | #[cfg(test)] |
| 6387 | fn evaluate_failed_armfortas( |
| 6388 | case: &CaseSpec, |
| 6389 | artifacts: &ExecutionArtifacts, |
| 6390 | failure: &CaptureFailure, |
| 6391 | ) -> Result<(), String> { |
| 6392 | let observed = legacy_failure_observed_program(&case.source, case, failure); |
| 6393 | evaluate_failed_armfortas_with_observed(case, artifacts, &observed) |
| 6394 | } |
| 6395 | |
| 6396 | fn evaluate_failed_armfortas_with_observed( |
| 6397 | case: &CaseSpec, |
| 6398 | artifacts: &ExecutionArtifacts, |
| 6399 | observed: &ObservedProgram, |
| 6400 | ) -> Result<(), String> { |
| 6401 | if has_failure_expectation(case) { |
| 6402 | evaluate_observation_failure_expectations(case, &observed.observation) |
| 6403 | } else { |
| 6404 | match evaluate_observation_expectations(case, observed) { |
| 6405 | Ok(()) => Err(compose_armfortas_failure_detail(artifacts)), |
| 6406 | Err(detail) if is_missing_stage_detail(&detail) => { |
| 6407 | Err(compose_armfortas_failure_detail(artifacts)) |
| 6408 | } |
| 6409 | Err(detail) => Err(detail), |
| 6410 | } |
| 6411 | } |
| 6412 | } |
| 6413 | |
| 6414 | fn legacy_failure_observed_program( |
| 6415 | program: &Path, |
| 6416 | case: &CaseSpec, |
| 6417 | failure: &CaptureFailure, |
| 6418 | ) -> ObservedProgram { |
| 6419 | let partial = failure.partial_result(); |
| 6420 | observed_program_from_armfortas_capture( |
| 6421 | program, |
| 6422 | failure.opt_level, |
| 6423 | expected_artifacts_for_legacy_case(case), |
| 6424 | &partial, |
| 6425 | Some(failure), |
| 6426 | ) |
| 6427 | } |
| 6428 | |
| 6429 | fn has_failure_expectation(case: &CaseSpec) -> bool { |
| 6430 | case.expectations.iter().any(|expectation| { |
| 6431 | matches!( |
| 6432 | expectation, |
| 6433 | Expectation::FailContains { .. } |
| 6434 | | Expectation::FailEquals { .. } |
| 6435 | | Expectation::FailSourceComments |
| 6436 | | Expectation::FailCommentPatterns(_) |
| 6437 | ) |
| 6438 | }) |
| 6439 | } |
| 6440 | |
| 6441 | fn legacy_unavailable_backend_detail( |
| 6442 | case: &CaseSpec, |
| 6443 | selected_backend: &SelectedPrimaryBackend, |
| 6444 | ) -> Option<String> { |
| 6445 | if selected_backend.kind == PrimaryCaptureBackendKind::Full |
| 6446 | && selected_backend.backend.mode_name() == "unavailable" |
| 6447 | { |
| 6448 | Some(format!( |
| 6449 | "case requires linked armfortas capture, but this build only provides the external-driver surface\nsource: {}\nrequired backend: {}\nuse scripts/bootstrap-linked-armfortas.sh for rich stages and legacy frontend/module suites, or run a generic suite-v2 / observable-only case instead", |
| 6450 | case.source_label(), |
| 6451 | selected_backend.backend.description() |
| 6452 | )) |
| 6453 | } else { |
| 6454 | None |
| 6455 | } |
| 6456 | } |
| 6457 | |
| 6458 | fn outcome_from_status_and_execution( |
| 6459 | suite: &SuiteSpec, |
| 6460 | case: &CaseSpec, |
| 6461 | opt_level: OptLevel, |
| 6462 | effective_status: EffectiveStatus, |
| 6463 | execution: Result<(), String>, |
| 6464 | primary_backend: Option<PrimaryBackendReport>, |
| 6465 | consistency_observations: Vec<ConsistencyObservation>, |
| 6466 | ) -> Outcome { |
| 6467 | match (effective_status, execution) { |
| 6468 | (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 6469 | suite: suite.name.clone(), |
| 6470 | case: case.name.clone(), |
| 6471 | opt_level, |
| 6472 | kind: OutcomeKind::Pass, |
| 6473 | detail: String::new(), |
| 6474 | bundle: None, |
| 6475 | primary_backend, |
| 6476 | consistency_observations, |
| 6477 | }, |
| 6478 | (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 6479 | suite: suite.name.clone(), |
| 6480 | case: case.name.clone(), |
| 6481 | opt_level, |
| 6482 | kind: OutcomeKind::Fail, |
| 6483 | detail, |
| 6484 | bundle: None, |
| 6485 | primary_backend, |
| 6486 | consistency_observations, |
| 6487 | }, |
| 6488 | (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 6489 | suite: suite.name.clone(), |
| 6490 | case: case.name.clone(), |
| 6491 | opt_level, |
| 6492 | kind: OutcomeKind::Xpass, |
| 6493 | detail: reason, |
| 6494 | bundle: None, |
| 6495 | primary_backend, |
| 6496 | consistency_observations, |
| 6497 | }, |
| 6498 | (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 6499 | suite: suite.name.clone(), |
| 6500 | case: case.name.clone(), |
| 6501 | opt_level, |
| 6502 | kind: OutcomeKind::Xfail, |
| 6503 | detail: format!("{}\n{}", reason, detail), |
| 6504 | bundle: None, |
| 6505 | primary_backend, |
| 6506 | consistency_observations, |
| 6507 | }, |
| 6508 | (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 6509 | suite: suite.name.clone(), |
| 6510 | case: case.name.clone(), |
| 6511 | opt_level, |
| 6512 | kind: OutcomeKind::Xpass, |
| 6513 | detail: reason, |
| 6514 | bundle: None, |
| 6515 | primary_backend, |
| 6516 | consistency_observations, |
| 6517 | }, |
| 6518 | (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 6519 | suite: suite.name.clone(), |
| 6520 | case: case.name.clone(), |
| 6521 | opt_level, |
| 6522 | kind: OutcomeKind::Future, |
| 6523 | detail: format!("{}\n{}", reason, detail), |
| 6524 | bundle: None, |
| 6525 | primary_backend, |
| 6526 | consistency_observations, |
| 6527 | }, |
| 6528 | } |
| 6529 | } |
| 6530 | |
| 6531 | fn expected_failure_description(case: &CaseSpec) -> String { |
| 6532 | let mut items = Vec::new(); |
| 6533 | for expectation in &case.expectations { |
| 6534 | match expectation { |
| 6535 | Expectation::FailContains { stage, needle } => { |
| 6536 | items.push(format!("{} contains {:?}", stage.as_str(), needle)); |
| 6537 | } |
| 6538 | Expectation::FailEquals { stage, value } => { |
| 6539 | items.push(format!("{} equals {:?}", stage.as_str(), value)); |
| 6540 | } |
| 6541 | Expectation::FailCommentPatterns(patterns) => { |
| 6542 | for needle in patterns { |
| 6543 | items.push(format!("comments contain {:?}", needle)); |
| 6544 | } |
| 6545 | } |
| 6546 | _ => {} |
| 6547 | } |
| 6548 | } |
| 6549 | if items.is_empty() { |
| 6550 | "declared failure".to_string() |
| 6551 | } else { |
| 6552 | items.join(", ") |
| 6553 | } |
| 6554 | } |
| 6555 | |
| 6556 | fn is_missing_stage_detail(detail: &str) -> bool { |
| 6557 | detail.starts_with("missing captured stage '") |
| 6558 | || detail == "missing captured run stage" |
| 6559 | || detail.starts_with("missing artifact '") |
| 6560 | } |
| 6561 | |
| 6562 | fn target_name(target: &Target) -> String { |
| 6563 | match target { |
| 6564 | Target::Stage(stage) => stage.as_str().to_string(), |
| 6565 | Target::Artifact(artifact) => artifact.as_str().to_string(), |
| 6566 | Target::CompareStatus => "compare.status".to_string(), |
| 6567 | Target::CompareClassification => "compare.classification".to_string(), |
| 6568 | Target::CompareChangedArtifacts => "compare.changed_artifacts".to_string(), |
| 6569 | Target::CompareDifferenceCount => "compare.difference_count".to_string(), |
| 6570 | Target::CompareBasis => "compare.basis".to_string(), |
| 6571 | Target::RunStdout => "run.stdout".to_string(), |
| 6572 | Target::RunStderr => "run.stderr".to_string(), |
| 6573 | Target::RunExitCode => "run.exit_code".to_string(), |
| 6574 | } |
| 6575 | } |
| 6576 | |
| 6577 | fn compare_target_text(result: &ComparisonResult, target: &Target) -> Result<String, String> { |
| 6578 | match target { |
| 6579 | Target::CompareStatus => Ok(compare_status(result).to_string()), |
| 6580 | Target::CompareClassification => Ok(compare_classification(result).to_string()), |
| 6581 | Target::CompareChangedArtifacts => { |
| 6582 | let changed = compare_changed_artifacts(result); |
| 6583 | if changed.is_empty() { |
| 6584 | Ok("none".to_string()) |
| 6585 | } else { |
| 6586 | Ok(changed.join(", ")) |
| 6587 | } |
| 6588 | } |
| 6589 | Target::CompareBasis => Ok(result.basis.clone()), |
| 6590 | Target::CompareDifferenceCount => Err( |
| 6591 | "compare.difference_count is numeric; use 'expect compare.difference_count equals <int>'" |
| 6592 | .into(), |
| 6593 | ), |
| 6594 | _ => Err(format!( |
| 6595 | "{} is not a compare text target", |
| 6596 | target_name(target) |
| 6597 | )), |
| 6598 | } |
| 6599 | } |
| 6600 | |
| 6601 | fn compare_target_int(result: &ComparisonResult, target: &Target) -> Result<i32, String> { |
| 6602 | match target { |
| 6603 | Target::CompareDifferenceCount => Ok(result.differences.len() as i32), |
| 6604 | _ => Err(format!( |
| 6605 | "{} is textual; use a string matcher instead", |
| 6606 | target_name(target) |
| 6607 | )), |
| 6608 | } |
| 6609 | } |
| 6610 | |
| 6611 | fn observation_target_text<'a>( |
| 6612 | observation: &'a CompilerObservation, |
| 6613 | target: &Target, |
| 6614 | ) -> Result<&'a str, String> { |
| 6615 | match target { |
| 6616 | Target::Stage(stage) => match stage { |
| 6617 | Stage::Asm => observation_artifact_text(observation, &ArtifactKey::Asm), |
| 6618 | Stage::Obj => observation_artifact_text(observation, &ArtifactKey::Obj), |
| 6619 | Stage::Run => { |
| 6620 | Err("run is structured; use run.stdout, run.stderr, or run.exit_code".into()) |
| 6621 | } |
| 6622 | other => observation_artifact_text( |
| 6623 | observation, |
| 6624 | &ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6625 | ), |
| 6626 | }, |
| 6627 | Target::Artifact(artifact) => observation_artifact_text(observation, artifact), |
| 6628 | Target::CompareStatus |
| 6629 | | Target::CompareClassification |
| 6630 | | Target::CompareChangedArtifacts |
| 6631 | | Target::CompareDifferenceCount |
| 6632 | | Target::CompareBasis => { |
| 6633 | Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6634 | } |
| 6635 | Target::RunStdout => observation_run_stdout(observation), |
| 6636 | Target::RunStderr => observation_run_stderr(observation), |
| 6637 | Target::RunExitCode => { |
| 6638 | Err("run.exit_code is numeric; use 'expect run.exit_code equals <int>'".into()) |
| 6639 | } |
| 6640 | } |
| 6641 | } |
| 6642 | |
| 6643 | fn observation_target_int( |
| 6644 | observation: &CompilerObservation, |
| 6645 | target: &Target, |
| 6646 | ) -> Result<i32, String> { |
| 6647 | match target { |
| 6648 | Target::RunExitCode => observation_run_exit_code(observation), |
| 6649 | Target::Artifact(ArtifactKey::ExitCode) => observation_run_exit_code(observation), |
| 6650 | Target::CompareStatus |
| 6651 | | Target::CompareClassification |
| 6652 | | Target::CompareChangedArtifacts |
| 6653 | | Target::CompareDifferenceCount |
| 6654 | | Target::CompareBasis => { |
| 6655 | Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6656 | } |
| 6657 | _ => Err(format!( |
| 6658 | "{} is textual; use a string matcher instead", |
| 6659 | target_name(target) |
| 6660 | )), |
| 6661 | } |
| 6662 | } |
| 6663 | |
| 6664 | fn observation_artifact_text<'a>( |
| 6665 | observation: &'a CompilerObservation, |
| 6666 | artifact: &ArtifactKey, |
| 6667 | ) -> Result<&'a str, String> { |
| 6668 | match observation.artifacts.get(artifact) { |
| 6669 | Some(ArtifactValue::Text(text)) => Ok(text), |
| 6670 | Some(ArtifactValue::Int(_)) => Err(format!( |
| 6671 | "artifact '{}' is numeric; use an integer matcher instead", |
| 6672 | artifact.as_str() |
| 6673 | )), |
| 6674 | Some(ArtifactValue::Run(_)) => Err(format!( |
| 6675 | "artifact '{}' is structured runtime data; use run.stdout, run.stderr, or run.exit_code", |
| 6676 | artifact.as_str() |
| 6677 | )), |
| 6678 | Some(ArtifactValue::Path(_)) => Err(format!( |
| 6679 | "artifact '{}' is binary/path data, not text", |
| 6680 | artifact.as_str() |
| 6681 | )), |
| 6682 | None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 6683 | } |
| 6684 | } |
| 6685 | |
| 6686 | fn observation_run_stdout(observation: &CompilerObservation) -> Result<&str, String> { |
| 6687 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6688 | Ok(&run.stdout) |
| 6689 | } else { |
| 6690 | observation_artifact_text(observation, &ArtifactKey::Stdout) |
| 6691 | } |
| 6692 | } |
| 6693 | |
| 6694 | fn observation_run_stderr(observation: &CompilerObservation) -> Result<&str, String> { |
| 6695 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6696 | Ok(&run.stderr) |
| 6697 | } else { |
| 6698 | observation_artifact_text(observation, &ArtifactKey::Stderr) |
| 6699 | } |
| 6700 | } |
| 6701 | |
| 6702 | fn observation_run_exit_code(observation: &CompilerObservation) -> Result<i32, String> { |
| 6703 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6704 | Ok(run.exit_code) |
| 6705 | } else { |
| 6706 | match observation.artifacts.get(&ArtifactKey::ExitCode) { |
| 6707 | Some(ArtifactValue::Int(value)) => Ok(*value), |
| 6708 | Some(_) => Err("artifact 'exit-code' is not numeric".into()), |
| 6709 | None => Err("missing artifact 'exit-code'".into()), |
| 6710 | } |
| 6711 | } |
| 6712 | } |
| 6713 | |
| 6714 | fn observation_diagnostics_text(observation: &CompilerObservation) -> Option<&str> { |
| 6715 | match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 6716 | Some(ArtifactValue::Text(text)) => Some(text.as_str()), |
| 6717 | _ => None, |
| 6718 | } |
| 6719 | } |
| 6720 | |
| 6721 | fn observation_failure_stage(observation: &CompilerObservation) -> Option<FailureStage> { |
| 6722 | observation |
| 6723 | .provenance |
| 6724 | .failure_stage |
| 6725 | .as_deref() |
| 6726 | .and_then(FailureStage::parse) |
| 6727 | } |
| 6728 | |
| 6729 | fn compare_differential( |
| 6730 | armfortas: &CompilerObservation, |
| 6731 | references: &[CompilerObservation], |
| 6732 | ) -> Result<(), String> { |
| 6733 | let requested = default_differential_artifacts(); |
| 6734 | let comparisons = references |
| 6735 | .iter() |
| 6736 | .cloned() |
| 6737 | .map(|reference| compare_observations(armfortas.clone(), reference, &requested)) |
| 6738 | .collect::<Vec<_>>(); |
| 6739 | let matching_refs = comparisons |
| 6740 | .iter() |
| 6741 | .filter(|comparison| comparison.differences.is_empty()) |
| 6742 | .count(); |
| 6743 | |
| 6744 | if matching_refs == comparisons.len() { |
| 6745 | return Ok(()); |
| 6746 | } |
| 6747 | |
| 6748 | let reference_disagreement = if references.len() > 1 { |
| 6749 | let baseline = references[0].clone(); |
| 6750 | references[1..].iter().cloned().any(|reference| { |
| 6751 | !compare_observations(baseline.clone(), reference, &requested) |
| 6752 | .differences |
| 6753 | .is_empty() |
| 6754 | }) |
| 6755 | } else { |
| 6756 | false |
| 6757 | }; |
| 6758 | |
| 6759 | let classification = if matching_refs == 0 && !reference_disagreement { |
| 6760 | "classification: armfortas-only divergence" |
| 6761 | } else if reference_disagreement { |
| 6762 | "classification: reference disagreement" |
| 6763 | } else { |
| 6764 | "classification: partial disagreement" |
| 6765 | }; |
| 6766 | |
| 6767 | let detail = comparisons |
| 6768 | .iter() |
| 6769 | .filter(|comparison| !comparison.differences.is_empty()) |
| 6770 | .map(render_compare_text) |
| 6771 | .collect::<Vec<_>>(); |
| 6772 | |
| 6773 | Err(format!( |
| 6774 | "behavior mismatch against reference compilers\n{}\n\n{}", |
| 6775 | classification, |
| 6776 | detail.join("\n\n") |
| 6777 | )) |
| 6778 | } |
| 6779 | |
| 6780 | fn compose_armfortas_failure_detail(artifacts: &ExecutionArtifacts) -> String { |
| 6781 | let mut detail = String::new(); |
| 6782 | if let Some(failure) = &artifacts.armfortas_failure { |
| 6783 | detail.push_str(&format!( |
| 6784 | "armfortas failed in {}\n{}", |
| 6785 | failure.stage.as_str(), |
| 6786 | failure.detail |
| 6787 | )); |
| 6788 | } else { |
| 6789 | detail.push_str("armfortas failed without an error message"); |
| 6790 | } |
| 6791 | |
| 6792 | if !artifacts.references.is_empty() { |
| 6793 | detail.push_str("\n\nreference compilers\n"); |
| 6794 | detail.push_str(&format_reference_summary(&artifacts.references)); |
| 6795 | } |
| 6796 | |
| 6797 | detail |
| 6798 | } |
| 6799 | |
| 6800 | fn compose_observation_failure_detail(observation: &CompilerObservation) -> String { |
| 6801 | if observation.provenance.backend_mode == "unavailable" { |
| 6802 | let mut detail = format!( |
| 6803 | "{} unavailable for requested artifacts in this build", |
| 6804 | observation.compiler.display_name() |
| 6805 | ); |
| 6806 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6807 | detail.push('\n'); |
| 6808 | detail.push_str(diagnostics); |
| 6809 | } |
| 6810 | return detail; |
| 6811 | } |
| 6812 | |
| 6813 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6814 | if diagnostics.contains("does not support requested artifacts in this adapter") { |
| 6815 | return format!( |
| 6816 | "{} does not support requested artifacts in this adapter\n{}", |
| 6817 | observation.compiler.display_name(), |
| 6818 | diagnostics |
| 6819 | ); |
| 6820 | } |
| 6821 | } |
| 6822 | |
| 6823 | let mut detail = String::new(); |
| 6824 | detail.push_str(&format!("{} failed", observation.compiler.display_name())); |
| 6825 | if let Some(stage) = &observation.provenance.failure_stage { |
| 6826 | detail.push_str(&format!(" in {}", stage)); |
| 6827 | } |
| 6828 | if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6829 | detail.push('\n'); |
| 6830 | detail.push_str(diagnostics); |
| 6831 | } |
| 6832 | detail |
| 6833 | } |
| 6834 | |
| 6835 | fn run_generic_differential( |
| 6836 | compiler: &CompilerSpec, |
| 6837 | program: &Path, |
| 6838 | opt_level: OptLevel, |
| 6839 | references: &[ReferenceCompiler], |
| 6840 | tools: &ToolchainConfig, |
| 6841 | ) -> Result<(), String> { |
| 6842 | let requested = default_differential_artifacts(); |
| 6843 | let primary = observe_compiler(compiler, program, opt_level, &requested, tools)?; |
| 6844 | let references = references |
| 6845 | .iter() |
| 6846 | .copied() |
| 6847 | .map(reference_compiler_spec) |
| 6848 | .map(|reference| observe_compiler(&reference, program, opt_level, &requested, tools)) |
| 6849 | .collect::<Result<Vec<_>, _>>()?; |
| 6850 | compare_differential(&primary, &references) |
| 6851 | } |
| 6852 | |
| 6853 | fn expected_artifacts_for_legacy_case(case: &CaseSpec) -> BTreeSet<ArtifactKey> { |
| 6854 | let mut requested = BTreeSet::new(); |
| 6855 | for stage in &case.requested { |
| 6856 | requested.insert(stage_to_artifact_key(*stage)); |
| 6857 | } |
| 6858 | for expectation in &case.expectations { |
| 6859 | match expectation { |
| 6860 | Expectation::CheckComments(target) |
| 6861 | | Expectation::Contains { target, .. } |
| 6862 | | Expectation::NotContains { target, .. } |
| 6863 | | Expectation::Equals { target, .. } |
| 6864 | | Expectation::IntEquals { target, .. } => match target { |
| 6865 | Target::Stage(stage) => { |
| 6866 | requested.insert(stage_to_artifact_key(*stage)); |
| 6867 | } |
| 6868 | Target::Artifact(artifact) => { |
| 6869 | requested.insert(artifact.clone()); |
| 6870 | } |
| 6871 | Target::RunStdout => { |
| 6872 | requested.insert(ArtifactKey::Stdout); |
| 6873 | } |
| 6874 | Target::RunStderr => { |
| 6875 | requested.insert(ArtifactKey::Stderr); |
| 6876 | } |
| 6877 | Target::RunExitCode => { |
| 6878 | requested.insert(ArtifactKey::ExitCode); |
| 6879 | } |
| 6880 | Target::CompareStatus |
| 6881 | | Target::CompareClassification |
| 6882 | | Target::CompareChangedArtifacts |
| 6883 | | Target::CompareDifferenceCount |
| 6884 | | Target::CompareBasis => {} |
| 6885 | }, |
| 6886 | Expectation::FailContains { .. } |
| 6887 | | Expectation::FailEquals { .. } |
| 6888 | | Expectation::FailSourceComments |
| 6889 | | Expectation::FailCommentPatterns(_) => {} |
| 6890 | } |
| 6891 | } |
| 6892 | requested |
| 6893 | } |
| 6894 | |
| 6895 | fn legacy_case_uses_generic_consistency_checks(case: &CaseSpec) -> bool { |
| 6896 | !case.consistency_checks.is_empty() |
| 6897 | && case |
| 6898 | .consistency_checks |
| 6899 | .iter() |
| 6900 | .copied() |
| 6901 | .all(|check| check.supports_generic_introspect()) |
| 6902 | } |
| 6903 | |
| 6904 | fn stage_to_artifact_key(stage: Stage) -> ArtifactKey { |
| 6905 | match stage { |
| 6906 | Stage::Asm => ArtifactKey::Asm, |
| 6907 | Stage::Obj => ArtifactKey::Obj, |
| 6908 | Stage::Run => ArtifactKey::Runtime, |
| 6909 | other => ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6910 | } |
| 6911 | } |
| 6912 | |
| 6913 | fn observed_program_from_armfortas_capture( |
| 6914 | program: &Path, |
| 6915 | opt_level: OptLevel, |
| 6916 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 6917 | result: &CaptureResult, |
| 6918 | failure: Option<&CaptureFailure>, |
| 6919 | ) -> ObservedProgram { |
| 6920 | let mut artifacts = BTreeMap::new(); |
| 6921 | for (stage, captured) in &result.stages { |
| 6922 | match (stage, captured) { |
| 6923 | (Stage::Asm, CapturedStage::Text(text)) |
| 6924 | if requested_artifacts.contains(&ArtifactKey::Asm) => |
| 6925 | { |
| 6926 | artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 6927 | } |
| 6928 | (Stage::Obj, CapturedStage::Text(text)) |
| 6929 | if requested_artifacts.contains(&ArtifactKey::Obj) => |
| 6930 | { |
| 6931 | artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 6932 | } |
| 6933 | (Stage::Run, CapturedStage::Run(run)) => { |
| 6934 | insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 6935 | } |
| 6936 | (stage, CapturedStage::Text(text)) => { |
| 6937 | let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 6938 | if requested_artifacts.contains(&key) { |
| 6939 | artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 6940 | } |
| 6941 | } |
| 6942 | _ => {} |
| 6943 | } |
| 6944 | } |
| 6945 | if let Some(failure) = failure { |
| 6946 | if requested_artifacts.contains(&ArtifactKey::Diagnostics) |
| 6947 | || !artifacts.contains_key(&ArtifactKey::Diagnostics) |
| 6948 | { |
| 6949 | artifacts.insert( |
| 6950 | ArtifactKey::Diagnostics, |
| 6951 | ArtifactValue::Text(failure.detail.clone()), |
| 6952 | ); |
| 6953 | } |
| 6954 | } |
| 6955 | let artifacts_captured = artifacts |
| 6956 | .keys() |
| 6957 | .map(|artifact| artifact.as_str().to_string()) |
| 6958 | .collect::<Vec<_>>(); |
| 6959 | ObservedProgram { |
| 6960 | observation: CompilerObservation { |
| 6961 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 6962 | program: program.to_path_buf(), |
| 6963 | opt_level, |
| 6964 | compile_exit_code: if failure.is_some() { 1 } else { 0 }, |
| 6965 | artifacts, |
| 6966 | provenance: ObservationProvenance { |
| 6967 | compiler_identity: "armfortas".into(), |
| 6968 | adapter_kind: "named".into(), |
| 6969 | backend_mode: "suite-legacy-capture".into(), |
| 6970 | backend_detail: "legacy suite cell capture converted into generic observation" |
| 6971 | .into(), |
| 6972 | artifacts_captured, |
| 6973 | comparison_basis: None, |
| 6974 | failure_stage: failure.map(|failure| failure.stage.as_str().to_string()), |
| 6975 | }, |
| 6976 | }, |
| 6977 | requested_artifacts, |
| 6978 | } |
| 6979 | } |
| 6980 | |
| 6981 | fn observed_program_from_reference_result( |
| 6982 | program: &Path, |
| 6983 | opt_level: OptLevel, |
| 6984 | requested_artifacts: BTreeSet<ArtifactKey>, |
| 6985 | reference: &ReferenceResult, |
| 6986 | ) -> ObservedProgram { |
| 6987 | let mut artifacts = BTreeMap::new(); |
| 6988 | let diagnostics = [ |
| 6989 | reference.compile_stdout.trim_end(), |
| 6990 | reference.compile_stderr.trim_end(), |
| 6991 | ] |
| 6992 | .iter() |
| 6993 | .filter(|part| !part.is_empty()) |
| 6994 | .copied() |
| 6995 | .collect::<Vec<_>>() |
| 6996 | .join("\n"); |
| 6997 | |
| 6998 | if requested_artifacts.contains(&ArtifactKey::Diagnostics) && !diagnostics.is_empty() { |
| 6999 | artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 7000 | } |
| 7001 | |
| 7002 | if let Some(run) = &reference.run { |
| 7003 | insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 7004 | } else if let Some(run_error) = &reference.run_error { |
| 7005 | let diagnostics = artifacts |
| 7006 | .entry(ArtifactKey::Diagnostics) |
| 7007 | .or_insert_with(|| ArtifactValue::Text(String::new())); |
| 7008 | if let ArtifactValue::Text(text) = diagnostics { |
| 7009 | if !text.is_empty() { |
| 7010 | text.push('\n'); |
| 7011 | } |
| 7012 | text.push_str(&format!("run error: {}", run_error)); |
| 7013 | } |
| 7014 | } |
| 7015 | |
| 7016 | let artifacts_captured = artifacts |
| 7017 | .keys() |
| 7018 | .map(|artifact| artifact.as_str().to_string()) |
| 7019 | .collect::<Vec<_>>(); |
| 7020 | ObservedProgram { |
| 7021 | observation: CompilerObservation { |
| 7022 | compiler: match reference.compiler { |
| 7023 | ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 7024 | ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 7025 | }, |
| 7026 | program: program.to_path_buf(), |
| 7027 | opt_level, |
| 7028 | compile_exit_code: reference.compile_exit_code, |
| 7029 | artifacts, |
| 7030 | provenance: ObservationProvenance { |
| 7031 | compiler_identity: reference.compiler.as_str().to_string(), |
| 7032 | adapter_kind: "named".into(), |
| 7033 | backend_mode: "legacy-reference".into(), |
| 7034 | backend_detail: format!( |
| 7035 | "legacy differential reference observation via {}", |
| 7036 | reference.compile_command |
| 7037 | ), |
| 7038 | artifacts_captured, |
| 7039 | comparison_basis: None, |
| 7040 | failure_stage: None, |
| 7041 | }, |
| 7042 | }, |
| 7043 | requested_artifacts, |
| 7044 | } |
| 7045 | } |
| 7046 | |
| 7047 | fn reference_compiler_spec(compiler: ReferenceCompiler) -> CompilerSpec { |
| 7048 | match compiler { |
| 7049 | ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 7050 | ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 7051 | } |
| 7052 | } |
| 7053 | |
| 7054 | fn run_generic_consistency_checks( |
| 7055 | compiler: &CompilerSpec, |
| 7056 | case: &CaseSpec, |
| 7057 | source: &Path, |
| 7058 | opt_level: OptLevel, |
| 7059 | tools: &ToolchainConfig, |
| 7060 | ) -> Vec<ConsistencyIssue> { |
| 7061 | let mut failures = Vec::new(); |
| 7062 | for check in &case.consistency_checks { |
| 7063 | let issue = match check { |
| 7064 | ConsistencyCheck::CliAsmReproducible => run_generic_cli_asm_reproducible( |
| 7065 | compiler, |
| 7066 | source, |
| 7067 | opt_level, |
| 7068 | case.repeat_count, |
| 7069 | tools, |
| 7070 | ), |
| 7071 | ConsistencyCheck::CliObjReproducible => run_generic_cli_obj_reproducible( |
| 7072 | compiler, |
| 7073 | source, |
| 7074 | opt_level, |
| 7075 | case.repeat_count, |
| 7076 | tools, |
| 7077 | ), |
| 7078 | ConsistencyCheck::CliRunReproducible => run_generic_cli_run_reproducible( |
| 7079 | compiler, |
| 7080 | source, |
| 7081 | opt_level, |
| 7082 | case.repeat_count, |
| 7083 | tools, |
| 7084 | ), |
| 7085 | _ => Some(ConsistencyIssue { |
| 7086 | check: *check, |
| 7087 | summary: "unsupported generic consistency check".into(), |
| 7088 | repeat_count: None, |
| 7089 | unique_variant_count: None, |
| 7090 | varying_components: Vec::new(), |
| 7091 | stable_components: Vec::new(), |
| 7092 | detail: format!( |
| 7093 | "generic compiler cases do not support '{}' yet", |
| 7094 | check.as_str() |
| 7095 | ), |
| 7096 | temp_root: next_consistency_temp_root(opt_level), |
| 7097 | }), |
| 7098 | }; |
| 7099 | if let Some(issue) = issue { |
| 7100 | failures.push(issue); |
| 7101 | } |
| 7102 | } |
| 7103 | failures |
| 7104 | } |
| 7105 | |
| 7106 | fn run_generic_cli_asm_reproducible( |
| 7107 | compiler: &CompilerSpec, |
| 7108 | source: &Path, |
| 7109 | opt_level: OptLevel, |
| 7110 | repeat_count: usize, |
| 7111 | tools: &ToolchainConfig, |
| 7112 | ) -> Option<ConsistencyIssue> { |
| 7113 | let temp_root = next_consistency_temp_root(opt_level); |
| 7114 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7115 | return Some(ConsistencyIssue { |
| 7116 | check: ConsistencyCheck::CliAsmReproducible, |
| 7117 | summary: "could not create consistency temp dir".into(), |
| 7118 | repeat_count: None, |
| 7119 | unique_variant_count: None, |
| 7120 | varying_components: Vec::new(), |
| 7121 | stable_components: Vec::new(), |
| 7122 | detail: format!( |
| 7123 | "cannot create consistency temp dir '{}': {}", |
| 7124 | temp_root.display(), |
| 7125 | err |
| 7126 | ), |
| 7127 | temp_root, |
| 7128 | }); |
| 7129 | } |
| 7130 | |
| 7131 | let requested = BTreeSet::from([ArtifactKey::Asm]); |
| 7132 | let mut runs = Vec::new(); |
| 7133 | for index in 0..repeat_count { |
| 7134 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7135 | Ok(observation) => observation, |
| 7136 | Err(detail) => { |
| 7137 | return Some(ConsistencyIssue { |
| 7138 | check: ConsistencyCheck::CliAsmReproducible, |
| 7139 | summary: "compiler observation failed during consistency check".into(), |
| 7140 | repeat_count: Some(repeat_count), |
| 7141 | unique_variant_count: None, |
| 7142 | varying_components: Vec::new(), |
| 7143 | stable_components: Vec::new(), |
| 7144 | detail, |
| 7145 | temp_root, |
| 7146 | }) |
| 7147 | } |
| 7148 | }; |
| 7149 | let asm = match observation_text_artifact(&observation, &ArtifactKey::Asm) { |
| 7150 | Ok(asm) => asm, |
| 7151 | Err(detail) => { |
| 7152 | return Some(ConsistencyIssue { |
| 7153 | check: ConsistencyCheck::CliAsmReproducible, |
| 7154 | summary: "missing asm artifact during consistency check".into(), |
| 7155 | repeat_count: Some(repeat_count), |
| 7156 | unique_variant_count: None, |
| 7157 | varying_components: Vec::new(), |
| 7158 | stable_components: Vec::new(), |
| 7159 | detail, |
| 7160 | temp_root, |
| 7161 | }) |
| 7162 | } |
| 7163 | }; |
| 7164 | runs.push(TextRun { |
| 7165 | label: format!("run {}", index + 1), |
| 7166 | command: observation_command_hint(&observation), |
| 7167 | normalized: normalize_text_artifact(&asm), |
| 7168 | }); |
| 7169 | } |
| 7170 | |
| 7171 | let unique_variant_count = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 7172 | if unique_variant_count > 1 { |
| 7173 | let (left, right) = first_distinct_text_pair(&runs).unwrap(); |
| 7174 | return Some(ConsistencyIssue { |
| 7175 | check: ConsistencyCheck::CliAsmReproducible, |
| 7176 | summary: format!( |
| 7177 | "repeat_count={} unique_variants={}", |
| 7178 | repeat_count, unique_variant_count |
| 7179 | ), |
| 7180 | repeat_count: Some(repeat_count), |
| 7181 | unique_variant_count: Some(unique_variant_count), |
| 7182 | varying_components: Vec::new(), |
| 7183 | stable_components: Vec::new(), |
| 7184 | detail: format!( |
| 7185 | "asm output was not reproducible for {}\n{}\n{}\n{}", |
| 7186 | compiler.display_name(), |
| 7187 | left.command, |
| 7188 | right.command, |
| 7189 | describe_text_difference( |
| 7190 | &left.normalized, |
| 7191 | &right.normalized, |
| 7192 | &left.label, |
| 7193 | &right.label |
| 7194 | ) |
| 7195 | ), |
| 7196 | temp_root, |
| 7197 | }); |
| 7198 | } |
| 7199 | |
| 7200 | let _ = fs::remove_dir_all(&temp_root); |
| 7201 | None |
| 7202 | } |
| 7203 | |
| 7204 | fn run_generic_cli_obj_reproducible( |
| 7205 | compiler: &CompilerSpec, |
| 7206 | source: &Path, |
| 7207 | opt_level: OptLevel, |
| 7208 | repeat_count: usize, |
| 7209 | tools: &ToolchainConfig, |
| 7210 | ) -> Option<ConsistencyIssue> { |
| 7211 | let temp_root = next_consistency_temp_root(opt_level); |
| 7212 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7213 | return Some(ConsistencyIssue { |
| 7214 | check: ConsistencyCheck::CliObjReproducible, |
| 7215 | summary: "could not create consistency temp dir".into(), |
| 7216 | repeat_count: None, |
| 7217 | unique_variant_count: None, |
| 7218 | varying_components: Vec::new(), |
| 7219 | stable_components: Vec::new(), |
| 7220 | detail: format!( |
| 7221 | "cannot create consistency temp dir '{}': {}", |
| 7222 | temp_root.display(), |
| 7223 | err |
| 7224 | ), |
| 7225 | temp_root, |
| 7226 | }); |
| 7227 | } |
| 7228 | |
| 7229 | let requested = BTreeSet::from([ArtifactKey::Obj]); |
| 7230 | let mut rendered_runs = Vec::new(); |
| 7231 | let mut object_runs = Vec::new(); |
| 7232 | let mut parseable = true; |
| 7233 | for index in 0..repeat_count { |
| 7234 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7235 | Ok(observation) => observation, |
| 7236 | Err(detail) => { |
| 7237 | return Some(ConsistencyIssue { |
| 7238 | check: ConsistencyCheck::CliObjReproducible, |
| 7239 | summary: "compiler observation failed during consistency check".into(), |
| 7240 | repeat_count: Some(repeat_count), |
| 7241 | unique_variant_count: None, |
| 7242 | varying_components: Vec::new(), |
| 7243 | stable_components: Vec::new(), |
| 7244 | detail, |
| 7245 | temp_root, |
| 7246 | }) |
| 7247 | } |
| 7248 | }; |
| 7249 | let obj_text = match observation_text_artifact(&observation, &ArtifactKey::Obj) { |
| 7250 | Ok(text) => text, |
| 7251 | Err(detail) => { |
| 7252 | return Some(ConsistencyIssue { |
| 7253 | check: ConsistencyCheck::CliObjReproducible, |
| 7254 | summary: "missing obj artifact during consistency check".into(), |
| 7255 | repeat_count: Some(repeat_count), |
| 7256 | unique_variant_count: None, |
| 7257 | varying_components: Vec::new(), |
| 7258 | stable_components: Vec::new(), |
| 7259 | detail, |
| 7260 | temp_root, |
| 7261 | }) |
| 7262 | } |
| 7263 | }; |
| 7264 | let label = format!("run {}", index + 1); |
| 7265 | let command = observation_command_hint(&observation); |
| 7266 | rendered_runs.push(TextRun { |
| 7267 | label: label.clone(), |
| 7268 | command: command.clone(), |
| 7269 | normalized: normalize_text_artifact(&obj_text), |
| 7270 | }); |
| 7271 | match parse_object_snapshot_text(&obj_text) { |
| 7272 | Ok(snapshot) => object_runs.push(ObjectRun { |
| 7273 | label, |
| 7274 | command, |
| 7275 | snapshot, |
| 7276 | }), |
| 7277 | Err(_) => parseable = false, |
| 7278 | } |
| 7279 | } |
| 7280 | |
| 7281 | if parseable { |
| 7282 | let rendered = object_runs |
| 7283 | .iter() |
| 7284 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 7285 | .collect::<Vec<_>>(); |
| 7286 | let unique_variant_count = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7287 | if unique_variant_count > 1 { |
| 7288 | let (left, right) = first_distinct_object_pair(&object_runs).unwrap(); |
| 7289 | let snapshots = object_runs |
| 7290 | .iter() |
| 7291 | .map(|run| &run.snapshot) |
| 7292 | .collect::<Vec<_>>(); |
| 7293 | let varying = varying_object_components(&snapshots) |
| 7294 | .into_iter() |
| 7295 | .map(str::to_string) |
| 7296 | .collect::<Vec<_>>(); |
| 7297 | let stable = stable_object_components(&snapshots) |
| 7298 | .into_iter() |
| 7299 | .map(str::to_string) |
| 7300 | .collect::<Vec<_>>(); |
| 7301 | return Some(ConsistencyIssue { |
| 7302 | check: ConsistencyCheck::CliObjReproducible, |
| 7303 | summary: format!( |
| 7304 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7305 | repeat_count, |
| 7306 | unique_variant_count, |
| 7307 | join_or_none_from_strings(&varying), |
| 7308 | join_or_none_from_strings(&stable) |
| 7309 | ), |
| 7310 | repeat_count: Some(repeat_count), |
| 7311 | unique_variant_count: Some(unique_variant_count), |
| 7312 | varying_components: varying, |
| 7313 | stable_components: stable, |
| 7314 | detail: format!( |
| 7315 | "object output was not reproducible for {}\n{}\n{}\n{}", |
| 7316 | compiler.display_name(), |
| 7317 | left.command, |
| 7318 | right.command, |
| 7319 | describe_object_difference( |
| 7320 | &left.snapshot, |
| 7321 | &right.snapshot, |
| 7322 | &left.label, |
| 7323 | &right.label |
| 7324 | ) |
| 7325 | ), |
| 7326 | temp_root, |
| 7327 | }); |
| 7328 | } |
| 7329 | } else { |
| 7330 | let unique_variant_count = |
| 7331 | count_unique_strings(rendered_runs.iter().map(|run| run.normalized.as_str())); |
| 7332 | if unique_variant_count > 1 { |
| 7333 | let (left, right) = first_distinct_text_pair(&rendered_runs).unwrap(); |
| 7334 | return Some(ConsistencyIssue { |
| 7335 | check: ConsistencyCheck::CliObjReproducible, |
| 7336 | summary: format!( |
| 7337 | "repeat_count={} unique_variants={}", |
| 7338 | repeat_count, unique_variant_count |
| 7339 | ), |
| 7340 | repeat_count: Some(repeat_count), |
| 7341 | unique_variant_count: Some(unique_variant_count), |
| 7342 | varying_components: Vec::new(), |
| 7343 | stable_components: Vec::new(), |
| 7344 | detail: format!( |
| 7345 | "object artifact text was not reproducible for {}\n{}\n{}\n{}", |
| 7346 | compiler.display_name(), |
| 7347 | left.command, |
| 7348 | right.command, |
| 7349 | describe_text_difference( |
| 7350 | &left.normalized, |
| 7351 | &right.normalized, |
| 7352 | &left.label, |
| 7353 | &right.label |
| 7354 | ) |
| 7355 | ), |
| 7356 | temp_root, |
| 7357 | }); |
| 7358 | } |
| 7359 | } |
| 7360 | |
| 7361 | let _ = fs::remove_dir_all(&temp_root); |
| 7362 | None |
| 7363 | } |
| 7364 | |
| 7365 | fn run_generic_cli_run_reproducible( |
| 7366 | compiler: &CompilerSpec, |
| 7367 | source: &Path, |
| 7368 | opt_level: OptLevel, |
| 7369 | repeat_count: usize, |
| 7370 | tools: &ToolchainConfig, |
| 7371 | ) -> Option<ConsistencyIssue> { |
| 7372 | let temp_root = next_consistency_temp_root(opt_level); |
| 7373 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7374 | return Some(ConsistencyIssue { |
| 7375 | check: ConsistencyCheck::CliRunReproducible, |
| 7376 | summary: "could not create consistency temp dir".into(), |
| 7377 | repeat_count: None, |
| 7378 | unique_variant_count: None, |
| 7379 | varying_components: Vec::new(), |
| 7380 | stable_components: Vec::new(), |
| 7381 | detail: format!( |
| 7382 | "cannot create consistency temp dir '{}': {}", |
| 7383 | temp_root.display(), |
| 7384 | err |
| 7385 | ), |
| 7386 | temp_root, |
| 7387 | }); |
| 7388 | } |
| 7389 | |
| 7390 | let requested = BTreeSet::from([ArtifactKey::Runtime]); |
| 7391 | let mut runs = Vec::new(); |
| 7392 | for index in 0..repeat_count { |
| 7393 | let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7394 | Ok(observation) => observation, |
| 7395 | Err(detail) => { |
| 7396 | return Some(ConsistencyIssue { |
| 7397 | check: ConsistencyCheck::CliRunReproducible, |
| 7398 | summary: "compiler observation failed during consistency check".into(), |
| 7399 | repeat_count: Some(repeat_count), |
| 7400 | unique_variant_count: None, |
| 7401 | varying_components: Vec::new(), |
| 7402 | stable_components: Vec::new(), |
| 7403 | detail, |
| 7404 | temp_root, |
| 7405 | }) |
| 7406 | } |
| 7407 | }; |
| 7408 | let run = match observation_run_capture(&observation) { |
| 7409 | Ok(run) => run, |
| 7410 | Err(detail) => { |
| 7411 | return Some(ConsistencyIssue { |
| 7412 | check: ConsistencyCheck::CliRunReproducible, |
| 7413 | summary: "missing runtime artifact during consistency check".into(), |
| 7414 | repeat_count: Some(repeat_count), |
| 7415 | unique_variant_count: None, |
| 7416 | varying_components: Vec::new(), |
| 7417 | stable_components: Vec::new(), |
| 7418 | detail, |
| 7419 | temp_root, |
| 7420 | }) |
| 7421 | } |
| 7422 | }; |
| 7423 | runs.push(BehaviorRun { |
| 7424 | label: format!("run {}", index + 1), |
| 7425 | command: observation_command_hint(&observation), |
| 7426 | signature: normalize_run_signature(&run), |
| 7427 | run, |
| 7428 | }); |
| 7429 | } |
| 7430 | |
| 7431 | let unique_variant_count = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 7432 | if unique_variant_count > 1 { |
| 7433 | let (left, right) = first_distinct_behavior_pair(&runs).unwrap(); |
| 7434 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 7435 | let varying = varying_run_components(&signatures) |
| 7436 | .into_iter() |
| 7437 | .map(str::to_string) |
| 7438 | .collect::<Vec<_>>(); |
| 7439 | let stable = stable_run_components(&signatures) |
| 7440 | .into_iter() |
| 7441 | .map(str::to_string) |
| 7442 | .collect::<Vec<_>>(); |
| 7443 | return Some(ConsistencyIssue { |
| 7444 | check: ConsistencyCheck::CliRunReproducible, |
| 7445 | summary: format!( |
| 7446 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7447 | repeat_count, |
| 7448 | unique_variant_count, |
| 7449 | join_or_none_from_strings(&varying), |
| 7450 | join_or_none_from_strings(&stable) |
| 7451 | ), |
| 7452 | repeat_count: Some(repeat_count), |
| 7453 | unique_variant_count: Some(unique_variant_count), |
| 7454 | varying_components: varying, |
| 7455 | stable_components: stable, |
| 7456 | detail: format!( |
| 7457 | "runtime behavior was not reproducible for {}\n{}\n{}\n{}", |
| 7458 | compiler.display_name(), |
| 7459 | left.command, |
| 7460 | right.command, |
| 7461 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 7462 | ), |
| 7463 | temp_root, |
| 7464 | }); |
| 7465 | } |
| 7466 | |
| 7467 | let _ = fs::remove_dir_all(&temp_root); |
| 7468 | None |
| 7469 | } |
| 7470 | |
| 7471 | fn observation_text_artifact( |
| 7472 | observation: &CompilerObservation, |
| 7473 | artifact: &ArtifactKey, |
| 7474 | ) -> Result<String, String> { |
| 7475 | match observation.artifacts.get(artifact) { |
| 7476 | Some(ArtifactValue::Text(text)) => Ok(text.clone()), |
| 7477 | Some(ArtifactValue::Int(_)) => Err(format!( |
| 7478 | "artifact '{}' is numeric, not text", |
| 7479 | artifact.as_str() |
| 7480 | )), |
| 7481 | Some(ArtifactValue::Run(_)) => Err(format!( |
| 7482 | "artifact '{}' is structured runtime data, not text", |
| 7483 | artifact.as_str() |
| 7484 | )), |
| 7485 | Some(ArtifactValue::Path(path)) => Err(format!( |
| 7486 | "artifact '{}' is path data ('{}'), not text", |
| 7487 | artifact.as_str(), |
| 7488 | path.display() |
| 7489 | )), |
| 7490 | None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 7491 | } |
| 7492 | } |
| 7493 | |
| 7494 | fn observation_run_capture(observation: &CompilerObservation) -> Result<RunCapture, String> { |
| 7495 | if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 7496 | return Ok(run.clone()); |
| 7497 | } |
| 7498 | |
| 7499 | Ok(RunCapture { |
| 7500 | exit_code: observation_run_exit_code(observation)?, |
| 7501 | stdout: observation_run_stdout(observation)?.to_string(), |
| 7502 | stderr: observation_run_stderr(observation)?.to_string(), |
| 7503 | }) |
| 7504 | } |
| 7505 | |
| 7506 | fn observation_command_hint(observation: &CompilerObservation) -> String { |
| 7507 | format!( |
| 7508 | "{} [{}; {}]", |
| 7509 | observation.compiler.display_name(), |
| 7510 | observation.provenance.backend_mode, |
| 7511 | observation.provenance.backend_detail |
| 7512 | ) |
| 7513 | } |
| 7514 | |
| 7515 | fn run_consistency_checks( |
| 7516 | case: &CaseSpec, |
| 7517 | prepared: &PreparedInput, |
| 7518 | opt_level: OptLevel, |
| 7519 | capture_result: &CaptureResult, |
| 7520 | tools: &ToolchainConfig, |
| 7521 | ) -> Vec<ConsistencyIssue> { |
| 7522 | let mut failures = Vec::new(); |
| 7523 | for check in &case.consistency_checks { |
| 7524 | let issue = match check { |
| 7525 | ConsistencyCheck::CliObjVsSystemAs => { |
| 7526 | run_cli_obj_vs_system_as(&prepared.compiler_source, opt_level, tools) |
| 7527 | } |
| 7528 | ConsistencyCheck::CliAsmReproducible => run_cli_asm_reproducible( |
| 7529 | &prepared.compiler_source, |
| 7530 | opt_level, |
| 7531 | case.repeat_count, |
| 7532 | tools, |
| 7533 | ), |
| 7534 | ConsistencyCheck::CliObjReproducible => run_cli_obj_reproducible( |
| 7535 | &prepared.compiler_source, |
| 7536 | opt_level, |
| 7537 | case.repeat_count, |
| 7538 | tools, |
| 7539 | ), |
| 7540 | ConsistencyCheck::CliRunReproducible => run_cli_run_reproducible( |
| 7541 | &prepared.compiler_source, |
| 7542 | opt_level, |
| 7543 | case.repeat_count, |
| 7544 | tools, |
| 7545 | ), |
| 7546 | ConsistencyCheck::CaptureAsmVsCliAsm => run_capture_asm_vs_cli_asm( |
| 7547 | &prepared.compiler_source, |
| 7548 | opt_level, |
| 7549 | case.repeat_count, |
| 7550 | capture_result, |
| 7551 | tools, |
| 7552 | ), |
| 7553 | ConsistencyCheck::CaptureObjVsCliObj => run_capture_obj_vs_cli_obj( |
| 7554 | &prepared.compiler_source, |
| 7555 | opt_level, |
| 7556 | case.repeat_count, |
| 7557 | capture_result, |
| 7558 | tools, |
| 7559 | ), |
| 7560 | ConsistencyCheck::CaptureRunVsCliRun => run_capture_run_vs_cli_run( |
| 7561 | &prepared.compiler_source, |
| 7562 | opt_level, |
| 7563 | case.repeat_count, |
| 7564 | capture_result, |
| 7565 | tools, |
| 7566 | ), |
| 7567 | ConsistencyCheck::CaptureAsmReproducible => run_capture_asm_reproducible( |
| 7568 | &prepared.compiler_source, |
| 7569 | opt_level, |
| 7570 | case.repeat_count, |
| 7571 | capture_result, |
| 7572 | tools, |
| 7573 | ), |
| 7574 | ConsistencyCheck::CaptureObjReproducible => run_capture_obj_reproducible( |
| 7575 | &prepared.compiler_source, |
| 7576 | opt_level, |
| 7577 | case.repeat_count, |
| 7578 | capture_result, |
| 7579 | tools, |
| 7580 | ), |
| 7581 | ConsistencyCheck::CaptureRunReproducible => run_capture_run_reproducible( |
| 7582 | &prepared.compiler_source, |
| 7583 | opt_level, |
| 7584 | case.repeat_count, |
| 7585 | capture_result, |
| 7586 | tools, |
| 7587 | ), |
| 7588 | }; |
| 7589 | if let Some(issue) = issue { |
| 7590 | failures.push(issue); |
| 7591 | } |
| 7592 | } |
| 7593 | failures |
| 7594 | } |
| 7595 | |
| 7596 | fn format_consistency_issues(issues: &[ConsistencyIssue]) -> String { |
| 7597 | issues |
| 7598 | .iter() |
| 7599 | .map(|issue| { |
| 7600 | format!( |
| 7601 | "consistency check '{}' failed\n{}", |
| 7602 | issue.check.as_str(), |
| 7603 | issue.detail |
| 7604 | ) |
| 7605 | }) |
| 7606 | .collect::<Vec<_>>() |
| 7607 | .join("\n\n") |
| 7608 | } |
| 7609 | |
| 7610 | fn cleanup_consistency_issues(issues: &[ConsistencyIssue]) { |
| 7611 | for issue in issues { |
| 7612 | let _ = fs::remove_dir_all(&issue.temp_root); |
| 7613 | } |
| 7614 | } |
| 7615 | |
| 7616 | fn run_cli_obj_vs_system_as( |
| 7617 | source: &Path, |
| 7618 | opt_level: OptLevel, |
| 7619 | tools: &ToolchainConfig, |
| 7620 | ) -> Option<ConsistencyIssue> { |
| 7621 | let temp_root = next_consistency_temp_root(opt_level); |
| 7622 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7623 | return Some(ConsistencyIssue { |
| 7624 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7625 | summary: "could not create consistency temp dir".into(), |
| 7626 | repeat_count: None, |
| 7627 | unique_variant_count: None, |
| 7628 | varying_components: Vec::new(), |
| 7629 | stable_components: Vec::new(), |
| 7630 | detail: format!( |
| 7631 | "cannot create consistency temp dir '{}': {}", |
| 7632 | temp_root.display(), |
| 7633 | err |
| 7634 | ), |
| 7635 | temp_root, |
| 7636 | }); |
| 7637 | } |
| 7638 | |
| 7639 | let asm_path = temp_root.join("from_cli.s"); |
| 7640 | let asm_obj_path = temp_root.join("from_cli_asm.o"); |
| 7641 | let obj_path = temp_root.join("from_cli_obj.o"); |
| 7642 | |
| 7643 | let asm_command = |
| 7644 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7645 | Ok(command) => command, |
| 7646 | Err(detail) => { |
| 7647 | return Some(ConsistencyIssue { |
| 7648 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7649 | summary: "armfortas -S failed during consistency check".into(), |
| 7650 | repeat_count: None, |
| 7651 | unique_variant_count: None, |
| 7652 | varying_components: Vec::new(), |
| 7653 | stable_components: Vec::new(), |
| 7654 | detail, |
| 7655 | temp_root, |
| 7656 | }) |
| 7657 | } |
| 7658 | }; |
| 7659 | |
| 7660 | let as_args = vec![ |
| 7661 | "-o".to_string(), |
| 7662 | asm_obj_path.display().to_string(), |
| 7663 | asm_path.display().to_string(), |
| 7664 | ]; |
| 7665 | let as_command = render_command(tools.system_as_bin(), &as_args); |
| 7666 | let as_output = match Command::new(tools.system_as_bin()) |
| 7667 | .args([ |
| 7668 | "-o", |
| 7669 | asm_obj_path.to_str().unwrap(), |
| 7670 | asm_path.to_str().unwrap(), |
| 7671 | ]) |
| 7672 | .output() |
| 7673 | { |
| 7674 | Ok(output) => output, |
| 7675 | Err(err) => { |
| 7676 | return Some(ConsistencyIssue { |
| 7677 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7678 | summary: "system assembler invocation failed".into(), |
| 7679 | repeat_count: None, |
| 7680 | unique_variant_count: None, |
| 7681 | varying_components: Vec::new(), |
| 7682 | stable_components: Vec::new(), |
| 7683 | detail: format!("{}\ncannot run assembler: {}", as_command, err), |
| 7684 | temp_root, |
| 7685 | }) |
| 7686 | } |
| 7687 | }; |
| 7688 | if !as_output.status.success() { |
| 7689 | let stderr = String::from_utf8_lossy(&as_output.stderr); |
| 7690 | return Some(ConsistencyIssue { |
| 7691 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7692 | summary: "system assembler rejected armfortas -S output".into(), |
| 7693 | repeat_count: None, |
| 7694 | unique_variant_count: None, |
| 7695 | varying_components: Vec::new(), |
| 7696 | stable_components: Vec::new(), |
| 7697 | detail: format!("{}\nassembler failed:\n{}", as_command, stderr), |
| 7698 | temp_root, |
| 7699 | }); |
| 7700 | } |
| 7701 | |
| 7702 | let obj_command = |
| 7703 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7704 | Ok(command) => command, |
| 7705 | Err(detail) => { |
| 7706 | return Some(ConsistencyIssue { |
| 7707 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7708 | summary: "armfortas -c failed during consistency check".into(), |
| 7709 | repeat_count: None, |
| 7710 | unique_variant_count: None, |
| 7711 | varying_components: Vec::new(), |
| 7712 | stable_components: Vec::new(), |
| 7713 | detail, |
| 7714 | temp_root, |
| 7715 | }) |
| 7716 | } |
| 7717 | }; |
| 7718 | |
| 7719 | let asm_snapshot = match object_snapshot(&asm_obj_path, tools) { |
| 7720 | Ok(snapshot) => snapshot, |
| 7721 | Err(detail) => { |
| 7722 | return Some(ConsistencyIssue { |
| 7723 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7724 | summary: "could not snapshot object assembled from -S output".into(), |
| 7725 | repeat_count: None, |
| 7726 | unique_variant_count: None, |
| 7727 | varying_components: Vec::new(), |
| 7728 | stable_components: Vec::new(), |
| 7729 | detail: format!("{}\n{}", as_command, detail), |
| 7730 | temp_root, |
| 7731 | }) |
| 7732 | } |
| 7733 | }; |
| 7734 | let obj_snapshot = match object_snapshot(&obj_path, tools) { |
| 7735 | Ok(snapshot) => snapshot, |
| 7736 | Err(detail) => { |
| 7737 | return Some(ConsistencyIssue { |
| 7738 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7739 | summary: "could not snapshot object from armfortas -c".into(), |
| 7740 | repeat_count: None, |
| 7741 | unique_variant_count: None, |
| 7742 | varying_components: Vec::new(), |
| 7743 | stable_components: Vec::new(), |
| 7744 | detail: format!("{}\n{}", obj_command, detail), |
| 7745 | temp_root, |
| 7746 | }) |
| 7747 | } |
| 7748 | }; |
| 7749 | |
| 7750 | if asm_snapshot != obj_snapshot { |
| 7751 | let snapshots = [&asm_snapshot, &obj_snapshot]; |
| 7752 | let varying = varying_object_components(&snapshots) |
| 7753 | .into_iter() |
| 7754 | .map(str::to_string) |
| 7755 | .collect::<Vec<_>>(); |
| 7756 | let stable = stable_object_components(&snapshots) |
| 7757 | .into_iter() |
| 7758 | .map(str::to_string) |
| 7759 | .collect::<Vec<_>>(); |
| 7760 | return Some(ConsistencyIssue { |
| 7761 | check: ConsistencyCheck::CliObjVsSystemAs, |
| 7762 | summary: format!( |
| 7763 | "varying_components={} stable_components={}", |
| 7764 | join_or_none_from_strings(&varying), |
| 7765 | join_or_none_from_strings(&stable) |
| 7766 | ), |
| 7767 | repeat_count: None, |
| 7768 | unique_variant_count: None, |
| 7769 | varying_components: varying, |
| 7770 | stable_components: stable, |
| 7771 | detail: format!( |
| 7772 | "object snapshot mismatch between armfortas -S | as and armfortas -c\n{}\n{}\n{}\n{}", |
| 7773 | asm_command, |
| 7774 | as_command, |
| 7775 | obj_command, |
| 7776 | describe_object_difference(&asm_snapshot, &obj_snapshot, "-S | as", "-c") |
| 7777 | ), |
| 7778 | temp_root, |
| 7779 | }); |
| 7780 | } |
| 7781 | |
| 7782 | let _ = fs::remove_dir_all(&temp_root); |
| 7783 | None |
| 7784 | } |
| 7785 | |
| 7786 | fn run_cli_asm_reproducible( |
| 7787 | source: &Path, |
| 7788 | opt_level: OptLevel, |
| 7789 | repeat_count: usize, |
| 7790 | tools: &ToolchainConfig, |
| 7791 | ) -> Option<ConsistencyIssue> { |
| 7792 | let temp_root = next_consistency_temp_root(opt_level); |
| 7793 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7794 | return Some(ConsistencyIssue { |
| 7795 | check: ConsistencyCheck::CliAsmReproducible, |
| 7796 | summary: "could not create consistency temp dir".into(), |
| 7797 | repeat_count: None, |
| 7798 | unique_variant_count: None, |
| 7799 | varying_components: Vec::new(), |
| 7800 | stable_components: Vec::new(), |
| 7801 | detail: format!( |
| 7802 | "cannot create consistency temp dir '{}': {}", |
| 7803 | temp_root.display(), |
| 7804 | err |
| 7805 | ), |
| 7806 | temp_root, |
| 7807 | }); |
| 7808 | } |
| 7809 | |
| 7810 | let mut runs = Vec::new(); |
| 7811 | for index in 0..repeat_count { |
| 7812 | let asm_path = temp_root.join(format!("run_{:02}.s", index)); |
| 7813 | let command = |
| 7814 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7815 | Ok(command) => command, |
| 7816 | Err(detail) => { |
| 7817 | return Some(ConsistencyIssue { |
| 7818 | check: ConsistencyCheck::CliAsmReproducible, |
| 7819 | summary: "armfortas -S failed during reproducibility check".into(), |
| 7820 | repeat_count: None, |
| 7821 | unique_variant_count: None, |
| 7822 | varying_components: Vec::new(), |
| 7823 | stable_components: Vec::new(), |
| 7824 | detail, |
| 7825 | temp_root, |
| 7826 | }) |
| 7827 | } |
| 7828 | }; |
| 7829 | let text = match read_text_artifact(&asm_path) { |
| 7830 | Ok(text) => text, |
| 7831 | Err(detail) => { |
| 7832 | return Some(ConsistencyIssue { |
| 7833 | check: ConsistencyCheck::CliAsmReproducible, |
| 7834 | summary: "could not read emitted assembly during reproducibility check".into(), |
| 7835 | repeat_count: None, |
| 7836 | unique_variant_count: None, |
| 7837 | varying_components: Vec::new(), |
| 7838 | stable_components: Vec::new(), |
| 7839 | detail, |
| 7840 | temp_root, |
| 7841 | }) |
| 7842 | } |
| 7843 | }; |
| 7844 | runs.push(TextRun { |
| 7845 | label: format!("run {} (-S)", index + 1), |
| 7846 | command, |
| 7847 | normalized: normalize_text_artifact(&text), |
| 7848 | }); |
| 7849 | } |
| 7850 | |
| 7851 | let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 7852 | if unique_variants > 1 { |
| 7853 | let (left, right) = |
| 7854 | first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7855 | return Some(ConsistencyIssue { |
| 7856 | check: ConsistencyCheck::CliAsmReproducible, |
| 7857 | summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 7858 | repeat_count: Some(repeat_count), |
| 7859 | unique_variant_count: Some(unique_variants), |
| 7860 | varying_components: Vec::new(), |
| 7861 | stable_components: Vec::new(), |
| 7862 | detail: format!( |
| 7863 | "assembly output is not reproducible across repeated armfortas -S runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 7864 | repeat_count, |
| 7865 | unique_variants, |
| 7866 | left.command, |
| 7867 | right.command, |
| 7868 | describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 7869 | ), |
| 7870 | temp_root, |
| 7871 | }); |
| 7872 | } |
| 7873 | |
| 7874 | let _ = fs::remove_dir_all(&temp_root); |
| 7875 | None |
| 7876 | } |
| 7877 | |
| 7878 | fn run_cli_obj_reproducible( |
| 7879 | source: &Path, |
| 7880 | opt_level: OptLevel, |
| 7881 | repeat_count: usize, |
| 7882 | tools: &ToolchainConfig, |
| 7883 | ) -> Option<ConsistencyIssue> { |
| 7884 | let temp_root = next_consistency_temp_root(opt_level); |
| 7885 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7886 | return Some(ConsistencyIssue { |
| 7887 | check: ConsistencyCheck::CliObjReproducible, |
| 7888 | summary: "could not create consistency temp dir".into(), |
| 7889 | repeat_count: None, |
| 7890 | unique_variant_count: None, |
| 7891 | varying_components: Vec::new(), |
| 7892 | stable_components: Vec::new(), |
| 7893 | detail: format!( |
| 7894 | "cannot create consistency temp dir '{}': {}", |
| 7895 | temp_root.display(), |
| 7896 | err |
| 7897 | ), |
| 7898 | temp_root, |
| 7899 | }); |
| 7900 | } |
| 7901 | |
| 7902 | let mut runs = Vec::new(); |
| 7903 | for index in 0..repeat_count { |
| 7904 | let obj_path = temp_root.join(format!("run_{:02}.o", index)); |
| 7905 | let command = |
| 7906 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7907 | Ok(command) => command, |
| 7908 | Err(detail) => { |
| 7909 | return Some(ConsistencyIssue { |
| 7910 | check: ConsistencyCheck::CliObjReproducible, |
| 7911 | summary: "armfortas -c failed during reproducibility check".into(), |
| 7912 | repeat_count: None, |
| 7913 | unique_variant_count: None, |
| 7914 | varying_components: Vec::new(), |
| 7915 | stable_components: Vec::new(), |
| 7916 | detail, |
| 7917 | temp_root, |
| 7918 | }) |
| 7919 | } |
| 7920 | }; |
| 7921 | let snapshot = match object_snapshot(&obj_path, tools) { |
| 7922 | Ok(snapshot) => snapshot, |
| 7923 | Err(detail) => { |
| 7924 | return Some(ConsistencyIssue { |
| 7925 | check: ConsistencyCheck::CliObjReproducible, |
| 7926 | summary: "could not snapshot object during reproducibility check".into(), |
| 7927 | repeat_count: None, |
| 7928 | unique_variant_count: None, |
| 7929 | varying_components: Vec::new(), |
| 7930 | stable_components: Vec::new(), |
| 7931 | detail: format!("{}\n{}", command, detail), |
| 7932 | temp_root, |
| 7933 | }) |
| 7934 | } |
| 7935 | }; |
| 7936 | runs.push(ObjectRun { |
| 7937 | label: format!("run {} (-c)", index + 1), |
| 7938 | command, |
| 7939 | snapshot, |
| 7940 | }); |
| 7941 | } |
| 7942 | |
| 7943 | let rendered = runs |
| 7944 | .iter() |
| 7945 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 7946 | .collect::<Vec<_>>(); |
| 7947 | let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7948 | if unique_variants > 1 { |
| 7949 | let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 7950 | let (left, right) = |
| 7951 | first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7952 | let varying = join_or_none(&varying_object_components(&snapshots)); |
| 7953 | let stable = join_or_none(&stable_object_components(&snapshots)); |
| 7954 | let varying_components = varying_object_components(&snapshots) |
| 7955 | .into_iter() |
| 7956 | .map(str::to_string) |
| 7957 | .collect::<Vec<_>>(); |
| 7958 | let stable_components = stable_object_components(&snapshots) |
| 7959 | .into_iter() |
| 7960 | .map(str::to_string) |
| 7961 | .collect::<Vec<_>>(); |
| 7962 | return Some(ConsistencyIssue { |
| 7963 | check: ConsistencyCheck::CliObjReproducible, |
| 7964 | summary: format!( |
| 7965 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7966 | repeat_count, unique_variants, varying, stable |
| 7967 | ), |
| 7968 | repeat_count: Some(repeat_count), |
| 7969 | unique_variant_count: Some(unique_variants), |
| 7970 | varying_components, |
| 7971 | stable_components, |
| 7972 | detail: format!( |
| 7973 | "object output is not reproducible across repeated armfortas -c runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 7974 | repeat_count, |
| 7975 | unique_variants, |
| 7976 | varying, |
| 7977 | stable, |
| 7978 | left.command, |
| 7979 | right.command, |
| 7980 | describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 7981 | ), |
| 7982 | temp_root, |
| 7983 | }); |
| 7984 | } |
| 7985 | |
| 7986 | let _ = fs::remove_dir_all(&temp_root); |
| 7987 | None |
| 7988 | } |
| 7989 | |
| 7990 | fn run_cli_run_reproducible( |
| 7991 | source: &Path, |
| 7992 | opt_level: OptLevel, |
| 7993 | repeat_count: usize, |
| 7994 | tools: &ToolchainConfig, |
| 7995 | ) -> Option<ConsistencyIssue> { |
| 7996 | let temp_root = next_consistency_temp_root(opt_level); |
| 7997 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7998 | return Some(ConsistencyIssue { |
| 7999 | check: ConsistencyCheck::CliRunReproducible, |
| 8000 | summary: "could not create consistency temp dir".into(), |
| 8001 | repeat_count: None, |
| 8002 | unique_variant_count: None, |
| 8003 | varying_components: Vec::new(), |
| 8004 | stable_components: Vec::new(), |
| 8005 | detail: format!( |
| 8006 | "cannot create consistency temp dir '{}': {}", |
| 8007 | temp_root.display(), |
| 8008 | err |
| 8009 | ), |
| 8010 | temp_root, |
| 8011 | }); |
| 8012 | } |
| 8013 | |
| 8014 | let mut runs = Vec::new(); |
| 8015 | for index in 0..repeat_count { |
| 8016 | let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 8017 | let build_command = match compile_with_driver( |
| 8018 | source, |
| 8019 | opt_level, |
| 8020 | DriverEmitMode::Binary, |
| 8021 | &binary_path, |
| 8022 | tools, |
| 8023 | ) { |
| 8024 | Ok(command) => command, |
| 8025 | Err(detail) => { |
| 8026 | return Some(ConsistencyIssue { |
| 8027 | check: ConsistencyCheck::CliRunReproducible, |
| 8028 | summary: "armfortas binary build failed during runtime reproducibility check" |
| 8029 | .into(), |
| 8030 | repeat_count: None, |
| 8031 | unique_variant_count: None, |
| 8032 | varying_components: Vec::new(), |
| 8033 | stable_components: Vec::new(), |
| 8034 | detail, |
| 8035 | temp_root, |
| 8036 | }) |
| 8037 | } |
| 8038 | }; |
| 8039 | let run_command = render_binary_run_command(&binary_path); |
| 8040 | let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 8041 | Ok(run) => run, |
| 8042 | Err(detail) => { |
| 8043 | return Some(ConsistencyIssue { |
| 8044 | check: ConsistencyCheck::CliRunReproducible, |
| 8045 | summary: "armfortas binary could not run during runtime reproducibility check" |
| 8046 | .into(), |
| 8047 | repeat_count: None, |
| 8048 | unique_variant_count: None, |
| 8049 | varying_components: Vec::new(), |
| 8050 | stable_components: Vec::new(), |
| 8051 | detail, |
| 8052 | temp_root, |
| 8053 | }) |
| 8054 | } |
| 8055 | }; |
| 8056 | let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 8057 | if let Err(err) = write_behavior_run_artifacts( |
| 8058 | &temp_root, |
| 8059 | &format!("cli_run_{:02}", index), |
| 8060 | &command, |
| 8061 | &run, |
| 8062 | ) { |
| 8063 | return Some(ConsistencyIssue { |
| 8064 | check: ConsistencyCheck::CliRunReproducible, |
| 8065 | summary: "could not write cli runtime artifact".into(), |
| 8066 | repeat_count: None, |
| 8067 | unique_variant_count: None, |
| 8068 | varying_components: Vec::new(), |
| 8069 | stable_components: Vec::new(), |
| 8070 | detail: format!("cannot write cli runtime artifact: {}", err), |
| 8071 | temp_root, |
| 8072 | }); |
| 8073 | } |
| 8074 | runs.push(BehaviorRun { |
| 8075 | label: format!("cli run {}", index + 1), |
| 8076 | command, |
| 8077 | signature: normalize_run_signature(&run), |
| 8078 | run, |
| 8079 | }); |
| 8080 | } |
| 8081 | |
| 8082 | let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 8083 | if unique_variants > 1 { |
| 8084 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 8085 | let varying = varying_run_components(&signatures); |
| 8086 | let stable = stable_run_components(&signatures); |
| 8087 | let (left, right) = first_distinct_behavior_pair(&runs) |
| 8088 | .expect("unique variants > 1 implies a distinct pair"); |
| 8089 | return Some(ConsistencyIssue { |
| 8090 | check: ConsistencyCheck::CliRunReproducible, |
| 8091 | summary: format!( |
| 8092 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8093 | repeat_count, |
| 8094 | unique_variants, |
| 8095 | join_or_none(&varying), |
| 8096 | join_or_none(&stable) |
| 8097 | ), |
| 8098 | repeat_count: Some(repeat_count), |
| 8099 | unique_variant_count: Some(unique_variants), |
| 8100 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8101 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8102 | detail: format!( |
| 8103 | "armfortas runtime behavior is not reproducible across repeated full CLI builds\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8104 | repeat_count, |
| 8105 | unique_variants, |
| 8106 | join_or_none(&varying), |
| 8107 | join_or_none(&stable), |
| 8108 | left.command, |
| 8109 | right.command, |
| 8110 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 8111 | ), |
| 8112 | temp_root, |
| 8113 | }); |
| 8114 | } |
| 8115 | |
| 8116 | let _ = fs::remove_dir_all(&temp_root); |
| 8117 | None |
| 8118 | } |
| 8119 | |
| 8120 | fn run_capture_asm_vs_cli_asm( |
| 8121 | source: &Path, |
| 8122 | opt_level: OptLevel, |
| 8123 | repeat_count: usize, |
| 8124 | capture_result: &CaptureResult, |
| 8125 | tools: &ToolchainConfig, |
| 8126 | ) -> Option<ConsistencyIssue> { |
| 8127 | let temp_root = next_consistency_temp_root(opt_level); |
| 8128 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8129 | return Some(ConsistencyIssue { |
| 8130 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8131 | summary: "could not create consistency temp dir".into(), |
| 8132 | repeat_count: None, |
| 8133 | unique_variant_count: None, |
| 8134 | varying_components: Vec::new(), |
| 8135 | stable_components: Vec::new(), |
| 8136 | detail: format!( |
| 8137 | "cannot create consistency temp dir '{}': {}", |
| 8138 | temp_root.display(), |
| 8139 | err |
| 8140 | ), |
| 8141 | temp_root, |
| 8142 | }); |
| 8143 | } |
| 8144 | |
| 8145 | let capture_command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 8146 | let capture_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 8147 | Ok(text) => text, |
| 8148 | Err(detail) => { |
| 8149 | return Some(ConsistencyIssue { |
| 8150 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8151 | summary: "capture result did not include assembly text".into(), |
| 8152 | repeat_count: None, |
| 8153 | unique_variant_count: None, |
| 8154 | varying_components: Vec::new(), |
| 8155 | stable_components: Vec::new(), |
| 8156 | detail, |
| 8157 | temp_root, |
| 8158 | }) |
| 8159 | } |
| 8160 | }; |
| 8161 | if let Err(err) = fs::write(temp_root.join("from_capture.s"), capture_text) { |
| 8162 | return Some(ConsistencyIssue { |
| 8163 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8164 | summary: "could not write captured assembly artifact".into(), |
| 8165 | repeat_count: None, |
| 8166 | unique_variant_count: None, |
| 8167 | varying_components: Vec::new(), |
| 8168 | stable_components: Vec::new(), |
| 8169 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 8170 | temp_root, |
| 8171 | }); |
| 8172 | } |
| 8173 | let capture_normalized = normalize_text_artifact(capture_text); |
| 8174 | |
| 8175 | let mut cli_runs = Vec::new(); |
| 8176 | let mut mismatch_indices = Vec::new(); |
| 8177 | for index in 0..repeat_count { |
| 8178 | let asm_path = temp_root.join(format!("cli_run_{:02}.s", index)); |
| 8179 | let command = |
| 8180 | match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 8181 | Ok(command) => command, |
| 8182 | Err(detail) => { |
| 8183 | return Some(ConsistencyIssue { |
| 8184 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8185 | summary: "armfortas -S failed during capture-vs-cli consistency check" |
| 8186 | .into(), |
| 8187 | repeat_count: None, |
| 8188 | unique_variant_count: None, |
| 8189 | varying_components: Vec::new(), |
| 8190 | stable_components: Vec::new(), |
| 8191 | detail, |
| 8192 | temp_root, |
| 8193 | }) |
| 8194 | } |
| 8195 | }; |
| 8196 | let text = match read_text_artifact(&asm_path) { |
| 8197 | Ok(text) => text, |
| 8198 | Err(detail) => { |
| 8199 | return Some(ConsistencyIssue { |
| 8200 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8201 | summary: "could not read cli assembly artifact".into(), |
| 8202 | repeat_count: None, |
| 8203 | unique_variant_count: None, |
| 8204 | varying_components: Vec::new(), |
| 8205 | stable_components: Vec::new(), |
| 8206 | detail, |
| 8207 | temp_root, |
| 8208 | }) |
| 8209 | } |
| 8210 | }; |
| 8211 | let normalized = normalize_text_artifact(&text); |
| 8212 | if normalized != capture_normalized { |
| 8213 | mismatch_indices.push(index); |
| 8214 | } |
| 8215 | cli_runs.push(TextRun { |
| 8216 | label: format!("cli run {} (-S)", index + 1), |
| 8217 | command, |
| 8218 | normalized, |
| 8219 | }); |
| 8220 | } |
| 8221 | |
| 8222 | if !mismatch_indices.is_empty() { |
| 8223 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8224 | let unique_cli_variants = |
| 8225 | count_unique_strings(cli_runs.iter().map(|run| run.normalized.as_str())); |
| 8226 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8227 | return Some(ConsistencyIssue { |
| 8228 | check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8229 | summary: format!( |
| 8230 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={}", |
| 8231 | repeat_count, |
| 8232 | matching_runs, |
| 8233 | mismatch_indices.len(), |
| 8234 | unique_cli_variants |
| 8235 | ), |
| 8236 | repeat_count: Some(repeat_count), |
| 8237 | unique_variant_count: Some(unique_cli_variants), |
| 8238 | varying_components: Vec::new(), |
| 8239 | stable_components: Vec::new(), |
| 8240 | detail: format!( |
| 8241 | "captured assembly does not match repeated armfortas -S runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8242 | repeat_count, |
| 8243 | matching_runs, |
| 8244 | mismatch_indices.len(), |
| 8245 | unique_cli_variants, |
| 8246 | capture_command, |
| 8247 | first_mismatch.command, |
| 8248 | describe_text_difference( |
| 8249 | &capture_normalized, |
| 8250 | &first_mismatch.normalized, |
| 8251 | "capture asm", |
| 8252 | &first_mismatch.label |
| 8253 | ) |
| 8254 | ), |
| 8255 | temp_root, |
| 8256 | }); |
| 8257 | } |
| 8258 | |
| 8259 | let _ = fs::remove_dir_all(&temp_root); |
| 8260 | None |
| 8261 | } |
| 8262 | |
| 8263 | fn run_capture_obj_vs_cli_obj( |
| 8264 | source: &Path, |
| 8265 | opt_level: OptLevel, |
| 8266 | repeat_count: usize, |
| 8267 | capture_result: &CaptureResult, |
| 8268 | tools: &ToolchainConfig, |
| 8269 | ) -> Option<ConsistencyIssue> { |
| 8270 | let temp_root = next_consistency_temp_root(opt_level); |
| 8271 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8272 | return Some(ConsistencyIssue { |
| 8273 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8274 | summary: "could not create consistency temp dir".into(), |
| 8275 | repeat_count: None, |
| 8276 | unique_variant_count: None, |
| 8277 | varying_components: Vec::new(), |
| 8278 | stable_components: Vec::new(), |
| 8279 | detail: format!( |
| 8280 | "cannot create consistency temp dir '{}': {}", |
| 8281 | temp_root.display(), |
| 8282 | err |
| 8283 | ), |
| 8284 | temp_root, |
| 8285 | }); |
| 8286 | } |
| 8287 | |
| 8288 | let capture_command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8289 | let capture_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8290 | Ok(text) => text, |
| 8291 | Err(detail) => { |
| 8292 | return Some(ConsistencyIssue { |
| 8293 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8294 | summary: "capture result did not include object snapshot text".into(), |
| 8295 | repeat_count: None, |
| 8296 | unique_variant_count: None, |
| 8297 | varying_components: Vec::new(), |
| 8298 | stable_components: Vec::new(), |
| 8299 | detail, |
| 8300 | temp_root, |
| 8301 | }) |
| 8302 | } |
| 8303 | }; |
| 8304 | if let Err(err) = fs::write(temp_root.join("from_capture.obj.txt"), capture_text) { |
| 8305 | return Some(ConsistencyIssue { |
| 8306 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8307 | summary: "could not write captured object snapshot artifact".into(), |
| 8308 | repeat_count: None, |
| 8309 | unique_variant_count: None, |
| 8310 | varying_components: Vec::new(), |
| 8311 | stable_components: Vec::new(), |
| 8312 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8313 | temp_root, |
| 8314 | }); |
| 8315 | } |
| 8316 | let capture_snapshot = match parse_object_snapshot_text(capture_text) { |
| 8317 | Ok(snapshot) => snapshot, |
| 8318 | Err(detail) => { |
| 8319 | return Some(ConsistencyIssue { |
| 8320 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8321 | summary: "captured object snapshot had an unexpected format".into(), |
| 8322 | repeat_count: None, |
| 8323 | unique_variant_count: None, |
| 8324 | varying_components: Vec::new(), |
| 8325 | stable_components: Vec::new(), |
| 8326 | detail, |
| 8327 | temp_root, |
| 8328 | }) |
| 8329 | } |
| 8330 | }; |
| 8331 | |
| 8332 | let mut cli_runs = Vec::new(); |
| 8333 | let mut mismatch_indices = Vec::new(); |
| 8334 | for index in 0..repeat_count { |
| 8335 | let obj_path = temp_root.join(format!("cli_run_{:02}.o", index)); |
| 8336 | let command = |
| 8337 | match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 8338 | Ok(command) => command, |
| 8339 | Err(detail) => { |
| 8340 | return Some(ConsistencyIssue { |
| 8341 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8342 | summary: "armfortas -c failed during capture-vs-cli consistency check" |
| 8343 | .into(), |
| 8344 | repeat_count: None, |
| 8345 | unique_variant_count: None, |
| 8346 | varying_components: Vec::new(), |
| 8347 | stable_components: Vec::new(), |
| 8348 | detail, |
| 8349 | temp_root, |
| 8350 | }) |
| 8351 | } |
| 8352 | }; |
| 8353 | let snapshot = match object_snapshot(&obj_path, tools) { |
| 8354 | Ok(snapshot) => snapshot, |
| 8355 | Err(detail) => { |
| 8356 | return Some(ConsistencyIssue { |
| 8357 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8358 | summary: "could not snapshot cli object artifact".into(), |
| 8359 | repeat_count: None, |
| 8360 | unique_variant_count: None, |
| 8361 | varying_components: Vec::new(), |
| 8362 | stable_components: Vec::new(), |
| 8363 | detail: format!("{}\n{}", command, detail), |
| 8364 | temp_root, |
| 8365 | }) |
| 8366 | } |
| 8367 | }; |
| 8368 | if let Err(err) = fs::write( |
| 8369 | temp_root.join(format!("cli_run_{:02}.obj.txt", index)), |
| 8370 | render_object_snapshot(&snapshot), |
| 8371 | ) { |
| 8372 | return Some(ConsistencyIssue { |
| 8373 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8374 | summary: "could not write cli object snapshot artifact".into(), |
| 8375 | repeat_count: None, |
| 8376 | unique_variant_count: None, |
| 8377 | varying_components: Vec::new(), |
| 8378 | stable_components: Vec::new(), |
| 8379 | detail: format!("cannot write cli object snapshot artifact: {}", err), |
| 8380 | temp_root, |
| 8381 | }); |
| 8382 | } |
| 8383 | if snapshot != capture_snapshot { |
| 8384 | mismatch_indices.push(index); |
| 8385 | } |
| 8386 | cli_runs.push(ObjectRun { |
| 8387 | label: format!("cli run {} (-c)", index + 1), |
| 8388 | command, |
| 8389 | snapshot, |
| 8390 | }); |
| 8391 | } |
| 8392 | |
| 8393 | if !mismatch_indices.is_empty() { |
| 8394 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8395 | let rendered = cli_runs |
| 8396 | .iter() |
| 8397 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 8398 | .collect::<Vec<_>>(); |
| 8399 | let unique_cli_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8400 | let mismatch_snapshots = mismatch_indices |
| 8401 | .iter() |
| 8402 | .map(|index| &cli_runs[*index].snapshot) |
| 8403 | .collect::<Vec<_>>(); |
| 8404 | let mut summary_snapshots = vec![&capture_snapshot]; |
| 8405 | summary_snapshots.extend(mismatch_snapshots.iter().copied()); |
| 8406 | let varying = varying_object_components(&summary_snapshots) |
| 8407 | .into_iter() |
| 8408 | .map(str::to_string) |
| 8409 | .collect::<Vec<_>>(); |
| 8410 | let stable = stable_object_components(&summary_snapshots) |
| 8411 | .into_iter() |
| 8412 | .map(str::to_string) |
| 8413 | .collect::<Vec<_>>(); |
| 8414 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8415 | return Some(ConsistencyIssue { |
| 8416 | check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8417 | summary: format!( |
| 8418 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8419 | repeat_count, |
| 8420 | matching_runs, |
| 8421 | mismatch_indices.len(), |
| 8422 | unique_cli_variants, |
| 8423 | join_or_none_from_strings(&varying), |
| 8424 | join_or_none_from_strings(&stable) |
| 8425 | ), |
| 8426 | repeat_count: Some(repeat_count), |
| 8427 | unique_variant_count: Some(unique_cli_variants), |
| 8428 | varying_components: varying, |
| 8429 | stable_components: stable, |
| 8430 | detail: format!( |
| 8431 | "captured object snapshot does not match repeated armfortas -c runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8432 | repeat_count, |
| 8433 | matching_runs, |
| 8434 | mismatch_indices.len(), |
| 8435 | unique_cli_variants, |
| 8436 | capture_command, |
| 8437 | first_mismatch.command, |
| 8438 | describe_object_difference( |
| 8439 | &capture_snapshot, |
| 8440 | &first_mismatch.snapshot, |
| 8441 | "capture obj", |
| 8442 | &first_mismatch.label |
| 8443 | ) |
| 8444 | ), |
| 8445 | temp_root, |
| 8446 | }); |
| 8447 | } |
| 8448 | |
| 8449 | let _ = fs::remove_dir_all(&temp_root); |
| 8450 | None |
| 8451 | } |
| 8452 | |
| 8453 | fn run_capture_run_vs_cli_run( |
| 8454 | source: &Path, |
| 8455 | opt_level: OptLevel, |
| 8456 | repeat_count: usize, |
| 8457 | capture_result: &CaptureResult, |
| 8458 | tools: &ToolchainConfig, |
| 8459 | ) -> Option<ConsistencyIssue> { |
| 8460 | let temp_root = next_consistency_temp_root(opt_level); |
| 8461 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8462 | return Some(ConsistencyIssue { |
| 8463 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8464 | summary: "could not create consistency temp dir".into(), |
| 8465 | repeat_count: None, |
| 8466 | unique_variant_count: None, |
| 8467 | varying_components: Vec::new(), |
| 8468 | stable_components: Vec::new(), |
| 8469 | detail: format!( |
| 8470 | "cannot create consistency temp dir '{}': {}", |
| 8471 | temp_root.display(), |
| 8472 | err |
| 8473 | ), |
| 8474 | temp_root, |
| 8475 | }); |
| 8476 | } |
| 8477 | |
| 8478 | let capture_command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8479 | let capture_run = match capture_run_stage(capture_result) { |
| 8480 | Ok(run) => run.clone(), |
| 8481 | Err(detail) => { |
| 8482 | return Some(ConsistencyIssue { |
| 8483 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8484 | summary: "capture result did not include runtime behavior".into(), |
| 8485 | repeat_count: None, |
| 8486 | unique_variant_count: None, |
| 8487 | varying_components: Vec::new(), |
| 8488 | stable_components: Vec::new(), |
| 8489 | detail, |
| 8490 | temp_root, |
| 8491 | }) |
| 8492 | } |
| 8493 | }; |
| 8494 | if let Err(err) = |
| 8495 | write_behavior_run_artifacts(&temp_root, "from_capture", &capture_command, &capture_run) |
| 8496 | { |
| 8497 | return Some(ConsistencyIssue { |
| 8498 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8499 | summary: "could not write captured runtime artifact".into(), |
| 8500 | repeat_count: None, |
| 8501 | unique_variant_count: None, |
| 8502 | varying_components: Vec::new(), |
| 8503 | stable_components: Vec::new(), |
| 8504 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 8505 | temp_root, |
| 8506 | }); |
| 8507 | } |
| 8508 | let capture_signature = normalize_run_signature(&capture_run); |
| 8509 | |
| 8510 | let mut cli_runs = Vec::new(); |
| 8511 | let mut mismatch_indices = Vec::new(); |
| 8512 | for index in 0..repeat_count { |
| 8513 | let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 8514 | let build_command = match compile_with_driver( |
| 8515 | source, |
| 8516 | opt_level, |
| 8517 | DriverEmitMode::Binary, |
| 8518 | &binary_path, |
| 8519 | tools, |
| 8520 | ) { |
| 8521 | Ok(command) => command, |
| 8522 | Err(detail) => { |
| 8523 | return Some(ConsistencyIssue { |
| 8524 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8525 | summary: "armfortas binary build failed during capture-vs-cli runtime check" |
| 8526 | .into(), |
| 8527 | repeat_count: None, |
| 8528 | unique_variant_count: None, |
| 8529 | varying_components: Vec::new(), |
| 8530 | stable_components: Vec::new(), |
| 8531 | detail, |
| 8532 | temp_root, |
| 8533 | }) |
| 8534 | } |
| 8535 | }; |
| 8536 | let run_command = render_binary_run_command(&binary_path); |
| 8537 | let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 8538 | Ok(run) => run, |
| 8539 | Err(detail) => { |
| 8540 | return Some(ConsistencyIssue { |
| 8541 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8542 | summary: "armfortas binary could not run during capture-vs-cli runtime check" |
| 8543 | .into(), |
| 8544 | repeat_count: None, |
| 8545 | unique_variant_count: None, |
| 8546 | varying_components: Vec::new(), |
| 8547 | stable_components: Vec::new(), |
| 8548 | detail, |
| 8549 | temp_root, |
| 8550 | }) |
| 8551 | } |
| 8552 | }; |
| 8553 | let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 8554 | if let Err(err) = write_behavior_run_artifacts( |
| 8555 | &temp_root, |
| 8556 | &format!("cli_run_{:02}", index), |
| 8557 | &command, |
| 8558 | &run, |
| 8559 | ) { |
| 8560 | return Some(ConsistencyIssue { |
| 8561 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8562 | summary: "could not write cli runtime artifact".into(), |
| 8563 | repeat_count: None, |
| 8564 | unique_variant_count: None, |
| 8565 | varying_components: Vec::new(), |
| 8566 | stable_components: Vec::new(), |
| 8567 | detail: format!("cannot write cli runtime artifact: {}", err), |
| 8568 | temp_root, |
| 8569 | }); |
| 8570 | } |
| 8571 | if normalize_run_signature(&run) != capture_signature { |
| 8572 | mismatch_indices.push(index); |
| 8573 | } |
| 8574 | cli_runs.push(BehaviorRun { |
| 8575 | label: format!("cli run {}", index + 1), |
| 8576 | command, |
| 8577 | signature: normalize_run_signature(&run), |
| 8578 | run, |
| 8579 | }); |
| 8580 | } |
| 8581 | |
| 8582 | if !mismatch_indices.is_empty() { |
| 8583 | let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8584 | let unique_cli_variants = |
| 8585 | count_unique_run_signatures(cli_runs.iter().map(|run| &run.signature)); |
| 8586 | let mismatch_signatures = mismatch_indices |
| 8587 | .iter() |
| 8588 | .map(|index| &cli_runs[*index].signature) |
| 8589 | .collect::<Vec<_>>(); |
| 8590 | let mut summary_signatures = vec![&capture_signature]; |
| 8591 | summary_signatures.extend(mismatch_signatures.iter().copied()); |
| 8592 | let varying = varying_run_components(&summary_signatures) |
| 8593 | .into_iter() |
| 8594 | .map(str::to_string) |
| 8595 | .collect::<Vec<_>>(); |
| 8596 | let stable = stable_run_components(&summary_signatures) |
| 8597 | .into_iter() |
| 8598 | .map(str::to_string) |
| 8599 | .collect::<Vec<_>>(); |
| 8600 | let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8601 | return Some(ConsistencyIssue { |
| 8602 | check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8603 | summary: format!( |
| 8604 | "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8605 | repeat_count, |
| 8606 | matching_runs, |
| 8607 | mismatch_indices.len(), |
| 8608 | unique_cli_variants, |
| 8609 | join_or_none_from_strings(&varying), |
| 8610 | join_or_none_from_strings(&stable) |
| 8611 | ), |
| 8612 | repeat_count: Some(repeat_count), |
| 8613 | unique_variant_count: Some(unique_cli_variants), |
| 8614 | varying_components: varying, |
| 8615 | stable_components: stable, |
| 8616 | detail: format!( |
| 8617 | "captured runtime behavior does not match repeated full CLI builds\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8618 | repeat_count, |
| 8619 | matching_runs, |
| 8620 | mismatch_indices.len(), |
| 8621 | unique_cli_variants, |
| 8622 | capture_command, |
| 8623 | first_mismatch.command, |
| 8624 | describe_run_difference( |
| 8625 | &capture_run, |
| 8626 | &first_mismatch.run, |
| 8627 | "capture run", |
| 8628 | &first_mismatch.label |
| 8629 | ) |
| 8630 | ), |
| 8631 | temp_root, |
| 8632 | }); |
| 8633 | } |
| 8634 | |
| 8635 | let _ = fs::remove_dir_all(&temp_root); |
| 8636 | None |
| 8637 | } |
| 8638 | |
| 8639 | fn run_capture_asm_reproducible( |
| 8640 | source: &Path, |
| 8641 | opt_level: OptLevel, |
| 8642 | repeat_count: usize, |
| 8643 | capture_result: &CaptureResult, |
| 8644 | tools: &ToolchainConfig, |
| 8645 | ) -> Option<ConsistencyIssue> { |
| 8646 | let temp_root = next_consistency_temp_root(opt_level); |
| 8647 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8648 | return Some(ConsistencyIssue { |
| 8649 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8650 | summary: "could not create consistency temp dir".into(), |
| 8651 | repeat_count: None, |
| 8652 | unique_variant_count: None, |
| 8653 | varying_components: Vec::new(), |
| 8654 | stable_components: Vec::new(), |
| 8655 | detail: format!( |
| 8656 | "cannot create consistency temp dir '{}': {}", |
| 8657 | temp_root.display(), |
| 8658 | err |
| 8659 | ), |
| 8660 | temp_root, |
| 8661 | }); |
| 8662 | } |
| 8663 | |
| 8664 | let mut runs = Vec::new(); |
| 8665 | let command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 8666 | let initial_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 8667 | Ok(text) => text, |
| 8668 | Err(detail) => { |
| 8669 | return Some(ConsistencyIssue { |
| 8670 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8671 | summary: "initial capture result did not include assembly text".into(), |
| 8672 | repeat_count: None, |
| 8673 | unique_variant_count: None, |
| 8674 | varying_components: Vec::new(), |
| 8675 | stable_components: Vec::new(), |
| 8676 | detail, |
| 8677 | temp_root, |
| 8678 | }) |
| 8679 | } |
| 8680 | }; |
| 8681 | if let Err(err) = fs::write(temp_root.join("capture_run_00.s"), initial_text) { |
| 8682 | return Some(ConsistencyIssue { |
| 8683 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8684 | summary: "could not write captured assembly artifact".into(), |
| 8685 | repeat_count: None, |
| 8686 | unique_variant_count: None, |
| 8687 | varying_components: Vec::new(), |
| 8688 | stable_components: Vec::new(), |
| 8689 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 8690 | temp_root, |
| 8691 | }); |
| 8692 | } |
| 8693 | runs.push(TextRun { |
| 8694 | label: "capture run 1".into(), |
| 8695 | command: command.clone(), |
| 8696 | normalized: normalize_text_artifact(initial_text), |
| 8697 | }); |
| 8698 | |
| 8699 | for index in 1..repeat_count { |
| 8700 | let text = match capture_text_from_testing(source, opt_level, Stage::Asm, tools) { |
| 8701 | Ok(text) => text, |
| 8702 | Err(detail) => { |
| 8703 | return Some(ConsistencyIssue { |
| 8704 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8705 | summary: "armfortas::testing capture failed during asm reproducibility check" |
| 8706 | .into(), |
| 8707 | repeat_count: None, |
| 8708 | unique_variant_count: None, |
| 8709 | varying_components: Vec::new(), |
| 8710 | stable_components: Vec::new(), |
| 8711 | detail, |
| 8712 | temp_root, |
| 8713 | }) |
| 8714 | } |
| 8715 | }; |
| 8716 | if let Err(err) = fs::write(temp_root.join(format!("capture_run_{:02}.s", index)), &text) { |
| 8717 | return Some(ConsistencyIssue { |
| 8718 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8719 | summary: "could not write captured assembly artifact".into(), |
| 8720 | repeat_count: None, |
| 8721 | unique_variant_count: None, |
| 8722 | varying_components: Vec::new(), |
| 8723 | stable_components: Vec::new(), |
| 8724 | detail: format!("cannot write captured assembly artifact: {}", err), |
| 8725 | temp_root, |
| 8726 | }); |
| 8727 | } |
| 8728 | runs.push(TextRun { |
| 8729 | label: format!("capture run {}", index + 1), |
| 8730 | command: command.clone(), |
| 8731 | normalized: normalize_text_artifact(&text), |
| 8732 | }); |
| 8733 | } |
| 8734 | |
| 8735 | let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 8736 | if unique_variants > 1 { |
| 8737 | let (left, right) = |
| 8738 | first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8739 | return Some(ConsistencyIssue { |
| 8740 | check: ConsistencyCheck::CaptureAsmReproducible, |
| 8741 | summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 8742 | repeat_count: Some(repeat_count), |
| 8743 | unique_variant_count: Some(unique_variants), |
| 8744 | varying_components: Vec::new(), |
| 8745 | stable_components: Vec::new(), |
| 8746 | detail: format!( |
| 8747 | "captured assembly is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 8748 | repeat_count, |
| 8749 | unique_variants, |
| 8750 | left.command, |
| 8751 | right.command, |
| 8752 | describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 8753 | ), |
| 8754 | temp_root, |
| 8755 | }); |
| 8756 | } |
| 8757 | |
| 8758 | let _ = fs::remove_dir_all(&temp_root); |
| 8759 | None |
| 8760 | } |
| 8761 | |
| 8762 | fn run_capture_obj_reproducible( |
| 8763 | source: &Path, |
| 8764 | opt_level: OptLevel, |
| 8765 | repeat_count: usize, |
| 8766 | capture_result: &CaptureResult, |
| 8767 | tools: &ToolchainConfig, |
| 8768 | ) -> Option<ConsistencyIssue> { |
| 8769 | let temp_root = next_consistency_temp_root(opt_level); |
| 8770 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8771 | return Some(ConsistencyIssue { |
| 8772 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8773 | summary: "could not create consistency temp dir".into(), |
| 8774 | repeat_count: None, |
| 8775 | unique_variant_count: None, |
| 8776 | varying_components: Vec::new(), |
| 8777 | stable_components: Vec::new(), |
| 8778 | detail: format!( |
| 8779 | "cannot create consistency temp dir '{}': {}", |
| 8780 | temp_root.display(), |
| 8781 | err |
| 8782 | ), |
| 8783 | temp_root, |
| 8784 | }); |
| 8785 | } |
| 8786 | |
| 8787 | let command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8788 | let initial_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8789 | Ok(text) => text, |
| 8790 | Err(detail) => { |
| 8791 | return Some(ConsistencyIssue { |
| 8792 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8793 | summary: "initial capture result did not include object snapshot text".into(), |
| 8794 | repeat_count: None, |
| 8795 | unique_variant_count: None, |
| 8796 | varying_components: Vec::new(), |
| 8797 | stable_components: Vec::new(), |
| 8798 | detail, |
| 8799 | temp_root, |
| 8800 | }) |
| 8801 | } |
| 8802 | }; |
| 8803 | let initial_snapshot = match parse_object_snapshot_text(initial_text) { |
| 8804 | Ok(snapshot) => snapshot, |
| 8805 | Err(detail) => { |
| 8806 | return Some(ConsistencyIssue { |
| 8807 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8808 | summary: "captured object snapshot had an unexpected format".into(), |
| 8809 | repeat_count: None, |
| 8810 | unique_variant_count: None, |
| 8811 | varying_components: Vec::new(), |
| 8812 | stable_components: Vec::new(), |
| 8813 | detail, |
| 8814 | temp_root, |
| 8815 | }) |
| 8816 | } |
| 8817 | }; |
| 8818 | if let Err(err) = fs::write(temp_root.join("capture_run_00.obj.txt"), initial_text) { |
| 8819 | return Some(ConsistencyIssue { |
| 8820 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8821 | summary: "could not write captured object snapshot artifact".into(), |
| 8822 | repeat_count: None, |
| 8823 | unique_variant_count: None, |
| 8824 | varying_components: Vec::new(), |
| 8825 | stable_components: Vec::new(), |
| 8826 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8827 | temp_root, |
| 8828 | }); |
| 8829 | } |
| 8830 | |
| 8831 | let mut runs = vec![ObjectRun { |
| 8832 | label: "capture run 1".into(), |
| 8833 | command: command.clone(), |
| 8834 | snapshot: initial_snapshot, |
| 8835 | }]; |
| 8836 | |
| 8837 | for index in 1..repeat_count { |
| 8838 | let text = match capture_text_from_testing(source, opt_level, Stage::Obj, tools) { |
| 8839 | Ok(text) => text, |
| 8840 | Err(detail) => { |
| 8841 | return Some(ConsistencyIssue { |
| 8842 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8843 | summary: "armfortas::testing capture failed during obj reproducibility check" |
| 8844 | .into(), |
| 8845 | repeat_count: None, |
| 8846 | unique_variant_count: None, |
| 8847 | varying_components: Vec::new(), |
| 8848 | stable_components: Vec::new(), |
| 8849 | detail, |
| 8850 | temp_root, |
| 8851 | }) |
| 8852 | } |
| 8853 | }; |
| 8854 | if let Err(err) = fs::write( |
| 8855 | temp_root.join(format!("capture_run_{:02}.obj.txt", index)), |
| 8856 | &text, |
| 8857 | ) { |
| 8858 | return Some(ConsistencyIssue { |
| 8859 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8860 | summary: "could not write captured object snapshot artifact".into(), |
| 8861 | repeat_count: None, |
| 8862 | unique_variant_count: None, |
| 8863 | varying_components: Vec::new(), |
| 8864 | stable_components: Vec::new(), |
| 8865 | detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8866 | temp_root, |
| 8867 | }); |
| 8868 | } |
| 8869 | let snapshot = match parse_object_snapshot_text(&text) { |
| 8870 | Ok(snapshot) => snapshot, |
| 8871 | Err(detail) => { |
| 8872 | return Some(ConsistencyIssue { |
| 8873 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8874 | summary: "captured object snapshot had an unexpected format".into(), |
| 8875 | repeat_count: None, |
| 8876 | unique_variant_count: None, |
| 8877 | varying_components: Vec::new(), |
| 8878 | stable_components: Vec::new(), |
| 8879 | detail, |
| 8880 | temp_root, |
| 8881 | }) |
| 8882 | } |
| 8883 | }; |
| 8884 | runs.push(ObjectRun { |
| 8885 | label: format!("capture run {}", index + 1), |
| 8886 | command: command.clone(), |
| 8887 | snapshot, |
| 8888 | }); |
| 8889 | } |
| 8890 | |
| 8891 | let rendered = runs |
| 8892 | .iter() |
| 8893 | .map(|run| render_object_snapshot(&run.snapshot)) |
| 8894 | .collect::<Vec<_>>(); |
| 8895 | let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8896 | if unique_variants > 1 { |
| 8897 | let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 8898 | let (left, right) = |
| 8899 | first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8900 | let varying = varying_object_components(&snapshots); |
| 8901 | let stable = stable_object_components(&snapshots); |
| 8902 | return Some(ConsistencyIssue { |
| 8903 | check: ConsistencyCheck::CaptureObjReproducible, |
| 8904 | summary: format!( |
| 8905 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8906 | repeat_count, |
| 8907 | unique_variants, |
| 8908 | join_or_none(&varying), |
| 8909 | join_or_none(&stable) |
| 8910 | ), |
| 8911 | repeat_count: Some(repeat_count), |
| 8912 | unique_variant_count: Some(unique_variants), |
| 8913 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8914 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8915 | detail: format!( |
| 8916 | "captured object snapshots are not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8917 | repeat_count, |
| 8918 | unique_variants, |
| 8919 | join_or_none(&varying), |
| 8920 | join_or_none(&stable), |
| 8921 | left.command, |
| 8922 | right.command, |
| 8923 | describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 8924 | ), |
| 8925 | temp_root, |
| 8926 | }); |
| 8927 | } |
| 8928 | |
| 8929 | let _ = fs::remove_dir_all(&temp_root); |
| 8930 | None |
| 8931 | } |
| 8932 | |
| 8933 | fn run_capture_run_reproducible( |
| 8934 | source: &Path, |
| 8935 | opt_level: OptLevel, |
| 8936 | repeat_count: usize, |
| 8937 | capture_result: &CaptureResult, |
| 8938 | tools: &ToolchainConfig, |
| 8939 | ) -> Option<ConsistencyIssue> { |
| 8940 | let temp_root = next_consistency_temp_root(opt_level); |
| 8941 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8942 | return Some(ConsistencyIssue { |
| 8943 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8944 | summary: "could not create consistency temp dir".into(), |
| 8945 | repeat_count: None, |
| 8946 | unique_variant_count: None, |
| 8947 | varying_components: Vec::new(), |
| 8948 | stable_components: Vec::new(), |
| 8949 | detail: format!( |
| 8950 | "cannot create consistency temp dir '{}': {}", |
| 8951 | temp_root.display(), |
| 8952 | err |
| 8953 | ), |
| 8954 | temp_root, |
| 8955 | }); |
| 8956 | } |
| 8957 | |
| 8958 | let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8959 | let initial_run = match capture_run_stage(capture_result) { |
| 8960 | Ok(run) => run.clone(), |
| 8961 | Err(detail) => { |
| 8962 | return Some(ConsistencyIssue { |
| 8963 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8964 | summary: "initial capture result did not include runtime behavior".into(), |
| 8965 | repeat_count: None, |
| 8966 | unique_variant_count: None, |
| 8967 | varying_components: Vec::new(), |
| 8968 | stable_components: Vec::new(), |
| 8969 | detail, |
| 8970 | temp_root, |
| 8971 | }) |
| 8972 | } |
| 8973 | }; |
| 8974 | if let Err(err) = |
| 8975 | write_behavior_run_artifacts(&temp_root, "capture_run_00", &command, &initial_run) |
| 8976 | { |
| 8977 | return Some(ConsistencyIssue { |
| 8978 | check: ConsistencyCheck::CaptureRunReproducible, |
| 8979 | summary: "could not write captured runtime artifact".into(), |
| 8980 | repeat_count: None, |
| 8981 | unique_variant_count: None, |
| 8982 | varying_components: Vec::new(), |
| 8983 | stable_components: Vec::new(), |
| 8984 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 8985 | temp_root, |
| 8986 | }); |
| 8987 | } |
| 8988 | let mut runs = vec![BehaviorRun { |
| 8989 | label: "capture run 1".into(), |
| 8990 | command: command.clone(), |
| 8991 | signature: normalize_run_signature(&initial_run), |
| 8992 | run: initial_run, |
| 8993 | }]; |
| 8994 | |
| 8995 | for index in 1..repeat_count { |
| 8996 | let run = match capture_run_from_testing(source, opt_level, tools) { |
| 8997 | Ok(run) => run, |
| 8998 | Err(detail) => { |
| 8999 | return Some(ConsistencyIssue { |
| 9000 | check: ConsistencyCheck::CaptureRunReproducible, |
| 9001 | summary: |
| 9002 | "armfortas::testing capture failed during runtime reproducibility check" |
| 9003 | .into(), |
| 9004 | repeat_count: None, |
| 9005 | unique_variant_count: None, |
| 9006 | varying_components: Vec::new(), |
| 9007 | stable_components: Vec::new(), |
| 9008 | detail, |
| 9009 | temp_root, |
| 9010 | }) |
| 9011 | } |
| 9012 | }; |
| 9013 | if let Err(err) = write_behavior_run_artifacts( |
| 9014 | &temp_root, |
| 9015 | &format!("capture_run_{:02}", index), |
| 9016 | &command, |
| 9017 | &run, |
| 9018 | ) { |
| 9019 | return Some(ConsistencyIssue { |
| 9020 | check: ConsistencyCheck::CaptureRunReproducible, |
| 9021 | summary: "could not write captured runtime artifact".into(), |
| 9022 | repeat_count: None, |
| 9023 | unique_variant_count: None, |
| 9024 | varying_components: Vec::new(), |
| 9025 | stable_components: Vec::new(), |
| 9026 | detail: format!("cannot write captured runtime artifact: {}", err), |
| 9027 | temp_root, |
| 9028 | }); |
| 9029 | } |
| 9030 | runs.push(BehaviorRun { |
| 9031 | label: format!("capture run {}", index + 1), |
| 9032 | command: command.clone(), |
| 9033 | signature: normalize_run_signature(&run), |
| 9034 | run, |
| 9035 | }); |
| 9036 | } |
| 9037 | |
| 9038 | let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 9039 | if unique_variants > 1 { |
| 9040 | let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 9041 | let varying = varying_run_components(&signatures); |
| 9042 | let stable = stable_run_components(&signatures); |
| 9043 | let (left, right) = first_distinct_behavior_pair(&runs) |
| 9044 | .expect("unique variants > 1 implies a distinct pair"); |
| 9045 | return Some(ConsistencyIssue { |
| 9046 | check: ConsistencyCheck::CaptureRunReproducible, |
| 9047 | summary: format!( |
| 9048 | "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 9049 | repeat_count, |
| 9050 | unique_variants, |
| 9051 | join_or_none(&varying), |
| 9052 | join_or_none(&stable) |
| 9053 | ), |
| 9054 | repeat_count: Some(repeat_count), |
| 9055 | unique_variant_count: Some(unique_variants), |
| 9056 | varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 9057 | stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 9058 | detail: format!( |
| 9059 | "captured runtime behavior is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 9060 | repeat_count, |
| 9061 | unique_variants, |
| 9062 | join_or_none(&varying), |
| 9063 | join_or_none(&stable), |
| 9064 | left.command, |
| 9065 | right.command, |
| 9066 | describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 9067 | ), |
| 9068 | temp_root, |
| 9069 | }); |
| 9070 | } |
| 9071 | |
| 9072 | let _ = fs::remove_dir_all(&temp_root); |
| 9073 | None |
| 9074 | } |
| 9075 | |
| 9076 | fn run_reference_compilers( |
| 9077 | prepared: &PreparedInput, |
| 9078 | case: &CaseSpec, |
| 9079 | opt_level: OptLevel, |
| 9080 | tools: &ToolchainConfig, |
| 9081 | ) -> Vec<ReferenceResult> { |
| 9082 | case.reference_compilers |
| 9083 | .iter() |
| 9084 | .copied() |
| 9085 | .map(|compiler| run_reference_case(&prepared.compiler_source, opt_level, compiler, tools)) |
| 9086 | .collect() |
| 9087 | } |
| 9088 | |
| 9089 | fn run_reference_case( |
| 9090 | source: &Path, |
| 9091 | opt_level: OptLevel, |
| 9092 | compiler: ReferenceCompiler, |
| 9093 | tools: &ToolchainConfig, |
| 9094 | ) -> ReferenceResult { |
| 9095 | let temp_root = next_report_temp_root(compiler, opt_level); |
| 9096 | let binary = temp_root.join("reference.out"); |
| 9097 | let uses_cpp = source_uses_cpp(source); |
| 9098 | |
| 9099 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 9100 | if uses_cpp { |
| 9101 | args.push("-cpp".to_string()); |
| 9102 | } |
| 9103 | args.push(source.display().to_string()); |
| 9104 | args.push("-o".to_string()); |
| 9105 | args.push(binary.display().to_string()); |
| 9106 | |
| 9107 | let compiler_bin = tools.reference_binary(compiler); |
| 9108 | let command_string = render_command(compiler_bin, &args); |
| 9109 | |
| 9110 | if let Err(err) = fs::create_dir_all(&temp_root) { |
| 9111 | return ReferenceResult::infrastructure_error( |
| 9112 | compiler, |
| 9113 | command_string, |
| 9114 | format!("cannot create temp dir '{}': {}", temp_root.display(), err), |
| 9115 | ); |
| 9116 | } |
| 9117 | |
| 9118 | let compile = match Command::new(compiler_bin) |
| 9119 | .current_dir(&temp_root) |
| 9120 | .args(&args) |
| 9121 | .output() |
| 9122 | { |
| 9123 | Ok(output) => output, |
| 9124 | Err(err) => { |
| 9125 | return ReferenceResult::infrastructure_error( |
| 9126 | compiler, |
| 9127 | command_string, |
| 9128 | format!("cannot run {}: {}", compiler_bin, err), |
| 9129 | ); |
| 9130 | } |
| 9131 | }; |
| 9132 | |
| 9133 | let mut result = ReferenceResult { |
| 9134 | compiler, |
| 9135 | compile_command: command_string, |
| 9136 | compile_exit_code: compile.status.code().unwrap_or(-1), |
| 9137 | compile_stdout: String::from_utf8_lossy(&compile.stdout).into_owned(), |
| 9138 | compile_stderr: String::from_utf8_lossy(&compile.stderr).into_owned(), |
| 9139 | run: None, |
| 9140 | run_error: None, |
| 9141 | }; |
| 9142 | |
| 9143 | if compile.status.success() { |
| 9144 | match Command::new(&binary).current_dir(&temp_root).output() { |
| 9145 | Ok(output) => { |
| 9146 | result.run = Some(RunCapture { |
| 9147 | exit_code: output.status.code().unwrap_or(-1), |
| 9148 | stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 9149 | stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 9150 | }); |
| 9151 | } |
| 9152 | Err(err) => { |
| 9153 | result.run_error = Some(format!("cannot run '{}': {}", binary.display(), err)); |
| 9154 | } |
| 9155 | } |
| 9156 | } |
| 9157 | |
| 9158 | let _ = fs::remove_dir_all(&temp_root); |
| 9159 | result |
| 9160 | } |
| 9161 | |
| 9162 | fn source_uses_cpp(source: &Path) -> bool { |
| 9163 | fs::read_to_string(source) |
| 9164 | .map(|text| text.lines().any(|line| line.trim_start().starts_with('#'))) |
| 9165 | .unwrap_or(false) |
| 9166 | } |
| 9167 | |
| 9168 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 9169 | enum DriverEmitMode { |
| 9170 | Asm, |
| 9171 | Obj, |
| 9172 | Binary, |
| 9173 | } |
| 9174 | |
| 9175 | fn compile_with_driver( |
| 9176 | source: &Path, |
| 9177 | opt_level: OptLevel, |
| 9178 | mode: DriverEmitMode, |
| 9179 | output: &Path, |
| 9180 | tools: &ToolchainConfig, |
| 9181 | ) -> Result<String, String> { |
| 9182 | let command = render_armfortas_command(source, opt_level, mode, output, tools); |
| 9183 | let emit_mode = match mode { |
| 9184 | DriverEmitMode::Asm => EmitMode::Asm, |
| 9185 | DriverEmitMode::Obj => EmitMode::Obj, |
| 9186 | DriverEmitMode::Binary => EmitMode::Binary, |
| 9187 | }; |
| 9188 | tools |
| 9189 | .armfortas_adapters() |
| 9190 | .compile_output(source, opt_level, emit_mode, output) |
| 9191 | .map_err(|detail| format!("{} failed:\n{}", command, detail))?; |
| 9192 | Ok(command) |
| 9193 | } |
| 9194 | |
| 9195 | fn render_armfortas_command( |
| 9196 | source: &Path, |
| 9197 | opt_level: OptLevel, |
| 9198 | mode: DriverEmitMode, |
| 9199 | output: &Path, |
| 9200 | tools: &ToolchainConfig, |
| 9201 | ) -> String { |
| 9202 | let armfortas = tools.armfortas_adapters(); |
| 9203 | let mut args = vec![opt_level.as_flag().to_string()]; |
| 9204 | match mode { |
| 9205 | DriverEmitMode::Asm => args.push("-S".to_string()), |
| 9206 | DriverEmitMode::Obj => args.push("-c".to_string()), |
| 9207 | DriverEmitMode::Binary => {} |
| 9208 | } |
| 9209 | args.push(source.display().to_string()); |
| 9210 | args.push("-o".to_string()); |
| 9211 | args.push(output.display().to_string()); |
| 9212 | render_command(armfortas.cli_command_name(), &args) |
| 9213 | } |
| 9214 | |
| 9215 | fn render_binary_run_command(binary: &Path) -> String { |
| 9216 | render_command(&binary.display().to_string(), &[]) |
| 9217 | } |
| 9218 | |
| 9219 | fn render_capture_command( |
| 9220 | source: &Path, |
| 9221 | opt_level: OptLevel, |
| 9222 | stage: Stage, |
| 9223 | tools: &ToolchainConfig, |
| 9224 | ) -> String { |
| 9225 | let armfortas = tools.armfortas_adapters(); |
| 9226 | format!( |
| 9227 | "{} {} --stage {} {}", |
| 9228 | armfortas.capture_command_name(), |
| 9229 | opt_level.as_flag(), |
| 9230 | stage.as_str(), |
| 9231 | quote_arg(&source.display().to_string()), |
| 9232 | ) |
| 9233 | } |
| 9234 | |
| 9235 | fn capture_text_from_testing( |
| 9236 | source: &Path, |
| 9237 | opt_level: OptLevel, |
| 9238 | stage: Stage, |
| 9239 | tools: &ToolchainConfig, |
| 9240 | ) -> Result<String, String> { |
| 9241 | let command = render_capture_command(source, opt_level, stage, tools); |
| 9242 | let request = CaptureRequest { |
| 9243 | input: source.to_path_buf(), |
| 9244 | requested: BTreeSet::from([stage]), |
| 9245 | opt_level, |
| 9246 | }; |
| 9247 | let result = tools |
| 9248 | .armfortas_adapters() |
| 9249 | .capture(&request) |
| 9250 | .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9251 | capture_text_stage(&result, stage).map(str::to_string) |
| 9252 | } |
| 9253 | |
| 9254 | fn capture_run_from_testing( |
| 9255 | source: &Path, |
| 9256 | opt_level: OptLevel, |
| 9257 | tools: &ToolchainConfig, |
| 9258 | ) -> Result<RunCapture, String> { |
| 9259 | let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 9260 | let request = CaptureRequest { |
| 9261 | input: source.to_path_buf(), |
| 9262 | requested: BTreeSet::from([Stage::Run]), |
| 9263 | opt_level, |
| 9264 | }; |
| 9265 | let result = tools |
| 9266 | .armfortas_adapters() |
| 9267 | .capture(&request) |
| 9268 | .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9269 | capture_run_stage(&result).cloned() |
| 9270 | } |
| 9271 | |
| 9272 | fn capture_text_stage<'a>(result: &'a CaptureResult, stage: Stage) -> Result<&'a str, String> { |
| 9273 | match result.get(stage) { |
| 9274 | Some(CapturedStage::Text(text)) => Ok(text), |
| 9275 | Some(CapturedStage::Run(_)) => Err(format!( |
| 9276 | "capture result contained non-text data for stage '{}'", |
| 9277 | stage.as_str() |
| 9278 | )), |
| 9279 | None => Err(format!( |
| 9280 | "capture result was missing requested stage '{}'", |
| 9281 | stage.as_str() |
| 9282 | )), |
| 9283 | } |
| 9284 | } |
| 9285 | |
| 9286 | fn capture_run_stage(result: &CaptureResult) -> Result<&RunCapture, String> { |
| 9287 | match result.get(Stage::Run) { |
| 9288 | Some(CapturedStage::Run(run)) => Ok(run), |
| 9289 | Some(CapturedStage::Text(_)) => { |
| 9290 | Err("capture result contained text data for the run stage".into()) |
| 9291 | } |
| 9292 | None => Err("capture result was missing requested stage 'run'".into()), |
| 9293 | } |
| 9294 | } |
| 9295 | |
| 9296 | fn run_binary_capture( |
| 9297 | binary: &Path, |
| 9298 | current_dir: &Path, |
| 9299 | command: &str, |
| 9300 | ) -> Result<RunCapture, String> { |
| 9301 | let output = Command::new(binary) |
| 9302 | .current_dir(current_dir) |
| 9303 | .output() |
| 9304 | .map_err(|err| { |
| 9305 | format!( |
| 9306 | "{} failed:\ncannot run '{}': {}", |
| 9307 | command, |
| 9308 | binary.display(), |
| 9309 | err |
| 9310 | ) |
| 9311 | })?; |
| 9312 | Ok(RunCapture { |
| 9313 | exit_code: output.status.code().unwrap_or(-1), |
| 9314 | stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 9315 | stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 9316 | }) |
| 9317 | } |
| 9318 | |
| 9319 | fn normalize_run_signature(run: &RunCapture) -> RunSignature { |
| 9320 | RunSignature { |
| 9321 | exit_code: run.exit_code, |
| 9322 | stdout: normalize_behavior_text(&run.stdout), |
| 9323 | stderr: normalize_behavior_text(&run.stderr), |
| 9324 | } |
| 9325 | } |
| 9326 | |
| 9327 | fn normalize_behavior_text(text: &str) -> String { |
| 9328 | text.replace("\r\n", "\n") |
| 9329 | .lines() |
| 9330 | .map(normalize_behavior_line) |
| 9331 | .collect::<Vec<_>>() |
| 9332 | .join("\n") |
| 9333 | .trim() |
| 9334 | .to_string() |
| 9335 | } |
| 9336 | |
| 9337 | fn normalize_behavior_line(line: &str) -> String { |
| 9338 | line.split_whitespace() |
| 9339 | .map(normalize_behavior_token) |
| 9340 | .collect::<Vec<_>>() |
| 9341 | .join(" ") |
| 9342 | } |
| 9343 | |
| 9344 | fn normalize_behavior_token(token: &str) -> String { |
| 9345 | if let Some(number) = parse_numeric_token(token) { |
| 9346 | format!("num:{:.6e}", number) |
| 9347 | } else { |
| 9348 | token.to_string() |
| 9349 | } |
| 9350 | } |
| 9351 | |
| 9352 | fn parse_numeric_token(token: &str) -> Option<f64> { |
| 9353 | if token.is_empty() { |
| 9354 | return None; |
| 9355 | } |
| 9356 | |
| 9357 | let normalized = token |
| 9358 | .trim() |
| 9359 | .trim_end_matches(',') |
| 9360 | .trim_end_matches(';') |
| 9361 | .replace('D', "E") |
| 9362 | .replace('d', "e"); |
| 9363 | |
| 9364 | normalized.parse::<f64>().ok() |
| 9365 | } |
| 9366 | |
| 9367 | fn format_reference_summary(references: &[ReferenceResult]) -> String { |
| 9368 | references |
| 9369 | .iter() |
| 9370 | .map(format_reference_result) |
| 9371 | .collect::<Vec<_>>() |
| 9372 | .join("\n\n") |
| 9373 | } |
| 9374 | |
| 9375 | fn format_reference_result(reference: &ReferenceResult) -> String { |
| 9376 | let mut lines = Vec::new(); |
| 9377 | lines.push(reference.compiler.as_str().to_string()); |
| 9378 | lines.push(format!("command: {}", reference.compile_command)); |
| 9379 | lines.push(format!("compile exit: {}", reference.compile_exit_code)); |
| 9380 | if !reference.compile_stdout.trim().is_empty() { |
| 9381 | lines.push(format!( |
| 9382 | "compile stdout:\n{}", |
| 9383 | reference.compile_stdout.trim_end() |
| 9384 | )); |
| 9385 | } |
| 9386 | if !reference.compile_stderr.trim().is_empty() { |
| 9387 | lines.push(format!( |
| 9388 | "compile stderr:\n{}", |
| 9389 | reference.compile_stderr.trim_end() |
| 9390 | )); |
| 9391 | } |
| 9392 | match (&reference.run, &reference.run_error) { |
| 9393 | (Some(run), _) => { |
| 9394 | lines.push(format!("run\n{}", format_run_capture(run))); |
| 9395 | } |
| 9396 | (None, Some(err)) => { |
| 9397 | lines.push(format!("run error: {}", err)); |
| 9398 | } |
| 9399 | (None, None) => {} |
| 9400 | } |
| 9401 | lines.join("\n") |
| 9402 | } |
| 9403 | |
| 9404 | fn format_run_capture(run: &RunCapture) -> String { |
| 9405 | let stdout = if run.stdout.is_empty() { |
| 9406 | "<empty>".to_string() |
| 9407 | } else { |
| 9408 | run.stdout.trim_end().to_string() |
| 9409 | }; |
| 9410 | let stderr = if run.stderr.is_empty() { |
| 9411 | "<empty>".to_string() |
| 9412 | } else { |
| 9413 | run.stderr.trim_end().to_string() |
| 9414 | }; |
| 9415 | format!( |
| 9416 | "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9417 | run.exit_code, stdout, stderr |
| 9418 | ) |
| 9419 | } |
| 9420 | |
| 9421 | fn format_run_signature(signature: &RunSignature) -> String { |
| 9422 | let stdout = if signature.stdout.is_empty() { |
| 9423 | "<empty>".to_string() |
| 9424 | } else { |
| 9425 | signature.stdout.clone() |
| 9426 | }; |
| 9427 | let stderr = if signature.stderr.is_empty() { |
| 9428 | "<empty>".to_string() |
| 9429 | } else { |
| 9430 | signature.stderr.clone() |
| 9431 | }; |
| 9432 | format!( |
| 9433 | "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9434 | signature.exit_code, stdout, stderr |
| 9435 | ) |
| 9436 | } |
| 9437 | |
| 9438 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 9439 | struct ObjectSnapshot { |
| 9440 | text: String, |
| 9441 | load_commands: String, |
| 9442 | relocations: String, |
| 9443 | symbols: String, |
| 9444 | } |
| 9445 | |
| 9446 | #[derive(Debug, Clone)] |
| 9447 | struct TextRun { |
| 9448 | label: String, |
| 9449 | command: String, |
| 9450 | normalized: String, |
| 9451 | } |
| 9452 | |
| 9453 | #[derive(Debug, Clone)] |
| 9454 | struct BehaviorRun { |
| 9455 | label: String, |
| 9456 | command: String, |
| 9457 | signature: RunSignature, |
| 9458 | run: RunCapture, |
| 9459 | } |
| 9460 | |
| 9461 | #[derive(Debug, Clone)] |
| 9462 | struct ObjectRun { |
| 9463 | label: String, |
| 9464 | command: String, |
| 9465 | snapshot: ObjectSnapshot, |
| 9466 | } |
| 9467 | |
| 9468 | fn object_snapshot(path: &Path, tools: &ToolchainConfig) -> Result<ObjectSnapshot, String> { |
| 9469 | let text = normalize_tool_output(&tool_output( |
| 9470 | tools.otool_bin(), |
| 9471 | &["-t", path.to_str().unwrap()], |
| 9472 | )?); |
| 9473 | let load_commands = normalize_tool_output(&tool_output( |
| 9474 | tools.otool_bin(), |
| 9475 | &["-l", path.to_str().unwrap()], |
| 9476 | )?); |
| 9477 | let relocations = normalize_tool_output(&tool_output( |
| 9478 | tools.otool_bin(), |
| 9479 | &["-rv", path.to_str().unwrap()], |
| 9480 | )?); |
| 9481 | let symbols = normalize_tool_output(&tool_output( |
| 9482 | tools.nm_bin(), |
| 9483 | &["-m", path.to_str().unwrap()], |
| 9484 | )?); |
| 9485 | |
| 9486 | Ok(ObjectSnapshot { |
| 9487 | text, |
| 9488 | load_commands, |
| 9489 | relocations, |
| 9490 | symbols, |
| 9491 | }) |
| 9492 | } |
| 9493 | |
| 9494 | fn tool_output(tool: &str, args: &[&str]) -> Result<String, String> { |
| 9495 | let output = Command::new(tool) |
| 9496 | .args(args) |
| 9497 | .output() |
| 9498 | .map_err(|e| format!("cannot run {}: {}", tool, e))?; |
| 9499 | if output.status.success() { |
| 9500 | Ok(String::from_utf8_lossy(&output.stdout).into_owned()) |
| 9501 | } else { |
| 9502 | Err(format!( |
| 9503 | "{} failed:\n{}", |
| 9504 | tool, |
| 9505 | String::from_utf8_lossy(&output.stderr) |
| 9506 | )) |
| 9507 | } |
| 9508 | } |
| 9509 | |
| 9510 | fn normalize_tool_output(text: &str) -> String { |
| 9511 | text.lines() |
| 9512 | .filter(|line| !line.trim_end().ends_with(".o:")) |
| 9513 | .map(str::trim_end) |
| 9514 | .collect::<Vec<_>>() |
| 9515 | .join("\n") |
| 9516 | } |
| 9517 | |
| 9518 | fn read_text_artifact(path: &Path) -> Result<String, String> { |
| 9519 | fs::read_to_string(path).map_err(|e| format!("cannot read '{}': {}", path.display(), e)) |
| 9520 | } |
| 9521 | |
| 9522 | fn normalize_text_artifact(text: &str) -> String { |
| 9523 | text.replace("\r\n", "\n") |
| 9524 | .lines() |
| 9525 | .map(str::trim_end) |
| 9526 | .collect::<Vec<_>>() |
| 9527 | .join("\n") |
| 9528 | } |
| 9529 | |
| 9530 | fn count_unique_strings<'a>(values: impl IntoIterator<Item = &'a str>) -> usize { |
| 9531 | values.into_iter().collect::<BTreeSet<_>>().len() |
| 9532 | } |
| 9533 | |
| 9534 | fn first_distinct_text_pair(runs: &[TextRun]) -> Option<(&TextRun, &TextRun)> { |
| 9535 | for left_index in 0..runs.len() { |
| 9536 | for right_index in (left_index + 1)..runs.len() { |
| 9537 | if runs[left_index].normalized != runs[right_index].normalized { |
| 9538 | return Some((&runs[left_index], &runs[right_index])); |
| 9539 | } |
| 9540 | } |
| 9541 | } |
| 9542 | None |
| 9543 | } |
| 9544 | |
| 9545 | fn count_unique_run_signatures<'a>(values: impl IntoIterator<Item = &'a RunSignature>) -> usize { |
| 9546 | values.into_iter().collect::<BTreeSet<_>>().len() |
| 9547 | } |
| 9548 | |
| 9549 | fn first_distinct_behavior_pair(runs: &[BehaviorRun]) -> Option<(&BehaviorRun, &BehaviorRun)> { |
| 9550 | for left_index in 0..runs.len() { |
| 9551 | for right_index in (left_index + 1)..runs.len() { |
| 9552 | if runs[left_index].signature != runs[right_index].signature { |
| 9553 | return Some((&runs[left_index], &runs[right_index])); |
| 9554 | } |
| 9555 | } |
| 9556 | } |
| 9557 | None |
| 9558 | } |
| 9559 | |
| 9560 | fn first_distinct_object_pair(runs: &[ObjectRun]) -> Option<(&ObjectRun, &ObjectRun)> { |
| 9561 | for left_index in 0..runs.len() { |
| 9562 | for right_index in (left_index + 1)..runs.len() { |
| 9563 | if runs[left_index].snapshot != runs[right_index].snapshot { |
| 9564 | return Some((&runs[left_index], &runs[right_index])); |
| 9565 | } |
| 9566 | } |
| 9567 | } |
| 9568 | None |
| 9569 | } |
| 9570 | |
| 9571 | fn render_object_snapshot(snapshot: &ObjectSnapshot) -> String { |
| 9572 | format!( |
| 9573 | "== text ==\n{}\n\n== load_commands ==\n{}\n\n== relocations ==\n{}\n\n== symbols ==\n{}", |
| 9574 | snapshot.text, snapshot.load_commands, snapshot.relocations, snapshot.symbols |
| 9575 | ) |
| 9576 | } |
| 9577 | |
| 9578 | fn parse_object_snapshot_text(text: &str) -> Result<ObjectSnapshot, String> { |
| 9579 | let text = text |
| 9580 | .strip_prefix("== text ==\n") |
| 9581 | .ok_or_else(|| "object snapshot was missing the '== text ==' header".to_string())?; |
| 9582 | let (text, rest) = text |
| 9583 | .split_once("\n\n== load_commands ==\n") |
| 9584 | .ok_or_else(|| { |
| 9585 | "object snapshot was missing the '== load_commands ==' section".to_string() |
| 9586 | })?; |
| 9587 | let (load_commands, rest) = rest |
| 9588 | .split_once("\n\n== relocations ==\n") |
| 9589 | .ok_or_else(|| "object snapshot was missing the '== relocations ==' section".to_string())?; |
| 9590 | let (relocations, symbols) = rest |
| 9591 | .split_once("\n\n== symbols ==\n") |
| 9592 | .ok_or_else(|| "object snapshot was missing the '== symbols ==' section".to_string())?; |
| 9593 | |
| 9594 | Ok(ObjectSnapshot { |
| 9595 | text: text.to_string(), |
| 9596 | load_commands: load_commands.to_string(), |
| 9597 | relocations: relocations.to_string(), |
| 9598 | symbols: symbols.to_string(), |
| 9599 | }) |
| 9600 | } |
| 9601 | |
| 9602 | fn describe_text_difference( |
| 9603 | expected: &str, |
| 9604 | actual: &str, |
| 9605 | left_label: &str, |
| 9606 | right_label: &str, |
| 9607 | ) -> String { |
| 9608 | let expected_lines: Vec<&str> = expected.lines().collect(); |
| 9609 | let actual_lines: Vec<&str> = actual.lines().collect(); |
| 9610 | let shared = expected_lines.len().min(actual_lines.len()); |
| 9611 | |
| 9612 | for index in 0..shared { |
| 9613 | if expected_lines[index] != actual_lines[index] { |
| 9614 | return format!( |
| 9615 | "first differing line: {}\n{}: {}\n{}: {}", |
| 9616 | index + 1, |
| 9617 | left_label, |
| 9618 | expected_lines[index], |
| 9619 | right_label, |
| 9620 | actual_lines[index] |
| 9621 | ); |
| 9622 | } |
| 9623 | } |
| 9624 | |
| 9625 | let mut detail = format!( |
| 9626 | "snapshot length differs\n{} lines: {}\n{} lines: {}", |
| 9627 | left_label, |
| 9628 | expected_lines.len(), |
| 9629 | right_label, |
| 9630 | actual_lines.len() |
| 9631 | ); |
| 9632 | if let Some(extra) = expected_lines.get(shared) { |
| 9633 | detail.push_str(&format!( |
| 9634 | "\nfirst extra line: {}\n{}: {}", |
| 9635 | shared + 1, |
| 9636 | left_label, |
| 9637 | extra |
| 9638 | )); |
| 9639 | } else if let Some(extra) = actual_lines.get(shared) { |
| 9640 | detail.push_str(&format!( |
| 9641 | "\nfirst extra line: {}\n{}: {}", |
| 9642 | shared + 1, |
| 9643 | right_label, |
| 9644 | extra |
| 9645 | )); |
| 9646 | } |
| 9647 | detail |
| 9648 | } |
| 9649 | |
| 9650 | fn describe_object_difference( |
| 9651 | expected: &ObjectSnapshot, |
| 9652 | actual: &ObjectSnapshot, |
| 9653 | left_label: &str, |
| 9654 | right_label: &str, |
| 9655 | ) -> String { |
| 9656 | let mut differing = Vec::new(); |
| 9657 | if expected.text != actual.text { |
| 9658 | differing.push(("text", &expected.text, &actual.text)); |
| 9659 | } |
| 9660 | if expected.load_commands != actual.load_commands { |
| 9661 | differing.push(( |
| 9662 | "load_commands", |
| 9663 | &expected.load_commands, |
| 9664 | &actual.load_commands, |
| 9665 | )); |
| 9666 | } |
| 9667 | if expected.relocations != actual.relocations { |
| 9668 | differing.push(("relocations", &expected.relocations, &actual.relocations)); |
| 9669 | } |
| 9670 | if expected.symbols != actual.symbols { |
| 9671 | differing.push(("symbols", &expected.symbols, &actual.symbols)); |
| 9672 | } |
| 9673 | |
| 9674 | if differing.is_empty() { |
| 9675 | return "object snapshots matched".to_string(); |
| 9676 | } |
| 9677 | |
| 9678 | let component_list = differing |
| 9679 | .iter() |
| 9680 | .map(|(name, _, _)| *name) |
| 9681 | .collect::<Vec<_>>() |
| 9682 | .join(", "); |
| 9683 | let (first_name, first_expected, first_actual) = differing[0]; |
| 9684 | |
| 9685 | format!( |
| 9686 | "differing object components: {}\n{}\n{}", |
| 9687 | component_list, |
| 9688 | format!("first differing component: {}", first_name), |
| 9689 | describe_text_difference(first_expected, first_actual, left_label, right_label) |
| 9690 | ) |
| 9691 | } |
| 9692 | |
| 9693 | fn describe_run_difference( |
| 9694 | expected: &RunCapture, |
| 9695 | actual: &RunCapture, |
| 9696 | left_label: &str, |
| 9697 | right_label: &str, |
| 9698 | ) -> String { |
| 9699 | let expected = normalize_run_signature(expected); |
| 9700 | let actual = normalize_run_signature(actual); |
| 9701 | let mut differing = Vec::new(); |
| 9702 | if expected.exit_code != actual.exit_code { |
| 9703 | differing.push("exit_code"); |
| 9704 | } |
| 9705 | if expected.stdout != actual.stdout { |
| 9706 | differing.push("stdout"); |
| 9707 | } |
| 9708 | if expected.stderr != actual.stderr { |
| 9709 | differing.push("stderr"); |
| 9710 | } |
| 9711 | |
| 9712 | if differing.is_empty() { |
| 9713 | return "runtime behavior matched".to_string(); |
| 9714 | } |
| 9715 | |
| 9716 | let component_list = differing.join(", "); |
| 9717 | match differing[0] { |
| 9718 | "exit_code" => format!( |
| 9719 | "differing runtime components: {}\nfirst differing component: exit_code\n{}: {}\n{}: {}", |
| 9720 | component_list, |
| 9721 | left_label, |
| 9722 | expected.exit_code, |
| 9723 | right_label, |
| 9724 | actual.exit_code |
| 9725 | ), |
| 9726 | "stdout" => format!( |
| 9727 | "differing runtime components: {}\nfirst differing component: stdout\n{}", |
| 9728 | component_list, |
| 9729 | describe_text_difference(&expected.stdout, &actual.stdout, left_label, right_label) |
| 9730 | ), |
| 9731 | "stderr" => format!( |
| 9732 | "differing runtime components: {}\nfirst differing component: stderr\n{}", |
| 9733 | component_list, |
| 9734 | describe_text_difference(&expected.stderr, &actual.stderr, left_label, right_label) |
| 9735 | ), |
| 9736 | _ => unreachable!("only known runtime components are compared"), |
| 9737 | } |
| 9738 | } |
| 9739 | |
| 9740 | fn varying_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9741 | object_components_by_variation(snapshots, true) |
| 9742 | } |
| 9743 | |
| 9744 | fn stable_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9745 | object_components_by_variation(snapshots, false) |
| 9746 | } |
| 9747 | |
| 9748 | fn varying_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9749 | run_components_by_variation(signatures, true) |
| 9750 | } |
| 9751 | |
| 9752 | fn stable_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9753 | run_components_by_variation(signatures, false) |
| 9754 | } |
| 9755 | |
| 9756 | fn object_components_by_variation( |
| 9757 | snapshots: &[&ObjectSnapshot], |
| 9758 | want_varying: bool, |
| 9759 | ) -> Vec<&'static str> { |
| 9760 | let components = [ |
| 9761 | ( |
| 9762 | "text", |
| 9763 | snapshots |
| 9764 | .iter() |
| 9765 | .map(|snapshot| snapshot.text.as_str()) |
| 9766 | .collect::<Vec<_>>(), |
| 9767 | ), |
| 9768 | ( |
| 9769 | "load_commands", |
| 9770 | snapshots |
| 9771 | .iter() |
| 9772 | .map(|snapshot| snapshot.load_commands.as_str()) |
| 9773 | .collect::<Vec<_>>(), |
| 9774 | ), |
| 9775 | ( |
| 9776 | "relocations", |
| 9777 | snapshots |
| 9778 | .iter() |
| 9779 | .map(|snapshot| snapshot.relocations.as_str()) |
| 9780 | .collect::<Vec<_>>(), |
| 9781 | ), |
| 9782 | ( |
| 9783 | "symbols", |
| 9784 | snapshots |
| 9785 | .iter() |
| 9786 | .map(|snapshot| snapshot.symbols.as_str()) |
| 9787 | .collect::<Vec<_>>(), |
| 9788 | ), |
| 9789 | ]; |
| 9790 | |
| 9791 | components |
| 9792 | .into_iter() |
| 9793 | .filter_map(|(name, values)| { |
| 9794 | let varies = count_unique_strings(values) > 1; |
| 9795 | if varies == want_varying { |
| 9796 | Some(name) |
| 9797 | } else { |
| 9798 | None |
| 9799 | } |
| 9800 | }) |
| 9801 | .collect() |
| 9802 | } |
| 9803 | |
| 9804 | fn run_components_by_variation( |
| 9805 | signatures: &[&RunSignature], |
| 9806 | want_varying: bool, |
| 9807 | ) -> Vec<&'static str> { |
| 9808 | let components = [ |
| 9809 | ( |
| 9810 | "exit_code", |
| 9811 | signatures |
| 9812 | .iter() |
| 9813 | .map(|signature| signature.exit_code.to_string()) |
| 9814 | .collect::<Vec<_>>(), |
| 9815 | ), |
| 9816 | ( |
| 9817 | "stdout", |
| 9818 | signatures |
| 9819 | .iter() |
| 9820 | .map(|signature| signature.stdout.clone()) |
| 9821 | .collect::<Vec<_>>(), |
| 9822 | ), |
| 9823 | ( |
| 9824 | "stderr", |
| 9825 | signatures |
| 9826 | .iter() |
| 9827 | .map(|signature| signature.stderr.clone()) |
| 9828 | .collect::<Vec<_>>(), |
| 9829 | ), |
| 9830 | ]; |
| 9831 | |
| 9832 | components |
| 9833 | .into_iter() |
| 9834 | .filter_map(|(name, values)| { |
| 9835 | let varies = count_unique_strings(values.iter().map(String::as_str)) > 1; |
| 9836 | if varies == want_varying { |
| 9837 | Some(name) |
| 9838 | } else { |
| 9839 | None |
| 9840 | } |
| 9841 | }) |
| 9842 | .collect() |
| 9843 | } |
| 9844 | |
| 9845 | fn join_or_none(values: &[&str]) -> String { |
| 9846 | if values.is_empty() { |
| 9847 | "none".to_string() |
| 9848 | } else { |
| 9849 | values.join(", ") |
| 9850 | } |
| 9851 | } |
| 9852 | |
| 9853 | fn join_or_none_from_strings(values: &[String]) -> String { |
| 9854 | if values.is_empty() { |
| 9855 | "none".to_string() |
| 9856 | } else { |
| 9857 | values.join(", ") |
| 9858 | } |
| 9859 | } |
| 9860 | |
| 9861 | fn join_usize_set(values: &BTreeSet<usize>) -> String { |
| 9862 | if values.is_empty() { |
| 9863 | "n/a".to_string() |
| 9864 | } else { |
| 9865 | values |
| 9866 | .iter() |
| 9867 | .map(|value| value.to_string()) |
| 9868 | .collect::<Vec<_>>() |
| 9869 | .join(", ") |
| 9870 | } |
| 9871 | } |
| 9872 | |
| 9873 | fn join_string_set(values: &BTreeSet<String>) -> String { |
| 9874 | if values.is_empty() { |
| 9875 | "none".to_string() |
| 9876 | } else { |
| 9877 | values.iter().cloned().collect::<Vec<_>>().join(", ") |
| 9878 | } |
| 9879 | } |
| 9880 | |
| 9881 | fn render_consistency_rollup(rollup: &ConsistencyRollup) -> String { |
| 9882 | let mut parts = vec![format!("{} cells", rollup.cells)]; |
| 9883 | if !rollup.repeat_counts.is_empty() { |
| 9884 | parts.push(format!( |
| 9885 | "repeat_count={}", |
| 9886 | join_usize_set(&rollup.repeat_counts) |
| 9887 | )); |
| 9888 | } |
| 9889 | if !rollup.unique_variant_counts.is_empty() { |
| 9890 | parts.push(format!( |
| 9891 | "unique_variants={}", |
| 9892 | join_usize_set(&rollup.unique_variant_counts) |
| 9893 | )); |
| 9894 | } |
| 9895 | if !rollup.varying_components.is_empty() { |
| 9896 | parts.push(format!( |
| 9897 | "varying={}", |
| 9898 | join_string_set(&rollup.varying_components) |
| 9899 | )); |
| 9900 | } |
| 9901 | if !rollup.stable_components.is_empty() { |
| 9902 | parts.push(format!( |
| 9903 | "stable={}", |
| 9904 | join_string_set(&rollup.stable_components) |
| 9905 | )); |
| 9906 | } |
| 9907 | parts.join("; ") |
| 9908 | } |
| 9909 | |
| 9910 | fn write_behavior_run_artifacts( |
| 9911 | root: &Path, |
| 9912 | prefix: &str, |
| 9913 | command: &str, |
| 9914 | run: &RunCapture, |
| 9915 | ) -> Result<(), std::io::Error> { |
| 9916 | let signature = normalize_run_signature(run); |
| 9917 | fs::write(root.join(format!("{}.command.txt", prefix)), command)?; |
| 9918 | fs::write(root.join(format!("{}.stdout.txt", prefix)), &run.stdout)?; |
| 9919 | fs::write(root.join(format!("{}.stderr.txt", prefix)), &run.stderr)?; |
| 9920 | fs::write( |
| 9921 | root.join(format!("{}.exit_code.txt", prefix)), |
| 9922 | format!("{}\n", run.exit_code), |
| 9923 | )?; |
| 9924 | fs::write( |
| 9925 | root.join(format!("{}.normalized.txt", prefix)), |
| 9926 | format_run_signature(&signature), |
| 9927 | )?; |
| 9928 | Ok(()) |
| 9929 | } |
| 9930 | |
| 9931 | fn render_summary(summary: &Summary) -> String { |
| 9932 | let mut lines = vec![ |
| 9933 | "Summary".to_string(), |
| 9934 | format!(" passed: {}", summary.passed), |
| 9935 | format!(" failed: {}", summary.failed), |
| 9936 | format!(" xfailed: {}", summary.xfailed), |
| 9937 | format!(" xpassed: {}", summary.xpassed), |
| 9938 | format!(" future: {}", summary.future), |
| 9939 | ]; |
| 9940 | |
| 9941 | if !summary.consistency.is_empty() { |
| 9942 | lines.push(String::new()); |
| 9943 | lines.push("Consistency".to_string()); |
| 9944 | lines.push(format!(" affected_checks: {}", summary.consistency.len())); |
| 9945 | lines.push(format!( |
| 9946 | " cells_with_issues: {}", |
| 9947 | summary |
| 9948 | .consistency |
| 9949 | .values() |
| 9950 | .map(|rollup| rollup.cells) |
| 9951 | .sum::<usize>() |
| 9952 | )); |
| 9953 | for (check, rollup) in &summary.consistency { |
| 9954 | lines.push(format!( |
| 9955 | " {}: {}", |
| 9956 | check.as_str(), |
| 9957 | render_consistency_rollup(rollup) |
| 9958 | )); |
| 9959 | } |
| 9960 | } |
| 9961 | |
| 9962 | lines.join("\n") |
| 9963 | } |
| 9964 | |
| 9965 | fn write_requested_reports(config: &RunConfig, summary: &Summary) -> Result<(), String> { |
| 9966 | if let Some(path) = &config.json_report { |
| 9967 | write_report(path, &render_json_report(summary), "json report")?; |
| 9968 | println!("json report: {}", path.display()); |
| 9969 | } |
| 9970 | if let Some(path) = &config.markdown_report { |
| 9971 | write_report(path, &render_markdown_report(summary), "markdown report")?; |
| 9972 | println!("markdown report: {}", path.display()); |
| 9973 | } |
| 9974 | Ok(()) |
| 9975 | } |
| 9976 | |
| 9977 | fn write_report(path: &Path, content: &str, label: &str) -> Result<(), String> { |
| 9978 | if let Some(parent) = path.parent() { |
| 9979 | fs::create_dir_all(parent).map_err(|e| { |
| 9980 | format!( |
| 9981 | "cannot create parent directory for {} '{}': {}", |
| 9982 | label, |
| 9983 | path.display(), |
| 9984 | e |
| 9985 | ) |
| 9986 | })?; |
| 9987 | } |
| 9988 | fs::write(path, content) |
| 9989 | .map_err(|e| format!("cannot write {} '{}': {}", label, path.display(), e)) |
| 9990 | } |
| 9991 | |
| 9992 | fn render_json_report(summary: &Summary) -> String { |
| 9993 | let mut lines = vec![ |
| 9994 | "{".to_string(), |
| 9995 | format!(" \"passed\": {},", summary.passed), |
| 9996 | format!(" \"failed\": {},", summary.failed), |
| 9997 | format!(" \"xfailed\": {},", summary.xfailed), |
| 9998 | format!(" \"xpassed\": {},", summary.xpassed), |
| 9999 | format!(" \"future\": {},", summary.future), |
| 10000 | " \"outcomes\": [".to_string(), |
| 10001 | ]; |
| 10002 | |
| 10003 | for (index, outcome) in summary.outcomes.iter().enumerate() { |
| 10004 | lines.push(" {".to_string()); |
| 10005 | lines.push(format!( |
| 10006 | " \"suite\": \"{}\",", |
| 10007 | json_escape(&outcome.suite) |
| 10008 | )); |
| 10009 | lines.push(format!( |
| 10010 | " \"case\": \"{}\",", |
| 10011 | json_escape(&outcome.case) |
| 10012 | )); |
| 10013 | lines.push(format!( |
| 10014 | " \"opt\": \"{}\",", |
| 10015 | outcome.opt_level.as_str() |
| 10016 | )); |
| 10017 | lines.push(format!( |
| 10018 | " \"kind\": \"{}\",", |
| 10019 | outcome_kind_name(outcome.kind) |
| 10020 | )); |
| 10021 | match &outcome.primary_backend { |
| 10022 | Some(backend) => { |
| 10023 | lines.push(" \"primary_backend\": {".to_string()); |
| 10024 | lines.push(format!( |
| 10025 | " \"kind\": \"{}\",", |
| 10026 | json_escape(&backend.kind) |
| 10027 | )); |
| 10028 | lines.push(format!( |
| 10029 | " \"mode\": \"{}\",", |
| 10030 | json_escape(&backend.mode) |
| 10031 | )); |
| 10032 | lines.push(format!( |
| 10033 | " \"detail\": \"{}\"", |
| 10034 | json_escape(&backend.detail) |
| 10035 | )); |
| 10036 | lines.push(" },".to_string()); |
| 10037 | } |
| 10038 | None => lines.push(" \"primary_backend\": null,".to_string()), |
| 10039 | } |
| 10040 | lines.push(format!( |
| 10041 | " \"detail\": \"{}\",", |
| 10042 | json_escape(&outcome.detail) |
| 10043 | )); |
| 10044 | match &outcome.bundle { |
| 10045 | Some(bundle) => lines.push(format!( |
| 10046 | " \"bundle\": \"{}\",", |
| 10047 | json_escape(&bundle.display().to_string()) |
| 10048 | )), |
| 10049 | None => lines.push(" \"bundle\": null,".to_string()), |
| 10050 | } |
| 10051 | lines.push(format!( |
| 10052 | " \"consistency\": {}", |
| 10053 | render_json_consistency_observations(&outcome.consistency_observations) |
| 10054 | )); |
| 10055 | lines.push(if index + 1 == summary.outcomes.len() { |
| 10056 | " }".to_string() |
| 10057 | } else { |
| 10058 | " },".to_string() |
| 10059 | }); |
| 10060 | } |
| 10061 | |
| 10062 | lines.push(" ],".to_string()); |
| 10063 | lines.push(" \"consistency\": {".to_string()); |
| 10064 | for (index, (check, rollup)) in summary.consistency.iter().enumerate() { |
| 10065 | lines.push(format!(" \"{}\": {{", check.as_str())); |
| 10066 | lines.push(format!(" \"cells\": {},", rollup.cells)); |
| 10067 | lines.push(format!( |
| 10068 | " \"repeat_counts\": {},", |
| 10069 | json_usize_set(&rollup.repeat_counts) |
| 10070 | )); |
| 10071 | lines.push(format!( |
| 10072 | " \"unique_variant_counts\": {},", |
| 10073 | json_usize_set(&rollup.unique_variant_counts) |
| 10074 | )); |
| 10075 | lines.push(format!( |
| 10076 | " \"varying_components\": {},", |
| 10077 | json_string_iter(rollup.varying_components.iter().map(|value| value.as_str())) |
| 10078 | )); |
| 10079 | lines.push(format!( |
| 10080 | " \"stable_components\": {}", |
| 10081 | json_string_iter(rollup.stable_components.iter().map(|value| value.as_str())) |
| 10082 | )); |
| 10083 | lines.push(if index + 1 == summary.consistency.len() { |
| 10084 | " }".to_string() |
| 10085 | } else { |
| 10086 | " },".to_string() |
| 10087 | }); |
| 10088 | } |
| 10089 | lines.push(" }".to_string()); |
| 10090 | lines.push("}".to_string()); |
| 10091 | lines.join("\n") + "\n" |
| 10092 | } |
| 10093 | |
| 10094 | fn render_json_consistency_observations(observations: &[ConsistencyObservation]) -> String { |
| 10095 | let mut rendered = String::from("["); |
| 10096 | for (index, observation) in observations.iter().enumerate() { |
| 10097 | if index > 0 { |
| 10098 | rendered.push_str(", "); |
| 10099 | } |
| 10100 | rendered.push('{'); |
| 10101 | rendered.push_str(&format!( |
| 10102 | "\"check\":\"{}\",\"summary\":\"{}\",", |
| 10103 | observation.check.as_str(), |
| 10104 | json_escape(&observation.summary) |
| 10105 | )); |
| 10106 | match observation.repeat_count { |
| 10107 | Some(count) => rendered.push_str(&format!("\"repeat_count\":{},", count)), |
| 10108 | None => rendered.push_str("\"repeat_count\":null,"), |
| 10109 | } |
| 10110 | match observation.unique_variant_count { |
| 10111 | Some(count) => rendered.push_str(&format!("\"unique_variant_count\":{},", count)), |
| 10112 | None => rendered.push_str("\"unique_variant_count\":null,"), |
| 10113 | } |
| 10114 | rendered.push_str(&format!( |
| 10115 | "\"varying_components\":{},\"stable_components\":{}", |
| 10116 | json_string_array(&observation.varying_components), |
| 10117 | json_string_array(&observation.stable_components) |
| 10118 | )); |
| 10119 | rendered.push('}'); |
| 10120 | } |
| 10121 | rendered.push(']'); |
| 10122 | rendered |
| 10123 | } |
| 10124 | |
| 10125 | fn render_markdown_report(summary: &Summary) -> String { |
| 10126 | let mut lines = vec![ |
| 10127 | "# afs-tests report".to_string(), |
| 10128 | String::new(), |
| 10129 | "## Summary".to_string(), |
| 10130 | String::new(), |
| 10131 | "| kind | count |".to_string(), |
| 10132 | "| --- | ---: |".to_string(), |
| 10133 | format!("| passed | {} |", summary.passed), |
| 10134 | format!("| failed | {} |", summary.failed), |
| 10135 | format!("| xfailed | {} |", summary.xfailed), |
| 10136 | format!("| xpassed | {} |", summary.xpassed), |
| 10137 | format!("| future | {} |", summary.future), |
| 10138 | ]; |
| 10139 | |
| 10140 | if !summary.consistency.is_empty() { |
| 10141 | lines.push(String::new()); |
| 10142 | lines.push("## Consistency".to_string()); |
| 10143 | lines.push(String::new()); |
| 10144 | lines.push("| check | cells | repeats | unique variants | varying | stable |".to_string()); |
| 10145 | lines.push("| --- | ---: | --- | --- | --- | --- |".to_string()); |
| 10146 | for (check, rollup) in &summary.consistency { |
| 10147 | lines.push(format!( |
| 10148 | "| `{}` | {} | {} | {} | {} | {} |", |
| 10149 | check.as_str(), |
| 10150 | rollup.cells, |
| 10151 | join_usize_set(&rollup.repeat_counts), |
| 10152 | join_usize_set(&rollup.unique_variant_counts), |
| 10153 | join_string_set(&rollup.varying_components), |
| 10154 | join_string_set(&rollup.stable_components), |
| 10155 | )); |
| 10156 | } |
| 10157 | } |
| 10158 | |
| 10159 | lines.push(String::new()); |
| 10160 | lines.push("## Outcomes".to_string()); |
| 10161 | for outcome in &summary.outcomes { |
| 10162 | lines.push(String::new()); |
| 10163 | lines.push(format!( |
| 10164 | "### `{}` / `{}` / `{}` / `{}`", |
| 10165 | outcome.suite, |
| 10166 | outcome.case, |
| 10167 | outcome.opt_level.as_str(), |
| 10168 | outcome_kind_name(outcome.kind) |
| 10169 | )); |
| 10170 | if let Some(backend) = &outcome.primary_backend { |
| 10171 | lines.push(format!( |
| 10172 | "primary_backend: `{}` (`{}`)", |
| 10173 | backend.kind, backend.mode |
| 10174 | )); |
| 10175 | lines.push(format!("primary_backend_detail: {}", backend.detail)); |
| 10176 | } |
| 10177 | if let Some(bundle) = &outcome.bundle { |
| 10178 | lines.push(format!("bundle: `{}`", bundle.display())); |
| 10179 | } |
| 10180 | if !outcome.detail.trim().is_empty() { |
| 10181 | lines.push(String::new()); |
| 10182 | lines.push("```text".to_string()); |
| 10183 | lines.extend( |
| 10184 | outcome |
| 10185 | .detail |
| 10186 | .trim_end() |
| 10187 | .lines() |
| 10188 | .map(|line| line.to_string()), |
| 10189 | ); |
| 10190 | lines.push("```".to_string()); |
| 10191 | } |
| 10192 | } |
| 10193 | |
| 10194 | lines.join("\n") + "\n" |
| 10195 | } |
| 10196 | |
| 10197 | fn outcome_kind_name(kind: OutcomeKind) -> &'static str { |
| 10198 | match kind { |
| 10199 | OutcomeKind::Pass => "pass", |
| 10200 | OutcomeKind::Fail => "fail", |
| 10201 | OutcomeKind::Xfail => "xfail", |
| 10202 | OutcomeKind::Xpass => "xpass", |
| 10203 | OutcomeKind::Future => "future", |
| 10204 | } |
| 10205 | } |
| 10206 | |
| 10207 | fn json_escape(text: &str) -> String { |
| 10208 | let mut escaped = String::new(); |
| 10209 | for ch in text.chars() { |
| 10210 | match ch { |
| 10211 | '\\' => escaped.push_str("\\\\"), |
| 10212 | '"' => escaped.push_str("\\\""), |
| 10213 | '\n' => escaped.push_str("\\n"), |
| 10214 | '\r' => escaped.push_str("\\r"), |
| 10215 | '\t' => escaped.push_str("\\t"), |
| 10216 | c if c.is_control() => escaped.push_str(&format!("\\u{:04x}", c as u32)), |
| 10217 | c => escaped.push(c), |
| 10218 | } |
| 10219 | } |
| 10220 | escaped |
| 10221 | } |
| 10222 | |
| 10223 | fn json_string_array(items: &[String]) -> String { |
| 10224 | json_string_iter(items.iter().map(|item| item.as_str())) |
| 10225 | } |
| 10226 | |
| 10227 | fn json_string_iter<'a>(items: impl Iterator<Item = &'a str>) -> String { |
| 10228 | let mut rendered = String::from("["); |
| 10229 | for (index, item) in items.enumerate() { |
| 10230 | if index > 0 { |
| 10231 | rendered.push_str(", "); |
| 10232 | } |
| 10233 | rendered.push('"'); |
| 10234 | rendered.push_str(&json_escape(item)); |
| 10235 | rendered.push('"'); |
| 10236 | } |
| 10237 | rendered.push(']'); |
| 10238 | rendered |
| 10239 | } |
| 10240 | |
| 10241 | fn json_usize_set(items: &BTreeSet<usize>) -> String { |
| 10242 | let mut rendered = String::from("["); |
| 10243 | for (index, item) in items.iter().enumerate() { |
| 10244 | if index > 0 { |
| 10245 | rendered.push_str(", "); |
| 10246 | } |
| 10247 | rendered.push_str(&item.to_string()); |
| 10248 | } |
| 10249 | rendered.push(']'); |
| 10250 | rendered |
| 10251 | } |
| 10252 | |
| 10253 | fn write_failure_bundle( |
| 10254 | suite: &SuiteSpec, |
| 10255 | case: &CaseSpec, |
| 10256 | prepared: &PreparedInput, |
| 10257 | outcome: &Outcome, |
| 10258 | artifacts: &ExecutionArtifacts, |
| 10259 | ) -> Result<PathBuf, String> { |
| 10260 | let bundle_root = default_report_root() |
| 10261 | .join(sanitize_component(&suite.name)) |
| 10262 | .join(sanitize_component(&case.name)) |
| 10263 | .join(next_report_suffix(outcome.opt_level)); |
| 10264 | fs::create_dir_all(&bundle_root).map_err(|e| { |
| 10265 | format!( |
| 10266 | "cannot create report bundle '{}': {}", |
| 10267 | bundle_root.display(), |
| 10268 | e |
| 10269 | ) |
| 10270 | })?; |
| 10271 | |
| 10272 | let stage_list = artifacts |
| 10273 | .requested |
| 10274 | .iter() |
| 10275 | .map(Stage::as_str) |
| 10276 | .collect::<Vec<_>>() |
| 10277 | .join(", "); |
| 10278 | let refs = if case.reference_compilers.is_empty() { |
| 10279 | "none".to_string() |
| 10280 | } else { |
| 10281 | case.reference_compilers |
| 10282 | .iter() |
| 10283 | .map(ReferenceCompiler::as_str) |
| 10284 | .collect::<Vec<_>>() |
| 10285 | .join(", ") |
| 10286 | }; |
| 10287 | let consistency = if case.consistency_checks.is_empty() { |
| 10288 | "none".to_string() |
| 10289 | } else { |
| 10290 | case.consistency_checks |
| 10291 | .iter() |
| 10292 | .map(ConsistencyCheck::as_str) |
| 10293 | .collect::<Vec<_>>() |
| 10294 | .join(", ") |
| 10295 | }; |
| 10296 | let primary_backend_kind = outcome |
| 10297 | .primary_backend |
| 10298 | .as_ref() |
| 10299 | .map(|backend| backend.kind.as_str()) |
| 10300 | .unwrap_or("none"); |
| 10301 | let primary_backend_mode = outcome |
| 10302 | .primary_backend |
| 10303 | .as_ref() |
| 10304 | .map(|backend| backend.mode.as_str()) |
| 10305 | .unwrap_or("none"); |
| 10306 | let primary_backend_detail = outcome |
| 10307 | .primary_backend |
| 10308 | .as_ref() |
| 10309 | .map(|backend| backend.detail.as_str()) |
| 10310 | .unwrap_or("none"); |
| 10311 | let metadata = format!( |
| 10312 | "suite: {}\ncase: {}\noutcome: {:?}\nopt: {}\nsource: {}\nrequested_stages: {}\nrepeat_count: {}\nreference_compilers: {}\nconsistency_checks: {}\nprimary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\n", |
| 10313 | suite.name, |
| 10314 | case.name, |
| 10315 | outcome.kind, |
| 10316 | outcome.opt_level.as_str(), |
| 10317 | case.source_label(), |
| 10318 | stage_list, |
| 10319 | case.repeat_count, |
| 10320 | refs, |
| 10321 | consistency, |
| 10322 | primary_backend_kind, |
| 10323 | primary_backend_mode, |
| 10324 | primary_backend_detail |
| 10325 | ); |
| 10326 | fs::write(bundle_root.join("metadata.txt"), metadata) |
| 10327 | .map_err(|e| format!("cannot write bundle metadata: {}", e))?; |
| 10328 | fs::write(bundle_root.join("detail.txt"), &outcome.detail) |
| 10329 | .map_err(|e| format!("cannot write bundle detail: {}", e))?; |
| 10330 | |
| 10331 | write_case_sources_bundle(&bundle_root, case, prepared)?; |
| 10332 | |
| 10333 | let armfortas_root = bundle_root.join("armfortas"); |
| 10334 | fs::create_dir_all(&armfortas_root) |
| 10335 | .map_err(|e| format!("cannot create armfortas bundle dir: {}", e))?; |
| 10336 | write_armfortas_bundle_metadata(&armfortas_root, outcome, artifacts)?; |
| 10337 | if let Some(result) = &artifacts.armfortas { |
| 10338 | write_capture_result(&armfortas_root, result)?; |
| 10339 | } |
| 10340 | if let Some(failure) = &artifacts.armfortas_failure { |
| 10341 | write_capture_result(&armfortas_root, &failure.partial_result())?; |
| 10342 | fs::write( |
| 10343 | armfortas_root.join("error.txt"), |
| 10344 | format!("stage: {}\n{}\n", failure.stage.as_str(), failure.detail), |
| 10345 | ) |
| 10346 | .map_err(|e| format!("cannot write armfortas error bundle: {}", e))?; |
| 10347 | } |
| 10348 | write_armfortas_observation_bundle(&armfortas_root, prepared, artifacts)?; |
| 10349 | |
| 10350 | if !artifacts.references.is_empty() { |
| 10351 | let refs_root = bundle_root.join("references"); |
| 10352 | fs::create_dir_all(&refs_root) |
| 10353 | .map_err(|e| format!("cannot create references bundle dir: {}", e))?; |
| 10354 | let reference_observations = reference_observations_for_bundle( |
| 10355 | &prepared.compiler_source, |
| 10356 | outcome.opt_level, |
| 10357 | artifacts, |
| 10358 | ); |
| 10359 | write_reference_summary_bundle(&refs_root, &artifacts.references, &reference_observations)?; |
| 10360 | for (index, reference) in artifacts.references.iter().enumerate() { |
| 10361 | write_reference_bundle( |
| 10362 | &refs_root, |
| 10363 | &prepared.compiler_source, |
| 10364 | outcome.opt_level, |
| 10365 | reference, |
| 10366 | reference_observations.get(index), |
| 10367 | )?; |
| 10368 | } |
| 10369 | } |
| 10370 | |
| 10371 | if !artifacts.consistency_issues.is_empty() { |
| 10372 | write_consistency_bundle(&bundle_root, &artifacts.consistency_issues)?; |
| 10373 | } |
| 10374 | |
| 10375 | Ok(bundle_root) |
| 10376 | } |
| 10377 | |
| 10378 | fn write_armfortas_bundle_metadata( |
| 10379 | armfortas_root: &Path, |
| 10380 | outcome: &Outcome, |
| 10381 | artifacts: &ExecutionArtifacts, |
| 10382 | ) -> Result<(), String> { |
| 10383 | let primary_backend_kind = outcome |
| 10384 | .primary_backend |
| 10385 | .as_ref() |
| 10386 | .map(|backend| backend.kind.as_str()) |
| 10387 | .unwrap_or("none"); |
| 10388 | let primary_backend_mode = outcome |
| 10389 | .primary_backend |
| 10390 | .as_ref() |
| 10391 | .map(|backend| backend.mode.as_str()) |
| 10392 | .unwrap_or("none"); |
| 10393 | let primary_backend_detail = outcome |
| 10394 | .primary_backend |
| 10395 | .as_ref() |
| 10396 | .map(|backend| backend.detail.as_str()) |
| 10397 | .unwrap_or("none"); |
| 10398 | let captured_stages = if let Some(result) = &artifacts.armfortas { |
| 10399 | join_or_none(&result.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10400 | } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10401 | join_or_none(&failure.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10402 | } else { |
| 10403 | "none".to_string() |
| 10404 | }; |
| 10405 | let error_stage = artifacts |
| 10406 | .armfortas_failure |
| 10407 | .as_ref() |
| 10408 | .map(|failure| failure.stage.as_str()) |
| 10409 | .unwrap_or("none"); |
| 10410 | let metadata = format!( |
| 10411 | "primary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\ncaptured_stages: {}\nerror_stage: {}\n", |
| 10412 | primary_backend_kind, |
| 10413 | primary_backend_mode, |
| 10414 | primary_backend_detail, |
| 10415 | captured_stages, |
| 10416 | error_stage |
| 10417 | ); |
| 10418 | fs::write(armfortas_root.join("metadata.txt"), metadata) |
| 10419 | .map_err(|e| format!("cannot write armfortas bundle metadata: {}", e)) |
| 10420 | } |
| 10421 | |
| 10422 | fn write_case_sources_bundle( |
| 10423 | bundle_root: &Path, |
| 10424 | case: &CaseSpec, |
| 10425 | prepared: &PreparedInput, |
| 10426 | ) -> Result<(), String> { |
| 10427 | if case.graph_files.is_empty() { |
| 10428 | let source_text = fs::read_to_string(&case.source) |
| 10429 | .map_err(|e| format!("cannot read case source '{}': {}", case.source.display(), e))?; |
| 10430 | fs::write(bundle_root.join("source.f90"), source_text) |
| 10431 | .map_err(|e| format!("cannot write bundle source copy: {}", e))?; |
| 10432 | return Ok(()); |
| 10433 | } |
| 10434 | |
| 10435 | let generated_source = prepared.generated_source.as_ref().ok_or_else(|| { |
| 10436 | format!( |
| 10437 | "graph case '{}' was missing a generated compiler source", |
| 10438 | case.name |
| 10439 | ) |
| 10440 | })?; |
| 10441 | let generated_text = fs::read_to_string(generated_source).map_err(|e| { |
| 10442 | format!( |
| 10443 | "cannot read generated graph source '{}': {}", |
| 10444 | generated_source.display(), |
| 10445 | e |
| 10446 | ) |
| 10447 | })?; |
| 10448 | fs::write(bundle_root.join("source.f90"), generated_text) |
| 10449 | .map_err(|e| format!("cannot write generated bundle source copy: {}", e))?; |
| 10450 | |
| 10451 | let sources_root = bundle_root.join("sources"); |
| 10452 | fs::create_dir_all(&sources_root) |
| 10453 | .map_err(|e| format!("cannot create bundle sources dir: {}", e))?; |
| 10454 | for (index, file) in case.graph_files.iter().enumerate() { |
| 10455 | let text = fs::read_to_string(file) |
| 10456 | .map_err(|e| format!("cannot read graph source '{}': {}", file.display(), e))?; |
| 10457 | let file_name = file |
| 10458 | .file_name() |
| 10459 | .and_then(|name| name.to_str()) |
| 10460 | .unwrap_or("source.f90"); |
| 10461 | let target = sources_root.join(format!("{:02}_{}", index, file_name)); |
| 10462 | fs::write(target, text).map_err(|e| format!("cannot write bundle graph source: {}", e))?; |
| 10463 | } |
| 10464 | |
| 10465 | Ok(()) |
| 10466 | } |
| 10467 | |
| 10468 | fn write_capture_result(root: &Path, result: &CaptureResult) -> Result<(), String> { |
| 10469 | for (stage, captured) in &result.stages { |
| 10470 | match captured { |
| 10471 | CapturedStage::Text(text) => { |
| 10472 | fs::write(root.join(format!("{}.txt", stage.as_str())), text).map_err(|e| { |
| 10473 | format!("cannot write '{}' stage bundle: {}", stage.as_str(), e) |
| 10474 | })?; |
| 10475 | } |
| 10476 | CapturedStage::Run(run) => { |
| 10477 | fs::write(root.join("run.stdout.txt"), &run.stdout) |
| 10478 | .map_err(|e| format!("cannot write run stdout bundle: {}", e))?; |
| 10479 | fs::write(root.join("run.stderr.txt"), &run.stderr) |
| 10480 | .map_err(|e| format!("cannot write run stderr bundle: {}", e))?; |
| 10481 | fs::write( |
| 10482 | root.join("run.exit_code.txt"), |
| 10483 | format!("{}\n", run.exit_code), |
| 10484 | ) |
| 10485 | .map_err(|e| format!("cannot write run exit-code bundle: {}", e))?; |
| 10486 | } |
| 10487 | } |
| 10488 | } |
| 10489 | Ok(()) |
| 10490 | } |
| 10491 | |
| 10492 | fn write_armfortas_observation_bundle( |
| 10493 | armfortas_root: &Path, |
| 10494 | prepared: &PreparedInput, |
| 10495 | artifacts: &ExecutionArtifacts, |
| 10496 | ) -> Result<(), String> { |
| 10497 | let observed = match observed_program_for_armfortas_bundle(prepared, artifacts) { |
| 10498 | Some(observed) => observed, |
| 10499 | None => return Ok(()), |
| 10500 | }; |
| 10501 | let render_config = IntrospectionRenderConfig { |
| 10502 | summary_only: false, |
| 10503 | max_artifact_lines: None, |
| 10504 | }; |
| 10505 | fs::write( |
| 10506 | armfortas_root.join("observation.txt"), |
| 10507 | render_introspection_text(&observed, render_config), |
| 10508 | ) |
| 10509 | .map_err(|e| format!("cannot write armfortas observation text bundle: {}", e))?; |
| 10510 | fs::write( |
| 10511 | armfortas_root.join("observation.json"), |
| 10512 | render_introspection_json(&observed), |
| 10513 | ) |
| 10514 | .map_err(|e| format!("cannot write armfortas observation json bundle: {}", e))?; |
| 10515 | fs::write( |
| 10516 | armfortas_root.join("observation.md"), |
| 10517 | render_introspection_markdown(&observed, render_config), |
| 10518 | ) |
| 10519 | .map_err(|e| format!("cannot write armfortas observation markdown bundle: {}", e))?; |
| 10520 | Ok(()) |
| 10521 | } |
| 10522 | |
| 10523 | fn observed_program_for_armfortas_bundle( |
| 10524 | prepared: &PreparedInput, |
| 10525 | artifacts: &ExecutionArtifacts, |
| 10526 | ) -> Option<ObservedProgram> { |
| 10527 | if let Some(observed) = &artifacts.armfortas_observation { |
| 10528 | Some(observed.clone()) |
| 10529 | } else if let Some(result) = &artifacts.armfortas { |
| 10530 | Some(observed_program_from_armfortas_capture( |
| 10531 | &prepared.compiler_source, |
| 10532 | result.opt_level, |
| 10533 | bundle_artifacts_for_capture_result(result), |
| 10534 | result, |
| 10535 | None, |
| 10536 | )) |
| 10537 | } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10538 | let partial = failure.partial_result(); |
| 10539 | Some(observed_program_from_armfortas_capture( |
| 10540 | &prepared.compiler_source, |
| 10541 | failure.opt_level, |
| 10542 | bundle_artifacts_for_capture_failure(failure), |
| 10543 | &partial, |
| 10544 | Some(failure), |
| 10545 | )) |
| 10546 | } else { |
| 10547 | None |
| 10548 | } |
| 10549 | } |
| 10550 | |
| 10551 | fn bundle_artifacts_for_capture_result(result: &CaptureResult) -> BTreeSet<ArtifactKey> { |
| 10552 | bundle_artifacts_for_stages(&result.stages) |
| 10553 | } |
| 10554 | |
| 10555 | fn bundle_artifacts_for_capture_failure(failure: &CaptureFailure) -> BTreeSet<ArtifactKey> { |
| 10556 | let mut requested = bundle_artifacts_for_stages(&failure.stages); |
| 10557 | requested.insert(ArtifactKey::Diagnostics); |
| 10558 | requested |
| 10559 | } |
| 10560 | |
| 10561 | fn bundle_artifacts_for_stages(stages: &BTreeMap<Stage, CapturedStage>) -> BTreeSet<ArtifactKey> { |
| 10562 | let mut requested = BTreeSet::new(); |
| 10563 | for (stage, captured) in stages { |
| 10564 | match (stage, captured) { |
| 10565 | (Stage::Asm, CapturedStage::Text(_)) => { |
| 10566 | requested.insert(ArtifactKey::Asm); |
| 10567 | } |
| 10568 | (Stage::Obj, CapturedStage::Text(_)) => { |
| 10569 | requested.insert(ArtifactKey::Obj); |
| 10570 | } |
| 10571 | (Stage::Run, CapturedStage::Run(_)) => { |
| 10572 | requested.insert(ArtifactKey::Runtime); |
| 10573 | } |
| 10574 | (stage, CapturedStage::Text(_)) => { |
| 10575 | requested.insert(ArtifactKey::Extra(format!("armfortas.{}", stage.as_str()))); |
| 10576 | } |
| 10577 | _ => {} |
| 10578 | } |
| 10579 | } |
| 10580 | requested |
| 10581 | } |
| 10582 | |
| 10583 | fn reference_observations_for_bundle( |
| 10584 | program: &Path, |
| 10585 | opt_level: OptLevel, |
| 10586 | artifacts: &ExecutionArtifacts, |
| 10587 | ) -> Vec<ObservedProgram> { |
| 10588 | if artifacts.reference_observations.len() == artifacts.references.len() { |
| 10589 | artifacts.reference_observations.clone() |
| 10590 | } else { |
| 10591 | artifacts |
| 10592 | .references |
| 10593 | .iter() |
| 10594 | .map(|reference| { |
| 10595 | observed_program_from_reference_result( |
| 10596 | program, |
| 10597 | opt_level, |
| 10598 | default_differential_artifacts(), |
| 10599 | reference, |
| 10600 | ) |
| 10601 | }) |
| 10602 | .collect() |
| 10603 | } |
| 10604 | } |
| 10605 | |
| 10606 | fn write_reference_summary_bundle( |
| 10607 | refs_root: &Path, |
| 10608 | references: &[ReferenceResult], |
| 10609 | observations: &[ObservedProgram], |
| 10610 | ) -> Result<(), String> { |
| 10611 | let summary = render_reference_bundle_summary(references, observations); |
| 10612 | fs::write(refs_root.join("summary.txt"), summary) |
| 10613 | .map_err(|e| format!("cannot write reference summary bundle: {}", e)) |
| 10614 | } |
| 10615 | |
| 10616 | fn render_reference_bundle_summary( |
| 10617 | references: &[ReferenceResult], |
| 10618 | observations: &[ObservedProgram], |
| 10619 | ) -> String { |
| 10620 | let mut lines = vec![ |
| 10621 | format!("reference_count: {}", references.len()), |
| 10622 | format!( |
| 10623 | "compilers: {}", |
| 10624 | if references.is_empty() { |
| 10625 | "none".to_string() |
| 10626 | } else { |
| 10627 | references |
| 10628 | .iter() |
| 10629 | .map(|reference| reference.compiler.as_str()) |
| 10630 | .collect::<Vec<_>>() |
| 10631 | .join(", ") |
| 10632 | } |
| 10633 | ), |
| 10634 | ]; |
| 10635 | |
| 10636 | for (reference, observed) in references.iter().zip(observations.iter()) { |
| 10637 | let observation = &observed.observation; |
| 10638 | lines.push(String::new()); |
| 10639 | lines.push(format!("compiler: {}", reference.compiler.as_str())); |
| 10640 | lines.push(format!("status: {}", introspection_status(observation))); |
| 10641 | lines.push(format!( |
| 10642 | "compile_exit_code: {}", |
| 10643 | observation.compile_exit_code |
| 10644 | )); |
| 10645 | lines.push(format!("command: {}", reference.compile_command)); |
| 10646 | lines.push(format!( |
| 10647 | "generic_artifacts: {}", |
| 10648 | join_or_none_from_strings( |
| 10649 | &observation_generic_artifacts(observation) |
| 10650 | .into_iter() |
| 10651 | .map(|(name, _)| name) |
| 10652 | .collect::<Vec<_>>() |
| 10653 | ) |
| 10654 | )); |
| 10655 | lines.push(format!( |
| 10656 | "adapter_extras: {}", |
| 10657 | format_adapter_extra_summary(&observation_adapter_extras(observation)) |
| 10658 | )); |
| 10659 | } |
| 10660 | |
| 10661 | lines.join("\n") + "\n" |
| 10662 | } |
| 10663 | |
| 10664 | fn write_reference_bundle( |
| 10665 | root: &Path, |
| 10666 | program: &Path, |
| 10667 | opt_level: OptLevel, |
| 10668 | reference: &ReferenceResult, |
| 10669 | observed: Option<&ObservedProgram>, |
| 10670 | ) -> Result<(), String> { |
| 10671 | let ref_root = root.join(sanitize_component(reference.compiler.as_str())); |
| 10672 | fs::create_dir_all(&ref_root) |
| 10673 | .map_err(|e| format!("cannot create reference bundle dir: {}", e))?; |
| 10674 | fs::write(ref_root.join("command.txt"), &reference.compile_command) |
| 10675 | .map_err(|e| format!("cannot write reference command bundle: {}", e))?; |
| 10676 | fs::write( |
| 10677 | ref_root.join("compile.exit_code.txt"), |
| 10678 | format!("{}\n", reference.compile_exit_code), |
| 10679 | ) |
| 10680 | .map_err(|e| format!("cannot write reference compile exit-code bundle: {}", e))?; |
| 10681 | fs::write( |
| 10682 | ref_root.join("compile.stdout.txt"), |
| 10683 | &reference.compile_stdout, |
| 10684 | ) |
| 10685 | .map_err(|e| format!("cannot write reference compile stdout bundle: {}", e))?; |
| 10686 | fs::write( |
| 10687 | ref_root.join("compile.stderr.txt"), |
| 10688 | &reference.compile_stderr, |
| 10689 | ) |
| 10690 | .map_err(|e| format!("cannot write reference compile stderr bundle: {}", e))?; |
| 10691 | if let Some(run) = &reference.run { |
| 10692 | fs::write(ref_root.join("run.stdout.txt"), &run.stdout) |
| 10693 | .map_err(|e| format!("cannot write reference run stdout bundle: {}", e))?; |
| 10694 | fs::write(ref_root.join("run.stderr.txt"), &run.stderr) |
| 10695 | .map_err(|e| format!("cannot write reference run stderr bundle: {}", e))?; |
| 10696 | fs::write( |
| 10697 | ref_root.join("run.exit_code.txt"), |
| 10698 | format!("{}\n", run.exit_code), |
| 10699 | ) |
| 10700 | .map_err(|e| format!("cannot write reference run exit-code bundle: {}", e))?; |
| 10701 | } |
| 10702 | if let Some(err) = &reference.run_error { |
| 10703 | fs::write(ref_root.join("run.error.txt"), err) |
| 10704 | .map_err(|e| format!("cannot write reference run error bundle: {}", e))?; |
| 10705 | } |
| 10706 | write_reference_observation_bundle(&ref_root, program, opt_level, reference, observed)?; |
| 10707 | Ok(()) |
| 10708 | } |
| 10709 | |
| 10710 | fn write_reference_observation_bundle( |
| 10711 | ref_root: &Path, |
| 10712 | program: &Path, |
| 10713 | opt_level: OptLevel, |
| 10714 | reference: &ReferenceResult, |
| 10715 | observed: Option<&ObservedProgram>, |
| 10716 | ) -> Result<(), String> { |
| 10717 | let observed = observed.cloned().unwrap_or_else(|| { |
| 10718 | observed_program_from_reference_result( |
| 10719 | program, |
| 10720 | opt_level, |
| 10721 | default_differential_artifacts(), |
| 10722 | reference, |
| 10723 | ) |
| 10724 | }); |
| 10725 | let render_config = IntrospectionRenderConfig { |
| 10726 | summary_only: false, |
| 10727 | max_artifact_lines: None, |
| 10728 | }; |
| 10729 | fs::write( |
| 10730 | ref_root.join("observation.txt"), |
| 10731 | render_introspection_text(&observed, render_config), |
| 10732 | ) |
| 10733 | .map_err(|e| format!("cannot write reference observation text bundle: {}", e))?; |
| 10734 | fs::write( |
| 10735 | ref_root.join("observation.json"), |
| 10736 | render_introspection_json(&observed), |
| 10737 | ) |
| 10738 | .map_err(|e| format!("cannot write reference observation json bundle: {}", e))?; |
| 10739 | fs::write( |
| 10740 | ref_root.join("observation.md"), |
| 10741 | render_introspection_markdown(&observed, render_config), |
| 10742 | ) |
| 10743 | .map_err(|e| format!("cannot write reference observation markdown bundle: {}", e))?; |
| 10744 | Ok(()) |
| 10745 | } |
| 10746 | |
| 10747 | fn render_consistency_bundle_summary(issues: &[ConsistencyIssue]) -> String { |
| 10748 | let mut rollups = BTreeMap::new(); |
| 10749 | for issue in issues { |
| 10750 | rollups |
| 10751 | .entry(issue.check) |
| 10752 | .or_insert_with(ConsistencyRollup::default) |
| 10753 | .record(&issue.observation()); |
| 10754 | } |
| 10755 | |
| 10756 | let mut aggregate = ConsistencyRollup::default(); |
| 10757 | for issue in issues { |
| 10758 | aggregate.record(&issue.observation()); |
| 10759 | } |
| 10760 | |
| 10761 | let checks = if rollups.is_empty() { |
| 10762 | "none".to_string() |
| 10763 | } else { |
| 10764 | rollups |
| 10765 | .keys() |
| 10766 | .map(ConsistencyCheck::as_str) |
| 10767 | .collect::<Vec<_>>() |
| 10768 | .join(", ") |
| 10769 | }; |
| 10770 | |
| 10771 | let mut lines = vec![ |
| 10772 | format!("issue_count: {}", issues.len()), |
| 10773 | format!("checks: {}", checks), |
| 10774 | ]; |
| 10775 | |
| 10776 | if !aggregate.repeat_counts.is_empty() { |
| 10777 | lines.push(format!( |
| 10778 | "repeat_counts: {}", |
| 10779 | join_usize_set(&aggregate.repeat_counts) |
| 10780 | )); |
| 10781 | } |
| 10782 | if !aggregate.unique_variant_counts.is_empty() { |
| 10783 | lines.push(format!( |
| 10784 | "unique_variants: {}", |
| 10785 | join_usize_set(&aggregate.unique_variant_counts) |
| 10786 | )); |
| 10787 | } |
| 10788 | if !aggregate.varying_components.is_empty() { |
| 10789 | lines.push(format!( |
| 10790 | "varying_components: {}", |
| 10791 | join_string_set(&aggregate.varying_components) |
| 10792 | )); |
| 10793 | } |
| 10794 | if !aggregate.stable_components.is_empty() { |
| 10795 | lines.push(format!( |
| 10796 | "stable_components: {}", |
| 10797 | join_string_set(&aggregate.stable_components) |
| 10798 | )); |
| 10799 | } |
| 10800 | |
| 10801 | if !rollups.is_empty() { |
| 10802 | lines.push(String::new()); |
| 10803 | lines.push("per_check:".to_string()); |
| 10804 | for (check, rollup) in rollups { |
| 10805 | lines.push(format!( |
| 10806 | " {}: {}", |
| 10807 | check.as_str(), |
| 10808 | render_consistency_rollup(&rollup) |
| 10809 | )); |
| 10810 | } |
| 10811 | } |
| 10812 | |
| 10813 | lines.push(String::new()); |
| 10814 | for issue in issues { |
| 10815 | lines.push(format!("check: {}", issue.check.as_str())); |
| 10816 | lines.push(format!("summary: {}", issue.summary)); |
| 10817 | if let Some(repeat_count) = issue.repeat_count { |
| 10818 | lines.push(format!("repeat_count: {}", repeat_count)); |
| 10819 | } |
| 10820 | if let Some(unique_variant_count) = issue.unique_variant_count { |
| 10821 | lines.push(format!("unique_variants: {}", unique_variant_count)); |
| 10822 | } |
| 10823 | if !issue.varying_components.is_empty() { |
| 10824 | lines.push(format!( |
| 10825 | "varying_components: {}", |
| 10826 | join_or_none_from_strings(&issue.varying_components) |
| 10827 | )); |
| 10828 | } |
| 10829 | if !issue.stable_components.is_empty() { |
| 10830 | lines.push(format!( |
| 10831 | "stable_components: {}", |
| 10832 | join_or_none_from_strings(&issue.stable_components) |
| 10833 | )); |
| 10834 | } |
| 10835 | lines.push(format!( |
| 10836 | "artifacts: {}", |
| 10837 | sanitize_component(issue.check.as_str()) |
| 10838 | )); |
| 10839 | lines.push(String::new()); |
| 10840 | } |
| 10841 | |
| 10842 | lines.join("\n") |
| 10843 | } |
| 10844 | |
| 10845 | fn write_consistency_bundle(root: &Path, issues: &[ConsistencyIssue]) -> Result<(), String> { |
| 10846 | let consistency_root = root.join("consistency"); |
| 10847 | fs::create_dir_all(&consistency_root) |
| 10848 | .map_err(|e| format!("cannot create consistency bundle dir: {}", e))?; |
| 10849 | |
| 10850 | let summary = render_consistency_bundle_summary(issues); |
| 10851 | fs::write(consistency_root.join("summary.txt"), summary) |
| 10852 | .map_err(|e| format!("cannot write consistency summary bundle: {}", e))?; |
| 10853 | |
| 10854 | for issue in issues { |
| 10855 | let issue_root = consistency_root.join(sanitize_component(issue.check.as_str())); |
| 10856 | fs::create_dir_all(&issue_root) |
| 10857 | .map_err(|e| format!("cannot create consistency issue bundle dir: {}", e))?; |
| 10858 | fs::write(issue_root.join("summary.txt"), &issue.summary) |
| 10859 | .map_err(|e| format!("cannot write consistency issue summary bundle: {}", e))?; |
| 10860 | fs::write(issue_root.join("detail.txt"), &issue.detail) |
| 10861 | .map_err(|e| format!("cannot write consistency issue detail bundle: {}", e))?; |
| 10862 | let runs_root = issue_root.join("artifacts"); |
| 10863 | copy_directory_recursive(&issue.temp_root, &runs_root)?; |
| 10864 | } |
| 10865 | |
| 10866 | Ok(()) |
| 10867 | } |
| 10868 | |
| 10869 | fn copy_directory_recursive(source: &Path, destination: &Path) -> Result<(), String> { |
| 10870 | fs::create_dir_all(destination).map_err(|e| { |
| 10871 | format!( |
| 10872 | "cannot create copied artifact dir '{}': {}", |
| 10873 | destination.display(), |
| 10874 | e |
| 10875 | ) |
| 10876 | })?; |
| 10877 | |
| 10878 | for entry in fs::read_dir(source) |
| 10879 | .map_err(|e| format!("cannot read artifact dir '{}': {}", source.display(), e))? |
| 10880 | { |
| 10881 | let entry = entry |
| 10882 | .map_err(|e| format!("cannot read artifact entry '{}': {}", source.display(), e))?; |
| 10883 | let source_path = entry.path(); |
| 10884 | let destination_path = destination.join(entry.file_name()); |
| 10885 | let file_type = entry.file_type().map_err(|e| { |
| 10886 | format!( |
| 10887 | "cannot read artifact type '{}': {}", |
| 10888 | source_path.display(), |
| 10889 | e |
| 10890 | ) |
| 10891 | })?; |
| 10892 | if file_type.is_dir() { |
| 10893 | copy_directory_recursive(&source_path, &destination_path)?; |
| 10894 | } else { |
| 10895 | fs::copy(&source_path, &destination_path).map_err(|e| { |
| 10896 | format!( |
| 10897 | "cannot copy artifact '{}' to '{}': {}", |
| 10898 | source_path.display(), |
| 10899 | destination_path.display(), |
| 10900 | e |
| 10901 | ) |
| 10902 | })?; |
| 10903 | } |
| 10904 | } |
| 10905 | |
| 10906 | Ok(()) |
| 10907 | } |
| 10908 | |
| 10909 | fn render_command(binary: &str, args: &[String]) -> String { |
| 10910 | let mut rendered = vec![quote_arg(binary)]; |
| 10911 | rendered.extend(args.iter().map(|arg| quote_arg(arg))); |
| 10912 | rendered.join(" ") |
| 10913 | } |
| 10914 | |
| 10915 | fn quote_arg(arg: &str) -> String { |
| 10916 | if arg |
| 10917 | .chars() |
| 10918 | .all(|ch| ch.is_ascii_alphanumeric() || "-_./".contains(ch)) |
| 10919 | { |
| 10920 | arg.to_string() |
| 10921 | } else { |
| 10922 | format!("{:?}", arg) |
| 10923 | } |
| 10924 | } |
| 10925 | |
| 10926 | fn sanitize_component(value: &str) -> String { |
| 10927 | let mut out = String::new(); |
| 10928 | for ch in value.chars() { |
| 10929 | if ch.is_ascii_alphanumeric() { |
| 10930 | out.push(ch.to_ascii_lowercase()); |
| 10931 | } else { |
| 10932 | out.push('_'); |
| 10933 | } |
| 10934 | } |
| 10935 | while out.contains("__") { |
| 10936 | out = out.replace("__", "_"); |
| 10937 | } |
| 10938 | out.trim_matches('_').to_string() |
| 10939 | } |
| 10940 | |
| 10941 | fn next_report_temp_root(compiler: ReferenceCompiler, opt_level: OptLevel) -> PathBuf { |
| 10942 | default_report_root().join(".tmp").join(format!( |
| 10943 | "{}_{}_{}", |
| 10944 | sanitize_component(compiler.as_str()), |
| 10945 | opt_level.as_str().to_ascii_lowercase(), |
| 10946 | next_report_suffix(opt_level) |
| 10947 | )) |
| 10948 | } |
| 10949 | |
| 10950 | fn next_primary_cli_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10951 | default_report_root().join(".tmp").join(format!( |
| 10952 | "primary_cli_{}_{}", |
| 10953 | opt_level.as_str().to_ascii_lowercase(), |
| 10954 | next_report_suffix(opt_level) |
| 10955 | )) |
| 10956 | } |
| 10957 | |
| 10958 | fn next_consistency_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10959 | default_report_root().join(".tmp").join(format!( |
| 10960 | "consistency_{}_{}", |
| 10961 | opt_level.as_str().to_ascii_lowercase(), |
| 10962 | next_report_suffix(opt_level) |
| 10963 | )) |
| 10964 | } |
| 10965 | |
| 10966 | fn next_report_suffix(opt_level: OptLevel) -> String { |
| 10967 | format!( |
| 10968 | "{}-{}-{:04}", |
| 10969 | opt_level.as_str().to_ascii_lowercase(), |
| 10970 | std::process::id(), |
| 10971 | REPORT_COUNTER.fetch_add(1, Ordering::Relaxed) |
| 10972 | ) |
| 10973 | } |
| 10974 | |
| 10975 | fn print_outcome(outcome: &Outcome) { |
| 10976 | let label = format!( |
| 10977 | "{}::{}[{}]", |
| 10978 | outcome.suite, |
| 10979 | outcome.case, |
| 10980 | outcome.opt_level.as_str() |
| 10981 | ); |
| 10982 | match outcome.kind { |
| 10983 | OutcomeKind::Pass => println!("PASS {}", label), |
| 10984 | OutcomeKind::Fail => { |
| 10985 | println!("FAIL {}", label); |
| 10986 | if !outcome.detail.is_empty() { |
| 10987 | println!("{}", outcome.detail); |
| 10988 | } |
| 10989 | } |
| 10990 | OutcomeKind::Xfail => { |
| 10991 | println!("XFAIL {}", label); |
| 10992 | if !outcome.detail.is_empty() { |
| 10993 | println!("{}", outcome.detail); |
| 10994 | } |
| 10995 | } |
| 10996 | OutcomeKind::Xpass => { |
| 10997 | println!("XPASS {}", label); |
| 10998 | if !outcome.detail.is_empty() { |
| 10999 | println!("{}", outcome.detail); |
| 11000 | } |
| 11001 | } |
| 11002 | OutcomeKind::Future => { |
| 11003 | println!("FUTURE {}", label); |
| 11004 | if !outcome.detail.is_empty() { |
| 11005 | println!("{}", outcome.detail); |
| 11006 | } |
| 11007 | } |
| 11008 | } |
| 11009 | if let Some(bundle) = &outcome.bundle { |
| 11010 | println!("bundle: {}", bundle.display()); |
| 11011 | } |
| 11012 | } |
| 11013 | |
| 11014 | fn print_summary(summary: &Summary) { |
| 11015 | println!(); |
| 11016 | println!("{}", render_summary(summary)); |
| 11017 | } |
| 11018 | |
| 11019 | #[derive(Debug, Clone)] |
| 11020 | struct Check { |
| 11021 | line_num: usize, |
| 11022 | pattern: String, |
| 11023 | negative: bool, |
| 11024 | kind: &'static str, |
| 11025 | } |
| 11026 | |
| 11027 | fn extract_checks(source: &str) -> Vec<Check> { |
| 11028 | source |
| 11029 | .lines() |
| 11030 | .enumerate() |
| 11031 | .filter_map(|(i, line)| { |
| 11032 | let trimmed = line.trim(); |
| 11033 | trimmed.strip_prefix("! CHECK:").map(|rest| Check { |
| 11034 | line_num: i + 1, |
| 11035 | pattern: rest.trim().to_string(), |
| 11036 | negative: false, |
| 11037 | kind: "CHECK", |
| 11038 | }) |
| 11039 | }) |
| 11040 | .collect() |
| 11041 | } |
| 11042 | |
| 11043 | fn extract_xfail_reason(source: &str) -> Option<String> { |
| 11044 | source.lines().find_map(|line| { |
| 11045 | line.trim() |
| 11046 | .strip_prefix("! XFAIL:") |
| 11047 | .map(|rest| rest.trim().to_string()) |
| 11048 | }) |
| 11049 | } |
| 11050 | |
| 11051 | fn extract_error_expected_patterns(source: &str) -> Vec<String> { |
| 11052 | source |
| 11053 | .lines() |
| 11054 | .filter_map(|line| { |
| 11055 | line.trim() |
| 11056 | .strip_prefix("! ERROR_EXPECTED:") |
| 11057 | .map(|rest| rest.trim().to_string()) |
| 11058 | }) |
| 11059 | .collect() |
| 11060 | } |
| 11061 | |
| 11062 | fn extract_ir_checks(source: &str) -> Vec<Check> { |
| 11063 | source |
| 11064 | .lines() |
| 11065 | .enumerate() |
| 11066 | .filter_map(|(i, line)| { |
| 11067 | let trimmed = line.trim(); |
| 11068 | if let Some(rest) = trimmed.strip_prefix("! IR_CHECK:") { |
| 11069 | Some(Check { |
| 11070 | line_num: i + 1, |
| 11071 | pattern: rest.trim().to_string(), |
| 11072 | negative: false, |
| 11073 | kind: "IR_CHECK", |
| 11074 | }) |
| 11075 | } else { |
| 11076 | trimmed.strip_prefix("! IR_NOT:").map(|rest| Check { |
| 11077 | line_num: i + 1, |
| 11078 | pattern: rest.trim().to_string(), |
| 11079 | negative: true, |
| 11080 | kind: "IR_NOT", |
| 11081 | }) |
| 11082 | } |
| 11083 | }) |
| 11084 | .collect() |
| 11085 | } |
| 11086 | |
| 11087 | fn match_checks(checks: &[Check], output: &str, case_name: &str) -> Result<(), String> { |
| 11088 | let output_lines: Vec<&str> = output.lines().collect(); |
| 11089 | let mut output_idx = 0; |
| 11090 | |
| 11091 | for check in checks { |
| 11092 | if check.negative { |
| 11093 | if output.contains(&check.pattern) { |
| 11094 | return Err(format!( |
| 11095 | "{}:{}: {} failed: substring '{}' appears in output\nfull output:\n{}", |
| 11096 | case_name, check.line_num, check.kind, check.pattern, output |
| 11097 | )); |
| 11098 | } |
| 11099 | continue; |
| 11100 | } |
| 11101 | |
| 11102 | let mut found = false; |
| 11103 | while output_idx < output_lines.len() { |
| 11104 | if output_lines[output_idx].trim().contains(&check.pattern) { |
| 11105 | found = true; |
| 11106 | output_idx += 1; |
| 11107 | break; |
| 11108 | } |
| 11109 | output_idx += 1; |
| 11110 | } |
| 11111 | if !found { |
| 11112 | return Err(format!( |
| 11113 | "{}:{}: {} failed: expected '{}' not found in remaining output\nfull output:\n{}", |
| 11114 | case_name, check.line_num, check.kind, check.pattern, output |
| 11115 | )); |
| 11116 | } |
| 11117 | } |
| 11118 | |
| 11119 | Ok(()) |
| 11120 | } |
| 11121 | |
| 11122 | fn target_uses_ir_comment_checks(target: &Target) -> bool { |
| 11123 | match target { |
| 11124 | Target::Stage(Stage::Ir) => true, |
| 11125 | Target::Artifact(ArtifactKey::Extra(name)) => name == "armfortas.ir", |
| 11126 | _ => false, |
| 11127 | } |
| 11128 | } |
| 11129 | |
| 11130 | #[cfg(test)] |
| 11131 | mod tests { |
| 11132 | use super::*; |
| 11133 | |
| 11134 | struct DummyBackend { |
| 11135 | mode: &'static str, |
| 11136 | detail: &'static str, |
| 11137 | } |
| 11138 | |
| 11139 | impl CaptureBackend for DummyBackend { |
| 11140 | fn mode_name(&self) -> &'static str { |
| 11141 | self.mode |
| 11142 | } |
| 11143 | |
| 11144 | fn description(&self) -> &'static str { |
| 11145 | self.detail |
| 11146 | } |
| 11147 | |
| 11148 | fn capture(&self, request: &CaptureRequest) -> Result<CaptureResult, CaptureFailure> { |
| 11149 | Err(CaptureFailure { |
| 11150 | input: request.input.clone(), |
| 11151 | opt_level: request.opt_level, |
| 11152 | stage: FailureStage::Ir, |
| 11153 | detail: self.detail.to_string(), |
| 11154 | stages: BTreeMap::new(), |
| 11155 | }) |
| 11156 | } |
| 11157 | } |
| 11158 | |
| 11159 | #[cfg(unix)] |
| 11160 | fn bencch_repo_root() -> PathBuf { |
| 11161 | PathBuf::from(env!("CARGO_MANIFEST_DIR")) |
| 11162 | .parent() |
| 11163 | .unwrap() |
| 11164 | .to_path_buf() |
| 11165 | } |
| 11166 | |
| 11167 | #[cfg(unix)] |
| 11168 | fn fake_compiler_fixture(name: &str) -> PathBuf { |
| 11169 | bencch_repo_root() |
| 11170 | .join("fixtures") |
| 11171 | .join("fake_compilers") |
| 11172 | .join(name) |
| 11173 | } |
| 11174 | |
| 11175 | #[cfg(unix)] |
| 11176 | fn runtime_fixture(name: &str) -> PathBuf { |
| 11177 | bencch_repo_root() |
| 11178 | .join("fixtures") |
| 11179 | .join("runtime") |
| 11180 | .join(name) |
| 11181 | } |
| 11182 | |
| 11183 | fn full_introspection_render_config() -> IntrospectionRenderConfig { |
| 11184 | IntrospectionRenderConfig { |
| 11185 | summary_only: false, |
| 11186 | max_artifact_lines: None, |
| 11187 | } |
| 11188 | } |
| 11189 | |
| 11190 | #[cfg(unix)] |
| 11191 | fn invalid_fixture(name: &str) -> PathBuf { |
| 11192 | bencch_repo_root() |
| 11193 | .join("fixtures") |
| 11194 | .join("invalid") |
| 11195 | .join(name) |
| 11196 | } |
| 11197 | |
| 11198 | #[cfg(unix)] |
| 11199 | fn ensure_fixture_executable(path: &Path) { |
| 11200 | let mut perms = fs::metadata(path).unwrap().permissions(); |
| 11201 | perms.set_mode(0o755); |
| 11202 | fs::set_permissions(path, perms).unwrap(); |
| 11203 | } |
| 11204 | |
| 11205 | #[cfg(unix)] |
| 11206 | fn write_probe_script(path: &Path, banner: &str) { |
| 11207 | fs::write(path, format!("#!/bin/sh\nprintf '%s\\n' {:?}\n", banner)).unwrap(); |
| 11208 | let mut perms = fs::metadata(path).unwrap().permissions(); |
| 11209 | perms.set_mode(0o755); |
| 11210 | fs::set_permissions(path, perms).unwrap(); |
| 11211 | } |
| 11212 | |
| 11213 | #[cfg(unix)] |
| 11214 | fn command_is_available(name: &str) -> bool { |
| 11215 | Command::new("which") |
| 11216 | .arg(name) |
| 11217 | .output() |
| 11218 | .map(|output| output.status.success()) |
| 11219 | .unwrap_or(false) |
| 11220 | } |
| 11221 | |
| 11222 | #[cfg(unix)] |
| 11223 | fn armfortas_smoke_binary() -> Option<PathBuf> { |
| 11224 | if let Some(path) = std::env::var_os("BENCCH_ARMFORTAS_SMOKE_BIN") { |
| 11225 | let path = PathBuf::from(path); |
| 11226 | if path.is_file() { |
| 11227 | return Some(path); |
| 11228 | } |
| 11229 | } |
| 11230 | |
| 11231 | let candidate = bencch_repo_root() |
| 11232 | .parent()? |
| 11233 | .join("target") |
| 11234 | .join("debug") |
| 11235 | .join("armfortas"); |
| 11236 | if candidate.is_file() { |
| 11237 | Some(candidate) |
| 11238 | } else { |
| 11239 | None |
| 11240 | } |
| 11241 | } |
| 11242 | |
| 11243 | #[cfg(unix)] |
| 11244 | fn stable_runtime_compare_corpus() -> Vec<PathBuf> { |
| 11245 | [ |
| 11246 | "allocatable.f90", |
| 11247 | "do_while.f90", |
| 11248 | "exit_cycle.f90", |
| 11249 | "nested_loops.f90", |
| 11250 | "subroutine_call.f90", |
| 11251 | "string_fixed.f90", |
| 11252 | "if_else.f90", |
| 11253 | "mixed_types.f90", |
| 11254 | "select_case.f90", |
| 11255 | "function_call.f90", |
| 11256 | "real_function.f90", |
| 11257 | "where_construct.f90", |
| 11258 | ] |
| 11259 | .into_iter() |
| 11260 | .map(runtime_fixture) |
| 11261 | .collect() |
| 11262 | } |
| 11263 | |
| 11264 | fn stable_runtime_compare_opt_levels() -> Vec<OptLevel> { |
| 11265 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 11266 | } |
| 11267 | use crate::compiler::test_support::{ |
| 11268 | verify_module, BlockParam, FloatWidth, Function, Inst, InstKind, IntWidth, IrType, Module, |
| 11269 | Position, Span, Terminator, ValueId, |
| 11270 | }; |
| 11271 | #[cfg(unix)] |
| 11272 | use std::os::unix::fs::PermissionsExt; |
| 11273 | |
| 11274 | fn dummy_span() -> Span { |
| 11275 | Span { |
| 11276 | file_id: 0, |
| 11277 | start: Position { line: 1, col: 1 }, |
| 11278 | end: Position { line: 1, col: 1 }, |
| 11279 | } |
| 11280 | } |
| 11281 | |
| 11282 | #[test] |
| 11283 | fn primary_backend_selection_uses_observable_backend_for_external_cases() { |
| 11284 | let case = CaseSpec { |
| 11285 | name: "runtime_case".into(), |
| 11286 | source: PathBuf::from("demo.f90"), |
| 11287 | graph_files: Vec::new(), |
| 11288 | requested: BTreeSet::from([Stage::Run]), |
| 11289 | generic_introspect: None, |
| 11290 | generic_compare: None, |
| 11291 | opt_levels: vec![OptLevel::O0], |
| 11292 | repeat_count: 3, |
| 11293 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 11294 | consistency_checks: vec![ConsistencyCheck::CliRunReproducible], |
| 11295 | expectations: vec![Expectation::Contains { |
| 11296 | target: Target::RunStdout, |
| 11297 | needle: "42".into(), |
| 11298 | }], |
| 11299 | status_rules: Vec::new(), |
| 11300 | capability_policy: None, |
| 11301 | }; |
| 11302 | let requested = BTreeSet::from([Stage::Run]); |
| 11303 | let external_tools = ToolchainConfig { |
| 11304 | armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 11305 | gfortran: "gfortran".into(), |
| 11306 | flang_new: "flang-new".into(), |
| 11307 | lfortran: "lfortran".into(), |
| 11308 | ifort: "ifort".into(), |
| 11309 | ifx: "ifx".into(), |
| 11310 | nvfortran: "nvfortran".into(), |
| 11311 | system_as: "as".into(), |
| 11312 | otool: "otool".into(), |
| 11313 | nm: "nm".into(), |
| 11314 | }; |
| 11315 | |
| 11316 | assert_eq!( |
| 11317 | primary_backend_kind_for_case(&case, &requested, &external_tools), |
| 11318 | PrimaryCaptureBackendKind::Observable |
| 11319 | ); |
| 11320 | let selected = |
| 11321 | select_primary_capture_backend(&case, &requested, OptLevel::O0, &external_tools); |
| 11322 | assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11323 | |
| 11324 | let linked_tools = ToolchainConfig { |
| 11325 | armfortas: ArmfortasCliAdapter::Linked, |
| 11326 | ..external_tools.clone() |
| 11327 | }; |
| 11328 | assert_eq!( |
| 11329 | primary_backend_kind_for_case(&case, &requested, &linked_tools), |
| 11330 | PrimaryCaptureBackendKind::Full |
| 11331 | ); |
| 11332 | |
| 11333 | let mut capture_check_case = case.clone(); |
| 11334 | capture_check_case.consistency_checks = vec![ConsistencyCheck::CaptureRunVsCliRun]; |
| 11335 | assert_eq!( |
| 11336 | primary_backend_kind_for_case(&capture_check_case, &requested, &external_tools), |
| 11337 | PrimaryCaptureBackendKind::Full |
| 11338 | ); |
| 11339 | |
| 11340 | let mut failure_case = case.clone(); |
| 11341 | failure_case.expectations.push(Expectation::FailContains { |
| 11342 | stage: FailureStage::Run, |
| 11343 | needle: "broken".into(), |
| 11344 | }); |
| 11345 | assert_eq!( |
| 11346 | primary_backend_kind_for_case(&failure_case, &requested, &external_tools), |
| 11347 | PrimaryCaptureBackendKind::Full |
| 11348 | ); |
| 11349 | |
| 11350 | let richer_request = BTreeSet::from([Stage::Run, Stage::Asm]); |
| 11351 | assert_eq!( |
| 11352 | primary_backend_kind_for_case(&case, &richer_request, &external_tools), |
| 11353 | PrimaryCaptureBackendKind::Observable |
| 11354 | ); |
| 11355 | |
| 11356 | let asm_only_request = BTreeSet::from([Stage::Asm]); |
| 11357 | assert_eq!( |
| 11358 | primary_backend_kind_for_case(&case, &asm_only_request, &external_tools), |
| 11359 | PrimaryCaptureBackendKind::Observable |
| 11360 | ); |
| 11361 | } |
| 11362 | |
| 11363 | #[test] |
| 11364 | fn legacy_unavailable_backend_detail_is_explicit() { |
| 11365 | let case = CaseSpec { |
| 11366 | name: "frontend_case".into(), |
| 11367 | source: PathBuf::from("frontend.f90"), |
| 11368 | graph_files: Vec::new(), |
| 11369 | requested: BTreeSet::from([Stage::Tokens]), |
| 11370 | generic_introspect: None, |
| 11371 | generic_compare: None, |
| 11372 | opt_levels: vec![OptLevel::O0], |
| 11373 | repeat_count: 1, |
| 11374 | reference_compilers: Vec::new(), |
| 11375 | consistency_checks: Vec::new(), |
| 11376 | expectations: vec![Expectation::Contains { |
| 11377 | target: Target::Stage(Stage::Tokens), |
| 11378 | needle: "program".into(), |
| 11379 | }], |
| 11380 | status_rules: Vec::new(), |
| 11381 | capability_policy: None, |
| 11382 | }; |
| 11383 | let backend = SelectedPrimaryBackend { |
| 11384 | kind: PrimaryCaptureBackendKind::Full, |
| 11385 | backend: Box::new(DummyBackend { |
| 11386 | mode: "unavailable", |
| 11387 | detail: "unavailable without linked-armfortas feature", |
| 11388 | }), |
| 11389 | }; |
| 11390 | |
| 11391 | let detail = legacy_unavailable_backend_detail(&case, &backend).unwrap(); |
| 11392 | assert!(detail.contains("case requires linked armfortas capture")); |
| 11393 | assert!(detail.contains("scripts/bootstrap-linked-armfortas.sh")); |
| 11394 | } |
| 11395 | |
| 11396 | #[test] |
| 11397 | fn legacy_cli_consistency_cases_use_generic_observation_path() { |
| 11398 | let cli_only_case = CaseSpec { |
| 11399 | name: "cli-consistency".into(), |
| 11400 | source: PathBuf::from("demo.f90"), |
| 11401 | graph_files: Vec::new(), |
| 11402 | requested: BTreeSet::from([Stage::Run]), |
| 11403 | generic_introspect: None, |
| 11404 | generic_compare: None, |
| 11405 | opt_levels: vec![OptLevel::O0], |
| 11406 | repeat_count: 3, |
| 11407 | reference_compilers: Vec::new(), |
| 11408 | consistency_checks: vec![ |
| 11409 | ConsistencyCheck::CliAsmReproducible, |
| 11410 | ConsistencyCheck::CliRunReproducible, |
| 11411 | ], |
| 11412 | expectations: vec![Expectation::Contains { |
| 11413 | target: Target::RunStdout, |
| 11414 | needle: "42".into(), |
| 11415 | }], |
| 11416 | status_rules: Vec::new(), |
| 11417 | capability_policy: None, |
| 11418 | }; |
| 11419 | assert!(legacy_case_uses_generic_consistency_checks(&cli_only_case)); |
| 11420 | |
| 11421 | let mixed_case = CaseSpec { |
| 11422 | consistency_checks: vec![ |
| 11423 | ConsistencyCheck::CliRunReproducible, |
| 11424 | ConsistencyCheck::CaptureRunReproducible, |
| 11425 | ], |
| 11426 | ..cli_only_case.clone() |
| 11427 | }; |
| 11428 | assert!(!legacy_case_uses_generic_consistency_checks(&mixed_case)); |
| 11429 | } |
| 11430 | |
| 11431 | #[test] |
| 11432 | fn legacy_observable_cases_use_generic_observation_execution() { |
| 11433 | let observable_case = CaseSpec { |
| 11434 | name: "observable".into(), |
| 11435 | source: PathBuf::from("demo.f90"), |
| 11436 | graph_files: Vec::new(), |
| 11437 | requested: BTreeSet::from([Stage::Run]), |
| 11438 | generic_introspect: None, |
| 11439 | generic_compare: None, |
| 11440 | opt_levels: vec![OptLevel::O0], |
| 11441 | repeat_count: 3, |
| 11442 | reference_compilers: Vec::new(), |
| 11443 | consistency_checks: Vec::new(), |
| 11444 | expectations: vec![Expectation::Contains { |
| 11445 | target: Target::RunStdout, |
| 11446 | needle: "42".into(), |
| 11447 | }], |
| 11448 | status_rules: Vec::new(), |
| 11449 | capability_policy: None, |
| 11450 | }; |
| 11451 | assert!(legacy_case_uses_generic_observation_execution( |
| 11452 | &observable_case, |
| 11453 | &observable_case.requested |
| 11454 | )); |
| 11455 | |
| 11456 | let richer_case = CaseSpec { |
| 11457 | requested: BTreeSet::from([Stage::Run, Stage::Ir]), |
| 11458 | ..observable_case.clone() |
| 11459 | }; |
| 11460 | assert!(!legacy_case_uses_generic_observation_execution( |
| 11461 | &richer_case, |
| 11462 | &richer_case.requested |
| 11463 | )); |
| 11464 | |
| 11465 | let failure_case = CaseSpec { |
| 11466 | expectations: vec![Expectation::FailContains { |
| 11467 | stage: FailureStage::Run, |
| 11468 | needle: "boom".into(), |
| 11469 | }], |
| 11470 | ..observable_case |
| 11471 | }; |
| 11472 | assert!(!legacy_case_uses_generic_observation_execution( |
| 11473 | &failure_case, |
| 11474 | &failure_case.requested |
| 11475 | )); |
| 11476 | } |
| 11477 | |
| 11478 | #[cfg(unix)] |
| 11479 | #[test] |
| 11480 | fn external_cli_primary_execution_returns_observable_stages() { |
| 11481 | let root = std::env::temp_dir().join("afs_tests_external_cli_primary"); |
| 11482 | let _ = fs::remove_dir_all(&root); |
| 11483 | fs::create_dir_all(&root).unwrap(); |
| 11484 | |
| 11485 | let source = root.join("demo.f90"); |
| 11486 | fs::write(&source, "program demo\nprint *, 42\nend program\n").unwrap(); |
| 11487 | |
| 11488 | let compiler = root.join("fake-armfortas"); |
| 11489 | fs::write( |
| 11490 | &compiler, |
| 11491 | "#!/bin/sh\nmode=bin\nout=\"\"\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n -S)\n mode=asm\n shift\n ;;\n -c)\n mode=obj\n shift\n ;;\n -o)\n out=\"$2\"\n shift 2\n ;;\n *)\n shift\n ;;\n esac\ndone\nif [ \"$mode\" = asm ]; then\n cat > \"$out\" <<'EOF'\n.globl _main\n_main:\n ret\nEOF\nelif [ \"$mode\" = obj ]; then\n printf 'fake object\\n' > \"$out\"\nelse\n cat > \"$out\" <<'EOF'\n#!/bin/sh\nprintf '42\\n'\nEOF\n chmod +x \"$out\"\nfi\n", |
| 11492 | ) |
| 11493 | .unwrap(); |
| 11494 | let mut perms = fs::metadata(&compiler).unwrap().permissions(); |
| 11495 | perms.set_mode(0o755); |
| 11496 | fs::set_permissions(&compiler, perms).unwrap(); |
| 11497 | |
| 11498 | let case = CaseSpec { |
| 11499 | name: "runtime_case".into(), |
| 11500 | source: source.clone(), |
| 11501 | graph_files: Vec::new(), |
| 11502 | requested: BTreeSet::from([Stage::Asm, Stage::Run]), |
| 11503 | generic_introspect: None, |
| 11504 | generic_compare: None, |
| 11505 | opt_levels: vec![OptLevel::O0], |
| 11506 | repeat_count: 3, |
| 11507 | reference_compilers: Vec::new(), |
| 11508 | consistency_checks: Vec::new(), |
| 11509 | expectations: vec![ |
| 11510 | Expectation::Contains { |
| 11511 | target: Target::Stage(Stage::Asm), |
| 11512 | needle: ".globl _main".into(), |
| 11513 | }, |
| 11514 | Expectation::Contains { |
| 11515 | target: Target::RunStdout, |
| 11516 | needle: "42".into(), |
| 11517 | }, |
| 11518 | ], |
| 11519 | status_rules: Vec::new(), |
| 11520 | capability_policy: None, |
| 11521 | }; |
| 11522 | let prepared = PreparedInput { |
| 11523 | compiler_source: source.clone(), |
| 11524 | generated_source: None, |
| 11525 | temp_root: None, |
| 11526 | }; |
| 11527 | let tools = ToolchainConfig { |
| 11528 | armfortas: ArmfortasCliAdapter::External(compiler.display().to_string()), |
| 11529 | gfortran: "gfortran".into(), |
| 11530 | flang_new: "flang-new".into(), |
| 11531 | lfortran: "lfortran".into(), |
| 11532 | ifort: "ifort".into(), |
| 11533 | ifx: "ifx".into(), |
| 11534 | nvfortran: "nvfortran".into(), |
| 11535 | system_as: "as".into(), |
| 11536 | otool: "otool".into(), |
| 11537 | nm: "nm".into(), |
| 11538 | }; |
| 11539 | let requested = BTreeSet::from([Stage::Asm, Stage::Run]); |
| 11540 | let selected = select_primary_capture_backend(&case, &requested, OptLevel::O0, &tools); |
| 11541 | assert_eq!(selected.kind, PrimaryCaptureBackendKind::Observable); |
| 11542 | assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11543 | |
| 11544 | let result = |
| 11545 | execute_primary_armfortas(&prepared, OptLevel::O0, &requested, &selected).unwrap(); |
| 11546 | let asm = capture_text_stage(&result, Stage::Asm).unwrap(); |
| 11547 | let run = capture_run_stage(&result).unwrap(); |
| 11548 | assert!(asm.contains(".globl _main")); |
| 11549 | assert_eq!(run.exit_code, 0); |
| 11550 | assert_eq!(run.stdout, "42\n"); |
| 11551 | assert!(run.stderr.is_empty()); |
| 11552 | assert_eq!(result.stages.len(), 2); |
| 11553 | |
| 11554 | let _ = fs::remove_dir_all(&root); |
| 11555 | } |
| 11556 | |
| 11557 | #[cfg(unix)] |
| 11558 | #[test] |
| 11559 | fn compare_uses_generic_external_driver_observations() { |
| 11560 | let compiler_a = fake_compiler_fixture("match_42_a.sh"); |
| 11561 | let compiler_b = fake_compiler_fixture("runtime_41.sh"); |
| 11562 | let source = runtime_fixture("mixed_types.f90"); |
| 11563 | ensure_fixture_executable(&compiler_a); |
| 11564 | ensure_fixture_executable(&compiler_b); |
| 11565 | |
| 11566 | let config = CompareConfig { |
| 11567 | left: CompilerSpec::Binary(compiler_a.clone()), |
| 11568 | right: CompilerSpec::Binary(compiler_b.clone()), |
| 11569 | program: source.clone(), |
| 11570 | opt_level: OptLevel::O0, |
| 11571 | artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 11572 | json_report: None, |
| 11573 | markdown_report: None, |
| 11574 | tools: ToolchainConfig::from_env(), |
| 11575 | }; |
| 11576 | |
| 11577 | let result = run_compare(&config).unwrap(); |
| 11578 | assert_eq!(result.left.provenance.backend_mode, "external-driver"); |
| 11579 | assert_eq!(result.right.provenance.backend_mode, "external-driver"); |
| 11580 | assert!(result |
| 11581 | .differences |
| 11582 | .iter() |
| 11583 | .any(|difference| difference.artifact == "runtime")); |
| 11584 | assert!(result |
| 11585 | .differences |
| 11586 | .iter() |
| 11587 | .any(|difference| difference.artifact == "asm")); |
| 11588 | let rendered = render_compare_text(&result); |
| 11589 | assert!(rendered.contains("status: diff")); |
| 11590 | assert!(rendered.contains("classification: mixed divergence")); |
| 11591 | assert!(rendered.contains("difference_count: 2")); |
| 11592 | } |
| 11593 | |
| 11594 | #[cfg(unix)] |
| 11595 | #[test] |
| 11596 | fn compare_fixture_compilers_report_compile_failures() { |
| 11597 | let source = runtime_fixture("mixed_types.f90"); |
| 11598 | let compiler_fail = fake_compiler_fixture("compile_fail.sh"); |
| 11599 | let compiler_ok = fake_compiler_fixture("match_42_a.sh"); |
| 11600 | ensure_fixture_executable(&compiler_fail); |
| 11601 | ensure_fixture_executable(&compiler_ok); |
| 11602 | |
| 11603 | let config = CompareConfig { |
| 11604 | left: CompilerSpec::Binary(compiler_fail), |
| 11605 | right: CompilerSpec::Binary(compiler_ok), |
| 11606 | program: source, |
| 11607 | opt_level: OptLevel::O0, |
| 11608 | artifacts: BTreeSet::new(), |
| 11609 | json_report: None, |
| 11610 | markdown_report: None, |
| 11611 | tools: ToolchainConfig::from_env(), |
| 11612 | }; |
| 11613 | |
| 11614 | let result = run_compare(&config).unwrap(); |
| 11615 | assert_eq!(compare_status(&result), "diff"); |
| 11616 | assert_eq!(compare_classification(&result), "compile divergence"); |
| 11617 | assert!(result |
| 11618 | .differences |
| 11619 | .iter() |
| 11620 | .any(|difference| difference.artifact == "compile-exit-code")); |
| 11621 | let diagnostics = result |
| 11622 | .differences |
| 11623 | .iter() |
| 11624 | .find(|difference| difference.artifact == "diagnostics") |
| 11625 | .unwrap(); |
| 11626 | assert!(diagnostics |
| 11627 | .detail |
| 11628 | .contains("fake compiler failure: missing lowering pass")); |
| 11629 | } |
| 11630 | |
| 11631 | #[test] |
| 11632 | fn compare_rejects_capability_mismatch_for_namespaced_artifacts() { |
| 11633 | let config = CompareConfig { |
| 11634 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11635 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11636 | program: runtime_fixture("if_else.f90"), |
| 11637 | opt_level: OptLevel::O0, |
| 11638 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 11639 | json_report: None, |
| 11640 | markdown_report: None, |
| 11641 | tools: ToolchainConfig::from_env(), |
| 11642 | }; |
| 11643 | |
| 11644 | let err = run_compare(&config).unwrap_err(); |
| 11645 | assert!(err.contains("compare request is not supported")); |
| 11646 | assert!(err.contains("right:")); |
| 11647 | assert!(err.contains("gfortran does not support requested artifacts")); |
| 11648 | assert!(err.contains("armfortas.ir")); |
| 11649 | } |
| 11650 | |
| 11651 | #[test] |
| 11652 | fn compare_executable_artifact_uses_file_contents_not_paths() { |
| 11653 | let root = std::env::temp_dir().join("bencch_compare_executable_paths"); |
| 11654 | let _ = fs::remove_dir_all(&root); |
| 11655 | fs::create_dir_all(&root).unwrap(); |
| 11656 | |
| 11657 | let left_exe = root.join("left.out"); |
| 11658 | let right_exe = root.join("right.out"); |
| 11659 | fs::write(&left_exe, b"same executable bytes").unwrap(); |
| 11660 | fs::write(&right_exe, b"same executable bytes").unwrap(); |
| 11661 | |
| 11662 | let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11663 | let left = CompilerObservation { |
| 11664 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11665 | program: PathBuf::from("demo.f90"), |
| 11666 | opt_level: OptLevel::O0, |
| 11667 | compile_exit_code: 0, |
| 11668 | artifacts: BTreeMap::from([( |
| 11669 | ArtifactKey::Executable, |
| 11670 | ArtifactValue::Path(left_exe.clone()), |
| 11671 | )]), |
| 11672 | provenance: ObservationProvenance { |
| 11673 | compiler_identity: "left".into(), |
| 11674 | adapter_kind: "explicit-path".into(), |
| 11675 | backend_mode: "external-driver".into(), |
| 11676 | backend_detail: "left detail".into(), |
| 11677 | artifacts_captured: vec!["executable".into()], |
| 11678 | comparison_basis: None, |
| 11679 | failure_stage: None, |
| 11680 | }, |
| 11681 | }; |
| 11682 | let right = CompilerObservation { |
| 11683 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11684 | program: PathBuf::from("demo.f90"), |
| 11685 | opt_level: OptLevel::O0, |
| 11686 | compile_exit_code: 0, |
| 11687 | artifacts: BTreeMap::from([( |
| 11688 | ArtifactKey::Executable, |
| 11689 | ArtifactValue::Path(right_exe.clone()), |
| 11690 | )]), |
| 11691 | provenance: ObservationProvenance { |
| 11692 | compiler_identity: "right".into(), |
| 11693 | adapter_kind: "explicit-path".into(), |
| 11694 | backend_mode: "external-driver".into(), |
| 11695 | backend_detail: "right detail".into(), |
| 11696 | artifacts_captured: vec!["executable".into()], |
| 11697 | comparison_basis: None, |
| 11698 | failure_stage: None, |
| 11699 | }, |
| 11700 | }; |
| 11701 | |
| 11702 | let result = compare_observations(left, right, &requested); |
| 11703 | assert!(result.differences.is_empty()); |
| 11704 | |
| 11705 | let _ = fs::remove_dir_all(&root); |
| 11706 | } |
| 11707 | |
| 11708 | #[test] |
| 11709 | fn compare_executable_artifact_reports_binary_difference() { |
| 11710 | let root = std::env::temp_dir().join("bencch_compare_executable_bytes"); |
| 11711 | let _ = fs::remove_dir_all(&root); |
| 11712 | fs::create_dir_all(&root).unwrap(); |
| 11713 | |
| 11714 | let left_exe = root.join("left.out"); |
| 11715 | let right_exe = root.join("right.out"); |
| 11716 | fs::write(&left_exe, b"abc").unwrap(); |
| 11717 | fs::write(&right_exe, b"axc").unwrap(); |
| 11718 | |
| 11719 | let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11720 | let left = CompilerObservation { |
| 11721 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11722 | program: PathBuf::from("demo.f90"), |
| 11723 | opt_level: OptLevel::O0, |
| 11724 | compile_exit_code: 0, |
| 11725 | artifacts: BTreeMap::from([( |
| 11726 | ArtifactKey::Executable, |
| 11727 | ArtifactValue::Path(left_exe.clone()), |
| 11728 | )]), |
| 11729 | provenance: ObservationProvenance { |
| 11730 | compiler_identity: "left".into(), |
| 11731 | adapter_kind: "explicit-path".into(), |
| 11732 | backend_mode: "external-driver".into(), |
| 11733 | backend_detail: "left detail".into(), |
| 11734 | artifacts_captured: vec!["executable".into()], |
| 11735 | comparison_basis: None, |
| 11736 | failure_stage: None, |
| 11737 | }, |
| 11738 | }; |
| 11739 | let right = CompilerObservation { |
| 11740 | compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11741 | program: PathBuf::from("demo.f90"), |
| 11742 | opt_level: OptLevel::O0, |
| 11743 | compile_exit_code: 0, |
| 11744 | artifacts: BTreeMap::from([( |
| 11745 | ArtifactKey::Executable, |
| 11746 | ArtifactValue::Path(right_exe.clone()), |
| 11747 | )]), |
| 11748 | provenance: ObservationProvenance { |
| 11749 | compiler_identity: "right".into(), |
| 11750 | adapter_kind: "explicit-path".into(), |
| 11751 | backend_mode: "external-driver".into(), |
| 11752 | backend_detail: "right detail".into(), |
| 11753 | artifacts_captured: vec!["executable".into()], |
| 11754 | comparison_basis: None, |
| 11755 | failure_stage: None, |
| 11756 | }, |
| 11757 | }; |
| 11758 | |
| 11759 | let result = compare_observations(left, right, &requested); |
| 11760 | assert_eq!(result.differences.len(), 1); |
| 11761 | assert_eq!(result.differences[0].artifact, "executable"); |
| 11762 | assert!(result.differences[0] |
| 11763 | .detail |
| 11764 | .contains("first differing byte: 1")); |
| 11765 | |
| 11766 | let _ = fs::remove_dir_all(&root); |
| 11767 | } |
| 11768 | |
| 11769 | #[cfg(unix)] |
| 11770 | #[test] |
| 11771 | fn compare_cli_with_fixture_compilers_writes_match_reports() { |
| 11772 | let left = fake_compiler_fixture("match_42_a.sh"); |
| 11773 | let right = fake_compiler_fixture("match_42_b.sh"); |
| 11774 | let source = runtime_fixture("mixed_types.f90"); |
| 11775 | ensure_fixture_executable(&left); |
| 11776 | ensure_fixture_executable(&right); |
| 11777 | |
| 11778 | let root = std::env::temp_dir().join("bencch_compare_fixture_reports"); |
| 11779 | let _ = fs::remove_dir_all(&root); |
| 11780 | fs::create_dir_all(&root).unwrap(); |
| 11781 | let json_report = root.join("compare.json"); |
| 11782 | let markdown_report = root.join("compare.md"); |
| 11783 | |
| 11784 | let args = vec![ |
| 11785 | "compare".to_string(), |
| 11786 | left.display().to_string(), |
| 11787 | right.display().to_string(), |
| 11788 | "--program".to_string(), |
| 11789 | source.display().to_string(), |
| 11790 | "--artifact".to_string(), |
| 11791 | "asm,obj".to_string(), |
| 11792 | "--json-report".to_string(), |
| 11793 | json_report.display().to_string(), |
| 11794 | "--markdown-report".to_string(), |
| 11795 | markdown_report.display().to_string(), |
| 11796 | ]; |
| 11797 | |
| 11798 | let exit = run_cli_named("bencch", &args); |
| 11799 | assert_eq!(exit, 0); |
| 11800 | |
| 11801 | let json = fs::read_to_string(&json_report).unwrap(); |
| 11802 | assert!(json.contains("\"status\": \"match\"")); |
| 11803 | assert!(json.contains("\"classification\": \"match\"")); |
| 11804 | assert!(json.contains("\"difference_count\": 0")); |
| 11805 | assert!(json.contains("\"changed_artifacts\": []")); |
| 11806 | |
| 11807 | let markdown = fs::read_to_string(&markdown_report).unwrap(); |
| 11808 | assert!(markdown.contains("status: match")); |
| 11809 | assert!(markdown.contains("classification: match")); |
| 11810 | assert!(markdown.contains("difference_count: 0")); |
| 11811 | assert!(markdown.contains("changed_artifacts: none")); |
| 11812 | |
| 11813 | let _ = fs::remove_dir_all(&root); |
| 11814 | } |
| 11815 | |
| 11816 | #[cfg(unix)] |
| 11817 | #[test] |
| 11818 | fn compare_named_real_compilers_match_on_runtime_corpus() { |
| 11819 | if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 11820 | return; |
| 11821 | } |
| 11822 | |
| 11823 | for opt_level in stable_runtime_compare_opt_levels() { |
| 11824 | for program in stable_runtime_compare_corpus() { |
| 11825 | let config = CompareConfig { |
| 11826 | left: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11827 | right: CompilerSpec::Named(NamedCompiler::FlangNew), |
| 11828 | program, |
| 11829 | opt_level, |
| 11830 | artifacts: BTreeSet::new(), |
| 11831 | json_report: None, |
| 11832 | markdown_report: None, |
| 11833 | tools: ToolchainConfig::from_env(), |
| 11834 | }; |
| 11835 | |
| 11836 | let result = run_compare(&config).unwrap(); |
| 11837 | assert_eq!(compare_status(&result), "match"); |
| 11838 | assert_eq!(compare_classification(&result), "match"); |
| 11839 | assert!(result.differences.is_empty()); |
| 11840 | assert_eq!(result.left.provenance.adapter_kind, "named"); |
| 11841 | assert_eq!(result.right.provenance.adapter_kind, "named"); |
| 11842 | assert_eq!(result.left.opt_level, opt_level); |
| 11843 | assert_eq!(result.right.opt_level, opt_level); |
| 11844 | } |
| 11845 | } |
| 11846 | } |
| 11847 | |
| 11848 | #[cfg(unix)] |
| 11849 | #[test] |
| 11850 | fn compare_armfortas_and_gfortran_match_on_runtime_corpus_when_available() { |
| 11851 | if !command_is_available("gfortran") { |
| 11852 | return; |
| 11853 | } |
| 11854 | let Some(armfortas_bin) = armfortas_smoke_binary() else { |
| 11855 | return; |
| 11856 | }; |
| 11857 | |
| 11858 | let mut tools = ToolchainConfig::from_env(); |
| 11859 | tools.armfortas = ArmfortasCliAdapter::External(armfortas_bin.display().to_string()); |
| 11860 | |
| 11861 | for opt_level in stable_runtime_compare_opt_levels() { |
| 11862 | for program in stable_runtime_compare_corpus() { |
| 11863 | let config = CompareConfig { |
| 11864 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11865 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11866 | program, |
| 11867 | opt_level, |
| 11868 | artifacts: BTreeSet::new(), |
| 11869 | json_report: None, |
| 11870 | markdown_report: None, |
| 11871 | tools: tools.clone(), |
| 11872 | }; |
| 11873 | |
| 11874 | let result = run_compare(&config).unwrap(); |
| 11875 | assert_eq!(compare_status(&result), "match"); |
| 11876 | assert_eq!(compare_classification(&result), "match"); |
| 11877 | assert!(result.differences.is_empty()); |
| 11878 | assert_eq!(result.left.provenance.backend_mode, "cli-observable"); |
| 11879 | assert_eq!(result.left.opt_level, opt_level); |
| 11880 | assert_eq!(result.right.opt_level, opt_level); |
| 11881 | } |
| 11882 | } |
| 11883 | } |
| 11884 | |
| 11885 | #[cfg(unix)] |
| 11886 | #[test] |
| 11887 | fn introspect_armfortas_rich_artifacts_on_runtime_fixture() { |
| 11888 | let config = IntrospectConfig { |
| 11889 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11890 | program: runtime_fixture("if_else.f90"), |
| 11891 | opt_level: OptLevel::O0, |
| 11892 | artifacts: BTreeSet::from([ |
| 11893 | ArtifactKey::Asm, |
| 11894 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11895 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11896 | ]), |
| 11897 | json_report: None, |
| 11898 | markdown_report: None, |
| 11899 | all_artifacts: false, |
| 11900 | summary_only: false, |
| 11901 | max_artifact_lines: None, |
| 11902 | tools: ToolchainConfig::from_env(), |
| 11903 | }; |
| 11904 | |
| 11905 | let observed = run_introspect(&config).unwrap(); |
| 11906 | let observation = &observed.observation; |
| 11907 | assert_eq!(observation.compile_exit_code, 0); |
| 11908 | assert_eq!(observation.provenance.backend_mode, "linked"); |
| 11909 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 11910 | assert!(observation |
| 11911 | .artifacts |
| 11912 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 11913 | assert!(observation |
| 11914 | .artifacts |
| 11915 | .contains_key(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 11916 | |
| 11917 | let ir = match observation |
| 11918 | .artifacts |
| 11919 | .get(&ArtifactKey::Extra("armfortas.ir".into())) |
| 11920 | .unwrap() |
| 11921 | { |
| 11922 | ArtifactValue::Text(text) => text, |
| 11923 | other => panic!("expected text ir artifact, got {:?}", other), |
| 11924 | }; |
| 11925 | assert!(ir.contains("func") || ir.contains("module")); |
| 11926 | |
| 11927 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 11928 | assert!(rendered.contains("Generic artifacts")); |
| 11929 | assert!(rendered.contains("Adapter extras")); |
| 11930 | assert!(rendered.contains("-- armfortas --")); |
| 11931 | assert!(rendered.contains("== ir ==")); |
| 11932 | } |
| 11933 | |
| 11934 | #[cfg(unix)] |
| 11935 | #[test] |
| 11936 | fn introspect_armfortas_all_artifacts_includes_stage_extras() { |
| 11937 | let config = IntrospectConfig { |
| 11938 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11939 | program: runtime_fixture("if_else.f90"), |
| 11940 | opt_level: OptLevel::O0, |
| 11941 | artifacts: BTreeSet::new(), |
| 11942 | json_report: None, |
| 11943 | markdown_report: None, |
| 11944 | all_artifacts: true, |
| 11945 | summary_only: false, |
| 11946 | max_artifact_lines: None, |
| 11947 | tools: ToolchainConfig::from_env(), |
| 11948 | }; |
| 11949 | |
| 11950 | let observed = run_introspect(&config).unwrap(); |
| 11951 | let observation = &observed.observation; |
| 11952 | for artifact in [ |
| 11953 | ArtifactKey::Asm, |
| 11954 | ArtifactKey::Obj, |
| 11955 | ArtifactKey::Runtime, |
| 11956 | ArtifactKey::Extra("armfortas.preprocess".into()), |
| 11957 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11958 | ArtifactKey::Extra("armfortas.ast".into()), |
| 11959 | ArtifactKey::Extra("armfortas.sema".into()), |
| 11960 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11961 | ArtifactKey::Extra("armfortas.optir".into()), |
| 11962 | ArtifactKey::Extra("armfortas.mir".into()), |
| 11963 | ArtifactKey::Extra("armfortas.regalloc".into()), |
| 11964 | ] { |
| 11965 | assert!( |
| 11966 | observation.artifacts.contains_key(&artifact), |
| 11967 | "missing artifact {}", |
| 11968 | artifact.as_str() |
| 11969 | ); |
| 11970 | } |
| 11971 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 11972 | } |
| 11973 | |
| 11974 | #[cfg(unix)] |
| 11975 | #[test] |
| 11976 | fn introspect_armfortas_failure_reports_stage_and_partial_capture() { |
| 11977 | let config = IntrospectConfig { |
| 11978 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11979 | program: invalid_fixture("parse_error.f90"), |
| 11980 | opt_level: OptLevel::O0, |
| 11981 | artifacts: BTreeSet::from([ |
| 11982 | ArtifactKey::Asm, |
| 11983 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 11984 | ArtifactKey::Extra("armfortas.ir".into()), |
| 11985 | ]), |
| 11986 | json_report: None, |
| 11987 | markdown_report: None, |
| 11988 | all_artifacts: false, |
| 11989 | summary_only: false, |
| 11990 | max_artifact_lines: None, |
| 11991 | tools: ToolchainConfig::from_env(), |
| 11992 | }; |
| 11993 | |
| 11994 | let observed = run_introspect(&config).unwrap(); |
| 11995 | let observation = &observed.observation; |
| 11996 | assert_eq!(observation.compile_exit_code, 1); |
| 11997 | assert_eq!( |
| 11998 | observation.provenance.failure_stage.as_deref(), |
| 11999 | Some("parser") |
| 12000 | ); |
| 12001 | assert!(observation |
| 12002 | .artifacts |
| 12003 | .contains_key(&ArtifactKey::Diagnostics)); |
| 12004 | assert!(observation |
| 12005 | .artifacts |
| 12006 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 12007 | assert!(missing_introspection_artifact_names(&observed).contains(&"asm".to_string())); |
| 12008 | assert!( |
| 12009 | missing_introspection_artifact_names(&observed).contains(&"armfortas.ir".to_string()) |
| 12010 | ); |
| 12011 | |
| 12012 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 12013 | assert!(rendered.contains("status: compile failed")); |
| 12014 | assert!(rendered.contains("failure_stage: parser")); |
| 12015 | assert!(rendered.contains("diagnostic_excerpt:")); |
| 12016 | } |
| 12017 | |
| 12018 | #[cfg(unix)] |
| 12019 | #[test] |
| 12020 | fn introspect_named_external_compiler_reports_generic_artifacts_when_available() { |
| 12021 | if !command_is_available("gfortran") { |
| 12022 | return; |
| 12023 | } |
| 12024 | |
| 12025 | let config = IntrospectConfig { |
| 12026 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12027 | program: runtime_fixture("if_else.f90"), |
| 12028 | opt_level: OptLevel::O0, |
| 12029 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12030 | json_report: None, |
| 12031 | markdown_report: None, |
| 12032 | all_artifacts: false, |
| 12033 | summary_only: false, |
| 12034 | max_artifact_lines: None, |
| 12035 | tools: ToolchainConfig::from_env(), |
| 12036 | }; |
| 12037 | |
| 12038 | let observed = run_introspect(&config).unwrap(); |
| 12039 | let observation = &observed.observation; |
| 12040 | assert_eq!(observation.compile_exit_code, 0); |
| 12041 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12042 | assert_eq!(observation.provenance.adapter_kind, "named"); |
| 12043 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 12044 | assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 12045 | assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 12046 | assert!(observation_adapter_extras(observation).is_empty()); |
| 12047 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 12048 | } |
| 12049 | |
| 12050 | #[cfg(unix)] |
| 12051 | #[test] |
| 12052 | fn introspect_explicit_path_compiler_reports_generic_artifacts_when_available() { |
| 12053 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12054 | ensure_fixture_executable(&compiler); |
| 12055 | |
| 12056 | let config = IntrospectConfig { |
| 12057 | compiler: CompilerSpec::Binary(compiler.clone()), |
| 12058 | program: runtime_fixture("if_else.f90"), |
| 12059 | opt_level: OptLevel::O0, |
| 12060 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12061 | json_report: None, |
| 12062 | markdown_report: None, |
| 12063 | all_artifacts: false, |
| 12064 | summary_only: false, |
| 12065 | max_artifact_lines: None, |
| 12066 | tools: ToolchainConfig::from_env(), |
| 12067 | }; |
| 12068 | |
| 12069 | let observed = run_introspect(&config).unwrap(); |
| 12070 | let observation = &observed.observation; |
| 12071 | assert_eq!(observation.compile_exit_code, 0); |
| 12072 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12073 | assert_eq!(observation.provenance.adapter_kind, "explicit-path"); |
| 12074 | assert!(observation |
| 12075 | .provenance |
| 12076 | .backend_detail |
| 12077 | .contains("match_42_a.sh")); |
| 12078 | assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 12079 | assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 12080 | assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 12081 | assert!(observation_adapter_extras(observation).is_empty()); |
| 12082 | assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 12083 | } |
| 12084 | |
| 12085 | #[cfg(unix)] |
| 12086 | #[test] |
| 12087 | fn introspect_external_failure_reports_missing_requested_artifacts() { |
| 12088 | let compiler = fake_compiler_fixture("compile_fail.sh"); |
| 12089 | ensure_fixture_executable(&compiler); |
| 12090 | |
| 12091 | let config = IntrospectConfig { |
| 12092 | compiler: CompilerSpec::Binary(compiler), |
| 12093 | program: runtime_fixture("if_else.f90"), |
| 12094 | opt_level: OptLevel::O0, |
| 12095 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12096 | json_report: None, |
| 12097 | markdown_report: None, |
| 12098 | all_artifacts: false, |
| 12099 | summary_only: false, |
| 12100 | max_artifact_lines: None, |
| 12101 | tools: ToolchainConfig::from_env(), |
| 12102 | }; |
| 12103 | |
| 12104 | let observed = run_introspect(&config).unwrap(); |
| 12105 | let observation = &observed.observation; |
| 12106 | assert_eq!(observation.compile_exit_code, 1); |
| 12107 | assert_eq!(observation.provenance.failure_stage, None); |
| 12108 | assert!(observation |
| 12109 | .artifacts |
| 12110 | .contains_key(&ArtifactKey::Diagnostics)); |
| 12111 | assert_eq!( |
| 12112 | missing_introspection_artifact_names(&observed), |
| 12113 | vec!["asm".to_string(), "obj".to_string(), "runtime".to_string()] |
| 12114 | ); |
| 12115 | |
| 12116 | let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 12117 | assert!(rendered.contains("status: compile failed")); |
| 12118 | assert!(rendered.contains("failure_stage: none")); |
| 12119 | } |
| 12120 | |
| 12121 | #[test] |
| 12122 | fn introspect_named_external_compiler_rejects_namespaced_artifacts() { |
| 12123 | let config = IntrospectConfig { |
| 12124 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12125 | program: runtime_fixture("if_else.f90"), |
| 12126 | opt_level: OptLevel::O0, |
| 12127 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12128 | json_report: None, |
| 12129 | markdown_report: None, |
| 12130 | all_artifacts: false, |
| 12131 | summary_only: false, |
| 12132 | max_artifact_lines: None, |
| 12133 | tools: ToolchainConfig::from_env(), |
| 12134 | }; |
| 12135 | |
| 12136 | let observed = run_introspect(&config).unwrap(); |
| 12137 | let observation = &observed.observation; |
| 12138 | assert_eq!(observation.compile_exit_code, 1); |
| 12139 | assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12140 | assert_eq!(observation.provenance.failure_stage, None); |
| 12141 | let diagnostics = match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 12142 | Some(ArtifactValue::Text(text)) => text, |
| 12143 | other => panic!("expected text diagnostics, got {:?}", other), |
| 12144 | }; |
| 12145 | assert!(diagnostics.contains("does not support requested artifacts")); |
| 12146 | assert!(diagnostics.contains("armfortas.ir")); |
| 12147 | } |
| 12148 | |
| 12149 | #[test] |
| 12150 | fn compose_observation_failure_detail_uses_unavailable_wording() { |
| 12151 | let observation = CompilerObservation { |
| 12152 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12153 | program: PathBuf::from("demo.f90"), |
| 12154 | opt_level: OptLevel::O0, |
| 12155 | compile_exit_code: 1, |
| 12156 | artifacts: BTreeMap::from([( |
| 12157 | ArtifactKey::Diagnostics, |
| 12158 | ArtifactValue::Text("linked armfortas capture is unavailable in this build".into()), |
| 12159 | )]), |
| 12160 | provenance: ObservationProvenance { |
| 12161 | compiler_identity: "armfortas".into(), |
| 12162 | adapter_kind: "named".into(), |
| 12163 | backend_mode: "unavailable".into(), |
| 12164 | backend_detail: "unavailable without linked-armfortas feature".into(), |
| 12165 | artifacts_captured: vec!["diagnostics".into()], |
| 12166 | comparison_basis: None, |
| 12167 | failure_stage: None, |
| 12168 | }, |
| 12169 | }; |
| 12170 | |
| 12171 | let detail = compose_observation_failure_detail(&observation); |
| 12172 | assert!(detail.contains("armfortas unavailable for requested artifacts in this build")); |
| 12173 | assert!(!detail.contains("failed in")); |
| 12174 | } |
| 12175 | |
| 12176 | #[test] |
| 12177 | fn compose_observation_failure_detail_uses_unsupported_wording() { |
| 12178 | let observation = CompilerObservation { |
| 12179 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12180 | program: PathBuf::from("demo.f90"), |
| 12181 | opt_level: OptLevel::O0, |
| 12182 | compile_exit_code: 1, |
| 12183 | artifacts: BTreeMap::from([( |
| 12184 | ArtifactKey::Diagnostics, |
| 12185 | ArtifactValue::Text( |
| 12186 | "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 12187 | .into(), |
| 12188 | ), |
| 12189 | )]), |
| 12190 | provenance: ObservationProvenance { |
| 12191 | compiler_identity: "gfortran".into(), |
| 12192 | adapter_kind: "named".into(), |
| 12193 | backend_mode: "external-driver".into(), |
| 12194 | backend_detail: "generic external driver adapter using gfortran".into(), |
| 12195 | artifacts_captured: vec!["diagnostics".into()], |
| 12196 | comparison_basis: None, |
| 12197 | failure_stage: None, |
| 12198 | }, |
| 12199 | }; |
| 12200 | |
| 12201 | let detail = compose_observation_failure_detail(&observation); |
| 12202 | assert!(detail.contains("gfortran does not support requested artifacts in this adapter")); |
| 12203 | assert!(!detail.contains("gfortran failed")); |
| 12204 | } |
| 12205 | |
| 12206 | #[test] |
| 12207 | fn execute_generic_introspect_case_reports_capability_mismatch_clearly() { |
| 12208 | let suite = SuiteSpec { |
| 12209 | name: "v2/generic-introspect".into(), |
| 12210 | path: PathBuf::from("suite.afs"), |
| 12211 | cases: Vec::new(), |
| 12212 | }; |
| 12213 | let case = CaseSpec { |
| 12214 | name: "gfortran-armfortas-ir".into(), |
| 12215 | source: runtime_fixture("if_else.f90"), |
| 12216 | graph_files: Vec::new(), |
| 12217 | requested: BTreeSet::new(), |
| 12218 | generic_introspect: Some(GenericIntrospectCase { |
| 12219 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12220 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12221 | }), |
| 12222 | generic_compare: None, |
| 12223 | opt_levels: vec![OptLevel::O0], |
| 12224 | repeat_count: 2, |
| 12225 | reference_compilers: Vec::new(), |
| 12226 | consistency_checks: Vec::new(), |
| 12227 | expectations: vec![Expectation::Contains { |
| 12228 | target: Target::Artifact(ArtifactKey::Extra("armfortas.ir".into())), |
| 12229 | needle: "func".into(), |
| 12230 | }], |
| 12231 | status_rules: Vec::new(), |
| 12232 | capability_policy: None, |
| 12233 | }; |
| 12234 | let config = RunConfig { |
| 12235 | suite_filter: None, |
| 12236 | case_filter: None, |
| 12237 | opt_filter: None, |
| 12238 | verbose: false, |
| 12239 | fail_fast: false, |
| 12240 | include_future: false, |
| 12241 | all_stages: false, |
| 12242 | json_report: None, |
| 12243 | markdown_report: None, |
| 12244 | tools: ToolchainConfig::from_env(), |
| 12245 | }; |
| 12246 | |
| 12247 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12248 | assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12249 | assert!(outcome |
| 12250 | .detail |
| 12251 | .contains("gfortran does not support requested artifacts in this adapter")); |
| 12252 | assert!(!outcome.detail.contains("gfortran failed")); |
| 12253 | } |
| 12254 | |
| 12255 | #[test] |
| 12256 | fn execute_generic_introspect_case_applies_future_capability_policy() { |
| 12257 | let suite = SuiteSpec { |
| 12258 | name: "v2/capability-policy".into(), |
| 12259 | path: PathBuf::from("suite.afs"), |
| 12260 | cases: Vec::new(), |
| 12261 | }; |
| 12262 | let case = CaseSpec { |
| 12263 | name: "gfortran-armfortas-ir".into(), |
| 12264 | source: runtime_fixture("if_else.f90"), |
| 12265 | graph_files: Vec::new(), |
| 12266 | requested: BTreeSet::new(), |
| 12267 | generic_introspect: Some(GenericIntrospectCase { |
| 12268 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12269 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12270 | }), |
| 12271 | generic_compare: None, |
| 12272 | opt_levels: vec![OptLevel::O0], |
| 12273 | repeat_count: 2, |
| 12274 | reference_compilers: Vec::new(), |
| 12275 | consistency_checks: Vec::new(), |
| 12276 | expectations: Vec::new(), |
| 12277 | status_rules: Vec::new(), |
| 12278 | capability_policy: Some(CapabilityPolicy { |
| 12279 | kind: StatusKind::Future, |
| 12280 | reason: "generic gfortran surface has no armfortas extras".into(), |
| 12281 | }), |
| 12282 | }; |
| 12283 | let config = RunConfig { |
| 12284 | suite_filter: None, |
| 12285 | case_filter: None, |
| 12286 | opt_filter: None, |
| 12287 | verbose: false, |
| 12288 | fail_fast: false, |
| 12289 | include_future: false, |
| 12290 | all_stages: false, |
| 12291 | json_report: None, |
| 12292 | markdown_report: None, |
| 12293 | tools: ToolchainConfig::from_env(), |
| 12294 | }; |
| 12295 | |
| 12296 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12297 | assert_eq!(outcome.kind, OutcomeKind::Future); |
| 12298 | assert!(outcome |
| 12299 | .detail |
| 12300 | .contains("generic gfortran surface has no armfortas extras")); |
| 12301 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12302 | } |
| 12303 | |
| 12304 | #[cfg(unix)] |
| 12305 | #[test] |
| 12306 | fn execute_generic_suite_case_uses_introspect_engine() { |
| 12307 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12308 | ensure_fixture_executable(&compiler); |
| 12309 | |
| 12310 | let suite = SuiteSpec { |
| 12311 | name: "v2/generic-introspect".into(), |
| 12312 | path: PathBuf::from("suite.afs"), |
| 12313 | cases: Vec::new(), |
| 12314 | }; |
| 12315 | let case = CaseSpec { |
| 12316 | name: "fake-runtime".into(), |
| 12317 | source: runtime_fixture("if_else.f90"), |
| 12318 | graph_files: Vec::new(), |
| 12319 | requested: BTreeSet::new(), |
| 12320 | generic_introspect: Some(GenericIntrospectCase { |
| 12321 | compiler: CompilerSpec::Binary(compiler), |
| 12322 | artifacts: BTreeSet::from([ |
| 12323 | ArtifactKey::Asm, |
| 12324 | ArtifactKey::Obj, |
| 12325 | ArtifactKey::Runtime, |
| 12326 | ]), |
| 12327 | }), |
| 12328 | generic_compare: None, |
| 12329 | opt_levels: vec![OptLevel::O0], |
| 12330 | repeat_count: 2, |
| 12331 | reference_compilers: Vec::new(), |
| 12332 | consistency_checks: Vec::new(), |
| 12333 | expectations: vec![ |
| 12334 | Expectation::Contains { |
| 12335 | target: Target::Artifact(ArtifactKey::Asm), |
| 12336 | needle: ".globl _main".into(), |
| 12337 | }, |
| 12338 | Expectation::Contains { |
| 12339 | target: Target::RunStdout, |
| 12340 | needle: "42".into(), |
| 12341 | }, |
| 12342 | ], |
| 12343 | status_rules: Vec::new(), |
| 12344 | capability_policy: None, |
| 12345 | }; |
| 12346 | let config = RunConfig { |
| 12347 | suite_filter: None, |
| 12348 | case_filter: None, |
| 12349 | opt_filter: None, |
| 12350 | verbose: false, |
| 12351 | fail_fast: false, |
| 12352 | include_future: false, |
| 12353 | all_stages: false, |
| 12354 | json_report: None, |
| 12355 | markdown_report: None, |
| 12356 | tools: ToolchainConfig::from_env(), |
| 12357 | }; |
| 12358 | |
| 12359 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12360 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12361 | assert!(outcome.detail.is_empty()); |
| 12362 | assert!(outcome.bundle.is_none()); |
| 12363 | } |
| 12364 | |
| 12365 | #[cfg(unix)] |
| 12366 | #[test] |
| 12367 | fn execute_generic_suite_case_supports_cli_consistency() { |
| 12368 | let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12369 | ensure_fixture_executable(&compiler); |
| 12370 | |
| 12371 | let suite = SuiteSpec { |
| 12372 | name: "v2/generic-consistency".into(), |
| 12373 | path: PathBuf::from("suite.afs"), |
| 12374 | cases: Vec::new(), |
| 12375 | }; |
| 12376 | let case = CaseSpec { |
| 12377 | name: "fake-runtime-consistency".into(), |
| 12378 | source: runtime_fixture("if_else.f90"), |
| 12379 | graph_files: Vec::new(), |
| 12380 | requested: BTreeSet::new(), |
| 12381 | generic_introspect: Some(GenericIntrospectCase { |
| 12382 | compiler: CompilerSpec::Binary(compiler), |
| 12383 | artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Runtime]), |
| 12384 | }), |
| 12385 | generic_compare: None, |
| 12386 | opt_levels: vec![OptLevel::O0], |
| 12387 | repeat_count: 3, |
| 12388 | reference_compilers: Vec::new(), |
| 12389 | consistency_checks: vec![ |
| 12390 | ConsistencyCheck::CliAsmReproducible, |
| 12391 | ConsistencyCheck::CliRunReproducible, |
| 12392 | ], |
| 12393 | expectations: vec![ |
| 12394 | Expectation::Contains { |
| 12395 | target: Target::Artifact(ArtifactKey::Asm), |
| 12396 | needle: ".globl _main".into(), |
| 12397 | }, |
| 12398 | Expectation::Contains { |
| 12399 | target: Target::RunStdout, |
| 12400 | needle: "42".into(), |
| 12401 | }, |
| 12402 | ], |
| 12403 | status_rules: Vec::new(), |
| 12404 | capability_policy: None, |
| 12405 | }; |
| 12406 | let config = RunConfig { |
| 12407 | suite_filter: None, |
| 12408 | case_filter: None, |
| 12409 | opt_filter: None, |
| 12410 | verbose: false, |
| 12411 | fail_fast: false, |
| 12412 | include_future: false, |
| 12413 | all_stages: false, |
| 12414 | json_report: None, |
| 12415 | markdown_report: None, |
| 12416 | tools: ToolchainConfig::from_env(), |
| 12417 | }; |
| 12418 | |
| 12419 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12420 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12421 | assert!(outcome.detail.is_empty()); |
| 12422 | assert!(outcome.consistency_observations.is_empty()); |
| 12423 | } |
| 12424 | |
| 12425 | #[cfg(unix)] |
| 12426 | #[test] |
| 12427 | fn execute_generic_suite_case_supports_differential_when_available() { |
| 12428 | if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 12429 | return; |
| 12430 | } |
| 12431 | |
| 12432 | let suite = SuiteSpec { |
| 12433 | name: "v2/generic-differential".into(), |
| 12434 | path: PathBuf::from("suite.afs"), |
| 12435 | cases: Vec::new(), |
| 12436 | }; |
| 12437 | let case = CaseSpec { |
| 12438 | name: "gfortran-vs-flang".into(), |
| 12439 | source: runtime_fixture("if_else.f90"), |
| 12440 | graph_files: Vec::new(), |
| 12441 | requested: BTreeSet::new(), |
| 12442 | generic_introspect: Some(GenericIntrospectCase { |
| 12443 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12444 | artifacts: BTreeSet::from([ArtifactKey::Runtime]), |
| 12445 | }), |
| 12446 | generic_compare: None, |
| 12447 | opt_levels: vec![OptLevel::O0], |
| 12448 | repeat_count: 2, |
| 12449 | reference_compilers: vec![ReferenceCompiler::FlangNew], |
| 12450 | consistency_checks: Vec::new(), |
| 12451 | expectations: vec![ |
| 12452 | Expectation::Contains { |
| 12453 | target: Target::RunStdout, |
| 12454 | needle: "positive".into(), |
| 12455 | }, |
| 12456 | Expectation::IntEquals { |
| 12457 | target: Target::RunExitCode, |
| 12458 | value: 0, |
| 12459 | }, |
| 12460 | ], |
| 12461 | status_rules: Vec::new(), |
| 12462 | capability_policy: None, |
| 12463 | }; |
| 12464 | let config = RunConfig { |
| 12465 | suite_filter: None, |
| 12466 | case_filter: None, |
| 12467 | opt_filter: None, |
| 12468 | verbose: false, |
| 12469 | fail_fast: false, |
| 12470 | include_future: false, |
| 12471 | all_stages: false, |
| 12472 | json_report: None, |
| 12473 | markdown_report: None, |
| 12474 | tools: ToolchainConfig::from_env(), |
| 12475 | }; |
| 12476 | |
| 12477 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12478 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12479 | assert!(outcome.detail.is_empty()); |
| 12480 | } |
| 12481 | |
| 12482 | #[cfg(unix)] |
| 12483 | #[test] |
| 12484 | fn execute_generic_compare_suite_case_uses_compare_engine() { |
| 12485 | let left = fake_compiler_fixture("match_42_a.sh"); |
| 12486 | let right = fake_compiler_fixture("runtime_41.sh"); |
| 12487 | ensure_fixture_executable(&left); |
| 12488 | ensure_fixture_executable(&right); |
| 12489 | |
| 12490 | let suite = SuiteSpec { |
| 12491 | name: "v2/generic-compare".into(), |
| 12492 | path: PathBuf::from("suite.afs"), |
| 12493 | cases: Vec::new(), |
| 12494 | }; |
| 12495 | let case = CaseSpec { |
| 12496 | name: "fake-divergence".into(), |
| 12497 | source: runtime_fixture("if_else.f90"), |
| 12498 | graph_files: Vec::new(), |
| 12499 | requested: BTreeSet::new(), |
| 12500 | generic_introspect: None, |
| 12501 | generic_compare: Some(GenericCompareCase { |
| 12502 | left: CompilerSpec::Binary(left), |
| 12503 | right: CompilerSpec::Binary(right), |
| 12504 | artifacts: BTreeSet::from([ |
| 12505 | ArtifactKey::Diagnostics, |
| 12506 | ArtifactKey::Runtime, |
| 12507 | ArtifactKey::Asm, |
| 12508 | ]), |
| 12509 | }), |
| 12510 | opt_levels: vec![OptLevel::O0], |
| 12511 | repeat_count: 2, |
| 12512 | reference_compilers: Vec::new(), |
| 12513 | consistency_checks: Vec::new(), |
| 12514 | expectations: vec![ |
| 12515 | Expectation::Equals { |
| 12516 | target: Target::CompareStatus, |
| 12517 | value: "diff".into(), |
| 12518 | }, |
| 12519 | Expectation::Equals { |
| 12520 | target: Target::CompareClassification, |
| 12521 | value: "mixed divergence".into(), |
| 12522 | }, |
| 12523 | Expectation::Contains { |
| 12524 | target: Target::CompareChangedArtifacts, |
| 12525 | needle: "asm".into(), |
| 12526 | }, |
| 12527 | Expectation::Contains { |
| 12528 | target: Target::CompareChangedArtifacts, |
| 12529 | needle: "runtime".into(), |
| 12530 | }, |
| 12531 | Expectation::IntEquals { |
| 12532 | target: Target::CompareDifferenceCount, |
| 12533 | value: 2, |
| 12534 | }, |
| 12535 | ], |
| 12536 | status_rules: Vec::new(), |
| 12537 | capability_policy: None, |
| 12538 | }; |
| 12539 | let config = RunConfig { |
| 12540 | suite_filter: None, |
| 12541 | case_filter: None, |
| 12542 | opt_filter: None, |
| 12543 | verbose: false, |
| 12544 | fail_fast: false, |
| 12545 | include_future: false, |
| 12546 | all_stages: false, |
| 12547 | json_report: None, |
| 12548 | markdown_report: None, |
| 12549 | tools: ToolchainConfig::from_env(), |
| 12550 | }; |
| 12551 | |
| 12552 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12553 | assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12554 | } |
| 12555 | |
| 12556 | #[test] |
| 12557 | fn execute_generic_compare_suite_case_reports_capability_mismatch() { |
| 12558 | let suite = SuiteSpec { |
| 12559 | name: "v2/generic-compare".into(), |
| 12560 | path: PathBuf::from("suite.afs"), |
| 12561 | cases: Vec::new(), |
| 12562 | }; |
| 12563 | let case = CaseSpec { |
| 12564 | name: "armfortas-ir-vs-gfortran".into(), |
| 12565 | source: runtime_fixture("if_else.f90"), |
| 12566 | graph_files: Vec::new(), |
| 12567 | requested: BTreeSet::new(), |
| 12568 | generic_introspect: None, |
| 12569 | generic_compare: Some(GenericCompareCase { |
| 12570 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12571 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12572 | artifacts: BTreeSet::from([ |
| 12573 | ArtifactKey::Diagnostics, |
| 12574 | ArtifactKey::Runtime, |
| 12575 | ArtifactKey::Extra("armfortas.ir".into()), |
| 12576 | ]), |
| 12577 | }), |
| 12578 | opt_levels: vec![OptLevel::O0], |
| 12579 | repeat_count: 2, |
| 12580 | reference_compilers: Vec::new(), |
| 12581 | consistency_checks: Vec::new(), |
| 12582 | expectations: vec![Expectation::Equals { |
| 12583 | target: Target::CompareStatus, |
| 12584 | value: "match".into(), |
| 12585 | }], |
| 12586 | status_rules: Vec::new(), |
| 12587 | capability_policy: None, |
| 12588 | }; |
| 12589 | let config = RunConfig { |
| 12590 | suite_filter: None, |
| 12591 | case_filter: None, |
| 12592 | opt_filter: None, |
| 12593 | verbose: false, |
| 12594 | fail_fast: false, |
| 12595 | include_future: false, |
| 12596 | all_stages: false, |
| 12597 | json_report: None, |
| 12598 | markdown_report: None, |
| 12599 | tools: ToolchainConfig::from_env(), |
| 12600 | }; |
| 12601 | |
| 12602 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12603 | assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12604 | assert!(outcome.detail.contains("compare request is not supported")); |
| 12605 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12606 | } |
| 12607 | |
| 12608 | #[test] |
| 12609 | fn execute_generic_compare_suite_case_applies_xfail_capability_policy() { |
| 12610 | let suite = SuiteSpec { |
| 12611 | name: "v2/capability-policy".into(), |
| 12612 | path: PathBuf::from("suite.afs"), |
| 12613 | cases: Vec::new(), |
| 12614 | }; |
| 12615 | let case = CaseSpec { |
| 12616 | name: "armfortas-ir-vs-gfortran".into(), |
| 12617 | source: runtime_fixture("if_else.f90"), |
| 12618 | graph_files: Vec::new(), |
| 12619 | requested: BTreeSet::new(), |
| 12620 | generic_introspect: None, |
| 12621 | generic_compare: Some(GenericCompareCase { |
| 12622 | left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12623 | right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12624 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12625 | }), |
| 12626 | opt_levels: vec![OptLevel::O0], |
| 12627 | repeat_count: 2, |
| 12628 | reference_compilers: Vec::new(), |
| 12629 | consistency_checks: Vec::new(), |
| 12630 | expectations: Vec::new(), |
| 12631 | status_rules: Vec::new(), |
| 12632 | capability_policy: Some(CapabilityPolicy { |
| 12633 | kind: StatusKind::Xfail, |
| 12634 | reason: "mixed-surface namespaced compare stays soft for now".into(), |
| 12635 | }), |
| 12636 | }; |
| 12637 | let config = RunConfig { |
| 12638 | suite_filter: None, |
| 12639 | case_filter: None, |
| 12640 | opt_filter: None, |
| 12641 | verbose: false, |
| 12642 | fail_fast: false, |
| 12643 | include_future: false, |
| 12644 | all_stages: false, |
| 12645 | json_report: None, |
| 12646 | markdown_report: None, |
| 12647 | tools: ToolchainConfig::from_env(), |
| 12648 | }; |
| 12649 | |
| 12650 | let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12651 | assert_eq!(outcome.kind, OutcomeKind::Xfail); |
| 12652 | assert!(outcome |
| 12653 | .detail |
| 12654 | .contains("mixed-surface namespaced compare stays soft for now")); |
| 12655 | assert!(outcome.detail.contains("armfortas.ir")); |
| 12656 | } |
| 12657 | |
| 12658 | #[test] |
| 12659 | fn parses_suite_and_case() { |
| 12660 | let root = std::env::temp_dir().join("afs_tests_parser_spec.afs"); |
| 12661 | fs::write( |
| 12662 | &root, |
| 12663 | r#"suite "runtime/smoke" |
| 12664 | |
| 12665 | case "hello" |
| 12666 | source "../../../test_programs/hello.f90" |
| 12667 | armfortas => run, ir |
| 12668 | expect run.stdout check-comments |
| 12669 | expect ir contains "module main" |
| 12670 | expect asm not-contains "x18" |
| 12671 | end |
| 12672 | "#, |
| 12673 | ) |
| 12674 | .unwrap(); |
| 12675 | |
| 12676 | let suite = parse_suite_file(&root).unwrap(); |
| 12677 | assert_eq!(suite.name, "runtime/smoke"); |
| 12678 | assert_eq!(suite.cases.len(), 1); |
| 12679 | assert!(suite.cases[0].requested.contains(&Stage::Run)); |
| 12680 | assert!(suite.cases[0].requested.contains(&Stage::Ir)); |
| 12681 | assert!(suite.cases[0].generic_introspect.is_none()); |
| 12682 | assert!(matches!( |
| 12683 | suite.cases[0].expectations[2], |
| 12684 | Expectation::NotContains { |
| 12685 | target: Target::Artifact(ArtifactKey::Asm), |
| 12686 | .. |
| 12687 | } |
| 12688 | )); |
| 12689 | assert_eq!(suite.cases[0].opt_levels, vec![OptLevel::O0]); |
| 12690 | let _ = fs::remove_file(&root); |
| 12691 | } |
| 12692 | |
| 12693 | #[test] |
| 12694 | fn parses_generic_compiler_case() { |
| 12695 | let root = std::env::temp_dir().join("bencch_generic_parser_spec.afs"); |
| 12696 | fs::write( |
| 12697 | &root, |
| 12698 | r#"suite "v2/generic-introspect" |
| 12699 | |
| 12700 | case "fake-runtime" |
| 12701 | source "../../fixtures/runtime/if_else.f90" |
| 12702 | compiler gfortran => asm, obj, runtime |
| 12703 | expect asm contains ".globl _main" |
| 12704 | expect run.stdout contains "42" |
| 12705 | end |
| 12706 | "#, |
| 12707 | ) |
| 12708 | .unwrap(); |
| 12709 | |
| 12710 | let suite = parse_suite_file(&root).unwrap(); |
| 12711 | let case = &suite.cases[0]; |
| 12712 | let generic = case.generic_introspect.as_ref().unwrap(); |
| 12713 | assert_eq!( |
| 12714 | generic.compiler, |
| 12715 | CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12716 | ); |
| 12717 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12718 | assert!(generic.artifacts.contains(&ArtifactKey::Obj)); |
| 12719 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12720 | assert!(case.requested.is_empty()); |
| 12721 | let _ = fs::remove_file(&root); |
| 12722 | } |
| 12723 | |
| 12724 | #[test] |
| 12725 | fn parses_generic_compiler_case_with_differential_and_cli_consistency() { |
| 12726 | let root = std::env::temp_dir().join("bencch_generic_differential_parser_spec.afs"); |
| 12727 | fs::write( |
| 12728 | &root, |
| 12729 | r#"suite "v2/generic-differential" |
| 12730 | |
| 12731 | case "gfortran_runtime_matrix" |
| 12732 | source "../../fixtures/runtime/if_else.f90" |
| 12733 | opts => O0, O1, O2 |
| 12734 | repeat => 3 |
| 12735 | compiler gfortran => runtime, asm |
| 12736 | differential => flang-new |
| 12737 | consistency => cli_asm_reproducible, cli_run_reproducible |
| 12738 | expect run.stdout check-comments |
| 12739 | expect run.exit_code equals 0 |
| 12740 | end |
| 12741 | "#, |
| 12742 | ) |
| 12743 | .unwrap(); |
| 12744 | |
| 12745 | let suite = parse_suite_file(&root).unwrap(); |
| 12746 | let case = &suite.cases[0]; |
| 12747 | let generic = case.generic_introspect.as_ref().unwrap(); |
| 12748 | assert_eq!( |
| 12749 | generic.compiler, |
| 12750 | CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12751 | ); |
| 12752 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12753 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12754 | assert_eq!(case.reference_compilers, vec![ReferenceCompiler::FlangNew]); |
| 12755 | assert_eq!( |
| 12756 | case.consistency_checks, |
| 12757 | vec![ |
| 12758 | ConsistencyCheck::CliAsmReproducible, |
| 12759 | ConsistencyCheck::CliRunReproducible, |
| 12760 | ] |
| 12761 | ); |
| 12762 | let _ = fs::remove_file(&root); |
| 12763 | } |
| 12764 | |
| 12765 | #[test] |
| 12766 | fn rejects_capture_consistency_on_generic_compiler_case() { |
| 12767 | let root = std::env::temp_dir().join("bencch_generic_capture_consistency_parser_spec.afs"); |
| 12768 | fs::write( |
| 12769 | &root, |
| 12770 | r#"suite "v2/generic-consistency" |
| 12771 | |
| 12772 | case "armfortas_capture_run" |
| 12773 | source "../../fixtures/runtime/if_else.f90" |
| 12774 | compiler armfortas => runtime |
| 12775 | consistency => capture_run_reproducible |
| 12776 | expect run.exit_code equals 0 |
| 12777 | end |
| 12778 | "#, |
| 12779 | ) |
| 12780 | .unwrap(); |
| 12781 | |
| 12782 | let err = parse_suite_file(&root).unwrap_err(); |
| 12783 | assert!(err.contains("generic compiler cases only support")); |
| 12784 | assert!(err.contains("capture_run_reproducible")); |
| 12785 | let _ = fs::remove_file(&root); |
| 12786 | } |
| 12787 | |
| 12788 | #[test] |
| 12789 | fn parses_generic_compare_case() { |
| 12790 | let root = std::env::temp_dir().join("bencch_generic_compare_parser_spec.afs"); |
| 12791 | fs::write( |
| 12792 | &root, |
| 12793 | r#"suite "v2/generic-compare" |
| 12794 | |
| 12795 | case "fake-match" |
| 12796 | source "../../fixtures/runtime/if_else.f90" |
| 12797 | opts => O0, O1, O2 |
| 12798 | compare gfortran flang-new => asm |
| 12799 | expect compare.status equals "match" |
| 12800 | expect compare.difference_count equals 0 |
| 12801 | end |
| 12802 | "#, |
| 12803 | ) |
| 12804 | .unwrap(); |
| 12805 | |
| 12806 | let suite = parse_suite_file(&root).unwrap(); |
| 12807 | let case = &suite.cases[0]; |
| 12808 | let generic = case.generic_compare.as_ref().unwrap(); |
| 12809 | assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Gfortran)); |
| 12810 | assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::FlangNew)); |
| 12811 | assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12812 | assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12813 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12814 | assert_eq!( |
| 12815 | case.opt_levels, |
| 12816 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12817 | ); |
| 12818 | let _ = fs::remove_file(&root); |
| 12819 | } |
| 12820 | |
| 12821 | #[test] |
| 12822 | fn parses_generic_compare_case_with_namespaced_artifact() { |
| 12823 | let root = std::env::temp_dir().join("bencch_generic_compare_namespaced_spec.afs"); |
| 12824 | fs::write( |
| 12825 | &root, |
| 12826 | r#"suite "v2/generic-compare" |
| 12827 | |
| 12828 | case "armfortas-ir" |
| 12829 | source "../../fixtures/runtime/if_else.f90" |
| 12830 | compare armfortas armfortas => armfortas.ir |
| 12831 | expect compare.status equals "match" |
| 12832 | end |
| 12833 | "#, |
| 12834 | ) |
| 12835 | .unwrap(); |
| 12836 | |
| 12837 | let suite = parse_suite_file(&root).unwrap(); |
| 12838 | let case = &suite.cases[0]; |
| 12839 | let generic = case.generic_compare.as_ref().unwrap(); |
| 12840 | assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12841 | assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12842 | assert!(generic |
| 12843 | .artifacts |
| 12844 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 12845 | assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12846 | assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12847 | |
| 12848 | let _ = fs::remove_file(&root); |
| 12849 | } |
| 12850 | |
| 12851 | #[test] |
| 12852 | fn parses_capability_policy_for_generic_case() { |
| 12853 | let root = std::env::temp_dir().join("bencch_generic_capability_policy_spec.afs"); |
| 12854 | fs::write( |
| 12855 | &root, |
| 12856 | r#"suite "v2/capability-policy" |
| 12857 | |
| 12858 | case "gfortran_armfortas_ir" |
| 12859 | source "../../fixtures/runtime/if_else.f90" |
| 12860 | compiler gfortran => armfortas.ir |
| 12861 | future capability "generic gfortran surface has no armfortas extras" |
| 12862 | end |
| 12863 | "#, |
| 12864 | ) |
| 12865 | .unwrap(); |
| 12866 | |
| 12867 | let suite = parse_suite_file(&root).unwrap(); |
| 12868 | let case = &suite.cases[0]; |
| 12869 | let policy = case.capability_policy.as_ref().unwrap(); |
| 12870 | assert!(matches!(policy.kind, StatusKind::Future)); |
| 12871 | assert_eq!( |
| 12872 | policy.reason, |
| 12873 | "generic gfortran surface has no armfortas extras" |
| 12874 | ); |
| 12875 | let _ = fs::remove_file(&root); |
| 12876 | } |
| 12877 | |
| 12878 | #[test] |
| 12879 | fn parses_matrix_status_and_differential() { |
| 12880 | let root = std::env::temp_dir().join("afs_tests_matrix_spec.afs"); |
| 12881 | fs::write( |
| 12882 | &root, |
| 12883 | r#"suite "runtime/matrix" |
| 12884 | |
| 12885 | case "hello" |
| 12886 | source "../../../test_programs/hello.f90" |
| 12887 | opts => O0, O1, O2 |
| 12888 | armfortas => run |
| 12889 | differential => gfortran, flang-new |
| 12890 | expect run.exit_code equals 0 |
| 12891 | xfail when O1, O2 because "known issue" |
| 12892 | end |
| 12893 | "#, |
| 12894 | ) |
| 12895 | .unwrap(); |
| 12896 | |
| 12897 | let suite = parse_suite_file(&root).unwrap(); |
| 12898 | let case = &suite.cases[0]; |
| 12899 | assert_eq!( |
| 12900 | case.opt_levels, |
| 12901 | vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12902 | ); |
| 12903 | assert_eq!( |
| 12904 | case.reference_compilers, |
| 12905 | vec![ReferenceCompiler::Gfortran, ReferenceCompiler::FlangNew] |
| 12906 | ); |
| 12907 | assert!(matches!( |
| 12908 | status_for_opt(case, OptLevel::O0), |
| 12909 | EffectiveStatus::Normal |
| 12910 | )); |
| 12911 | assert!(matches!( |
| 12912 | status_for_opt(case, OptLevel::O1), |
| 12913 | EffectiveStatus::Xfail(_) |
| 12914 | )); |
| 12915 | let _ = fs::remove_file(&root); |
| 12916 | } |
| 12917 | |
| 12918 | #[test] |
| 12919 | fn parses_consistency_checks() { |
| 12920 | let root = std::env::temp_dir().join("afs_tests_consistency_spec.afs"); |
| 12921 | fs::write( |
| 12922 | &root, |
| 12923 | r#"suite "consistency/object" |
| 12924 | |
| 12925 | case "driver_paths" |
| 12926 | source "../../fixtures/backend/runtime_calls.f90" |
| 12927 | armfortas => asm, obj |
| 12928 | repeat => 5 |
| 12929 | consistency => cli_obj_vs_system_as, cli-obj-vs-system-as, cli_asm_reproducible, cli-obj-reproducible, cli_run_reproducible, capture_asm_vs_cli_asm, capture-obj-vs-cli-obj, capture_run_vs_cli_run, capture_asm_reproducible, capture-obj-reproducible, capture_run_reproducible |
| 12930 | expect obj contains "_main" |
| 12931 | end |
| 12932 | "#, |
| 12933 | ) |
| 12934 | .unwrap(); |
| 12935 | |
| 12936 | let suite = parse_suite_file(&root).unwrap(); |
| 12937 | let case = &suite.cases[0]; |
| 12938 | assert_eq!( |
| 12939 | case.consistency_checks, |
| 12940 | vec![ |
| 12941 | ConsistencyCheck::CliObjVsSystemAs, |
| 12942 | ConsistencyCheck::CliAsmReproducible, |
| 12943 | ConsistencyCheck::CliObjReproducible, |
| 12944 | ConsistencyCheck::CliRunReproducible, |
| 12945 | ConsistencyCheck::CaptureAsmVsCliAsm, |
| 12946 | ConsistencyCheck::CaptureObjVsCliObj, |
| 12947 | ConsistencyCheck::CaptureRunVsCliRun, |
| 12948 | ConsistencyCheck::CaptureAsmReproducible, |
| 12949 | ConsistencyCheck::CaptureObjReproducible, |
| 12950 | ConsistencyCheck::CaptureRunReproducible, |
| 12951 | ] |
| 12952 | ); |
| 12953 | assert_eq!(case.repeat_count, 5); |
| 12954 | let _ = fs::remove_file(&root); |
| 12955 | } |
| 12956 | |
| 12957 | #[test] |
| 12958 | fn parses_graph_case() { |
| 12959 | let root = std::env::temp_dir().join("afs_tests_graph_spec"); |
| 12960 | let _ = fs::remove_dir_all(&root); |
| 12961 | fs::create_dir_all(&root).unwrap(); |
| 12962 | fs::write( |
| 12963 | root.join("math_values.f90"), |
| 12964 | "module math_values\nend module\n", |
| 12965 | ) |
| 12966 | .unwrap(); |
| 12967 | fs::write(root.join("main.f90"), "program main\nend program\n").unwrap(); |
| 12968 | fs::write( |
| 12969 | root.join("graph.afs"), |
| 12970 | r#"suite "modules/graph" |
| 12971 | |
| 12972 | case "basic_use" |
| 12973 | entry "main.f90" |
| 12974 | file "math_values.f90" |
| 12975 | file "main.f90" |
| 12976 | armfortas => run |
| 12977 | expect run.exit_code equals 0 |
| 12978 | end |
| 12979 | "#, |
| 12980 | ) |
| 12981 | .unwrap(); |
| 12982 | |
| 12983 | let suite = parse_suite_file(&root.join("graph.afs")).unwrap(); |
| 12984 | let case = &suite.cases[0]; |
| 12985 | assert_eq!(case.source, root.join("main.f90")); |
| 12986 | assert_eq!( |
| 12987 | case.graph_files, |
| 12988 | vec![root.join("math_values.f90"), root.join("main.f90")] |
| 12989 | ); |
| 12990 | |
| 12991 | let _ = fs::remove_dir_all(&root); |
| 12992 | } |
| 12993 | |
| 12994 | #[test] |
| 12995 | fn parse_cli_collects_tool_overrides() { |
| 12996 | let args = vec![ |
| 12997 | "run".to_string(), |
| 12998 | "--suite".to_string(), |
| 12999 | "consistency/runtime".to_string(), |
| 13000 | "--json-report".to_string(), |
| 13001 | "/tmp/report.json".to_string(), |
| 13002 | "--markdown-report".to_string(), |
| 13003 | "/tmp/report.md".to_string(), |
| 13004 | "--armfortas-bin".to_string(), |
| 13005 | "/tmp/armfortas".to_string(), |
| 13006 | "--gfortran-bin".to_string(), |
| 13007 | "/tmp/gfortran".to_string(), |
| 13008 | "--flang-bin".to_string(), |
| 13009 | "/tmp/flang-new".to_string(), |
| 13010 | "--lfortran-bin".to_string(), |
| 13011 | "/tmp/lfortran".to_string(), |
| 13012 | "--ifort-bin".to_string(), |
| 13013 | "/tmp/ifort".to_string(), |
| 13014 | "--ifx-bin".to_string(), |
| 13015 | "/tmp/ifx".to_string(), |
| 13016 | "--nvfortran-bin".to_string(), |
| 13017 | "/tmp/nvfortran".to_string(), |
| 13018 | "--as-bin".to_string(), |
| 13019 | "/tmp/as".to_string(), |
| 13020 | "--otool-bin".to_string(), |
| 13021 | "/tmp/otool".to_string(), |
| 13022 | "--nm-bin".to_string(), |
| 13023 | "/tmp/nm".to_string(), |
| 13024 | ]; |
| 13025 | |
| 13026 | let command = parse_cli(&args).unwrap(); |
| 13027 | let config = match command { |
| 13028 | CommandKind::Run(config) => config, |
| 13029 | other => panic!( |
| 13030 | "expected run command, got {:?}", |
| 13031 | std::mem::discriminant(&other) |
| 13032 | ), |
| 13033 | }; |
| 13034 | |
| 13035 | assert_eq!(config.suite_filter.as_deref(), Some("consistency/runtime")); |
| 13036 | assert_eq!( |
| 13037 | config.json_report.as_deref(), |
| 13038 | Some(Path::new("/tmp/report.json")) |
| 13039 | ); |
| 13040 | assert_eq!( |
| 13041 | config.markdown_report.as_deref(), |
| 13042 | Some(Path::new("/tmp/report.md")) |
| 13043 | ); |
| 13044 | assert_eq!( |
| 13045 | config.tools.armfortas, |
| 13046 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13047 | ); |
| 13048 | assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 13049 | assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 13050 | assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 13051 | assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 13052 | assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 13053 | assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 13054 | assert_eq!(config.tools.system_as, "/tmp/as"); |
| 13055 | assert_eq!(config.tools.otool, "/tmp/otool"); |
| 13056 | assert_eq!(config.tools.nm, "/tmp/nm"); |
| 13057 | } |
| 13058 | |
| 13059 | #[test] |
| 13060 | fn parse_cli_collects_list_config() { |
| 13061 | let args = vec![ |
| 13062 | "list".to_string(), |
| 13063 | "--suite".to_string(), |
| 13064 | "v2/generic".to_string(), |
| 13065 | "--verbose".to_string(), |
| 13066 | "--armfortas-bin".to_string(), |
| 13067 | "/tmp/armfortas".to_string(), |
| 13068 | ]; |
| 13069 | |
| 13070 | let command = parse_cli(&args).unwrap(); |
| 13071 | let config = match command { |
| 13072 | CommandKind::List(config) => config, |
| 13073 | other => panic!( |
| 13074 | "expected list command, got {:?}", |
| 13075 | std::mem::discriminant(&other) |
| 13076 | ), |
| 13077 | }; |
| 13078 | |
| 13079 | assert_eq!(config.suite_filter.as_deref(), Some("v2/generic")); |
| 13080 | assert!(config.verbose); |
| 13081 | assert_eq!( |
| 13082 | config.tools.armfortas, |
| 13083 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13084 | ); |
| 13085 | } |
| 13086 | |
| 13087 | #[test] |
| 13088 | fn parse_cli_collects_doctor_tool_overrides() { |
| 13089 | let args = vec![ |
| 13090 | "doctor".to_string(), |
| 13091 | "--json-report".to_string(), |
| 13092 | "/tmp/doctor.json".to_string(), |
| 13093 | "--markdown-report".to_string(), |
| 13094 | "/tmp/doctor.md".to_string(), |
| 13095 | "--armfortas-bin".to_string(), |
| 13096 | "/tmp/armfortas".to_string(), |
| 13097 | "--gfortran-bin".to_string(), |
| 13098 | "/tmp/gfortran".to_string(), |
| 13099 | "--flang-bin".to_string(), |
| 13100 | "/tmp/flang-new".to_string(), |
| 13101 | "--lfortran-bin".to_string(), |
| 13102 | "/tmp/lfortran".to_string(), |
| 13103 | "--ifort-bin".to_string(), |
| 13104 | "/tmp/ifort".to_string(), |
| 13105 | "--ifx-bin".to_string(), |
| 13106 | "/tmp/ifx".to_string(), |
| 13107 | "--nvfortran-bin".to_string(), |
| 13108 | "/tmp/nvfortran".to_string(), |
| 13109 | "--as-bin".to_string(), |
| 13110 | "/tmp/as".to_string(), |
| 13111 | "--otool-bin".to_string(), |
| 13112 | "/tmp/otool".to_string(), |
| 13113 | "--nm-bin".to_string(), |
| 13114 | "/tmp/nm".to_string(), |
| 13115 | ]; |
| 13116 | |
| 13117 | let command = parse_cli(&args).unwrap(); |
| 13118 | let config = match command { |
| 13119 | CommandKind::Doctor(config) => config, |
| 13120 | other => panic!( |
| 13121 | "expected doctor command, got {:?}", |
| 13122 | std::mem::discriminant(&other) |
| 13123 | ), |
| 13124 | }; |
| 13125 | |
| 13126 | assert_eq!( |
| 13127 | config.tools.armfortas, |
| 13128 | ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13129 | ); |
| 13130 | assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 13131 | assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 13132 | assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 13133 | assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 13134 | assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 13135 | assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 13136 | assert_eq!(config.tools.system_as, "/tmp/as"); |
| 13137 | assert_eq!(config.tools.otool, "/tmp/otool"); |
| 13138 | assert_eq!(config.tools.nm, "/tmp/nm"); |
| 13139 | assert_eq!( |
| 13140 | config.json_report.as_deref(), |
| 13141 | Some(Path::new("/tmp/doctor.json")) |
| 13142 | ); |
| 13143 | assert_eq!( |
| 13144 | config.markdown_report.as_deref(), |
| 13145 | Some(Path::new("/tmp/doctor.md")) |
| 13146 | ); |
| 13147 | } |
| 13148 | |
| 13149 | #[test] |
| 13150 | fn parse_cli_collects_compare_config() { |
| 13151 | let args = vec![ |
| 13152 | "compare".to_string(), |
| 13153 | "armfortas".to_string(), |
| 13154 | "/tmp/other-compiler".to_string(), |
| 13155 | "--program".to_string(), |
| 13156 | "/tmp/demo.f90".to_string(), |
| 13157 | "--opt".to_string(), |
| 13158 | "O2".to_string(), |
| 13159 | "--artifact".to_string(), |
| 13160 | "asm,obj,armfortas.ir".to_string(), |
| 13161 | "--json-report".to_string(), |
| 13162 | "/tmp/compare.json".to_string(), |
| 13163 | "--markdown-report".to_string(), |
| 13164 | "/tmp/compare.md".to_string(), |
| 13165 | ]; |
| 13166 | |
| 13167 | let command = parse_cli(&args).unwrap(); |
| 13168 | let config = match command { |
| 13169 | CommandKind::Compare(config) => config, |
| 13170 | other => panic!( |
| 13171 | "expected compare command, got {:?}", |
| 13172 | std::mem::discriminant(&other) |
| 13173 | ), |
| 13174 | }; |
| 13175 | |
| 13176 | assert_eq!(config.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 13177 | assert_eq!( |
| 13178 | config.right, |
| 13179 | CompilerSpec::Binary(PathBuf::from("/tmp/other-compiler")) |
| 13180 | ); |
| 13181 | assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 13182 | assert_eq!(config.opt_level, OptLevel::O2); |
| 13183 | assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 13184 | assert!(config.artifacts.contains(&ArtifactKey::Obj)); |
| 13185 | assert!(config |
| 13186 | .artifacts |
| 13187 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 13188 | assert_eq!( |
| 13189 | config.json_report.as_deref(), |
| 13190 | Some(Path::new("/tmp/compare.json")) |
| 13191 | ); |
| 13192 | assert_eq!( |
| 13193 | config.markdown_report.as_deref(), |
| 13194 | Some(Path::new("/tmp/compare.md")) |
| 13195 | ); |
| 13196 | } |
| 13197 | |
| 13198 | #[test] |
| 13199 | fn parse_cli_collects_introspect_config() { |
| 13200 | let args = vec![ |
| 13201 | "introspect".to_string(), |
| 13202 | "armfortas".to_string(), |
| 13203 | "/tmp/demo.f90".to_string(), |
| 13204 | "--artifact".to_string(), |
| 13205 | "armfortas.ir,asm".to_string(), |
| 13206 | "--all".to_string(), |
| 13207 | "--summary-only".to_string(), |
| 13208 | "--max-artifact-lines".to_string(), |
| 13209 | "12".to_string(), |
| 13210 | "--json-report".to_string(), |
| 13211 | "/tmp/introspect.json".to_string(), |
| 13212 | ]; |
| 13213 | |
| 13214 | let command = parse_cli(&args).unwrap(); |
| 13215 | let config = match command { |
| 13216 | CommandKind::Introspect(config) => config, |
| 13217 | other => panic!( |
| 13218 | "expected introspect command, got {:?}", |
| 13219 | std::mem::discriminant(&other) |
| 13220 | ), |
| 13221 | }; |
| 13222 | |
| 13223 | assert_eq!( |
| 13224 | config.compiler, |
| 13225 | CompilerSpec::Named(NamedCompiler::Armfortas) |
| 13226 | ); |
| 13227 | assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 13228 | assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 13229 | assert!(config |
| 13230 | .artifacts |
| 13231 | .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 13232 | assert!(config.all_artifacts); |
| 13233 | assert!(config.summary_only); |
| 13234 | assert_eq!(config.max_artifact_lines, Some(12)); |
| 13235 | assert_eq!( |
| 13236 | config.json_report.as_deref(), |
| 13237 | Some(Path::new("/tmp/introspect.json")) |
| 13238 | ); |
| 13239 | } |
| 13240 | |
| 13241 | #[test] |
| 13242 | fn parses_failure_expectation() { |
| 13243 | let root = std::env::temp_dir().join("afs_tests_failure_spec.afs"); |
| 13244 | fs::write( |
| 13245 | &root, |
| 13246 | r#"suite "frontend/parser" |
| 13247 | |
| 13248 | case "missing_then" |
| 13249 | source "../../fixtures/frontend/parser/missing_then.f90" |
| 13250 | armfortas => tokens |
| 13251 | expect tokens contains "if" |
| 13252 | expect-fail parser contains "expected" |
| 13253 | end |
| 13254 | "#, |
| 13255 | ) |
| 13256 | .unwrap(); |
| 13257 | |
| 13258 | let suite = parse_suite_file(&root).unwrap(); |
| 13259 | assert_eq!(suite.cases.len(), 1); |
| 13260 | assert!(has_failure_expectation(&suite.cases[0])); |
| 13261 | let _ = fs::remove_file(&root); |
| 13262 | } |
| 13263 | |
| 13264 | #[test] |
| 13265 | fn parses_failure_expectation_from_source_comments() { |
| 13266 | let root = std::env::temp_dir().join("afs_tests_failure_comment_spec"); |
| 13267 | let _ = fs::remove_dir_all(&root); |
| 13268 | fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13269 | fs::create_dir_all(root.join("suites")).unwrap(); |
| 13270 | |
| 13271 | let source = root.join("fixtures/error_expected.f90"); |
| 13272 | fs::write( |
| 13273 | &source, |
| 13274 | "! ERROR_EXPECTED: hidden\nprogram error_expected\n print *, hidden\nend program\n", |
| 13275 | ) |
| 13276 | .unwrap(); |
| 13277 | |
| 13278 | let suite_path = root.join("suites/spec.afs"); |
| 13279 | fs::write( |
| 13280 | &suite_path, |
| 13281 | r#"suite "v2/comment-failure" |
| 13282 | |
| 13283 | case "error_expected_comments" |
| 13284 | source "../fixtures/error_expected.f90" |
| 13285 | compiler armfortas => diagnostics |
| 13286 | expect-fail comments |
| 13287 | end |
| 13288 | "#, |
| 13289 | ) |
| 13290 | .unwrap(); |
| 13291 | |
| 13292 | let suite = parse_suite_file(&suite_path).unwrap(); |
| 13293 | match &suite.cases[0].expectations[0] { |
| 13294 | Expectation::FailCommentPatterns(patterns) => { |
| 13295 | assert_eq!(patterns, &vec!["hidden".to_string()]); |
| 13296 | } |
| 13297 | other => panic!("expected source-comment failure expectation, got {other:?}"), |
| 13298 | } |
| 13299 | |
| 13300 | let _ = fs::remove_dir_all(&root); |
| 13301 | } |
| 13302 | |
| 13303 | #[test] |
| 13304 | fn resolves_xfail_comments_from_source() { |
| 13305 | let root = std::env::temp_dir().join("afs_tests_xfail_comment_spec"); |
| 13306 | let _ = fs::remove_dir_all(&root); |
| 13307 | fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13308 | fs::create_dir_all(root.join("suites")).unwrap(); |
| 13309 | |
| 13310 | let source = root.join("fixtures/xfail_case.f90"); |
| 13311 | fs::write( |
| 13312 | &source, |
| 13313 | "! XFAIL: audit BLOCKING-1 (demo)\nprogram xfail_case\nend program\n", |
| 13314 | ) |
| 13315 | .unwrap(); |
| 13316 | |
| 13317 | let suite_path = root.join("suites/spec.afs"); |
| 13318 | fs::write( |
| 13319 | &suite_path, |
| 13320 | r#"suite "v2/comment-xfail" |
| 13321 | |
| 13322 | case "xfail_comments" |
| 13323 | source "../fixtures/xfail_case.f90" |
| 13324 | compiler armfortas => runtime |
| 13325 | xfail comments |
| 13326 | end |
| 13327 | "#, |
| 13328 | ) |
| 13329 | .unwrap(); |
| 13330 | |
| 13331 | let suite = parse_suite_file(&suite_path).unwrap(); |
| 13332 | match status_for_opt(&suite.cases[0], OptLevel::O0) { |
| 13333 | EffectiveStatus::Xfail(reason) => { |
| 13334 | assert_eq!(reason, "audit BLOCKING-1 (demo)"); |
| 13335 | } |
| 13336 | other => panic!("expected xfail status, got {other:?}"), |
| 13337 | } |
| 13338 | |
| 13339 | let _ = fs::remove_dir_all(&root); |
| 13340 | } |
| 13341 | |
| 13342 | #[test] |
| 13343 | fn check_matching_preserves_order() { |
| 13344 | let checks = vec![ |
| 13345 | Check { |
| 13346 | line_num: 1, |
| 13347 | pattern: "alpha".into(), |
| 13348 | negative: false, |
| 13349 | kind: "CHECK", |
| 13350 | }, |
| 13351 | Check { |
| 13352 | line_num: 2, |
| 13353 | pattern: "omega".into(), |
| 13354 | negative: false, |
| 13355 | kind: "CHECK", |
| 13356 | }, |
| 13357 | ]; |
| 13358 | assert!(match_checks(&checks, "alpha\nmiddle\nomega\n", "demo").is_ok()); |
| 13359 | assert!(match_checks(&checks, "omega\nalpha\n", "demo").is_err()); |
| 13360 | } |
| 13361 | |
| 13362 | #[test] |
| 13363 | fn ir_check_matching_supports_negative_patterns() { |
| 13364 | let checks = vec![ |
| 13365 | Check { |
| 13366 | line_num: 1, |
| 13367 | pattern: "func @demo".into(), |
| 13368 | negative: false, |
| 13369 | kind: "IR_CHECK", |
| 13370 | }, |
| 13371 | Check { |
| 13372 | line_num: 2, |
| 13373 | pattern: "zeroinit".into(), |
| 13374 | negative: true, |
| 13375 | kind: "IR_NOT", |
| 13376 | }, |
| 13377 | ]; |
| 13378 | let ok_ir = "module main\n\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13379 | assert!(match_checks(&checks, ok_ir, "demo").is_ok()); |
| 13380 | |
| 13381 | let bad_ir = "module main\n global @value: i32 = zeroinit\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13382 | let err = match_checks(&checks, bad_ir, "demo").unwrap_err(); |
| 13383 | assert!(err.contains("IR_NOT failed")); |
| 13384 | } |
| 13385 | |
| 13386 | fn run_only_result(stdout: &str, stderr: &str, exit_code: i32) -> CaptureResult { |
| 13387 | CaptureResult { |
| 13388 | input: PathBuf::from("demo.f90"), |
| 13389 | opt_level: OptLevel::O0, |
| 13390 | stages: std::collections::BTreeMap::from([( |
| 13391 | Stage::Run, |
| 13392 | CapturedStage::Run(RunCapture { |
| 13393 | exit_code, |
| 13394 | stdout: stdout.into(), |
| 13395 | stderr: stderr.into(), |
| 13396 | }), |
| 13397 | )]), |
| 13398 | } |
| 13399 | } |
| 13400 | |
| 13401 | fn reference_run( |
| 13402 | compiler: ReferenceCompiler, |
| 13403 | stdout: &str, |
| 13404 | stderr: &str, |
| 13405 | exit_code: i32, |
| 13406 | ) -> ReferenceResult { |
| 13407 | ReferenceResult { |
| 13408 | compiler, |
| 13409 | compile_command: format!("{} demo.f90 -o demo", compiler.as_str()), |
| 13410 | compile_exit_code: 0, |
| 13411 | compile_stdout: String::new(), |
| 13412 | compile_stderr: String::new(), |
| 13413 | run: Some(RunCapture { |
| 13414 | exit_code, |
| 13415 | stdout: stdout.into(), |
| 13416 | stderr: stderr.into(), |
| 13417 | }), |
| 13418 | run_error: None, |
| 13419 | } |
| 13420 | } |
| 13421 | |
| 13422 | fn differential_armfortas_observation( |
| 13423 | stdout: &str, |
| 13424 | stderr: &str, |
| 13425 | exit_code: i32, |
| 13426 | ) -> CompilerObservation { |
| 13427 | observed_program_from_armfortas_capture( |
| 13428 | Path::new("demo.f90"), |
| 13429 | OptLevel::O0, |
| 13430 | default_differential_artifacts(), |
| 13431 | &run_only_result(stdout, stderr, exit_code), |
| 13432 | None, |
| 13433 | ) |
| 13434 | .observation |
| 13435 | } |
| 13436 | |
| 13437 | fn differential_reference_observation( |
| 13438 | compiler: ReferenceCompiler, |
| 13439 | stdout: &str, |
| 13440 | stderr: &str, |
| 13441 | exit_code: i32, |
| 13442 | ) -> CompilerObservation { |
| 13443 | observed_program_from_reference_result( |
| 13444 | Path::new("demo.f90"), |
| 13445 | OptLevel::O0, |
| 13446 | default_differential_artifacts(), |
| 13447 | &reference_run(compiler, stdout, stderr, exit_code), |
| 13448 | ) |
| 13449 | .observation |
| 13450 | } |
| 13451 | |
| 13452 | #[test] |
| 13453 | fn not_contains_expectation_checks_text_absence() { |
| 13454 | let case = CaseSpec { |
| 13455 | name: "no_reserved_register".into(), |
| 13456 | source: PathBuf::from("demo.f90"), |
| 13457 | graph_files: Vec::new(), |
| 13458 | requested: BTreeSet::from([Stage::Asm]), |
| 13459 | generic_introspect: None, |
| 13460 | generic_compare: None, |
| 13461 | opt_levels: vec![OptLevel::O0], |
| 13462 | repeat_count: 2, |
| 13463 | reference_compilers: Vec::new(), |
| 13464 | consistency_checks: Vec::new(), |
| 13465 | expectations: vec![Expectation::NotContains { |
| 13466 | target: Target::Stage(Stage::Asm), |
| 13467 | needle: "x18".into(), |
| 13468 | }], |
| 13469 | status_rules: Vec::new(), |
| 13470 | capability_policy: None, |
| 13471 | }; |
| 13472 | let result = CaptureResult { |
| 13473 | input: PathBuf::from("demo.f90"), |
| 13474 | opt_level: OptLevel::O0, |
| 13475 | stages: std::collections::BTreeMap::from([( |
| 13476 | Stage::Asm, |
| 13477 | CapturedStage::Text("mov x19, x0\nret\n".into()), |
| 13478 | )]), |
| 13479 | }; |
| 13480 | let observed = observed_program_from_armfortas_capture( |
| 13481 | Path::new("demo.f90"), |
| 13482 | OptLevel::O0, |
| 13483 | expected_artifacts_for_legacy_case(&case), |
| 13484 | &result, |
| 13485 | None, |
| 13486 | ); |
| 13487 | assert!(evaluate_observation_expectations(&case, &observed).is_ok()); |
| 13488 | |
| 13489 | let bad = CaptureResult { |
| 13490 | input: PathBuf::from("demo.f90"), |
| 13491 | opt_level: OptLevel::O0, |
| 13492 | stages: std::collections::BTreeMap::from([( |
| 13493 | Stage::Asm, |
| 13494 | CapturedStage::Text("mov x18, x0\nret\n".into()), |
| 13495 | )]), |
| 13496 | }; |
| 13497 | let observed = observed_program_from_armfortas_capture( |
| 13498 | Path::new("demo.f90"), |
| 13499 | OptLevel::O0, |
| 13500 | expected_artifacts_for_legacy_case(&case), |
| 13501 | &bad, |
| 13502 | None, |
| 13503 | ); |
| 13504 | let err = evaluate_observation_expectations(&case, &observed).unwrap_err(); |
| 13505 | assert!(err.contains("expected asm to not contain")); |
| 13506 | } |
| 13507 | |
| 13508 | #[test] |
| 13509 | fn writes_failure_bundle_with_artifacts() { |
| 13510 | let source = std::env::temp_dir().join("afs_tests_bundle_source.f90"); |
| 13511 | fs::write(&source, "program hello\nprint *, 'hello'\nend program\n").unwrap(); |
| 13512 | |
| 13513 | let suite = SuiteSpec { |
| 13514 | name: "runtime/bundles".into(), |
| 13515 | path: PathBuf::from("/tmp/runtime/bundles.afs"), |
| 13516 | cases: Vec::new(), |
| 13517 | }; |
| 13518 | let case = CaseSpec { |
| 13519 | name: "hello_bundle".into(), |
| 13520 | source: source.clone(), |
| 13521 | graph_files: Vec::new(), |
| 13522 | requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13523 | generic_introspect: None, |
| 13524 | generic_compare: None, |
| 13525 | opt_levels: vec![OptLevel::O0], |
| 13526 | repeat_count: 3, |
| 13527 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 13528 | consistency_checks: vec![ConsistencyCheck::CliObjVsSystemAs], |
| 13529 | expectations: Vec::new(), |
| 13530 | status_rules: Vec::new(), |
| 13531 | capability_policy: None, |
| 13532 | }; |
| 13533 | let mut stages = std::collections::BTreeMap::new(); |
| 13534 | stages.insert(Stage::Ir, CapturedStage::Text("module main".into())); |
| 13535 | stages.insert( |
| 13536 | Stage::Run, |
| 13537 | CapturedStage::Run(RunCapture { |
| 13538 | exit_code: 1, |
| 13539 | stdout: "oops\n".into(), |
| 13540 | stderr: "broken\n".into(), |
| 13541 | }), |
| 13542 | ); |
| 13543 | let artifacts = ExecutionArtifacts { |
| 13544 | requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13545 | armfortas: None, |
| 13546 | armfortas_failure: Some(CaptureFailure { |
| 13547 | input: source.clone(), |
| 13548 | opt_level: OptLevel::O0, |
| 13549 | stage: FailureStage::Sema, |
| 13550 | detail: "compiler failed".into(), |
| 13551 | stages, |
| 13552 | }), |
| 13553 | armfortas_observation: Some(ObservedProgram { |
| 13554 | observation: CompilerObservation { |
| 13555 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13556 | program: source.clone(), |
| 13557 | opt_level: OptLevel::O0, |
| 13558 | compile_exit_code: 1, |
| 13559 | artifacts: BTreeMap::from([ |
| 13560 | ( |
| 13561 | ArtifactKey::Diagnostics, |
| 13562 | ArtifactValue::Text("cached observation failure".into()), |
| 13563 | ), |
| 13564 | ( |
| 13565 | ArtifactKey::Extra("armfortas.ast".into()), |
| 13566 | ArtifactValue::Text("program hello".into()), |
| 13567 | ), |
| 13568 | ]), |
| 13569 | provenance: ObservationProvenance { |
| 13570 | compiler_identity: "armfortas".into(), |
| 13571 | adapter_kind: "named".into(), |
| 13572 | backend_mode: "linked".into(), |
| 13573 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13574 | artifacts_captured: vec!["diagnostics".into(), "armfortas.ast".into()], |
| 13575 | comparison_basis: None, |
| 13576 | failure_stage: Some("sema".into()), |
| 13577 | }, |
| 13578 | }, |
| 13579 | requested_artifacts: BTreeSet::from([ |
| 13580 | ArtifactKey::Diagnostics, |
| 13581 | ArtifactKey::Extra("armfortas.ast".into()), |
| 13582 | ]), |
| 13583 | }), |
| 13584 | references: vec![ReferenceResult { |
| 13585 | compiler: ReferenceCompiler::Gfortran, |
| 13586 | compile_command: "gfortran hello.f90 -o hello".into(), |
| 13587 | compile_exit_code: 0, |
| 13588 | compile_stdout: String::new(), |
| 13589 | compile_stderr: String::new(), |
| 13590 | run: Some(RunCapture { |
| 13591 | exit_code: 0, |
| 13592 | stdout: "hello\n".into(), |
| 13593 | stderr: String::new(), |
| 13594 | }), |
| 13595 | run_error: None, |
| 13596 | }], |
| 13597 | reference_observations: vec![ObservedProgram { |
| 13598 | observation: CompilerObservation { |
| 13599 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 13600 | program: source.clone(), |
| 13601 | opt_level: OptLevel::O0, |
| 13602 | compile_exit_code: 0, |
| 13603 | artifacts: BTreeMap::from([( |
| 13604 | ArtifactKey::Asm, |
| 13605 | ArtifactValue::Text(".globl _main".into()), |
| 13606 | )]), |
| 13607 | provenance: ObservationProvenance { |
| 13608 | compiler_identity: "gfortran".into(), |
| 13609 | adapter_kind: "named".into(), |
| 13610 | backend_mode: "legacy-reference".into(), |
| 13611 | backend_detail: "cached reference observation".into(), |
| 13612 | artifacts_captured: vec!["asm".into()], |
| 13613 | comparison_basis: None, |
| 13614 | failure_stage: None, |
| 13615 | }, |
| 13616 | }, |
| 13617 | requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 13618 | }], |
| 13619 | consistency_issues: { |
| 13620 | let asm_temp_root = |
| 13621 | std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm"); |
| 13622 | fs::create_dir_all(&asm_temp_root).unwrap(); |
| 13623 | fs::write(asm_temp_root.join("run_00.s"), "mov x19, x0\n").unwrap(); |
| 13624 | |
| 13625 | let obj_temp_root = |
| 13626 | std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj"); |
| 13627 | fs::create_dir_all(&obj_temp_root).unwrap(); |
| 13628 | fs::write(obj_temp_root.join("run_00.o"), "fake object bytes\n").unwrap(); |
| 13629 | |
| 13630 | vec![ |
| 13631 | ConsistencyIssue { |
| 13632 | check: ConsistencyCheck::CliAsmReproducible, |
| 13633 | summary: "repeat_count=3 unique_variants=3".into(), |
| 13634 | repeat_count: Some(3), |
| 13635 | unique_variant_count: Some(3), |
| 13636 | varying_components: Vec::new(), |
| 13637 | stable_components: Vec::new(), |
| 13638 | detail: "assembly output is not reproducible".into(), |
| 13639 | temp_root: asm_temp_root, |
| 13640 | }, |
| 13641 | ConsistencyIssue { |
| 13642 | check: ConsistencyCheck::CliObjReproducible, |
| 13643 | summary: "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols".into(), |
| 13644 | repeat_count: Some(3), |
| 13645 | unique_variant_count: Some(2), |
| 13646 | varying_components: vec!["text".into()], |
| 13647 | stable_components: vec![ |
| 13648 | "load_commands".into(), |
| 13649 | "relocations".into(), |
| 13650 | "symbols".into(), |
| 13651 | ], |
| 13652 | detail: "object output is not reproducible".into(), |
| 13653 | temp_root: obj_temp_root, |
| 13654 | }, |
| 13655 | ] |
| 13656 | }, |
| 13657 | }; |
| 13658 | let outcome = Outcome { |
| 13659 | suite: suite.name.clone(), |
| 13660 | case: case.name.clone(), |
| 13661 | opt_level: OptLevel::O0, |
| 13662 | kind: OutcomeKind::Fail, |
| 13663 | detail: "boom".into(), |
| 13664 | bundle: None, |
| 13665 | primary_backend: Some(PrimaryBackendReport { |
| 13666 | kind: "full".into(), |
| 13667 | mode: "linked".into(), |
| 13668 | detail: "linked armfortas::testing capture adapter".into(), |
| 13669 | }), |
| 13670 | consistency_observations: Vec::new(), |
| 13671 | }; |
| 13672 | let prepared = PreparedInput { |
| 13673 | compiler_source: source.clone(), |
| 13674 | generated_source: None, |
| 13675 | temp_root: None, |
| 13676 | }; |
| 13677 | |
| 13678 | let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13679 | assert!(bundle.join("metadata.txt").exists()); |
| 13680 | assert!(bundle.join("detail.txt").exists()); |
| 13681 | assert!(bundle.join("source.f90").exists()); |
| 13682 | assert!(bundle.join("armfortas").join("ir.txt").exists()); |
| 13683 | assert!(bundle.join("armfortas").join("metadata.txt").exists()); |
| 13684 | assert!(bundle.join("armfortas").join("observation.txt").exists()); |
| 13685 | assert!(bundle.join("armfortas").join("observation.json").exists()); |
| 13686 | assert!(bundle.join("armfortas").join("observation.md").exists()); |
| 13687 | assert!(bundle.join("armfortas").join("run.stdout.txt").exists()); |
| 13688 | assert!(bundle.join("armfortas").join("error.txt").exists()); |
| 13689 | assert!(bundle |
| 13690 | .join("references") |
| 13691 | .join("gfortran") |
| 13692 | .join("observation.txt") |
| 13693 | .exists()); |
| 13694 | assert!(bundle |
| 13695 | .join("references") |
| 13696 | .join("gfortran") |
| 13697 | .join("observation.json") |
| 13698 | .exists()); |
| 13699 | assert!(bundle |
| 13700 | .join("references") |
| 13701 | .join("gfortran") |
| 13702 | .join("observation.md") |
| 13703 | .exists()); |
| 13704 | assert!(bundle.join("references").join("summary.txt").exists()); |
| 13705 | assert!(bundle |
| 13706 | .join("references") |
| 13707 | .join("gfortran") |
| 13708 | .join("run.stdout.txt") |
| 13709 | .exists()); |
| 13710 | assert!(bundle.join("consistency").join("summary.txt").exists()); |
| 13711 | let metadata = fs::read_to_string(bundle.join("metadata.txt")).unwrap(); |
| 13712 | assert!(metadata.contains("primary_backend_kind: full")); |
| 13713 | assert!(metadata.contains("primary_backend_mode: linked")); |
| 13714 | assert!( |
| 13715 | metadata.contains("primary_backend_detail: linked armfortas::testing capture adapter") |
| 13716 | ); |
| 13717 | let armfortas_metadata = |
| 13718 | fs::read_to_string(bundle.join("armfortas").join("metadata.txt")).unwrap(); |
| 13719 | assert!(armfortas_metadata.contains("primary_backend_kind: full")); |
| 13720 | assert!(armfortas_metadata.contains("primary_backend_mode: linked")); |
| 13721 | assert!(armfortas_metadata |
| 13722 | .contains("primary_backend_detail: linked armfortas::testing capture adapter")); |
| 13723 | assert!(armfortas_metadata.contains("captured_stages: ir, run")); |
| 13724 | assert!(armfortas_metadata.contains("error_stage: sema")); |
| 13725 | let observation = |
| 13726 | fs::read_to_string(bundle.join("armfortas").join("observation.txt")).unwrap(); |
| 13727 | assert!(observation.contains("Introspect")); |
| 13728 | assert!(observation.contains("compiler: armfortas")); |
| 13729 | assert!(observation.contains("failure_stage: sema")); |
| 13730 | assert!(observation.contains("generic_artifacts: diagnostics")); |
| 13731 | assert!(observation.contains("adapter_extras: armfortas(ast)")); |
| 13732 | assert!(observation.contains("cached observation failure")); |
| 13733 | let reference_observation = fs::read_to_string( |
| 13734 | bundle |
| 13735 | .join("references") |
| 13736 | .join("gfortran") |
| 13737 | .join("observation.txt"), |
| 13738 | ) |
| 13739 | .unwrap(); |
| 13740 | assert!(reference_observation.contains("Introspect")); |
| 13741 | assert!(reference_observation.contains("compiler: gfortran")); |
| 13742 | assert!(reference_observation.contains("status: compile ok")); |
| 13743 | assert!(reference_observation.contains("requested_artifacts: asm")); |
| 13744 | assert!(reference_observation.contains("generic_artifacts: asm")); |
| 13745 | assert!(reference_observation.contains("cached reference observation")); |
| 13746 | let reference_summary = |
| 13747 | fs::read_to_string(bundle.join("references").join("summary.txt")).unwrap(); |
| 13748 | assert!(reference_summary.contains("reference_count: 1")); |
| 13749 | assert!(reference_summary.contains("compilers: gfortran")); |
| 13750 | assert!(reference_summary.contains("compiler: gfortran")); |
| 13751 | assert!(reference_summary.contains("status: compile ok")); |
| 13752 | assert!(reference_summary.contains("compile_exit_code: 0")); |
| 13753 | assert!(reference_summary.contains("command: gfortran hello.f90 -o hello")); |
| 13754 | assert!(reference_summary.contains("generic_artifacts: asm")); |
| 13755 | assert!(reference_summary.contains("adapter_extras: none")); |
| 13756 | let consistency_summary = |
| 13757 | fs::read_to_string(bundle.join("consistency").join("summary.txt")).unwrap(); |
| 13758 | assert!(consistency_summary.contains("issue_count: 2")); |
| 13759 | assert!(consistency_summary.contains("checks: cli_asm_reproducible, cli_obj_reproducible")); |
| 13760 | assert!(consistency_summary.contains("repeat_counts: 3")); |
| 13761 | assert!(consistency_summary.contains("unique_variants: 2, 3")); |
| 13762 | assert!(consistency_summary.contains("varying_components: text")); |
| 13763 | assert!( |
| 13764 | consistency_summary.contains("stable_components: load_commands, relocations, symbols") |
| 13765 | ); |
| 13766 | assert!(bundle |
| 13767 | .join("consistency") |
| 13768 | .join("cli_asm_reproducible") |
| 13769 | .join("summary.txt") |
| 13770 | .exists()); |
| 13771 | assert!(bundle |
| 13772 | .join("consistency") |
| 13773 | .join("cli_asm_reproducible") |
| 13774 | .join("detail.txt") |
| 13775 | .exists()); |
| 13776 | assert!(bundle |
| 13777 | .join("consistency") |
| 13778 | .join("cli_asm_reproducible") |
| 13779 | .join("artifacts") |
| 13780 | .join("run_00.s") |
| 13781 | .exists()); |
| 13782 | assert!(bundle |
| 13783 | .join("consistency") |
| 13784 | .join("cli_obj_reproducible") |
| 13785 | .join("artifacts") |
| 13786 | .join("run_00.o") |
| 13787 | .exists()); |
| 13788 | |
| 13789 | let _ = fs::remove_dir_all(bundle); |
| 13790 | let _ = |
| 13791 | fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm")); |
| 13792 | let _ = |
| 13793 | fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj")); |
| 13794 | let _ = fs::remove_file(source); |
| 13795 | } |
| 13796 | |
| 13797 | #[test] |
| 13798 | fn materializes_graph_input_in_declared_file_order() { |
| 13799 | let root = std::env::temp_dir().join("afs_tests_graph_materialize"); |
| 13800 | let _ = fs::remove_dir_all(&root); |
| 13801 | fs::create_dir_all(&root).unwrap(); |
| 13802 | let module = root.join("math_values.f90"); |
| 13803 | let main = root.join("main.f90"); |
| 13804 | fs::write(&module, "module math_values\ncontains\nend module\n").unwrap(); |
| 13805 | fs::write(&main, "program main\nuse math_values\nend program\n").unwrap(); |
| 13806 | |
| 13807 | let suite = SuiteSpec { |
| 13808 | name: "modules/graph".into(), |
| 13809 | path: root.join("graph.afs"), |
| 13810 | cases: Vec::new(), |
| 13811 | }; |
| 13812 | let case = CaseSpec { |
| 13813 | name: "basic_use".into(), |
| 13814 | source: main.clone(), |
| 13815 | graph_files: vec![module.clone(), main.clone()], |
| 13816 | requested: BTreeSet::from([Stage::Run]), |
| 13817 | generic_introspect: None, |
| 13818 | generic_compare: None, |
| 13819 | opt_levels: vec![OptLevel::O0], |
| 13820 | repeat_count: 2, |
| 13821 | reference_compilers: Vec::new(), |
| 13822 | consistency_checks: Vec::new(), |
| 13823 | expectations: Vec::new(), |
| 13824 | status_rules: Vec::new(), |
| 13825 | capability_policy: None, |
| 13826 | }; |
| 13827 | |
| 13828 | let prepared = prepare_case_input(&case, &suite, OptLevel::O0).unwrap(); |
| 13829 | let generated = fs::read_to_string(&prepared.compiler_source).unwrap(); |
| 13830 | assert!(generated.contains("module math_values")); |
| 13831 | assert!(generated.contains("program main")); |
| 13832 | assert!( |
| 13833 | generated.find("module math_values").unwrap() < generated.find("program main").unwrap() |
| 13834 | ); |
| 13835 | |
| 13836 | cleanup_prepared_input(&prepared); |
| 13837 | let _ = fs::remove_dir_all(&root); |
| 13838 | } |
| 13839 | |
| 13840 | #[test] |
| 13841 | fn graph_failure_bundle_writes_authored_sources() { |
| 13842 | let root = std::env::temp_dir().join("afs_tests_graph_bundle"); |
| 13843 | let _ = fs::remove_dir_all(&root); |
| 13844 | fs::create_dir_all(&root).unwrap(); |
| 13845 | let module = root.join("math_values.f90"); |
| 13846 | let main = root.join("main.f90"); |
| 13847 | let generated = root.join("generated.f90"); |
| 13848 | fs::write( |
| 13849 | &module, |
| 13850 | "module math_values\n integer :: answer = 42\nend module\n", |
| 13851 | ) |
| 13852 | .unwrap(); |
| 13853 | fs::write( |
| 13854 | &main, |
| 13855 | "program main\n use math_values\n print *, answer\nend program\n", |
| 13856 | ) |
| 13857 | .unwrap(); |
| 13858 | fs::write(&generated, "module math_values\n integer :: answer = 42\nend module\n\nprogram main\n use math_values\n print *, answer\nend program\n").unwrap(); |
| 13859 | |
| 13860 | let suite = SuiteSpec { |
| 13861 | name: "modules/bundles".into(), |
| 13862 | path: root.join("bundle.afs"), |
| 13863 | cases: Vec::new(), |
| 13864 | }; |
| 13865 | let case = CaseSpec { |
| 13866 | name: "graph_bundle".into(), |
| 13867 | source: main.clone(), |
| 13868 | graph_files: vec![module.clone(), main.clone()], |
| 13869 | requested: BTreeSet::from([Stage::Run]), |
| 13870 | generic_introspect: None, |
| 13871 | generic_compare: None, |
| 13872 | opt_levels: vec![OptLevel::O0], |
| 13873 | repeat_count: 2, |
| 13874 | reference_compilers: Vec::new(), |
| 13875 | consistency_checks: Vec::new(), |
| 13876 | expectations: Vec::new(), |
| 13877 | status_rules: Vec::new(), |
| 13878 | capability_policy: None, |
| 13879 | }; |
| 13880 | let outcome = Outcome { |
| 13881 | suite: suite.name.clone(), |
| 13882 | case: case.name.clone(), |
| 13883 | opt_level: OptLevel::O0, |
| 13884 | kind: OutcomeKind::Fail, |
| 13885 | detail: "boom".into(), |
| 13886 | bundle: None, |
| 13887 | primary_backend: Some(PrimaryBackendReport { |
| 13888 | kind: "full".into(), |
| 13889 | mode: "linked".into(), |
| 13890 | detail: "linked armfortas::testing capture adapter".into(), |
| 13891 | }), |
| 13892 | consistency_observations: Vec::new(), |
| 13893 | }; |
| 13894 | let artifacts = ExecutionArtifacts { |
| 13895 | requested: BTreeSet::from([Stage::Run]), |
| 13896 | armfortas: Some(run_only_result("42\n", "", 0)), |
| 13897 | armfortas_failure: None, |
| 13898 | armfortas_observation: None, |
| 13899 | references: Vec::new(), |
| 13900 | reference_observations: Vec::new(), |
| 13901 | consistency_issues: Vec::new(), |
| 13902 | }; |
| 13903 | let prepared = PreparedInput { |
| 13904 | compiler_source: generated.clone(), |
| 13905 | generated_source: Some(generated.clone()), |
| 13906 | temp_root: None, |
| 13907 | }; |
| 13908 | |
| 13909 | let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13910 | assert!(bundle.join("source.f90").exists()); |
| 13911 | assert!(bundle.join("sources").join("00_math_values.f90").exists()); |
| 13912 | assert!(bundle.join("sources").join("01_main.f90").exists()); |
| 13913 | |
| 13914 | let _ = fs::remove_dir_all(bundle); |
| 13915 | let _ = fs::remove_dir_all(&root); |
| 13916 | } |
| 13917 | |
| 13918 | #[test] |
| 13919 | fn armfortas_bundle_observation_prefers_cached_observation() { |
| 13920 | let prepared = PreparedInput { |
| 13921 | compiler_source: PathBuf::from("demo.f90"), |
| 13922 | generated_source: None, |
| 13923 | temp_root: None, |
| 13924 | }; |
| 13925 | let artifacts = ExecutionArtifacts { |
| 13926 | requested: BTreeSet::from([Stage::Run]), |
| 13927 | armfortas: Some(run_only_result("42\n", "", 0)), |
| 13928 | armfortas_failure: None, |
| 13929 | armfortas_observation: Some(ObservedProgram { |
| 13930 | observation: CompilerObservation { |
| 13931 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13932 | program: PathBuf::from("demo.f90"), |
| 13933 | opt_level: OptLevel::O0, |
| 13934 | compile_exit_code: 0, |
| 13935 | artifacts: BTreeMap::from([( |
| 13936 | ArtifactKey::Extra("armfortas.sema".into()), |
| 13937 | ArtifactValue::Text("ok".into()), |
| 13938 | )]), |
| 13939 | provenance: ObservationProvenance { |
| 13940 | compiler_identity: "armfortas".into(), |
| 13941 | adapter_kind: "named".into(), |
| 13942 | backend_mode: "linked".into(), |
| 13943 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13944 | artifacts_captured: vec!["armfortas.sema".into()], |
| 13945 | comparison_basis: None, |
| 13946 | failure_stage: None, |
| 13947 | }, |
| 13948 | }, |
| 13949 | requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.sema".into())]), |
| 13950 | }), |
| 13951 | references: Vec::new(), |
| 13952 | reference_observations: Vec::new(), |
| 13953 | consistency_issues: Vec::new(), |
| 13954 | }; |
| 13955 | |
| 13956 | let observed = observed_program_for_armfortas_bundle(&prepared, &artifacts).unwrap(); |
| 13957 | assert!(observed |
| 13958 | .observation |
| 13959 | .artifacts |
| 13960 | .contains_key(&ArtifactKey::Extra("armfortas.sema".into()))); |
| 13961 | assert!(!observed |
| 13962 | .observation |
| 13963 | .artifacts |
| 13964 | .contains_key(&ArtifactKey::Runtime)); |
| 13965 | } |
| 13966 | |
| 13967 | #[test] |
| 13968 | fn render_summary_includes_consistency_rollups() { |
| 13969 | let mut summary = Summary::default(); |
| 13970 | summary.record_consistency(&[ |
| 13971 | ConsistencyObservation { |
| 13972 | check: ConsistencyCheck::CliAsmReproducible, |
| 13973 | summary: "repeat_count=3 unique_variants=3".into(), |
| 13974 | repeat_count: Some(3), |
| 13975 | unique_variant_count: Some(3), |
| 13976 | varying_components: Vec::new(), |
| 13977 | stable_components: Vec::new(), |
| 13978 | }, |
| 13979 | ConsistencyObservation { |
| 13980 | check: ConsistencyCheck::CliObjReproducible, |
| 13981 | summary: |
| 13982 | "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols" |
| 13983 | .into(), |
| 13984 | repeat_count: Some(3), |
| 13985 | unique_variant_count: Some(2), |
| 13986 | varying_components: vec!["text".into()], |
| 13987 | stable_components: vec![ |
| 13988 | "load_commands".into(), |
| 13989 | "relocations".into(), |
| 13990 | "symbols".into(), |
| 13991 | ], |
| 13992 | }, |
| 13993 | ]); |
| 13994 | |
| 13995 | let rendered = render_summary(&summary); |
| 13996 | assert!(rendered.contains("Consistency")); |
| 13997 | assert!(rendered.contains("affected_checks: 2")); |
| 13998 | assert!(rendered.contains("cells_with_issues: 2")); |
| 13999 | assert!( |
| 14000 | rendered.contains("cli_asm_reproducible: 1 cells; repeat_count=3; unique_variants=3") |
| 14001 | ); |
| 14002 | assert!(rendered.contains( |
| 14003 | "cli_obj_reproducible: 1 cells; repeat_count=3; unique_variants=2; varying=text; stable=load_commands, relocations, symbols" |
| 14004 | )); |
| 14005 | } |
| 14006 | |
| 14007 | #[test] |
| 14008 | fn render_reports_include_outcomes() { |
| 14009 | let mut summary = Summary::default(); |
| 14010 | summary.record_outcome(&Outcome { |
| 14011 | suite: "modules/runtime-graphs".into(), |
| 14012 | case: "module_chain_runtime".into(), |
| 14013 | opt_level: OptLevel::O0, |
| 14014 | kind: OutcomeKind::Xfail, |
| 14015 | detail: "expected 42, got 0".into(), |
| 14016 | bundle: Some(PathBuf::from("/tmp/bundle")), |
| 14017 | primary_backend: Some(PrimaryBackendReport { |
| 14018 | kind: "observable".into(), |
| 14019 | mode: "cli-observable".into(), |
| 14020 | detail: "cli-observable armfortas driver capture adapter".into(), |
| 14021 | }), |
| 14022 | consistency_observations: vec![ConsistencyObservation { |
| 14023 | check: ConsistencyCheck::CliRunReproducible, |
| 14024 | summary: "repeat_count=3 unique_variants=1".into(), |
| 14025 | repeat_count: Some(3), |
| 14026 | unique_variant_count: Some(1), |
| 14027 | varying_components: Vec::new(), |
| 14028 | stable_components: vec!["exit_code".into(), "stdout".into(), "stderr".into()], |
| 14029 | }], |
| 14030 | }); |
| 14031 | |
| 14032 | let json = render_json_report(&summary); |
| 14033 | assert!(json.contains("\"outcomes\": [")); |
| 14034 | assert!(json.contains("\"suite\": \"modules/runtime-graphs\"")); |
| 14035 | assert!(json.contains("\"bundle\": \"/tmp/bundle\"")); |
| 14036 | assert!(json.contains("\"primary_backend\": {")); |
| 14037 | assert!(json.contains("\"mode\": \"cli-observable\"")); |
| 14038 | |
| 14039 | let markdown = render_markdown_report(&summary); |
| 14040 | assert!(markdown.contains("# afs-tests report")); |
| 14041 | assert!(markdown |
| 14042 | .contains("### `modules/runtime-graphs` / `module_chain_runtime` / `O0` / `xfail`")); |
| 14043 | assert!(markdown.contains("primary_backend: `observable` (`cli-observable`)")); |
| 14044 | assert!(markdown.contains("bundle: `/tmp/bundle`")); |
| 14045 | assert!(markdown.contains("expected 42, got 0")); |
| 14046 | } |
| 14047 | |
| 14048 | #[test] |
| 14049 | fn render_generic_reports_include_provenance() { |
| 14050 | let observation = CompilerObservation { |
| 14051 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14052 | program: PathBuf::from("demo.f90"), |
| 14053 | opt_level: OptLevel::O0, |
| 14054 | compile_exit_code: 0, |
| 14055 | artifacts: BTreeMap::from([ |
| 14056 | ( |
| 14057 | ArtifactKey::Asm, |
| 14058 | ArtifactValue::Text(".globl _main\n".into()), |
| 14059 | ), |
| 14060 | ( |
| 14061 | ArtifactKey::Extra("armfortas.ir".into()), |
| 14062 | ArtifactValue::Text("module main".into()), |
| 14063 | ), |
| 14064 | ]), |
| 14065 | provenance: ObservationProvenance { |
| 14066 | compiler_identity: "armfortas".into(), |
| 14067 | adapter_kind: "named".into(), |
| 14068 | backend_mode: "linked".into(), |
| 14069 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14070 | artifacts_captured: vec!["asm".into(), "armfortas.ir".into()], |
| 14071 | comparison_basis: None, |
| 14072 | failure_stage: None, |
| 14073 | }, |
| 14074 | }; |
| 14075 | let compare = ComparisonResult { |
| 14076 | left: observation.clone(), |
| 14077 | right: CompilerObservation { |
| 14078 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14079 | program: PathBuf::from("demo.f90"), |
| 14080 | opt_level: OptLevel::O0, |
| 14081 | compile_exit_code: 0, |
| 14082 | artifacts: BTreeMap::from([( |
| 14083 | ArtifactKey::Asm, |
| 14084 | ArtifactValue::Text(".arch armv8.5-a".into()), |
| 14085 | )]), |
| 14086 | provenance: ObservationProvenance { |
| 14087 | compiler_identity: "gfortran".into(), |
| 14088 | adapter_kind: "named".into(), |
| 14089 | backend_mode: "external-driver".into(), |
| 14090 | backend_detail: "generic external driver adapter using gfortran".into(), |
| 14091 | artifacts_captured: vec!["asm".into()], |
| 14092 | comparison_basis: Some("compile-status, diagnostics, runtime, asm".into()), |
| 14093 | failure_stage: None, |
| 14094 | }, |
| 14095 | }, |
| 14096 | basis: "compile-status, diagnostics, runtime, asm".into(), |
| 14097 | differences: vec![ArtifactDifference { |
| 14098 | artifact: "asm".into(), |
| 14099 | detail: "first differing line: 1".into(), |
| 14100 | }], |
| 14101 | }; |
| 14102 | |
| 14103 | let observed = ObservedProgram { |
| 14104 | observation: observation.clone(), |
| 14105 | requested_artifacts: BTreeSet::from([ |
| 14106 | ArtifactKey::Asm, |
| 14107 | ArtifactKey::Extra("armfortas.ir".into()), |
| 14108 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 14109 | ]), |
| 14110 | }; |
| 14111 | |
| 14112 | let introspection_text = |
| 14113 | render_introspection_text(&observed, full_introspection_render_config()); |
| 14114 | assert!(introspection_text.contains("status: compile ok")); |
| 14115 | assert!(introspection_text.contains("artifact_count: 2")); |
| 14116 | assert!( |
| 14117 | introspection_text.contains("requested_artifacts: asm, armfortas.ir, armfortas.tokens") |
| 14118 | ); |
| 14119 | assert!(introspection_text.contains("missing_artifacts: armfortas.tokens")); |
| 14120 | assert!(introspection_text.contains("generic_artifacts: asm")); |
| 14121 | assert!(introspection_text.contains("adapter_extras: armfortas(ir)")); |
| 14122 | assert!(introspection_text.contains("Generic artifacts")); |
| 14123 | assert!(introspection_text.contains("Adapter extras")); |
| 14124 | |
| 14125 | let introspection_json = render_introspection_json(&observed); |
| 14126 | assert!(introspection_json.contains("\"status\": \"compile ok\"")); |
| 14127 | assert!(introspection_json.contains("\"artifact_count\": 2")); |
| 14128 | assert!(introspection_json.contains( |
| 14129 | "\"requested_artifacts\": [\"asm\", \"armfortas.ir\", \"armfortas.tokens\"]" |
| 14130 | )); |
| 14131 | assert!(introspection_json.contains("\"missing_artifacts\": [\"armfortas.tokens\"]")); |
| 14132 | assert!(introspection_json.contains("\"artifact_summaries\":")); |
| 14133 | assert!(introspection_json.contains("\"asm\": {\"kind\":\"text\"")); |
| 14134 | assert!(introspection_json.contains("\"line_count\":1")); |
| 14135 | assert!(introspection_json.contains("\"generic_artifacts\": [\"asm\"]")); |
| 14136 | assert!(introspection_json.contains("\"adapter_extras\": {\"armfortas\": [\"ir\"]}")); |
| 14137 | assert!(introspection_json.contains("\"backend_mode\": \"linked\"")); |
| 14138 | assert!(introspection_json.contains("\"armfortas.ir\"")); |
| 14139 | |
| 14140 | let introspection_markdown = |
| 14141 | render_introspection_markdown(&observed, full_introspection_render_config()); |
| 14142 | assert!(introspection_markdown.contains("# bencch introspect report")); |
| 14143 | assert!(introspection_markdown.contains("status: compile ok")); |
| 14144 | assert!(introspection_markdown.contains("failure_stage: `none`")); |
| 14145 | assert!(introspection_markdown.contains("artifact_count: 2")); |
| 14146 | assert!(introspection_markdown |
| 14147 | .contains("requested_artifacts: `asm`, `armfortas.ir`, `armfortas.tokens`")); |
| 14148 | assert!(introspection_markdown.contains("missing_artifacts: `armfortas.tokens`")); |
| 14149 | assert!(introspection_markdown.contains("## Generic artifacts")); |
| 14150 | assert!(introspection_markdown.contains("## Adapter extras")); |
| 14151 | assert!(introspection_markdown.contains("### `armfortas`")); |
| 14152 | assert!(introspection_markdown.contains("#### `ir`")); |
| 14153 | |
| 14154 | let compare_markdown = render_compare_markdown(&compare); |
| 14155 | assert!(compare_markdown.contains("# bencch compare report")); |
| 14156 | assert!(compare_markdown.contains("status: diff")); |
| 14157 | assert!(compare_markdown.contains("classification: artifact divergence")); |
| 14158 | assert!(compare_markdown.contains("difference_count: 1")); |
| 14159 | assert!(compare_markdown.contains("changed_artifacts: asm")); |
| 14160 | assert!(compare_markdown.contains("backend_mode: `external-driver`")); |
| 14161 | assert!(compare_markdown.contains("### `asm`")); |
| 14162 | |
| 14163 | let compare_json = render_compare_json(&compare); |
| 14164 | assert!(compare_json.contains("\"status\": \"diff\"")); |
| 14165 | assert!(compare_json.contains("\"classification\": \"artifact divergence\"")); |
| 14166 | assert!(compare_json.contains("\"difference_count\": 1")); |
| 14167 | assert!(compare_json.contains("\"changed_artifacts\": [\"asm\"]")); |
| 14168 | } |
| 14169 | |
| 14170 | #[test] |
| 14171 | fn render_failure_introspection_reports_stage_and_excerpt() { |
| 14172 | let observed = ObservedProgram { |
| 14173 | observation: CompilerObservation { |
| 14174 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14175 | program: PathBuf::from("broken.f90"), |
| 14176 | opt_level: OptLevel::O0, |
| 14177 | compile_exit_code: 1, |
| 14178 | artifacts: BTreeMap::from([ |
| 14179 | ( |
| 14180 | ArtifactKey::Diagnostics, |
| 14181 | ArtifactValue::Text("undefined symbol: missing_value\nmore detail".into()), |
| 14182 | ), |
| 14183 | ( |
| 14184 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 14185 | ArtifactValue::Text("token stream".into()), |
| 14186 | ), |
| 14187 | ]), |
| 14188 | provenance: ObservationProvenance { |
| 14189 | compiler_identity: "armfortas".into(), |
| 14190 | adapter_kind: "named".into(), |
| 14191 | backend_mode: "linked".into(), |
| 14192 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14193 | artifacts_captured: vec!["diagnostics".into(), "armfortas.tokens".into()], |
| 14194 | comparison_basis: None, |
| 14195 | failure_stage: Some("sema".into()), |
| 14196 | }, |
| 14197 | }, |
| 14198 | requested_artifacts: BTreeSet::from([ |
| 14199 | ArtifactKey::Asm, |
| 14200 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 14201 | ]), |
| 14202 | }; |
| 14203 | |
| 14204 | let text = render_introspection_text(&observed, full_introspection_render_config()); |
| 14205 | assert!(text.contains("status: compile failed")); |
| 14206 | assert!(text.contains("failure_stage: sema")); |
| 14207 | assert!(text.contains("diagnostic_excerpt: undefined symbol: missing_value")); |
| 14208 | assert!(text.contains("missing_artifacts: asm")); |
| 14209 | |
| 14210 | let json = render_introspection_json(&observed); |
| 14211 | assert!(json.contains("\"stage\": \"sema\"")); |
| 14212 | assert!(json.contains("\"diagnostic_excerpt\": \"undefined symbol: missing_value\"")); |
| 14213 | assert!(json.contains("\"failure_stage\": \"sema\"")); |
| 14214 | assert!(json.contains("\"diagnostics\": {\"kind\":\"text\"")); |
| 14215 | assert!(json.contains("\"summary\":\"text, 2 lines, ")); |
| 14216 | assert!(json.contains("\"line_count\":2")); |
| 14217 | |
| 14218 | let markdown = render_introspection_markdown(&observed, full_introspection_render_config()); |
| 14219 | assert!(markdown.contains("status: compile failed")); |
| 14220 | assert!(markdown.contains("failure_stage: `sema`")); |
| 14221 | assert!(markdown.contains("diagnostic_excerpt: `undefined symbol: missing_value`")); |
| 14222 | } |
| 14223 | |
| 14224 | #[test] |
| 14225 | fn render_introspection_summary_only_omits_artifact_bodies() { |
| 14226 | let observed = ObservedProgram { |
| 14227 | observation: CompilerObservation { |
| 14228 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14229 | program: PathBuf::from("demo.f90"), |
| 14230 | opt_level: OptLevel::O0, |
| 14231 | compile_exit_code: 0, |
| 14232 | artifacts: BTreeMap::from([( |
| 14233 | ArtifactKey::Extra("armfortas.tokens".into()), |
| 14234 | ArtifactValue::Text("line1\nline2\nline3".into()), |
| 14235 | )]), |
| 14236 | provenance: ObservationProvenance { |
| 14237 | compiler_identity: "armfortas".into(), |
| 14238 | adapter_kind: "named".into(), |
| 14239 | backend_mode: "linked".into(), |
| 14240 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14241 | artifacts_captured: vec!["armfortas.tokens".into()], |
| 14242 | comparison_basis: None, |
| 14243 | failure_stage: None, |
| 14244 | }, |
| 14245 | }, |
| 14246 | requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.tokens".into())]), |
| 14247 | }; |
| 14248 | |
| 14249 | let config = IntrospectionRenderConfig { |
| 14250 | summary_only: true, |
| 14251 | max_artifact_lines: Some(1), |
| 14252 | }; |
| 14253 | let text = render_introspection_text(&observed, config); |
| 14254 | assert!(text.contains("content_mode: summary-only")); |
| 14255 | assert!(text.contains("summary: text, 3 lines, 17 chars")); |
| 14256 | assert!(text.contains("[content omitted by --summary-only]")); |
| 14257 | assert!(!text.contains("line2")); |
| 14258 | |
| 14259 | let markdown = render_introspection_markdown(&observed, config); |
| 14260 | assert!(markdown.contains("content_mode: `summary-only`")); |
| 14261 | assert!(markdown.contains("[content omitted by --summary-only]")); |
| 14262 | } |
| 14263 | |
| 14264 | #[test] |
| 14265 | fn render_introspection_truncates_large_artifacts() { |
| 14266 | let observed = ObservedProgram { |
| 14267 | observation: CompilerObservation { |
| 14268 | compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14269 | program: PathBuf::from("demo.f90"), |
| 14270 | opt_level: OptLevel::O0, |
| 14271 | compile_exit_code: 0, |
| 14272 | artifacts: BTreeMap::from([( |
| 14273 | ArtifactKey::Asm, |
| 14274 | ArtifactValue::Text("a\nb\nc\nd".into()), |
| 14275 | )]), |
| 14276 | provenance: ObservationProvenance { |
| 14277 | compiler_identity: "armfortas".into(), |
| 14278 | adapter_kind: "named".into(), |
| 14279 | backend_mode: "linked".into(), |
| 14280 | backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14281 | artifacts_captured: vec!["asm".into()], |
| 14282 | comparison_basis: None, |
| 14283 | failure_stage: None, |
| 14284 | }, |
| 14285 | }, |
| 14286 | requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 14287 | }; |
| 14288 | |
| 14289 | let config = IntrospectionRenderConfig { |
| 14290 | summary_only: false, |
| 14291 | max_artifact_lines: Some(2), |
| 14292 | }; |
| 14293 | let text = render_introspection_text(&observed, config); |
| 14294 | assert!(text.contains("content_mode: first 2 lines per artifact")); |
| 14295 | assert!(text.contains("a\nb\n... (truncated; showing first 2 of 4 lines)")); |
| 14296 | assert!(!text.contains("\nc\nd")); |
| 14297 | |
| 14298 | let markdown = render_introspection_markdown(&observed, config); |
| 14299 | assert!(markdown.contains("content_mode: `first 2 lines per artifact`")); |
| 14300 | assert!(markdown.contains("... (truncated; showing first 2 of 4 lines)")); |
| 14301 | } |
| 14302 | |
| 14303 | #[test] |
| 14304 | fn write_requested_reports_emits_files() { |
| 14305 | let root = std::env::temp_dir().join("afs_tests_report_output"); |
| 14306 | let _ = fs::remove_dir_all(&root); |
| 14307 | let json_path = root.join("result.json"); |
| 14308 | let markdown_path = root.join("result.md"); |
| 14309 | let config = RunConfig { |
| 14310 | suite_filter: None, |
| 14311 | case_filter: None, |
| 14312 | opt_filter: None, |
| 14313 | verbose: false, |
| 14314 | fail_fast: false, |
| 14315 | include_future: false, |
| 14316 | all_stages: false, |
| 14317 | json_report: Some(json_path.clone()), |
| 14318 | markdown_report: Some(markdown_path.clone()), |
| 14319 | tools: ToolchainConfig::from_env(), |
| 14320 | }; |
| 14321 | let mut summary = Summary::default(); |
| 14322 | summary.record_outcome(&Outcome { |
| 14323 | suite: "frontend/parser".into(), |
| 14324 | case: "where_construct".into(), |
| 14325 | opt_level: OptLevel::O0, |
| 14326 | kind: OutcomeKind::Pass, |
| 14327 | detail: String::new(), |
| 14328 | bundle: None, |
| 14329 | primary_backend: Some(PrimaryBackendReport { |
| 14330 | kind: "full".into(), |
| 14331 | mode: "linked".into(), |
| 14332 | detail: "linked armfortas::testing capture adapter".into(), |
| 14333 | }), |
| 14334 | consistency_observations: Vec::new(), |
| 14335 | }); |
| 14336 | |
| 14337 | write_requested_reports(&config, &summary).unwrap(); |
| 14338 | |
| 14339 | let json = fs::read_to_string(&json_path).unwrap(); |
| 14340 | let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14341 | assert!(json.contains("\"passed\": 1")); |
| 14342 | assert!(markdown.contains("| passed | 1 |")); |
| 14343 | |
| 14344 | let _ = fs::remove_dir_all(&root); |
| 14345 | } |
| 14346 | |
| 14347 | #[test] |
| 14348 | fn render_doctor_report_includes_tool_status() { |
| 14349 | let root = std::env::temp_dir().join("afs_tests_doctor_paths"); |
| 14350 | let _ = fs::remove_dir_all(&root); |
| 14351 | fs::create_dir_all(&root).unwrap(); |
| 14352 | let armfortas_bin = root.join("armfortas"); |
| 14353 | let gfortran_bin = root.join("gfortran"); |
| 14354 | write_probe_script(&armfortas_bin, "armfortas dev build"); |
| 14355 | write_probe_script(&gfortran_bin, "GNU Fortran 99.1"); |
| 14356 | |
| 14357 | let config = DoctorConfig { |
| 14358 | tools: ToolchainConfig { |
| 14359 | armfortas: ArmfortasCliAdapter::External(armfortas_bin.display().to_string()), |
| 14360 | gfortran: gfortran_bin.display().to_string(), |
| 14361 | flang_new: "/tmp/does-not-exist-flang".into(), |
| 14362 | lfortran: "/tmp/does-not-exist-lfortran".into(), |
| 14363 | ifort: "/tmp/does-not-exist-ifort".into(), |
| 14364 | ifx: "/tmp/does-not-exist-ifx".into(), |
| 14365 | nvfortran: "/tmp/does-not-exist-nvfortran".into(), |
| 14366 | system_as: "/tmp/does-not-exist-as".into(), |
| 14367 | otool: "/tmp/does-not-exist-otool".into(), |
| 14368 | nm: "/tmp/does-not-exist-nm".into(), |
| 14369 | }, |
| 14370 | json_report: None, |
| 14371 | markdown_report: None, |
| 14372 | }; |
| 14373 | |
| 14374 | let rendered = render_doctor_report(&config); |
| 14375 | assert!(rendered.contains("Doctor")); |
| 14376 | assert!(rendered.contains("armfortas_cli_adapter: external armfortas binary adapter")); |
| 14377 | assert!(rendered.contains("armfortas_cli_mode: external")); |
| 14378 | assert!(rendered |
| 14379 | .contains("armfortas_capture_adapter: linked armfortas::testing capture adapter")); |
| 14380 | assert!(rendered.contains("armfortas_capture_mode: linked")); |
| 14381 | assert!(rendered.contains("armfortas_capture_manifest:")); |
| 14382 | assert!( |
| 14383 | rendered.contains("primary_backend_full: linked armfortas::testing capture adapter") |
| 14384 | ); |
| 14385 | assert!(rendered.contains( |
| 14386 | "linked_mode_surface: rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 14387 | )); |
| 14388 | assert!(rendered.contains( |
| 14389 | "primary_backend_observable: cli-observable armfortas driver capture adapter" |
| 14390 | )); |
| 14391 | assert!(rendered.contains( |
| 14392 | "primary_backend_selection: observable backend is selected for asm/obj/run-only cells" |
| 14393 | )); |
| 14394 | assert!(rendered.contains("named_compiler.armfortas: cli=external capture=linked")); |
| 14395 | assert!(rendered.contains( |
| 14396 | "named_compiler.armfortas.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14397 | )); |
| 14398 | assert!(rendered.contains("named_compiler.armfortas.adapter_extras: armfortas(")); |
| 14399 | assert!(rendered.contains("named_compiler.armfortas.unavailable_artifacts: none")); |
| 14400 | assert!(rendered.contains("named_compiler.gfortran:")); |
| 14401 | assert!(rendered.contains( |
| 14402 | "named_compiler.gfortran.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14403 | )); |
| 14404 | assert!(rendered.contains("named_compiler.gfortran.adapter_extras: none")); |
| 14405 | assert!(rendered.contains("named_compiler.lfortran:")); |
| 14406 | assert!(rendered.contains("named_compiler.lfortran.accepted_names: lfortran")); |
| 14407 | assert!(rendered.contains("named_compiler.lfortran.candidate_binaries: lfortran")); |
| 14408 | assert!(rendered.contains("named_compiler.ifx.accepted_names: ifx")); |
| 14409 | assert!(rendered.contains("named_compiler.nvfortran.accepted_names: nvfortran, pgfortran")); |
| 14410 | assert!(rendered.contains("named_compiler.armfortas.probe_status: invokable")); |
| 14411 | assert!(rendered.contains("named_compiler.armfortas.probe_banner: armfortas dev build")); |
| 14412 | assert!(rendered.contains("named_compiler.gfortran.probe_status: invokable")); |
| 14413 | assert!(rendered.contains("named_compiler.gfortran.probe_banner: GNU Fortran 99.1")); |
| 14414 | assert!(rendered.contains( |
| 14415 | "explicit_compiler_path: any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 14416 | )); |
| 14417 | assert!(rendered.contains( |
| 14418 | "explicit_compiler_path.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14419 | )); |
| 14420 | assert!(rendered.contains(&format!( |
| 14421 | "configured={} resolved={}", |
| 14422 | armfortas_bin.display(), |
| 14423 | armfortas_bin.display() |
| 14424 | ))); |
| 14425 | assert!(rendered.contains(&format!( |
| 14426 | "configured={} resolved={}", |
| 14427 | gfortran_bin.display(), |
| 14428 | gfortran_bin.display() |
| 14429 | ))); |
| 14430 | assert!(rendered.contains("configured=/tmp/does-not-exist-flang resolved=missing")); |
| 14431 | let rendered_json = render_doctor_json(&config); |
| 14432 | let rendered_markdown = render_doctor_markdown(&config); |
| 14433 | assert!(rendered_json.contains("\"command\": \"doctor\"")); |
| 14434 | assert!(rendered_json.contains("\"workspace\": {")); |
| 14435 | assert!(rendered_json.contains("\"named_compilers\": {")); |
| 14436 | assert!(rendered_json.contains("\"tools\": {")); |
| 14437 | assert!(rendered_json.contains("\"lfortran\": {")); |
| 14438 | assert!(rendered_json.contains("\"named_compiler.armfortas.adapter_extras\"")); |
| 14439 | assert!(rendered_json.contains("\"probe\": {")); |
| 14440 | assert!(rendered_markdown.contains("# bencch doctor report")); |
| 14441 | assert!(rendered_markdown.contains("| `named_compiler.armfortas` |")); |
| 14442 | |
| 14443 | let _ = fs::remove_dir_all(&root); |
| 14444 | } |
| 14445 | |
| 14446 | #[cfg(unix)] |
| 14447 | #[test] |
| 14448 | fn tool_probe_reads_banner_from_executable() { |
| 14449 | let root = std::env::temp_dir().join("bencch_tool_probe_banner"); |
| 14450 | let _ = fs::remove_dir_all(&root); |
| 14451 | fs::create_dir_all(&root).unwrap(); |
| 14452 | let probe_bin = root.join("fakefortran"); |
| 14453 | write_probe_script(&probe_bin, "Fake Fortran 1.2.3"); |
| 14454 | |
| 14455 | let probe = tool_probe(&probe_bin.display().to_string(), true); |
| 14456 | assert_eq!(probe.status, "invokable"); |
| 14457 | assert_eq!(probe.banner.as_deref(), Some("Fake Fortran 1.2.3")); |
| 14458 | assert!(probe |
| 14459 | .detail |
| 14460 | .as_deref() |
| 14461 | .unwrap_or_default() |
| 14462 | .contains("--version")); |
| 14463 | |
| 14464 | let _ = fs::remove_dir_all(&root); |
| 14465 | } |
| 14466 | |
| 14467 | #[test] |
| 14468 | fn case_discovery_lines_report_capability_block_for_generic_introspect() { |
| 14469 | let case = CaseSpec { |
| 14470 | name: "unsupported_extra".into(), |
| 14471 | source: PathBuf::from("demo.f90"), |
| 14472 | graph_files: Vec::new(), |
| 14473 | requested: BTreeSet::new(), |
| 14474 | generic_introspect: Some(GenericIntrospectCase { |
| 14475 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14476 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 14477 | }), |
| 14478 | generic_compare: None, |
| 14479 | opt_levels: vec![OptLevel::O0], |
| 14480 | repeat_count: 2, |
| 14481 | reference_compilers: Vec::new(), |
| 14482 | consistency_checks: Vec::new(), |
| 14483 | expectations: Vec::new(), |
| 14484 | status_rules: Vec::new(), |
| 14485 | capability_policy: None, |
| 14486 | }; |
| 14487 | |
| 14488 | let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14489 | assert!(lines.contains(&"capability_status: blocked".to_string())); |
| 14490 | assert!(lines.iter().any(|line| line.contains( |
| 14491 | "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 14492 | ))); |
| 14493 | } |
| 14494 | |
| 14495 | #[test] |
| 14496 | fn case_discovery_lines_report_capability_policy_as_deferred() { |
| 14497 | let case = CaseSpec { |
| 14498 | name: "unsupported_extra".into(), |
| 14499 | source: PathBuf::from("demo.f90"), |
| 14500 | graph_files: Vec::new(), |
| 14501 | requested: BTreeSet::new(), |
| 14502 | generic_introspect: Some(GenericIntrospectCase { |
| 14503 | compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14504 | artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 14505 | }), |
| 14506 | generic_compare: None, |
| 14507 | opt_levels: vec![OptLevel::O0], |
| 14508 | repeat_count: 2, |
| 14509 | reference_compilers: Vec::new(), |
| 14510 | consistency_checks: Vec::new(), |
| 14511 | expectations: Vec::new(), |
| 14512 | status_rules: Vec::new(), |
| 14513 | capability_policy: Some(CapabilityPolicy { |
| 14514 | kind: StatusKind::Future, |
| 14515 | reason: "generic gfortran surface has no armfortas extras".into(), |
| 14516 | }), |
| 14517 | }; |
| 14518 | |
| 14519 | let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14520 | assert!(lines.contains(&"capability_status: deferred".to_string())); |
| 14521 | assert!(lines.contains( |
| 14522 | &"capability_policy: future when blocked (generic gfortran surface has no armfortas extras)" |
| 14523 | .to_string() |
| 14524 | )); |
| 14525 | } |
| 14526 | |
| 14527 | #[cfg(unix)] |
| 14528 | #[test] |
| 14529 | fn case_discovery_lines_include_compiler_probe_banner() { |
| 14530 | let root = std::env::temp_dir().join("bencch_case_discovery_probe"); |
| 14531 | let _ = fs::remove_dir_all(&root); |
| 14532 | fs::create_dir_all(&root).unwrap(); |
| 14533 | let compiler = root.join("probe-compiler"); |
| 14534 | write_probe_script(&compiler, "Probe Compiler 7.4"); |
| 14535 | |
| 14536 | let case = CaseSpec { |
| 14537 | name: "probe".into(), |
| 14538 | source: PathBuf::from("demo.f90"), |
| 14539 | graph_files: Vec::new(), |
| 14540 | requested: BTreeSet::new(), |
| 14541 | generic_introspect: Some(GenericIntrospectCase { |
| 14542 | compiler: CompilerSpec::Binary(compiler.clone()), |
| 14543 | artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 14544 | }), |
| 14545 | generic_compare: None, |
| 14546 | opt_levels: vec![OptLevel::O0], |
| 14547 | repeat_count: 2, |
| 14548 | reference_compilers: Vec::new(), |
| 14549 | consistency_checks: Vec::new(), |
| 14550 | expectations: Vec::new(), |
| 14551 | status_rules: Vec::new(), |
| 14552 | capability_policy: None, |
| 14553 | }; |
| 14554 | |
| 14555 | let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14556 | assert!(lines.contains(&"compiler_probe_status: invokable".to_string())); |
| 14557 | assert!(lines |
| 14558 | .iter() |
| 14559 | .any(|line| line.contains("compiler_probe_banner: Probe Compiler 7.4"))); |
| 14560 | |
| 14561 | let _ = fs::remove_dir_all(&root); |
| 14562 | } |
| 14563 | |
| 14564 | #[test] |
| 14565 | fn case_discovery_lines_distinguish_legacy_surfaces() { |
| 14566 | let observable_case = CaseSpec { |
| 14567 | name: "observable".into(), |
| 14568 | source: PathBuf::from("demo.f90"), |
| 14569 | graph_files: Vec::new(), |
| 14570 | requested: BTreeSet::from([Stage::Run]), |
| 14571 | generic_introspect: None, |
| 14572 | generic_compare: None, |
| 14573 | opt_levels: vec![OptLevel::O0], |
| 14574 | repeat_count: 2, |
| 14575 | reference_compilers: Vec::new(), |
| 14576 | consistency_checks: Vec::new(), |
| 14577 | expectations: Vec::new(), |
| 14578 | status_rules: Vec::new(), |
| 14579 | capability_policy: None, |
| 14580 | }; |
| 14581 | let linked_case = CaseSpec { |
| 14582 | name: "linked".into(), |
| 14583 | source: PathBuf::from("demo.f90"), |
| 14584 | graph_files: Vec::new(), |
| 14585 | requested: BTreeSet::from([Stage::Tokens]), |
| 14586 | generic_introspect: None, |
| 14587 | generic_compare: None, |
| 14588 | opt_levels: vec![OptLevel::O0, OptLevel::O1], |
| 14589 | repeat_count: 2, |
| 14590 | reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 14591 | consistency_checks: vec![ConsistencyCheck::CaptureAsmReproducible], |
| 14592 | expectations: Vec::new(), |
| 14593 | status_rules: Vec::new(), |
| 14594 | capability_policy: None, |
| 14595 | }; |
| 14596 | |
| 14597 | let tools = ToolchainConfig { |
| 14598 | armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 14599 | ..ToolchainConfig::from_env() |
| 14600 | }; |
| 14601 | let observable_lines = case_discovery_lines(&observable_case, &tools); |
| 14602 | let linked_lines = case_discovery_lines(&linked_case, &tools); |
| 14603 | |
| 14604 | assert!(observable_lines.contains(&"surface: observable-only legacy path".to_string())); |
| 14605 | assert!(observable_lines.contains(&"capability_status: ready".to_string())); |
| 14606 | assert!(linked_lines.contains(&"surface: linked armfortas capture".to_string())); |
| 14607 | assert!(linked_lines |
| 14608 | .iter() |
| 14609 | .any(|line| line.contains("differential: gfortran"))); |
| 14610 | assert!(linked_lines |
| 14611 | .iter() |
| 14612 | .any(|line| line.contains("consistency: capture_asm_reproducible"))); |
| 14613 | } |
| 14614 | |
| 14615 | #[test] |
| 14616 | fn write_doctor_reports_emits_files() { |
| 14617 | let root = std::env::temp_dir().join("afs_tests_doctor_report_output"); |
| 14618 | let _ = fs::remove_dir_all(&root); |
| 14619 | fs::create_dir_all(&root).unwrap(); |
| 14620 | let json_path = root.join("doctor.json"); |
| 14621 | let markdown_path = root.join("doctor.md"); |
| 14622 | let config = DoctorConfig { |
| 14623 | tools: ToolchainConfig::from_env(), |
| 14624 | json_report: Some(json_path.clone()), |
| 14625 | markdown_report: Some(markdown_path.clone()), |
| 14626 | }; |
| 14627 | |
| 14628 | write_doctor_reports(&config).unwrap(); |
| 14629 | |
| 14630 | let json = fs::read_to_string(&json_path).unwrap(); |
| 14631 | let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14632 | assert!(json.contains("\"command\": \"doctor\"")); |
| 14633 | assert!(json.contains("\"workspace_root\"")); |
| 14634 | assert!(markdown.contains("# bencch doctor report")); |
| 14635 | assert!(markdown.contains("| `workspace_root` |")); |
| 14636 | |
| 14637 | let _ = fs::remove_dir_all(&root); |
| 14638 | } |
| 14639 | |
| 14640 | #[test] |
| 14641 | fn differential_reports_armfortas_only_divergence() { |
| 14642 | let armfortas = differential_armfortas_observation("0\n", "", 0); |
| 14643 | let refs = vec![ |
| 14644 | differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14645 | differential_reference_observation(ReferenceCompiler::FlangNew, "42\n", "", 0), |
| 14646 | ]; |
| 14647 | |
| 14648 | let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 14649 | assert!(err.contains("classification: armfortas-only divergence")); |
| 14650 | assert!(err.contains("basis: compile-status, diagnostics, runtime")); |
| 14651 | } |
| 14652 | |
| 14653 | #[test] |
| 14654 | fn differential_reports_reference_disagreement() { |
| 14655 | let armfortas = differential_armfortas_observation("42\n", "", 0); |
| 14656 | let refs = vec![ |
| 14657 | differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14658 | differential_reference_observation(ReferenceCompiler::FlangNew, "99\n", "", 0), |
| 14659 | ]; |
| 14660 | |
| 14661 | let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 14662 | assert!(err.contains("classification: reference disagreement")); |
| 14663 | } |
| 14664 | |
| 14665 | #[test] |
| 14666 | fn differential_tolerates_numeric_formatting_differences() { |
| 14667 | let armfortas = differential_armfortas_observation(" 5.5000000E0\n", "", 0); |
| 14668 | let refs = vec![ |
| 14669 | differential_reference_observation( |
| 14670 | ReferenceCompiler::Gfortran, |
| 14671 | " 5.50000000\n", |
| 14672 | "", |
| 14673 | 0, |
| 14674 | ), |
| 14675 | differential_reference_observation(ReferenceCompiler::FlangNew, " 5.5\n", "", 0), |
| 14676 | ]; |
| 14677 | |
| 14678 | assert!(compare_differential(&armfortas, &refs).is_ok()); |
| 14679 | } |
| 14680 | |
| 14681 | #[test] |
| 14682 | fn consistency_diff_reports_first_mismatch() { |
| 14683 | let detail = describe_text_difference("alpha\nbeta\n", "alpha\ngamma\n", "left", "right"); |
| 14684 | assert!(detail.contains("first differing line: 2")); |
| 14685 | assert!(detail.contains("left: beta")); |
| 14686 | assert!(detail.contains("right: gamma")); |
| 14687 | } |
| 14688 | |
| 14689 | #[test] |
| 14690 | fn consistency_diff_reports_first_extra_line() { |
| 14691 | let detail = describe_text_difference("alpha\nbeta\n", "", "left", "right"); |
| 14692 | assert!(detail.contains("snapshot length differs")); |
| 14693 | assert!(detail.contains("first extra line: 1")); |
| 14694 | assert!(detail.contains("left: alpha")); |
| 14695 | } |
| 14696 | |
| 14697 | #[test] |
| 14698 | fn object_diff_reports_changed_components() { |
| 14699 | let expected = ObjectSnapshot { |
| 14700 | text: "alpha\nbeta\n".into(), |
| 14701 | load_commands: "same".into(), |
| 14702 | relocations: "same".into(), |
| 14703 | symbols: "same".into(), |
| 14704 | }; |
| 14705 | let actual = ObjectSnapshot { |
| 14706 | text: "alpha\ngamma\n".into(), |
| 14707 | load_commands: "same".into(), |
| 14708 | relocations: "same".into(), |
| 14709 | symbols: "same".into(), |
| 14710 | }; |
| 14711 | |
| 14712 | let detail = describe_object_difference(&expected, &actual, "first -c", "second -c"); |
| 14713 | assert!(detail.contains("differing object components: text")); |
| 14714 | assert!(detail.contains("first differing component: text")); |
| 14715 | assert!(detail.contains("first -c: beta")); |
| 14716 | assert!(detail.contains("second -c: gamma")); |
| 14717 | } |
| 14718 | |
| 14719 | #[test] |
| 14720 | fn object_component_variation_classifies_text_only_instability() { |
| 14721 | let first = ObjectSnapshot { |
| 14722 | text: "alpha".into(), |
| 14723 | load_commands: "load".into(), |
| 14724 | relocations: "reloc".into(), |
| 14725 | symbols: "symbols".into(), |
| 14726 | }; |
| 14727 | let second = ObjectSnapshot { |
| 14728 | text: "beta".into(), |
| 14729 | load_commands: "load".into(), |
| 14730 | relocations: "reloc".into(), |
| 14731 | symbols: "symbols".into(), |
| 14732 | }; |
| 14733 | let snapshots = vec![&first, &second]; |
| 14734 | |
| 14735 | assert_eq!(varying_object_components(&snapshots), vec!["text"]); |
| 14736 | assert_eq!( |
| 14737 | stable_object_components(&snapshots), |
| 14738 | vec!["load_commands", "relocations", "symbols"] |
| 14739 | ); |
| 14740 | } |
| 14741 | |
| 14742 | #[test] |
| 14743 | fn run_diff_reports_changed_components() { |
| 14744 | let left = RunCapture { |
| 14745 | exit_code: 0, |
| 14746 | stdout: "alpha\nbeta\n".into(), |
| 14747 | stderr: String::new(), |
| 14748 | }; |
| 14749 | let right = RunCapture { |
| 14750 | exit_code: 0, |
| 14751 | stdout: "alpha\ngamma\n".into(), |
| 14752 | stderr: String::new(), |
| 14753 | }; |
| 14754 | |
| 14755 | let detail = describe_run_difference(&left, &right, "capture run", "cli run 2"); |
| 14756 | assert!(detail.contains("differing runtime components: stdout")); |
| 14757 | assert!(detail.contains("first differing component: stdout")); |
| 14758 | assert!(detail.contains("capture run: beta")); |
| 14759 | assert!(detail.contains("cli run 2: gamma")); |
| 14760 | } |
| 14761 | |
| 14762 | #[test] |
| 14763 | fn run_component_variation_classifies_stdout_only_instability() { |
| 14764 | let first = RunSignature { |
| 14765 | exit_code: 0, |
| 14766 | stdout: "alpha".into(), |
| 14767 | stderr: String::new(), |
| 14768 | }; |
| 14769 | let second = RunSignature { |
| 14770 | exit_code: 0, |
| 14771 | stdout: "beta".into(), |
| 14772 | stderr: String::new(), |
| 14773 | }; |
| 14774 | let signatures = vec![&first, &second]; |
| 14775 | |
| 14776 | assert_eq!(varying_run_components(&signatures), vec!["stdout"]); |
| 14777 | assert_eq!( |
| 14778 | stable_run_components(&signatures), |
| 14779 | vec!["exit_code", "stderr"] |
| 14780 | ); |
| 14781 | } |
| 14782 | |
| 14783 | #[test] |
| 14784 | fn parse_object_snapshot_text_round_trips_rendered_snapshot() { |
| 14785 | let snapshot = ObjectSnapshot { |
| 14786 | text: "text bytes".into(), |
| 14787 | load_commands: "load commands".into(), |
| 14788 | relocations: "relocations".into(), |
| 14789 | symbols: "symbols".into(), |
| 14790 | }; |
| 14791 | |
| 14792 | let rendered = render_object_snapshot(&snapshot); |
| 14793 | let parsed = parse_object_snapshot_text(&rendered).unwrap(); |
| 14794 | assert_eq!(parsed, snapshot); |
| 14795 | } |
| 14796 | |
| 14797 | #[test] |
| 14798 | fn verifier_regression_detects_integer_op_on_float_values() { |
| 14799 | let mut module = Module::new("verify".into()); |
| 14800 | let mut func = Function::new("broken".into(), vec![], IrType::Void); |
| 14801 | func.blocks[0].insts.push(Inst { |
| 14802 | id: ValueId(0), |
| 14803 | kind: InstKind::ConstFloat(1.0, FloatWidth::F32), |
| 14804 | ty: IrType::Float(FloatWidth::F32), |
| 14805 | span: dummy_span(), |
| 14806 | }); |
| 14807 | func.blocks[0].insts.push(Inst { |
| 14808 | id: ValueId(1), |
| 14809 | kind: InstKind::ConstFloat(2.0, FloatWidth::F32), |
| 14810 | ty: IrType::Float(FloatWidth::F32), |
| 14811 | span: dummy_span(), |
| 14812 | }); |
| 14813 | func.blocks[0].insts.push(Inst { |
| 14814 | id: ValueId(2), |
| 14815 | kind: InstKind::IAdd(ValueId(0), ValueId(1)), |
| 14816 | ty: IrType::Int(IntWidth::I32), |
| 14817 | span: dummy_span(), |
| 14818 | }); |
| 14819 | func.blocks[0].terminator = Some(Terminator::Return(None)); |
| 14820 | module.add_function(func); |
| 14821 | |
| 14822 | let errors = verify_module(&module); |
| 14823 | assert!( |
| 14824 | errors |
| 14825 | .iter() |
| 14826 | .any(|error| error.msg.contains("non-integer operand")), |
| 14827 | "expected verifier error, got: {:?}", |
| 14828 | errors |
| 14829 | ); |
| 14830 | } |
| 14831 | |
| 14832 | #[test] |
| 14833 | fn verifier_regression_detects_branch_argument_mismatch() { |
| 14834 | let mut module = Module::new("verify".into()); |
| 14835 | let mut func = Function::new("broken_branch".into(), vec![], IrType::Void); |
| 14836 | let target = func.create_block("target"); |
| 14837 | func.block_mut(target).params.push(BlockParam { |
| 14838 | id: ValueId(0), |
| 14839 | ty: IrType::Int(IntWidth::I32), |
| 14840 | }); |
| 14841 | func.blocks[0].terminator = Some(Terminator::Branch(target, vec![])); |
| 14842 | func.block_mut(target).terminator = Some(Terminator::Return(None)); |
| 14843 | module.add_function(func); |
| 14844 | |
| 14845 | let errors = verify_module(&module); |
| 14846 | assert!( |
| 14847 | errors |
| 14848 | .iter() |
| 14849 | .any(|error| error.msg.contains("expected 1 args, got 0")), |
| 14850 | "expected verifier error, got: {:?}", |
| 14851 | errors |
| 14852 | ); |
| 14853 | } |
| 14854 | |
| 14855 | #[test] |
| 14856 | fn verifier_regression_detects_store_to_non_pointer() { |
| 14857 | let mut module = Module::new("verify".into()); |
| 14858 | let mut func = Function::new("broken_store".into(), vec![], IrType::Void); |
| 14859 | func.blocks[0].insts.push(Inst { |
| 14860 | id: ValueId(0), |
| 14861 | kind: InstKind::ConstInt(42, IntWidth::I32), |
| 14862 | ty: IrType::Int(IntWidth::I32), |
| 14863 | span: dummy_span(), |
| 14864 | }); |
| 14865 | func.blocks[0].insts.push(Inst { |
| 14866 | id: ValueId(1), |
| 14867 | kind: InstKind::ConstInt(0, IntWidth::I32), |
| 14868 | ty: IrType::Int(IntWidth::I32), |
| 14869 | span: dummy_span(), |
| 14870 | }); |
| 14871 | func.blocks[0].insts.push(Inst { |
| 14872 | id: ValueId(2), |
| 14873 | kind: InstKind::Store(ValueId(0), ValueId(1)), |
| 14874 | ty: IrType::Void, |
| 14875 | span: dummy_span(), |
| 14876 | }); |
| 14877 | func.blocks[0].terminator = Some(Terminator::Return(None)); |
| 14878 | module.add_function(func); |
| 14879 | |
| 14880 | let errors = verify_module(&module); |
| 14881 | assert!( |
| 14882 | errors.iter().any(|error| error.msg.contains("non-pointer")), |
| 14883 | "expected verifier error, got: {:?}", |
| 14884 | errors |
| 14885 | ); |
| 14886 | } |
| 14887 | |
| 14888 | #[test] |
| 14889 | fn failure_expectation_precedes_partial_stage_checks() { |
| 14890 | let case = CaseSpec { |
| 14891 | name: "missing_then".into(), |
| 14892 | source: PathBuf::from("demo.f90"), |
| 14893 | graph_files: Vec::new(), |
| 14894 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14895 | generic_introspect: None, |
| 14896 | generic_compare: None, |
| 14897 | opt_levels: vec![OptLevel::O0], |
| 14898 | repeat_count: 3, |
| 14899 | reference_compilers: Vec::new(), |
| 14900 | consistency_checks: Vec::new(), |
| 14901 | expectations: vec![ |
| 14902 | Expectation::Contains { |
| 14903 | target: Target::RunStdout, |
| 14904 | needle: "42".into(), |
| 14905 | }, |
| 14906 | Expectation::FailContains { |
| 14907 | stage: FailureStage::Parser, |
| 14908 | needle: "expected 'then'".into(), |
| 14909 | }, |
| 14910 | ], |
| 14911 | status_rules: Vec::new(), |
| 14912 | capability_policy: None, |
| 14913 | }; |
| 14914 | let artifacts = ExecutionArtifacts { |
| 14915 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14916 | armfortas: None, |
| 14917 | armfortas_failure: None, |
| 14918 | armfortas_observation: None, |
| 14919 | references: Vec::new(), |
| 14920 | reference_observations: Vec::new(), |
| 14921 | consistency_issues: Vec::new(), |
| 14922 | }; |
| 14923 | let failure = CaptureFailure { |
| 14924 | input: PathBuf::from("demo.f90"), |
| 14925 | opt_level: OptLevel::O0, |
| 14926 | stage: FailureStage::Parser, |
| 14927 | detail: "expected 'then'".into(), |
| 14928 | stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14929 | }; |
| 14930 | |
| 14931 | assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14932 | } |
| 14933 | |
| 14934 | #[test] |
| 14935 | fn legacy_capture_failures_use_observation_failure_semantics() { |
| 14936 | let case = CaseSpec { |
| 14937 | name: "hidden_use_only".into(), |
| 14938 | source: PathBuf::from("demo.f90"), |
| 14939 | graph_files: Vec::new(), |
| 14940 | requested: BTreeSet::from([Stage::Run]), |
| 14941 | generic_introspect: None, |
| 14942 | generic_compare: None, |
| 14943 | opt_levels: vec![OptLevel::O0], |
| 14944 | repeat_count: 3, |
| 14945 | reference_compilers: Vec::new(), |
| 14946 | consistency_checks: Vec::new(), |
| 14947 | expectations: vec![Expectation::FailCommentPatterns(vec!["hidden".into()])], |
| 14948 | status_rules: Vec::new(), |
| 14949 | capability_policy: None, |
| 14950 | }; |
| 14951 | let artifacts = ExecutionArtifacts { |
| 14952 | requested: BTreeSet::from([Stage::Run]), |
| 14953 | armfortas: None, |
| 14954 | armfortas_failure: None, |
| 14955 | armfortas_observation: None, |
| 14956 | references: Vec::new(), |
| 14957 | reference_observations: Vec::new(), |
| 14958 | consistency_issues: Vec::new(), |
| 14959 | }; |
| 14960 | let failure = CaptureFailure { |
| 14961 | input: PathBuf::from("demo.f90"), |
| 14962 | opt_level: OptLevel::O0, |
| 14963 | stage: FailureStage::Sema, |
| 14964 | detail: "demo.f90:4:3: semantic error: hidden".into(), |
| 14965 | stages: BTreeMap::new(), |
| 14966 | }; |
| 14967 | |
| 14968 | assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14969 | } |
| 14970 | |
| 14971 | #[test] |
| 14972 | fn legacy_failure_observed_program_uses_prepared_source_and_partial_stages() { |
| 14973 | let case = CaseSpec { |
| 14974 | name: "missing_then".into(), |
| 14975 | source: PathBuf::from("authored.f90"), |
| 14976 | graph_files: Vec::new(), |
| 14977 | requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14978 | generic_introspect: None, |
| 14979 | generic_compare: None, |
| 14980 | opt_levels: vec![OptLevel::O0], |
| 14981 | repeat_count: 3, |
| 14982 | reference_compilers: Vec::new(), |
| 14983 | consistency_checks: Vec::new(), |
| 14984 | expectations: vec![Expectation::FailContains { |
| 14985 | stage: FailureStage::Parser, |
| 14986 | needle: "expected 'then'".into(), |
| 14987 | }], |
| 14988 | status_rules: Vec::new(), |
| 14989 | capability_policy: None, |
| 14990 | }; |
| 14991 | let failure = CaptureFailure { |
| 14992 | input: PathBuf::from("generated.f90"), |
| 14993 | opt_level: OptLevel::O0, |
| 14994 | stage: FailureStage::Parser, |
| 14995 | detail: "expected 'then'".into(), |
| 14996 | stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14997 | }; |
| 14998 | |
| 14999 | let observed = legacy_failure_observed_program(Path::new("prepared.f90"), &case, &failure); |
| 15000 | assert_eq!(observed.observation.program, PathBuf::from("prepared.f90")); |
| 15001 | assert_eq!(observed.observation.compile_exit_code, 1); |
| 15002 | assert_eq!( |
| 15003 | observed.observation.provenance.failure_stage.as_deref(), |
| 15004 | Some("parser") |
| 15005 | ); |
| 15006 | assert!(observed |
| 15007 | .observation |
| 15008 | .artifacts |
| 15009 | .contains_key(&ArtifactKey::Diagnostics)); |
| 15010 | assert!(observed |
| 15011 | .observation |
| 15012 | .artifacts |
| 15013 | .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 15014 | } |
| 15015 | |
| 15016 | #[test] |
| 15017 | fn unexpected_capture_failure_reports_compiler_failure_detail() { |
| 15018 | let case = CaseSpec { |
| 15019 | name: "module_procedure_runtime".into(), |
| 15020 | source: PathBuf::from("graph.f90"), |
| 15021 | graph_files: Vec::new(), |
| 15022 | requested: BTreeSet::from([Stage::Run]), |
| 15023 | generic_introspect: None, |
| 15024 | generic_compare: None, |
| 15025 | opt_levels: vec![OptLevel::O0], |
| 15026 | repeat_count: 3, |
| 15027 | reference_compilers: Vec::new(), |
| 15028 | consistency_checks: Vec::new(), |
| 15029 | expectations: vec![Expectation::Contains { |
| 15030 | target: Target::RunStdout, |
| 15031 | needle: "42".into(), |
| 15032 | }], |
| 15033 | status_rules: Vec::new(), |
| 15034 | capability_policy: None, |
| 15035 | }; |
| 15036 | let failure = CaptureFailure { |
| 15037 | input: PathBuf::from("graph.f90"), |
| 15038 | opt_level: OptLevel::O0, |
| 15039 | stage: FailureStage::Run, |
| 15040 | detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 15041 | stages: BTreeMap::new(), |
| 15042 | }; |
| 15043 | let artifacts = ExecutionArtifacts { |
| 15044 | requested: BTreeSet::from([Stage::Run]), |
| 15045 | armfortas: None, |
| 15046 | armfortas_failure: Some(failure.clone()), |
| 15047 | armfortas_observation: None, |
| 15048 | references: Vec::new(), |
| 15049 | reference_observations: Vec::new(), |
| 15050 | consistency_issues: Vec::new(), |
| 15051 | }; |
| 15052 | |
| 15053 | let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 15054 | assert!(err.contains("armfortas failed in run")); |
| 15055 | assert!(err.contains("_add_one")); |
| 15056 | assert!(!err.contains("missing captured run stage")); |
| 15057 | } |
| 15058 | |
| 15059 | #[test] |
| 15060 | fn partial_stage_expectation_failure_is_preserved_on_capture_failure() { |
| 15061 | let case = CaseSpec { |
| 15062 | name: "module_procedure_backend".into(), |
| 15063 | source: PathBuf::from("graph.f90"), |
| 15064 | graph_files: Vec::new(), |
| 15065 | requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 15066 | generic_introspect: None, |
| 15067 | generic_compare: None, |
| 15068 | opt_levels: vec![OptLevel::O0], |
| 15069 | repeat_count: 3, |
| 15070 | reference_compilers: Vec::new(), |
| 15071 | consistency_checks: Vec::new(), |
| 15072 | expectations: vec![Expectation::Contains { |
| 15073 | target: Target::Stage(Stage::Asm), |
| 15074 | needle: ".globl _add_one".into(), |
| 15075 | }], |
| 15076 | status_rules: Vec::new(), |
| 15077 | capability_policy: None, |
| 15078 | }; |
| 15079 | let failure = CaptureFailure { |
| 15080 | input: PathBuf::from("graph.f90"), |
| 15081 | opt_level: OptLevel::O0, |
| 15082 | stage: FailureStage::Run, |
| 15083 | detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 15084 | stages: BTreeMap::from([(Stage::Asm, CapturedStage::Text(".globl _main\n".into()))]), |
| 15085 | }; |
| 15086 | let artifacts = ExecutionArtifacts { |
| 15087 | requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 15088 | armfortas: None, |
| 15089 | armfortas_failure: Some(failure.clone()), |
| 15090 | armfortas_observation: None, |
| 15091 | references: Vec::new(), |
| 15092 | reference_observations: Vec::new(), |
| 15093 | consistency_issues: Vec::new(), |
| 15094 | }; |
| 15095 | |
| 15096 | let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 15097 | assert!(err.contains("expected asm to contain")); |
| 15098 | assert!(!err.contains("armfortas failed in run")); |
| 15099 | } |
| 15100 | } |
| 15101 |