@@ -7,8 +7,13 @@ use std::process::Command; |
| 7 | 7 | use std::sync::atomic::{AtomicU64, Ordering}; |
| 8 | 8 | |
| 9 | 9 | use crate::compiler::{ |
| 10 | | - capture_from_path, compile_output, CaptureFailure, CaptureRequest, CaptureResult, |
| 11 | | - CapturedStage, EmitMode, FailureStage, OptLevel, RunCapture, Stage, |
| 10 | + linked_capture_available, object_snapshot_text, ArmfortasAdapters, ArmfortasCliAdapter, |
| 11 | + CaptureBackend, CaptureFailure, CaptureRequest, CaptureResult, CapturedStage, |
| 12 | + CliObservableCaptureBackend, EmitMode, FailureStage, OptLevel, RunCapture, Stage, |
| 13 | +}; |
| 14 | +use bencch_core::{ |
| 15 | + ArtifactDifference, ArtifactKey, ArtifactValue, ComparisonResult, CompilerCapabilities, |
| 16 | + CompilerObservation, CompilerSpec, NamedCompiler, ObservationProvenance, |
| 12 | 17 | }; |
| 13 | 18 | |
| 14 | 19 | const SUITE_EXTENSION: &str = "afs"; |
@@ -28,12 +33,15 @@ struct CaseSpec { |
| 28 | 33 | source: PathBuf, |
| 29 | 34 | graph_files: Vec<PathBuf>, |
| 30 | 35 | requested: BTreeSet<Stage>, |
| 36 | + generic_introspect: Option<GenericIntrospectCase>, |
| 37 | + generic_compare: Option<GenericCompareCase>, |
| 31 | 38 | opt_levels: Vec<OptLevel>, |
| 32 | 39 | repeat_count: usize, |
| 33 | 40 | reference_compilers: Vec<ReferenceCompiler>, |
| 34 | 41 | consistency_checks: Vec<ConsistencyCheck>, |
| 35 | 42 | expectations: Vec<Expectation>, |
| 36 | 43 | status_rules: Vec<StatusRule>, |
| 44 | + capability_policy: Option<CapabilityPolicy>, |
| 37 | 45 | } |
| 38 | 46 | |
| 39 | 47 | impl CaseSpec { |
@@ -41,6 +49,14 @@ impl CaseSpec { |
| 41 | 49 | !self.graph_files.is_empty() |
| 42 | 50 | } |
| 43 | 51 | |
| 52 | + fn is_generic_introspect(&self) -> bool { |
| 53 | + self.generic_introspect.is_some() |
| 54 | + } |
| 55 | + |
| 56 | + fn is_generic_compare(&self) -> bool { |
| 57 | + self.generic_compare.is_some() |
| 58 | + } |
| 59 | + |
| 44 | 60 | fn source_label(&self) -> String { |
| 45 | 61 | if self.is_graph() { |
| 46 | 62 | format!( |
@@ -54,6 +70,19 @@ impl CaseSpec { |
| 54 | 70 | } |
| 55 | 71 | } |
| 56 | 72 | |
| 73 | +#[derive(Debug, Clone)] |
| 74 | +struct GenericIntrospectCase { |
| 75 | + compiler: CompilerSpec, |
| 76 | + artifacts: BTreeSet<ArtifactKey>, |
| 77 | +} |
| 78 | + |
| 79 | +#[derive(Debug, Clone)] |
| 80 | +struct GenericCompareCase { |
| 81 | + left: CompilerSpec, |
| 82 | + right: CompilerSpec, |
| 83 | + artifacts: BTreeSet<ArtifactKey>, |
| 84 | +} |
| 85 | + |
| 57 | 86 | #[derive(Debug, Clone)] |
| 58 | 87 | struct PreparedInput { |
| 59 | 88 | compiler_source: PathBuf, |
@@ -68,6 +97,18 @@ struct StatusRule { |
| 68 | 97 | reason: String, |
| 69 | 98 | } |
| 70 | 99 | |
| 100 | +#[derive(Debug, Clone)] |
| 101 | +struct CapabilityPolicy { |
| 102 | + kind: StatusKind, |
| 103 | + reason: String, |
| 104 | +} |
| 105 | + |
| 106 | +#[derive(Debug, Clone)] |
| 107 | +enum PendingStatusRule { |
| 108 | + Explicit(StatusRule), |
| 109 | + XfailSourceComments, |
| 110 | +} |
| 111 | + |
| 71 | 112 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 72 | 113 | enum StatusKind { |
| 73 | 114 | Xfail, |
@@ -159,6 +200,25 @@ impl ConsistencyCheck { |
| 159 | 200 | Self::CaptureRunVsCliRun | Self::CaptureRunReproducible => Some(Stage::Run), |
| 160 | 201 | } |
| 161 | 202 | } |
| 203 | + |
| 204 | + fn requires_capture_result(&self) -> bool { |
| 205 | + matches!( |
| 206 | + self, |
| 207 | + Self::CaptureAsmVsCliAsm |
| 208 | + | Self::CaptureObjVsCliObj |
| 209 | + | Self::CaptureRunVsCliRun |
| 210 | + | Self::CaptureAsmReproducible |
| 211 | + | Self::CaptureObjReproducible |
| 212 | + | Self::CaptureRunReproducible |
| 213 | + ) |
| 214 | + } |
| 215 | + |
| 216 | + fn supports_generic_introspect(&self) -> bool { |
| 217 | + matches!( |
| 218 | + self, |
| 219 | + Self::CliAsmReproducible | Self::CliObjReproducible | Self::CliRunReproducible |
| 220 | + ) |
| 221 | + } |
| 162 | 222 | } |
| 163 | 223 | |
| 164 | 224 | #[derive(Debug, Clone)] |
@@ -170,11 +230,19 @@ enum Expectation { |
| 170 | 230 | IntEquals { target: Target, value: i32 }, |
| 171 | 231 | FailContains { stage: FailureStage, needle: String }, |
| 172 | 232 | FailEquals { stage: FailureStage, value: String }, |
| 233 | + FailSourceComments, |
| 234 | + FailCommentPatterns(Vec<String>), |
| 173 | 235 | } |
| 174 | 236 | |
| 175 | | -#[derive(Debug, Clone, Copy)] |
| 237 | +#[derive(Debug, Clone)] |
| 176 | 238 | enum Target { |
| 177 | 239 | Stage(Stage), |
| 240 | + Artifact(ArtifactKey), |
| 241 | + CompareStatus, |
| 242 | + CompareClassification, |
| 243 | + CompareChangedArtifacts, |
| 244 | + CompareDifferenceCount, |
| 245 | + CompareBasis, |
| 178 | 246 | RunStdout, |
| 179 | 247 | RunStderr, |
| 180 | 248 | RunExitCode, |
@@ -207,17 +275,15 @@ impl ReferenceCompiler { |
| 207 | 275 | } |
| 208 | 276 | } |
| 209 | 277 | |
| 210 | | -#[derive(Debug, Clone, PartialEq, Eq)] |
| 211 | | -enum ArmfortasCliAdapter { |
| 212 | | - Linked, |
| 213 | | - External(String), |
| 214 | | -} |
| 215 | | - |
| 216 | 278 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 217 | 279 | struct ToolchainConfig { |
| 218 | 280 | armfortas: ArmfortasCliAdapter, |
| 219 | 281 | gfortran: String, |
| 220 | 282 | flang_new: String, |
| 283 | + lfortran: String, |
| 284 | + ifort: String, |
| 285 | + ifx: String, |
| 286 | + nvfortran: String, |
| 221 | 287 | system_as: String, |
| 222 | 288 | otool: String, |
| 223 | 289 | nm: String, |
@@ -228,28 +294,32 @@ impl ToolchainConfig { |
| 228 | 294 | Self { |
| 229 | 295 | armfortas: match std::env::var("BENCCH_ARMFORTAS_BIN") { |
| 230 | 296 | Ok(value) if !value.trim().is_empty() => ArmfortasCliAdapter::External(value), |
| 231 | | - _ => ArmfortasCliAdapter::Linked, |
| 297 | + _ if linked_capture_available() => ArmfortasCliAdapter::Linked, |
| 298 | + _ => ArmfortasCliAdapter::External("armfortas".into()), |
| 232 | 299 | }, |
| 233 | 300 | gfortran: tool_override("BENCCH_GFORTRAN_BIN", "gfortran"), |
| 234 | 301 | flang_new: tool_override("BENCCH_FLANG_BIN", "flang-new"), |
| 302 | + lfortran: tool_override("BENCCH_LFORTRAN_BIN", "lfortran"), |
| 303 | + ifort: tool_override("BENCCH_IFORT_BIN", "ifort"), |
| 304 | + ifx: tool_override("BENCCH_IFX_BIN", "ifx"), |
| 305 | + nvfortran: tool_override("BENCCH_NVFORTRAN_BIN", "nvfortran"), |
| 235 | 306 | system_as: tool_override("BENCCH_AS_BIN", "as"), |
| 236 | 307 | otool: tool_override("BENCCH_OTOOL_BIN", "otool"), |
| 237 | 308 | nm: tool_override("BENCCH_NM_BIN", "nm"), |
| 238 | 309 | } |
| 239 | 310 | } |
| 240 | 311 | |
| 241 | | - fn armfortas_command_name(&self) -> &str { |
| 242 | | - match &self.armfortas { |
| 243 | | - ArmfortasCliAdapter::Linked => "armfortas (linked)", |
| 244 | | - ArmfortasCliAdapter::External(binary) => binary, |
| 245 | | - } |
| 312 | + fn armfortas_adapters(&self) -> ArmfortasAdapters { |
| 313 | + ArmfortasAdapters::new(self.armfortas.clone()) |
| 246 | 314 | } |
| 247 | 315 | |
| 248 | | - fn armfortas_external_bin(&self) -> Option<&str> { |
| 249 | | - match &self.armfortas { |
| 250 | | - ArmfortasCliAdapter::Linked => None, |
| 251 | | - ArmfortasCliAdapter::External(binary) => Some(binary), |
| 252 | | - } |
| 316 | + fn cli_observable_capture_backend(&self, work_root: PathBuf) -> CliObservableCaptureBackend { |
| 317 | + CliObservableCaptureBackend::new( |
| 318 | + self.armfortas.clone(), |
| 319 | + work_root, |
| 320 | + self.otool.clone(), |
| 321 | + self.nm.clone(), |
| 322 | + ) |
| 253 | 323 | } |
| 254 | 324 | |
| 255 | 325 | fn reference_binary(&self, compiler: ReferenceCompiler) -> &str { |
@@ -270,6 +340,297 @@ impl ToolchainConfig { |
| 270 | 340 | fn nm_bin(&self) -> &str { |
| 271 | 341 | &self.nm |
| 272 | 342 | } |
| 343 | + |
| 344 | + fn named_compiler_binary(&self, compiler: NamedCompiler) -> Option<String> { |
| 345 | + match compiler { |
| 346 | + NamedCompiler::Armfortas => match &self.armfortas { |
| 347 | + ArmfortasCliAdapter::Linked => None, |
| 348 | + ArmfortasCliAdapter::External(binary) => Some(binary.clone()), |
| 349 | + }, |
| 350 | + NamedCompiler::Gfortran => Some(self.gfortran.clone()), |
| 351 | + NamedCompiler::FlangNew => Some(self.flang_new.clone()), |
| 352 | + NamedCompiler::LFortran => Some(self.lfortran.clone()), |
| 353 | + NamedCompiler::Ifort => Some(self.ifort.clone()), |
| 354 | + NamedCompiler::Ifx => Some(self.ifx.clone()), |
| 355 | + NamedCompiler::Nvfortran => Some(self.nvfortran.clone()), |
| 356 | + } |
| 357 | + } |
| 358 | +} |
| 359 | + |
| 360 | +fn generic_external_capabilities(spec: CompilerSpec) -> CompilerCapabilities { |
| 361 | + CompilerCapabilities::new(spec).support_all([ |
| 362 | + ArtifactKey::Diagnostics, |
| 363 | + ArtifactKey::ExitCode, |
| 364 | + ArtifactKey::Stdout, |
| 365 | + ArtifactKey::Stderr, |
| 366 | + ArtifactKey::Asm, |
| 367 | + ArtifactKey::Obj, |
| 368 | + ArtifactKey::Executable, |
| 369 | + ArtifactKey::Runtime, |
| 370 | + ]) |
| 371 | +} |
| 372 | + |
| 373 | +fn armfortas_capabilities(tools: &ToolchainConfig) -> CompilerCapabilities { |
| 374 | + let mut capabilities = |
| 375 | + generic_external_capabilities(CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 376 | + let linked_reason = "linked armfortas capture is unavailable in this build; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary".to_string(); |
| 377 | + let capture_available = tools.armfortas_adapters().capture_mode_name() != "unavailable"; |
| 378 | + for stage in Stage::ALL { |
| 379 | + if matches!(stage, Stage::Asm | Stage::Obj | Stage::Run) { |
| 380 | + continue; |
| 381 | + } |
| 382 | + let artifact = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 383 | + capabilities = if capture_available { |
| 384 | + capabilities.support(artifact) |
| 385 | + } else { |
| 386 | + capabilities.mark_unavailable(artifact, linked_reason.clone()) |
| 387 | + }; |
| 388 | + } |
| 389 | + capabilities |
| 390 | +} |
| 391 | + |
| 392 | +fn compiler_capabilities(spec: &CompilerSpec, tools: &ToolchainConfig) -> CompilerCapabilities { |
| 393 | + match spec { |
| 394 | + CompilerSpec::Named(NamedCompiler::Armfortas) => armfortas_capabilities(tools), |
| 395 | + CompilerSpec::Named(named) => generic_external_capabilities(CompilerSpec::Named(*named)), |
| 396 | + CompilerSpec::Binary(path) => { |
| 397 | + generic_external_capabilities(CompilerSpec::Binary(path.clone())) |
| 398 | + } |
| 399 | + } |
| 400 | +} |
| 401 | + |
| 402 | +fn capability_extra_summary(extras: &BTreeMap<String, Vec<String>>) -> String { |
| 403 | + if extras.is_empty() { |
| 404 | + return "none".to_string(); |
| 405 | + } |
| 406 | + |
| 407 | + extras |
| 408 | + .iter() |
| 409 | + .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 410 | + .collect::<Vec<_>>() |
| 411 | + .join(", ") |
| 412 | +} |
| 413 | + |
| 414 | +fn capability_unavailable_summary(capabilities: &CompilerCapabilities) -> String { |
| 415 | + if capabilities.unavailable_artifacts.is_empty() { |
| 416 | + return "none".to_string(); |
| 417 | + } |
| 418 | + |
| 419 | + let mut grouped = BTreeMap::<String, Vec<String>>::new(); |
| 420 | + for artifact in capabilities.unavailable_artifacts.keys() { |
| 421 | + if let Some((namespace, local_name)) = artifact.extra_parts() { |
| 422 | + grouped |
| 423 | + .entry(namespace.to_string()) |
| 424 | + .or_insert_with(Vec::new) |
| 425 | + .push(local_name.to_string()); |
| 426 | + } else { |
| 427 | + grouped |
| 428 | + .entry("generic".to_string()) |
| 429 | + .or_insert_with(Vec::new) |
| 430 | + .push(artifact.as_str().to_string()); |
| 431 | + } |
| 432 | + } |
| 433 | + |
| 434 | + grouped |
| 435 | + .iter() |
| 436 | + .map(|(namespace, names)| format!("{}({})", namespace, names.join(", "))) |
| 437 | + .collect::<Vec<_>>() |
| 438 | + .join(", ") |
| 439 | +} |
| 440 | + |
| 441 | +fn named_compiler_status_value( |
| 442 | + named: NamedCompiler, |
| 443 | + tools: &ToolchainConfig, |
| 444 | + capture_root: Option<&PathBuf>, |
| 445 | +) -> String { |
| 446 | + let probe = named_compiler_probe(named, tools, capture_root); |
| 447 | + match probe.resolved_path { |
| 448 | + Some(path) => format!( |
| 449 | + "configured={} resolved={}", |
| 450 | + probe.configured, |
| 451 | + path.display() |
| 452 | + ), |
| 453 | + None => { |
| 454 | + if probe.status == "linked" { |
| 455 | + probe |
| 456 | + .detail |
| 457 | + .unwrap_or_else(|| "linked via Cargo".to_string()) |
| 458 | + } else { |
| 459 | + format!("configured={} resolved=missing", probe.configured) |
| 460 | + } |
| 461 | + } |
| 462 | + } |
| 463 | +} |
| 464 | + |
| 465 | +fn append_named_compiler_fields( |
| 466 | + fields: &mut Vec<(String, String)>, |
| 467 | + named: NamedCompiler, |
| 468 | + tools: &ToolchainConfig, |
| 469 | + capture_root: Option<&PathBuf>, |
| 470 | +) { |
| 471 | + let prefix = format!("named_compiler.{}", named.as_str()); |
| 472 | + let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 473 | + let probe = named_compiler_probe(named, tools, capture_root); |
| 474 | + if named == NamedCompiler::Armfortas { |
| 475 | + let armfortas = tools.armfortas_adapters(); |
| 476 | + fields.push(( |
| 477 | + prefix.clone(), |
| 478 | + format!( |
| 479 | + "cli={} capture={}", |
| 480 | + armfortas.cli_mode_name(), |
| 481 | + armfortas.capture_mode_name() |
| 482 | + ), |
| 483 | + )); |
| 484 | + } else { |
| 485 | + fields.push(( |
| 486 | + prefix.clone(), |
| 487 | + named_compiler_status_value(named, tools, capture_root), |
| 488 | + )); |
| 489 | + } |
| 490 | + fields.push(( |
| 491 | + format!("{}.accepted_names", prefix), |
| 492 | + named.accepted_names().join(", "), |
| 493 | + )); |
| 494 | + fields.push(( |
| 495 | + format!("{}.candidate_binaries", prefix), |
| 496 | + named.candidate_binaries().join(", "), |
| 497 | + )); |
| 498 | + fields.push(( |
| 499 | + format!("{}.generic_artifacts", prefix), |
| 500 | + format_artifact_name_list(&capabilities.generic_artifacts()), |
| 501 | + )); |
| 502 | + fields.push(( |
| 503 | + format!("{}.adapter_extras", prefix), |
| 504 | + capability_extra_summary(&capabilities.adapter_extras()), |
| 505 | + )); |
| 506 | + fields.push(( |
| 507 | + format!("{}.unavailable_artifacts", prefix), |
| 508 | + capability_unavailable_summary(&capabilities), |
| 509 | + )); |
| 510 | + fields.push((format!("{}.probe_status", prefix), probe.status.clone())); |
| 511 | + fields.push(( |
| 512 | + format!("{}.probe_resolved_path", prefix), |
| 513 | + probe |
| 514 | + .resolved_path |
| 515 | + .as_ref() |
| 516 | + .map(|path| display_path(path)) |
| 517 | + .unwrap_or_else(|| "none".to_string()), |
| 518 | + )); |
| 519 | + fields.push(( |
| 520 | + format!("{}.probe_banner", prefix), |
| 521 | + probe.banner.clone().unwrap_or_else(|| "none".to_string()), |
| 522 | + )); |
| 523 | + fields.push(( |
| 524 | + format!("{}.probe_detail", prefix), |
| 525 | + probe.detail.clone().unwrap_or_else(|| "none".to_string()), |
| 526 | + )); |
| 527 | +} |
| 528 | + |
| 529 | +fn compiler_capability_backend(spec: &CompilerSpec, tools: &ToolchainConfig) -> (String, String) { |
| 530 | + match spec { |
| 531 | + CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 532 | + let adapters = tools.armfortas_adapters(); |
| 533 | + ( |
| 534 | + adapters.capture_mode_name().to_string(), |
| 535 | + adapters.capture_description().to_string(), |
| 536 | + ) |
| 537 | + } |
| 538 | + CompilerSpec::Named(named) => { |
| 539 | + let binary = tools |
| 540 | + .named_compiler_binary(*named) |
| 541 | + .unwrap_or_else(|| named.as_str().to_string()); |
| 542 | + ( |
| 543 | + "external-driver".to_string(), |
| 544 | + format!("generic external driver adapter using {}", binary), |
| 545 | + ) |
| 546 | + } |
| 547 | + CompilerSpec::Binary(path) => ( |
| 548 | + "external-driver".to_string(), |
| 549 | + format!("generic external driver adapter using {}", path.display()), |
| 550 | + ), |
| 551 | + } |
| 552 | +} |
| 553 | + |
| 554 | +fn observation_from_capability_mismatch( |
| 555 | + spec: &CompilerSpec, |
| 556 | + program: &Path, |
| 557 | + opt_level: OptLevel, |
| 558 | + requested: BTreeSet<ArtifactKey>, |
| 559 | + backend_mode: String, |
| 560 | + backend_detail: String, |
| 561 | + detail: String, |
| 562 | +) -> ObservedProgram { |
| 563 | + ObservedProgram { |
| 564 | + observation: CompilerObservation { |
| 565 | + compiler: spec.clone(), |
| 566 | + program: program.to_path_buf(), |
| 567 | + opt_level, |
| 568 | + compile_exit_code: 1, |
| 569 | + artifacts: BTreeMap::from([(ArtifactKey::Diagnostics, ArtifactValue::Text(detail))]), |
| 570 | + provenance: ObservationProvenance { |
| 571 | + compiler_identity: spec.display_name(), |
| 572 | + adapter_kind: match spec { |
| 573 | + CompilerSpec::Named(_) => "named".into(), |
| 574 | + CompilerSpec::Binary(_) => "explicit-path".into(), |
| 575 | + }, |
| 576 | + backend_mode, |
| 577 | + backend_detail, |
| 578 | + artifacts_captured: vec!["diagnostics".into()], |
| 579 | + comparison_basis: None, |
| 580 | + failure_stage: None, |
| 581 | + }, |
| 582 | + }, |
| 583 | + requested_artifacts: requested, |
| 584 | + } |
| 585 | +} |
| 586 | + |
| 587 | +fn preflight_introspection_request( |
| 588 | + spec: &CompilerSpec, |
| 589 | + program: &Path, |
| 590 | + opt_level: OptLevel, |
| 591 | + requested: &BTreeSet<ArtifactKey>, |
| 592 | + tools: &ToolchainConfig, |
| 593 | +) -> Option<ObservedProgram> { |
| 594 | + let capabilities = compiler_capabilities(spec, tools); |
| 595 | + let (backend_mode, backend_detail) = compiler_capability_backend(spec, tools); |
| 596 | + |
| 597 | + let unavailable = capabilities.unavailable_requests(requested); |
| 598 | + if !unavailable.is_empty() { |
| 599 | + let detail = unavailable |
| 600 | + .into_iter() |
| 601 | + .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 602 | + .collect::<Vec<_>>() |
| 603 | + .join("\n"); |
| 604 | + return Some(observation_from_capability_mismatch( |
| 605 | + spec, |
| 606 | + program, |
| 607 | + opt_level, |
| 608 | + requested.clone(), |
| 609 | + backend_mode, |
| 610 | + backend_detail, |
| 611 | + detail, |
| 612 | + )); |
| 613 | + } |
| 614 | + |
| 615 | + let unsupported = capabilities.unsupported_requests(requested); |
| 616 | + if !unsupported.is_empty() { |
| 617 | + let detail = format!( |
| 618 | + "{} does not support requested artifacts in this adapter: {}", |
| 619 | + spec.display_name(), |
| 620 | + unsupported.join(", ") |
| 621 | + ); |
| 622 | + return Some(observation_from_capability_mismatch( |
| 623 | + spec, |
| 624 | + program, |
| 625 | + opt_level, |
| 626 | + requested.clone(), |
| 627 | + backend_mode, |
| 628 | + backend_detail, |
| 629 | + detail, |
| 630 | + )); |
| 631 | + } |
| 632 | + |
| 633 | + None |
| 273 | 634 | } |
| 274 | 635 | |
| 275 | 636 | fn tool_override(var: &str, default: &str) -> String { |
@@ -296,6 +657,7 @@ struct Outcome { |
| 296 | 657 | kind: OutcomeKind, |
| 297 | 658 | detail: String, |
| 298 | 659 | bundle: Option<PathBuf>, |
| 660 | + primary_backend: Option<PrimaryBackendReport>, |
| 299 | 661 | consistency_observations: Vec<ConsistencyObservation>, |
| 300 | 662 | } |
| 301 | 663 | |
@@ -306,6 +668,7 @@ struct Summary { |
| 306 | 668 | xfailed: usize, |
| 307 | 669 | xpassed: usize, |
| 308 | 670 | future: usize, |
| 671 | + outcomes: Vec<Outcome>, |
| 309 | 672 | consistency: BTreeMap<ConsistencyCheck, ConsistencyRollup>, |
| 310 | 673 | } |
| 311 | 674 | |
@@ -328,6 +691,13 @@ struct ConsistencyObservation { |
| 328 | 691 | stable_components: Vec<String>, |
| 329 | 692 | } |
| 330 | 693 | |
| 694 | +#[derive(Debug, Clone)] |
| 695 | +struct ListConfig { |
| 696 | + suite_filter: Option<String>, |
| 697 | + verbose: bool, |
| 698 | + tools: ToolchainConfig, |
| 699 | +} |
| 700 | + |
| 331 | 701 | #[derive(Debug, Clone)] |
| 332 | 702 | struct RunConfig { |
| 333 | 703 | suite_filter: Option<String>, |
@@ -337,6 +707,34 @@ struct RunConfig { |
| 337 | 707 | fail_fast: bool, |
| 338 | 708 | include_future: bool, |
| 339 | 709 | all_stages: bool, |
| 710 | + json_report: Option<PathBuf>, |
| 711 | + markdown_report: Option<PathBuf>, |
| 712 | + tools: ToolchainConfig, |
| 713 | +} |
| 714 | + |
| 715 | +#[derive(Debug, Clone)] |
| 716 | +struct CompareConfig { |
| 717 | + left: CompilerSpec, |
| 718 | + right: CompilerSpec, |
| 719 | + program: PathBuf, |
| 720 | + opt_level: OptLevel, |
| 721 | + artifacts: BTreeSet<ArtifactKey>, |
| 722 | + json_report: Option<PathBuf>, |
| 723 | + markdown_report: Option<PathBuf>, |
| 724 | + tools: ToolchainConfig, |
| 725 | +} |
| 726 | + |
| 727 | +#[derive(Debug, Clone)] |
| 728 | +struct IntrospectConfig { |
| 729 | + compiler: CompilerSpec, |
| 730 | + program: PathBuf, |
| 731 | + opt_level: OptLevel, |
| 732 | + artifacts: BTreeSet<ArtifactKey>, |
| 733 | + json_report: Option<PathBuf>, |
| 734 | + markdown_report: Option<PathBuf>, |
| 735 | + all_artifacts: bool, |
| 736 | + summary_only: bool, |
| 737 | + max_artifact_lines: Option<usize>, |
| 340 | 738 | tools: ToolchainConfig, |
| 341 | 739 | } |
| 342 | 740 | |
@@ -345,10 +743,61 @@ struct ExecutionArtifacts { |
| 345 | 743 | requested: BTreeSet<Stage>, |
| 346 | 744 | armfortas: Option<CaptureResult>, |
| 347 | 745 | armfortas_failure: Option<CaptureFailure>, |
| 746 | + armfortas_observation: Option<ObservedProgram>, |
| 348 | 747 | references: Vec<ReferenceResult>, |
| 748 | + reference_observations: Vec<ObservedProgram>, |
| 349 | 749 | consistency_issues: Vec<ConsistencyIssue>, |
| 350 | 750 | } |
| 351 | 751 | |
| 752 | +#[derive(Debug, Clone)] |
| 753 | +struct ObservedProgram { |
| 754 | + observation: CompilerObservation, |
| 755 | + requested_artifacts: BTreeSet<ArtifactKey>, |
| 756 | +} |
| 757 | + |
| 758 | +#[derive(Debug, Clone, Copy)] |
| 759 | +struct IntrospectionRenderConfig { |
| 760 | + summary_only: bool, |
| 761 | + max_artifact_lines: Option<usize>, |
| 762 | +} |
| 763 | + |
| 764 | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 765 | +enum PrimaryCaptureBackendKind { |
| 766 | + Full, |
| 767 | + Observable, |
| 768 | +} |
| 769 | + |
| 770 | +impl PrimaryCaptureBackendKind { |
| 771 | + fn as_str(&self) -> &'static str { |
| 772 | + match self { |
| 773 | + Self::Full => "full", |
| 774 | + Self::Observable => "observable", |
| 775 | + } |
| 776 | + } |
| 777 | +} |
| 778 | + |
| 779 | +struct SelectedPrimaryBackend { |
| 780 | + kind: PrimaryCaptureBackendKind, |
| 781 | + backend: Box<dyn CaptureBackend>, |
| 782 | +} |
| 783 | + |
| 784 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 785 | +struct PrimaryBackendReport { |
| 786 | + kind: String, |
| 787 | + mode: String, |
| 788 | + detail: String, |
| 789 | +} |
| 790 | + |
| 791 | +impl PrimaryBackendReport { |
| 792 | + fn from_selected(selected: &SelectedPrimaryBackend) -> Self { |
| 793 | + Self { |
| 794 | + kind: selected.kind.as_str().to_string(), |
| 795 | + mode: selected.backend.mode_name().to_string(), |
| 796 | + detail: selected.backend.description().to_string(), |
| 797 | + } |
| 798 | + } |
| 799 | +} |
| 800 | + |
| 352 | 801 | #[derive(Debug, Clone)] |
| 353 | 802 | struct ReferenceResult { |
| 354 | 803 | compiler: ReferenceCompiler, |
@@ -381,6 +830,7 @@ impl Summary { |
| 381 | 830 | OutcomeKind::Xpass => self.xpassed += 1, |
| 382 | 831 | OutcomeKind::Future => self.future += 1, |
| 383 | 832 | } |
| 833 | + self.outcomes.push(outcome.clone()); |
| 384 | 834 | self.record_consistency(&outcome.consistency_observations); |
| 385 | 835 | } |
| 386 | 836 | |
@@ -435,10 +885,6 @@ impl ReferenceResult { |
| 435 | 885 | run_error: None, |
| 436 | 886 | } |
| 437 | 887 | } |
| 438 | | - |
| 439 | | - fn run_signature(&self) -> Option<RunSignature> { |
| 440 | | - self.run.as_ref().map(normalize_run_signature) |
| 441 | | - } |
| 442 | 888 | } |
| 443 | 889 | |
| 444 | 890 | #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] |
@@ -449,20 +895,31 @@ struct RunSignature { |
| 449 | 895 | } |
| 450 | 896 | |
| 451 | 897 | pub fn run_cli(args: &[String]) -> i32 { |
| 898 | + run_cli_named("afs-tests", args) |
| 899 | +} |
| 900 | + |
| 901 | +pub fn run_cli_named(program_name: &str, args: &[String]) -> i32 { |
| 452 | 902 | match parse_cli(args) { |
| 453 | | - Ok(CommandKind::List { suite_filter }) => match discover_suites(default_suite_root()) { |
| 903 | + Ok(CommandKind::List(config)) => match discover_suites(default_suite_root()) { |
| 454 | 904 | Ok(suites) => { |
| 455 | | - print_suites(&filter_suites(&suites, suite_filter.as_deref())); |
| 905 | + print_suites( |
| 906 | + &filter_suites(&suites, config.suite_filter.as_deref()), |
| 907 | + &config, |
| 908 | + ); |
| 456 | 909 | 0 |
| 457 | 910 | } |
| 458 | 911 | Err(err) => { |
| 459 | | - eprintln!("afs-tests: {}", err); |
| 912 | + eprintln!("{}: {}", program_name, err); |
| 460 | 913 | 1 |
| 461 | 914 | } |
| 462 | 915 | }, |
| 463 | 916 | Ok(CommandKind::Run(config)) => match run_suites(&config) { |
| 464 | 917 | Ok(summary) => { |
| 465 | 918 | print_summary(&summary); |
| 919 | + if let Err(err) = write_requested_reports(&config, &summary) { |
| 920 | + eprintln!("afs-tests: {}", err); |
| 921 | + return 1; |
| 922 | + } |
| 466 | 923 | if summary.failed == 0 && summary.xpassed == 0 { |
| 467 | 924 | 0 |
| 468 | 925 | } else { |
@@ -470,28 +927,146 @@ pub fn run_cli(args: &[String]) -> i32 { |
| 470 | 927 | } |
| 471 | 928 | } |
| 472 | 929 | Err(err) => { |
| 473 | | - eprintln!("afs-tests: {}", err); |
| 930 | + eprintln!("{}: {}", program_name, err); |
| 931 | + 1 |
| 932 | + } |
| 933 | + }, |
| 934 | + Ok(CommandKind::Compare(config)) => match run_compare(&config) { |
| 935 | + Ok(result) => { |
| 936 | + print_compare_result(&result); |
| 937 | + if let Err(err) = write_compare_reports(&config, &result) { |
| 938 | + eprintln!("{}: {}", program_name, err); |
| 939 | + return 1; |
| 940 | + } |
| 941 | + if result.differences.is_empty() { |
| 942 | + 0 |
| 943 | + } else { |
| 944 | + 1 |
| 945 | + } |
| 946 | + } |
| 947 | + Err(err) => { |
| 948 | + eprintln!("{}: {}", program_name, err); |
| 949 | + 1 |
| 950 | + } |
| 951 | + }, |
| 952 | + Ok(CommandKind::Introspect(config)) => match run_introspect(&config) { |
| 953 | + Ok(observation) => { |
| 954 | + print_introspection(&config, &observation); |
| 955 | + if let Err(err) = write_introspection_reports(&config, &observation) { |
| 956 | + eprintln!("{}: {}", program_name, err); |
| 957 | + return 1; |
| 958 | + } |
| 959 | + if observation.observation.compile_exit_code == 0 { |
| 960 | + 0 |
| 961 | + } else { |
| 962 | + 1 |
| 963 | + } |
| 964 | + } |
| 965 | + Err(err) => { |
| 966 | + eprintln!("{}: {}", program_name, err); |
| 474 | 967 | 1 |
| 475 | 968 | } |
| 476 | 969 | }, |
| 970 | + Ok(CommandKind::Doctor(config)) => { |
| 971 | + println!("{}", render_doctor_report(&config)); |
| 972 | + if let Err(err) = write_doctor_reports(&config) { |
| 973 | + eprintln!("{}: {}", program_name, err); |
| 974 | + return 1; |
| 975 | + } |
| 976 | + 0 |
| 977 | + } |
| 477 | 978 | Ok(CommandKind::Help) => { |
| 478 | | - print_usage(); |
| 979 | + print_usage(program_name); |
| 479 | 980 | 0 |
| 480 | 981 | } |
| 481 | 982 | Err(err) => { |
| 482 | | - eprintln!("afs-tests: {}", err); |
| 483 | | - print_usage(); |
| 983 | + eprintln!("{}: {}", program_name, err); |
| 984 | + print_usage(program_name); |
| 484 | 985 | 2 |
| 485 | 986 | } |
| 486 | 987 | } |
| 487 | 988 | } |
| 488 | 989 | |
| 489 | 990 | enum CommandKind { |
| 490 | | - List { suite_filter: Option<String> }, |
| 991 | + List(ListConfig), |
| 491 | 992 | Run(RunConfig), |
| 993 | + Compare(CompareConfig), |
| 994 | + Introspect(IntrospectConfig), |
| 995 | + Doctor(DoctorConfig), |
| 492 | 996 | Help, |
| 493 | 997 | } |
| 494 | 998 | |
| 999 | +#[derive(Debug, Clone)] |
| 1000 | +struct DoctorConfig { |
| 1001 | + tools: ToolchainConfig, |
| 1002 | + json_report: Option<PathBuf>, |
| 1003 | + markdown_report: Option<PathBuf>, |
| 1004 | +} |
| 1005 | + |
| 1006 | +fn parse_tool_override_arg( |
| 1007 | + arg: &str, |
| 1008 | + queue: &mut VecDeque<&String>, |
| 1009 | + tools: &mut ToolchainConfig, |
| 1010 | +) -> Result<bool, String> { |
| 1011 | + match arg { |
| 1012 | + "--armfortas-bin" => { |
| 1013 | + let value = queue |
| 1014 | + .pop_front() |
| 1015 | + .ok_or("--armfortas-bin requires a value")?; |
| 1016 | + tools.armfortas = ArmfortasCliAdapter::External(value.clone()); |
| 1017 | + Ok(true) |
| 1018 | + } |
| 1019 | + "--gfortran-bin" => { |
| 1020 | + let value = queue.pop_front().ok_or("--gfortran-bin requires a value")?; |
| 1021 | + tools.gfortran = value.clone(); |
| 1022 | + Ok(true) |
| 1023 | + } |
| 1024 | + "--flang-bin" => { |
| 1025 | + let value = queue.pop_front().ok_or("--flang-bin requires a value")?; |
| 1026 | + tools.flang_new = value.clone(); |
| 1027 | + Ok(true) |
| 1028 | + } |
| 1029 | + "--lfortran-bin" => { |
| 1030 | + let value = queue.pop_front().ok_or("--lfortran-bin requires a value")?; |
| 1031 | + tools.lfortran = value.clone(); |
| 1032 | + Ok(true) |
| 1033 | + } |
| 1034 | + "--ifort-bin" => { |
| 1035 | + let value = queue.pop_front().ok_or("--ifort-bin requires a value")?; |
| 1036 | + tools.ifort = value.clone(); |
| 1037 | + Ok(true) |
| 1038 | + } |
| 1039 | + "--ifx-bin" => { |
| 1040 | + let value = queue.pop_front().ok_or("--ifx-bin requires a value")?; |
| 1041 | + tools.ifx = value.clone(); |
| 1042 | + Ok(true) |
| 1043 | + } |
| 1044 | + "--nvfortran-bin" => { |
| 1045 | + let value = queue |
| 1046 | + .pop_front() |
| 1047 | + .ok_or("--nvfortran-bin requires a value")?; |
| 1048 | + tools.nvfortran = value.clone(); |
| 1049 | + Ok(true) |
| 1050 | + } |
| 1051 | + "--as-bin" => { |
| 1052 | + let value = queue.pop_front().ok_or("--as-bin requires a value")?; |
| 1053 | + tools.system_as = value.clone(); |
| 1054 | + Ok(true) |
| 1055 | + } |
| 1056 | + "--otool-bin" => { |
| 1057 | + let value = queue.pop_front().ok_or("--otool-bin requires a value")?; |
| 1058 | + tools.otool = value.clone(); |
| 1059 | + Ok(true) |
| 1060 | + } |
| 1061 | + "--nm-bin" => { |
| 1062 | + let value = queue.pop_front().ok_or("--nm-bin requires a value")?; |
| 1063 | + tools.nm = value.clone(); |
| 1064 | + Ok(true) |
| 1065 | + } |
| 1066 | + _ => Ok(false), |
| 1067 | + } |
| 1068 | +} |
| 1069 | + |
| 495 | 1070 | fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 496 | 1071 | if args.is_empty() { |
| 497 | 1072 | return Ok(CommandKind::Help); |
@@ -499,19 +1074,27 @@ fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 499 | 1074 | |
| 500 | 1075 | match args[0].as_str() { |
| 501 | 1076 | "list" => { |
| 502 | | - let mut suite_filter = None; |
| 1077 | + let mut config = ListConfig { |
| 1078 | + suite_filter: None, |
| 1079 | + verbose: false, |
| 1080 | + tools: ToolchainConfig::from_env(), |
| 1081 | + }; |
| 503 | 1082 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 504 | 1083 | while let Some(arg) = queue.pop_front() { |
| 1084 | + if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1085 | + continue; |
| 1086 | + } |
| 505 | 1087 | match arg.as_str() { |
| 506 | 1088 | "--suite" => { |
| 507 | 1089 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
| 508 | | - suite_filter = Some(value.clone()); |
| 1090 | + config.suite_filter = Some(value.clone()); |
| 509 | 1091 | } |
| 1092 | + "--verbose" => config.verbose = true, |
| 510 | 1093 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 511 | 1094 | other => return Err(format!("unknown list option: {}", other)), |
| 512 | 1095 | } |
| 513 | 1096 | } |
| 514 | | - Ok(CommandKind::List { suite_filter }) |
| 1097 | + Ok(CommandKind::List(config)) |
| 515 | 1098 | } |
| 516 | 1099 | "run" => { |
| 517 | 1100 | let mut config = RunConfig { |
@@ -522,10 +1105,15 @@ fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 522 | 1105 | fail_fast: false, |
| 523 | 1106 | include_future: false, |
| 524 | 1107 | all_stages: false, |
| 1108 | + json_report: None, |
| 1109 | + markdown_report: None, |
| 525 | 1110 | tools: ToolchainConfig::from_env(), |
| 526 | 1111 | }; |
| 527 | 1112 | let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 528 | 1113 | while let Some(arg) = queue.pop_front() { |
| 1114 | + if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1115 | + continue; |
| 1116 | + } |
| 529 | 1117 | match arg.as_str() { |
| 530 | 1118 | "--suite" => { |
| 531 | 1119 | let value = queue.pop_front().ok_or("--suite requires a value")?; |
@@ -545,4982 +1133,13549 @@ fn parse_cli(args: &[String]) -> Result<CommandKind, String> { |
| 545 | 1133 | "--fail-fast" => config.fail_fast = true, |
| 546 | 1134 | "--include-future" => config.include_future = true, |
| 547 | 1135 | "--all" => config.all_stages = true, |
| 548 | | - "--armfortas-bin" => { |
| 1136 | + "--json-report" => { |
| 1137 | + let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1138 | + config.json_report = Some(PathBuf::from(value)); |
| 1139 | + } |
| 1140 | + "--markdown-report" => { |
| 549 | 1141 | let value = queue |
| 550 | 1142 | .pop_front() |
| 551 | | - .ok_or("--armfortas-bin requires a value")?; |
| 552 | | - config.tools.armfortas = ArmfortasCliAdapter::External(value.clone()); |
| 1143 | + .ok_or("--markdown-report requires a value")?; |
| 1144 | + config.markdown_report = Some(PathBuf::from(value)); |
| 553 | 1145 | } |
| 554 | | - "--gfortran-bin" => { |
| 555 | | - let value = queue.pop_front().ok_or("--gfortran-bin requires a value")?; |
| 556 | | - config.tools.gfortran = value.clone(); |
| 1146 | + "--help" | "-h" => return Ok(CommandKind::Help), |
| 1147 | + other => return Err(format!("unknown run option: {}", other)), |
| 1148 | + } |
| 1149 | + } |
| 1150 | + Ok(CommandKind::Run(config)) |
| 1151 | + } |
| 1152 | + "compare" => { |
| 1153 | + if args.len() < 3 { |
| 1154 | + return Err( |
| 1155 | + "compare requires <compiler-a> <compiler-b> and --program <path>".to_string(), |
| 1156 | + ); |
| 1157 | + } |
| 1158 | + let left = CompilerSpec::parse(&args[1]); |
| 1159 | + let right = CompilerSpec::parse(&args[2]); |
| 1160 | + let mut config = CompareConfig { |
| 1161 | + left, |
| 1162 | + right, |
| 1163 | + program: PathBuf::new(), |
| 1164 | + opt_level: OptLevel::O0, |
| 1165 | + artifacts: BTreeSet::new(), |
| 1166 | + json_report: None, |
| 1167 | + markdown_report: None, |
| 1168 | + tools: ToolchainConfig::from_env(), |
| 1169 | + }; |
| 1170 | + let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1171 | + while let Some(arg) = queue.pop_front() { |
| 1172 | + if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1173 | + continue; |
| 1174 | + } |
| 1175 | + match arg.as_str() { |
| 1176 | + "--program" => { |
| 1177 | + let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1178 | + config.program = PathBuf::from(value); |
| 557 | 1179 | } |
| 558 | | - "--flang-bin" => { |
| 559 | | - let value = queue.pop_front().ok_or("--flang-bin requires a value")?; |
| 560 | | - config.tools.flang_new = value.clone(); |
| 1180 | + "--opt" => { |
| 1181 | + let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1182 | + let parsed = parse_opt_level_list(value)?; |
| 1183 | + let opt = parsed |
| 1184 | + .into_iter() |
| 1185 | + .next() |
| 1186 | + .ok_or("--opt requires at least one optimization level")?; |
| 1187 | + config.opt_level = opt; |
| 561 | 1188 | } |
| 562 | | - "--as-bin" => { |
| 563 | | - let value = queue.pop_front().ok_or("--as-bin requires a value")?; |
| 564 | | - config.tools.system_as = value.clone(); |
| 1189 | + "--artifact" => { |
| 1190 | + let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1191 | + config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 565 | 1192 | } |
| 566 | | - "--otool-bin" => { |
| 567 | | - let value = queue.pop_front().ok_or("--otool-bin requires a value")?; |
| 568 | | - config.tools.otool = value.clone(); |
| 1193 | + "--json-report" => { |
| 1194 | + let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1195 | + config.json_report = Some(PathBuf::from(value)); |
| 569 | 1196 | } |
| 570 | | - "--nm-bin" => { |
| 571 | | - let value = queue.pop_front().ok_or("--nm-bin requires a value")?; |
| 572 | | - config.tools.nm = value.clone(); |
| 1197 | + "--markdown-report" => { |
| 1198 | + let value = queue |
| 1199 | + .pop_front() |
| 1200 | + .ok_or("--markdown-report requires a value")?; |
| 1201 | + config.markdown_report = Some(PathBuf::from(value)); |
| 573 | 1202 | } |
| 574 | 1203 | "--help" | "-h" => return Ok(CommandKind::Help), |
| 575 | | - other => return Err(format!("unknown run option: {}", other)), |
| 1204 | + other => return Err(format!("unknown compare option: {}", other)), |
| 576 | 1205 | } |
| 577 | 1206 | } |
| 578 | | - Ok(CommandKind::Run(config)) |
| 1207 | + if config.program.as_os_str().is_empty() { |
| 1208 | + return Err("compare requires --program <path>".to_string()); |
| 1209 | + } |
| 1210 | + Ok(CommandKind::Compare(config)) |
| 579 | 1211 | } |
| 580 | | - "--help" | "-h" | "help" => Ok(CommandKind::Help), |
| 581 | | - other => Err(format!("unknown command: {}", other)), |
| 582 | | - } |
| 583 | | -} |
| 584 | | - |
| 585 | | -fn print_usage() { |
| 586 | | - eprintln!("afs-tests — structured ARMFORTAS bench runner"); |
| 587 | | - eprintln!(); |
| 588 | | - eprintln!("usage:"); |
| 589 | | - eprintln!(" cargo run -p afs-tests -- list [--suite <filter>]"); |
| 590 | | - eprintln!( |
| 591 | | - " cargo run -p afs-tests -- run [--suite <filter>] [--case <filter>] [--opt <O0,O1,...>] [--verbose] [--fail-fast] [--include-future] [--all] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]" |
| 592 | | - ); |
| 1212 | + "introspect" => { |
| 1213 | + if args.len() < 3 { |
| 1214 | + return Err("introspect requires <compiler> <program>".to_string()); |
| 1215 | + } |
| 1216 | + let compiler = CompilerSpec::parse(&args[1]); |
| 1217 | + let mut config = IntrospectConfig { |
| 1218 | + compiler, |
| 1219 | + program: PathBuf::from(&args[2]), |
| 1220 | + opt_level: OptLevel::O0, |
| 1221 | + artifacts: BTreeSet::new(), |
| 1222 | + json_report: None, |
| 1223 | + markdown_report: None, |
| 1224 | + all_artifacts: false, |
| 1225 | + summary_only: false, |
| 1226 | + max_artifact_lines: None, |
| 1227 | + tools: ToolchainConfig::from_env(), |
| 1228 | + }; |
| 1229 | + let mut queue: VecDeque<&String> = args[3..].iter().collect(); |
| 1230 | + while let Some(arg) = queue.pop_front() { |
| 1231 | + if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1232 | + continue; |
| 1233 | + } |
| 1234 | + match arg.as_str() { |
| 1235 | + "--program" => { |
| 1236 | + let value = queue.pop_front().ok_or("--program requires a value")?; |
| 1237 | + config.program = PathBuf::from(value); |
| 1238 | + } |
| 1239 | + "--opt" => { |
| 1240 | + let value = queue.pop_front().ok_or("--opt requires a value")?; |
| 1241 | + let parsed = parse_opt_level_list(value)?; |
| 1242 | + let opt = parsed |
| 1243 | + .into_iter() |
| 1244 | + .next() |
| 1245 | + .ok_or("--opt requires at least one optimization level")?; |
| 1246 | + config.opt_level = opt; |
| 1247 | + } |
| 1248 | + "--artifact" => { |
| 1249 | + let value = queue.pop_front().ok_or("--artifact requires a value")?; |
| 1250 | + config.artifacts.extend(ArtifactKey::parse_list(value)?); |
| 1251 | + } |
| 1252 | + "--all" => config.all_artifacts = true, |
| 1253 | + "--summary-only" => config.summary_only = true, |
| 1254 | + "--max-artifact-lines" => { |
| 1255 | + let value = queue |
| 1256 | + .pop_front() |
| 1257 | + .ok_or("--max-artifact-lines requires a value")?; |
| 1258 | + let parsed = value.parse::<usize>().map_err(|_| { |
| 1259 | + format!("invalid --max-artifact-lines value '{}'", value) |
| 1260 | + })?; |
| 1261 | + if parsed == 0 { |
| 1262 | + return Err("--max-artifact-lines must be greater than 0".to_string()); |
| 1263 | + } |
| 1264 | + config.max_artifact_lines = Some(parsed); |
| 1265 | + } |
| 1266 | + "--json-report" => { |
| 1267 | + let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1268 | + config.json_report = Some(PathBuf::from(value)); |
| 1269 | + } |
| 1270 | + "--markdown-report" => { |
| 1271 | + let value = queue |
| 1272 | + .pop_front() |
| 1273 | + .ok_or("--markdown-report requires a value")?; |
| 1274 | + config.markdown_report = Some(PathBuf::from(value)); |
| 1275 | + } |
| 1276 | + "--help" | "-h" => return Ok(CommandKind::Help), |
| 1277 | + other => return Err(format!("unknown introspect option: {}", other)), |
| 1278 | + } |
| 1279 | + } |
| 1280 | + Ok(CommandKind::Introspect(config)) |
| 1281 | + } |
| 1282 | + "doctor" => { |
| 1283 | + let mut config = DoctorConfig { |
| 1284 | + tools: ToolchainConfig::from_env(), |
| 1285 | + json_report: None, |
| 1286 | + markdown_report: None, |
| 1287 | + }; |
| 1288 | + let mut queue: VecDeque<&String> = args[1..].iter().collect(); |
| 1289 | + while let Some(arg) = queue.pop_front() { |
| 1290 | + if parse_tool_override_arg(arg, &mut queue, &mut config.tools)? { |
| 1291 | + continue; |
| 1292 | + } |
| 1293 | + match arg.as_str() { |
| 1294 | + "--json-report" => { |
| 1295 | + let value = queue.pop_front().ok_or("--json-report requires a value")?; |
| 1296 | + config.json_report = Some(PathBuf::from(value)); |
| 1297 | + } |
| 1298 | + "--markdown-report" => { |
| 1299 | + let value = queue |
| 1300 | + .pop_front() |
| 1301 | + .ok_or("--markdown-report requires a value")?; |
| 1302 | + config.markdown_report = Some(PathBuf::from(value)); |
| 1303 | + } |
| 1304 | + "--help" | "-h" => return Ok(CommandKind::Help), |
| 1305 | + other => return Err(format!("unknown doctor option: {}", other)), |
| 1306 | + } |
| 1307 | + } |
| 1308 | + Ok(CommandKind::Doctor(config)) |
| 1309 | + } |
| 1310 | + "--help" | "-h" | "help" => Ok(CommandKind::Help), |
| 1311 | + other => Err(format!("unknown command: {}", other)), |
| 1312 | + } |
| 1313 | +} |
| 1314 | + |
| 1315 | +fn print_usage(program_name: &str) { |
| 1316 | + eprintln!( |
| 1317 | + "{} — generic compiler bench runner (afs-tests compatibility preserved)", |
| 1318 | + program_name |
| 1319 | + ); |
| 1320 | + eprintln!(); |
| 1321 | + eprintln!("usage:"); |
| 1322 | + eprintln!( |
| 1323 | + " {} list [--suite <filter>] [--verbose] [tool overrides]", |
| 1324 | + program_name |
| 1325 | + ); |
| 1326 | + eprintln!( |
| 1327 | + " {} run [--suite <filter>] [--case <filter>] [--opt <O0,O1,...>] [--verbose] [--fail-fast] [--include-future] [--all] [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1328 | + program_name |
| 1329 | + ); |
| 1330 | + eprintln!( |
| 1331 | + " {} compare <compiler-a> <compiler-b> --program <path> [--opt <O0>] [--artifact <asm,obj,stdout,stderr,exit-code,executable>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1332 | + program_name |
| 1333 | + ); |
| 1334 | + eprintln!( |
| 1335 | + " {} introspect <compiler> <program> [--opt <O0>] [--artifact <list>] [--all] [--summary-only] [--max-artifact-lines <n>] [--json-report <path>] [--markdown-report <path>] [tool overrides]", |
| 1336 | + program_name |
| 1337 | + ); |
| 1338 | + eprintln!( |
| 1339 | + " {} doctor [--json-report <path>] [--markdown-report <path>] [--armfortas-bin <path>] [--gfortran-bin <path>] [--flang-bin <path>] [--lfortran-bin <path>] [--ifort-bin <path>] [--ifx-bin <path>] [--nvfortran-bin <path>] [--as-bin <path>] [--otool-bin <path>] [--nm-bin <path>]", |
| 1340 | + program_name |
| 1341 | + ); |
| 593 | 1342 | eprintln!(); |
| 594 | 1343 | eprintln!("env overrides:"); |
| 595 | | - eprintln!(" BENCCH_ARMFORTAS_BIN, BENCCH_GFORTRAN_BIN, BENCCH_FLANG_BIN"); |
| 1344 | + eprintln!(" BENCCH_ARMFORTAS_BIN, BENCCH_GFORTRAN_BIN, BENCCH_FLANG_BIN, BENCCH_LFORTRAN_BIN"); |
| 1345 | + eprintln!(" BENCCH_IFORT_BIN, BENCCH_IFX_BIN, BENCCH_NVFORTRAN_BIN"); |
| 596 | 1346 | eprintln!(" BENCCH_AS_BIN, BENCCH_OTOOL_BIN, BENCCH_NM_BIN"); |
| 1347 | + eprintln!(); |
| 1348 | + if linked_capture_available() { |
| 1349 | + eprintln!("mode:"); |
| 1350 | + eprintln!(" linked armfortas capture is available in this build"); |
| 1351 | + } else { |
| 1352 | + eprintln!("mode:"); |
| 1353 | + eprintln!(" linked armfortas capture is unavailable in this build"); |
| 1354 | + eprintln!(" compare, introspect, and generic/observable suite runs still work"); |
| 1355 | + eprintln!(" use scripts/bootstrap-linked-armfortas.sh for rich armfortas stages and legacy frontend/module suites"); |
| 1356 | + } |
| 1357 | +} |
| 1358 | + |
| 1359 | +fn default_compare_artifacts(extra: &BTreeSet<ArtifactKey>) -> BTreeSet<ArtifactKey> { |
| 1360 | + let mut requested = BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]); |
| 1361 | + requested.extend(extra.iter().cloned()); |
| 1362 | + requested |
| 1363 | +} |
| 1364 | + |
| 1365 | +fn default_differential_artifacts() -> BTreeSet<ArtifactKey> { |
| 1366 | + BTreeSet::from([ArtifactKey::Diagnostics, ArtifactKey::Runtime]) |
| 1367 | +} |
| 1368 | + |
| 1369 | +fn default_introspection_artifacts( |
| 1370 | + compiler: &CompilerSpec, |
| 1371 | + all_artifacts: bool, |
| 1372 | +) -> BTreeSet<ArtifactKey> { |
| 1373 | + let mut requested = BTreeSet::from([ |
| 1374 | + ArtifactKey::Diagnostics, |
| 1375 | + ArtifactKey::Runtime, |
| 1376 | + ArtifactKey::Asm, |
| 1377 | + ArtifactKey::Obj, |
| 1378 | + ]); |
| 1379 | + if matches!(compiler, CompilerSpec::Named(NamedCompiler::Armfortas)) { |
| 1380 | + requested.insert(ArtifactKey::Extra("armfortas.ir".into())); |
| 1381 | + if all_artifacts { |
| 1382 | + for name in [ |
| 1383 | + "armfortas.preprocess", |
| 1384 | + "armfortas.tokens", |
| 1385 | + "armfortas.ast", |
| 1386 | + "armfortas.sema", |
| 1387 | + "armfortas.ir", |
| 1388 | + "armfortas.optir", |
| 1389 | + "armfortas.mir", |
| 1390 | + "armfortas.regalloc", |
| 1391 | + ] { |
| 1392 | + requested.insert(ArtifactKey::Extra(name.to_string())); |
| 1393 | + } |
| 1394 | + } |
| 1395 | + } |
| 1396 | + requested |
| 597 | 1397 | } |
| 598 | 1398 | |
| 599 | | -fn default_suite_root() -> PathBuf { |
| 600 | | - Path::new(env!("CARGO_MANIFEST_DIR")) |
| 601 | | - .join("..") |
| 602 | | - .join("suites") |
| 1399 | +fn run_compare(config: &CompareConfig) -> Result<ComparisonResult, String> { |
| 1400 | + let requested = default_compare_artifacts(&config.artifacts); |
| 1401 | + preflight_compare_request(config, &requested)?; |
| 1402 | + let left = observe_compiler( |
| 1403 | + &config.left, |
| 1404 | + &config.program, |
| 1405 | + config.opt_level, |
| 1406 | + &requested, |
| 1407 | + &config.tools, |
| 1408 | + )?; |
| 1409 | + let right = observe_compiler( |
| 1410 | + &config.right, |
| 1411 | + &config.program, |
| 1412 | + config.opt_level, |
| 1413 | + &requested, |
| 1414 | + &config.tools, |
| 1415 | + )?; |
| 1416 | + Ok(compare_observations(left, right, &requested)) |
| 603 | 1417 | } |
| 604 | 1418 | |
| 605 | | -fn default_report_root() -> PathBuf { |
| 606 | | - Path::new(env!("CARGO_MANIFEST_DIR")) |
| 607 | | - .join("..") |
| 608 | | - .join("reports") |
| 609 | | -} |
| 1419 | +fn capability_request_issue( |
| 1420 | + spec: &CompilerSpec, |
| 1421 | + requested: &BTreeSet<ArtifactKey>, |
| 1422 | + tools: &ToolchainConfig, |
| 1423 | +) -> Option<String> { |
| 1424 | + let capabilities = compiler_capabilities(spec, tools); |
| 1425 | + let unavailable = capabilities.unavailable_requests(requested); |
| 1426 | + let unsupported = capabilities.unsupported_requests(requested); |
| 1427 | + if unavailable.is_empty() && unsupported.is_empty() { |
| 1428 | + return None; |
| 1429 | + } |
| 610 | 1430 | |
| 611 | | -fn discover_suites(root: PathBuf) -> Result<Vec<SuiteSpec>, String> { |
| 612 | | - let mut files = Vec::new(); |
| 613 | | - collect_suite_files(&root, &mut files)?; |
| 614 | | - files.sort(); |
| 1431 | + let mut sections = Vec::new(); |
| 1432 | + if !unavailable.is_empty() { |
| 1433 | + let detail = unavailable |
| 1434 | + .into_iter() |
| 1435 | + .map(|(artifact, reason)| format!("requested {}: {}", artifact, reason)) |
| 1436 | + .collect::<Vec<_>>() |
| 1437 | + .join("\n"); |
| 1438 | + sections.push(format!( |
| 1439 | + "{} unavailable for requested artifacts in this build\n{}", |
| 1440 | + spec.display_name(), |
| 1441 | + detail |
| 1442 | + )); |
| 1443 | + } |
| 1444 | + if !unsupported.is_empty() { |
| 1445 | + sections.push(format!( |
| 1446 | + "{} does not support requested artifacts in this adapter: {}", |
| 1447 | + spec.display_name(), |
| 1448 | + unsupported.join(", ") |
| 1449 | + )); |
| 1450 | + } |
| 1451 | + Some(sections.join("\n")) |
| 1452 | +} |
| 615 | 1453 | |
| 616 | | - let mut suites = Vec::new(); |
| 617 | | - for file in files { |
| 618 | | - suites.push(parse_suite_file(&file)?); |
| 1454 | +fn compare_capability_issue( |
| 1455 | + left: &CompilerSpec, |
| 1456 | + right: &CompilerSpec, |
| 1457 | + requested: &BTreeSet<ArtifactKey>, |
| 1458 | + tools: &ToolchainConfig, |
| 1459 | +) -> Option<String> { |
| 1460 | + let mut issues = Vec::new(); |
| 1461 | + if let Some(issue) = capability_request_issue(left, requested, tools) { |
| 1462 | + issues.push(format!("left:\n{}", issue)); |
| 1463 | + } |
| 1464 | + if let Some(issue) = capability_request_issue(right, requested, tools) { |
| 1465 | + issues.push(format!("right:\n{}", issue)); |
| 1466 | + } |
| 1467 | + if issues.is_empty() { |
| 1468 | + None |
| 1469 | + } else { |
| 1470 | + Some(format!( |
| 1471 | + "compare request is not supported for the selected compiler surfaces\n{}", |
| 1472 | + issues.join("\n") |
| 1473 | + )) |
| 619 | 1474 | } |
| 620 | | - suites.sort_by(|a, b| a.name.cmp(&b.name)); |
| 621 | | - Ok(suites) |
| 622 | 1475 | } |
| 623 | 1476 | |
| 624 | | -fn collect_suite_files(root: &Path, files: &mut Vec<PathBuf>) -> Result<(), String> { |
| 625 | | - let entries = fs::read_dir(root) |
| 626 | | - .map_err(|e| format!("cannot read suite root '{}': {}", root.display(), e))?; |
| 627 | | - for entry in entries { |
| 628 | | - let entry = |
| 629 | | - entry.map_err(|e| format!("cannot read entry in '{}': {}", root.display(), e))?; |
| 630 | | - let path = entry.path(); |
| 631 | | - if path.is_dir() { |
| 632 | | - collect_suite_files(&path, files)?; |
| 633 | | - } else if path.extension().and_then(|ext| ext.to_str()) == Some(SUITE_EXTENSION) { |
| 634 | | - files.push(path); |
| 635 | | - } |
| 1477 | +fn preflight_compare_request( |
| 1478 | + config: &CompareConfig, |
| 1479 | + requested: &BTreeSet<ArtifactKey>, |
| 1480 | +) -> Result<(), String> { |
| 1481 | + match compare_capability_issue(&config.left, &config.right, requested, &config.tools) { |
| 1482 | + Some(issue) => Err(issue), |
| 1483 | + None => Ok(()), |
| 636 | 1484 | } |
| 637 | | - Ok(()) |
| 638 | 1485 | } |
| 639 | 1486 | |
| 640 | | -fn parse_suite_file(path: &Path) -> Result<SuiteSpec, String> { |
| 641 | | - let text = fs::read_to_string(path) |
| 642 | | - .map_err(|e| format!("cannot read suite '{}': {}", path.display(), e))?; |
| 1487 | +fn run_introspect(config: &IntrospectConfig) -> Result<ObservedProgram, String> { |
| 1488 | + let requested = if config.artifacts.is_empty() { |
| 1489 | + default_introspection_artifacts(&config.compiler, config.all_artifacts) |
| 1490 | + } else { |
| 1491 | + let mut requested = config.artifacts.clone(); |
| 1492 | + if config.all_artifacts |
| 1493 | + && matches!( |
| 1494 | + config.compiler, |
| 1495 | + CompilerSpec::Named(NamedCompiler::Armfortas) |
| 1496 | + ) |
| 1497 | + { |
| 1498 | + requested.extend(default_introspection_artifacts(&config.compiler, true)); |
| 1499 | + } |
| 1500 | + requested |
| 1501 | + }; |
| 1502 | + if let Some(observed) = preflight_introspection_request( |
| 1503 | + &config.compiler, |
| 1504 | + &config.program, |
| 1505 | + config.opt_level, |
| 1506 | + &requested, |
| 1507 | + &config.tools, |
| 1508 | + ) { |
| 1509 | + return Ok(observed); |
| 1510 | + } |
| 1511 | + Ok(ObservedProgram { |
| 1512 | + observation: observe_compiler( |
| 1513 | + &config.compiler, |
| 1514 | + &config.program, |
| 1515 | + config.opt_level, |
| 1516 | + &requested, |
| 1517 | + &config.tools, |
| 1518 | + )?, |
| 1519 | + requested_artifacts: requested, |
| 1520 | + }) |
| 1521 | +} |
| 643 | 1522 | |
| 644 | | - let mut suite_name = None; |
| 645 | | - let mut cases = Vec::new(); |
| 646 | | - let mut current = None; |
| 1523 | +fn requested_linked_armfortas_artifacts(requested: &BTreeSet<ArtifactKey>) -> Vec<String> { |
| 1524 | + requested |
| 1525 | + .iter() |
| 1526 | + .filter_map(|artifact| match artifact { |
| 1527 | + ArtifactKey::Extra(name) if name.starts_with("armfortas.") => Some(name.clone()), |
| 1528 | + _ => None, |
| 1529 | + }) |
| 1530 | + .collect() |
| 1531 | +} |
| 647 | 1532 | |
| 648 | | - for (index, raw_line) in text.lines().enumerate() { |
| 649 | | - let line_no = index + 1; |
| 650 | | - let line = raw_line.trim(); |
| 651 | | - if line.is_empty() || line.starts_with('#') { |
| 652 | | - continue; |
| 1533 | +fn observe_compiler( |
| 1534 | + spec: &CompilerSpec, |
| 1535 | + program: &Path, |
| 1536 | + opt_level: OptLevel, |
| 1537 | + requested: &BTreeSet<ArtifactKey>, |
| 1538 | + tools: &ToolchainConfig, |
| 1539 | +) -> Result<CompilerObservation, String> { |
| 1540 | + match spec { |
| 1541 | + CompilerSpec::Named(NamedCompiler::Armfortas) => { |
| 1542 | + observe_armfortas(program, opt_level, requested, tools) |
| 653 | 1543 | } |
| 654 | | - |
| 655 | | - if let Some(rest) = line.strip_prefix("suite ") { |
| 656 | | - if suite_name.is_some() { |
| 657 | | - return Err(format!( |
| 658 | | - "{}:{}: duplicate suite declaration", |
| 659 | | - path.display(), |
| 660 | | - line_no |
| 661 | | - )); |
| 662 | | - } |
| 663 | | - suite_name = Some(parse_quoted(rest, path, line_no)?); |
| 664 | | - continue; |
| 1544 | + CompilerSpec::Named(named) => { |
| 1545 | + let binary = tools.named_compiler_binary(*named).ok_or_else(|| { |
| 1546 | + format!("named compiler '{}' has no resolved binary", named.as_str()) |
| 1547 | + })?; |
| 1548 | + observe_external_driver( |
| 1549 | + spec, |
| 1550 | + &binary, |
| 1551 | + program, |
| 1552 | + opt_level, |
| 1553 | + requested, |
| 1554 | + matches!(named, NamedCompiler::Gfortran | NamedCompiler::FlangNew) |
| 1555 | + && source_uses_cpp(program), |
| 1556 | + "named".to_string(), |
| 1557 | + tools.otool_bin(), |
| 1558 | + tools.nm_bin(), |
| 1559 | + ) |
| 665 | 1560 | } |
| 1561 | + CompilerSpec::Binary(path) => observe_external_driver( |
| 1562 | + spec, |
| 1563 | + &path.display().to_string(), |
| 1564 | + program, |
| 1565 | + opt_level, |
| 1566 | + requested, |
| 1567 | + false, |
| 1568 | + "explicit-path".to_string(), |
| 1569 | + tools.otool_bin(), |
| 1570 | + tools.nm_bin(), |
| 1571 | + ), |
| 1572 | + } |
| 1573 | +} |
| 666 | 1574 | |
| 667 | | - if let Some(rest) = line.strip_prefix("case ") { |
| 668 | | - if current.is_some() { |
| 669 | | - return Err(format!( |
| 670 | | - "{}:{}: nested case without end", |
| 671 | | - path.display(), |
| 672 | | - line_no |
| 673 | | - )); |
| 1575 | +fn observe_armfortas( |
| 1576 | + program: &Path, |
| 1577 | + opt_level: OptLevel, |
| 1578 | + requested: &BTreeSet<ArtifactKey>, |
| 1579 | + tools: &ToolchainConfig, |
| 1580 | +) -> Result<CompilerObservation, String> { |
| 1581 | + let stages = armfortas_requested_stages(requested)?; |
| 1582 | + let linked_only_artifacts = requested_linked_armfortas_artifacts(requested); |
| 1583 | + let linked_backend = tools.armfortas_adapters(); |
| 1584 | + if !linked_only_artifacts.is_empty() && linked_backend.capture_mode_name() == "unavailable" { |
| 1585 | + let detail = format!( |
| 1586 | + "linked armfortas capture is unavailable in this build; requested {}; use scripts/bootstrap-linked-armfortas.sh or request only asm/obj/run from an external armfortas binary", |
| 1587 | + linked_only_artifacts.join(", ") |
| 1588 | + ); |
| 1589 | + return Ok(CompilerObservation { |
| 1590 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1591 | + program: program.to_path_buf(), |
| 1592 | + opt_level, |
| 1593 | + compile_exit_code: 1, |
| 1594 | + artifacts: BTreeMap::from([( |
| 1595 | + ArtifactKey::Diagnostics, |
| 1596 | + ArtifactValue::Text(detail.clone()), |
| 1597 | + )]), |
| 1598 | + provenance: ObservationProvenance { |
| 1599 | + compiler_identity: "armfortas".into(), |
| 1600 | + adapter_kind: "named".into(), |
| 1601 | + backend_mode: linked_backend.capture_mode_name().into(), |
| 1602 | + backend_detail: linked_backend.capture_description().into(), |
| 1603 | + artifacts_captured: vec!["diagnostics".into()], |
| 1604 | + comparison_basis: None, |
| 1605 | + failure_stage: None, |
| 1606 | + }, |
| 1607 | + }); |
| 1608 | + } |
| 1609 | + let cli_observable_only = requested.iter().all(|artifact| { |
| 1610 | + matches!( |
| 1611 | + artifact, |
| 1612 | + ArtifactKey::Diagnostics |
| 1613 | + | ArtifactKey::Runtime |
| 1614 | + | ArtifactKey::Stdout |
| 1615 | + | ArtifactKey::Stderr |
| 1616 | + | ArtifactKey::ExitCode |
| 1617 | + | ArtifactKey::Asm |
| 1618 | + | ArtifactKey::Obj |
| 1619 | + | ArtifactKey::Executable |
| 1620 | + ) |
| 1621 | + }); |
| 1622 | + let (backend_mode, backend_detail, capture) = if cli_observable_only |
| 1623 | + && matches!(tools.armfortas, ArmfortasCliAdapter::External(_)) |
| 1624 | + { |
| 1625 | + let backend = tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level)); |
| 1626 | + let detail = backend.description().to_string(); |
| 1627 | + let mode = backend.mode_name().to_string(); |
| 1628 | + let request = CaptureRequest { |
| 1629 | + input: program.to_path_buf(), |
| 1630 | + requested: stages.clone(), |
| 1631 | + opt_level, |
| 1632 | + }; |
| 1633 | + (mode, detail, backend.capture(&request)) |
| 1634 | + } else { |
| 1635 | + let backend = linked_backend; |
| 1636 | + let detail = backend.capture_description().to_string(); |
| 1637 | + let mode = backend.capture_mode_name().to_string(); |
| 1638 | + let request = CaptureRequest { |
| 1639 | + input: program.to_path_buf(), |
| 1640 | + requested: stages.clone(), |
| 1641 | + opt_level, |
| 1642 | + }; |
| 1643 | + (mode, detail, backend.capture(&request)) |
| 1644 | + }; |
| 1645 | + |
| 1646 | + let mut artifacts = BTreeMap::new(); |
| 1647 | + let mut compile_exit_code = 0; |
| 1648 | + let mut failure_stage = None; |
| 1649 | + match capture { |
| 1650 | + Ok(result) => { |
| 1651 | + for (stage, captured) in &result.stages { |
| 1652 | + match (stage, captured) { |
| 1653 | + (Stage::Asm, CapturedStage::Text(text)) |
| 1654 | + if requested.contains(&ArtifactKey::Asm) => |
| 1655 | + { |
| 1656 | + artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1657 | + } |
| 1658 | + (Stage::Obj, CapturedStage::Text(text)) |
| 1659 | + if requested.contains(&ArtifactKey::Obj) => |
| 1660 | + { |
| 1661 | + artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1662 | + } |
| 1663 | + (Stage::Run, CapturedStage::Run(run)) => { |
| 1664 | + insert_run_artifacts(requested, run, &mut artifacts); |
| 1665 | + } |
| 1666 | + (stage, CapturedStage::Text(text)) => { |
| 1667 | + let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1668 | + if requested.contains(&key) { |
| 1669 | + artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1670 | + } |
| 1671 | + } |
| 1672 | + _ => {} |
| 1673 | + } |
| 674 | 1674 | } |
| 675 | | - current = Some(CaseBuilder::new(parse_quoted(rest, path, line_no)?)); |
| 676 | | - continue; |
| 677 | 1675 | } |
| 678 | | - |
| 679 | | - if line == "end" { |
| 680 | | - let builder = current.take().ok_or_else(|| { |
| 681 | | - format!("{}:{}: stray end outside of case", path.display(), line_no) |
| 682 | | - })?; |
| 683 | | - cases.push(builder.build(path)?); |
| 684 | | - continue; |
| 1676 | + Err(failure) => { |
| 1677 | + compile_exit_code = 1; |
| 1678 | + failure_stage = Some(failure.stage.as_str().to_string()); |
| 1679 | + artifacts.insert( |
| 1680 | + ArtifactKey::Diagnostics, |
| 1681 | + ArtifactValue::Text(failure.detail.clone()), |
| 1682 | + ); |
| 1683 | + for (stage, captured) in &failure.stages { |
| 1684 | + match (stage, captured) { |
| 1685 | + (Stage::Asm, CapturedStage::Text(text)) |
| 1686 | + if requested.contains(&ArtifactKey::Asm) => |
| 1687 | + { |
| 1688 | + artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 1689 | + } |
| 1690 | + (Stage::Obj, CapturedStage::Text(text)) |
| 1691 | + if requested.contains(&ArtifactKey::Obj) => |
| 1692 | + { |
| 1693 | + artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 1694 | + } |
| 1695 | + (Stage::Run, CapturedStage::Run(run)) => { |
| 1696 | + insert_run_artifacts(requested, run, &mut artifacts); |
| 1697 | + } |
| 1698 | + (stage, CapturedStage::Text(text)) => { |
| 1699 | + let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 1700 | + if requested.contains(&key) { |
| 1701 | + artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 1702 | + } |
| 1703 | + } |
| 1704 | + _ => {} |
| 1705 | + } |
| 1706 | + } |
| 685 | 1707 | } |
| 1708 | + } |
| 686 | 1709 | |
| 687 | | - let builder = current.as_mut().ok_or_else(|| { |
| 1710 | + if requested.contains(&ArtifactKey::Executable) && compile_exit_code == 0 { |
| 1711 | + let temp_root = next_observation_temp_root("armfortas", opt_level); |
| 1712 | + fs::create_dir_all(&temp_root).map_err(|e| { |
| 688 | 1713 | format!( |
| 689 | | - "{}:{}: expected suite/case declaration first", |
| 690 | | - path.display(), |
| 691 | | - line_no |
| 1714 | + "cannot create introspection temp dir '{}': {}", |
| 1715 | + temp_root.display(), |
| 1716 | + e |
| 692 | 1717 | ) |
| 693 | 1718 | })?; |
| 694 | | - |
| 695 | | - if let Some(rest) = line.strip_prefix("source ") { |
| 696 | | - builder.source = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 697 | | - } else if let Some(rest) = line.strip_prefix("entry ") { |
| 698 | | - builder.graph_entry = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 699 | | - } else if let Some(rest) = line.strip_prefix("file ") { |
| 700 | | - builder |
| 701 | | - .graph_files |
| 702 | | - .push(resolve_suite_relative_path(rest, path, line_no)?); |
| 703 | | - } else if let Some(rest) = line.strip_prefix("armfortas =>") { |
| 704 | | - builder.requested = parse_stage_list(rest, path, line_no)?; |
| 705 | | - } else if let Some(rest) = line.strip_prefix("repeat =>") { |
| 706 | | - builder.repeat_count = parse_repeat_count(rest, path, line_no)?; |
| 707 | | - } else if let Some(rest) = line.strip_prefix("opts =>") { |
| 708 | | - builder.opt_levels = parse_opt_levels(rest, path, line_no)?; |
| 709 | | - } else if let Some(rest) = line.strip_prefix("differential =>") { |
| 710 | | - builder.reference_compilers = parse_reference_compilers(rest, path, line_no)?; |
| 711 | | - } else if let Some(rest) = line.strip_prefix("consistency =>") { |
| 712 | | - builder.consistency_checks = parse_consistency_checks(rest, path, line_no)?; |
| 713 | | - } else if let Some(rest) = line.strip_prefix("expect-fail ") { |
| 714 | | - builder |
| 715 | | - .expectations |
| 716 | | - .push(parse_failure_expectation(rest, path, line_no)?); |
| 717 | | - } else if let Some(rest) = line.strip_prefix("expect ") { |
| 718 | | - builder |
| 719 | | - .expectations |
| 720 | | - .push(parse_expectation(rest, path, line_no)?); |
| 721 | | - } else if let Some(rest) = line.strip_prefix("xfail ") { |
| 722 | | - builder |
| 723 | | - .status_rules |
| 724 | | - .push(parse_status_rule(StatusKind::Xfail, rest, path, line_no)?); |
| 725 | | - } else if let Some(rest) = line.strip_prefix("future ") { |
| 726 | | - builder |
| 727 | | - .status_rules |
| 728 | | - .push(parse_status_rule(StatusKind::Future, rest, path, line_no)?); |
| 729 | | - } else { |
| 730 | | - return Err(format!( |
| 731 | | - "{}:{}: unrecognized line '{}'", |
| 732 | | - path.display(), |
| 733 | | - line_no, |
| 734 | | - line |
| 735 | | - )); |
| 736 | | - } |
| 1719 | + let binary = temp_root.join("introspect.out"); |
| 1720 | + tools |
| 1721 | + .armfortas_adapters() |
| 1722 | + .compile_output(program, opt_level, EmitMode::Binary, &binary) |
| 1723 | + .map_err(|detail| { |
| 1724 | + format!("failed to build armfortas executable artifact:\n{}", detail) |
| 1725 | + })?; |
| 1726 | + artifacts.insert(ArtifactKey::Executable, ArtifactValue::Path(binary)); |
| 737 | 1727 | } |
| 738 | 1728 | |
| 739 | | - if current.is_some() { |
| 740 | | - return Err(format!("{}: unterminated case block", path.display())); |
| 741 | | - } |
| 742 | | - |
| 743 | | - let suite_name = |
| 744 | | - suite_name.ok_or_else(|| format!("{}: missing suite declaration", path.display()))?; |
| 745 | | - if cases.is_empty() { |
| 746 | | - return Err(format!("{}: suite has no cases", path.display())); |
| 747 | | - } |
| 1729 | + let artifacts_captured = artifacts |
| 1730 | + .keys() |
| 1731 | + .map(|artifact| artifact.as_str().to_string()) |
| 1732 | + .collect::<Vec<_>>(); |
| 748 | 1733 | |
| 749 | | - Ok(SuiteSpec { |
| 750 | | - name: suite_name, |
| 751 | | - path: path.to_path_buf(), |
| 752 | | - cases, |
| 1734 | + Ok(CompilerObservation { |
| 1735 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 1736 | + program: program.to_path_buf(), |
| 1737 | + opt_level, |
| 1738 | + compile_exit_code, |
| 1739 | + artifacts, |
| 1740 | + provenance: ObservationProvenance { |
| 1741 | + compiler_identity: "armfortas".into(), |
| 1742 | + adapter_kind: "named".into(), |
| 1743 | + backend_mode, |
| 1744 | + backend_detail, |
| 1745 | + artifacts_captured, |
| 1746 | + comparison_basis: None, |
| 1747 | + failure_stage, |
| 1748 | + }, |
| 753 | 1749 | }) |
| 754 | 1750 | } |
| 755 | 1751 | |
| 756 | | -struct CaseBuilder { |
| 757 | | - name: String, |
| 758 | | - source: Option<PathBuf>, |
| 759 | | - graph_entry: Option<PathBuf>, |
| 760 | | - graph_files: Vec<PathBuf>, |
| 761 | | - requested: BTreeSet<Stage>, |
| 762 | | - opt_levels: Vec<OptLevel>, |
| 763 | | - repeat_count: usize, |
| 764 | | - reference_compilers: Vec<ReferenceCompiler>, |
| 765 | | - consistency_checks: Vec<ConsistencyCheck>, |
| 766 | | - expectations: Vec<Expectation>, |
| 767 | | - status_rules: Vec<StatusRule>, |
| 1752 | +#[derive(Debug, Clone)] |
| 1753 | +struct DriverCompileResult { |
| 1754 | + command: String, |
| 1755 | + exit_code: i32, |
| 1756 | + stdout: String, |
| 1757 | + stderr: String, |
| 1758 | + output: PathBuf, |
| 768 | 1759 | } |
| 769 | 1760 | |
| 770 | | -impl CaseBuilder { |
| 771 | | - fn new(name: String) -> Self { |
| 772 | | - Self { |
| 773 | | - name, |
| 774 | | - source: None, |
| 775 | | - graph_entry: None, |
| 776 | | - graph_files: Vec::new(), |
| 777 | | - requested: BTreeSet::new(), |
| 778 | | - opt_levels: Vec::new(), |
| 779 | | - repeat_count: 2, |
| 780 | | - reference_compilers: Vec::new(), |
| 781 | | - consistency_checks: Vec::new(), |
| 782 | | - expectations: Vec::new(), |
| 783 | | - status_rules: Vec::new(), |
| 784 | | - } |
| 785 | | - } |
| 1761 | +fn observe_external_driver( |
| 1762 | + spec: &CompilerSpec, |
| 1763 | + binary: &str, |
| 1764 | + program: &Path, |
| 1765 | + opt_level: OptLevel, |
| 1766 | + requested: &BTreeSet<ArtifactKey>, |
| 1767 | + uses_cpp: bool, |
| 1768 | + adapter_kind: String, |
| 1769 | + otool: &str, |
| 1770 | + nm: &str, |
| 1771 | +) -> Result<CompilerObservation, String> { |
| 1772 | + let temp_root = next_observation_temp_root(&spec.display_name(), opt_level); |
| 1773 | + fs::create_dir_all(&temp_root).map_err(|e| { |
| 1774 | + format!( |
| 1775 | + "cannot create observation temp dir '{}': {}", |
| 1776 | + temp_root.display(), |
| 1777 | + e |
| 1778 | + ) |
| 1779 | + })?; |
| 786 | 1780 | |
| 787 | | - fn build(self, suite_path: &Path) -> Result<CaseSpec, String> { |
| 788 | | - if self.source.is_some() && (self.graph_entry.is_some() || !self.graph_files.is_empty()) { |
| 789 | | - return Err(format!( |
| 790 | | - "{}: case '{}' mixes source with graph entry/file declarations", |
| 791 | | - suite_path.display(), |
| 792 | | - self.name |
| 793 | | - )); |
| 794 | | - } |
| 1781 | + let needs_runtime = requested.contains(&ArtifactKey::Runtime) |
| 1782 | + || requested.contains(&ArtifactKey::Stdout) |
| 1783 | + || requested.contains(&ArtifactKey::Stderr) |
| 1784 | + || requested.contains(&ArtifactKey::ExitCode) |
| 1785 | + || requested.contains(&ArtifactKey::Executable); |
| 1786 | + let primary_mode = if needs_runtime { |
| 1787 | + DriverEmitMode::Binary |
| 1788 | + } else if requested.contains(&ArtifactKey::Asm) { |
| 1789 | + DriverEmitMode::Asm |
| 1790 | + } else if requested.contains(&ArtifactKey::Obj) { |
| 1791 | + DriverEmitMode::Obj |
| 1792 | + } else { |
| 1793 | + DriverEmitMode::Binary |
| 1794 | + }; |
| 1795 | + let primary_name = match primary_mode { |
| 1796 | + DriverEmitMode::Binary => "observe.out", |
| 1797 | + DriverEmitMode::Asm => "observe.s", |
| 1798 | + DriverEmitMode::Obj => "observe.o", |
| 1799 | + }; |
| 1800 | + let primary = compile_with_external_driver( |
| 1801 | + binary, |
| 1802 | + program, |
| 1803 | + opt_level, |
| 1804 | + primary_mode, |
| 1805 | + &temp_root.join(primary_name), |
| 1806 | + uses_cpp, |
| 1807 | + )?; |
| 795 | 1808 | |
| 796 | | - if self.graph_entry.is_some() && self.graph_files.is_empty() { |
| 797 | | - return Err(format!( |
| 798 | | - "{}: case '{}' declares an entry without any file members", |
| 799 | | - suite_path.display(), |
| 800 | | - self.name |
| 801 | | - )); |
| 1809 | + let mut artifacts = BTreeMap::new(); |
| 1810 | + if !primary.stdout.trim().is_empty() |
| 1811 | + || !primary.stderr.trim().is_empty() |
| 1812 | + || primary.exit_code != 0 |
| 1813 | + { |
| 1814 | + let diagnostics = [primary.stdout.trim_end(), primary.stderr.trim_end()] |
| 1815 | + .iter() |
| 1816 | + .filter(|part| !part.is_empty()) |
| 1817 | + .copied() |
| 1818 | + .collect::<Vec<_>>() |
| 1819 | + .join("\n"); |
| 1820 | + artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 1821 | + } |
| 1822 | + |
| 1823 | + let mut compile_exit_code = primary.exit_code; |
| 1824 | + if primary.exit_code == 0 { |
| 1825 | + match primary_mode { |
| 1826 | + DriverEmitMode::Binary => { |
| 1827 | + if requested.contains(&ArtifactKey::Executable) { |
| 1828 | + artifacts.insert( |
| 1829 | + ArtifactKey::Executable, |
| 1830 | + ArtifactValue::Path(primary.output.clone()), |
| 1831 | + ); |
| 1832 | + } |
| 1833 | + if needs_runtime { |
| 1834 | + let run_command = render_binary_run_command(&primary.output); |
| 1835 | + let run = run_binary_capture(&primary.output, &temp_root, &run_command) |
| 1836 | + .map_err(|detail| format!("build: {}\n{}", primary.command, detail))?; |
| 1837 | + insert_run_artifacts(requested, &run, &mut artifacts); |
| 1838 | + } |
| 1839 | + } |
| 1840 | + DriverEmitMode::Asm if requested.contains(&ArtifactKey::Asm) => { |
| 1841 | + artifacts.insert( |
| 1842 | + ArtifactKey::Asm, |
| 1843 | + ArtifactValue::Text(fs::read_to_string(&primary.output).map_err(|e| { |
| 1844 | + format!( |
| 1845 | + "cannot read asm artifact '{}': {}", |
| 1846 | + primary.output.display(), |
| 1847 | + e |
| 1848 | + ) |
| 1849 | + })?), |
| 1850 | + ); |
| 1851 | + } |
| 1852 | + DriverEmitMode::Obj if requested.contains(&ArtifactKey::Obj) => { |
| 1853 | + artifacts.insert( |
| 1854 | + ArtifactKey::Obj, |
| 1855 | + ArtifactValue::Text( |
| 1856 | + object_snapshot_text(&primary.output, otool, nm) |
| 1857 | + .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1858 | + ), |
| 1859 | + ); |
| 1860 | + } |
| 1861 | + _ => {} |
| 802 | 1862 | } |
| 803 | 1863 | |
| 804 | | - if self.graph_entry.is_none() && !self.graph_files.is_empty() { |
| 805 | | - return Err(format!( |
| 806 | | - "{}: case '{}' declares file members without an entry", |
| 807 | | - suite_path.display(), |
| 808 | | - self.name |
| 809 | | - )); |
| 1864 | + if requested.contains(&ArtifactKey::Asm) && primary_mode != DriverEmitMode::Asm { |
| 1865 | + let asm = compile_with_external_driver( |
| 1866 | + binary, |
| 1867 | + program, |
| 1868 | + opt_level, |
| 1869 | + DriverEmitMode::Asm, |
| 1870 | + &temp_root.join("observe-extra.s"), |
| 1871 | + uses_cpp, |
| 1872 | + )?; |
| 1873 | + if asm.exit_code != 0 { |
| 1874 | + compile_exit_code = asm.exit_code; |
| 1875 | + artifacts.insert( |
| 1876 | + ArtifactKey::Diagnostics, |
| 1877 | + ArtifactValue::Text(asm.stderr.trim_end().to_string()), |
| 1878 | + ); |
| 1879 | + } else { |
| 1880 | + artifacts.insert( |
| 1881 | + ArtifactKey::Asm, |
| 1882 | + ArtifactValue::Text(fs::read_to_string(&asm.output).map_err(|e| { |
| 1883 | + format!("cannot read asm artifact '{}': {}", asm.output.display(), e) |
| 1884 | + })?), |
| 1885 | + ); |
| 1886 | + } |
| 810 | 1887 | } |
| 811 | 1888 | |
| 812 | | - let (source, graph_files) = if let Some(source) = self.source { |
| 813 | | - (source, Vec::new()) |
| 814 | | - } else if let Some(entry) = self.graph_entry { |
| 815 | | - if !self.graph_files.iter().any(|file| file == &entry) { |
| 816 | | - return Err(format!( |
| 817 | | - "{}: case '{}' entry '{}' is not listed in file declarations", |
| 818 | | - suite_path.display(), |
| 819 | | - self.name, |
| 820 | | - entry.display() |
| 821 | | - )); |
| 1889 | + if requested.contains(&ArtifactKey::Obj) && primary_mode != DriverEmitMode::Obj { |
| 1890 | + let obj = compile_with_external_driver( |
| 1891 | + binary, |
| 1892 | + program, |
| 1893 | + opt_level, |
| 1894 | + DriverEmitMode::Obj, |
| 1895 | + &temp_root.join("observe-extra.o"), |
| 1896 | + uses_cpp, |
| 1897 | + )?; |
| 1898 | + if obj.exit_code != 0 { |
| 1899 | + compile_exit_code = obj.exit_code; |
| 1900 | + artifacts.insert( |
| 1901 | + ArtifactKey::Diagnostics, |
| 1902 | + ArtifactValue::Text(obj.stderr.trim_end().to_string()), |
| 1903 | + ); |
| 1904 | + } else { |
| 1905 | + artifacts.insert( |
| 1906 | + ArtifactKey::Obj, |
| 1907 | + ArtifactValue::Text( |
| 1908 | + object_snapshot_text(&obj.output, otool, nm) |
| 1909 | + .unwrap_or_else(|_| "object snapshot unavailable".into()), |
| 1910 | + ), |
| 1911 | + ); |
| 822 | 1912 | } |
| 823 | | - (entry, self.graph_files) |
| 824 | | - } else { |
| 825 | | - return Err(format!( |
| 826 | | - "{}: case '{}' is missing a source path or graph entry", |
| 827 | | - suite_path.display(), |
| 828 | | - self.name |
| 829 | | - )); |
| 830 | | - }; |
| 831 | | - |
| 832 | | - let mut requested = self.requested; |
| 833 | | - if requested.is_empty() { |
| 834 | | - requested.insert(Stage::Run); |
| 835 | 1913 | } |
| 1914 | + } |
| 836 | 1915 | |
| 837 | | - let opt_levels = if self.opt_levels.is_empty() { |
| 838 | | - vec![OptLevel::O0] |
| 839 | | - } else { |
| 840 | | - self.opt_levels |
| 841 | | - }; |
| 1916 | + let artifacts_captured = artifacts |
| 1917 | + .keys() |
| 1918 | + .map(|artifact| artifact.as_str().to_string()) |
| 1919 | + .collect::<Vec<_>>(); |
| 842 | 1920 | |
| 843 | | - Ok(CaseSpec { |
| 844 | | - name: self.name, |
| 845 | | - source, |
| 846 | | - graph_files, |
| 847 | | - requested, |
| 848 | | - opt_levels, |
| 849 | | - repeat_count: self.repeat_count, |
| 850 | | - reference_compilers: self.reference_compilers, |
| 851 | | - consistency_checks: self.consistency_checks, |
| 852 | | - expectations: self.expectations, |
| 853 | | - status_rules: self.status_rules, |
| 854 | | - }) |
| 1921 | + Ok(CompilerObservation { |
| 1922 | + compiler: spec.clone(), |
| 1923 | + program: program.to_path_buf(), |
| 1924 | + opt_level, |
| 1925 | + compile_exit_code, |
| 1926 | + artifacts, |
| 1927 | + provenance: ObservationProvenance { |
| 1928 | + compiler_identity: spec.display_name(), |
| 1929 | + adapter_kind, |
| 1930 | + backend_mode: "external-driver".into(), |
| 1931 | + backend_detail: format!("generic external driver adapter using {}", binary), |
| 1932 | + artifacts_captured, |
| 1933 | + comparison_basis: None, |
| 1934 | + failure_stage: None, |
| 1935 | + }, |
| 1936 | + }) |
| 1937 | +} |
| 1938 | + |
| 1939 | +fn compile_with_external_driver( |
| 1940 | + binary: &str, |
| 1941 | + source: &Path, |
| 1942 | + opt_level: OptLevel, |
| 1943 | + mode: DriverEmitMode, |
| 1944 | + output: &Path, |
| 1945 | + uses_cpp: bool, |
| 1946 | +) -> Result<DriverCompileResult, String> { |
| 1947 | + let mut args = vec![opt_level.as_flag().to_string()]; |
| 1948 | + if uses_cpp { |
| 1949 | + args.push("-cpp".to_string()); |
| 1950 | + } |
| 1951 | + match mode { |
| 1952 | + DriverEmitMode::Asm => args.push("-S".to_string()), |
| 1953 | + DriverEmitMode::Obj => args.push("-c".to_string()), |
| 1954 | + DriverEmitMode::Binary => {} |
| 855 | 1955 | } |
| 1956 | + args.push(source.display().to_string()); |
| 1957 | + args.push("-o".to_string()); |
| 1958 | + args.push(output.display().to_string()); |
| 1959 | + let command = render_command(binary, &args); |
| 1960 | + let output_result = Command::new(binary) |
| 1961 | + .args(&args) |
| 1962 | + .output() |
| 1963 | + .map_err(|e| format!("cannot run '{}': {}", binary, e))?; |
| 1964 | + Ok(DriverCompileResult { |
| 1965 | + command, |
| 1966 | + exit_code: output_result.status.code().unwrap_or(-1), |
| 1967 | + stdout: String::from_utf8_lossy(&output_result.stdout).into_owned(), |
| 1968 | + stderr: String::from_utf8_lossy(&output_result.stderr).into_owned(), |
| 1969 | + output: output.to_path_buf(), |
| 1970 | + }) |
| 856 | 1971 | } |
| 857 | 1972 | |
| 858 | | -fn resolve_suite_relative_path(rest: &str, path: &Path, line_no: usize) -> Result<PathBuf, String> { |
| 859 | | - let relative = parse_quoted(rest, path, line_no)?; |
| 860 | | - Ok(path |
| 861 | | - .parent() |
| 862 | | - .unwrap_or_else(|| Path::new(".")) |
| 863 | | - .join(relative)) |
| 1973 | +fn next_observation_temp_root(label: &str, opt_level: OptLevel) -> PathBuf { |
| 1974 | + default_report_root().join(".tmp").join(format!( |
| 1975 | + "observe_{}_{}", |
| 1976 | + sanitize_component(label), |
| 1977 | + next_report_suffix(opt_level) |
| 1978 | + )) |
| 864 | 1979 | } |
| 865 | 1980 | |
| 866 | | -fn parse_stage_list(rest: &str, path: &Path, line_no: usize) -> Result<BTreeSet<Stage>, String> { |
| 1981 | +fn armfortas_requested_stages( |
| 1982 | + requested: &BTreeSet<ArtifactKey>, |
| 1983 | +) -> Result<BTreeSet<Stage>, String> { |
| 867 | 1984 | let mut stages = BTreeSet::new(); |
| 868 | | - for raw in rest.split(',') { |
| 869 | | - let name = raw.trim(); |
| 870 | | - if name.is_empty() { |
| 871 | | - continue; |
| 1985 | + for artifact in requested { |
| 1986 | + match artifact { |
| 1987 | + ArtifactKey::Asm => { |
| 1988 | + stages.insert(Stage::Asm); |
| 1989 | + } |
| 1990 | + ArtifactKey::Obj => { |
| 1991 | + stages.insert(Stage::Obj); |
| 1992 | + } |
| 1993 | + ArtifactKey::Runtime |
| 1994 | + | ArtifactKey::Stdout |
| 1995 | + | ArtifactKey::Stderr |
| 1996 | + | ArtifactKey::ExitCode => { |
| 1997 | + stages.insert(Stage::Run); |
| 1998 | + } |
| 1999 | + ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 2000 | + ArtifactKey::Extra(name) => { |
| 2001 | + let suffix = name |
| 2002 | + .strip_prefix("armfortas.") |
| 2003 | + .ok_or_else(|| format!("unsupported adapter-specific artifact '{}'", name))?; |
| 2004 | + let stage = Stage::parse(suffix) |
| 2005 | + .ok_or_else(|| format!("unknown armfortas artifact '{}'", name))?; |
| 2006 | + stages.insert(stage); |
| 2007 | + } |
| 872 | 2008 | } |
| 873 | | - let stage = Stage::parse(name) |
| 874 | | - .ok_or_else(|| format!("{}:{}: unknown stage '{}'", path.display(), line_no, name))?; |
| 875 | | - stages.insert(stage); |
| 876 | 2009 | } |
| 877 | 2010 | if stages.is_empty() { |
| 878 | | - return Err(format!( |
| 879 | | - "{}:{}: armfortas stage list is empty", |
| 880 | | - path.display(), |
| 881 | | - line_no |
| 882 | | - )); |
| 2011 | + stages.insert(Stage::Run); |
| 883 | 2012 | } |
| 884 | 2013 | Ok(stages) |
| 885 | 2014 | } |
| 886 | 2015 | |
| 887 | | -fn parse_opt_levels(rest: &str, path: &Path, line_no: usize) -> Result<Vec<OptLevel>, String> { |
| 888 | | - let mut levels = BTreeSet::new(); |
| 889 | | - for raw in rest.split(',') { |
| 890 | | - let name = raw.trim(); |
| 891 | | - if name.is_empty() { |
| 892 | | - continue; |
| 893 | | - } |
| 894 | | - if name.eq_ignore_ascii_case("all") { |
| 895 | | - levels.extend(all_opt_levels()); |
| 896 | | - continue; |
| 897 | | - } |
| 898 | | - let level = parse_opt_level_token(name).ok_or_else(|| { |
| 899 | | - format!( |
| 900 | | - "{}:{}: unknown opt level '{}'", |
| 901 | | - path.display(), |
| 902 | | - line_no, |
| 903 | | - name |
| 904 | | - ) |
| 905 | | - })?; |
| 906 | | - levels.insert(level); |
| 2016 | +fn insert_run_artifacts( |
| 2017 | + requested: &BTreeSet<ArtifactKey>, |
| 2018 | + run: &RunCapture, |
| 2019 | + artifacts: &mut BTreeMap<ArtifactKey, ArtifactValue>, |
| 2020 | +) { |
| 2021 | + if requested.contains(&ArtifactKey::Runtime) { |
| 2022 | + artifacts.insert(ArtifactKey::Runtime, ArtifactValue::Run(run.clone())); |
| 907 | 2023 | } |
| 908 | | - if levels.is_empty() { |
| 909 | | - return Err(format!( |
| 910 | | - "{}:{}: opt level list is empty", |
| 911 | | - path.display(), |
| 912 | | - line_no |
| 913 | | - )); |
| 2024 | + if requested.contains(&ArtifactKey::Stdout) { |
| 2025 | + artifacts.insert(ArtifactKey::Stdout, ArtifactValue::Text(run.stdout.clone())); |
| 2026 | + } |
| 2027 | + if requested.contains(&ArtifactKey::Stderr) { |
| 2028 | + artifacts.insert(ArtifactKey::Stderr, ArtifactValue::Text(run.stderr.clone())); |
| 2029 | + } |
| 2030 | + if requested.contains(&ArtifactKey::ExitCode) { |
| 2031 | + artifacts.insert(ArtifactKey::ExitCode, ArtifactValue::Int(run.exit_code)); |
| 914 | 2032 | } |
| 915 | | - Ok(levels.into_iter().collect()) |
| 916 | 2033 | } |
| 917 | 2034 | |
| 918 | | -fn parse_reference_compilers( |
| 919 | | - rest: &str, |
| 920 | | - path: &Path, |
| 921 | | - line_no: usize, |
| 922 | | -) -> Result<Vec<ReferenceCompiler>, String> { |
| 923 | | - let mut compilers = BTreeSet::new(); |
| 924 | | - for raw in rest.split(',') { |
| 925 | | - let name = raw.trim(); |
| 926 | | - if name.is_empty() { |
| 927 | | - continue; |
| 2035 | +fn compare_observations( |
| 2036 | + mut left: CompilerObservation, |
| 2037 | + mut right: CompilerObservation, |
| 2038 | + requested: &BTreeSet<ArtifactKey>, |
| 2039 | +) -> ComparisonResult { |
| 2040 | + let basis = format!( |
| 2041 | + "compile-status, diagnostics, runtime{}", |
| 2042 | + if requested.is_empty() { |
| 2043 | + String::new() |
| 2044 | + } else { |
| 2045 | + let extras = requested |
| 2046 | + .iter() |
| 2047 | + .filter(|artifact| { |
| 2048 | + !matches!(artifact, ArtifactKey::Diagnostics | ArtifactKey::Runtime) |
| 2049 | + }) |
| 2050 | + .map(|artifact| artifact.as_str().to_string()) |
| 2051 | + .collect::<Vec<_>>(); |
| 2052 | + if extras.is_empty() { |
| 2053 | + String::new() |
| 2054 | + } else { |
| 2055 | + format!(", {}", extras.join(", ")) |
| 2056 | + } |
| 928 | 2057 | } |
| 929 | | - let compiler = ReferenceCompiler::parse(name).ok_or_else(|| { |
| 930 | | - format!( |
| 931 | | - "{}:{}: unknown reference compiler '{}'", |
| 932 | | - path.display(), |
| 933 | | - line_no, |
| 934 | | - name |
| 935 | | - ) |
| 936 | | - })?; |
| 937 | | - compilers.insert(compiler); |
| 938 | | - } |
| 939 | | - if compilers.is_empty() { |
| 940 | | - return Err(format!( |
| 941 | | - "{}:{}: differential compiler list is empty", |
| 942 | | - path.display(), |
| 943 | | - line_no |
| 944 | | - )); |
| 2058 | + ); |
| 2059 | + left.provenance.comparison_basis = Some(basis.clone()); |
| 2060 | + right.provenance.comparison_basis = Some(basis.clone()); |
| 2061 | + |
| 2062 | + let mut differences = Vec::new(); |
| 2063 | + if left.compile_exit_code != right.compile_exit_code { |
| 2064 | + differences.push(ArtifactDifference { |
| 2065 | + artifact: "compile-exit-code".into(), |
| 2066 | + detail: format!( |
| 2067 | + "{}: {}\n{}: {}", |
| 2068 | + left.compiler.display_name(), |
| 2069 | + left.compile_exit_code, |
| 2070 | + right.compiler.display_name(), |
| 2071 | + right.compile_exit_code |
| 2072 | + ), |
| 2073 | + }); |
| 945 | 2074 | } |
| 946 | | - Ok(compilers.into_iter().collect()) |
| 947 | | -} |
| 948 | 2075 | |
| 949 | | -fn parse_repeat_count(rest: &str, path: &Path, line_no: usize) -> Result<usize, String> { |
| 950 | | - let count = rest.trim().parse::<usize>().map_err(|_| { |
| 951 | | - format!( |
| 952 | | - "{}:{}: repeat count must be an integer >= 2", |
| 953 | | - path.display(), |
| 954 | | - line_no |
| 955 | | - ) |
| 956 | | - })?; |
| 957 | | - if count < 2 { |
| 958 | | - return Err(format!( |
| 959 | | - "{}:{}: repeat count must be >= 2", |
| 960 | | - path.display(), |
| 961 | | - line_no |
| 962 | | - )); |
| 963 | | - } |
| 964 | | - Ok(count) |
| 965 | | -} |
| 2076 | + compare_artifact_text( |
| 2077 | + &left, |
| 2078 | + &right, |
| 2079 | + &ArtifactKey::Diagnostics, |
| 2080 | + "diagnostics", |
| 2081 | + &mut differences, |
| 2082 | + ); |
| 966 | 2083 | |
| 967 | | -fn parse_consistency_checks( |
| 968 | | - rest: &str, |
| 969 | | - path: &Path, |
| 970 | | - line_no: usize, |
| 971 | | -) -> Result<Vec<ConsistencyCheck>, String> { |
| 972 | | - let mut checks = Vec::new(); |
| 973 | | - for raw in rest.split(',') { |
| 974 | | - let name = raw.trim(); |
| 975 | | - if name.is_empty() { |
| 976 | | - continue; |
| 2084 | + if left.compile_exit_code == 0 && right.compile_exit_code == 0 { |
| 2085 | + if requested.contains(&ArtifactKey::Runtime) { |
| 2086 | + compare_artifact_runtime(&left, &right, &mut differences); |
| 977 | 2087 | } |
| 978 | | - let check = ConsistencyCheck::parse(name).ok_or_else(|| { |
| 979 | | - format!( |
| 980 | | - "{}:{}: unknown consistency check '{}'", |
| 981 | | - path.display(), |
| 982 | | - line_no, |
| 983 | | - name |
| 984 | | - ) |
| 985 | | - })?; |
| 986 | | - if !checks.contains(&check) { |
| 987 | | - checks.push(check); |
| 2088 | + for artifact in requested { |
| 2089 | + match artifact { |
| 2090 | + ArtifactKey::Diagnostics | ArtifactKey::Runtime => {} |
| 2091 | + ArtifactKey::Stdout | ArtifactKey::Stderr | ArtifactKey::Asm | ArtifactKey::Obj => { |
| 2092 | + compare_artifact_text( |
| 2093 | + &left, |
| 2094 | + &right, |
| 2095 | + artifact, |
| 2096 | + artifact.as_str(), |
| 2097 | + &mut differences, |
| 2098 | + ); |
| 2099 | + } |
| 2100 | + ArtifactKey::ExitCode => { |
| 2101 | + compare_artifact_int(&left, &right, artifact, &mut differences) |
| 2102 | + } |
| 2103 | + ArtifactKey::Executable => { |
| 2104 | + compare_artifact_path(&left, &right, artifact, &mut differences) |
| 2105 | + } |
| 2106 | + ArtifactKey::Extra(name) => { |
| 2107 | + compare_artifact_text(&left, &right, artifact, name, &mut differences) |
| 2108 | + } |
| 2109 | + } |
| 988 | 2110 | } |
| 989 | 2111 | } |
| 990 | | - if checks.is_empty() { |
| 991 | | - return Err(format!( |
| 992 | | - "{}:{}: consistency check list is empty", |
| 993 | | - path.display(), |
| 994 | | - line_no |
| 995 | | - )); |
| 2112 | + |
| 2113 | + ComparisonResult { |
| 2114 | + left, |
| 2115 | + right, |
| 2116 | + basis, |
| 2117 | + differences, |
| 996 | 2118 | } |
| 997 | | - Ok(checks) |
| 998 | 2119 | } |
| 999 | 2120 | |
| 1000 | | -fn parse_expectation(rest: &str, path: &Path, line_no: usize) -> Result<Expectation, String> { |
| 1001 | | - if let Some(prefix) = rest.strip_suffix(" check-comments") { |
| 1002 | | - return Ok(Expectation::CheckComments(parse_target( |
| 1003 | | - prefix.trim(), |
| 1004 | | - path, |
| 1005 | | - line_no, |
| 1006 | | - )?)); |
| 2121 | +fn compare_artifact_text( |
| 2122 | + left: &CompilerObservation, |
| 2123 | + right: &CompilerObservation, |
| 2124 | + artifact: &ArtifactKey, |
| 2125 | + label: &str, |
| 2126 | + differences: &mut Vec<ArtifactDifference>, |
| 2127 | +) { |
| 2128 | + let left_text = match left.artifacts.get(artifact) { |
| 2129 | + Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2130 | + _ => "", |
| 2131 | + }; |
| 2132 | + let right_text = match right.artifacts.get(artifact) { |
| 2133 | + Some(ArtifactValue::Text(text)) => text.as_str(), |
| 2134 | + _ => "", |
| 2135 | + }; |
| 2136 | + if left_text != right_text { |
| 2137 | + differences.push(ArtifactDifference { |
| 2138 | + artifact: label.to_string(), |
| 2139 | + detail: describe_text_difference( |
| 2140 | + left_text, |
| 2141 | + right_text, |
| 2142 | + &left.compiler.display_name(), |
| 2143 | + &right.compiler.display_name(), |
| 2144 | + ), |
| 2145 | + }); |
| 1007 | 2146 | } |
| 2147 | +} |
| 1008 | 2148 | |
| 1009 | | - if let Some((target, value)) = rest.split_once(" not-contains ") { |
| 1010 | | - return Ok(Expectation::NotContains { |
| 1011 | | - target: parse_target(target.trim(), path, line_no)?, |
| 1012 | | - needle: parse_quoted(value.trim(), path, line_no)?, |
| 2149 | +fn compare_artifact_runtime( |
| 2150 | + left: &CompilerObservation, |
| 2151 | + right: &CompilerObservation, |
| 2152 | + differences: &mut Vec<ArtifactDifference>, |
| 2153 | +) { |
| 2154 | + let left_run = match left.artifacts.get(&ArtifactKey::Runtime) { |
| 2155 | + Some(ArtifactValue::Run(run)) => Some(run), |
| 2156 | + _ => None, |
| 2157 | + }; |
| 2158 | + let right_run = match right.artifacts.get(&ArtifactKey::Runtime) { |
| 2159 | + Some(ArtifactValue::Run(run)) => Some(run), |
| 2160 | + _ => None, |
| 2161 | + }; |
| 2162 | + match (left_run, right_run) { |
| 2163 | + (Some(left_run), Some(right_run)) => { |
| 2164 | + if normalize_run_signature(left_run) != normalize_run_signature(right_run) { |
| 2165 | + differences.push(ArtifactDifference { |
| 2166 | + artifact: "runtime".into(), |
| 2167 | + detail: describe_run_difference( |
| 2168 | + left_run, |
| 2169 | + right_run, |
| 2170 | + &left.compiler.display_name(), |
| 2171 | + &right.compiler.display_name(), |
| 2172 | + ), |
| 2173 | + }); |
| 2174 | + } |
| 2175 | + } |
| 2176 | + _ => differences.push(ArtifactDifference { |
| 2177 | + artifact: "runtime".into(), |
| 2178 | + detail: "one side did not produce a runtime result".into(), |
| 2179 | + }), |
| 2180 | + } |
| 2181 | +} |
| 2182 | + |
| 2183 | +fn compare_artifact_int( |
| 2184 | + left: &CompilerObservation, |
| 2185 | + right: &CompilerObservation, |
| 2186 | + artifact: &ArtifactKey, |
| 2187 | + differences: &mut Vec<ArtifactDifference>, |
| 2188 | +) { |
| 2189 | + let left_value = match left.artifacts.get(artifact) { |
| 2190 | + Some(ArtifactValue::Int(value)) => Some(*value), |
| 2191 | + _ => None, |
| 2192 | + }; |
| 2193 | + let right_value = match right.artifacts.get(artifact) { |
| 2194 | + Some(ArtifactValue::Int(value)) => Some(*value), |
| 2195 | + _ => None, |
| 2196 | + }; |
| 2197 | + if left_value != right_value { |
| 2198 | + differences.push(ArtifactDifference { |
| 2199 | + artifact: artifact.as_str().to_string(), |
| 2200 | + detail: format!( |
| 2201 | + "{}: {:?}\n{}: {:?}", |
| 2202 | + left.compiler.display_name(), |
| 2203 | + left_value, |
| 2204 | + right.compiler.display_name(), |
| 2205 | + right_value |
| 2206 | + ), |
| 1013 | 2207 | }); |
| 1014 | 2208 | } |
| 2209 | +} |
| 1015 | 2210 | |
| 1016 | | - if let Some((target, value)) = rest.split_once(" contains ") { |
| 1017 | | - return Ok(Expectation::Contains { |
| 1018 | | - target: parse_target(target.trim(), path, line_no)?, |
| 1019 | | - needle: parse_quoted(value.trim(), path, line_no)?, |
| 1020 | | - }); |
| 2211 | +fn compare_artifact_path( |
| 2212 | + left: &CompilerObservation, |
| 2213 | + right: &CompilerObservation, |
| 2214 | + artifact: &ArtifactKey, |
| 2215 | + differences: &mut Vec<ArtifactDifference>, |
| 2216 | +) { |
| 2217 | + let left_path = match left.artifacts.get(artifact) { |
| 2218 | + Some(ArtifactValue::Path(path)) => Some(path), |
| 2219 | + _ => None, |
| 2220 | + }; |
| 2221 | + let right_path = match right.artifacts.get(artifact) { |
| 2222 | + Some(ArtifactValue::Path(path)) => Some(path), |
| 2223 | + _ => None, |
| 2224 | + }; |
| 2225 | + |
| 2226 | + match (left_path, right_path) { |
| 2227 | + (Some(left_path), Some(right_path)) => { |
| 2228 | + let left_bytes = fs::read(left_path); |
| 2229 | + let right_bytes = fs::read(right_path); |
| 2230 | + match (left_bytes, right_bytes) { |
| 2231 | + (Ok(left_bytes), Ok(right_bytes)) => { |
| 2232 | + if left_bytes != right_bytes { |
| 2233 | + differences.push(ArtifactDifference { |
| 2234 | + artifact: artifact.as_str().to_string(), |
| 2235 | + detail: describe_binary_difference( |
| 2236 | + &left_bytes, |
| 2237 | + &right_bytes, |
| 2238 | + &left.compiler.display_name(), |
| 2239 | + &right.compiler.display_name(), |
| 2240 | + left_path, |
| 2241 | + right_path, |
| 2242 | + ), |
| 2243 | + }); |
| 2244 | + } |
| 2245 | + } |
| 2246 | + (Err(left_err), Err(right_err)) => { |
| 2247 | + differences.push(ArtifactDifference { |
| 2248 | + artifact: artifact.as_str().to_string(), |
| 2249 | + detail: format!( |
| 2250 | + "{}: unable to read '{}': {}\n{}: unable to read '{}': {}", |
| 2251 | + left.compiler.display_name(), |
| 2252 | + left_path.display(), |
| 2253 | + left_err, |
| 2254 | + right.compiler.display_name(), |
| 2255 | + right_path.display(), |
| 2256 | + right_err |
| 2257 | + ), |
| 2258 | + }); |
| 2259 | + } |
| 2260 | + (Err(left_err), Ok(_)) => { |
| 2261 | + differences.push(ArtifactDifference { |
| 2262 | + artifact: artifact.as_str().to_string(), |
| 2263 | + detail: format!( |
| 2264 | + "{}: unable to read '{}': {}\n{}: readable '{}'", |
| 2265 | + left.compiler.display_name(), |
| 2266 | + left_path.display(), |
| 2267 | + left_err, |
| 2268 | + right.compiler.display_name(), |
| 2269 | + right_path.display() |
| 2270 | + ), |
| 2271 | + }); |
| 2272 | + } |
| 2273 | + (Ok(_), Err(right_err)) => { |
| 2274 | + differences.push(ArtifactDifference { |
| 2275 | + artifact: artifact.as_str().to_string(), |
| 2276 | + detail: format!( |
| 2277 | + "{}: readable '{}'\n{}: unable to read '{}': {}", |
| 2278 | + left.compiler.display_name(), |
| 2279 | + left_path.display(), |
| 2280 | + right.compiler.display_name(), |
| 2281 | + right_path.display(), |
| 2282 | + right_err |
| 2283 | + ), |
| 2284 | + }); |
| 2285 | + } |
| 2286 | + } |
| 2287 | + } |
| 2288 | + _ => { |
| 2289 | + let left_value = left_path.map(|path| path.display().to_string()); |
| 2290 | + let right_value = right_path.map(|path| path.display().to_string()); |
| 2291 | + if left_value != right_value { |
| 2292 | + differences.push(ArtifactDifference { |
| 2293 | + artifact: artifact.as_str().to_string(), |
| 2294 | + detail: format!( |
| 2295 | + "{}: {:?}\n{}: {:?}", |
| 2296 | + left.compiler.display_name(), |
| 2297 | + left_value, |
| 2298 | + right.compiler.display_name(), |
| 2299 | + right_value |
| 2300 | + ), |
| 2301 | + }); |
| 2302 | + } |
| 2303 | + } |
| 1021 | 2304 | } |
| 2305 | +} |
| 1022 | 2306 | |
| 1023 | | - if let Some((target, value)) = rest.split_once(" equals ") { |
| 1024 | | - let target = parse_target(target.trim(), path, line_no)?; |
| 1025 | | - if matches!(target, Target::RunExitCode) { |
| 1026 | | - let value = parse_integer(value.trim(), path, line_no)?; |
| 1027 | | - return Ok(Expectation::IntEquals { target, value }); |
| 2307 | +fn describe_binary_difference( |
| 2308 | + left: &[u8], |
| 2309 | + right: &[u8], |
| 2310 | + left_label: &str, |
| 2311 | + right_label: &str, |
| 2312 | + left_path: &Path, |
| 2313 | + right_path: &Path, |
| 2314 | +) -> String { |
| 2315 | + let shared = left.len().min(right.len()); |
| 2316 | + for index in 0..shared { |
| 2317 | + if left[index] != right[index] { |
| 2318 | + return format!( |
| 2319 | + "first differing byte: {}\n{}: {} bytes ({})\n{}: 0x{:02x}\n{}: {} bytes ({})\n{}: 0x{:02x}", |
| 2320 | + index, |
| 2321 | + left_label, |
| 2322 | + left.len(), |
| 2323 | + left_path.display(), |
| 2324 | + left_label, |
| 2325 | + left[index], |
| 2326 | + right_label, |
| 2327 | + right.len(), |
| 2328 | + right_path.display(), |
| 2329 | + right_label, |
| 2330 | + right[index] |
| 2331 | + ); |
| 1028 | 2332 | } |
| 1029 | | - return Ok(Expectation::Equals { |
| 1030 | | - target, |
| 1031 | | - value: parse_quoted(value.trim(), path, line_no)?, |
| 1032 | | - }); |
| 1033 | 2333 | } |
| 1034 | 2334 | |
| 1035 | | - Err(format!( |
| 1036 | | - "{}:{}: unsupported expectation '{}'", |
| 1037 | | - path.display(), |
| 1038 | | - line_no, |
| 1039 | | - rest |
| 1040 | | - )) |
| 2335 | + format!( |
| 2336 | + "binary length differs\n{}: {} bytes ({})\n{}: {} bytes ({})", |
| 2337 | + left_label, |
| 2338 | + left.len(), |
| 2339 | + left_path.display(), |
| 2340 | + right_label, |
| 2341 | + right.len(), |
| 2342 | + right_path.display() |
| 2343 | + ) |
| 1041 | 2344 | } |
| 1042 | 2345 | |
| 1043 | | -fn parse_failure_expectation( |
| 1044 | | - rest: &str, |
| 1045 | | - path: &Path, |
| 1046 | | - line_no: usize, |
| 1047 | | -) -> Result<Expectation, String> { |
| 1048 | | - if let Some((target, value)) = rest.split_once(" contains ") { |
| 1049 | | - return Ok(Expectation::FailContains { |
| 1050 | | - stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 1051 | | - needle: parse_quoted(value.trim(), path, line_no)?, |
| 1052 | | - }); |
| 2346 | +fn compare_status(result: &ComparisonResult) -> &'static str { |
| 2347 | + if result.differences.is_empty() { |
| 2348 | + "match" |
| 2349 | + } else { |
| 2350 | + "diff" |
| 1053 | 2351 | } |
| 2352 | +} |
| 1054 | 2353 | |
| 1055 | | - if let Some((target, value)) = rest.split_once(" equals ") { |
| 1056 | | - return Ok(Expectation::FailEquals { |
| 1057 | | - stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 1058 | | - value: parse_quoted(value.trim(), path, line_no)?, |
| 1059 | | - }); |
| 2354 | +fn compare_classification(result: &ComparisonResult) -> &'static str { |
| 2355 | + if result.differences.is_empty() { |
| 2356 | + return "match"; |
| 1060 | 2357 | } |
| 1061 | 2358 | |
| 1062 | | - Err(format!( |
| 1063 | | - "{}:{}: unsupported failure expectation '{}'", |
| 1064 | | - path.display(), |
| 1065 | | - line_no, |
| 1066 | | - rest |
| 1067 | | - )) |
| 1068 | | -} |
| 2359 | + let mut has_compile = false; |
| 2360 | + let mut has_diagnostics = false; |
| 2361 | + let mut has_runtime = false; |
| 2362 | + let mut has_artifact = false; |
| 1069 | 2363 | |
| 1070 | | -fn parse_status_rule( |
| 1071 | | - kind: StatusKind, |
| 1072 | | - rest: &str, |
| 1073 | | - path: &Path, |
| 1074 | | - line_no: usize, |
| 1075 | | -) -> Result<StatusRule, String> { |
| 1076 | | - let rest = rest.trim(); |
| 1077 | | - if rest.starts_with('"') { |
| 1078 | | - return Ok(StatusRule { |
| 1079 | | - kind, |
| 1080 | | - selector: OptSelector::All, |
| 1081 | | - reason: parse_quoted(rest, path, line_no)?, |
| 1082 | | - }); |
| 2364 | + for difference in &result.differences { |
| 2365 | + match difference.artifact.as_str() { |
| 2366 | + "compile-exit-code" => has_compile = true, |
| 2367 | + "diagnostics" => has_diagnostics = true, |
| 2368 | + "runtime" => has_runtime = true, |
| 2369 | + _ => has_artifact = true, |
| 2370 | + } |
| 1083 | 2371 | } |
| 1084 | 2372 | |
| 1085 | | - let conditional = rest.strip_prefix("when ").ok_or_else(|| { |
| 2373 | + if has_compile { |
| 2374 | + if has_runtime || has_artifact { |
| 2375 | + "mixed divergence" |
| 2376 | + } else { |
| 2377 | + "compile divergence" |
| 2378 | + } |
| 2379 | + } else if has_runtime && !has_diagnostics && !has_artifact { |
| 2380 | + "runtime divergence" |
| 2381 | + } else if has_artifact && !has_runtime && !has_diagnostics { |
| 2382 | + "artifact divergence" |
| 2383 | + } else if has_diagnostics && !has_runtime && !has_artifact { |
| 2384 | + "diagnostics divergence" |
| 2385 | + } else { |
| 2386 | + "mixed divergence" |
| 2387 | + } |
| 2388 | +} |
| 2389 | + |
| 2390 | +fn compare_changed_artifacts(result: &ComparisonResult) -> Vec<String> { |
| 2391 | + result |
| 2392 | + .differences |
| 2393 | + .iter() |
| 2394 | + .map(|difference| difference.artifact.clone()) |
| 2395 | + .collect() |
| 2396 | +} |
| 2397 | + |
| 2398 | +fn render_compare_text(result: &ComparisonResult) -> String { |
| 2399 | + let changed_artifacts = compare_changed_artifacts(result); |
| 2400 | + let mut lines = vec![ |
| 2401 | + "Compare".to_string(), |
| 2402 | + format!(" left: {}", result.left.compiler.display_name()), |
| 2403 | + format!(" right: {}", result.right.compiler.display_name()), |
| 2404 | + format!(" program: {}", result.left.program.display()), |
| 2405 | + format!(" opt: {}", result.left.opt_level.as_str()), |
| 2406 | + format!(" status: {}", compare_status(result)), |
| 2407 | + format!(" classification: {}", compare_classification(result)), |
| 2408 | + format!(" basis: {}", result.basis), |
| 2409 | + format!(" difference_count: {}", result.differences.len()), |
| 1086 | 2410 | format!( |
| 1087 | | - "{}:{}: expected quoted reason or 'when <opts> because \"...\"'", |
| 1088 | | - path.display(), |
| 1089 | | - line_no |
| 1090 | | - ) |
| 1091 | | - })?; |
| 1092 | | - let (selector, reason) = conditional.split_once(" because ").ok_or_else(|| { |
| 2411 | + " changed_artifacts: {}", |
| 2412 | + if changed_artifacts.is_empty() { |
| 2413 | + "none".to_string() |
| 2414 | + } else { |
| 2415 | + changed_artifacts.join(", ") |
| 2416 | + } |
| 2417 | + ), |
| 1093 | 2418 | format!( |
| 1094 | | - "{}:{}: conditional status must use 'when <opts> because \"...\"'", |
| 1095 | | - path.display(), |
| 1096 | | - line_no |
| 1097 | | - ) |
| 1098 | | - })?; |
| 2419 | + " left_backend: {} ({})", |
| 2420 | + result.left.provenance.backend_mode, result.left.provenance.backend_detail |
| 2421 | + ), |
| 2422 | + format!( |
| 2423 | + " right_backend: {} ({})", |
| 2424 | + result.right.provenance.backend_mode, result.right.provenance.backend_detail |
| 2425 | + ), |
| 2426 | + ]; |
| 1099 | 2427 | |
| 1100 | | - Ok(StatusRule { |
| 1101 | | - kind, |
| 1102 | | - selector: parse_opt_selector(selector.trim(), path, line_no)?, |
| 1103 | | - reason: parse_quoted(reason.trim(), path, line_no)?, |
| 1104 | | - }) |
| 2428 | + if result.differences.is_empty() { |
| 2429 | + lines.push(String::new()); |
| 2430 | + lines.push("No differences detected.".to_string()); |
| 2431 | + } else { |
| 2432 | + for difference in &result.differences { |
| 2433 | + lines.push(String::new()); |
| 2434 | + lines.push(format!("== {} ==", difference.artifact)); |
| 2435 | + lines.push(difference.detail.clone()); |
| 2436 | + } |
| 2437 | + } |
| 2438 | + |
| 2439 | + lines.join("\n") |
| 1105 | 2440 | } |
| 1106 | 2441 | |
| 1107 | | -fn parse_opt_selector(raw: &str, path: &Path, line_no: usize) -> Result<OptSelector, String> { |
| 1108 | | - let raw = raw.trim(); |
| 1109 | | - if raw.eq_ignore_ascii_case("all") { |
| 1110 | | - return Ok(OptSelector::All); |
| 2442 | +fn print_compare_result(result: &ComparisonResult) { |
| 2443 | + println!("{}", render_compare_text(result)); |
| 2444 | +} |
| 2445 | + |
| 2446 | +fn print_introspection(config: &IntrospectConfig, observed: &ObservedProgram) { |
| 2447 | + println!( |
| 2448 | + "{}", |
| 2449 | + render_introspection_text( |
| 2450 | + observed, |
| 2451 | + IntrospectionRenderConfig { |
| 2452 | + summary_only: config.summary_only, |
| 2453 | + max_artifact_lines: config.max_artifact_lines, |
| 2454 | + } |
| 2455 | + ) |
| 2456 | + ); |
| 2457 | +} |
| 2458 | + |
| 2459 | +fn write_compare_reports(config: &CompareConfig, result: &ComparisonResult) -> Result<(), String> { |
| 2460 | + if let Some(path) = &config.json_report { |
| 2461 | + write_report(path, &render_compare_json(result), "json report")?; |
| 2462 | + println!("json report: {}", path.display()); |
| 1111 | 2463 | } |
| 1112 | | - if let Some(rest) = raw.strip_prefix("opts =>") { |
| 1113 | | - return Ok(OptSelector::Only(parse_opt_levels(rest, path, line_no)?)); |
| 2464 | + if let Some(path) = &config.markdown_report { |
| 2465 | + write_report(path, &render_compare_markdown(result), "markdown report")?; |
| 2466 | + println!("markdown report: {}", path.display()); |
| 1114 | 2467 | } |
| 1115 | | - Ok(OptSelector::Only(parse_opt_levels(raw, path, line_no)?)) |
| 2468 | + Ok(()) |
| 1116 | 2469 | } |
| 1117 | 2470 | |
| 1118 | | -fn parse_target(raw: &str, path: &Path, line_no: usize) -> Result<Target, String> { |
| 1119 | | - match raw { |
| 1120 | | - "run.stdout" => Ok(Target::RunStdout), |
| 1121 | | - "run.stderr" => Ok(Target::RunStderr), |
| 1122 | | - "run.exit_code" => Ok(Target::RunExitCode), |
| 1123 | | - _ => { |
| 1124 | | - let stage = Stage::parse(raw).ok_or_else(|| { |
| 1125 | | - format!( |
| 1126 | | - "{}:{}: unsupported expectation target '{}'", |
| 1127 | | - path.display(), |
| 1128 | | - line_no, |
| 1129 | | - raw |
| 1130 | | - ) |
| 1131 | | - })?; |
| 1132 | | - Ok(Target::Stage(stage)) |
| 1133 | | - } |
| 2471 | +fn write_introspection_reports( |
| 2472 | + config: &IntrospectConfig, |
| 2473 | + observed: &ObservedProgram, |
| 2474 | +) -> Result<(), String> { |
| 2475 | + let render_config = IntrospectionRenderConfig { |
| 2476 | + summary_only: config.summary_only, |
| 2477 | + max_artifact_lines: config.max_artifact_lines, |
| 2478 | + }; |
| 2479 | + if let Some(path) = &config.json_report { |
| 2480 | + write_report(path, &render_introspection_json(observed), "json report")?; |
| 2481 | + println!("json report: {}", path.display()); |
| 2482 | + } |
| 2483 | + if let Some(path) = &config.markdown_report { |
| 2484 | + write_report( |
| 2485 | + path, |
| 2486 | + &render_introspection_markdown(observed, render_config), |
| 2487 | + "markdown report", |
| 2488 | + )?; |
| 2489 | + println!("markdown report: {}", path.display()); |
| 1134 | 2490 | } |
| 2491 | + Ok(()) |
| 1135 | 2492 | } |
| 1136 | 2493 | |
| 1137 | | -fn parse_failure_stage(raw: &str, path: &Path, line_no: usize) -> Result<FailureStage, String> { |
| 1138 | | - FailureStage::parse(raw).ok_or_else(|| { |
| 1139 | | - format!( |
| 1140 | | - "{}:{}: unsupported failure stage '{}'", |
| 1141 | | - path.display(), |
| 1142 | | - line_no, |
| 1143 | | - raw |
| 1144 | | - ) |
| 1145 | | - }) |
| 2494 | +fn introspection_status(observation: &CompilerObservation) -> &'static str { |
| 2495 | + if observation.compile_exit_code == 0 { |
| 2496 | + "compile ok" |
| 2497 | + } else { |
| 2498 | + "compile failed" |
| 2499 | + } |
| 1146 | 2500 | } |
| 1147 | 2501 | |
| 1148 | | -fn parse_quoted(raw: &str, path: &Path, line_no: usize) -> Result<String, String> { |
| 1149 | | - let raw = raw.trim(); |
| 1150 | | - if !(raw.starts_with('"') && raw.ends_with('"')) { |
| 1151 | | - return Err(format!( |
| 1152 | | - "{}:{}: expected quoted string, got '{}'", |
| 1153 | | - path.display(), |
| 1154 | | - line_no, |
| 1155 | | - raw |
| 1156 | | - )); |
| 2502 | +fn diagnostic_excerpt(observation: &CompilerObservation) -> Option<String> { |
| 2503 | + match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 2504 | + Some(ArtifactValue::Text(text)) => text |
| 2505 | + .lines() |
| 2506 | + .map(str::trim) |
| 2507 | + .find(|line| !line.is_empty()) |
| 2508 | + .map(|line| line.to_string()), |
| 2509 | + _ => None, |
| 1157 | 2510 | } |
| 1158 | | - let body = &raw[1..raw.len() - 1]; |
| 1159 | | - Ok(body.replace("\\\"", "\"").replace("\\n", "\n")) |
| 1160 | 2511 | } |
| 1161 | 2512 | |
| 1162 | | -fn parse_integer(raw: &str, path: &Path, line_no: usize) -> Result<i32, String> { |
| 1163 | | - let value = if raw.starts_with('"') { |
| 1164 | | - parse_quoted(raw, path, line_no)? |
| 1165 | | - } else { |
| 1166 | | - raw.trim().to_string() |
| 1167 | | - }; |
| 1168 | | - value.parse::<i32>().map_err(|_| { |
| 1169 | | - format!( |
| 1170 | | - "{}:{}: expected integer literal, got '{}'", |
| 1171 | | - path.display(), |
| 1172 | | - line_no, |
| 1173 | | - raw |
| 1174 | | - ) |
| 1175 | | - }) |
| 2513 | +fn failure_stage_summary(observation: &CompilerObservation) -> &str { |
| 2514 | + observation |
| 2515 | + .provenance |
| 2516 | + .failure_stage |
| 2517 | + .as_deref() |
| 2518 | + .unwrap_or("none") |
| 1176 | 2519 | } |
| 1177 | 2520 | |
| 1178 | | -fn parse_opt_level_token(raw: &str) -> Option<OptLevel> { |
| 1179 | | - let raw = raw.trim(); |
| 1180 | | - let raw = raw.strip_prefix('-').unwrap_or(raw); |
| 1181 | | - OptLevel::parse_flag(raw) |
| 2521 | +fn requested_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2522 | + observed |
| 2523 | + .requested_artifacts |
| 2524 | + .iter() |
| 2525 | + .map(|artifact| artifact.as_str().to_string()) |
| 2526 | + .collect() |
| 1182 | 2527 | } |
| 1183 | 2528 | |
| 1184 | | -fn parse_opt_level_list(raw: &str) -> Result<Vec<OptLevel>, String> { |
| 1185 | | - let mut levels = BTreeSet::new(); |
| 1186 | | - for value in raw.split(',') { |
| 1187 | | - let value = value.trim(); |
| 1188 | | - if value.is_empty() { |
| 1189 | | - continue; |
| 1190 | | - } |
| 1191 | | - if value.eq_ignore_ascii_case("all") { |
| 1192 | | - levels.extend(all_opt_levels()); |
| 1193 | | - continue; |
| 2529 | +fn missing_introspection_artifact_names(observed: &ObservedProgram) -> Vec<String> { |
| 2530 | + observed |
| 2531 | + .requested_artifacts |
| 2532 | + .iter() |
| 2533 | + .filter(|artifact| { |
| 2534 | + if matches!(artifact, ArtifactKey::Diagnostics) |
| 2535 | + && observed.observation.compile_exit_code == 0 |
| 2536 | + && !observed.observation.artifacts.contains_key(*artifact) |
| 2537 | + { |
| 2538 | + return false; |
| 2539 | + } |
| 2540 | + !observed.observation.artifacts.contains_key(*artifact) |
| 2541 | + }) |
| 2542 | + .map(|artifact| artifact.as_str().to_string()) |
| 2543 | + .collect() |
| 2544 | +} |
| 2545 | + |
| 2546 | +fn observation_generic_artifacts<'a>( |
| 2547 | + observation: &'a CompilerObservation, |
| 2548 | +) -> Vec<(String, &'a ArtifactValue)> { |
| 2549 | + observation |
| 2550 | + .artifacts |
| 2551 | + .iter() |
| 2552 | + .filter(|(artifact, _)| artifact.is_generic()) |
| 2553 | + .map(|(artifact, value)| (artifact.as_str().to_string(), value)) |
| 2554 | + .collect() |
| 2555 | +} |
| 2556 | + |
| 2557 | +fn observation_adapter_extras<'a>( |
| 2558 | + observation: &'a CompilerObservation, |
| 2559 | +) -> BTreeMap<String, Vec<(String, &'a ArtifactValue)>> { |
| 2560 | + let mut extras = BTreeMap::new(); |
| 2561 | + for (artifact, value) in &observation.artifacts { |
| 2562 | + if let ArtifactKey::Extra(name) = artifact { |
| 2563 | + let (namespace, local_name) = artifact |
| 2564 | + .extra_parts() |
| 2565 | + .map(|(namespace, local_name)| (namespace.to_string(), local_name.to_string())) |
| 2566 | + .unwrap_or_else(|| ("extra".to_string(), name.clone())); |
| 2567 | + extras |
| 2568 | + .entry(namespace) |
| 2569 | + .or_insert_with(Vec::new) |
| 2570 | + .push((local_name, value)); |
| 1194 | 2571 | } |
| 1195 | | - let level = |
| 1196 | | - parse_opt_level_token(value).ok_or_else(|| format!("unknown opt level '{}'", value))?; |
| 1197 | | - levels.insert(level); |
| 1198 | | - } |
| 1199 | | - if levels.is_empty() { |
| 1200 | | - return Err("opt filter is empty".into()); |
| 1201 | 2572 | } |
| 1202 | | - Ok(levels.into_iter().collect()) |
| 2573 | + extras |
| 1203 | 2574 | } |
| 1204 | 2575 | |
| 1205 | | -fn all_opt_levels() -> [OptLevel; 5] { |
| 1206 | | - [ |
| 1207 | | - OptLevel::O0, |
| 1208 | | - OptLevel::O1, |
| 1209 | | - OptLevel::O2, |
| 1210 | | - OptLevel::O3, |
| 1211 | | - OptLevel::Ofast, |
| 1212 | | - ] |
| 2576 | +fn format_artifact_name_list(names: &[String]) -> String { |
| 2577 | + if names.is_empty() { |
| 2578 | + "none".to_string() |
| 2579 | + } else { |
| 2580 | + names.join(", ") |
| 2581 | + } |
| 1213 | 2582 | } |
| 1214 | 2583 | |
| 1215 | | -fn filter_suites<'a>(suites: &'a [SuiteSpec], suite_filter: Option<&str>) -> Vec<&'a SuiteSpec> { |
| 1216 | | - let filter = suite_filter.map(|value| value.to_ascii_lowercase()); |
| 1217 | | - suites |
| 2584 | +fn format_adapter_extra_summary( |
| 2585 | + extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2586 | +) -> String { |
| 2587 | + if extras.is_empty() { |
| 2588 | + return "none".to_string(); |
| 2589 | + } |
| 2590 | + |
| 2591 | + extras |
| 1218 | 2592 | .iter() |
| 1219 | | - .filter(|suite| { |
| 1220 | | - if let Some(filter) = &filter { |
| 1221 | | - suite.name.to_ascii_lowercase().contains(filter) |
| 1222 | | - } else { |
| 1223 | | - true |
| 1224 | | - } |
| 2593 | + .map(|(namespace, entries)| { |
| 2594 | + format!( |
| 2595 | + "{}({})", |
| 2596 | + namespace, |
| 2597 | + entries |
| 2598 | + .iter() |
| 2599 | + .map(|(name, _)| name.as_str()) |
| 2600 | + .collect::<Vec<_>>() |
| 2601 | + .join(", ") |
| 2602 | + ) |
| 1225 | 2603 | }) |
| 1226 | | - .collect() |
| 2604 | + .collect::<Vec<_>>() |
| 2605 | + .join(", ") |
| 1227 | 2606 | } |
| 1228 | 2607 | |
| 1229 | | -fn print_suites(suites: &[&SuiteSpec]) { |
| 1230 | | - for suite in suites { |
| 1231 | | - println!("{} ({})", suite.name, suite.cases.len()); |
| 1232 | | - println!(" {}", suite.path.display()); |
| 2608 | +fn render_named_artifact_map_json(entries: &[(String, &ArtifactValue)]) -> String { |
| 2609 | + let mut rendered = String::from("{"); |
| 2610 | + for (index, (name, value)) in entries.iter().enumerate() { |
| 2611 | + if index > 0 { |
| 2612 | + rendered.push_str(", "); |
| 2613 | + } |
| 2614 | + rendered.push_str(&format!( |
| 2615 | + "\"{}\": {}", |
| 2616 | + json_escape(name), |
| 2617 | + render_artifact_value_json(value) |
| 2618 | + )); |
| 1233 | 2619 | } |
| 2620 | + rendered.push('}'); |
| 2621 | + rendered |
| 1234 | 2622 | } |
| 1235 | 2623 | |
| 1236 | | -fn run_suites(config: &RunConfig) -> Result<Summary, String> { |
| 1237 | | - let suites = discover_suites(default_suite_root())?; |
| 1238 | | - let suites = filter_suites(&suites, config.suite_filter.as_deref()); |
| 1239 | | - if suites.is_empty() { |
| 1240 | | - return Err("no suites matched the requested filter".into()); |
| 2624 | +fn render_flat_artifacts_json(observation: &CompilerObservation) -> String { |
| 2625 | + let mut entries = Vec::new(); |
| 2626 | + for (artifact, value) in &observation.artifacts { |
| 2627 | + entries.push((artifact.as_str().to_string(), value)); |
| 1241 | 2628 | } |
| 2629 | + render_named_artifact_map_json(&entries) |
| 2630 | +} |
| 1242 | 2631 | |
| 1243 | | - let case_filter = config |
| 1244 | | - .case_filter |
| 1245 | | - .as_ref() |
| 1246 | | - .map(|value| value.to_ascii_lowercase()); |
| 1247 | | - let mut summary = Summary::default(); |
| 1248 | | - let mut matched_cells = 0usize; |
| 1249 | | - |
| 1250 | | - for suite in suites { |
| 1251 | | - println!("=== {} ===", suite.name); |
| 1252 | | - for case in &suite.cases { |
| 1253 | | - if let Some(filter) = &case_filter { |
| 1254 | | - if !case.name.to_ascii_lowercase().contains(filter) { |
| 1255 | | - continue; |
| 1256 | | - } |
| 1257 | | - } |
| 2632 | +fn render_artifact_summary_json(value: &ArtifactValue) -> String { |
| 2633 | + match value { |
| 2634 | + ArtifactValue::Text(text) => format!( |
| 2635 | + "{{\"kind\":\"text\",\"summary\":\"{}\",\"line_count\":{},\"char_count\":{}}}", |
| 2636 | + json_escape(&artifact_value_summary(value)), |
| 2637 | + text_line_count(text), |
| 2638 | + text.len() |
| 2639 | + ), |
| 2640 | + ArtifactValue::Int(number) => format!( |
| 2641 | + "{{\"kind\":\"int\",\"summary\":\"{}\",\"value\":{}}}", |
| 2642 | + json_escape(&artifact_value_summary(value)), |
| 2643 | + number |
| 2644 | + ), |
| 2645 | + ArtifactValue::Run(run) => format!( |
| 2646 | + "{{\"kind\":\"runtime\",\"summary\":\"{}\",\"exit_code\":{},\"stdout_lines\":{},\"stderr_lines\":{}}}", |
| 2647 | + json_escape(&artifact_value_summary(value)), |
| 2648 | + run.exit_code, |
| 2649 | + text_line_count(&run.stdout), |
| 2650 | + text_line_count(&run.stderr) |
| 2651 | + ), |
| 2652 | + ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2653 | + Ok(metadata) => format!( |
| 2654 | + "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":{}}}", |
| 2655 | + json_escape(&artifact_value_summary(value)), |
| 2656 | + metadata.len() |
| 2657 | + ), |
| 2658 | + Err(_) => format!( |
| 2659 | + "{{\"kind\":\"path\",\"summary\":\"{}\",\"byte_count\":null}}", |
| 2660 | + json_escape(&artifact_value_summary(value)) |
| 2661 | + ), |
| 2662 | + }, |
| 2663 | + } |
| 2664 | +} |
| 1258 | 2665 | |
| 1259 | | - let opt_levels = selected_opt_levels(case, config); |
| 1260 | | - for opt_level in opt_levels { |
| 1261 | | - matched_cells += 1; |
| 1262 | | - let outcome = execute_case_cell(suite, case, opt_level, config)?; |
| 1263 | | - print_outcome(&outcome); |
| 1264 | | - summary.record_outcome(&outcome); |
| 2666 | +fn render_artifact_summaries_json(observation: &CompilerObservation) -> String { |
| 2667 | + let mut rendered = String::from("{"); |
| 2668 | + for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 2669 | + if index > 0 { |
| 2670 | + rendered.push_str(", "); |
| 2671 | + } |
| 2672 | + rendered.push_str(&format!( |
| 2673 | + "\"{}\": {}", |
| 2674 | + json_escape(artifact.as_str()), |
| 2675 | + render_artifact_summary_json(value) |
| 2676 | + )); |
| 2677 | + } |
| 2678 | + rendered.push('}'); |
| 2679 | + rendered |
| 2680 | +} |
| 1265 | 2681 | |
| 1266 | | - if config.fail_fast |
| 1267 | | - && matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 1268 | | - { |
| 1269 | | - return Ok(summary); |
| 1270 | | - } |
| 1271 | | - } |
| 2682 | +fn render_adapter_extra_summary_json( |
| 2683 | + extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2684 | +) -> String { |
| 2685 | + let mut rendered = String::from("{"); |
| 2686 | + for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2687 | + if index > 0 { |
| 2688 | + rendered.push_str(", "); |
| 1272 | 2689 | } |
| 2690 | + let names = entries |
| 2691 | + .iter() |
| 2692 | + .map(|(name, _)| name.clone()) |
| 2693 | + .collect::<Vec<_>>(); |
| 2694 | + rendered.push_str(&format!( |
| 2695 | + "\"{}\": {}", |
| 2696 | + json_escape(namespace), |
| 2697 | + json_string_array(&names) |
| 2698 | + )); |
| 1273 | 2699 | } |
| 2700 | + rendered.push('}'); |
| 2701 | + rendered |
| 2702 | +} |
| 1274 | 2703 | |
| 1275 | | - if matched_cells == 0 { |
| 1276 | | - return Err("no cases matched the requested filters".into()); |
| 2704 | +fn render_namespaced_artifacts_json( |
| 2705 | + extras: &BTreeMap<String, Vec<(String, &ArtifactValue)>>, |
| 2706 | +) -> String { |
| 2707 | + let mut rendered = String::from("{"); |
| 2708 | + for (index, (namespace, entries)) in extras.iter().enumerate() { |
| 2709 | + if index > 0 { |
| 2710 | + rendered.push_str(", "); |
| 2711 | + } |
| 2712 | + rendered.push_str(&format!( |
| 2713 | + "\"{}\": {}", |
| 2714 | + json_escape(namespace), |
| 2715 | + render_named_artifact_map_json(entries) |
| 2716 | + )); |
| 1277 | 2717 | } |
| 2718 | + rendered.push('}'); |
| 2719 | + rendered |
| 2720 | +} |
| 1278 | 2721 | |
| 1279 | | - Ok(summary) |
| 2722 | +fn text_line_count(text: &str) -> usize { |
| 2723 | + if text.is_empty() { |
| 2724 | + 0 |
| 2725 | + } else { |
| 2726 | + text.lines().count() |
| 2727 | + } |
| 1280 | 2728 | } |
| 1281 | 2729 | |
| 1282 | | -fn selected_opt_levels(case: &CaseSpec, config: &RunConfig) -> Vec<OptLevel> { |
| 1283 | | - case.opt_levels |
| 1284 | | - .iter() |
| 1285 | | - .copied() |
| 1286 | | - .filter(|level| { |
| 1287 | | - config |
| 1288 | | - .opt_filter |
| 1289 | | - .as_ref() |
| 1290 | | - .map(|filter| filter.contains(level)) |
| 1291 | | - .unwrap_or(true) |
| 1292 | | - }) |
| 1293 | | - .collect() |
| 2730 | +fn render_config_summary(config: IntrospectionRenderConfig) -> String { |
| 2731 | + if config.summary_only { |
| 2732 | + "summary-only".to_string() |
| 2733 | + } else if let Some(limit) = config.max_artifact_lines { |
| 2734 | + format!("first {} lines per artifact", limit) |
| 2735 | + } else { |
| 2736 | + "full artifact bodies".to_string() |
| 2737 | + } |
| 1294 | 2738 | } |
| 1295 | 2739 | |
| 1296 | | -fn execute_case_cell( |
| 1297 | | - suite: &SuiteSpec, |
| 1298 | | - case: &CaseSpec, |
| 1299 | | - opt_level: OptLevel, |
| 1300 | | - config: &RunConfig, |
| 1301 | | -) -> Result<Outcome, String> { |
| 1302 | | - let effective_status = status_for_opt(case, opt_level); |
| 1303 | | - if let EffectiveStatus::Future(reason) = &effective_status { |
| 1304 | | - if !config.include_future { |
| 1305 | | - return Ok(Outcome { |
| 1306 | | - suite: suite.name.clone(), |
| 1307 | | - case: case.name.clone(), |
| 1308 | | - opt_level, |
| 1309 | | - kind: OutcomeKind::Future, |
| 1310 | | - detail: reason.clone(), |
| 1311 | | - bundle: None, |
| 1312 | | - consistency_observations: Vec::new(), |
| 1313 | | - }); |
| 2740 | +fn artifact_value_summary(value: &ArtifactValue) -> String { |
| 2741 | + match value { |
| 2742 | + ArtifactValue::Text(text) => { |
| 2743 | + format!( |
| 2744 | + "text, {} lines, {} chars", |
| 2745 | + text_line_count(text), |
| 2746 | + text.len() |
| 2747 | + ) |
| 1314 | 2748 | } |
| 2749 | + ArtifactValue::Int(value) => format!("int, value {}", value), |
| 2750 | + ArtifactValue::Run(run) => format!( |
| 2751 | + "runtime, exit {}, stdout {} lines, stderr {} lines", |
| 2752 | + run.exit_code, |
| 2753 | + text_line_count(&run.stdout), |
| 2754 | + text_line_count(&run.stderr) |
| 2755 | + ), |
| 2756 | + ArtifactValue::Path(path) => match fs::metadata(path) { |
| 2757 | + Ok(metadata) => format!("path, {} bytes", metadata.len()), |
| 2758 | + Err(_) => "path".to_string(), |
| 2759 | + }, |
| 1315 | 2760 | } |
| 2761 | +} |
| 1316 | 2762 | |
| 1317 | | - let mut requested = case.requested.clone(); |
| 1318 | | - if config.all_stages { |
| 1319 | | - requested.extend(Stage::ALL); |
| 2763 | +fn render_artifact_body_lines( |
| 2764 | + value: &ArtifactValue, |
| 2765 | + config: IntrospectionRenderConfig, |
| 2766 | +) -> Vec<String> { |
| 2767 | + if config.summary_only { |
| 2768 | + return vec!["[content omitted by --summary-only]".to_string()]; |
| 1320 | 2769 | } |
| 1321 | | - for expectation in &case.expectations { |
| 1322 | | - ensure_target_stage(expectation, &mut requested); |
| 2770 | + |
| 2771 | + let rendered = render_artifact_value_text(value); |
| 2772 | + let mut lines = rendered |
| 2773 | + .lines() |
| 2774 | + .map(|line| line.to_string()) |
| 2775 | + .collect::<Vec<_>>(); |
| 2776 | + if lines.is_empty() { |
| 2777 | + return vec!["<empty>".to_string()]; |
| 1323 | 2778 | } |
| 1324 | | - if !case.reference_compilers.is_empty() { |
| 1325 | | - requested.insert(Stage::Run); |
| 2779 | + |
| 2780 | + if let Some(limit) = config.max_artifact_lines { |
| 2781 | + if lines.len() > limit { |
| 2782 | + let total = lines.len(); |
| 2783 | + lines.truncate(limit); |
| 2784 | + lines.push(format!( |
| 2785 | + "... (truncated; showing first {} of {} lines)", |
| 2786 | + limit, total |
| 2787 | + )); |
| 2788 | + } |
| 1326 | 2789 | } |
| 1327 | | - for check in &case.consistency_checks { |
| 1328 | | - ensure_consistency_stage(*check, &mut requested); |
| 2790 | + |
| 2791 | + lines |
| 2792 | +} |
| 2793 | + |
| 2794 | +fn render_introspection_text( |
| 2795 | + observed: &ObservedProgram, |
| 2796 | + render_config: IntrospectionRenderConfig, |
| 2797 | +) -> String { |
| 2798 | + let observation = &observed.observation; |
| 2799 | + let generic_artifacts = observation_generic_artifacts(observation); |
| 2800 | + let generic_names = generic_artifacts |
| 2801 | + .iter() |
| 2802 | + .map(|(name, _)| name.clone()) |
| 2803 | + .collect::<Vec<_>>(); |
| 2804 | + let adapter_extras = observation_adapter_extras(observation); |
| 2805 | + let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2806 | + let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2807 | + let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2808 | + let mut lines = vec![ |
| 2809 | + "Introspect".to_string(), |
| 2810 | + format!(" status: {}", introspection_status(observation)), |
| 2811 | + format!(" compiler: {}", observation.compiler.display_name()), |
| 2812 | + format!(" program: {}", observation.program.display()), |
| 2813 | + format!(" opt: {}", observation.opt_level.as_str()), |
| 2814 | + format!(" compile_exit_code: {}", observation.compile_exit_code), |
| 2815 | + format!(" adapter_kind: {}", observation.provenance.adapter_kind), |
| 2816 | + format!(" backend_mode: {}", observation.provenance.backend_mode), |
| 2817 | + format!( |
| 2818 | + " backend_detail: {}", |
| 2819 | + observation.provenance.backend_detail |
| 2820 | + ), |
| 2821 | + format!(" failure_stage: {}", failure_stage_summary(observation)), |
| 2822 | + format!( |
| 2823 | + " diagnostic_excerpt: {}", |
| 2824 | + diagnostic_excerpt |
| 2825 | + .clone() |
| 2826 | + .unwrap_or_else(|| "none".to_string()) |
| 2827 | + ), |
| 2828 | + format!(" content_mode: {}", render_config_summary(render_config)), |
| 2829 | + format!(" artifact_count: {}", observation.artifacts.len()), |
| 2830 | + format!( |
| 2831 | + " requested_artifacts: {}", |
| 2832 | + format_artifact_name_list(&requested_artifacts) |
| 2833 | + ), |
| 2834 | + format!( |
| 2835 | + " missing_artifacts: {}", |
| 2836 | + format_artifact_name_list(&missing_artifacts) |
| 2837 | + ), |
| 2838 | + format!( |
| 2839 | + " generic_artifacts: {}", |
| 2840 | + format_artifact_name_list(&generic_names) |
| 2841 | + ), |
| 2842 | + format!( |
| 2843 | + " adapter_extras: {}", |
| 2844 | + format_adapter_extra_summary(&adapter_extras) |
| 2845 | + ), |
| 2846 | + ]; |
| 2847 | + if !observation.provenance.artifacts_captured.is_empty() { |
| 2848 | + lines.push(format!( |
| 2849 | + " captured_artifacts: {}", |
| 2850 | + observation.provenance.artifacts_captured.join(", ") |
| 2851 | + )); |
| 1329 | 2852 | } |
| 1330 | 2853 | |
| 1331 | | - let prepared = prepare_case_input(case, suite, opt_level)?; |
| 2854 | + if !generic_artifacts.is_empty() { |
| 2855 | + lines.push(String::new()); |
| 2856 | + lines.push("Generic artifacts".to_string()); |
| 2857 | + for (artifact, value) in generic_artifacts { |
| 2858 | + lines.push(String::new()); |
| 2859 | + lines.push(format!("== {} ==", artifact)); |
| 2860 | + lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2861 | + lines.extend(render_artifact_body_lines(value, render_config)); |
| 2862 | + } |
| 2863 | + } |
| 1332 | 2864 | |
| 1333 | | - if config.verbose { |
| 1334 | | - let stage_list = requested |
| 1335 | | - .iter() |
| 1336 | | - .map(Stage::as_str) |
| 1337 | | - .collect::<Vec<_>>() |
| 1338 | | - .join(", "); |
| 1339 | | - let refs = if case.reference_compilers.is_empty() { |
| 1340 | | - "none".to_string() |
| 1341 | | - } else { |
| 1342 | | - case.reference_compilers |
| 1343 | | - .iter() |
| 1344 | | - .map(ReferenceCompiler::as_str) |
| 1345 | | - .collect::<Vec<_>>() |
| 1346 | | - .join(", ") |
| 1347 | | - }; |
| 1348 | | - println!(" source: {}", case.source_label()); |
| 1349 | | - if case.is_graph() { |
| 1350 | | - for file in &case.graph_files { |
| 1351 | | - println!(" file: {}", file.display()); |
| 2865 | + if !adapter_extras.is_empty() { |
| 2866 | + lines.push(String::new()); |
| 2867 | + lines.push("Adapter extras".to_string()); |
| 2868 | + for (namespace, entries) in adapter_extras { |
| 2869 | + lines.push(String::new()); |
| 2870 | + lines.push(format!("-- {} --", namespace)); |
| 2871 | + for (name, value) in entries { |
| 2872 | + lines.push(String::new()); |
| 2873 | + lines.push(format!("== {} ==", name)); |
| 2874 | + lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 2875 | + lines.extend(render_artifact_body_lines(value, render_config)); |
| 1352 | 2876 | } |
| 1353 | | - println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 1354 | | - } |
| 1355 | | - println!(" opt: {}", opt_level.as_str()); |
| 1356 | | - println!(" stages: {}", stage_list); |
| 1357 | | - println!(" refs: {}", refs); |
| 1358 | | - if !case.consistency_checks.is_empty() { |
| 1359 | | - println!(" repeat: {}", case.repeat_count); |
| 1360 | 2877 | } |
| 1361 | 2878 | } |
| 2879 | + lines.join("\n") |
| 2880 | +} |
| 1362 | 2881 | |
| 1363 | | - let request = CaptureRequest { |
| 1364 | | - input: prepared.compiler_source.clone(), |
| 1365 | | - requested: requested.clone(), |
| 1366 | | - opt_level, |
| 1367 | | - }; |
| 1368 | | - |
| 1369 | | - let references = run_reference_compilers(&prepared, case, opt_level, &config.tools); |
| 1370 | | - let mut artifacts = ExecutionArtifacts { |
| 1371 | | - requested, |
| 1372 | | - armfortas: None, |
| 1373 | | - armfortas_failure: None, |
| 1374 | | - references, |
| 1375 | | - consistency_issues: Vec::new(), |
| 1376 | | - }; |
| 1377 | | - |
| 1378 | | - match capture_from_path(&request) { |
| 1379 | | - Ok(result) => artifacts.armfortas = Some(result), |
| 1380 | | - Err(failure) => artifacts.armfortas_failure = Some(failure), |
| 1381 | | - } |
| 2882 | +fn render_compare_json(result: &ComparisonResult) -> String { |
| 2883 | + let changed_artifacts = compare_changed_artifacts(result); |
| 2884 | + format!( |
| 2885 | + "{{\n \"status\": \"{}\",\n \"classification\": \"{}\",\n \"difference_count\": {},\n \"changed_artifacts\": {},\n \"basis\": \"{}\",\n \"left\": {},\n \"right\": {},\n \"differences\": {}\n}}\n", |
| 2886 | + compare_status(result), |
| 2887 | + compare_classification(result), |
| 2888 | + result.differences.len(), |
| 2889 | + json_string_array(&changed_artifacts), |
| 2890 | + json_escape(&result.basis), |
| 2891 | + render_observation_json(&result.left), |
| 2892 | + render_observation_json(&result.right), |
| 2893 | + render_differences_json(&result.differences) |
| 2894 | + ) |
| 2895 | +} |
| 1382 | 2896 | |
| 1383 | | - let execution = match (&artifacts.armfortas, &artifacts.armfortas_failure) { |
| 1384 | | - (Some(result), None) => { |
| 1385 | | - if has_failure_expectation(case) { |
| 1386 | | - Err(format!( |
| 1387 | | - "expected armfortas to fail ({}) but compilation succeeded", |
| 1388 | | - expected_failure_description(case) |
| 1389 | | - )) |
| 2897 | +fn render_compare_markdown(result: &ComparisonResult) -> String { |
| 2898 | + let changed_artifacts = compare_changed_artifacts(result); |
| 2899 | + let mut lines = vec![ |
| 2900 | + "# bencch compare report".to_string(), |
| 2901 | + String::new(), |
| 2902 | + format!("status: {}", compare_status(result)), |
| 2903 | + format!("classification: {}", compare_classification(result)), |
| 2904 | + format!( |
| 2905 | + "compilers: `{}` vs `{}`", |
| 2906 | + result.left.compiler.display_name(), |
| 2907 | + result.right.compiler.display_name() |
| 2908 | + ), |
| 2909 | + format!("basis: {}", result.basis), |
| 2910 | + format!("difference_count: {}", result.differences.len()), |
| 2911 | + format!( |
| 2912 | + "changed_artifacts: {}", |
| 2913 | + if changed_artifacts.is_empty() { |
| 2914 | + "none".to_string() |
| 1390 | 2915 | } else { |
| 1391 | | - let mut execution = evaluate_positive_expectations(case, result); |
| 1392 | | - if execution.is_ok() && !artifacts.references.is_empty() { |
| 1393 | | - execution = compare_differential(result, &artifacts.references); |
| 1394 | | - } |
| 1395 | | - if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 1396 | | - artifacts.consistency_issues = |
| 1397 | | - run_consistency_checks(case, &prepared, opt_level, result, &config.tools); |
| 1398 | | - if !artifacts.consistency_issues.is_empty() { |
| 1399 | | - execution = Err(format_consistency_issues(&artifacts.consistency_issues)); |
| 1400 | | - } |
| 1401 | | - } |
| 1402 | | - execution |
| 1403 | | - } |
| 1404 | | - } |
| 1405 | | - (None, Some(failure)) => { |
| 1406 | | - let partial = failure.partial_result(); |
| 1407 | | - let mut execution = evaluate_positive_expectations(case, &partial); |
| 1408 | | - if execution.is_ok() { |
| 1409 | | - if has_failure_expectation(case) { |
| 1410 | | - execution = evaluate_failure_expectations(case, failure); |
| 1411 | | - } else { |
| 1412 | | - execution = Err(compose_armfortas_failure_detail(&artifacts)); |
| 1413 | | - } |
| 1414 | | - } |
| 1415 | | - if execution.is_ok() && !artifacts.references.is_empty() { |
| 1416 | | - execution = |
| 1417 | | - Err("differential comparison requires a successful armfortas run".to_string()); |
| 2916 | + changed_artifacts.join(", ") |
| 1418 | 2917 | } |
| 1419 | | - execution |
| 2918 | + ), |
| 2919 | + String::new(), |
| 2920 | + "## Left".to_string(), |
| 2921 | + render_observation_markdown(&result.left), |
| 2922 | + String::new(), |
| 2923 | + "## Right".to_string(), |
| 2924 | + render_observation_markdown(&result.right), |
| 2925 | + String::new(), |
| 2926 | + "## Differences".to_string(), |
| 2927 | + ]; |
| 2928 | + if result.differences.is_empty() { |
| 2929 | + lines.push("none".to_string()); |
| 2930 | + } else { |
| 2931 | + for difference in &result.differences { |
| 2932 | + lines.push(String::new()); |
| 2933 | + lines.push(format!("### `{}`", difference.artifact)); |
| 2934 | + lines.push("```text".to_string()); |
| 2935 | + lines.extend(difference.detail.lines().map(|line| line.to_string())); |
| 2936 | + lines.push("```".to_string()); |
| 1420 | 2937 | } |
| 1421 | | - (Some(_), Some(_)) => Err("armfortas produced both a result and a failure".into()), |
| 1422 | | - (None, None) => Err("armfortas produced neither a result nor a failure".into()), |
| 1423 | | - }; |
| 2938 | + } |
| 2939 | + lines.join("\n") + "\n" |
| 2940 | +} |
| 1424 | 2941 | |
| 1425 | | - let consistency_observations = artifacts |
| 1426 | | - .consistency_issues |
| 2942 | +fn render_introspection_json(observed: &ObservedProgram) -> String { |
| 2943 | + let observation = &observed.observation; |
| 2944 | + let generic_artifacts = observation_generic_artifacts(observation); |
| 2945 | + let generic_names = generic_artifacts |
| 1427 | 2946 | .iter() |
| 1428 | | - .map(ConsistencyIssue::observation) |
| 2947 | + .map(|(name, _)| name.clone()) |
| 1429 | 2948 | .collect::<Vec<_>>(); |
| 1430 | | - |
| 1431 | | - let mut outcome = match (effective_status, execution) { |
| 1432 | | - (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 1433 | | - suite: suite.name.clone(), |
| 1434 | | - case: case.name.clone(), |
| 1435 | | - opt_level, |
| 1436 | | - kind: OutcomeKind::Pass, |
| 1437 | | - detail: String::new(), |
| 1438 | | - bundle: None, |
| 1439 | | - consistency_observations: consistency_observations.clone(), |
| 1440 | | - }, |
| 1441 | | - (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 1442 | | - suite: suite.name.clone(), |
| 1443 | | - case: case.name.clone(), |
| 1444 | | - opt_level, |
| 1445 | | - kind: OutcomeKind::Fail, |
| 1446 | | - detail, |
| 1447 | | - bundle: None, |
| 1448 | | - consistency_observations: consistency_observations.clone(), |
| 1449 | | - }, |
| 1450 | | - (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 1451 | | - suite: suite.name.clone(), |
| 1452 | | - case: case.name.clone(), |
| 1453 | | - opt_level, |
| 1454 | | - kind: OutcomeKind::Xpass, |
| 1455 | | - detail: reason, |
| 1456 | | - bundle: None, |
| 1457 | | - consistency_observations: consistency_observations.clone(), |
| 2949 | + let adapter_extras = observation_adapter_extras(observation); |
| 2950 | + let requested_artifacts = requested_introspection_artifact_names(observed); |
| 2951 | + let missing_artifacts = missing_introspection_artifact_names(observed); |
| 2952 | + let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 2953 | + format!( |
| 2954 | + "{{\n \"status\": \"{}\",\n \"compiler\": \"{}\",\n \"program\": \"{}\",\n \"opt\": \"{}\",\n \"compile_exit_code\": {},\n \"failure\": {{\n \"stage\": {},\n \"diagnostic_excerpt\": {}\n }},\n \"artifact_summary\": {{\n \"artifact_count\": {},\n \"requested_artifacts\": {},\n \"captured_artifacts\": {},\n \"missing_artifacts\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {}\n }},\n \"provenance\": {{\n \"compiler_identity\": \"{}\",\n \"adapter_kind\": \"{}\",\n \"backend_mode\": \"{}\",\n \"backend_detail\": \"{}\",\n \"artifacts_captured\": {},\n \"comparison_basis\": {},\n \"failure_stage\": {}\n }},\n \"artifact_summaries\": {},\n \"generic_artifacts\": {},\n \"adapter_extras\": {},\n \"artifacts\": {}\n}}\n", |
| 2955 | + json_escape(introspection_status(observation)), |
| 2956 | + json_escape(&observation.compiler.display_name()), |
| 2957 | + json_escape(&observation.program.display().to_string()), |
| 2958 | + observation.opt_level.as_str(), |
| 2959 | + observation.compile_exit_code, |
| 2960 | + match &observation.provenance.failure_stage { |
| 2961 | + Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2962 | + None => "null".to_string(), |
| 1458 | 2963 | }, |
| 1459 | | - (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 1460 | | - suite: suite.name.clone(), |
| 1461 | | - case: case.name.clone(), |
| 1462 | | - opt_level, |
| 1463 | | - kind: OutcomeKind::Xfail, |
| 1464 | | - detail: format!("{}\n{}", reason, detail), |
| 1465 | | - bundle: None, |
| 1466 | | - consistency_observations: consistency_observations.clone(), |
| 2964 | + match &diagnostic_excerpt { |
| 2965 | + Some(line) => format!("\"{}\"", json_escape(line)), |
| 2966 | + None => "null".to_string(), |
| 1467 | 2967 | }, |
| 1468 | | - (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 1469 | | - suite: suite.name.clone(), |
| 1470 | | - case: case.name.clone(), |
| 1471 | | - opt_level, |
| 1472 | | - kind: OutcomeKind::Pass, |
| 1473 | | - detail: reason, |
| 1474 | | - bundle: None, |
| 1475 | | - consistency_observations: consistency_observations.clone(), |
| 2968 | + observation.artifacts.len(), |
| 2969 | + json_string_array(&requested_artifacts), |
| 2970 | + json_string_array(&observation.provenance.artifacts_captured), |
| 2971 | + json_string_array(&missing_artifacts), |
| 2972 | + json_string_array(&generic_names), |
| 2973 | + render_adapter_extra_summary_json(&adapter_extras), |
| 2974 | + json_escape(&observation.provenance.compiler_identity), |
| 2975 | + json_escape(&observation.provenance.adapter_kind), |
| 2976 | + json_escape(&observation.provenance.backend_mode), |
| 2977 | + json_escape(&observation.provenance.backend_detail), |
| 2978 | + json_string_array(&observation.provenance.artifacts_captured), |
| 2979 | + match &observation.provenance.comparison_basis { |
| 2980 | + Some(basis) => format!("\"{}\"", json_escape(basis)), |
| 2981 | + None => "null".to_string(), |
| 1476 | 2982 | }, |
| 1477 | | - (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 1478 | | - suite: suite.name.clone(), |
| 1479 | | - case: case.name.clone(), |
| 1480 | | - opt_level, |
| 1481 | | - kind: OutcomeKind::Fail, |
| 1482 | | - detail: format!("{}\n{}", reason, detail), |
| 1483 | | - bundle: None, |
| 1484 | | - consistency_observations, |
| 2983 | + match &observation.provenance.failure_stage { |
| 2984 | + Some(stage) => format!("\"{}\"", json_escape(stage)), |
| 2985 | + None => "null".to_string(), |
| 1485 | 2986 | }, |
| 1486 | | - }; |
| 1487 | | - |
| 1488 | | - let should_bundle = matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 1489 | | - || (matches!(outcome.kind, OutcomeKind::Xfail) && !artifacts.consistency_issues.is_empty()); |
| 2987 | + render_artifact_summaries_json(observation), |
| 2988 | + render_named_artifact_map_json(&generic_artifacts), |
| 2989 | + render_namespaced_artifacts_json(&adapter_extras), |
| 2990 | + render_flat_artifacts_json(observation) |
| 2991 | + ) |
| 2992 | +} |
| 1490 | 2993 | |
| 1491 | | - if should_bundle { |
| 1492 | | - match write_failure_bundle(suite, case, &prepared, &outcome, &artifacts) { |
| 1493 | | - Ok(bundle) => outcome.bundle = Some(bundle), |
| 1494 | | - Err(err) => { |
| 1495 | | - if outcome.detail.is_empty() { |
| 1496 | | - outcome.detail = format!("failed to write failure bundle: {}", err); |
| 1497 | | - } else { |
| 1498 | | - outcome.detail.push_str(&format!( |
| 1499 | | - "\n\nwarning: failed to write failure bundle: {}", |
| 1500 | | - err |
| 1501 | | - )); |
| 1502 | | - } |
| 2994 | +fn render_introspection_markdown( |
| 2995 | + observed: &ObservedProgram, |
| 2996 | + render_config: IntrospectionRenderConfig, |
| 2997 | +) -> String { |
| 2998 | + let observation = &observed.observation; |
| 2999 | + let generic_artifacts = observation_generic_artifacts(observation); |
| 3000 | + let generic_names = generic_artifacts |
| 3001 | + .iter() |
| 3002 | + .map(|(name, _)| name.clone()) |
| 3003 | + .collect::<Vec<_>>(); |
| 3004 | + let adapter_extras = observation_adapter_extras(observation); |
| 3005 | + let requested_artifacts = requested_introspection_artifact_names(observed); |
| 3006 | + let missing_artifacts = missing_introspection_artifact_names(observed); |
| 3007 | + let diagnostic_excerpt = diagnostic_excerpt(observation); |
| 3008 | + let mut lines = vec![ |
| 3009 | + "# bencch introspect report".to_string(), |
| 3010 | + String::new(), |
| 3011 | + format!("status: {}", introspection_status(observation)), |
| 3012 | + format!("compiler: `{}`", observation.compiler.display_name()), |
| 3013 | + format!("program: `{}`", observation.program.display()), |
| 3014 | + format!("opt: `{}`", observation.opt_level.as_str()), |
| 3015 | + format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 3016 | + format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 3017 | + format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 3018 | + format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3019 | + format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3020 | + format!( |
| 3021 | + "diagnostic_excerpt: {}", |
| 3022 | + diagnostic_excerpt |
| 3023 | + .map(|line| format!("`{}`", line)) |
| 3024 | + .unwrap_or_else(|| "none".to_string()) |
| 3025 | + ), |
| 3026 | + format!("content_mode: `{}`", render_config_summary(render_config)), |
| 3027 | + format!("artifact_count: {}", observation.artifacts.len()), |
| 3028 | + format!( |
| 3029 | + "requested_artifacts: {}", |
| 3030 | + if requested_artifacts.is_empty() { |
| 3031 | + "none".to_string() |
| 3032 | + } else { |
| 3033 | + format!("`{}`", requested_artifacts.join("`, `")) |
| 3034 | + } |
| 3035 | + ), |
| 3036 | + format!( |
| 3037 | + "missing_artifacts: {}", |
| 3038 | + if missing_artifacts.is_empty() { |
| 3039 | + "none".to_string() |
| 3040 | + } else { |
| 3041 | + format!("`{}`", missing_artifacts.join("`, `")) |
| 3042 | + } |
| 3043 | + ), |
| 3044 | + format!( |
| 3045 | + "generic_artifacts: {}", |
| 3046 | + if generic_names.is_empty() { |
| 3047 | + "none".to_string() |
| 3048 | + } else { |
| 3049 | + format!("`{}`", generic_names.join("`, `")) |
| 1503 | 3050 | } |
| 3051 | + ), |
| 3052 | + format!( |
| 3053 | + "adapter_extras: {}", |
| 3054 | + format_adapter_extra_summary(&adapter_extras) |
| 3055 | + ), |
| 3056 | + ]; |
| 3057 | + if !observation.provenance.artifacts_captured.is_empty() { |
| 3058 | + lines.push(format!( |
| 3059 | + "captured_artifacts: `{}`", |
| 3060 | + observation.provenance.artifacts_captured.join("`, `") |
| 3061 | + )); |
| 3062 | + } |
| 3063 | + |
| 3064 | + if !generic_artifacts.is_empty() { |
| 3065 | + lines.push(String::new()); |
| 3066 | + lines.push("## Generic artifacts".to_string()); |
| 3067 | + for (artifact, value) in generic_artifacts { |
| 3068 | + lines.push(String::new()); |
| 3069 | + lines.push(format!("### `{}`", artifact)); |
| 3070 | + lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3071 | + lines.push("```text".to_string()); |
| 3072 | + lines.extend(render_artifact_body_lines(value, render_config)); |
| 3073 | + lines.push("```".to_string()); |
| 1504 | 3074 | } |
| 1505 | 3075 | } |
| 1506 | 3076 | |
| 1507 | | - cleanup_prepared_input(&prepared); |
| 1508 | | - cleanup_consistency_issues(&artifacts.consistency_issues); |
| 3077 | + if !adapter_extras.is_empty() { |
| 3078 | + lines.push(String::new()); |
| 3079 | + lines.push("## Adapter extras".to_string()); |
| 3080 | + for (namespace, entries) in adapter_extras { |
| 3081 | + lines.push(String::new()); |
| 3082 | + lines.push(format!("### `{}`", namespace)); |
| 3083 | + for (name, value) in entries { |
| 3084 | + lines.push(String::new()); |
| 3085 | + lines.push(format!("#### `{}`", name)); |
| 3086 | + lines.push(format!("summary: {}", artifact_value_summary(value))); |
| 3087 | + lines.push("```text".to_string()); |
| 3088 | + lines.extend(render_artifact_body_lines(value, render_config)); |
| 3089 | + lines.push("```".to_string()); |
| 3090 | + } |
| 3091 | + } |
| 3092 | + } |
| 1509 | 3093 | |
| 1510 | | - Ok(outcome) |
| 3094 | + lines.join("\n") + "\n" |
| 1511 | 3095 | } |
| 1512 | 3096 | |
| 1513 | | -fn prepare_case_input( |
| 1514 | | - case: &CaseSpec, |
| 1515 | | - suite: &SuiteSpec, |
| 1516 | | - opt_level: OptLevel, |
| 1517 | | -) -> Result<PreparedInput, String> { |
| 1518 | | - if case.graph_files.is_empty() { |
| 1519 | | - return Ok(PreparedInput { |
| 1520 | | - compiler_source: case.source.clone(), |
| 1521 | | - generated_source: None, |
| 1522 | | - temp_root: None, |
| 1523 | | - }); |
| 1524 | | - } |
| 1525 | | - |
| 1526 | | - let temp_root = default_report_root().join(".tmp").join(format!( |
| 1527 | | - "graph_{}_{}_{}", |
| 1528 | | - sanitize_component(&suite.name), |
| 1529 | | - sanitize_component(&case.name), |
| 1530 | | - next_report_suffix(opt_level) |
| 1531 | | - )); |
| 1532 | | - fs::create_dir_all(&temp_root).map_err(|e| { |
| 3097 | +fn render_observation_json(observation: &CompilerObservation) -> String { |
| 3098 | + let mut lines = vec![ |
| 3099 | + "{".to_string(), |
| 1533 | 3100 | format!( |
| 1534 | | - "cannot create graph temp dir '{}': {}", |
| 1535 | | - temp_root.display(), |
| 1536 | | - e |
| 1537 | | - ) |
| 1538 | | - })?; |
| 3101 | + " \"compiler\": \"{}\",", |
| 3102 | + json_escape(&observation.compiler.display_name()) |
| 3103 | + ), |
| 3104 | + format!( |
| 3105 | + " \"program\": \"{}\",", |
| 3106 | + json_escape(&observation.program.display().to_string()) |
| 3107 | + ), |
| 3108 | + format!(" \"opt\": \"{}\",", observation.opt_level.as_str()), |
| 3109 | + format!( |
| 3110 | + " \"compile_exit_code\": {},", |
| 3111 | + observation.compile_exit_code |
| 3112 | + ), |
| 3113 | + " \"provenance\": {".to_string(), |
| 3114 | + format!( |
| 3115 | + " \"compiler_identity\": \"{}\",", |
| 3116 | + json_escape(&observation.provenance.compiler_identity) |
| 3117 | + ), |
| 3118 | + format!( |
| 3119 | + " \"adapter_kind\": \"{}\",", |
| 3120 | + json_escape(&observation.provenance.adapter_kind) |
| 3121 | + ), |
| 3122 | + format!( |
| 3123 | + " \"backend_mode\": \"{}\",", |
| 3124 | + json_escape(&observation.provenance.backend_mode) |
| 3125 | + ), |
| 3126 | + format!( |
| 3127 | + " \"backend_detail\": \"{}\",", |
| 3128 | + json_escape(&observation.provenance.backend_detail) |
| 3129 | + ), |
| 3130 | + format!( |
| 3131 | + " \"artifacts_captured\": {},", |
| 3132 | + json_string_array(&observation.provenance.artifacts_captured) |
| 3133 | + ), |
| 3134 | + match &observation.provenance.failure_stage { |
| 3135 | + Some(stage) => format!(" \"failure_stage\": \"{}\",", json_escape(stage)), |
| 3136 | + None => " \"failure_stage\": null,".to_string(), |
| 3137 | + }, |
| 3138 | + match &observation.provenance.comparison_basis { |
| 3139 | + Some(basis) => format!(" \"comparison_basis\": \"{}\"", json_escape(basis)), |
| 3140 | + None => " \"comparison_basis\": null".to_string(), |
| 3141 | + }, |
| 3142 | + " },".to_string(), |
| 3143 | + " \"artifacts\": {".to_string(), |
| 3144 | + ]; |
| 3145 | + for (index, (artifact, value)) in observation.artifacts.iter().enumerate() { |
| 3146 | + lines.push(format!( |
| 3147 | + " \"{}\": {}{}", |
| 3148 | + json_escape(artifact.as_str()), |
| 3149 | + render_artifact_value_json(value), |
| 3150 | + if index + 1 == observation.artifacts.len() { |
| 3151 | + "" |
| 3152 | + } else { |
| 3153 | + "," |
| 3154 | + } |
| 3155 | + )); |
| 3156 | + } |
| 3157 | + lines.push(" }".to_string()); |
| 3158 | + lines.push("}".to_string()); |
| 3159 | + lines.join("\n") |
| 3160 | +} |
| 1539 | 3161 | |
| 1540 | | - let extension = case |
| 1541 | | - .source |
| 1542 | | - .extension() |
| 1543 | | - .and_then(|ext| ext.to_str()) |
| 1544 | | - .filter(|ext| !ext.is_empty()) |
| 1545 | | - .unwrap_or("f90"); |
| 1546 | | - let generated_source = temp_root.join(format!( |
| 1547 | | - "{}_graph.{}", |
| 1548 | | - sanitize_component(&case.name), |
| 1549 | | - extension |
| 1550 | | - )); |
| 3162 | +fn render_observation_markdown(observation: &CompilerObservation) -> String { |
| 3163 | + let mut lines = vec![ |
| 3164 | + format!("compiler: `{}`", observation.compiler.display_name()), |
| 3165 | + format!("program: `{}`", observation.program.display()), |
| 3166 | + format!("opt: `{}`", observation.opt_level.as_str()), |
| 3167 | + format!("compile_exit_code: `{}`", observation.compile_exit_code), |
| 3168 | + format!("adapter_kind: `{}`", observation.provenance.adapter_kind), |
| 3169 | + format!("backend_mode: `{}`", observation.provenance.backend_mode), |
| 3170 | + format!("backend_detail: {}", observation.provenance.backend_detail), |
| 3171 | + format!("failure_stage: `{}`", failure_stage_summary(observation)), |
| 3172 | + ]; |
| 3173 | + if !observation.provenance.artifacts_captured.is_empty() { |
| 3174 | + lines.push(format!( |
| 3175 | + "artifacts: `{}`", |
| 3176 | + observation.provenance.artifacts_captured.join("`, `") |
| 3177 | + )); |
| 3178 | + } |
| 3179 | + for (artifact, value) in &observation.artifacts { |
| 3180 | + lines.push(String::new()); |
| 3181 | + lines.push(format!("## `{}`", artifact.as_str())); |
| 3182 | + lines.push("```text".to_string()); |
| 3183 | + lines.extend( |
| 3184 | + render_artifact_value_text(value) |
| 3185 | + .lines() |
| 3186 | + .map(|line| line.to_string()), |
| 3187 | + ); |
| 3188 | + lines.push("```".to_string()); |
| 3189 | + } |
| 3190 | + lines.join("\n") |
| 3191 | +} |
| 1551 | 3192 | |
| 1552 | | - let mut combined = String::new(); |
| 1553 | | - for (index, file) in case.graph_files.iter().enumerate() { |
| 1554 | | - let text = fs::read_to_string(file) |
| 1555 | | - .map_err(|e| format!("cannot read graph file '{}': {}", file.display(), e))?; |
| 3193 | +fn render_differences_json(differences: &[ArtifactDifference]) -> String { |
| 3194 | + let mut rendered = String::from("["); |
| 3195 | + for (index, difference) in differences.iter().enumerate() { |
| 1556 | 3196 | if index > 0 { |
| 1557 | | - combined.push('\n'); |
| 3197 | + rendered.push_str(", "); |
| 1558 | 3198 | } |
| 1559 | | - combined.push_str(&text); |
| 1560 | | - if !text.ends_with('\n') { |
| 1561 | | - combined.push('\n'); |
| 3199 | + rendered.push_str(&format!( |
| 3200 | + "{{\"artifact\":\"{}\",\"detail\":\"{}\"}}", |
| 3201 | + json_escape(&difference.artifact), |
| 3202 | + json_escape(&difference.detail) |
| 3203 | + )); |
| 3204 | + } |
| 3205 | + rendered.push(']'); |
| 3206 | + rendered |
| 3207 | +} |
| 3208 | + |
| 3209 | +fn render_artifact_value_json(value: &ArtifactValue) -> String { |
| 3210 | + match value { |
| 3211 | + ArtifactValue::Text(text) => { |
| 3212 | + format!("{{\"kind\":\"text\",\"value\":\"{}\"}}", json_escape(text)) |
| 1562 | 3213 | } |
| 3214 | + ArtifactValue::Int(value) => format!("{{\"kind\":\"int\",\"value\":{}}}", value), |
| 3215 | + ArtifactValue::Run(run) => format!( |
| 3216 | + "{{\"kind\":\"runtime\",\"exit_code\":{},\"stdout\":\"{}\",\"stderr\":\"{}\"}}", |
| 3217 | + run.exit_code, |
| 3218 | + json_escape(&run.stdout), |
| 3219 | + json_escape(&run.stderr) |
| 3220 | + ), |
| 3221 | + ArtifactValue::Path(path) => format!( |
| 3222 | + "{{\"kind\":\"path\",\"value\":\"{}\"}}", |
| 3223 | + json_escape(&path.display().to_string()) |
| 3224 | + ), |
| 1563 | 3225 | } |
| 3226 | +} |
| 1564 | 3227 | |
| 1565 | | - fs::write(&generated_source, combined).map_err(|e| { |
| 1566 | | - format!( |
| 1567 | | - "cannot write generated graph input '{}': {}", |
| 1568 | | - generated_source.display(), |
| 1569 | | - e |
| 1570 | | - ) |
| 1571 | | - })?; |
| 3228 | +fn render_artifact_value_text(value: &ArtifactValue) -> String { |
| 3229 | + match value { |
| 3230 | + ArtifactValue::Text(text) => text.trim_end().to_string(), |
| 3231 | + ArtifactValue::Int(value) => value.to_string(), |
| 3232 | + ArtifactValue::Run(run) => format_run_capture(run), |
| 3233 | + ArtifactValue::Path(path) => path.display().to_string(), |
| 3234 | + } |
| 3235 | +} |
| 1572 | 3236 | |
| 1573 | | - Ok(PreparedInput { |
| 1574 | | - compiler_source: generated_source.clone(), |
| 1575 | | - generated_source: Some(generated_source), |
| 1576 | | - temp_root: Some(temp_root), |
| 1577 | | - }) |
| 1578 | | -} |
| 1579 | | - |
| 1580 | | -fn cleanup_prepared_input(prepared: &PreparedInput) { |
| 1581 | | - if let Some(temp_root) = &prepared.temp_root { |
| 1582 | | - let _ = fs::remove_dir_all(temp_root); |
| 1583 | | - } |
| 3237 | +fn default_suite_root() -> PathBuf { |
| 3238 | + Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3239 | + .join("..") |
| 3240 | + .join("suites") |
| 1584 | 3241 | } |
| 1585 | 3242 | |
| 1586 | | -fn status_for_opt(case: &CaseSpec, opt_level: OptLevel) -> EffectiveStatus { |
| 1587 | | - let mut status = EffectiveStatus::Normal; |
| 1588 | | - for rule in &case.status_rules { |
| 1589 | | - if rule.selector.matches(opt_level) { |
| 1590 | | - status = match rule.kind { |
| 1591 | | - StatusKind::Xfail => EffectiveStatus::Xfail(rule.reason.clone()), |
| 1592 | | - StatusKind::Future => EffectiveStatus::Future(rule.reason.clone()), |
| 1593 | | - }; |
| 1594 | | - } |
| 1595 | | - } |
| 1596 | | - status |
| 3243 | +fn default_report_root() -> PathBuf { |
| 3244 | + Path::new(env!("CARGO_MANIFEST_DIR")) |
| 3245 | + .join("..") |
| 3246 | + .join("reports") |
| 1597 | 3247 | } |
| 1598 | 3248 | |
| 1599 | | -fn ensure_target_stage(expectation: &Expectation, requested: &mut BTreeSet<Stage>) { |
| 1600 | | - match expectation { |
| 1601 | | - Expectation::CheckComments(target) |
| 1602 | | - | Expectation::Contains { target, .. } |
| 1603 | | - | Expectation::NotContains { target, .. } |
| 1604 | | - | Expectation::Equals { target, .. } |
| 1605 | | - | Expectation::IntEquals { target, .. } => match target { |
| 1606 | | - Target::Stage(stage) => { |
| 1607 | | - requested.insert(*stage); |
| 1608 | | - } |
| 1609 | | - Target::RunStdout | Target::RunStderr | Target::RunExitCode => { |
| 1610 | | - requested.insert(Stage::Run); |
| 1611 | | - } |
| 1612 | | - }, |
| 1613 | | - Expectation::FailContains { .. } | Expectation::FailEquals { .. } => {} |
| 1614 | | - } |
| 3249 | +fn workspace_root() -> PathBuf { |
| 3250 | + Path::new(env!("CARGO_MANIFEST_DIR")).join("..") |
| 1615 | 3251 | } |
| 1616 | 3252 | |
| 1617 | | -fn ensure_consistency_stage(check: ConsistencyCheck, requested: &mut BTreeSet<Stage>) { |
| 1618 | | - if let Some(stage) = check.required_stage() { |
| 1619 | | - requested.insert(stage); |
| 1620 | | - } |
| 1621 | | -} |
| 3253 | +fn doctor_report_fields(config: &DoctorConfig) -> Vec<(String, String)> { |
| 3254 | + let workspace_root = workspace_root(); |
| 3255 | + let suite_root = default_suite_root(); |
| 3256 | + let report_root = default_report_root(); |
| 3257 | + let armfortas = config.tools.armfortas_adapters(); |
| 3258 | + let observable_backend = config |
| 3259 | + .tools |
| 3260 | + .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3261 | + let capture_root = armfortas.capture_root(); |
| 3262 | + let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 1622 | 3263 | |
| 1623 | | -fn evaluate_positive_expectations(case: &CaseSpec, result: &CaptureResult) -> Result<(), String> { |
| 1624 | | - for expectation in &case.expectations { |
| 1625 | | - match expectation { |
| 1626 | | - Expectation::CheckComments(target) => { |
| 1627 | | - let text = target_text(result, target)?; |
| 1628 | | - let source = fs::read_to_string(&case.source) |
| 1629 | | - .map_err(|e| format!("cannot read '{}': {}", case.source.display(), e))?; |
| 1630 | | - let checks = extract_checks(&source); |
| 1631 | | - if checks.is_empty() { |
| 1632 | | - return Err(format!( |
| 1633 | | - "case '{}' requested check-comments but '{}' has no ! CHECK: lines", |
| 1634 | | - case.name, |
| 1635 | | - case.source.display() |
| 1636 | | - )); |
| 1637 | | - } |
| 1638 | | - match_checks(&checks, text, &case.name)?; |
| 1639 | | - } |
| 1640 | | - Expectation::Contains { target, needle } => { |
| 1641 | | - let text = target_text(result, target)?; |
| 1642 | | - if !text.contains(needle) { |
| 1643 | | - return Err(format!( |
| 1644 | | - "expected {} to contain {:?}\nactual:\n{}", |
| 1645 | | - target_name(*target), |
| 1646 | | - needle, |
| 1647 | | - text |
| 1648 | | - )); |
| 1649 | | - } |
| 1650 | | - } |
| 1651 | | - Expectation::NotContains { target, needle } => { |
| 1652 | | - let text = target_text(result, target)?; |
| 1653 | | - if text.contains(needle) { |
| 1654 | | - return Err(format!( |
| 1655 | | - "expected {} to not contain {:?}\nactual:\n{}", |
| 1656 | | - target_name(*target), |
| 1657 | | - needle, |
| 1658 | | - text |
| 1659 | | - )); |
| 1660 | | - } |
| 1661 | | - } |
| 1662 | | - Expectation::Equals { target, value } => { |
| 1663 | | - let text = target_text(result, target)?; |
| 1664 | | - if text.trim_end() != value { |
| 1665 | | - return Err(format!( |
| 1666 | | - "expected {} to equal {:?}\nactual:\n{}", |
| 1667 | | - target_name(*target), |
| 1668 | | - value, |
| 1669 | | - text |
| 1670 | | - )); |
| 1671 | | - } |
| 1672 | | - } |
| 1673 | | - Expectation::IntEquals { target, value } => { |
| 1674 | | - let actual = target_int(result, target)?; |
| 1675 | | - if actual != *value { |
| 1676 | | - return Err(format!( |
| 1677 | | - "expected {} to equal {}\nactual: {}", |
| 1678 | | - target_name(*target), |
| 1679 | | - value, |
| 1680 | | - actual |
| 1681 | | - )); |
| 1682 | | - } |
| 1683 | | - } |
| 1684 | | - Expectation::FailContains { .. } | Expectation::FailEquals { .. } => {} |
| 3264 | + let mut fields = vec![ |
| 3265 | + ("workspace_root".to_string(), display_path(&workspace_root)), |
| 3266 | + ("suite_root".to_string(), display_path(&suite_root)), |
| 3267 | + ("report_root".to_string(), display_path(&report_root)), |
| 3268 | + ( |
| 3269 | + "armfortas_cli_adapter".to_string(), |
| 3270 | + armfortas.cli_description().to_string(), |
| 3271 | + ), |
| 3272 | + ( |
| 3273 | + "armfortas_capture_adapter".to_string(), |
| 3274 | + armfortas.capture_description().to_string(), |
| 3275 | + ), |
| 3276 | + ( |
| 3277 | + "primary_backend_full".to_string(), |
| 3278 | + armfortas.capture_description().to_string(), |
| 3279 | + ), |
| 3280 | + ( |
| 3281 | + "primary_backend_observable".to_string(), |
| 3282 | + observable_backend.description().to_string(), |
| 3283 | + ), |
| 3284 | + ( |
| 3285 | + "armfortas_capture_root".to_string(), |
| 3286 | + capture_root |
| 3287 | + .as_ref() |
| 3288 | + .map(|root| display_path(root)) |
| 3289 | + .unwrap_or_else(|| "unavailable".to_string()), |
| 3290 | + ), |
| 3291 | + ( |
| 3292 | + "armfortas_capture_manifest".to_string(), |
| 3293 | + capture_manifest |
| 3294 | + .as_ref() |
| 3295 | + .map(|manifest| { |
| 3296 | + if manifest.exists() { |
| 3297 | + display_path(manifest) |
| 3298 | + } else { |
| 3299 | + "missing".to_string() |
| 3300 | + } |
| 3301 | + }) |
| 3302 | + .unwrap_or_else(|| "unavailable".to_string()), |
| 3303 | + ), |
| 3304 | + ( |
| 3305 | + "armfortas_cli_mode".to_string(), |
| 3306 | + armfortas.cli_mode_name().to_string(), |
| 3307 | + ), |
| 3308 | + ]; |
| 3309 | + match armfortas.cli() { |
| 3310 | + ArmfortasCliAdapter::Linked => { |
| 3311 | + fields.push(( |
| 3312 | + "armfortas_cli_status".to_string(), |
| 3313 | + capture_root |
| 3314 | + .as_ref() |
| 3315 | + .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3316 | + .unwrap_or_else(|| { |
| 3317 | + "linked adapter requested but unavailable in this build".to_string() |
| 3318 | + }), |
| 3319 | + )); |
| 1685 | 3320 | } |
| 1686 | | - } |
| 1687 | | - Ok(()) |
| 1688 | | -} |
| 1689 | | - |
| 1690 | | -fn evaluate_failure_expectations(case: &CaseSpec, failure: &CaptureFailure) -> Result<(), String> { |
| 1691 | | - let mut saw_failure_expectation = false; |
| 1692 | | - for expectation in &case.expectations { |
| 1693 | | - match expectation { |
| 1694 | | - Expectation::FailContains { stage, needle } => { |
| 1695 | | - saw_failure_expectation = true; |
| 1696 | | - if failure.stage != *stage { |
| 1697 | | - return Err(format!( |
| 1698 | | - "expected failure stage {} but armfortas failed in {}\n{}", |
| 1699 | | - stage.as_str(), |
| 1700 | | - failure.stage.as_str(), |
| 1701 | | - failure.detail |
| 1702 | | - )); |
| 1703 | | - } |
| 1704 | | - if !failure.detail.contains(needle) { |
| 1705 | | - return Err(format!( |
| 1706 | | - "expected failure detail at {} to contain {:?}\nactual:\n{}", |
| 1707 | | - stage.as_str(), |
| 1708 | | - needle, |
| 1709 | | - failure.detail |
| 1710 | | - )); |
| 1711 | | - } |
| 1712 | | - } |
| 1713 | | - Expectation::FailEquals { stage, value } => { |
| 1714 | | - saw_failure_expectation = true; |
| 1715 | | - if failure.stage != *stage { |
| 1716 | | - return Err(format!( |
| 1717 | | - "expected failure stage {} but armfortas failed in {}\n{}", |
| 1718 | | - stage.as_str(), |
| 1719 | | - failure.stage.as_str(), |
| 1720 | | - failure.detail |
| 1721 | | - )); |
| 1722 | | - } |
| 1723 | | - if failure.detail.trim_end() != value { |
| 1724 | | - return Err(format!( |
| 1725 | | - "expected failure detail at {} to equal {:?}\nactual:\n{}", |
| 1726 | | - stage.as_str(), |
| 1727 | | - value, |
| 1728 | | - failure.detail |
| 1729 | | - )); |
| 1730 | | - } |
| 1731 | | - } |
| 1732 | | - Expectation::CheckComments(_) |
| 1733 | | - | Expectation::Contains { .. } |
| 1734 | | - | Expectation::NotContains { .. } |
| 1735 | | - | Expectation::Equals { .. } |
| 1736 | | - | Expectation::IntEquals { .. } => {} |
| 3321 | + ArmfortasCliAdapter::External(binary) => { |
| 3322 | + fields.push(( |
| 3323 | + "armfortas_cli_status".to_string(), |
| 3324 | + tool_probe_status(binary, false), |
| 3325 | + )); |
| 1737 | 3326 | } |
| 1738 | 3327 | } |
| 1739 | | - |
| 1740 | | - if !saw_failure_expectation { |
| 1741 | | - return Err(format!( |
| 1742 | | - "armfortas failed in {} but the case did not declare an expect-fail rule\n{}", |
| 1743 | | - failure.stage.as_str(), |
| 1744 | | - failure.detail |
| 1745 | | - )); |
| 3328 | + fields.push(( |
| 3329 | + "armfortas_capture_mode".to_string(), |
| 3330 | + armfortas.capture_mode_name().to_string(), |
| 3331 | + )); |
| 3332 | + fields.push(( |
| 3333 | + "armfortas_capture_status".to_string(), |
| 3334 | + capture_root |
| 3335 | + .as_ref() |
| 3336 | + .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3337 | + .unwrap_or_else(|| { |
| 3338 | + "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh".to_string() |
| 3339 | + }), |
| 3340 | + )); |
| 3341 | + fields.push(( |
| 3342 | + "primary_backend_selection".to_string(), |
| 3343 | + "observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend".to_string(), |
| 3344 | + )); |
| 3345 | + for named in NamedCompiler::ALL { |
| 3346 | + append_named_compiler_fields(&mut fields, named, &config.tools, capture_root.as_ref()); |
| 1746 | 3347 | } |
| 3348 | + fields.push(( |
| 3349 | + "explicit_compiler_path".to_string(), |
| 3350 | + "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3351 | + .to_string(), |
| 3352 | + )); |
| 3353 | + let explicit_capabilities = compiler_capabilities( |
| 3354 | + &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3355 | + &config.tools, |
| 3356 | + ); |
| 3357 | + fields.push(( |
| 3358 | + "explicit_compiler_path.generic_artifacts".to_string(), |
| 3359 | + format_artifact_name_list(&explicit_capabilities.generic_artifacts()), |
| 3360 | + )); |
| 3361 | + fields.push(( |
| 3362 | + "explicit_compiler_path.adapter_extras".to_string(), |
| 3363 | + capability_extra_summary(&explicit_capabilities.adapter_extras()), |
| 3364 | + )); |
| 3365 | + fields.push(( |
| 3366 | + "gfortran".to_string(), |
| 3367 | + tool_probe_status(&config.tools.gfortran, false), |
| 3368 | + )); |
| 3369 | + fields.push(( |
| 3370 | + "flang-new".to_string(), |
| 3371 | + tool_probe_status(&config.tools.flang_new, false), |
| 3372 | + )); |
| 3373 | + fields.push(( |
| 3374 | + "lfortran".to_string(), |
| 3375 | + tool_probe_status(&config.tools.lfortran, false), |
| 3376 | + )); |
| 3377 | + fields.push(( |
| 3378 | + "ifort".to_string(), |
| 3379 | + tool_probe_status(&config.tools.ifort, false), |
| 3380 | + )); |
| 3381 | + fields.push(( |
| 3382 | + "ifx".to_string(), |
| 3383 | + tool_probe_status(&config.tools.ifx, false), |
| 3384 | + )); |
| 3385 | + fields.push(( |
| 3386 | + "nvfortran".to_string(), |
| 3387 | + tool_probe_status(&config.tools.nvfortran, false), |
| 3388 | + )); |
| 3389 | + fields.push(( |
| 3390 | + "as".to_string(), |
| 3391 | + tool_probe_status(&config.tools.system_as, false), |
| 3392 | + )); |
| 3393 | + fields.push(( |
| 3394 | + "otool".to_string(), |
| 3395 | + tool_probe_status(&config.tools.otool, false), |
| 3396 | + )); |
| 3397 | + fields.push(("nm".to_string(), tool_probe_status(&config.tools.nm, false))); |
| 3398 | + fields.push(( |
| 3399 | + "note".to_string(), |
| 3400 | + if capture_root.is_some() { |
| 3401 | + "linked capture still depends on the surrounding armfortas checkout".to_string() |
| 3402 | + } else { |
| 3403 | + "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work".to_string() |
| 3404 | + }, |
| 3405 | + )); |
| 3406 | + fields.push(( |
| 3407 | + if capture_root.is_some() { |
| 3408 | + "linked_mode_surface".to_string() |
| 3409 | + } else { |
| 3410 | + "external_only_surface".to_string() |
| 3411 | + }, |
| 3412 | + if capture_root.is_some() { |
| 3413 | + "rich armfortas stages, legacy frontend/module suites, capture consistency".to_string() |
| 3414 | + } else { |
| 3415 | + "compare, introspect, generic suite-v2, observable-only run cells".to_string() |
| 3416 | + }, |
| 3417 | + )); |
| 3418 | + fields.push(( |
| 3419 | + if capture_root.is_some() { |
| 3420 | + "external_only_limits".to_string() |
| 3421 | + } else { |
| 3422 | + "linked_only_surface".to_string() |
| 3423 | + }, |
| 3424 | + if capture_root.is_some() { |
| 3425 | + "none in this build".to_string() |
| 3426 | + } else { |
| 3427 | + "armfortas.* extras, legacy frontend/module suites, capture consistency".to_string() |
| 3428 | + }, |
| 3429 | + )); |
| 1747 | 3430 | |
| 1748 | | - Ok(()) |
| 3431 | + fields |
| 1749 | 3432 | } |
| 1750 | 3433 | |
| 1751 | | -fn has_failure_expectation(case: &CaseSpec) -> bool { |
| 1752 | | - case.expectations.iter().any(|expectation| { |
| 1753 | | - matches!( |
| 1754 | | - expectation, |
| 1755 | | - Expectation::FailContains { .. } | Expectation::FailEquals { .. } |
| 1756 | | - ) |
| 1757 | | - }) |
| 3434 | +fn render_doctor_report(config: &DoctorConfig) -> String { |
| 3435 | + let mut lines = vec!["Doctor".to_string()]; |
| 3436 | + for (field, value) in doctor_report_fields(config) { |
| 3437 | + lines.push(format!(" {}: {}", field, value)); |
| 3438 | + } |
| 3439 | + lines.join("\n") |
| 1758 | 3440 | } |
| 1759 | 3441 | |
| 1760 | | -fn expected_failure_description(case: &CaseSpec) -> String { |
| 1761 | | - let mut items = Vec::new(); |
| 1762 | | - for expectation in &case.expectations { |
| 1763 | | - match expectation { |
| 1764 | | - Expectation::FailContains { stage, needle } => { |
| 1765 | | - items.push(format!("{} contains {:?}", stage.as_str(), needle)); |
| 1766 | | - } |
| 1767 | | - Expectation::FailEquals { stage, value } => { |
| 1768 | | - items.push(format!("{} equals {:?}", stage.as_str(), value)); |
| 1769 | | - } |
| 1770 | | - _ => {} |
| 1771 | | - } |
| 1772 | | - } |
| 1773 | | - if items.is_empty() { |
| 1774 | | - "declared failure".to_string() |
| 1775 | | - } else { |
| 1776 | | - items.join(", ") |
| 1777 | | - } |
| 3442 | +fn render_doctor_capabilities_json(capabilities: &CompilerCapabilities) -> String { |
| 3443 | + let unavailable = capabilities |
| 3444 | + .unavailable_artifacts |
| 3445 | + .iter() |
| 3446 | + .map(|(artifact, reason)| { |
| 3447 | + format!( |
| 3448 | + "{{\"artifact\":\"{}\",\"reason\":\"{}\"}}", |
| 3449 | + json_escape(artifact.as_str()), |
| 3450 | + json_escape(reason) |
| 3451 | + ) |
| 3452 | + }) |
| 3453 | + .collect::<Vec<_>>() |
| 3454 | + .join(", "); |
| 3455 | + format!( |
| 3456 | + "{{\"generic_artifacts\":{},\"adapter_extras\":{},\"unavailable_artifacts\":[{}]}}", |
| 3457 | + json_string_iter( |
| 3458 | + capabilities |
| 3459 | + .generic_artifacts() |
| 3460 | + .iter() |
| 3461 | + .map(|artifact| artifact.as_str()) |
| 3462 | + ), |
| 3463 | + json_string_vec_map(&capabilities.adapter_extras()), |
| 3464 | + unavailable |
| 3465 | + ) |
| 1778 | 3466 | } |
| 1779 | 3467 | |
| 1780 | | -fn target_text<'a>(result: &'a CaptureResult, target: &Target) -> Result<&'a str, String> { |
| 1781 | | - match target { |
| 1782 | | - Target::Stage(stage) => match result.get(*stage) { |
| 1783 | | - Some(CapturedStage::Text(text)) => Ok(text), |
| 1784 | | - Some(CapturedStage::Run(_)) => { |
| 1785 | | - Err(format!("stage '{}' is not textual", stage.as_str())) |
| 1786 | | - } |
| 1787 | | - None => Err(format!("missing captured stage '{}'", stage.as_str())), |
| 1788 | | - }, |
| 1789 | | - Target::RunStdout => match result.get(Stage::Run).and_then(CapturedStage::as_run) { |
| 1790 | | - Some(run) => Ok(&run.stdout), |
| 1791 | | - None => Err("missing captured run stage".into()), |
| 3468 | +fn render_tool_probe_json(probe: &ToolProbe) -> String { |
| 3469 | + format!( |
| 3470 | + "{{\"configured\":\"{}\",\"status\":\"{}\",\"resolved_path\":{},\"banner\":{},\"detail\":{}}}", |
| 3471 | + json_escape(&probe.configured), |
| 3472 | + json_escape(&probe.status), |
| 3473 | + match probe.resolved_path.as_ref() { |
| 3474 | + Some(path) => format!("\"{}\"", json_escape(&display_path(path))), |
| 3475 | + None => "null".to_string(), |
| 1792 | 3476 | }, |
| 1793 | | - Target::RunStderr => match result.get(Stage::Run).and_then(CapturedStage::as_run) { |
| 1794 | | - Some(run) => Ok(&run.stderr), |
| 1795 | | - None => Err("missing captured run stage".into()), |
| 3477 | + match probe.banner.as_ref() { |
| 3478 | + Some(banner) => format!("\"{}\"", json_escape(banner)), |
| 3479 | + None => "null".to_string(), |
| 1796 | 3480 | }, |
| 1797 | | - Target::RunExitCode => { |
| 1798 | | - Err("run.exit_code is numeric; use 'expect run.exit_code equals <int>'".into()) |
| 1799 | | - } |
| 1800 | | - } |
| 1801 | | -} |
| 1802 | | - |
| 1803 | | -fn target_int(result: &CaptureResult, target: &Target) -> Result<i32, String> { |
| 1804 | | - match target { |
| 1805 | | - Target::RunExitCode => match result.get(Stage::Run).and_then(CapturedStage::as_run) { |
| 1806 | | - Some(run) => Ok(run.exit_code), |
| 1807 | | - None => Err("missing captured run stage".into()), |
| 3481 | + match probe.detail.as_ref() { |
| 3482 | + Some(detail) => format!("\"{}\"", json_escape(detail)), |
| 3483 | + None => "null".to_string(), |
| 1808 | 3484 | }, |
| 1809 | | - _ => Err(format!( |
| 1810 | | - "{} is textual; use a string matcher instead", |
| 1811 | | - target_name(*target) |
| 1812 | | - )), |
| 1813 | | - } |
| 3485 | + ) |
| 1814 | 3486 | } |
| 1815 | 3487 | |
| 1816 | | -fn target_name(target: Target) -> &'static str { |
| 1817 | | - match target { |
| 1818 | | - Target::Stage(stage) => stage.as_str(), |
| 1819 | | - Target::RunStdout => "run.stdout", |
| 1820 | | - Target::RunStderr => "run.stderr", |
| 1821 | | - Target::RunExitCode => "run.exit_code", |
| 3488 | +fn json_string_vec_map(map: &BTreeMap<String, Vec<String>>) -> String { |
| 3489 | + let mut rendered = String::from("{"); |
| 3490 | + for (index, (key, values)) in map.iter().enumerate() { |
| 3491 | + if index > 0 { |
| 3492 | + rendered.push_str(", "); |
| 3493 | + } |
| 3494 | + rendered.push('"'); |
| 3495 | + rendered.push_str(&json_escape(key)); |
| 3496 | + rendered.push_str("\": "); |
| 3497 | + rendered.push_str(&json_string_iter(values.iter().map(|value| value.as_str()))); |
| 1822 | 3498 | } |
| 3499 | + rendered.push('}'); |
| 3500 | + rendered |
| 1823 | 3501 | } |
| 1824 | 3502 | |
| 1825 | | -fn compare_differential( |
| 1826 | | - result: &CaptureResult, |
| 1827 | | - references: &[ReferenceResult], |
| 1828 | | -) -> Result<(), String> { |
| 1829 | | - let arm_run = result |
| 1830 | | - .get(Stage::Run) |
| 1831 | | - .and_then(CapturedStage::as_run) |
| 1832 | | - .ok_or("differential comparison requires the run stage")?; |
| 1833 | | - let arm_sig = normalize_run_signature(arm_run); |
| 1834 | | - |
| 1835 | | - let mut reference_sigs = BTreeSet::new(); |
| 1836 | | - let mut matching_refs = 0usize; |
| 1837 | | - let mut detail = Vec::new(); |
| 1838 | | - |
| 1839 | | - for reference in references { |
| 1840 | | - if reference.compile_exit_code != 0 { |
| 1841 | | - return Err(format!( |
| 1842 | | - "reference compiler '{}' failed to compile\n{}", |
| 1843 | | - reference.compiler.as_str(), |
| 1844 | | - format_reference_result(reference) |
| 1845 | | - )); |
| 1846 | | - } |
| 1847 | | - |
| 1848 | | - if let Some(run_error) = &reference.run_error { |
| 1849 | | - return Err(format!( |
| 1850 | | - "reference compiler '{}' built but could not run: {}\n{}", |
| 1851 | | - reference.compiler.as_str(), |
| 1852 | | - run_error, |
| 1853 | | - format_reference_result(reference) |
| 1854 | | - )); |
| 1855 | | - } |
| 1856 | | - |
| 1857 | | - let signature = reference.run_signature().ok_or_else(|| { |
| 3503 | +fn render_named_compiler_entry_json( |
| 3504 | + named: NamedCompiler, |
| 3505 | + tools: &ToolchainConfig, |
| 3506 | + capture_root: Option<&PathBuf>, |
| 3507 | +) -> String { |
| 3508 | + let capabilities = compiler_capabilities(&CompilerSpec::Named(named), tools); |
| 3509 | + let probe = named_compiler_probe(named, tools, capture_root); |
| 3510 | + let mut fields = vec![ |
| 3511 | + format!( |
| 3512 | + "\"accepted_names\": {}", |
| 3513 | + json_string_iter(named.accepted_names().iter().copied()) |
| 3514 | + ), |
| 3515 | + format!( |
| 3516 | + "\"candidate_binaries\": {}", |
| 3517 | + json_string_iter(named.candidate_binaries().iter().copied()) |
| 3518 | + ), |
| 3519 | + format!( |
| 3520 | + "\"capabilities\": {}", |
| 3521 | + render_doctor_capabilities_json(&capabilities) |
| 3522 | + ), |
| 3523 | + format!("\"probe\": {}", render_tool_probe_json(&probe)), |
| 3524 | + ]; |
| 3525 | + if named == NamedCompiler::Armfortas { |
| 3526 | + let armfortas = tools.armfortas_adapters(); |
| 3527 | + fields.insert( |
| 3528 | + 0, |
| 3529 | + format!( |
| 3530 | + "\"surface\": \"{}\"", |
| 3531 | + json_escape(&format!( |
| 3532 | + "cli={} capture={}", |
| 3533 | + armfortas.cli_mode_name(), |
| 3534 | + armfortas.capture_mode_name() |
| 3535 | + )) |
| 3536 | + ), |
| 3537 | + ); |
| 3538 | + } else { |
| 3539 | + fields.insert( |
| 3540 | + 0, |
| 3541 | + format!( |
| 3542 | + "\"status\": \"{}\"", |
| 3543 | + json_escape(&named_compiler_status_value(named, tools, capture_root)) |
| 3544 | + ), |
| 3545 | + ); |
| 3546 | + } |
| 3547 | + format!("{{{}}}", fields.join(", ")) |
| 3548 | +} |
| 3549 | + |
| 3550 | +fn render_doctor_json(config: &DoctorConfig) -> String { |
| 3551 | + let fields = doctor_report_fields(config); |
| 3552 | + let workspace_root = workspace_root(); |
| 3553 | + let suite_root = default_suite_root(); |
| 3554 | + let report_root = default_report_root(); |
| 3555 | + let armfortas = config.tools.armfortas_adapters(); |
| 3556 | + let observable_backend = config |
| 3557 | + .tools |
| 3558 | + .cli_observable_capture_backend(report_root.join(".tmp").join("doctor")); |
| 3559 | + let capture_root = armfortas.capture_root(); |
| 3560 | + let capture_manifest = capture_root.as_ref().map(|root| root.join("Cargo.toml")); |
| 3561 | + let explicit_capabilities = compiler_capabilities( |
| 3562 | + &CompilerSpec::Binary(PathBuf::from("/path/to/compiler")), |
| 3563 | + &config.tools, |
| 3564 | + ); |
| 3565 | + let named_entries = NamedCompiler::ALL |
| 3566 | + .iter() |
| 3567 | + .enumerate() |
| 3568 | + .map(|(index, named)| { |
| 1858 | 3569 | format!( |
| 1859 | | - "reference compiler '{}' did not produce a run result", |
| 1860 | | - reference.compiler.as_str() |
| 3570 | + " \"{}\": {}{}", |
| 3571 | + named.as_str(), |
| 3572 | + render_named_compiler_entry_json(*named, &config.tools, capture_root.as_ref()), |
| 3573 | + if index + 1 == NamedCompiler::ALL.len() { |
| 3574 | + "" |
| 3575 | + } else { |
| 3576 | + "," |
| 3577 | + } |
| 1861 | 3578 | ) |
| 1862 | | - })?; |
| 3579 | + }) |
| 3580 | + .collect::<Vec<_>>(); |
| 3581 | + let mut lines = vec![ |
| 3582 | + "{".to_string(), |
| 3583 | + " \"command\": \"doctor\",".to_string(), |
| 3584 | + " \"workspace\": {".to_string(), |
| 3585 | + format!( |
| 3586 | + " \"workspace_root\": \"{}\",", |
| 3587 | + json_escape(&display_path(&workspace_root)) |
| 3588 | + ), |
| 3589 | + format!( |
| 3590 | + " \"suite_root\": \"{}\",", |
| 3591 | + json_escape(&display_path(&suite_root)) |
| 3592 | + ), |
| 3593 | + format!( |
| 3594 | + " \"report_root\": \"{}\"", |
| 3595 | + json_escape(&display_path(&report_root)) |
| 3596 | + ), |
| 3597 | + " },".to_string(), |
| 3598 | + " \"armfortas\": {".to_string(), |
| 3599 | + format!( |
| 3600 | + " \"cli_adapter\": \"{}\",", |
| 3601 | + json_escape(armfortas.cli_description()) |
| 3602 | + ), |
| 3603 | + format!( |
| 3604 | + " \"capture_adapter\": \"{}\",", |
| 3605 | + json_escape(armfortas.capture_description()) |
| 3606 | + ), |
| 3607 | + format!( |
| 3608 | + " \"cli_mode\": \"{}\",", |
| 3609 | + json_escape(armfortas.cli_mode_name()) |
| 3610 | + ), |
| 3611 | + format!( |
| 3612 | + " \"cli_status\": \"{}\",", |
| 3613 | + json_escape( |
| 3614 | + &match armfortas.cli() { |
| 3615 | + ArmfortasCliAdapter::Linked => capture_root |
| 3616 | + .as_ref() |
| 3617 | + .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3618 | + .unwrap_or_else(|| { |
| 3619 | + "linked adapter requested but unavailable in this build".to_string() |
| 3620 | + }), |
| 3621 | + ArmfortasCliAdapter::External(binary) => tool_probe_status(binary, false), |
| 3622 | + } |
| 3623 | + ) |
| 3624 | + ), |
| 3625 | + format!( |
| 3626 | + " \"capture_mode\": \"{}\",", |
| 3627 | + json_escape(armfortas.capture_mode_name()) |
| 3628 | + ), |
| 3629 | + format!( |
| 3630 | + " \"capture_status\": \"{}\",", |
| 3631 | + json_escape( |
| 3632 | + &capture_root |
| 3633 | + .as_ref() |
| 3634 | + .map(|root| format!("linked via Cargo to {}", display_path(root))) |
| 3635 | + .unwrap_or_else(|| { |
| 3636 | + "unavailable in this build; use scripts/bootstrap-linked-armfortas.sh" |
| 3637 | + .to_string() |
| 3638 | + }) |
| 3639 | + ) |
| 3640 | + ), |
| 3641 | + format!( |
| 3642 | + " \"capture_root\": {},", |
| 3643 | + match capture_root.as_ref() { |
| 3644 | + Some(root) => format!("\"{}\"", json_escape(&display_path(root))), |
| 3645 | + None => "null".to_string(), |
| 3646 | + } |
| 3647 | + ), |
| 3648 | + format!( |
| 3649 | + " \"capture_manifest\": \"{}\"", |
| 3650 | + json_escape( |
| 3651 | + &capture_manifest |
| 3652 | + .as_ref() |
| 3653 | + .map(|manifest| { |
| 3654 | + if manifest.exists() { |
| 3655 | + display_path(manifest) |
| 3656 | + } else { |
| 3657 | + "missing".to_string() |
| 3658 | + } |
| 3659 | + }) |
| 3660 | + .unwrap_or_else(|| "unavailable".to_string()) |
| 3661 | + ) |
| 3662 | + ), |
| 3663 | + " },".to_string(), |
| 3664 | + " \"primary_backends\": {".to_string(), |
| 3665 | + format!( |
| 3666 | + " \"full\": \"{}\",", |
| 3667 | + json_escape(armfortas.capture_description()) |
| 3668 | + ), |
| 3669 | + format!( |
| 3670 | + " \"observable\": \"{}\",", |
| 3671 | + json_escape(observable_backend.description()) |
| 3672 | + ), |
| 3673 | + format!( |
| 3674 | + " \"selection\": \"{}\"", |
| 3675 | + json_escape("observable backend is selected for asm/obj/run-only cells when the armfortas CLI is external and the case does not require expect-fail or capture-consistency semantics; otherwise full backend") |
| 3676 | + ), |
| 3677 | + " },".to_string(), |
| 3678 | + " \"named_compilers\": {".to_string(), |
| 3679 | + ]; |
| 3680 | + lines.extend(named_entries); |
| 3681 | + lines.extend([ |
| 3682 | + " },".to_string(), |
| 3683 | + format!( |
| 3684 | + " \"explicit_compiler_path\": {{\"description\":\"{}\",\"capabilities\":{}}},", |
| 3685 | + json_escape( |
| 3686 | + "any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 3687 | + ), |
| 3688 | + render_doctor_capabilities_json(&explicit_capabilities) |
| 3689 | + ), |
| 3690 | + " \"tools\": {".to_string(), |
| 3691 | + format!( |
| 3692 | + " \"gfortran\": \"{}\",", |
| 3693 | + json_escape(&tool_probe_status(&config.tools.gfortran, false)) |
| 3694 | + ), |
| 3695 | + format!( |
| 3696 | + " \"flang-new\": \"{}\",", |
| 3697 | + json_escape(&tool_probe_status(&config.tools.flang_new, false)) |
| 3698 | + ), |
| 3699 | + format!( |
| 3700 | + " \"lfortran\": \"{}\",", |
| 3701 | + json_escape(&tool_probe_status(&config.tools.lfortran, false)) |
| 3702 | + ), |
| 3703 | + format!( |
| 3704 | + " \"ifort\": \"{}\",", |
| 3705 | + json_escape(&tool_probe_status(&config.tools.ifort, false)) |
| 3706 | + ), |
| 3707 | + format!( |
| 3708 | + " \"ifx\": \"{}\",", |
| 3709 | + json_escape(&tool_probe_status(&config.tools.ifx, false)) |
| 3710 | + ), |
| 3711 | + format!( |
| 3712 | + " \"nvfortran\": \"{}\",", |
| 3713 | + json_escape(&tool_probe_status(&config.tools.nvfortran, false)) |
| 3714 | + ), |
| 3715 | + format!( |
| 3716 | + " \"as\": \"{}\",", |
| 3717 | + json_escape(&tool_probe_status(&config.tools.system_as, false)) |
| 3718 | + ), |
| 3719 | + format!( |
| 3720 | + " \"otool\": \"{}\",", |
| 3721 | + json_escape(&tool_probe_status(&config.tools.otool, false)) |
| 3722 | + ), |
| 3723 | + format!( |
| 3724 | + " \"nm\": \"{}\"", |
| 3725 | + json_escape(&tool_probe_status(&config.tools.nm, false)) |
| 3726 | + ), |
| 3727 | + " },".to_string(), |
| 3728 | + " \"mode\": {".to_string(), |
| 3729 | + format!( |
| 3730 | + " \"note\": \"{}\",", |
| 3731 | + json_escape( |
| 3732 | + if capture_root.is_some() { |
| 3733 | + "linked capture still depends on the surrounding armfortas checkout" |
| 3734 | + } else { |
| 3735 | + "linked capture is unavailable in this build; external compiler compare/introspect surfaces still work" |
| 3736 | + } |
| 3737 | + ) |
| 3738 | + ), |
| 3739 | + format!( |
| 3740 | + " \"surface_key\": \"{}\",", |
| 3741 | + if capture_root.is_some() { |
| 3742 | + "linked_mode_surface" |
| 3743 | + } else { |
| 3744 | + "external_only_surface" |
| 3745 | + } |
| 3746 | + ), |
| 3747 | + format!( |
| 3748 | + " \"surface_value\": \"{}\",", |
| 3749 | + json_escape( |
| 3750 | + if capture_root.is_some() { |
| 3751 | + "rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 3752 | + } else { |
| 3753 | + "compare, introspect, generic suite-v2, observable-only run cells" |
| 3754 | + } |
| 3755 | + ) |
| 3756 | + ), |
| 3757 | + format!( |
| 3758 | + " \"limits_key\": \"{}\",", |
| 3759 | + if capture_root.is_some() { |
| 3760 | + "external_only_limits" |
| 3761 | + } else { |
| 3762 | + "linked_only_surface" |
| 3763 | + } |
| 3764 | + ), |
| 3765 | + format!( |
| 3766 | + " \"limits_value\": \"{}\"", |
| 3767 | + json_escape( |
| 3768 | + if capture_root.is_some() { |
| 3769 | + "none in this build" |
| 3770 | + } else { |
| 3771 | + "armfortas.* extras, legacy frontend/module suites, capture consistency" |
| 3772 | + } |
| 3773 | + ) |
| 3774 | + ), |
| 3775 | + " },".to_string(), |
| 3776 | + " \"fields\": {".to_string(), |
| 3777 | + ]); |
| 3778 | + for (index, (field, value)) in fields.iter().enumerate() { |
| 3779 | + lines.push(format!( |
| 3780 | + " \"{}\": \"{}\"{}", |
| 3781 | + json_escape(field), |
| 3782 | + json_escape(value), |
| 3783 | + if index + 1 == fields.len() { "" } else { "," } |
| 3784 | + )); |
| 3785 | + } |
| 3786 | + lines.push(" }".to_string()); |
| 3787 | + lines.push("}".to_string()); |
| 3788 | + lines.join("\n") + "\n" |
| 3789 | +} |
| 1863 | 3790 | |
| 1864 | | - if signature == arm_sig { |
| 1865 | | - matching_refs += 1; |
| 1866 | | - } else { |
| 1867 | | - detail.push(format_reference_result(reference)); |
| 1868 | | - } |
| 1869 | | - reference_sigs.insert(signature); |
| 3791 | +fn render_doctor_markdown(config: &DoctorConfig) -> String { |
| 3792 | + let mut lines = vec![ |
| 3793 | + "# bencch doctor report".to_string(), |
| 3794 | + String::new(), |
| 3795 | + "| field | value |".to_string(), |
| 3796 | + "| --- | --- |".to_string(), |
| 3797 | + ]; |
| 3798 | + for (field, value) in doctor_report_fields(config) { |
| 3799 | + lines.push(format!( |
| 3800 | + "| `{}` | {} |", |
| 3801 | + field, |
| 3802 | + doctor_markdown_cell(&value) |
| 3803 | + )); |
| 1870 | 3804 | } |
| 3805 | + lines.join("\n") + "\n" |
| 3806 | +} |
| 1871 | 3807 | |
| 1872 | | - if matching_refs == references.len() { |
| 1873 | | - return Ok(()); |
| 3808 | +fn write_doctor_reports(config: &DoctorConfig) -> Result<(), String> { |
| 3809 | + if let Some(path) = &config.json_report { |
| 3810 | + write_report(path, &render_doctor_json(config), "json report")?; |
| 3811 | + println!("json report: {}", path.display()); |
| 1874 | 3812 | } |
| 3813 | + if let Some(path) = &config.markdown_report { |
| 3814 | + write_report(path, &render_doctor_markdown(config), "markdown report")?; |
| 3815 | + println!("markdown report: {}", path.display()); |
| 3816 | + } |
| 3817 | + Ok(()) |
| 3818 | +} |
| 1875 | 3819 | |
| 1876 | | - let classification = if matching_refs == 0 && reference_sigs.len() == 1 { |
| 1877 | | - "classification: armfortas-only divergence" |
| 1878 | | - } else if reference_sigs.len() > 1 { |
| 1879 | | - "classification: reference disagreement" |
| 3820 | +fn doctor_markdown_cell(value: &str) -> String { |
| 3821 | + value.replace('|', "\\|").replace('\n', "<br>") |
| 3822 | +} |
| 3823 | + |
| 3824 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 3825 | +struct ToolProbe { |
| 3826 | + configured: String, |
| 3827 | + resolved_path: Option<PathBuf>, |
| 3828 | + status: String, |
| 3829 | + banner: Option<String>, |
| 3830 | + detail: Option<String>, |
| 3831 | +} |
| 3832 | + |
| 3833 | +fn tool_probe(configured: &str, already_resolved_path: bool) -> ToolProbe { |
| 3834 | + let resolved_path = if already_resolved_path { |
| 3835 | + let path = PathBuf::from(configured); |
| 3836 | + if path.exists() { |
| 3837 | + Some(path) |
| 3838 | + } else { |
| 3839 | + None |
| 3840 | + } |
| 1880 | 3841 | } else { |
| 1881 | | - "classification: partial disagreement" |
| 3842 | + resolve_tool_path(configured) |
| 1882 | 3843 | }; |
| 1883 | 3844 | |
| 1884 | | - Err(format!( |
| 1885 | | - "behavior mismatch against reference compilers\n{}\n\narmfortas\n{}\n\n{}", |
| 1886 | | - classification, |
| 1887 | | - format_run_capture(arm_run), |
| 1888 | | - detail.join("\n\n") |
| 1889 | | - )) |
| 3845 | + match resolved_path { |
| 3846 | + Some(path) => match probe_tool_banner(&path) { |
| 3847 | + Ok((arg, banner)) => ToolProbe { |
| 3848 | + configured: configured.to_string(), |
| 3849 | + resolved_path: Some(path), |
| 3850 | + status: "invokable".into(), |
| 3851 | + banner: Some(banner), |
| 3852 | + detail: Some(format!("probe succeeded with {}", arg)), |
| 3853 | + }, |
| 3854 | + Err(detail) => ToolProbe { |
| 3855 | + configured: configured.to_string(), |
| 3856 | + resolved_path: Some(path), |
| 3857 | + status: "resolved".into(), |
| 3858 | + banner: None, |
| 3859 | + detail: Some(detail), |
| 3860 | + }, |
| 3861 | + }, |
| 3862 | + None => ToolProbe { |
| 3863 | + configured: configured.to_string(), |
| 3864 | + resolved_path: None, |
| 3865 | + status: "missing".into(), |
| 3866 | + banner: None, |
| 3867 | + detail: Some("binary not found on disk or PATH".into()), |
| 3868 | + }, |
| 3869 | + } |
| 1890 | 3870 | } |
| 1891 | 3871 | |
| 1892 | | -fn compose_armfortas_failure_detail(artifacts: &ExecutionArtifacts) -> String { |
| 1893 | | - let mut detail = String::new(); |
| 1894 | | - if let Some(failure) = &artifacts.armfortas_failure { |
| 1895 | | - detail.push_str(&format!( |
| 1896 | | - "armfortas failed in {}\n{}", |
| 1897 | | - failure.stage.as_str(), |
| 1898 | | - failure.detail |
| 1899 | | - )); |
| 1900 | | - } else { |
| 1901 | | - detail.push_str("armfortas failed without an error message"); |
| 3872 | +fn linked_tool_probe(capture_root: Option<&PathBuf>) -> ToolProbe { |
| 3873 | + match capture_root { |
| 3874 | + Some(root) => ToolProbe { |
| 3875 | + configured: "linked".into(), |
| 3876 | + resolved_path: Some(root.clone()), |
| 3877 | + status: "linked".into(), |
| 3878 | + banner: None, |
| 3879 | + detail: Some(format!("linked via Cargo to {}", display_path(root))), |
| 3880 | + }, |
| 3881 | + None => ToolProbe { |
| 3882 | + configured: "linked".into(), |
| 3883 | + resolved_path: None, |
| 3884 | + status: "unavailable".into(), |
| 3885 | + banner: None, |
| 3886 | + detail: Some("linked adapter requested but unavailable in this build".into()), |
| 3887 | + }, |
| 1902 | 3888 | } |
| 3889 | +} |
| 1903 | 3890 | |
| 1904 | | - if !artifacts.references.is_empty() { |
| 1905 | | - detail.push_str("\n\nreference compilers\n"); |
| 1906 | | - detail.push_str(&format_reference_summary(&artifacts.references)); |
| 3891 | +fn named_compiler_probe( |
| 3892 | + named: NamedCompiler, |
| 3893 | + tools: &ToolchainConfig, |
| 3894 | + capture_root: Option<&PathBuf>, |
| 3895 | +) -> ToolProbe { |
| 3896 | + match named { |
| 3897 | + NamedCompiler::Armfortas => match &tools.armfortas { |
| 3898 | + ArmfortasCliAdapter::Linked => linked_tool_probe(capture_root), |
| 3899 | + ArmfortasCliAdapter::External(binary) => tool_probe(binary, false), |
| 3900 | + }, |
| 3901 | + _ => tool_probe( |
| 3902 | + &tools |
| 3903 | + .named_compiler_binary(named) |
| 3904 | + .unwrap_or_else(|| named.as_str().to_string()), |
| 3905 | + false, |
| 3906 | + ), |
| 1907 | 3907 | } |
| 1908 | | - |
| 1909 | | - detail |
| 1910 | 3908 | } |
| 1911 | 3909 | |
| 1912 | | -fn run_consistency_checks( |
| 1913 | | - case: &CaseSpec, |
| 1914 | | - prepared: &PreparedInput, |
| 1915 | | - opt_level: OptLevel, |
| 1916 | | - capture_result: &CaptureResult, |
| 3910 | +fn compiler_spec_probe( |
| 3911 | + spec: &CompilerSpec, |
| 1917 | 3912 | tools: &ToolchainConfig, |
| 1918 | | -) -> Vec<ConsistencyIssue> { |
| 1919 | | - let mut failures = Vec::new(); |
| 1920 | | - for check in &case.consistency_checks { |
| 1921 | | - let issue = match check { |
| 1922 | | - ConsistencyCheck::CliObjVsSystemAs => { |
| 1923 | | - run_cli_obj_vs_system_as(&prepared.compiler_source, opt_level, tools) |
| 3913 | + capture_root: Option<&PathBuf>, |
| 3914 | +) -> ToolProbe { |
| 3915 | + match spec { |
| 3916 | + CompilerSpec::Named(named) => named_compiler_probe(*named, tools, capture_root), |
| 3917 | + CompilerSpec::Binary(path) => tool_probe(&path.display().to_string(), true), |
| 3918 | + } |
| 3919 | +} |
| 3920 | + |
| 3921 | +fn probe_tool_banner(path: &Path) -> Result<(String, String), String> { |
| 3922 | + let mut last_detail = None; |
| 3923 | + for arg in ["--version", "-V", "-v"] { |
| 3924 | + match Command::new(path).arg(arg).output() { |
| 3925 | + Ok(output) => { |
| 3926 | + let stdout = String::from_utf8_lossy(&output.stdout); |
| 3927 | + let stderr = String::from_utf8_lossy(&output.stderr); |
| 3928 | + let combined = stdout |
| 3929 | + .lines() |
| 3930 | + .chain(stderr.lines()) |
| 3931 | + .map(str::trim) |
| 3932 | + .find(|line| !line.is_empty()) |
| 3933 | + .map(|line| compact_probe_banner(line)); |
| 3934 | + if let Some(line) = combined { |
| 3935 | + return Ok((arg.to_string(), line)); |
| 3936 | + } |
| 3937 | + last_detail = Some(format!( |
| 3938 | + "{} returned no banner output (exit={})", |
| 3939 | + arg, |
| 3940 | + output.status.code().unwrap_or(-1) |
| 3941 | + )); |
| 3942 | + } |
| 3943 | + Err(err) => { |
| 3944 | + last_detail = Some(format!("{} failed: {}", arg, err)); |
| 1924 | 3945 | } |
| 1925 | | - ConsistencyCheck::CliAsmReproducible => run_cli_asm_reproducible( |
| 1926 | | - &prepared.compiler_source, |
| 1927 | | - opt_level, |
| 1928 | | - case.repeat_count, |
| 1929 | | - tools, |
| 1930 | | - ), |
| 1931 | | - ConsistencyCheck::CliObjReproducible => run_cli_obj_reproducible( |
| 1932 | | - &prepared.compiler_source, |
| 1933 | | - opt_level, |
| 1934 | | - case.repeat_count, |
| 1935 | | - tools, |
| 1936 | | - ), |
| 1937 | | - ConsistencyCheck::CliRunReproducible => run_cli_run_reproducible( |
| 1938 | | - &prepared.compiler_source, |
| 1939 | | - opt_level, |
| 1940 | | - case.repeat_count, |
| 1941 | | - tools, |
| 1942 | | - ), |
| 1943 | | - ConsistencyCheck::CaptureAsmVsCliAsm => run_capture_asm_vs_cli_asm( |
| 1944 | | - &prepared.compiler_source, |
| 1945 | | - opt_level, |
| 1946 | | - case.repeat_count, |
| 1947 | | - capture_result, |
| 1948 | | - tools, |
| 1949 | | - ), |
| 1950 | | - ConsistencyCheck::CaptureObjVsCliObj => run_capture_obj_vs_cli_obj( |
| 1951 | | - &prepared.compiler_source, |
| 1952 | | - opt_level, |
| 1953 | | - case.repeat_count, |
| 1954 | | - capture_result, |
| 1955 | | - tools, |
| 1956 | | - ), |
| 1957 | | - ConsistencyCheck::CaptureRunVsCliRun => run_capture_run_vs_cli_run( |
| 1958 | | - &prepared.compiler_source, |
| 1959 | | - opt_level, |
| 1960 | | - case.repeat_count, |
| 1961 | | - capture_result, |
| 1962 | | - tools, |
| 1963 | | - ), |
| 1964 | | - ConsistencyCheck::CaptureAsmReproducible => run_capture_asm_reproducible( |
| 1965 | | - &prepared.compiler_source, |
| 1966 | | - opt_level, |
| 1967 | | - case.repeat_count, |
| 1968 | | - capture_result, |
| 1969 | | - tools, |
| 1970 | | - ), |
| 1971 | | - ConsistencyCheck::CaptureObjReproducible => run_capture_obj_reproducible( |
| 1972 | | - &prepared.compiler_source, |
| 1973 | | - opt_level, |
| 1974 | | - case.repeat_count, |
| 1975 | | - capture_result, |
| 1976 | | - tools, |
| 1977 | | - ), |
| 1978 | | - ConsistencyCheck::CaptureRunReproducible => run_capture_run_reproducible( |
| 1979 | | - &prepared.compiler_source, |
| 1980 | | - opt_level, |
| 1981 | | - case.repeat_count, |
| 1982 | | - capture_result, |
| 1983 | | - tools, |
| 1984 | | - ), |
| 1985 | | - }; |
| 1986 | | - if let Some(issue) = issue { |
| 1987 | | - failures.push(issue); |
| 1988 | 3946 | } |
| 1989 | 3947 | } |
| 1990 | | - failures |
| 3948 | + |
| 3949 | + Err(last_detail.unwrap_or_else(|| "probe failed".to_string())) |
| 1991 | 3950 | } |
| 1992 | 3951 | |
| 1993 | | -fn format_consistency_issues(issues: &[ConsistencyIssue]) -> String { |
| 1994 | | - issues |
| 1995 | | - .iter() |
| 1996 | | - .map(|issue| { |
| 1997 | | - format!( |
| 1998 | | - "consistency check '{}' failed\n{}", |
| 1999 | | - issue.check.as_str(), |
| 2000 | | - issue.detail |
| 2001 | | - ) |
| 2002 | | - }) |
| 2003 | | - .collect::<Vec<_>>() |
| 2004 | | - .join("\n\n") |
| 3952 | +fn compact_probe_banner(line: &str) -> String { |
| 3953 | + let compact = line.split_whitespace().collect::<Vec<_>>().join(" "); |
| 3954 | + let chars = compact.chars().collect::<Vec<_>>(); |
| 3955 | + if chars.len() > 120 { |
| 3956 | + chars.into_iter().take(117).collect::<String>() + "..." |
| 3957 | + } else { |
| 3958 | + compact |
| 3959 | + } |
| 2005 | 3960 | } |
| 2006 | 3961 | |
| 2007 | | -fn cleanup_consistency_issues(issues: &[ConsistencyIssue]) { |
| 2008 | | - for issue in issues { |
| 2009 | | - let _ = fs::remove_dir_all(&issue.temp_root); |
| 3962 | +fn tool_probe_status(configured: &str, already_resolved_path: bool) -> String { |
| 3963 | + let probe = tool_probe(configured, already_resolved_path); |
| 3964 | + match probe.resolved_path { |
| 3965 | + Some(path) => format!("configured={} resolved={}", configured, path.display()), |
| 3966 | + None => format!("configured={} resolved=missing", configured), |
| 2010 | 3967 | } |
| 2011 | 3968 | } |
| 2012 | 3969 | |
| 2013 | | -fn run_cli_obj_vs_system_as( |
| 2014 | | - source: &Path, |
| 2015 | | - opt_level: OptLevel, |
| 2016 | | - tools: &ToolchainConfig, |
| 2017 | | -) -> Option<ConsistencyIssue> { |
| 2018 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2019 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2020 | | - return Some(ConsistencyIssue { |
| 2021 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2022 | | - summary: "could not create consistency temp dir".into(), |
| 2023 | | - repeat_count: None, |
| 2024 | | - unique_variant_count: None, |
| 2025 | | - varying_components: Vec::new(), |
| 2026 | | - stable_components: Vec::new(), |
| 2027 | | - detail: format!( |
| 2028 | | - "cannot create consistency temp dir '{}': {}", |
| 2029 | | - temp_root.display(), |
| 2030 | | - err |
| 2031 | | - ), |
| 2032 | | - temp_root, |
| 2033 | | - }); |
| 3970 | +fn resolve_tool_path(configured: &str) -> Option<PathBuf> { |
| 3971 | + let configured_path = Path::new(configured); |
| 3972 | + if configured.contains('/') || configured.starts_with('.') { |
| 3973 | + return configured_path |
| 3974 | + .exists() |
| 3975 | + .then(|| configured_path.to_path_buf()); |
| 2034 | 3976 | } |
| 2035 | 3977 | |
| 2036 | | - let asm_path = temp_root.join("from_cli.s"); |
| 2037 | | - let asm_obj_path = temp_root.join("from_cli_asm.o"); |
| 2038 | | - let obj_path = temp_root.join("from_cli_obj.o"); |
| 3978 | + let path_var = std::env::var_os("PATH")?; |
| 3979 | + for entry in std::env::split_paths(&path_var) { |
| 3980 | + let candidate = entry.join(configured); |
| 3981 | + if candidate.exists() { |
| 3982 | + return Some(candidate); |
| 3983 | + } |
| 3984 | + } |
| 3985 | + None |
| 3986 | +} |
| 2039 | 3987 | |
| 2040 | | - let asm_command = |
| 2041 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 2042 | | - Ok(command) => command, |
| 2043 | | - Err(detail) => { |
| 2044 | | - return Some(ConsistencyIssue { |
| 2045 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2046 | | - summary: "armfortas -S failed during consistency check".into(), |
| 2047 | | - repeat_count: None, |
| 2048 | | - unique_variant_count: None, |
| 2049 | | - varying_components: Vec::new(), |
| 2050 | | - stable_components: Vec::new(), |
| 2051 | | - detail, |
| 2052 | | - temp_root, |
| 2053 | | - }) |
| 2054 | | - } |
| 2055 | | - }; |
| 3988 | +fn display_path(path: &Path) -> String { |
| 3989 | + fs::canonicalize(path) |
| 3990 | + .unwrap_or_else(|_| path.to_path_buf()) |
| 3991 | + .display() |
| 3992 | + .to_string() |
| 3993 | +} |
| 2056 | 3994 | |
| 2057 | | - let as_args = vec![ |
| 2058 | | - "-o".to_string(), |
| 2059 | | - asm_obj_path.display().to_string(), |
| 2060 | | - asm_path.display().to_string(), |
| 2061 | | - ]; |
| 2062 | | - let as_command = render_command(tools.system_as_bin(), &as_args); |
| 2063 | | - let as_output = match Command::new(tools.system_as_bin()) |
| 2064 | | - .args([ |
| 2065 | | - "-o", |
| 2066 | | - asm_obj_path.to_str().unwrap(), |
| 2067 | | - asm_path.to_str().unwrap(), |
| 2068 | | - ]) |
| 2069 | | - .output() |
| 2070 | | - { |
| 2071 | | - Ok(output) => output, |
| 2072 | | - Err(err) => { |
| 2073 | | - return Some(ConsistencyIssue { |
| 2074 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2075 | | - summary: "system assembler invocation failed".into(), |
| 2076 | | - repeat_count: None, |
| 2077 | | - unique_variant_count: None, |
| 2078 | | - varying_components: Vec::new(), |
| 2079 | | - stable_components: Vec::new(), |
| 2080 | | - detail: format!("{}\ncannot run assembler: {}", as_command, err), |
| 2081 | | - temp_root, |
| 2082 | | - }) |
| 3995 | +fn discover_suites(root: PathBuf) -> Result<Vec<SuiteSpec>, String> { |
| 3996 | + let mut files = Vec::new(); |
| 3997 | + collect_suite_files(&root, &mut files)?; |
| 3998 | + files.sort(); |
| 3999 | + |
| 4000 | + let mut suites = Vec::new(); |
| 4001 | + for file in files { |
| 4002 | + suites.push(parse_suite_file(&file)?); |
| 4003 | + } |
| 4004 | + suites.sort_by(|a, b| a.name.cmp(&b.name)); |
| 4005 | + Ok(suites) |
| 4006 | +} |
| 4007 | + |
| 4008 | +fn collect_suite_files(root: &Path, files: &mut Vec<PathBuf>) -> Result<(), String> { |
| 4009 | + let entries = fs::read_dir(root) |
| 4010 | + .map_err(|e| format!("cannot read suite root '{}': {}", root.display(), e))?; |
| 4011 | + for entry in entries { |
| 4012 | + let entry = |
| 4013 | + entry.map_err(|e| format!("cannot read entry in '{}': {}", root.display(), e))?; |
| 4014 | + let path = entry.path(); |
| 4015 | + if path.is_dir() { |
| 4016 | + collect_suite_files(&path, files)?; |
| 4017 | + } else if path.extension().and_then(|ext| ext.to_str()) == Some(SUITE_EXTENSION) { |
| 4018 | + files.push(path); |
| 2083 | 4019 | } |
| 2084 | | - }; |
| 2085 | | - if !as_output.status.success() { |
| 2086 | | - let stderr = String::from_utf8_lossy(&as_output.stderr); |
| 2087 | | - return Some(ConsistencyIssue { |
| 2088 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2089 | | - summary: "system assembler rejected armfortas -S output".into(), |
| 2090 | | - repeat_count: None, |
| 2091 | | - unique_variant_count: None, |
| 2092 | | - varying_components: Vec::new(), |
| 2093 | | - stable_components: Vec::new(), |
| 2094 | | - detail: format!("{}\nassembler failed:\n{}", as_command, stderr), |
| 2095 | | - temp_root, |
| 2096 | | - }); |
| 2097 | 4020 | } |
| 4021 | + Ok(()) |
| 4022 | +} |
| 2098 | 4023 | |
| 2099 | | - let obj_command = |
| 2100 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 2101 | | - Ok(command) => command, |
| 2102 | | - Err(detail) => { |
| 2103 | | - return Some(ConsistencyIssue { |
| 2104 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2105 | | - summary: "armfortas -c failed during consistency check".into(), |
| 2106 | | - repeat_count: None, |
| 2107 | | - unique_variant_count: None, |
| 2108 | | - varying_components: Vec::new(), |
| 2109 | | - stable_components: Vec::new(), |
| 2110 | | - detail, |
| 2111 | | - temp_root, |
| 2112 | | - }) |
| 2113 | | - } |
| 2114 | | - }; |
| 4024 | +fn parse_suite_file(path: &Path) -> Result<SuiteSpec, String> { |
| 4025 | + let text = fs::read_to_string(path) |
| 4026 | + .map_err(|e| format!("cannot read suite '{}': {}", path.display(), e))?; |
| 2115 | 4027 | |
| 2116 | | - let asm_snapshot = match object_snapshot(&asm_obj_path, tools) { |
| 2117 | | - Ok(snapshot) => snapshot, |
| 2118 | | - Err(detail) => { |
| 2119 | | - return Some(ConsistencyIssue { |
| 2120 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2121 | | - summary: "could not snapshot object assembled from -S output".into(), |
| 2122 | | - repeat_count: None, |
| 2123 | | - unique_variant_count: None, |
| 2124 | | - varying_components: Vec::new(), |
| 2125 | | - stable_components: Vec::new(), |
| 2126 | | - detail: format!("{}\n{}", as_command, detail), |
| 2127 | | - temp_root, |
| 2128 | | - }) |
| 4028 | + let mut suite_name = None; |
| 4029 | + let mut cases = Vec::new(); |
| 4030 | + let mut current = None; |
| 4031 | + |
| 4032 | + for (index, raw_line) in text.lines().enumerate() { |
| 4033 | + let line_no = index + 1; |
| 4034 | + let line = raw_line.trim(); |
| 4035 | + if line.is_empty() || line.starts_with('#') { |
| 4036 | + continue; |
| 2129 | 4037 | } |
| 2130 | | - }; |
| 2131 | | - let obj_snapshot = match object_snapshot(&obj_path, tools) { |
| 2132 | | - Ok(snapshot) => snapshot, |
| 2133 | | - Err(detail) => { |
| 2134 | | - return Some(ConsistencyIssue { |
| 2135 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2136 | | - summary: "could not snapshot object from armfortas -c".into(), |
| 2137 | | - repeat_count: None, |
| 2138 | | - unique_variant_count: None, |
| 2139 | | - varying_components: Vec::new(), |
| 2140 | | - stable_components: Vec::new(), |
| 2141 | | - detail: format!("{}\n{}", obj_command, detail), |
| 2142 | | - temp_root, |
| 2143 | | - }) |
| 4038 | + |
| 4039 | + if let Some(rest) = line.strip_prefix("suite ") { |
| 4040 | + if suite_name.is_some() { |
| 4041 | + return Err(format!( |
| 4042 | + "{}:{}: duplicate suite declaration", |
| 4043 | + path.display(), |
| 4044 | + line_no |
| 4045 | + )); |
| 4046 | + } |
| 4047 | + suite_name = Some(parse_quoted(rest, path, line_no)?); |
| 4048 | + continue; |
| 2144 | 4049 | } |
| 2145 | | - }; |
| 2146 | 4050 | |
| 2147 | | - if asm_snapshot != obj_snapshot { |
| 2148 | | - let snapshots = [&asm_snapshot, &obj_snapshot]; |
| 2149 | | - let varying = varying_object_components(&snapshots) |
| 2150 | | - .into_iter() |
| 2151 | | - .map(str::to_string) |
| 2152 | | - .collect::<Vec<_>>(); |
| 2153 | | - let stable = stable_object_components(&snapshots) |
| 2154 | | - .into_iter() |
| 2155 | | - .map(str::to_string) |
| 2156 | | - .collect::<Vec<_>>(); |
| 2157 | | - return Some(ConsistencyIssue { |
| 2158 | | - check: ConsistencyCheck::CliObjVsSystemAs, |
| 2159 | | - summary: format!( |
| 2160 | | - "varying_components={} stable_components={}", |
| 2161 | | - join_or_none_from_strings(&varying), |
| 2162 | | - join_or_none_from_strings(&stable) |
| 2163 | | - ), |
| 2164 | | - repeat_count: None, |
| 2165 | | - unique_variant_count: None, |
| 2166 | | - varying_components: varying, |
| 2167 | | - stable_components: stable, |
| 2168 | | - detail: format!( |
| 2169 | | - "object snapshot mismatch between armfortas -S | as and armfortas -c\n{}\n{}\n{}\n{}", |
| 2170 | | - asm_command, |
| 2171 | | - as_command, |
| 2172 | | - obj_command, |
| 2173 | | - describe_object_difference(&asm_snapshot, &obj_snapshot, "-S | as", "-c") |
| 2174 | | - ), |
| 2175 | | - temp_root, |
| 2176 | | - }); |
| 2177 | | - } |
| 4051 | + if let Some(rest) = line.strip_prefix("case ") { |
| 4052 | + if current.is_some() { |
| 4053 | + return Err(format!( |
| 4054 | + "{}:{}: nested case without end", |
| 4055 | + path.display(), |
| 4056 | + line_no |
| 4057 | + )); |
| 4058 | + } |
| 4059 | + current = Some(CaseBuilder::new(parse_quoted(rest, path, line_no)?)); |
| 4060 | + continue; |
| 4061 | + } |
| 2178 | 4062 | |
| 2179 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2180 | | - None |
| 2181 | | -} |
| 4063 | + if line == "end" { |
| 4064 | + let builder = current.take().ok_or_else(|| { |
| 4065 | + format!("{}:{}: stray end outside of case", path.display(), line_no) |
| 4066 | + })?; |
| 4067 | + cases.push(builder.build(path)?); |
| 4068 | + continue; |
| 4069 | + } |
| 2182 | 4070 | |
| 2183 | | -fn run_cli_asm_reproducible( |
| 2184 | | - source: &Path, |
| 2185 | | - opt_level: OptLevel, |
| 2186 | | - repeat_count: usize, |
| 2187 | | - tools: &ToolchainConfig, |
| 2188 | | -) -> Option<ConsistencyIssue> { |
| 2189 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2190 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2191 | | - return Some(ConsistencyIssue { |
| 2192 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 2193 | | - summary: "could not create consistency temp dir".into(), |
| 2194 | | - repeat_count: None, |
| 2195 | | - unique_variant_count: None, |
| 2196 | | - varying_components: Vec::new(), |
| 2197 | | - stable_components: Vec::new(), |
| 2198 | | - detail: format!( |
| 2199 | | - "cannot create consistency temp dir '{}': {}", |
| 2200 | | - temp_root.display(), |
| 2201 | | - err |
| 2202 | | - ), |
| 2203 | | - temp_root, |
| 2204 | | - }); |
| 2205 | | - } |
| 4071 | + let builder = current.as_mut().ok_or_else(|| { |
| 4072 | + format!( |
| 4073 | + "{}:{}: expected suite/case declaration first", |
| 4074 | + path.display(), |
| 4075 | + line_no |
| 4076 | + ) |
| 4077 | + })?; |
| 2206 | 4078 | |
| 2207 | | - let mut runs = Vec::new(); |
| 2208 | | - for index in 0..repeat_count { |
| 2209 | | - let asm_path = temp_root.join(format!("run_{:02}.s", index)); |
| 2210 | | - let command = |
| 2211 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 2212 | | - Ok(command) => command, |
| 2213 | | - Err(detail) => { |
| 2214 | | - return Some(ConsistencyIssue { |
| 2215 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 2216 | | - summary: "armfortas -S failed during reproducibility check".into(), |
| 2217 | | - repeat_count: None, |
| 2218 | | - unique_variant_count: None, |
| 2219 | | - varying_components: Vec::new(), |
| 2220 | | - stable_components: Vec::new(), |
| 2221 | | - detail, |
| 2222 | | - temp_root, |
| 2223 | | - }) |
| 2224 | | - } |
| 2225 | | - }; |
| 2226 | | - let text = match read_text_artifact(&asm_path) { |
| 2227 | | - Ok(text) => text, |
| 2228 | | - Err(detail) => { |
| 2229 | | - return Some(ConsistencyIssue { |
| 2230 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 2231 | | - summary: "could not read emitted assembly during reproducibility check".into(), |
| 2232 | | - repeat_count: None, |
| 2233 | | - unique_variant_count: None, |
| 2234 | | - varying_components: Vec::new(), |
| 2235 | | - stable_components: Vec::new(), |
| 2236 | | - detail, |
| 2237 | | - temp_root, |
| 2238 | | - }) |
| 4079 | + if let Some(rest) = line.strip_prefix("source ") { |
| 4080 | + builder.source = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 4081 | + } else if let Some(rest) = line.strip_prefix("entry ") { |
| 4082 | + builder.graph_entry = Some(resolve_suite_relative_path(rest, path, line_no)?); |
| 4083 | + } else if let Some(rest) = line.strip_prefix("file ") { |
| 4084 | + builder |
| 4085 | + .graph_files |
| 4086 | + .push(resolve_suite_relative_path(rest, path, line_no)?); |
| 4087 | + } else if let Some(rest) = line.strip_prefix("compiler ") { |
| 4088 | + let (compiler, artifacts) = parse_compiler_artifact_declaration(rest, path, line_no)?; |
| 4089 | + builder.generic_compiler = Some(compiler); |
| 4090 | + builder.generic_artifacts = artifacts; |
| 4091 | + } else if let Some(rest) = line.strip_prefix("compare ") { |
| 4092 | + let (left, right, artifacts) = parse_compare_declaration(rest, path, line_no)?; |
| 4093 | + builder.generic_compare = Some((left, right)); |
| 4094 | + builder.generic_compare_artifacts = artifacts; |
| 4095 | + } else if let Some(rest) = line.strip_prefix("armfortas =>") { |
| 4096 | + builder.requested = parse_stage_list(rest, path, line_no)?; |
| 4097 | + } else if let Some(rest) = line.strip_prefix("repeat =>") { |
| 4098 | + builder.repeat_count = parse_repeat_count(rest, path, line_no)?; |
| 4099 | + } else if let Some(rest) = line.strip_prefix("opts =>") { |
| 4100 | + builder.opt_levels = parse_opt_levels(rest, path, line_no)?; |
| 4101 | + } else if let Some(rest) = line.strip_prefix("differential =>") { |
| 4102 | + builder.reference_compilers = parse_reference_compilers(rest, path, line_no)?; |
| 4103 | + } else if let Some(rest) = line.strip_prefix("consistency =>") { |
| 4104 | + builder.consistency_checks = parse_consistency_checks(rest, path, line_no)?; |
| 4105 | + } else if let Some(rest) = line.strip_prefix("expect-fail ") { |
| 4106 | + builder |
| 4107 | + .expectations |
| 4108 | + .push(parse_failure_expectation(rest, path, line_no)?); |
| 4109 | + } else if let Some(rest) = line.strip_prefix("expect ") { |
| 4110 | + builder |
| 4111 | + .expectations |
| 4112 | + .push(parse_expectation(rest, path, line_no)?); |
| 4113 | + } else if let Some(rest) = line.strip_prefix("xfail capability ") { |
| 4114 | + if builder.capability_policy.is_some() { |
| 4115 | + return Err(format!( |
| 4116 | + "{}:{}: duplicate capability policy", |
| 4117 | + path.display(), |
| 4118 | + line_no |
| 4119 | + )); |
| 2239 | 4120 | } |
| 2240 | | - }; |
| 2241 | | - runs.push(TextRun { |
| 2242 | | - label: format!("run {} (-S)", index + 1), |
| 2243 | | - command, |
| 2244 | | - normalized: normalize_text_artifact(&text), |
| 2245 | | - }); |
| 4121 | + builder.capability_policy = Some(parse_capability_policy( |
| 4122 | + StatusKind::Xfail, |
| 4123 | + rest, |
| 4124 | + path, |
| 4125 | + line_no, |
| 4126 | + )?); |
| 4127 | + } else if let Some(rest) = line.strip_prefix("future capability ") { |
| 4128 | + if builder.capability_policy.is_some() { |
| 4129 | + return Err(format!( |
| 4130 | + "{}:{}: duplicate capability policy", |
| 4131 | + path.display(), |
| 4132 | + line_no |
| 4133 | + )); |
| 4134 | + } |
| 4135 | + builder.capability_policy = Some(parse_capability_policy( |
| 4136 | + StatusKind::Future, |
| 4137 | + rest, |
| 4138 | + path, |
| 4139 | + line_no, |
| 4140 | + )?); |
| 4141 | + } else if let Some(rest) = line.strip_prefix("xfail ") { |
| 4142 | + builder |
| 4143 | + .status_rules |
| 4144 | + .push(parse_status_rule(StatusKind::Xfail, rest, path, line_no)?); |
| 4145 | + } else if let Some(rest) = line.strip_prefix("future ") { |
| 4146 | + builder |
| 4147 | + .status_rules |
| 4148 | + .push(parse_status_rule(StatusKind::Future, rest, path, line_no)?); |
| 4149 | + } else { |
| 4150 | + return Err(format!( |
| 4151 | + "{}:{}: unrecognized line '{}'", |
| 4152 | + path.display(), |
| 4153 | + line_no, |
| 4154 | + line |
| 4155 | + )); |
| 4156 | + } |
| 2246 | 4157 | } |
| 2247 | 4158 | |
| 2248 | | - let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 2249 | | - if unique_variants > 1 { |
| 2250 | | - let (left, right) = |
| 2251 | | - first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 2252 | | - return Some(ConsistencyIssue { |
| 2253 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 2254 | | - summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 2255 | | - repeat_count: Some(repeat_count), |
| 2256 | | - unique_variant_count: Some(unique_variants), |
| 2257 | | - varying_components: Vec::new(), |
| 2258 | | - stable_components: Vec::new(), |
| 2259 | | - detail: format!( |
| 2260 | | - "assembly output is not reproducible across repeated armfortas -S runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 2261 | | - repeat_count, |
| 2262 | | - unique_variants, |
| 2263 | | - left.command, |
| 2264 | | - right.command, |
| 2265 | | - describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 2266 | | - ), |
| 2267 | | - temp_root, |
| 2268 | | - }); |
| 4159 | + if current.is_some() { |
| 4160 | + return Err(format!("{}: unterminated case block", path.display())); |
| 2269 | 4161 | } |
| 2270 | 4162 | |
| 2271 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2272 | | - None |
| 4163 | + let suite_name = |
| 4164 | + suite_name.ok_or_else(|| format!("{}: missing suite declaration", path.display()))?; |
| 4165 | + if cases.is_empty() { |
| 4166 | + return Err(format!("{}: suite has no cases", path.display())); |
| 4167 | + } |
| 4168 | + |
| 4169 | + Ok(SuiteSpec { |
| 4170 | + name: suite_name, |
| 4171 | + path: path.to_path_buf(), |
| 4172 | + cases, |
| 4173 | + }) |
| 2273 | 4174 | } |
| 2274 | 4175 | |
| 2275 | | -fn run_cli_obj_reproducible( |
| 2276 | | - source: &Path, |
| 2277 | | - opt_level: OptLevel, |
| 4176 | +struct CaseBuilder { |
| 4177 | + name: String, |
| 4178 | + source: Option<PathBuf>, |
| 4179 | + graph_entry: Option<PathBuf>, |
| 4180 | + graph_files: Vec<PathBuf>, |
| 4181 | + requested: BTreeSet<Stage>, |
| 4182 | + generic_compiler: Option<CompilerSpec>, |
| 4183 | + generic_artifacts: BTreeSet<ArtifactKey>, |
| 4184 | + generic_compare: Option<(CompilerSpec, CompilerSpec)>, |
| 4185 | + generic_compare_artifacts: BTreeSet<ArtifactKey>, |
| 4186 | + opt_levels: Vec<OptLevel>, |
| 2278 | 4187 | repeat_count: usize, |
| 2279 | | - tools: &ToolchainConfig, |
| 2280 | | -) -> Option<ConsistencyIssue> { |
| 2281 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2282 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2283 | | - return Some(ConsistencyIssue { |
| 2284 | | - check: ConsistencyCheck::CliObjReproducible, |
| 2285 | | - summary: "could not create consistency temp dir".into(), |
| 2286 | | - repeat_count: None, |
| 2287 | | - unique_variant_count: None, |
| 2288 | | - varying_components: Vec::new(), |
| 2289 | | - stable_components: Vec::new(), |
| 2290 | | - detail: format!( |
| 2291 | | - "cannot create consistency temp dir '{}': {}", |
| 2292 | | - temp_root.display(), |
| 2293 | | - err |
| 2294 | | - ), |
| 2295 | | - temp_root, |
| 2296 | | - }); |
| 2297 | | - } |
| 4188 | + reference_compilers: Vec<ReferenceCompiler>, |
| 4189 | + consistency_checks: Vec<ConsistencyCheck>, |
| 4190 | + expectations: Vec<Expectation>, |
| 4191 | + status_rules: Vec<PendingStatusRule>, |
| 4192 | + capability_policy: Option<CapabilityPolicy>, |
| 4193 | +} |
| 2298 | 4194 | |
| 2299 | | - let mut runs = Vec::new(); |
| 2300 | | - for index in 0..repeat_count { |
| 2301 | | - let obj_path = temp_root.join(format!("run_{:02}.o", index)); |
| 2302 | | - let command = |
| 2303 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 2304 | | - Ok(command) => command, |
| 2305 | | - Err(detail) => { |
| 2306 | | - return Some(ConsistencyIssue { |
| 2307 | | - check: ConsistencyCheck::CliObjReproducible, |
| 2308 | | - summary: "armfortas -c failed during reproducibility check".into(), |
| 2309 | | - repeat_count: None, |
| 2310 | | - unique_variant_count: None, |
| 2311 | | - varying_components: Vec::new(), |
| 2312 | | - stable_components: Vec::new(), |
| 2313 | | - detail, |
| 2314 | | - temp_root, |
| 2315 | | - }) |
| 2316 | | - } |
| 2317 | | - }; |
| 2318 | | - let snapshot = match object_snapshot(&obj_path, tools) { |
| 2319 | | - Ok(snapshot) => snapshot, |
| 2320 | | - Err(detail) => { |
| 2321 | | - return Some(ConsistencyIssue { |
| 2322 | | - check: ConsistencyCheck::CliObjReproducible, |
| 2323 | | - summary: "could not snapshot object during reproducibility check".into(), |
| 2324 | | - repeat_count: None, |
| 2325 | | - unique_variant_count: None, |
| 2326 | | - varying_components: Vec::new(), |
| 2327 | | - stable_components: Vec::new(), |
| 2328 | | - detail: format!("{}\n{}", command, detail), |
| 2329 | | - temp_root, |
| 2330 | | - }) |
| 2331 | | - } |
| 2332 | | - }; |
| 2333 | | - runs.push(ObjectRun { |
| 2334 | | - label: format!("run {} (-c)", index + 1), |
| 2335 | | - command, |
| 2336 | | - snapshot, |
| 2337 | | - }); |
| 4195 | +impl CaseBuilder { |
| 4196 | + fn new(name: String) -> Self { |
| 4197 | + Self { |
| 4198 | + name, |
| 4199 | + source: None, |
| 4200 | + graph_entry: None, |
| 4201 | + graph_files: Vec::new(), |
| 4202 | + requested: BTreeSet::new(), |
| 4203 | + generic_compiler: None, |
| 4204 | + generic_artifacts: BTreeSet::new(), |
| 4205 | + generic_compare: None, |
| 4206 | + generic_compare_artifacts: BTreeSet::new(), |
| 4207 | + opt_levels: Vec::new(), |
| 4208 | + repeat_count: 2, |
| 4209 | + reference_compilers: Vec::new(), |
| 4210 | + consistency_checks: Vec::new(), |
| 4211 | + expectations: Vec::new(), |
| 4212 | + status_rules: Vec::new(), |
| 4213 | + capability_policy: None, |
| 4214 | + } |
| 2338 | 4215 | } |
| 2339 | 4216 | |
| 2340 | | - let rendered = runs |
| 2341 | | - .iter() |
| 2342 | | - .map(|run| render_object_snapshot(&run.snapshot)) |
| 2343 | | - .collect::<Vec<_>>(); |
| 2344 | | - let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 2345 | | - if unique_variants > 1 { |
| 2346 | | - let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 2347 | | - let (left, right) = |
| 2348 | | - first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 2349 | | - let varying = join_or_none(&varying_object_components(&snapshots)); |
| 2350 | | - let stable = join_or_none(&stable_object_components(&snapshots)); |
| 2351 | | - let varying_components = varying_object_components(&snapshots) |
| 2352 | | - .into_iter() |
| 2353 | | - .map(str::to_string) |
| 2354 | | - .collect::<Vec<_>>(); |
| 2355 | | - let stable_components = stable_object_components(&snapshots) |
| 2356 | | - .into_iter() |
| 2357 | | - .map(str::to_string) |
| 2358 | | - .collect::<Vec<_>>(); |
| 2359 | | - return Some(ConsistencyIssue { |
| 2360 | | - check: ConsistencyCheck::CliObjReproducible, |
| 2361 | | - summary: format!( |
| 2362 | | - "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 2363 | | - repeat_count, unique_variants, varying, stable |
| 2364 | | - ), |
| 2365 | | - repeat_count: Some(repeat_count), |
| 2366 | | - unique_variant_count: Some(unique_variants), |
| 2367 | | - varying_components, |
| 2368 | | - stable_components, |
| 2369 | | - detail: format!( |
| 2370 | | - "object output is not reproducible across repeated armfortas -c runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 2371 | | - repeat_count, |
| 2372 | | - unique_variants, |
| 2373 | | - varying, |
| 2374 | | - stable, |
| 2375 | | - left.command, |
| 2376 | | - right.command, |
| 2377 | | - describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 2378 | | - ), |
| 2379 | | - temp_root, |
| 2380 | | - }); |
| 2381 | | - } |
| 4217 | + fn build(self, suite_path: &Path) -> Result<CaseSpec, String> { |
| 4218 | + let generic_mode_count = usize::from(self.generic_compiler.is_some()) |
| 4219 | + + usize::from(self.generic_compare.is_some()); |
| 4220 | + if generic_mode_count > 1 { |
| 4221 | + return Err(format!( |
| 4222 | + "{}: case '{}' mixes multiple suite-v2 execution forms", |
| 4223 | + suite_path.display(), |
| 4224 | + self.name |
| 4225 | + )); |
| 4226 | + } |
| 2382 | 4227 | |
| 2383 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2384 | | - None |
| 2385 | | -} |
| 4228 | + if generic_mode_count > 0 && !self.requested.is_empty() { |
| 4229 | + return Err(format!( |
| 4230 | + "{}: case '{}' mixes generic suite-v2 syntax with legacy 'armfortas => ...' stages", |
| 4231 | + suite_path.display(), |
| 4232 | + self.name |
| 4233 | + )); |
| 4234 | + } |
| 2386 | 4235 | |
| 2387 | | -fn run_cli_run_reproducible( |
| 2388 | | - source: &Path, |
| 2389 | | - opt_level: OptLevel, |
| 2390 | | - repeat_count: usize, |
| 2391 | | - tools: &ToolchainConfig, |
| 2392 | | -) -> Option<ConsistencyIssue> { |
| 2393 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2394 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2395 | | - return Some(ConsistencyIssue { |
| 2396 | | - check: ConsistencyCheck::CliRunReproducible, |
| 2397 | | - summary: "could not create consistency temp dir".into(), |
| 2398 | | - repeat_count: None, |
| 2399 | | - unique_variant_count: None, |
| 2400 | | - varying_components: Vec::new(), |
| 2401 | | - stable_components: Vec::new(), |
| 2402 | | - detail: format!( |
| 2403 | | - "cannot create consistency temp dir '{}': {}", |
| 2404 | | - temp_root.display(), |
| 2405 | | - err |
| 2406 | | - ), |
| 2407 | | - temp_root, |
| 2408 | | - }); |
| 2409 | | - } |
| 4236 | + if self.source.is_some() && (self.graph_entry.is_some() || !self.graph_files.is_empty()) { |
| 4237 | + return Err(format!( |
| 4238 | + "{}: case '{}' mixes source with graph entry/file declarations", |
| 4239 | + suite_path.display(), |
| 4240 | + self.name |
| 4241 | + )); |
| 4242 | + } |
| 2410 | 4243 | |
| 2411 | | - let mut runs = Vec::new(); |
| 2412 | | - for index in 0..repeat_count { |
| 2413 | | - let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 2414 | | - let build_command = match compile_with_driver( |
| 2415 | | - source, |
| 2416 | | - opt_level, |
| 2417 | | - DriverEmitMode::Binary, |
| 2418 | | - &binary_path, |
| 2419 | | - tools, |
| 2420 | | - ) { |
| 2421 | | - Ok(command) => command, |
| 2422 | | - Err(detail) => { |
| 2423 | | - return Some(ConsistencyIssue { |
| 2424 | | - check: ConsistencyCheck::CliRunReproducible, |
| 2425 | | - summary: "armfortas binary build failed during runtime reproducibility check" |
| 2426 | | - .into(), |
| 2427 | | - repeat_count: None, |
| 2428 | | - unique_variant_count: None, |
| 2429 | | - varying_components: Vec::new(), |
| 2430 | | - stable_components: Vec::new(), |
| 2431 | | - detail, |
| 2432 | | - temp_root, |
| 2433 | | - }) |
| 4244 | + if self.graph_entry.is_some() && self.graph_files.is_empty() { |
| 4245 | + return Err(format!( |
| 4246 | + "{}: case '{}' declares an entry without any file members", |
| 4247 | + suite_path.display(), |
| 4248 | + self.name |
| 4249 | + )); |
| 4250 | + } |
| 4251 | + |
| 4252 | + if self.graph_entry.is_none() && !self.graph_files.is_empty() { |
| 4253 | + return Err(format!( |
| 4254 | + "{}: case '{}' declares file members without an entry", |
| 4255 | + suite_path.display(), |
| 4256 | + self.name |
| 4257 | + )); |
| 4258 | + } |
| 4259 | + |
| 4260 | + let (source, graph_files) = if let Some(source) = self.source { |
| 4261 | + (source, Vec::new()) |
| 4262 | + } else if let Some(entry) = self.graph_entry { |
| 4263 | + if !self.graph_files.iter().any(|file| file == &entry) { |
| 4264 | + return Err(format!( |
| 4265 | + "{}: case '{}' entry '{}' is not listed in file declarations", |
| 4266 | + suite_path.display(), |
| 4267 | + self.name, |
| 4268 | + entry.display() |
| 4269 | + )); |
| 2434 | 4270 | } |
| 4271 | + (entry, self.graph_files) |
| 4272 | + } else { |
| 4273 | + return Err(format!( |
| 4274 | + "{}: case '{}' is missing a source path or graph entry", |
| 4275 | + suite_path.display(), |
| 4276 | + self.name |
| 4277 | + )); |
| 2435 | 4278 | }; |
| 2436 | | - let run_command = render_binary_run_command(&binary_path); |
| 2437 | | - let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 2438 | | - Ok(run) => run, |
| 2439 | | - Err(detail) => { |
| 2440 | | - return Some(ConsistencyIssue { |
| 2441 | | - check: ConsistencyCheck::CliRunReproducible, |
| 2442 | | - summary: "armfortas binary could not run during runtime reproducibility check" |
| 2443 | | - .into(), |
| 2444 | | - repeat_count: None, |
| 2445 | | - unique_variant_count: None, |
| 2446 | | - varying_components: Vec::new(), |
| 2447 | | - stable_components: Vec::new(), |
| 2448 | | - detail, |
| 2449 | | - temp_root, |
| 2450 | | - }) |
| 4279 | + |
| 4280 | + if self.generic_compare.is_some() |
| 4281 | + && (!self.reference_compilers.is_empty() || !self.consistency_checks.is_empty()) |
| 4282 | + { |
| 4283 | + return Err(format!( |
| 4284 | + "{}: case '{}' compare suite-v2 cases do not support differential/consistency rules", |
| 4285 | + suite_path.display(), |
| 4286 | + self.name |
| 4287 | + )); |
| 4288 | + } |
| 4289 | + |
| 4290 | + if self.generic_compiler.is_some() { |
| 4291 | + let unsupported = self |
| 4292 | + .consistency_checks |
| 4293 | + .iter() |
| 4294 | + .copied() |
| 4295 | + .filter(|check| !check.supports_generic_introspect()) |
| 4296 | + .collect::<Vec<_>>(); |
| 4297 | + if !unsupported.is_empty() { |
| 4298 | + return Err(format!( |
| 4299 | + "{}: case '{}' generic compiler cases only support cli_asm_reproducible, cli_obj_reproducible, and cli_run_reproducible today (unsupported: {})", |
| 4300 | + suite_path.display(), |
| 4301 | + self.name, |
| 4302 | + unsupported |
| 4303 | + .iter() |
| 4304 | + .map(ConsistencyCheck::as_str) |
| 4305 | + .collect::<Vec<_>>() |
| 4306 | + .join(", ") |
| 4307 | + )); |
| 2451 | 4308 | } |
| 4309 | + } |
| 4310 | + |
| 4311 | + let needs_source_comment_resolution = self |
| 4312 | + .expectations |
| 4313 | + .iter() |
| 4314 | + .any(|expectation| matches!(expectation, Expectation::FailSourceComments)) |
| 4315 | + || self |
| 4316 | + .status_rules |
| 4317 | + .iter() |
| 4318 | + .any(|rule| matches!(rule, PendingStatusRule::XfailSourceComments)); |
| 4319 | + let source_text = if needs_source_comment_resolution { |
| 4320 | + Some(fs::read_to_string(&source).map_err(|e| { |
| 4321 | + format!( |
| 4322 | + "{}: case '{}': cannot read source '{}' for comment-based directives: {}", |
| 4323 | + suite_path.display(), |
| 4324 | + self.name, |
| 4325 | + source.display(), |
| 4326 | + e |
| 4327 | + ) |
| 4328 | + })?) |
| 4329 | + } else { |
| 4330 | + None |
| 2452 | 4331 | }; |
| 2453 | | - let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 2454 | | - if let Err(err) = write_behavior_run_artifacts( |
| 2455 | | - &temp_root, |
| 2456 | | - &format!("cli_run_{:02}", index), |
| 2457 | | - &command, |
| 2458 | | - &run, |
| 2459 | | - ) { |
| 2460 | | - return Some(ConsistencyIssue { |
| 2461 | | - check: ConsistencyCheck::CliRunReproducible, |
| 2462 | | - summary: "could not write cli runtime artifact".into(), |
| 2463 | | - repeat_count: None, |
| 2464 | | - unique_variant_count: None, |
| 2465 | | - varying_components: Vec::new(), |
| 2466 | | - stable_components: Vec::new(), |
| 2467 | | - detail: format!("cannot write cli runtime artifact: {}", err), |
| 2468 | | - temp_root, |
| 2469 | | - }); |
| 4332 | + |
| 4333 | + let generic_introspect = if let Some(compiler) = self.generic_compiler { |
| 4334 | + if self.generic_artifacts.is_empty() { |
| 4335 | + return Err(format!( |
| 4336 | + "{}: case '{}' generic compiler artifact list is empty", |
| 4337 | + suite_path.display(), |
| 4338 | + self.name |
| 4339 | + )); |
| 4340 | + } |
| 4341 | + Some(GenericIntrospectCase { |
| 4342 | + compiler, |
| 4343 | + artifacts: self.generic_artifacts, |
| 4344 | + }) |
| 4345 | + } else { |
| 4346 | + None |
| 4347 | + }; |
| 4348 | + |
| 4349 | + let generic_compare = if let Some((left, right)) = self.generic_compare { |
| 4350 | + let mut artifacts = self.generic_compare_artifacts; |
| 4351 | + if artifacts.is_empty() { |
| 4352 | + return Err(format!( |
| 4353 | + "{}: case '{}' compare artifact list is empty", |
| 4354 | + suite_path.display(), |
| 4355 | + self.name |
| 4356 | + )); |
| 4357 | + } |
| 4358 | + artifacts.extend(default_compare_artifacts(&artifacts)); |
| 4359 | + Some(GenericCompareCase { |
| 4360 | + left, |
| 4361 | + right, |
| 4362 | + artifacts, |
| 4363 | + }) |
| 4364 | + } else { |
| 4365 | + None |
| 4366 | + }; |
| 4367 | + |
| 4368 | + let mut requested = self.requested; |
| 4369 | + if requested.is_empty() && generic_introspect.is_none() && generic_compare.is_none() { |
| 4370 | + requested.insert(Stage::Run); |
| 2470 | 4371 | } |
| 2471 | | - runs.push(BehaviorRun { |
| 2472 | | - label: format!("cli run {}", index + 1), |
| 2473 | | - command, |
| 2474 | | - signature: normalize_run_signature(&run), |
| 2475 | | - run, |
| 2476 | | - }); |
| 2477 | | - } |
| 2478 | 4372 | |
| 2479 | | - let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 2480 | | - if unique_variants > 1 { |
| 2481 | | - let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 2482 | | - let varying = varying_run_components(&signatures); |
| 2483 | | - let stable = stable_run_components(&signatures); |
| 2484 | | - let (left, right) = first_distinct_behavior_pair(&runs) |
| 2485 | | - .expect("unique variants > 1 implies a distinct pair"); |
| 2486 | | - return Some(ConsistencyIssue { |
| 2487 | | - check: ConsistencyCheck::CliRunReproducible, |
| 2488 | | - summary: format!( |
| 2489 | | - "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 2490 | | - repeat_count, |
| 2491 | | - unique_variants, |
| 2492 | | - join_or_none(&varying), |
| 2493 | | - join_or_none(&stable) |
| 2494 | | - ), |
| 2495 | | - repeat_count: Some(repeat_count), |
| 2496 | | - unique_variant_count: Some(unique_variants), |
| 2497 | | - varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 2498 | | - stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 2499 | | - detail: format!( |
| 2500 | | - "armfortas runtime behavior is not reproducible across repeated full CLI builds\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 2501 | | - repeat_count, |
| 2502 | | - unique_variants, |
| 2503 | | - join_or_none(&varying), |
| 2504 | | - join_or_none(&stable), |
| 2505 | | - left.command, |
| 2506 | | - right.command, |
| 2507 | | - describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 2508 | | - ), |
| 2509 | | - temp_root, |
| 2510 | | - }); |
| 4373 | + let opt_levels = if self.opt_levels.is_empty() { |
| 4374 | + vec![OptLevel::O0] |
| 4375 | + } else { |
| 4376 | + self.opt_levels |
| 4377 | + }; |
| 4378 | + let expectations = resolve_source_comment_expectations( |
| 4379 | + self.expectations, |
| 4380 | + source_text.as_deref(), |
| 4381 | + suite_path, |
| 4382 | + &self.name, |
| 4383 | + &source, |
| 4384 | + )?; |
| 4385 | + let status_rules = resolve_source_comment_status_rules( |
| 4386 | + self.status_rules, |
| 4387 | + source_text.as_deref(), |
| 4388 | + suite_path, |
| 4389 | + &self.name, |
| 4390 | + &source, |
| 4391 | + )?; |
| 4392 | + |
| 4393 | + Ok(CaseSpec { |
| 4394 | + name: self.name, |
| 4395 | + source, |
| 4396 | + graph_files, |
| 4397 | + requested, |
| 4398 | + generic_introspect, |
| 4399 | + generic_compare, |
| 4400 | + opt_levels, |
| 4401 | + repeat_count: self.repeat_count, |
| 4402 | + reference_compilers: self.reference_compilers, |
| 4403 | + consistency_checks: self.consistency_checks, |
| 4404 | + expectations, |
| 4405 | + status_rules, |
| 4406 | + capability_policy: self.capability_policy, |
| 4407 | + }) |
| 2511 | 4408 | } |
| 4409 | +} |
| 2512 | 4410 | |
| 2513 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2514 | | - None |
| 4411 | +fn resolve_suite_relative_path(rest: &str, path: &Path, line_no: usize) -> Result<PathBuf, String> { |
| 4412 | + let relative = parse_quoted(rest, path, line_no)?; |
| 4413 | + Ok(path |
| 4414 | + .parent() |
| 4415 | + .unwrap_or_else(|| Path::new(".")) |
| 4416 | + .join(relative)) |
| 2515 | 4417 | } |
| 2516 | 4418 | |
| 2517 | | -fn run_capture_asm_vs_cli_asm( |
| 2518 | | - source: &Path, |
| 2519 | | - opt_level: OptLevel, |
| 2520 | | - repeat_count: usize, |
| 2521 | | - capture_result: &CaptureResult, |
| 2522 | | - tools: &ToolchainConfig, |
| 2523 | | -) -> Option<ConsistencyIssue> { |
| 2524 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2525 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2526 | | - return Some(ConsistencyIssue { |
| 2527 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2528 | | - summary: "could not create consistency temp dir".into(), |
| 2529 | | - repeat_count: None, |
| 2530 | | - unique_variant_count: None, |
| 2531 | | - varying_components: Vec::new(), |
| 2532 | | - stable_components: Vec::new(), |
| 2533 | | - detail: format!( |
| 2534 | | - "cannot create consistency temp dir '{}': {}", |
| 2535 | | - temp_root.display(), |
| 2536 | | - err |
| 2537 | | - ), |
| 2538 | | - temp_root, |
| 2539 | | - }); |
| 4419 | +fn parse_stage_list(rest: &str, path: &Path, line_no: usize) -> Result<BTreeSet<Stage>, String> { |
| 4420 | + let mut stages = BTreeSet::new(); |
| 4421 | + for raw in rest.split(',') { |
| 4422 | + let name = raw.trim(); |
| 4423 | + if name.is_empty() { |
| 4424 | + continue; |
| 4425 | + } |
| 4426 | + let stage = Stage::parse(name) |
| 4427 | + .ok_or_else(|| format!("{}:{}: unknown stage '{}'", path.display(), line_no, name))?; |
| 4428 | + stages.insert(stage); |
| 4429 | + } |
| 4430 | + if stages.is_empty() { |
| 4431 | + return Err(format!( |
| 4432 | + "{}:{}: armfortas stage list is empty", |
| 4433 | + path.display(), |
| 4434 | + line_no |
| 4435 | + )); |
| 2540 | 4436 | } |
| 4437 | + Ok(stages) |
| 4438 | +} |
| 2541 | 4439 | |
| 2542 | | - let capture_command = render_capture_command(source, opt_level, Stage::Asm); |
| 2543 | | - let capture_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 2544 | | - Ok(text) => text, |
| 2545 | | - Err(detail) => { |
| 2546 | | - return Some(ConsistencyIssue { |
| 2547 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2548 | | - summary: "capture result did not include assembly text".into(), |
| 2549 | | - repeat_count: None, |
| 2550 | | - unique_variant_count: None, |
| 2551 | | - varying_components: Vec::new(), |
| 2552 | | - stable_components: Vec::new(), |
| 2553 | | - detail, |
| 2554 | | - temp_root, |
| 2555 | | - }) |
| 2556 | | - } |
| 2557 | | - }; |
| 2558 | | - if let Err(err) = fs::write(temp_root.join("from_capture.s"), capture_text) { |
| 2559 | | - return Some(ConsistencyIssue { |
| 2560 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2561 | | - summary: "could not write captured assembly artifact".into(), |
| 2562 | | - repeat_count: None, |
| 2563 | | - unique_variant_count: None, |
| 2564 | | - varying_components: Vec::new(), |
| 2565 | | - stable_components: Vec::new(), |
| 2566 | | - detail: format!("cannot write captured assembly artifact: {}", err), |
| 2567 | | - temp_root, |
| 2568 | | - }); |
| 4440 | +fn parse_compiler_artifact_declaration( |
| 4441 | + rest: &str, |
| 4442 | + path: &Path, |
| 4443 | + line_no: usize, |
| 4444 | +) -> Result<(CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4445 | + let (compiler_raw, artifact_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4446 | + format!( |
| 4447 | + "{}:{}: compiler declaration must use 'compiler <spec> => <artifacts>'", |
| 4448 | + path.display(), |
| 4449 | + line_no |
| 4450 | + ) |
| 4451 | + })?; |
| 4452 | + let compiler = parse_compiler_spec_token(compiler_raw.trim(), path, line_no)?; |
| 4453 | + let artifacts = ArtifactKey::parse_list(artifact_raw.trim()) |
| 4454 | + .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4455 | + if artifacts.is_empty() { |
| 4456 | + return Err(format!( |
| 4457 | + "{}:{}: generic compiler artifact list is empty", |
| 4458 | + path.display(), |
| 4459 | + line_no |
| 4460 | + )); |
| 2569 | 4461 | } |
| 2570 | | - let capture_normalized = normalize_text_artifact(capture_text); |
| 4462 | + Ok((compiler, artifacts)) |
| 4463 | +} |
| 2571 | 4464 | |
| 2572 | | - let mut cli_runs = Vec::new(); |
| 2573 | | - let mut mismatch_indices = Vec::new(); |
| 2574 | | - for index in 0..repeat_count { |
| 2575 | | - let asm_path = temp_root.join(format!("cli_run_{:02}.s", index)); |
| 2576 | | - let command = |
| 2577 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 2578 | | - Ok(command) => command, |
| 2579 | | - Err(detail) => { |
| 2580 | | - return Some(ConsistencyIssue { |
| 2581 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2582 | | - summary: "armfortas -S failed during capture-vs-cli consistency check" |
| 2583 | | - .into(), |
| 2584 | | - repeat_count: None, |
| 2585 | | - unique_variant_count: None, |
| 2586 | | - varying_components: Vec::new(), |
| 2587 | | - stable_components: Vec::new(), |
| 2588 | | - detail, |
| 2589 | | - temp_root, |
| 2590 | | - }) |
| 4465 | +fn parse_compare_declaration( |
| 4466 | + rest: &str, |
| 4467 | + path: &Path, |
| 4468 | + line_no: usize, |
| 4469 | +) -> Result<(CompilerSpec, CompilerSpec, BTreeSet<ArtifactKey>), String> { |
| 4470 | + let (compilers_raw, artifacts_raw) = rest.split_once("=>").ok_or_else(|| { |
| 4471 | + format!( |
| 4472 | + "{}:{}: compare declaration must use 'compare <left> <right> => <artifacts>'", |
| 4473 | + path.display(), |
| 4474 | + line_no |
| 4475 | + ) |
| 4476 | + })?; |
| 4477 | + let tokens = split_compiler_tokens(compilers_raw.trim(), path, line_no)?; |
| 4478 | + if tokens.len() != 2 { |
| 4479 | + return Err(format!( |
| 4480 | + "{}:{}: compare declaration requires exactly two compiler specs", |
| 4481 | + path.display(), |
| 4482 | + line_no |
| 4483 | + )); |
| 4484 | + } |
| 4485 | + let left = parse_compiler_spec_token(&tokens[0], path, line_no)?; |
| 4486 | + let right = parse_compiler_spec_token(&tokens[1], path, line_no)?; |
| 4487 | + let artifacts = ArtifactKey::parse_list(artifacts_raw.trim()) |
| 4488 | + .map_err(|err| format!("{}:{}: {}", path.display(), line_no, err))?; |
| 4489 | + Ok((left, right, artifacts)) |
| 4490 | +} |
| 4491 | + |
| 4492 | +fn split_compiler_tokens(raw: &str, path: &Path, line_no: usize) -> Result<Vec<String>, String> { |
| 4493 | + let mut tokens = Vec::new(); |
| 4494 | + let mut current = String::new(); |
| 4495 | + let mut quoted = false; |
| 4496 | + |
| 4497 | + for ch in raw.chars() { |
| 4498 | + match ch { |
| 4499 | + '"' => { |
| 4500 | + quoted = !quoted; |
| 4501 | + current.push(ch); |
| 4502 | + } |
| 4503 | + c if c.is_whitespace() && !quoted => { |
| 4504 | + if !current.trim().is_empty() { |
| 4505 | + tokens.push(current.trim().to_string()); |
| 4506 | + current.clear(); |
| 2591 | 4507 | } |
| 2592 | | - }; |
| 2593 | | - let text = match read_text_artifact(&asm_path) { |
| 2594 | | - Ok(text) => text, |
| 2595 | | - Err(detail) => { |
| 2596 | | - return Some(ConsistencyIssue { |
| 2597 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2598 | | - summary: "could not read cli assembly artifact".into(), |
| 2599 | | - repeat_count: None, |
| 2600 | | - unique_variant_count: None, |
| 2601 | | - varying_components: Vec::new(), |
| 2602 | | - stable_components: Vec::new(), |
| 2603 | | - detail, |
| 2604 | | - temp_root, |
| 2605 | | - }) |
| 2606 | 4508 | } |
| 2607 | | - }; |
| 2608 | | - let normalized = normalize_text_artifact(&text); |
| 2609 | | - if normalized != capture_normalized { |
| 2610 | | - mismatch_indices.push(index); |
| 4509 | + other => current.push(other), |
| 2611 | 4510 | } |
| 2612 | | - cli_runs.push(TextRun { |
| 2613 | | - label: format!("cli run {} (-S)", index + 1), |
| 2614 | | - command, |
| 2615 | | - normalized, |
| 2616 | | - }); |
| 2617 | 4511 | } |
| 2618 | 4512 | |
| 2619 | | - if !mismatch_indices.is_empty() { |
| 2620 | | - let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 2621 | | - let unique_cli_variants = |
| 2622 | | - count_unique_strings(cli_runs.iter().map(|run| run.normalized.as_str())); |
| 2623 | | - let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 2624 | | - return Some(ConsistencyIssue { |
| 2625 | | - check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 2626 | | - summary: format!( |
| 2627 | | - "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={}", |
| 2628 | | - repeat_count, |
| 2629 | | - matching_runs, |
| 2630 | | - mismatch_indices.len(), |
| 2631 | | - unique_cli_variants |
| 2632 | | - ), |
| 2633 | | - repeat_count: Some(repeat_count), |
| 2634 | | - unique_variant_count: Some(unique_cli_variants), |
| 2635 | | - varying_components: Vec::new(), |
| 2636 | | - stable_components: Vec::new(), |
| 2637 | | - detail: format!( |
| 2638 | | - "captured assembly does not match repeated armfortas -S runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 2639 | | - repeat_count, |
| 2640 | | - matching_runs, |
| 2641 | | - mismatch_indices.len(), |
| 2642 | | - unique_cli_variants, |
| 2643 | | - capture_command, |
| 2644 | | - first_mismatch.command, |
| 2645 | | - describe_text_difference( |
| 2646 | | - &capture_normalized, |
| 2647 | | - &first_mismatch.normalized, |
| 2648 | | - "capture asm", |
| 2649 | | - &first_mismatch.label |
| 2650 | | - ) |
| 2651 | | - ), |
| 2652 | | - temp_root, |
| 2653 | | - }); |
| 4513 | + if quoted { |
| 4514 | + return Err(format!( |
| 4515 | + "{}:{}: unterminated quoted compiler spec '{}'", |
| 4516 | + path.display(), |
| 4517 | + line_no, |
| 4518 | + raw |
| 4519 | + )); |
| 2654 | 4520 | } |
| 2655 | 4521 | |
| 2656 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2657 | | - None |
| 2658 | | -} |
| 2659 | | - |
| 2660 | | -fn run_capture_obj_vs_cli_obj( |
| 2661 | | - source: &Path, |
| 2662 | | - opt_level: OptLevel, |
| 2663 | | - repeat_count: usize, |
| 2664 | | - capture_result: &CaptureResult, |
| 2665 | | - tools: &ToolchainConfig, |
| 2666 | | -) -> Option<ConsistencyIssue> { |
| 2667 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2668 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2669 | | - return Some(ConsistencyIssue { |
| 2670 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2671 | | - summary: "could not create consistency temp dir".into(), |
| 2672 | | - repeat_count: None, |
| 2673 | | - unique_variant_count: None, |
| 2674 | | - varying_components: Vec::new(), |
| 2675 | | - stable_components: Vec::new(), |
| 2676 | | - detail: format!( |
| 2677 | | - "cannot create consistency temp dir '{}': {}", |
| 2678 | | - temp_root.display(), |
| 2679 | | - err |
| 2680 | | - ), |
| 2681 | | - temp_root, |
| 2682 | | - }); |
| 4522 | + if !current.trim().is_empty() { |
| 4523 | + tokens.push(current.trim().to_string()); |
| 2683 | 4524 | } |
| 2684 | 4525 | |
| 2685 | | - let capture_command = render_capture_command(source, opt_level, Stage::Obj); |
| 2686 | | - let capture_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 2687 | | - Ok(text) => text, |
| 2688 | | - Err(detail) => { |
| 2689 | | - return Some(ConsistencyIssue { |
| 2690 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2691 | | - summary: "capture result did not include object snapshot text".into(), |
| 2692 | | - repeat_count: None, |
| 2693 | | - unique_variant_count: None, |
| 2694 | | - varying_components: Vec::new(), |
| 2695 | | - stable_components: Vec::new(), |
| 2696 | | - detail, |
| 2697 | | - temp_root, |
| 2698 | | - }) |
| 2699 | | - } |
| 4526 | + Ok(tokens) |
| 4527 | +} |
| 4528 | + |
| 4529 | +fn parse_compiler_spec_token( |
| 4530 | + raw: &str, |
| 4531 | + path: &Path, |
| 4532 | + line_no: usize, |
| 4533 | +) -> Result<CompilerSpec, String> { |
| 4534 | + let token = if raw.starts_with('"') { |
| 4535 | + parse_quoted(raw, path, line_no)? |
| 4536 | + } else { |
| 4537 | + raw.trim().to_string() |
| 2700 | 4538 | }; |
| 2701 | | - if let Err(err) = fs::write(temp_root.join("from_capture.obj.txt"), capture_text) { |
| 2702 | | - return Some(ConsistencyIssue { |
| 2703 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2704 | | - summary: "could not write captured object snapshot artifact".into(), |
| 2705 | | - repeat_count: None, |
| 2706 | | - unique_variant_count: None, |
| 2707 | | - varying_components: Vec::new(), |
| 2708 | | - stable_components: Vec::new(), |
| 2709 | | - detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 2710 | | - temp_root, |
| 2711 | | - }); |
| 4539 | + if token.is_empty() { |
| 4540 | + return Err(format!( |
| 4541 | + "{}:{}: compiler declaration is missing a compiler spec", |
| 4542 | + path.display(), |
| 4543 | + line_no |
| 4544 | + )); |
| 2712 | 4545 | } |
| 2713 | | - let capture_snapshot = match parse_object_snapshot_text(capture_text) { |
| 2714 | | - Ok(snapshot) => snapshot, |
| 2715 | | - Err(detail) => { |
| 2716 | | - return Some(ConsistencyIssue { |
| 2717 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2718 | | - summary: "captured object snapshot had an unexpected format".into(), |
| 2719 | | - repeat_count: None, |
| 2720 | | - unique_variant_count: None, |
| 2721 | | - varying_components: Vec::new(), |
| 2722 | | - stable_components: Vec::new(), |
| 2723 | | - detail, |
| 2724 | | - temp_root, |
| 2725 | | - }) |
| 2726 | | - } |
| 4546 | + if let Some(named) = NamedCompiler::parse(&token) { |
| 4547 | + return Ok(CompilerSpec::Named(named)); |
| 4548 | + } |
| 4549 | + let parsed = PathBuf::from(&token); |
| 4550 | + let resolved = if parsed.is_absolute() { |
| 4551 | + parsed |
| 4552 | + } else { |
| 4553 | + path.parent().unwrap_or_else(|| Path::new(".")).join(parsed) |
| 2727 | 4554 | }; |
| 4555 | + Ok(CompilerSpec::Binary(resolved)) |
| 4556 | +} |
| 2728 | 4557 | |
| 2729 | | - let mut cli_runs = Vec::new(); |
| 2730 | | - let mut mismatch_indices = Vec::new(); |
| 2731 | | - for index in 0..repeat_count { |
| 2732 | | - let obj_path = temp_root.join(format!("cli_run_{:02}.o", index)); |
| 2733 | | - let command = |
| 2734 | | - match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 2735 | | - Ok(command) => command, |
| 2736 | | - Err(detail) => { |
| 2737 | | - return Some(ConsistencyIssue { |
| 2738 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2739 | | - summary: "armfortas -c failed during capture-vs-cli consistency check" |
| 2740 | | - .into(), |
| 2741 | | - repeat_count: None, |
| 2742 | | - unique_variant_count: None, |
| 2743 | | - varying_components: Vec::new(), |
| 2744 | | - stable_components: Vec::new(), |
| 2745 | | - detail, |
| 2746 | | - temp_root, |
| 2747 | | - }) |
| 2748 | | - } |
| 2749 | | - }; |
| 2750 | | - let snapshot = match object_snapshot(&obj_path, tools) { |
| 2751 | | - Ok(snapshot) => snapshot, |
| 2752 | | - Err(detail) => { |
| 2753 | | - return Some(ConsistencyIssue { |
| 2754 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2755 | | - summary: "could not snapshot cli object artifact".into(), |
| 2756 | | - repeat_count: None, |
| 2757 | | - unique_variant_count: None, |
| 2758 | | - varying_components: Vec::new(), |
| 2759 | | - stable_components: Vec::new(), |
| 2760 | | - detail: format!("{}\n{}", command, detail), |
| 2761 | | - temp_root, |
| 2762 | | - }) |
| 2763 | | - } |
| 2764 | | - }; |
| 2765 | | - if let Err(err) = fs::write( |
| 2766 | | - temp_root.join(format!("cli_run_{:02}.obj.txt", index)), |
| 2767 | | - render_object_snapshot(&snapshot), |
| 2768 | | - ) { |
| 2769 | | - return Some(ConsistencyIssue { |
| 2770 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2771 | | - summary: "could not write cli object snapshot artifact".into(), |
| 2772 | | - repeat_count: None, |
| 2773 | | - unique_variant_count: None, |
| 2774 | | - varying_components: Vec::new(), |
| 2775 | | - stable_components: Vec::new(), |
| 2776 | | - detail: format!("cannot write cli object snapshot artifact: {}", err), |
| 2777 | | - temp_root, |
| 2778 | | - }); |
| 4558 | +fn parse_opt_levels(rest: &str, path: &Path, line_no: usize) -> Result<Vec<OptLevel>, String> { |
| 4559 | + let mut levels = BTreeSet::new(); |
| 4560 | + for raw in rest.split(',') { |
| 4561 | + let name = raw.trim(); |
| 4562 | + if name.is_empty() { |
| 4563 | + continue; |
| 2779 | 4564 | } |
| 2780 | | - if snapshot != capture_snapshot { |
| 2781 | | - mismatch_indices.push(index); |
| 4565 | + if name.eq_ignore_ascii_case("all") { |
| 4566 | + levels.extend(all_opt_levels()); |
| 4567 | + continue; |
| 2782 | 4568 | } |
| 2783 | | - cli_runs.push(ObjectRun { |
| 2784 | | - label: format!("cli run {} (-c)", index + 1), |
| 2785 | | - command, |
| 2786 | | - snapshot, |
| 2787 | | - }); |
| 4569 | + let level = parse_opt_level_token(name).ok_or_else(|| { |
| 4570 | + format!( |
| 4571 | + "{}:{}: unknown opt level '{}'", |
| 4572 | + path.display(), |
| 4573 | + line_no, |
| 4574 | + name |
| 4575 | + ) |
| 4576 | + })?; |
| 4577 | + levels.insert(level); |
| 2788 | 4578 | } |
| 2789 | | - |
| 2790 | | - if !mismatch_indices.is_empty() { |
| 2791 | | - let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 2792 | | - let rendered = cli_runs |
| 2793 | | - .iter() |
| 2794 | | - .map(|run| render_object_snapshot(&run.snapshot)) |
| 2795 | | - .collect::<Vec<_>>(); |
| 2796 | | - let unique_cli_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 2797 | | - let mismatch_snapshots = mismatch_indices |
| 2798 | | - .iter() |
| 2799 | | - .map(|index| &cli_runs[*index].snapshot) |
| 2800 | | - .collect::<Vec<_>>(); |
| 2801 | | - let mut summary_snapshots = vec![&capture_snapshot]; |
| 2802 | | - summary_snapshots.extend(mismatch_snapshots.iter().copied()); |
| 2803 | | - let varying = varying_object_components(&summary_snapshots) |
| 2804 | | - .into_iter() |
| 2805 | | - .map(str::to_string) |
| 2806 | | - .collect::<Vec<_>>(); |
| 2807 | | - let stable = stable_object_components(&summary_snapshots) |
| 2808 | | - .into_iter() |
| 2809 | | - .map(str::to_string) |
| 2810 | | - .collect::<Vec<_>>(); |
| 2811 | | - let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 2812 | | - return Some(ConsistencyIssue { |
| 2813 | | - check: ConsistencyCheck::CaptureObjVsCliObj, |
| 2814 | | - summary: format!( |
| 2815 | | - "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 2816 | | - repeat_count, |
| 2817 | | - matching_runs, |
| 2818 | | - mismatch_indices.len(), |
| 2819 | | - unique_cli_variants, |
| 2820 | | - join_or_none_from_strings(&varying), |
| 2821 | | - join_or_none_from_strings(&stable) |
| 2822 | | - ), |
| 2823 | | - repeat_count: Some(repeat_count), |
| 2824 | | - unique_variant_count: Some(unique_cli_variants), |
| 2825 | | - varying_components: varying, |
| 2826 | | - stable_components: stable, |
| 2827 | | - detail: format!( |
| 2828 | | - "captured object snapshot does not match repeated armfortas -c runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 2829 | | - repeat_count, |
| 2830 | | - matching_runs, |
| 2831 | | - mismatch_indices.len(), |
| 2832 | | - unique_cli_variants, |
| 2833 | | - capture_command, |
| 2834 | | - first_mismatch.command, |
| 2835 | | - describe_object_difference( |
| 2836 | | - &capture_snapshot, |
| 2837 | | - &first_mismatch.snapshot, |
| 2838 | | - "capture obj", |
| 2839 | | - &first_mismatch.label |
| 2840 | | - ) |
| 2841 | | - ), |
| 2842 | | - temp_root, |
| 2843 | | - }); |
| 4579 | + if levels.is_empty() { |
| 4580 | + return Err(format!( |
| 4581 | + "{}:{}: opt level list is empty", |
| 4582 | + path.display(), |
| 4583 | + line_no |
| 4584 | + )); |
| 2844 | 4585 | } |
| 4586 | + Ok(levels.into_iter().collect()) |
| 4587 | +} |
| 2845 | 4588 | |
| 2846 | | - let _ = fs::remove_dir_all(&temp_root); |
| 2847 | | - None |
| 4589 | +fn parse_reference_compilers( |
| 4590 | + rest: &str, |
| 4591 | + path: &Path, |
| 4592 | + line_no: usize, |
| 4593 | +) -> Result<Vec<ReferenceCompiler>, String> { |
| 4594 | + let mut compilers = BTreeSet::new(); |
| 4595 | + for raw in rest.split(',') { |
| 4596 | + let name = raw.trim(); |
| 4597 | + if name.is_empty() { |
| 4598 | + continue; |
| 4599 | + } |
| 4600 | + let compiler = ReferenceCompiler::parse(name).ok_or_else(|| { |
| 4601 | + format!( |
| 4602 | + "{}:{}: unknown reference compiler '{}'", |
| 4603 | + path.display(), |
| 4604 | + line_no, |
| 4605 | + name |
| 4606 | + ) |
| 4607 | + })?; |
| 4608 | + compilers.insert(compiler); |
| 4609 | + } |
| 4610 | + if compilers.is_empty() { |
| 4611 | + return Err(format!( |
| 4612 | + "{}:{}: differential compiler list is empty", |
| 4613 | + path.display(), |
| 4614 | + line_no |
| 4615 | + )); |
| 4616 | + } |
| 4617 | + Ok(compilers.into_iter().collect()) |
| 2848 | 4618 | } |
| 2849 | 4619 | |
| 2850 | | -fn run_capture_run_vs_cli_run( |
| 2851 | | - source: &Path, |
| 2852 | | - opt_level: OptLevel, |
| 2853 | | - repeat_count: usize, |
| 2854 | | - capture_result: &CaptureResult, |
| 2855 | | - tools: &ToolchainConfig, |
| 2856 | | -) -> Option<ConsistencyIssue> { |
| 2857 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 2858 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 2859 | | - return Some(ConsistencyIssue { |
| 2860 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2861 | | - summary: "could not create consistency temp dir".into(), |
| 2862 | | - repeat_count: None, |
| 2863 | | - unique_variant_count: None, |
| 2864 | | - varying_components: Vec::new(), |
| 2865 | | - stable_components: Vec::new(), |
| 2866 | | - detail: format!( |
| 2867 | | - "cannot create consistency temp dir '{}': {}", |
| 2868 | | - temp_root.display(), |
| 2869 | | - err |
| 2870 | | - ), |
| 2871 | | - temp_root, |
| 2872 | | - }); |
| 4620 | +fn parse_repeat_count(rest: &str, path: &Path, line_no: usize) -> Result<usize, String> { |
| 4621 | + let count = rest.trim().parse::<usize>().map_err(|_| { |
| 4622 | + format!( |
| 4623 | + "{}:{}: repeat count must be an integer >= 2", |
| 4624 | + path.display(), |
| 4625 | + line_no |
| 4626 | + ) |
| 4627 | + })?; |
| 4628 | + if count < 2 { |
| 4629 | + return Err(format!( |
| 4630 | + "{}:{}: repeat count must be >= 2", |
| 4631 | + path.display(), |
| 4632 | + line_no |
| 4633 | + )); |
| 2873 | 4634 | } |
| 4635 | + Ok(count) |
| 4636 | +} |
| 2874 | 4637 | |
| 2875 | | - let capture_command = render_capture_command(source, opt_level, Stage::Run); |
| 2876 | | - let capture_run = match capture_run_stage(capture_result) { |
| 2877 | | - Ok(run) => run.clone(), |
| 2878 | | - Err(detail) => { |
| 2879 | | - return Some(ConsistencyIssue { |
| 2880 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2881 | | - summary: "capture result did not include runtime behavior".into(), |
| 2882 | | - repeat_count: None, |
| 2883 | | - unique_variant_count: None, |
| 2884 | | - varying_components: Vec::new(), |
| 2885 | | - stable_components: Vec::new(), |
| 2886 | | - detail, |
| 2887 | | - temp_root, |
| 2888 | | - }) |
| 4638 | +fn parse_consistency_checks( |
| 4639 | + rest: &str, |
| 4640 | + path: &Path, |
| 4641 | + line_no: usize, |
| 4642 | +) -> Result<Vec<ConsistencyCheck>, String> { |
| 4643 | + let mut checks = Vec::new(); |
| 4644 | + for raw in rest.split(',') { |
| 4645 | + let name = raw.trim(); |
| 4646 | + if name.is_empty() { |
| 4647 | + continue; |
| 4648 | + } |
| 4649 | + let check = ConsistencyCheck::parse(name).ok_or_else(|| { |
| 4650 | + format!( |
| 4651 | + "{}:{}: unknown consistency check '{}'", |
| 4652 | + path.display(), |
| 4653 | + line_no, |
| 4654 | + name |
| 4655 | + ) |
| 4656 | + })?; |
| 4657 | + if !checks.contains(&check) { |
| 4658 | + checks.push(check); |
| 2889 | 4659 | } |
| 2890 | | - }; |
| 2891 | | - if let Err(err) = |
| 2892 | | - write_behavior_run_artifacts(&temp_root, "from_capture", &capture_command, &capture_run) |
| 2893 | | - { |
| 2894 | | - return Some(ConsistencyIssue { |
| 2895 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2896 | | - summary: "could not write captured runtime artifact".into(), |
| 2897 | | - repeat_count: None, |
| 2898 | | - unique_variant_count: None, |
| 2899 | | - varying_components: Vec::new(), |
| 2900 | | - stable_components: Vec::new(), |
| 2901 | | - detail: format!("cannot write captured runtime artifact: {}", err), |
| 2902 | | - temp_root, |
| 2903 | | - }); |
| 2904 | 4660 | } |
| 2905 | | - let capture_signature = normalize_run_signature(&capture_run); |
| 4661 | + if checks.is_empty() { |
| 4662 | + return Err(format!( |
| 4663 | + "{}:{}: consistency check list is empty", |
| 4664 | + path.display(), |
| 4665 | + line_no |
| 4666 | + )); |
| 4667 | + } |
| 4668 | + Ok(checks) |
| 4669 | +} |
| 2906 | 4670 | |
| 2907 | | - let mut cli_runs = Vec::new(); |
| 2908 | | - let mut mismatch_indices = Vec::new(); |
| 2909 | | - for index in 0..repeat_count { |
| 2910 | | - let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 2911 | | - let build_command = match compile_with_driver( |
| 2912 | | - source, |
| 2913 | | - opt_level, |
| 2914 | | - DriverEmitMode::Binary, |
| 2915 | | - &binary_path, |
| 2916 | | - tools, |
| 2917 | | - ) { |
| 2918 | | - Ok(command) => command, |
| 2919 | | - Err(detail) => { |
| 2920 | | - return Some(ConsistencyIssue { |
| 2921 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2922 | | - summary: "armfortas binary build failed during capture-vs-cli runtime check" |
| 2923 | | - .into(), |
| 2924 | | - repeat_count: None, |
| 2925 | | - unique_variant_count: None, |
| 2926 | | - varying_components: Vec::new(), |
| 2927 | | - stable_components: Vec::new(), |
| 2928 | | - detail, |
| 2929 | | - temp_root, |
| 2930 | | - }) |
| 2931 | | - } |
| 2932 | | - }; |
| 2933 | | - let run_command = render_binary_run_command(&binary_path); |
| 2934 | | - let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 2935 | | - Ok(run) => run, |
| 2936 | | - Err(detail) => { |
| 2937 | | - return Some(ConsistencyIssue { |
| 2938 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2939 | | - summary: "armfortas binary could not run during capture-vs-cli runtime check" |
| 2940 | | - .into(), |
| 2941 | | - repeat_count: None, |
| 2942 | | - unique_variant_count: None, |
| 2943 | | - varying_components: Vec::new(), |
| 2944 | | - stable_components: Vec::new(), |
| 2945 | | - detail, |
| 2946 | | - temp_root, |
| 2947 | | - }) |
| 2948 | | - } |
| 2949 | | - }; |
| 2950 | | - let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 2951 | | - if let Err(err) = write_behavior_run_artifacts( |
| 2952 | | - &temp_root, |
| 2953 | | - &format!("cli_run_{:02}", index), |
| 2954 | | - &command, |
| 2955 | | - &run, |
| 2956 | | - ) { |
| 2957 | | - return Some(ConsistencyIssue { |
| 2958 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 2959 | | - summary: "could not write cli runtime artifact".into(), |
| 2960 | | - repeat_count: None, |
| 2961 | | - unique_variant_count: None, |
| 2962 | | - varying_components: Vec::new(), |
| 2963 | | - stable_components: Vec::new(), |
| 2964 | | - detail: format!("cannot write cli runtime artifact: {}", err), |
| 2965 | | - temp_root, |
| 2966 | | - }); |
| 2967 | | - } |
| 2968 | | - if normalize_run_signature(&run) != capture_signature { |
| 2969 | | - mismatch_indices.push(index); |
| 2970 | | - } |
| 2971 | | - cli_runs.push(BehaviorRun { |
| 2972 | | - label: format!("cli run {}", index + 1), |
| 2973 | | - command, |
| 2974 | | - signature: normalize_run_signature(&run), |
| 2975 | | - run, |
| 2976 | | - }); |
| 4671 | +fn parse_expectation(rest: &str, path: &Path, line_no: usize) -> Result<Expectation, String> { |
| 4672 | + if let Some(prefix) = rest.strip_suffix(" check-comments") { |
| 4673 | + return Ok(Expectation::CheckComments(parse_target( |
| 4674 | + prefix.trim(), |
| 4675 | + path, |
| 4676 | + line_no, |
| 4677 | + )?)); |
| 2977 | 4678 | } |
| 2978 | 4679 | |
| 2979 | | - if !mismatch_indices.is_empty() { |
| 2980 | | - let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 2981 | | - let unique_cli_variants = |
| 2982 | | - count_unique_run_signatures(cli_runs.iter().map(|run| &run.signature)); |
| 2983 | | - let mismatch_signatures = mismatch_indices |
| 2984 | | - .iter() |
| 2985 | | - .map(|index| &cli_runs[*index].signature) |
| 2986 | | - .collect::<Vec<_>>(); |
| 2987 | | - let mut summary_signatures = vec![&capture_signature]; |
| 2988 | | - summary_signatures.extend(mismatch_signatures.iter().copied()); |
| 2989 | | - let varying = varying_run_components(&summary_signatures) |
| 2990 | | - .into_iter() |
| 2991 | | - .map(str::to_string) |
| 2992 | | - .collect::<Vec<_>>(); |
| 2993 | | - let stable = stable_run_components(&summary_signatures) |
| 2994 | | - .into_iter() |
| 2995 | | - .map(str::to_string) |
| 2996 | | - .collect::<Vec<_>>(); |
| 2997 | | - let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 2998 | | - return Some(ConsistencyIssue { |
| 2999 | | - check: ConsistencyCheck::CaptureRunVsCliRun, |
| 3000 | | - summary: format!( |
| 3001 | | - "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 3002 | | - repeat_count, |
| 3003 | | - matching_runs, |
| 3004 | | - mismatch_indices.len(), |
| 3005 | | - unique_cli_variants, |
| 3006 | | - join_or_none_from_strings(&varying), |
| 3007 | | - join_or_none_from_strings(&stable) |
| 3008 | | - ), |
| 3009 | | - repeat_count: Some(repeat_count), |
| 3010 | | - unique_variant_count: Some(unique_cli_variants), |
| 3011 | | - varying_components: varying, |
| 3012 | | - stable_components: stable, |
| 3013 | | - detail: format!( |
| 3014 | | - "captured runtime behavior does not match repeated full CLI builds\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 3015 | | - repeat_count, |
| 3016 | | - matching_runs, |
| 3017 | | - mismatch_indices.len(), |
| 3018 | | - unique_cli_variants, |
| 3019 | | - capture_command, |
| 3020 | | - first_mismatch.command, |
| 3021 | | - describe_run_difference( |
| 3022 | | - &capture_run, |
| 3023 | | - &first_mismatch.run, |
| 3024 | | - "capture run", |
| 3025 | | - &first_mismatch.label |
| 3026 | | - ) |
| 3027 | | - ), |
| 3028 | | - temp_root, |
| 4680 | + if let Some((target, value)) = rest.split_once(" not-contains ") { |
| 4681 | + return Ok(Expectation::NotContains { |
| 4682 | + target: parse_target(target.trim(), path, line_no)?, |
| 4683 | + needle: parse_quoted(value.trim(), path, line_no)?, |
| 3029 | 4684 | }); |
| 3030 | 4685 | } |
| 3031 | 4686 | |
| 3032 | | - let _ = fs::remove_dir_all(&temp_root); |
| 3033 | | - None |
| 3034 | | -} |
| 3035 | | - |
| 3036 | | -fn run_capture_asm_reproducible( |
| 3037 | | - source: &Path, |
| 3038 | | - opt_level: OptLevel, |
| 3039 | | - repeat_count: usize, |
| 3040 | | - capture_result: &CaptureResult, |
| 3041 | | - _tools: &ToolchainConfig, |
| 3042 | | -) -> Option<ConsistencyIssue> { |
| 3043 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 3044 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 3045 | | - return Some(ConsistencyIssue { |
| 3046 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3047 | | - summary: "could not create consistency temp dir".into(), |
| 3048 | | - repeat_count: None, |
| 3049 | | - unique_variant_count: None, |
| 3050 | | - varying_components: Vec::new(), |
| 3051 | | - stable_components: Vec::new(), |
| 3052 | | - detail: format!( |
| 3053 | | - "cannot create consistency temp dir '{}': {}", |
| 3054 | | - temp_root.display(), |
| 3055 | | - err |
| 3056 | | - ), |
| 3057 | | - temp_root, |
| 4687 | + if let Some((target, value)) = rest.split_once(" contains ") { |
| 4688 | + return Ok(Expectation::Contains { |
| 4689 | + target: parse_target(target.trim(), path, line_no)?, |
| 4690 | + needle: parse_quoted(value.trim(), path, line_no)?, |
| 3058 | 4691 | }); |
| 3059 | 4692 | } |
| 3060 | 4693 | |
| 3061 | | - let mut runs = Vec::new(); |
| 3062 | | - let command = render_capture_command(source, opt_level, Stage::Asm); |
| 3063 | | - let initial_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 3064 | | - Ok(text) => text, |
| 3065 | | - Err(detail) => { |
| 3066 | | - return Some(ConsistencyIssue { |
| 3067 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3068 | | - summary: "initial capture result did not include assembly text".into(), |
| 3069 | | - repeat_count: None, |
| 3070 | | - unique_variant_count: None, |
| 3071 | | - varying_components: Vec::new(), |
| 3072 | | - stable_components: Vec::new(), |
| 3073 | | - detail, |
| 3074 | | - temp_root, |
| 3075 | | - }) |
| 4694 | + if let Some((target, value)) = rest.split_once(" equals ") { |
| 4695 | + let target = parse_target(target.trim(), path, line_no)?; |
| 4696 | + if matches!( |
| 4697 | + target, |
| 4698 | + Target::RunExitCode |
| 4699 | + | Target::Artifact(ArtifactKey::ExitCode) |
| 4700 | + | Target::CompareDifferenceCount |
| 4701 | + ) { |
| 4702 | + let value = parse_integer(value.trim(), path, line_no)?; |
| 4703 | + return Ok(Expectation::IntEquals { target, value }); |
| 3076 | 4704 | } |
| 3077 | | - }; |
| 3078 | | - if let Err(err) = fs::write(temp_root.join("capture_run_00.s"), initial_text) { |
| 3079 | | - return Some(ConsistencyIssue { |
| 3080 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3081 | | - summary: "could not write captured assembly artifact".into(), |
| 3082 | | - repeat_count: None, |
| 3083 | | - unique_variant_count: None, |
| 3084 | | - varying_components: Vec::new(), |
| 3085 | | - stable_components: Vec::new(), |
| 3086 | | - detail: format!("cannot write captured assembly artifact: {}", err), |
| 3087 | | - temp_root, |
| 4705 | + return Ok(Expectation::Equals { |
| 4706 | + target, |
| 4707 | + value: parse_quoted(value.trim(), path, line_no)?, |
| 3088 | 4708 | }); |
| 3089 | 4709 | } |
| 3090 | | - runs.push(TextRun { |
| 3091 | | - label: "capture run 1".into(), |
| 3092 | | - command: command.clone(), |
| 3093 | | - normalized: normalize_text_artifact(initial_text), |
| 3094 | | - }); |
| 3095 | 4710 | |
| 3096 | | - for index in 1..repeat_count { |
| 3097 | | - let text = match capture_text_from_testing(source, opt_level, Stage::Asm) { |
| 3098 | | - Ok(text) => text, |
| 3099 | | - Err(detail) => { |
| 3100 | | - return Some(ConsistencyIssue { |
| 3101 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3102 | | - summary: "armfortas::testing capture failed during asm reproducibility check" |
| 3103 | | - .into(), |
| 3104 | | - repeat_count: None, |
| 3105 | | - unique_variant_count: None, |
| 3106 | | - varying_components: Vec::new(), |
| 3107 | | - stable_components: Vec::new(), |
| 3108 | | - detail, |
| 3109 | | - temp_root, |
| 3110 | | - }) |
| 3111 | | - } |
| 3112 | | - }; |
| 3113 | | - if let Err(err) = fs::write(temp_root.join(format!("capture_run_{:02}.s", index)), &text) { |
| 3114 | | - return Some(ConsistencyIssue { |
| 3115 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3116 | | - summary: "could not write captured assembly artifact".into(), |
| 3117 | | - repeat_count: None, |
| 3118 | | - unique_variant_count: None, |
| 3119 | | - varying_components: Vec::new(), |
| 3120 | | - stable_components: Vec::new(), |
| 3121 | | - detail: format!("cannot write captured assembly artifact: {}", err), |
| 3122 | | - temp_root, |
| 3123 | | - }); |
| 3124 | | - } |
| 3125 | | - runs.push(TextRun { |
| 3126 | | - label: format!("capture run {}", index + 1), |
| 3127 | | - command: command.clone(), |
| 3128 | | - normalized: normalize_text_artifact(&text), |
| 3129 | | - }); |
| 4711 | + Err(format!( |
| 4712 | + "{}:{}: unsupported expectation '{}'", |
| 4713 | + path.display(), |
| 4714 | + line_no, |
| 4715 | + rest |
| 4716 | + )) |
| 4717 | +} |
| 4718 | + |
| 4719 | +fn parse_failure_expectation( |
| 4720 | + rest: &str, |
| 4721 | + path: &Path, |
| 4722 | + line_no: usize, |
| 4723 | +) -> Result<Expectation, String> { |
| 4724 | + if rest.trim().eq_ignore_ascii_case("comments") { |
| 4725 | + return Ok(Expectation::FailSourceComments); |
| 3130 | 4726 | } |
| 3131 | 4727 | |
| 3132 | | - let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 3133 | | - if unique_variants > 1 { |
| 3134 | | - let (left, right) = |
| 3135 | | - first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 3136 | | - return Some(ConsistencyIssue { |
| 3137 | | - check: ConsistencyCheck::CaptureAsmReproducible, |
| 3138 | | - summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 3139 | | - repeat_count: Some(repeat_count), |
| 3140 | | - unique_variant_count: Some(unique_variants), |
| 3141 | | - varying_components: Vec::new(), |
| 3142 | | - stable_components: Vec::new(), |
| 3143 | | - detail: format!( |
| 3144 | | - "captured assembly is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 3145 | | - repeat_count, |
| 3146 | | - unique_variants, |
| 3147 | | - left.command, |
| 3148 | | - right.command, |
| 3149 | | - describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 3150 | | - ), |
| 3151 | | - temp_root, |
| 4728 | + if let Some((target, value)) = rest.split_once(" contains ") { |
| 4729 | + return Ok(Expectation::FailContains { |
| 4730 | + stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4731 | + needle: parse_quoted(value.trim(), path, line_no)?, |
| 3152 | 4732 | }); |
| 3153 | 4733 | } |
| 3154 | 4734 | |
| 3155 | | - let _ = fs::remove_dir_all(&temp_root); |
| 3156 | | - None |
| 3157 | | -} |
| 3158 | | - |
| 3159 | | -fn run_capture_obj_reproducible( |
| 3160 | | - source: &Path, |
| 3161 | | - opt_level: OptLevel, |
| 3162 | | - repeat_count: usize, |
| 3163 | | - capture_result: &CaptureResult, |
| 3164 | | - _tools: &ToolchainConfig, |
| 3165 | | -) -> Option<ConsistencyIssue> { |
| 3166 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 3167 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 3168 | | - return Some(ConsistencyIssue { |
| 3169 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3170 | | - summary: "could not create consistency temp dir".into(), |
| 3171 | | - repeat_count: None, |
| 3172 | | - unique_variant_count: None, |
| 3173 | | - varying_components: Vec::new(), |
| 3174 | | - stable_components: Vec::new(), |
| 3175 | | - detail: format!( |
| 3176 | | - "cannot create consistency temp dir '{}': {}", |
| 3177 | | - temp_root.display(), |
| 3178 | | - err |
| 3179 | | - ), |
| 3180 | | - temp_root, |
| 4735 | + if let Some((target, value)) = rest.split_once(" equals ") { |
| 4736 | + return Ok(Expectation::FailEquals { |
| 4737 | + stage: parse_failure_stage(target.trim(), path, line_no)?, |
| 4738 | + value: parse_quoted(value.trim(), path, line_no)?, |
| 3181 | 4739 | }); |
| 3182 | 4740 | } |
| 3183 | 4741 | |
| 3184 | | - let command = render_capture_command(source, opt_level, Stage::Obj); |
| 3185 | | - let initial_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 3186 | | - Ok(text) => text, |
| 3187 | | - Err(detail) => { |
| 3188 | | - return Some(ConsistencyIssue { |
| 3189 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3190 | | - summary: "initial capture result did not include object snapshot text".into(), |
| 3191 | | - repeat_count: None, |
| 3192 | | - unique_variant_count: None, |
| 3193 | | - varying_components: Vec::new(), |
| 3194 | | - stable_components: Vec::new(), |
| 3195 | | - detail, |
| 3196 | | - temp_root, |
| 3197 | | - }) |
| 3198 | | - } |
| 3199 | | - }; |
| 3200 | | - let initial_snapshot = match parse_object_snapshot_text(initial_text) { |
| 3201 | | - Ok(snapshot) => snapshot, |
| 3202 | | - Err(detail) => { |
| 3203 | | - return Some(ConsistencyIssue { |
| 3204 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3205 | | - summary: "captured object snapshot had an unexpected format".into(), |
| 3206 | | - repeat_count: None, |
| 3207 | | - unique_variant_count: None, |
| 3208 | | - varying_components: Vec::new(), |
| 3209 | | - stable_components: Vec::new(), |
| 3210 | | - detail, |
| 3211 | | - temp_root, |
| 3212 | | - }) |
| 3213 | | - } |
| 3214 | | - }; |
| 3215 | | - if let Err(err) = fs::write(temp_root.join("capture_run_00.obj.txt"), initial_text) { |
| 3216 | | - return Some(ConsistencyIssue { |
| 3217 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3218 | | - summary: "could not write captured object snapshot artifact".into(), |
| 3219 | | - repeat_count: None, |
| 3220 | | - unique_variant_count: None, |
| 3221 | | - varying_components: Vec::new(), |
| 3222 | | - stable_components: Vec::new(), |
| 3223 | | - detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 3224 | | - temp_root, |
| 3225 | | - }); |
| 4742 | + Err(format!( |
| 4743 | + "{}:{}: unsupported failure expectation '{}'", |
| 4744 | + path.display(), |
| 4745 | + line_no, |
| 4746 | + rest |
| 4747 | + )) |
| 4748 | +} |
| 4749 | + |
| 4750 | +fn parse_status_rule( |
| 4751 | + kind: StatusKind, |
| 4752 | + rest: &str, |
| 4753 | + path: &Path, |
| 4754 | + line_no: usize, |
| 4755 | +) -> Result<PendingStatusRule, String> { |
| 4756 | + let rest = rest.trim(); |
| 4757 | + if kind == StatusKind::Xfail && rest.eq_ignore_ascii_case("comments") { |
| 4758 | + return Ok(PendingStatusRule::XfailSourceComments); |
| 4759 | + } |
| 4760 | + if rest.starts_with('"') { |
| 4761 | + return Ok(PendingStatusRule::Explicit(StatusRule { |
| 4762 | + kind, |
| 4763 | + selector: OptSelector::All, |
| 4764 | + reason: parse_quoted(rest, path, line_no)?, |
| 4765 | + })); |
| 3226 | 4766 | } |
| 3227 | 4767 | |
| 3228 | | - let mut runs = vec![ObjectRun { |
| 3229 | | - label: "capture run 1".into(), |
| 3230 | | - command: command.clone(), |
| 3231 | | - snapshot: initial_snapshot, |
| 3232 | | - }]; |
| 4768 | + let conditional = rest.strip_prefix("when ").ok_or_else(|| { |
| 4769 | + format!( |
| 4770 | + "{}:{}: expected quoted reason or 'when <opts> because \"...\"'", |
| 4771 | + path.display(), |
| 4772 | + line_no |
| 4773 | + ) |
| 4774 | + })?; |
| 4775 | + let (selector, reason) = conditional.split_once(" because ").ok_or_else(|| { |
| 4776 | + format!( |
| 4777 | + "{}:{}: conditional status must use 'when <opts> because \"...\"'", |
| 4778 | + path.display(), |
| 4779 | + line_no |
| 4780 | + ) |
| 4781 | + })?; |
| 3233 | 4782 | |
| 3234 | | - for index in 1..repeat_count { |
| 3235 | | - let text = match capture_text_from_testing(source, opt_level, Stage::Obj) { |
| 3236 | | - Ok(text) => text, |
| 3237 | | - Err(detail) => { |
| 3238 | | - return Some(ConsistencyIssue { |
| 3239 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3240 | | - summary: "armfortas::testing capture failed during obj reproducibility check" |
| 3241 | | - .into(), |
| 3242 | | - repeat_count: None, |
| 3243 | | - unique_variant_count: None, |
| 3244 | | - varying_components: Vec::new(), |
| 3245 | | - stable_components: Vec::new(), |
| 3246 | | - detail, |
| 3247 | | - temp_root, |
| 3248 | | - }) |
| 4783 | + Ok(PendingStatusRule::Explicit(StatusRule { |
| 4784 | + kind, |
| 4785 | + selector: parse_opt_selector(selector.trim(), path, line_no)?, |
| 4786 | + reason: parse_quoted(reason.trim(), path, line_no)?, |
| 4787 | + })) |
| 4788 | +} |
| 4789 | + |
| 4790 | +fn parse_capability_policy( |
| 4791 | + kind: StatusKind, |
| 4792 | + rest: &str, |
| 4793 | + path: &Path, |
| 4794 | + line_no: usize, |
| 4795 | +) -> Result<CapabilityPolicy, String> { |
| 4796 | + Ok(CapabilityPolicy { |
| 4797 | + kind, |
| 4798 | + reason: parse_quoted(rest.trim(), path, line_no)?, |
| 4799 | + }) |
| 4800 | +} |
| 4801 | + |
| 4802 | +fn resolve_source_comment_expectations( |
| 4803 | + expectations: Vec<Expectation>, |
| 4804 | + source_text: Option<&str>, |
| 4805 | + suite_path: &Path, |
| 4806 | + case_name: &str, |
| 4807 | + source_path: &Path, |
| 4808 | +) -> Result<Vec<Expectation>, String> { |
| 4809 | + let mut resolved = Vec::with_capacity(expectations.len()); |
| 4810 | + for expectation in expectations { |
| 4811 | + match expectation { |
| 4812 | + Expectation::FailSourceComments => { |
| 4813 | + let source_text = source_text.ok_or_else(|| { |
| 4814 | + format!( |
| 4815 | + "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4816 | + suite_path.display(), |
| 4817 | + case_name, |
| 4818 | + source_path.display() |
| 4819 | + ) |
| 4820 | + })?; |
| 4821 | + let patterns = extract_error_expected_patterns(source_text); |
| 4822 | + if patterns.is_empty() { |
| 4823 | + return Err(format!( |
| 4824 | + "{}: case '{}' requests expect-fail comments but '{}' has no ! ERROR_EXPECTED: lines", |
| 4825 | + suite_path.display(), |
| 4826 | + case_name, |
| 4827 | + source_path.display() |
| 4828 | + )); |
| 4829 | + } |
| 4830 | + resolved.push(Expectation::FailCommentPatterns(patterns)); |
| 3249 | 4831 | } |
| 3250 | | - }; |
| 3251 | | - if let Err(err) = fs::write( |
| 3252 | | - temp_root.join(format!("capture_run_{:02}.obj.txt", index)), |
| 3253 | | - &text, |
| 3254 | | - ) { |
| 3255 | | - return Some(ConsistencyIssue { |
| 3256 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3257 | | - summary: "could not write captured object snapshot artifact".into(), |
| 3258 | | - repeat_count: None, |
| 3259 | | - unique_variant_count: None, |
| 3260 | | - varying_components: Vec::new(), |
| 3261 | | - stable_components: Vec::new(), |
| 3262 | | - detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 3263 | | - temp_root, |
| 3264 | | - }); |
| 4832 | + other => resolved.push(other), |
| 3265 | 4833 | } |
| 3266 | | - let snapshot = match parse_object_snapshot_text(&text) { |
| 3267 | | - Ok(snapshot) => snapshot, |
| 3268 | | - Err(detail) => { |
| 3269 | | - return Some(ConsistencyIssue { |
| 3270 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3271 | | - summary: "captured object snapshot had an unexpected format".into(), |
| 3272 | | - repeat_count: None, |
| 3273 | | - unique_variant_count: None, |
| 3274 | | - varying_components: Vec::new(), |
| 3275 | | - stable_components: Vec::new(), |
| 3276 | | - detail, |
| 3277 | | - temp_root, |
| 3278 | | - }) |
| 4834 | + } |
| 4835 | + Ok(resolved) |
| 4836 | +} |
| 4837 | + |
| 4838 | +fn resolve_source_comment_status_rules( |
| 4839 | + status_rules: Vec<PendingStatusRule>, |
| 4840 | + source_text: Option<&str>, |
| 4841 | + suite_path: &Path, |
| 4842 | + case_name: &str, |
| 4843 | + source_path: &Path, |
| 4844 | +) -> Result<Vec<StatusRule>, String> { |
| 4845 | + let mut resolved = Vec::with_capacity(status_rules.len()); |
| 4846 | + for rule in status_rules { |
| 4847 | + match rule { |
| 4848 | + PendingStatusRule::Explicit(rule) => resolved.push(rule), |
| 4849 | + PendingStatusRule::XfailSourceComments => { |
| 4850 | + let source_text = source_text.ok_or_else(|| { |
| 4851 | + format!( |
| 4852 | + "{}: case '{}': source comments were required but '{}' was not loaded", |
| 4853 | + suite_path.display(), |
| 4854 | + case_name, |
| 4855 | + source_path.display() |
| 4856 | + ) |
| 4857 | + })?; |
| 4858 | + let reason = extract_xfail_reason(source_text).ok_or_else(|| { |
| 4859 | + format!( |
| 4860 | + "{}: case '{}' requests xfail comments but '{}' has no ! XFAIL: lines", |
| 4861 | + suite_path.display(), |
| 4862 | + case_name, |
| 4863 | + source_path.display() |
| 4864 | + ) |
| 4865 | + })?; |
| 4866 | + resolved.push(StatusRule { |
| 4867 | + kind: StatusKind::Xfail, |
| 4868 | + selector: OptSelector::All, |
| 4869 | + reason, |
| 4870 | + }); |
| 3279 | 4871 | } |
| 3280 | | - }; |
| 3281 | | - runs.push(ObjectRun { |
| 3282 | | - label: format!("capture run {}", index + 1), |
| 3283 | | - command: command.clone(), |
| 3284 | | - snapshot, |
| 3285 | | - }); |
| 4872 | + } |
| 3286 | 4873 | } |
| 4874 | + Ok(resolved) |
| 4875 | +} |
| 3287 | 4876 | |
| 3288 | | - let rendered = runs |
| 3289 | | - .iter() |
| 3290 | | - .map(|run| render_object_snapshot(&run.snapshot)) |
| 3291 | | - .collect::<Vec<_>>(); |
| 3292 | | - let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 3293 | | - if unique_variants > 1 { |
| 3294 | | - let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 3295 | | - let (left, right) = |
| 3296 | | - first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 3297 | | - let varying = varying_object_components(&snapshots); |
| 3298 | | - let stable = stable_object_components(&snapshots); |
| 3299 | | - return Some(ConsistencyIssue { |
| 3300 | | - check: ConsistencyCheck::CaptureObjReproducible, |
| 3301 | | - summary: format!( |
| 3302 | | - "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 3303 | | - repeat_count, |
| 3304 | | - unique_variants, |
| 3305 | | - join_or_none(&varying), |
| 3306 | | - join_or_none(&stable) |
| 3307 | | - ), |
| 3308 | | - repeat_count: Some(repeat_count), |
| 3309 | | - unique_variant_count: Some(unique_variants), |
| 3310 | | - varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 3311 | | - stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 3312 | | - detail: format!( |
| 3313 | | - "captured object snapshots are not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 3314 | | - repeat_count, |
| 3315 | | - unique_variants, |
| 3316 | | - join_or_none(&varying), |
| 3317 | | - join_or_none(&stable), |
| 3318 | | - left.command, |
| 3319 | | - right.command, |
| 3320 | | - describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 3321 | | - ), |
| 3322 | | - temp_root, |
| 3323 | | - }); |
| 4877 | +fn parse_opt_selector(raw: &str, path: &Path, line_no: usize) -> Result<OptSelector, String> { |
| 4878 | + let raw = raw.trim(); |
| 4879 | + if raw.eq_ignore_ascii_case("all") { |
| 4880 | + return Ok(OptSelector::All); |
| 3324 | 4881 | } |
| 3325 | | - |
| 3326 | | - let _ = fs::remove_dir_all(&temp_root); |
| 3327 | | - None |
| 4882 | + if let Some(rest) = raw.strip_prefix("opts =>") { |
| 4883 | + return Ok(OptSelector::Only(parse_opt_levels(rest, path, line_no)?)); |
| 4884 | + } |
| 4885 | + Ok(OptSelector::Only(parse_opt_levels(raw, path, line_no)?)) |
| 3328 | 4886 | } |
| 3329 | 4887 | |
| 3330 | | -fn run_capture_run_reproducible( |
| 3331 | | - source: &Path, |
| 3332 | | - opt_level: OptLevel, |
| 3333 | | - repeat_count: usize, |
| 3334 | | - capture_result: &CaptureResult, |
| 3335 | | - _tools: &ToolchainConfig, |
| 3336 | | -) -> Option<ConsistencyIssue> { |
| 3337 | | - let temp_root = next_consistency_temp_root(opt_level); |
| 3338 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 3339 | | - return Some(ConsistencyIssue { |
| 3340 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3341 | | - summary: "could not create consistency temp dir".into(), |
| 3342 | | - repeat_count: None, |
| 3343 | | - unique_variant_count: None, |
| 3344 | | - varying_components: Vec::new(), |
| 3345 | | - stable_components: Vec::new(), |
| 3346 | | - detail: format!( |
| 3347 | | - "cannot create consistency temp dir '{}': {}", |
| 3348 | | - temp_root.display(), |
| 3349 | | - err |
| 3350 | | - ), |
| 3351 | | - temp_root, |
| 3352 | | - }); |
| 4888 | +fn parse_target(raw: &str, path: &Path, line_no: usize) -> Result<Target, String> { |
| 4889 | + match raw { |
| 4890 | + "compare.status" => Ok(Target::CompareStatus), |
| 4891 | + "compare.classification" => Ok(Target::CompareClassification), |
| 4892 | + "compare.changed_artifacts" => Ok(Target::CompareChangedArtifacts), |
| 4893 | + "compare.difference_count" => Ok(Target::CompareDifferenceCount), |
| 4894 | + "compare.basis" => Ok(Target::CompareBasis), |
| 4895 | + "run.stdout" => Ok(Target::RunStdout), |
| 4896 | + "run.stderr" => Ok(Target::RunStderr), |
| 4897 | + "run.exit_code" => Ok(Target::RunExitCode), |
| 4898 | + _ => { |
| 4899 | + if let Some(artifact) = ArtifactKey::parse(raw) { |
| 4900 | + return Ok(Target::Artifact(artifact)); |
| 4901 | + } |
| 4902 | + let stage = Stage::parse(raw).ok_or_else(|| { |
| 4903 | + format!( |
| 4904 | + "{}:{}: unsupported expectation target '{}'", |
| 4905 | + path.display(), |
| 4906 | + line_no, |
| 4907 | + raw |
| 4908 | + ) |
| 4909 | + })?; |
| 4910 | + Ok(Target::Stage(stage)) |
| 4911 | + } |
| 3353 | 4912 | } |
| 4913 | +} |
| 3354 | 4914 | |
| 3355 | | - let command = render_capture_command(source, opt_level, Stage::Run); |
| 3356 | | - let initial_run = match capture_run_stage(capture_result) { |
| 3357 | | - Ok(run) => run.clone(), |
| 3358 | | - Err(detail) => { |
| 3359 | | - return Some(ConsistencyIssue { |
| 3360 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3361 | | - summary: "initial capture result did not include runtime behavior".into(), |
| 3362 | | - repeat_count: None, |
| 3363 | | - unique_variant_count: None, |
| 3364 | | - varying_components: Vec::new(), |
| 3365 | | - stable_components: Vec::new(), |
| 3366 | | - detail, |
| 3367 | | - temp_root, |
| 3368 | | - }) |
| 3369 | | - } |
| 3370 | | - }; |
| 3371 | | - if let Err(err) = |
| 3372 | | - write_behavior_run_artifacts(&temp_root, "capture_run_00", &command, &initial_run) |
| 3373 | | - { |
| 3374 | | - return Some(ConsistencyIssue { |
| 3375 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3376 | | - summary: "could not write captured runtime artifact".into(), |
| 3377 | | - repeat_count: None, |
| 3378 | | - unique_variant_count: None, |
| 3379 | | - varying_components: Vec::new(), |
| 3380 | | - stable_components: Vec::new(), |
| 3381 | | - detail: format!("cannot write captured runtime artifact: {}", err), |
| 3382 | | - temp_root, |
| 3383 | | - }); |
| 4915 | +fn parse_failure_stage(raw: &str, path: &Path, line_no: usize) -> Result<FailureStage, String> { |
| 4916 | + FailureStage::parse(raw).ok_or_else(|| { |
| 4917 | + format!( |
| 4918 | + "{}:{}: unsupported failure stage '{}'", |
| 4919 | + path.display(), |
| 4920 | + line_no, |
| 4921 | + raw |
| 4922 | + ) |
| 4923 | + }) |
| 4924 | +} |
| 4925 | + |
| 4926 | +fn parse_quoted(raw: &str, path: &Path, line_no: usize) -> Result<String, String> { |
| 4927 | + let raw = raw.trim(); |
| 4928 | + if !(raw.starts_with('"') && raw.ends_with('"')) { |
| 4929 | + return Err(format!( |
| 4930 | + "{}:{}: expected quoted string, got '{}'", |
| 4931 | + path.display(), |
| 4932 | + line_no, |
| 4933 | + raw |
| 4934 | + )); |
| 3384 | 4935 | } |
| 3385 | | - let mut runs = vec![BehaviorRun { |
| 3386 | | - label: "capture run 1".into(), |
| 3387 | | - command: command.clone(), |
| 3388 | | - signature: normalize_run_signature(&initial_run), |
| 3389 | | - run: initial_run, |
| 3390 | | - }]; |
| 4936 | + let body = &raw[1..raw.len() - 1]; |
| 4937 | + Ok(body.replace("\\\"", "\"").replace("\\n", "\n")) |
| 4938 | +} |
| 3391 | 4939 | |
| 3392 | | - for index in 1..repeat_count { |
| 3393 | | - let run = match capture_run_from_testing(source, opt_level) { |
| 3394 | | - Ok(run) => run, |
| 3395 | | - Err(detail) => { |
| 3396 | | - return Some(ConsistencyIssue { |
| 3397 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3398 | | - summary: |
| 3399 | | - "armfortas::testing capture failed during runtime reproducibility check" |
| 3400 | | - .into(), |
| 3401 | | - repeat_count: None, |
| 3402 | | - unique_variant_count: None, |
| 3403 | | - varying_components: Vec::new(), |
| 3404 | | - stable_components: Vec::new(), |
| 3405 | | - detail, |
| 3406 | | - temp_root, |
| 3407 | | - }) |
| 3408 | | - } |
| 3409 | | - }; |
| 3410 | | - if let Err(err) = write_behavior_run_artifacts( |
| 3411 | | - &temp_root, |
| 3412 | | - &format!("capture_run_{:02}", index), |
| 3413 | | - &command, |
| 3414 | | - &run, |
| 3415 | | - ) { |
| 3416 | | - return Some(ConsistencyIssue { |
| 3417 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3418 | | - summary: "could not write captured runtime artifact".into(), |
| 3419 | | - repeat_count: None, |
| 3420 | | - unique_variant_count: None, |
| 3421 | | - varying_components: Vec::new(), |
| 3422 | | - stable_components: Vec::new(), |
| 3423 | | - detail: format!("cannot write captured runtime artifact: {}", err), |
| 3424 | | - temp_root, |
| 3425 | | - }); |
| 4940 | +fn parse_integer(raw: &str, path: &Path, line_no: usize) -> Result<i32, String> { |
| 4941 | + let value = if raw.starts_with('"') { |
| 4942 | + parse_quoted(raw, path, line_no)? |
| 4943 | + } else { |
| 4944 | + raw.trim().to_string() |
| 4945 | + }; |
| 4946 | + value.parse::<i32>().map_err(|_| { |
| 4947 | + format!( |
| 4948 | + "{}:{}: expected integer literal, got '{}'", |
| 4949 | + path.display(), |
| 4950 | + line_no, |
| 4951 | + raw |
| 4952 | + ) |
| 4953 | + }) |
| 4954 | +} |
| 4955 | + |
| 4956 | +fn parse_opt_level_token(raw: &str) -> Option<OptLevel> { |
| 4957 | + let raw = raw.trim(); |
| 4958 | + let raw = raw.strip_prefix('-').unwrap_or(raw); |
| 4959 | + OptLevel::parse_flag(raw) |
| 4960 | +} |
| 4961 | + |
| 4962 | +fn parse_opt_level_list(raw: &str) -> Result<Vec<OptLevel>, String> { |
| 4963 | + let mut levels = BTreeSet::new(); |
| 4964 | + for value in raw.split(',') { |
| 4965 | + let value = value.trim(); |
| 4966 | + if value.is_empty() { |
| 4967 | + continue; |
| 3426 | 4968 | } |
| 3427 | | - runs.push(BehaviorRun { |
| 3428 | | - label: format!("capture run {}", index + 1), |
| 3429 | | - command: command.clone(), |
| 3430 | | - signature: normalize_run_signature(&run), |
| 3431 | | - run, |
| 3432 | | - }); |
| 4969 | + if value.eq_ignore_ascii_case("all") { |
| 4970 | + levels.extend(all_opt_levels()); |
| 4971 | + continue; |
| 4972 | + } |
| 4973 | + let level = |
| 4974 | + parse_opt_level_token(value).ok_or_else(|| format!("unknown opt level '{}'", value))?; |
| 4975 | + levels.insert(level); |
| 3433 | 4976 | } |
| 3434 | | - |
| 3435 | | - let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 3436 | | - if unique_variants > 1 { |
| 3437 | | - let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 3438 | | - let varying = varying_run_components(&signatures); |
| 3439 | | - let stable = stable_run_components(&signatures); |
| 3440 | | - let (left, right) = first_distinct_behavior_pair(&runs) |
| 3441 | | - .expect("unique variants > 1 implies a distinct pair"); |
| 3442 | | - return Some(ConsistencyIssue { |
| 3443 | | - check: ConsistencyCheck::CaptureRunReproducible, |
| 3444 | | - summary: format!( |
| 3445 | | - "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 3446 | | - repeat_count, |
| 3447 | | - unique_variants, |
| 3448 | | - join_or_none(&varying), |
| 3449 | | - join_or_none(&stable) |
| 3450 | | - ), |
| 3451 | | - repeat_count: Some(repeat_count), |
| 3452 | | - unique_variant_count: Some(unique_variants), |
| 3453 | | - varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 3454 | | - stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 3455 | | - detail: format!( |
| 3456 | | - "captured runtime behavior is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 3457 | | - repeat_count, |
| 3458 | | - unique_variants, |
| 3459 | | - join_or_none(&varying), |
| 3460 | | - join_or_none(&stable), |
| 3461 | | - left.command, |
| 3462 | | - right.command, |
| 3463 | | - describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 3464 | | - ), |
| 3465 | | - temp_root, |
| 3466 | | - }); |
| 4977 | + if levels.is_empty() { |
| 4978 | + return Err("opt filter is empty".into()); |
| 3467 | 4979 | } |
| 4980 | + Ok(levels.into_iter().collect()) |
| 4981 | +} |
| 3468 | 4982 | |
| 3469 | | - let _ = fs::remove_dir_all(&temp_root); |
| 3470 | | - None |
| 4983 | +fn all_opt_levels() -> [OptLevel; 5] { |
| 4984 | + [ |
| 4985 | + OptLevel::O0, |
| 4986 | + OptLevel::O1, |
| 4987 | + OptLevel::O2, |
| 4988 | + OptLevel::O3, |
| 4989 | + OptLevel::Ofast, |
| 4990 | + ] |
| 3471 | 4991 | } |
| 3472 | 4992 | |
| 3473 | | -fn run_reference_compilers( |
| 3474 | | - prepared: &PreparedInput, |
| 3475 | | - case: &CaseSpec, |
| 3476 | | - opt_level: OptLevel, |
| 3477 | | - tools: &ToolchainConfig, |
| 3478 | | -) -> Vec<ReferenceResult> { |
| 3479 | | - case.reference_compilers |
| 4993 | +fn filter_suites<'a>(suites: &'a [SuiteSpec], suite_filter: Option<&str>) -> Vec<&'a SuiteSpec> { |
| 4994 | + let filter = suite_filter.map(|value| value.to_ascii_lowercase()); |
| 4995 | + suites |
| 3480 | 4996 | .iter() |
| 3481 | | - .copied() |
| 3482 | | - .map(|compiler| run_reference_case(&prepared.compiler_source, opt_level, compiler, tools)) |
| 4997 | + .filter(|suite| { |
| 4998 | + if let Some(filter) = &filter { |
| 4999 | + suite.name.to_ascii_lowercase().contains(filter) |
| 5000 | + } else { |
| 5001 | + true |
| 5002 | + } |
| 5003 | + }) |
| 3483 | 5004 | .collect() |
| 3484 | 5005 | } |
| 3485 | 5006 | |
| 3486 | | -fn run_reference_case( |
| 3487 | | - source: &Path, |
| 3488 | | - opt_level: OptLevel, |
| 3489 | | - compiler: ReferenceCompiler, |
| 3490 | | - tools: &ToolchainConfig, |
| 3491 | | -) -> ReferenceResult { |
| 3492 | | - let temp_root = next_report_temp_root(compiler, opt_level); |
| 3493 | | - let binary = temp_root.join("reference.out"); |
| 3494 | | - let uses_cpp = source_uses_cpp(source); |
| 5007 | +fn print_suites(suites: &[&SuiteSpec], config: &ListConfig) { |
| 5008 | + for suite in suites { |
| 5009 | + println!("{} ({})", suite.name, suite.cases.len()); |
| 5010 | + println!(" {}", suite.path.display()); |
| 5011 | + if config.verbose { |
| 5012 | + for case in &suite.cases { |
| 5013 | + println!(" - {} [{}]", case.name, case_discovery_mode_label(case)); |
| 5014 | + for line in case_discovery_lines(case, &config.tools) { |
| 5015 | + println!(" {}", line); |
| 5016 | + } |
| 5017 | + } |
| 5018 | + } |
| 5019 | + } |
| 5020 | +} |
| 3495 | 5021 | |
| 3496 | | - let mut args = vec![opt_level.as_flag().to_string()]; |
| 3497 | | - if uses_cpp { |
| 3498 | | - args.push("-cpp".to_string()); |
| 5022 | +fn case_discovery_mode_label(case: &CaseSpec) -> &'static str { |
| 5023 | + if case.is_generic_compare() { |
| 5024 | + "generic-compare" |
| 5025 | + } else if case.is_generic_introspect() { |
| 5026 | + "generic-introspect" |
| 5027 | + } else { |
| 5028 | + "legacy" |
| 3499 | 5029 | } |
| 3500 | | - args.push(source.display().to_string()); |
| 3501 | | - args.push("-o".to_string()); |
| 3502 | | - args.push(binary.display().to_string()); |
| 5030 | +} |
| 3503 | 5031 | |
| 3504 | | - let compiler_bin = tools.reference_binary(compiler); |
| 3505 | | - let command_string = render_command(compiler_bin, &args); |
| 5032 | +fn format_opt_level_list(levels: &[OptLevel]) -> String { |
| 5033 | + levels |
| 5034 | + .iter() |
| 5035 | + .map(OptLevel::as_str) |
| 5036 | + .collect::<Vec<_>>() |
| 5037 | + .join(", ") |
| 5038 | +} |
| 3506 | 5039 | |
| 3507 | | - if let Err(err) = fs::create_dir_all(&temp_root) { |
| 3508 | | - return ReferenceResult::infrastructure_error( |
| 3509 | | - compiler, |
| 3510 | | - command_string, |
| 3511 | | - format!("cannot create temp dir '{}': {}", temp_root.display(), err), |
| 3512 | | - ); |
| 5040 | +fn case_discovery_lines(case: &CaseSpec, tools: &ToolchainConfig) -> Vec<String> { |
| 5041 | + let mut lines = vec![ |
| 5042 | + format!("source: {}", case.source_label()), |
| 5043 | + format!("opts: {}", format_opt_level_list(&case.opt_levels)), |
| 5044 | + ]; |
| 5045 | + if let Some(policy) = &case.capability_policy { |
| 5046 | + lines.push(format!( |
| 5047 | + "capability_policy: {} when blocked ({})", |
| 5048 | + match policy.kind { |
| 5049 | + StatusKind::Xfail => "xfail", |
| 5050 | + StatusKind::Future => "future", |
| 5051 | + }, |
| 5052 | + policy.reason |
| 5053 | + )); |
| 3513 | 5054 | } |
| 3514 | 5055 | |
| 3515 | | - let compile = match Command::new(compiler_bin) |
| 3516 | | - .current_dir(&temp_root) |
| 3517 | | - .args(&args) |
| 3518 | | - .output() |
| 3519 | | - { |
| 3520 | | - Ok(output) => output, |
| 3521 | | - Err(err) => { |
| 3522 | | - return ReferenceResult::infrastructure_error( |
| 3523 | | - compiler, |
| 3524 | | - command_string, |
| 3525 | | - format!("cannot run {}: {}", compiler_bin, err), |
| 3526 | | - ); |
| 5056 | + if let Some(generic) = &case.generic_introspect { |
| 5057 | + let capture_root = tools.armfortas_adapters().capture_root(); |
| 5058 | + let probe = compiler_spec_probe(&generic.compiler, tools, capture_root.as_ref()); |
| 5059 | + lines.push(format!("compiler: {}", generic.compiler.display_name())); |
| 5060 | + lines.push(format!("compiler_probe_status: {}", probe.status)); |
| 5061 | + lines.push(format!( |
| 5062 | + "compiler_probe_resolved_path: {}", |
| 5063 | + probe |
| 5064 | + .resolved_path |
| 5065 | + .as_ref() |
| 5066 | + .map(|path| display_path(path)) |
| 5067 | + .unwrap_or_else(|| "none".to_string()) |
| 5068 | + )); |
| 5069 | + if let Some(banner) = &probe.banner { |
| 5070 | + lines.push(format!("compiler_probe_banner: {}", banner)); |
| 3527 | 5071 | } |
| 3528 | | - }; |
| 3529 | | - |
| 3530 | | - let mut result = ReferenceResult { |
| 3531 | | - compiler, |
| 3532 | | - compile_command: command_string, |
| 3533 | | - compile_exit_code: compile.status.code().unwrap_or(-1), |
| 3534 | | - compile_stdout: String::from_utf8_lossy(&compile.stdout).into_owned(), |
| 3535 | | - compile_stderr: String::from_utf8_lossy(&compile.stderr).into_owned(), |
| 3536 | | - run: None, |
| 3537 | | - run_error: None, |
| 3538 | | - }; |
| 3539 | | - |
| 3540 | | - if compile.status.success() { |
| 3541 | | - match Command::new(&binary).current_dir(&temp_root).output() { |
| 3542 | | - Ok(output) => { |
| 3543 | | - result.run = Some(RunCapture { |
| 3544 | | - exit_code: output.status.code().unwrap_or(-1), |
| 3545 | | - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 3546 | | - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 5072 | + lines.push(format!( |
| 5073 | + "artifacts: {}", |
| 5074 | + format_artifact_name_list( |
| 5075 | + &generic |
| 5076 | + .artifacts |
| 5077 | + .iter() |
| 5078 | + .map(|artifact| artifact.as_str().to_string()) |
| 5079 | + .collect::<Vec<_>>() |
| 5080 | + ) |
| 5081 | + )); |
| 5082 | + match capability_request_issue(&generic.compiler, &generic.artifacts, tools) { |
| 5083 | + Some(issue) => { |
| 5084 | + lines.push(if case.capability_policy.is_some() { |
| 5085 | + "capability_status: deferred".to_string() |
| 5086 | + } else { |
| 5087 | + "capability_status: blocked".to_string() |
| 3547 | 5088 | }); |
| 5089 | + lines.extend( |
| 5090 | + issue |
| 5091 | + .lines() |
| 5092 | + .map(|line| format!("capability_detail: {}", line)), |
| 5093 | + ); |
| 3548 | 5094 | } |
| 3549 | | - Err(err) => { |
| 3550 | | - result.run_error = Some(format!("cannot run '{}': {}", binary.display(), err)); |
| 3551 | | - } |
| 5095 | + None => lines.push("capability_status: ready".to_string()), |
| 3552 | 5096 | } |
| 5097 | + return lines; |
| 3553 | 5098 | } |
| 3554 | 5099 | |
| 3555 | | - let _ = fs::remove_dir_all(&temp_root); |
| 3556 | | - result |
| 3557 | | -} |
| 3558 | | - |
| 3559 | | -fn source_uses_cpp(source: &Path) -> bool { |
| 3560 | | - fs::read_to_string(source) |
| 3561 | | - .map(|text| text.lines().any(|line| line.trim_start().starts_with('#'))) |
| 3562 | | - .unwrap_or(false) |
| 3563 | | -} |
| 3564 | | - |
| 3565 | | -#[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 3566 | | -enum DriverEmitMode { |
| 3567 | | - Asm, |
| 3568 | | - Obj, |
| 3569 | | - Binary, |
| 3570 | | -} |
| 5100 | + if let Some(generic) = &case.generic_compare { |
| 5101 | + let capture_root = tools.armfortas_adapters().capture_root(); |
| 5102 | + let left_probe = compiler_spec_probe(&generic.left, tools, capture_root.as_ref()); |
| 5103 | + let right_probe = compiler_spec_probe(&generic.right, tools, capture_root.as_ref()); |
| 5104 | + lines.push(format!( |
| 5105 | + "compare: {} vs {}", |
| 5106 | + generic.left.display_name(), |
| 5107 | + generic.right.display_name() |
| 5108 | + )); |
| 5109 | + lines.push(format!("left_probe_status: {}", left_probe.status)); |
| 5110 | + lines.push(format!( |
| 5111 | + "left_probe_resolved_path: {}", |
| 5112 | + left_probe |
| 5113 | + .resolved_path |
| 5114 | + .as_ref() |
| 5115 | + .map(|path| display_path(path)) |
| 5116 | + .unwrap_or_else(|| "none".to_string()) |
| 5117 | + )); |
| 5118 | + if let Some(banner) = &left_probe.banner { |
| 5119 | + lines.push(format!("left_probe_banner: {}", banner)); |
| 5120 | + } |
| 5121 | + lines.push(format!("right_probe_status: {}", right_probe.status)); |
| 5122 | + lines.push(format!( |
| 5123 | + "right_probe_resolved_path: {}", |
| 5124 | + right_probe |
| 5125 | + .resolved_path |
| 5126 | + .as_ref() |
| 5127 | + .map(|path| display_path(path)) |
| 5128 | + .unwrap_or_else(|| "none".to_string()) |
| 5129 | + )); |
| 5130 | + if let Some(banner) = &right_probe.banner { |
| 5131 | + lines.push(format!("right_probe_banner: {}", banner)); |
| 5132 | + } |
| 5133 | + lines.push(format!( |
| 5134 | + "artifacts: {}", |
| 5135 | + format_artifact_name_list( |
| 5136 | + &generic |
| 5137 | + .artifacts |
| 5138 | + .iter() |
| 5139 | + .map(|artifact| artifact.as_str().to_string()) |
| 5140 | + .collect::<Vec<_>>() |
| 5141 | + ) |
| 5142 | + )); |
| 5143 | + let mut issues = Vec::new(); |
| 5144 | + if let Some(issue) = capability_request_issue(&generic.left, &generic.artifacts, tools) { |
| 5145 | + issues.push(format!("left:\n{}", issue)); |
| 5146 | + } |
| 5147 | + if let Some(issue) = capability_request_issue(&generic.right, &generic.artifacts, tools) { |
| 5148 | + issues.push(format!("right:\n{}", issue)); |
| 5149 | + } |
| 5150 | + if issues.is_empty() { |
| 5151 | + lines.push("capability_status: ready".to_string()); |
| 5152 | + } else { |
| 5153 | + lines.push(if case.capability_policy.is_some() { |
| 5154 | + "capability_status: deferred".to_string() |
| 5155 | + } else { |
| 5156 | + "capability_status: blocked".to_string() |
| 5157 | + }); |
| 5158 | + lines.extend(issues.into_iter().flat_map(|issue| { |
| 5159 | + issue |
| 5160 | + .lines() |
| 5161 | + .map(|line| format!("capability_detail: {}", line)) |
| 5162 | + .collect::<Vec<_>>() |
| 5163 | + })); |
| 5164 | + } |
| 5165 | + return lines; |
| 5166 | + } |
| 3571 | 5167 | |
| 3572 | | -fn compile_with_driver( |
| 3573 | | - source: &Path, |
| 3574 | | - opt_level: OptLevel, |
| 3575 | | - mode: DriverEmitMode, |
| 3576 | | - output: &Path, |
| 3577 | | - tools: &ToolchainConfig, |
| 3578 | | -) -> Result<String, String> { |
| 3579 | | - let command = render_armfortas_command(source, opt_level, mode, output, tools); |
| 3580 | | - if let Some(binary) = tools.armfortas_external_bin() { |
| 3581 | | - let mut args = vec![opt_level.as_flag().to_string()]; |
| 3582 | | - match mode { |
| 3583 | | - DriverEmitMode::Asm => args.push("-S".to_string()), |
| 3584 | | - DriverEmitMode::Obj => args.push("-c".to_string()), |
| 3585 | | - DriverEmitMode::Binary => {} |
| 3586 | | - } |
| 3587 | | - args.push(source.display().to_string()); |
| 3588 | | - args.push("-o".to_string()); |
| 3589 | | - args.push(output.display().to_string()); |
| 3590 | | - |
| 3591 | | - let compile = Command::new(binary) |
| 3592 | | - .args(&args) |
| 3593 | | - .output() |
| 3594 | | - .map_err(|err| format!("{} failed:\ncannot run '{}': {}", command, binary, err))?; |
| 3595 | | - if !compile.status.success() { |
| 3596 | | - let stderr = String::from_utf8_lossy(&compile.stderr); |
| 3597 | | - return Err(format!("{} failed:\n{}", command, stderr.trim_end())); |
| 5168 | + let needs_linked_capture = primary_backend_kind_for_case(case, &case.requested, tools) |
| 5169 | + == PrimaryCaptureBackendKind::Full; |
| 5170 | + if case.requested.is_empty() { |
| 5171 | + lines.push("stages: run".to_string()); |
| 5172 | + } else { |
| 5173 | + let stages = case |
| 5174 | + .requested |
| 5175 | + .iter() |
| 5176 | + .map(Stage::as_str) |
| 5177 | + .collect::<Vec<_>>() |
| 5178 | + .join(", "); |
| 5179 | + lines.push(format!("stages: {}", stages)); |
| 5180 | + } |
| 5181 | + if !case.reference_compilers.is_empty() { |
| 5182 | + lines.push(format!( |
| 5183 | + "differential: {}", |
| 5184 | + case.reference_compilers |
| 5185 | + .iter() |
| 5186 | + .map(ReferenceCompiler::as_str) |
| 5187 | + .collect::<Vec<_>>() |
| 5188 | + .join(", ") |
| 5189 | + )); |
| 5190 | + } |
| 5191 | + if !case.consistency_checks.is_empty() { |
| 5192 | + lines.push(format!( |
| 5193 | + "consistency: {}", |
| 5194 | + case.consistency_checks |
| 5195 | + .iter() |
| 5196 | + .map(ConsistencyCheck::as_str) |
| 5197 | + .collect::<Vec<_>>() |
| 5198 | + .join(", ") |
| 5199 | + )); |
| 5200 | + } |
| 5201 | + if needs_linked_capture { |
| 5202 | + lines.push("surface: linked armfortas capture".to_string()); |
| 5203 | + if linked_capture_available() { |
| 5204 | + lines.push("capability_status: ready".to_string()); |
| 5205 | + } else { |
| 5206 | + lines.push(if case.capability_policy.is_some() { |
| 5207 | + "capability_status: deferred".to_string() |
| 5208 | + } else { |
| 5209 | + "capability_status: blocked".to_string() |
| 5210 | + }); |
| 5211 | + lines.push( |
| 5212 | + "capability_detail: linked armfortas capture is unavailable in this build" |
| 5213 | + .to_string(), |
| 5214 | + ); |
| 3598 | 5215 | } |
| 3599 | 5216 | } else { |
| 3600 | | - let emit_mode = match mode { |
| 3601 | | - DriverEmitMode::Asm => EmitMode::Asm, |
| 3602 | | - DriverEmitMode::Obj => EmitMode::Obj, |
| 3603 | | - DriverEmitMode::Binary => EmitMode::Binary, |
| 3604 | | - }; |
| 3605 | | - compile_output(source, opt_level, emit_mode, output) |
| 3606 | | - .map_err(|detail| format!("{} failed:\n{}", command, detail))?; |
| 5217 | + lines.push("surface: observable-only legacy path".to_string()); |
| 5218 | + lines.push("capability_status: ready".to_string()); |
| 3607 | 5219 | } |
| 3608 | | - Ok(command) |
| 3609 | | -} |
| 3610 | 5220 | |
| 3611 | | -fn render_armfortas_command( |
| 3612 | | - source: &Path, |
| 3613 | | - opt_level: OptLevel, |
| 3614 | | - mode: DriverEmitMode, |
| 3615 | | - output: &Path, |
| 3616 | | - tools: &ToolchainConfig, |
| 3617 | | -) -> String { |
| 3618 | | - let mut args = vec![opt_level.as_flag().to_string()]; |
| 3619 | | - match mode { |
| 3620 | | - DriverEmitMode::Asm => args.push("-S".to_string()), |
| 3621 | | - DriverEmitMode::Obj => args.push("-c".to_string()), |
| 3622 | | - DriverEmitMode::Binary => {} |
| 3623 | | - } |
| 3624 | | - args.push(source.display().to_string()); |
| 3625 | | - args.push("-o".to_string()); |
| 3626 | | - args.push(output.display().to_string()); |
| 3627 | | - render_command(tools.armfortas_command_name(), &args) |
| 5221 | + lines |
| 3628 | 5222 | } |
| 3629 | 5223 | |
| 3630 | | -fn render_binary_run_command(binary: &Path) -> String { |
| 3631 | | - render_command(&binary.display().to_string(), &[]) |
| 3632 | | -} |
| 5224 | +fn run_suites(config: &RunConfig) -> Result<Summary, String> { |
| 5225 | + let suites = discover_suites(default_suite_root())?; |
| 5226 | + let suites = filter_suites(&suites, config.suite_filter.as_deref()); |
| 5227 | + if suites.is_empty() { |
| 5228 | + return Err("no suites matched the requested filter".into()); |
| 5229 | + } |
| 3633 | 5230 | |
| 3634 | | -fn render_capture_command(source: &Path, opt_level: OptLevel, stage: Stage) -> String { |
| 3635 | | - format!( |
| 3636 | | - "armfortas::testing capture {} --stage {} {}", |
| 3637 | | - opt_level.as_flag(), |
| 3638 | | - stage.as_str(), |
| 3639 | | - quote_arg(&source.display().to_string()) |
| 3640 | | - ) |
| 3641 | | -} |
| 5231 | + let case_filter = config |
| 5232 | + .case_filter |
| 5233 | + .as_ref() |
| 5234 | + .map(|value| value.to_ascii_lowercase()); |
| 5235 | + let mut summary = Summary::default(); |
| 5236 | + let mut matched_cells = 0usize; |
| 3642 | 5237 | |
| 3643 | | -fn capture_text_from_testing( |
| 3644 | | - source: &Path, |
| 3645 | | - opt_level: OptLevel, |
| 3646 | | - stage: Stage, |
| 3647 | | -) -> Result<String, String> { |
| 3648 | | - let command = render_capture_command(source, opt_level, stage); |
| 3649 | | - let request = CaptureRequest { |
| 3650 | | - input: source.to_path_buf(), |
| 3651 | | - requested: BTreeSet::from([stage]), |
| 3652 | | - opt_level, |
| 3653 | | - }; |
| 3654 | | - let result = capture_from_path(&request) |
| 3655 | | - .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 3656 | | - capture_text_stage(&result, stage).map(str::to_string) |
| 3657 | | -} |
| 5238 | + for suite in suites { |
| 5239 | + println!("=== {} ===", suite.name); |
| 5240 | + for case in &suite.cases { |
| 5241 | + if let Some(filter) = &case_filter { |
| 5242 | + if !case.name.to_ascii_lowercase().contains(filter) { |
| 5243 | + continue; |
| 5244 | + } |
| 5245 | + } |
| 3658 | 5246 | |
| 3659 | | -fn capture_run_from_testing(source: &Path, opt_level: OptLevel) -> Result<RunCapture, String> { |
| 3660 | | - let command = render_capture_command(source, opt_level, Stage::Run); |
| 3661 | | - let request = CaptureRequest { |
| 3662 | | - input: source.to_path_buf(), |
| 3663 | | - requested: BTreeSet::from([Stage::Run]), |
| 3664 | | - opt_level, |
| 3665 | | - }; |
| 3666 | | - let result = capture_from_path(&request) |
| 3667 | | - .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 3668 | | - capture_run_stage(&result).cloned() |
| 3669 | | -} |
| 5247 | + let opt_levels = selected_opt_levels(case, config); |
| 5248 | + for opt_level in opt_levels { |
| 5249 | + matched_cells += 1; |
| 5250 | + let outcome = execute_case_cell(suite, case, opt_level, config)?; |
| 5251 | + print_outcome(&outcome); |
| 5252 | + summary.record_outcome(&outcome); |
| 3670 | 5253 | |
| 3671 | | -fn capture_text_stage<'a>(result: &'a CaptureResult, stage: Stage) -> Result<&'a str, String> { |
| 3672 | | - match result.get(stage) { |
| 3673 | | - Some(CapturedStage::Text(text)) => Ok(text), |
| 3674 | | - Some(CapturedStage::Run(_)) => Err(format!( |
| 3675 | | - "capture result contained non-text data for stage '{}'", |
| 3676 | | - stage.as_str() |
| 3677 | | - )), |
| 3678 | | - None => Err(format!( |
| 3679 | | - "capture result was missing requested stage '{}'", |
| 3680 | | - stage.as_str() |
| 3681 | | - )), |
| 5254 | + if config.fail_fast |
| 5255 | + && matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5256 | + { |
| 5257 | + return Ok(summary); |
| 5258 | + } |
| 5259 | + } |
| 5260 | + } |
| 3682 | 5261 | } |
| 3683 | | -} |
| 3684 | 5262 | |
| 3685 | | -fn capture_run_stage(result: &CaptureResult) -> Result<&RunCapture, String> { |
| 3686 | | - match result.get(Stage::Run) { |
| 3687 | | - Some(CapturedStage::Run(run)) => Ok(run), |
| 3688 | | - Some(CapturedStage::Text(_)) => { |
| 3689 | | - Err("capture result contained text data for the run stage".into()) |
| 3690 | | - } |
| 3691 | | - None => Err("capture result was missing requested stage 'run'".into()), |
| 5263 | + if matched_cells == 0 { |
| 5264 | + return Err("no cases matched the requested filters".into()); |
| 3692 | 5265 | } |
| 3693 | | -} |
| 3694 | 5266 | |
| 3695 | | -fn run_binary_capture( |
| 3696 | | - binary: &Path, |
| 3697 | | - current_dir: &Path, |
| 3698 | | - command: &str, |
| 3699 | | -) -> Result<RunCapture, String> { |
| 3700 | | - let output = Command::new(binary) |
| 3701 | | - .current_dir(current_dir) |
| 3702 | | - .output() |
| 3703 | | - .map_err(|err| { |
| 3704 | | - format!( |
| 3705 | | - "{} failed:\ncannot run '{}': {}", |
| 3706 | | - command, |
| 3707 | | - binary.display(), |
| 3708 | | - err |
| 3709 | | - ) |
| 3710 | | - })?; |
| 3711 | | - Ok(RunCapture { |
| 3712 | | - exit_code: output.status.code().unwrap_or(-1), |
| 3713 | | - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 3714 | | - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 3715 | | - }) |
| 5267 | + Ok(summary) |
| 3716 | 5268 | } |
| 3717 | 5269 | |
| 3718 | | -fn normalize_run_signature(run: &RunCapture) -> RunSignature { |
| 3719 | | - RunSignature { |
| 3720 | | - exit_code: run.exit_code, |
| 3721 | | - stdout: normalize_behavior_text(&run.stdout), |
| 3722 | | - stderr: normalize_behavior_text(&run.stderr), |
| 3723 | | - } |
| 3724 | | -} |
| 3725 | | - |
| 3726 | | -fn normalize_behavior_text(text: &str) -> String { |
| 3727 | | - text.replace("\r\n", "\n") |
| 3728 | | - .lines() |
| 3729 | | - .map(normalize_behavior_line) |
| 3730 | | - .collect::<Vec<_>>() |
| 3731 | | - .join("\n") |
| 3732 | | - .trim() |
| 3733 | | - .to_string() |
| 3734 | | -} |
| 3735 | | - |
| 3736 | | -fn normalize_behavior_line(line: &str) -> String { |
| 3737 | | - line.split_whitespace() |
| 3738 | | - .map(normalize_behavior_token) |
| 3739 | | - .collect::<Vec<_>>() |
| 3740 | | - .join(" ") |
| 5270 | +fn selected_opt_levels(case: &CaseSpec, config: &RunConfig) -> Vec<OptLevel> { |
| 5271 | + case.opt_levels |
| 5272 | + .iter() |
| 5273 | + .copied() |
| 5274 | + .filter(|level| { |
| 5275 | + config |
| 5276 | + .opt_filter |
| 5277 | + .as_ref() |
| 5278 | + .map(|filter| filter.contains(level)) |
| 5279 | + .unwrap_or(true) |
| 5280 | + }) |
| 5281 | + .collect() |
| 3741 | 5282 | } |
| 3742 | 5283 | |
| 3743 | | -fn normalize_behavior_token(token: &str) -> String { |
| 3744 | | - if let Some(number) = parse_numeric_token(token) { |
| 3745 | | - format!("num:{:.6e}", number) |
| 3746 | | - } else { |
| 3747 | | - token.to_string() |
| 5284 | +fn execute_case_cell( |
| 5285 | + suite: &SuiteSpec, |
| 5286 | + case: &CaseSpec, |
| 5287 | + opt_level: OptLevel, |
| 5288 | + config: &RunConfig, |
| 5289 | +) -> Result<Outcome, String> { |
| 5290 | + if case.is_generic_compare() { |
| 5291 | + return execute_generic_compare_case_cell(suite, case, opt_level, config); |
| 3748 | 5292 | } |
| 3749 | | -} |
| 3750 | | - |
| 3751 | | -fn parse_numeric_token(token: &str) -> Option<f64> { |
| 3752 | | - if token.is_empty() { |
| 3753 | | - return None; |
| 5293 | + if case.is_generic_introspect() { |
| 5294 | + return execute_generic_introspect_case_cell(suite, case, opt_level, config); |
| 3754 | 5295 | } |
| 3755 | 5296 | |
| 3756 | | - let normalized = token |
| 3757 | | - .trim() |
| 3758 | | - .trim_end_matches(',') |
| 3759 | | - .trim_end_matches(';') |
| 3760 | | - .replace('D', "E") |
| 3761 | | - .replace('d', "e"); |
| 5297 | + let effective_status = status_for_opt(case, opt_level); |
| 5298 | + if let EffectiveStatus::Future(reason) = &effective_status { |
| 5299 | + if !config.include_future { |
| 5300 | + return Ok(Outcome { |
| 5301 | + suite: suite.name.clone(), |
| 5302 | + case: case.name.clone(), |
| 5303 | + opt_level, |
| 5304 | + kind: OutcomeKind::Future, |
| 5305 | + detail: reason.clone(), |
| 5306 | + bundle: None, |
| 5307 | + primary_backend: None, |
| 5308 | + consistency_observations: Vec::new(), |
| 5309 | + }); |
| 5310 | + } |
| 5311 | + } |
| 3762 | 5312 | |
| 3763 | | - normalized.parse::<f64>().ok() |
| 3764 | | -} |
| 5313 | + let mut requested = case.requested.clone(); |
| 5314 | + if config.all_stages { |
| 5315 | + requested.extend(Stage::ALL); |
| 5316 | + } |
| 5317 | + for expectation in &case.expectations { |
| 5318 | + ensure_target_stage(expectation, &mut requested); |
| 5319 | + } |
| 5320 | + if !case.reference_compilers.is_empty() { |
| 5321 | + requested.insert(Stage::Run); |
| 5322 | + } |
| 5323 | + for check in &case.consistency_checks { |
| 5324 | + ensure_consistency_stage(*check, &mut requested); |
| 5325 | + } |
| 3765 | 5326 | |
| 3766 | | -fn format_reference_summary(references: &[ReferenceResult]) -> String { |
| 3767 | | - references |
| 3768 | | - .iter() |
| 3769 | | - .map(format_reference_result) |
| 3770 | | - .collect::<Vec<_>>() |
| 3771 | | - .join("\n\n") |
| 3772 | | -} |
| 5327 | + let prepared = prepare_case_input(case, suite, opt_level)?; |
| 5328 | + let selected_backend = |
| 5329 | + select_primary_capture_backend(case, &requested, opt_level, &config.tools); |
| 3773 | 5330 | |
| 3774 | | -fn format_reference_result(reference: &ReferenceResult) -> String { |
| 3775 | | - let mut lines = Vec::new(); |
| 3776 | | - lines.push(reference.compiler.as_str().to_string()); |
| 3777 | | - lines.push(format!("command: {}", reference.compile_command)); |
| 3778 | | - lines.push(format!("compile exit: {}", reference.compile_exit_code)); |
| 3779 | | - if !reference.compile_stdout.trim().is_empty() { |
| 3780 | | - lines.push(format!( |
| 3781 | | - "compile stdout:\n{}", |
| 3782 | | - reference.compile_stdout.trim_end() |
| 3783 | | - )); |
| 3784 | | - } |
| 3785 | | - if !reference.compile_stderr.trim().is_empty() { |
| 3786 | | - lines.push(format!( |
| 3787 | | - "compile stderr:\n{}", |
| 3788 | | - reference.compile_stderr.trim_end() |
| 5331 | + if let Some(detail) = legacy_unavailable_backend_detail(case, &selected_backend) { |
| 5332 | + cleanup_prepared_input(&prepared); |
| 5333 | + return Ok(outcome_from_status_and_execution( |
| 5334 | + suite, |
| 5335 | + case, |
| 5336 | + opt_level, |
| 5337 | + capability_effective_status(&effective_status, case), |
| 5338 | + Err(detail), |
| 5339 | + Some(PrimaryBackendReport::from_selected(&selected_backend)), |
| 5340 | + Vec::new(), |
| 3789 | 5341 | )); |
| 3790 | 5342 | } |
| 3791 | | - match (&reference.run, &reference.run_error) { |
| 3792 | | - (Some(run), _) => { |
| 3793 | | - lines.push(format!("run\n{}", format_run_capture(run))); |
| 5343 | + |
| 5344 | + if config.verbose { |
| 5345 | + let stage_list = requested |
| 5346 | + .iter() |
| 5347 | + .map(Stage::as_str) |
| 5348 | + .collect::<Vec<_>>() |
| 5349 | + .join(", "); |
| 5350 | + let refs = if case.reference_compilers.is_empty() { |
| 5351 | + "none".to_string() |
| 5352 | + } else { |
| 5353 | + case.reference_compilers |
| 5354 | + .iter() |
| 5355 | + .map(ReferenceCompiler::as_str) |
| 5356 | + .collect::<Vec<_>>() |
| 5357 | + .join(", ") |
| 5358 | + }; |
| 5359 | + println!(" source: {}", case.source_label()); |
| 5360 | + if case.is_graph() { |
| 5361 | + for file in &case.graph_files { |
| 5362 | + println!(" file: {}", file.display()); |
| 5363 | + } |
| 5364 | + println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 3794 | 5365 | } |
| 3795 | | - (None, Some(err)) => { |
| 3796 | | - lines.push(format!("run error: {}", err)); |
| 5366 | + println!(" opt: {}", opt_level.as_str()); |
| 5367 | + println!(" stages: {}", stage_list); |
| 5368 | + println!( |
| 5369 | + " primary_backend: {} ({})", |
| 5370 | + selected_backend.kind.as_str(), |
| 5371 | + selected_backend.backend.mode_name() |
| 5372 | + ); |
| 5373 | + println!( |
| 5374 | + " primary_backend_detail: {}", |
| 5375 | + selected_backend.backend.description() |
| 5376 | + ); |
| 5377 | + println!(" refs: {}", refs); |
| 5378 | + if !case.consistency_checks.is_empty() { |
| 5379 | + println!(" repeat: {}", case.repeat_count); |
| 3797 | 5380 | } |
| 3798 | | - (None, None) => {} |
| 3799 | 5381 | } |
| 3800 | | - lines.join("\n") |
| 3801 | | -} |
| 3802 | 5382 | |
| 3803 | | -fn format_run_capture(run: &RunCapture) -> String { |
| 3804 | | - let stdout = if run.stdout.is_empty() { |
| 3805 | | - "<empty>".to_string() |
| 3806 | | - } else { |
| 3807 | | - run.stdout.trim_end().to_string() |
| 3808 | | - }; |
| 3809 | | - let stderr = if run.stderr.is_empty() { |
| 3810 | | - "<empty>".to_string() |
| 3811 | | - } else { |
| 3812 | | - run.stderr.trim_end().to_string() |
| 5383 | + let references = run_reference_compilers(&prepared, case, opt_level, &config.tools); |
| 5384 | + let mut artifacts = ExecutionArtifacts { |
| 5385 | + requested, |
| 5386 | + armfortas: None, |
| 5387 | + armfortas_failure: None, |
| 5388 | + armfortas_observation: None, |
| 5389 | + references, |
| 5390 | + reference_observations: Vec::new(), |
| 5391 | + consistency_issues: Vec::new(), |
| 3813 | 5392 | }; |
| 3814 | | - format!( |
| 3815 | | - "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 3816 | | - run.exit_code, stdout, stderr |
| 3817 | | - ) |
| 3818 | | -} |
| 5393 | + artifacts.reference_observations = artifacts |
| 5394 | + .references |
| 5395 | + .iter() |
| 5396 | + .map(|reference| { |
| 5397 | + observed_program_from_reference_result( |
| 5398 | + &prepared.compiler_source, |
| 5399 | + opt_level, |
| 5400 | + default_differential_artifacts(), |
| 5401 | + reference, |
| 5402 | + ) |
| 5403 | + }) |
| 5404 | + .collect(); |
| 3819 | 5405 | |
| 3820 | | -fn format_run_signature(signature: &RunSignature) -> String { |
| 3821 | | - let stdout = if signature.stdout.is_empty() { |
| 3822 | | - "<empty>".to_string() |
| 3823 | | - } else { |
| 3824 | | - signature.stdout.clone() |
| 3825 | | - }; |
| 3826 | | - let stderr = if signature.stderr.is_empty() { |
| 3827 | | - "<empty>".to_string() |
| 3828 | | - } else { |
| 3829 | | - signature.stderr.clone() |
| 5406 | + match execute_primary_armfortas( |
| 5407 | + &prepared, |
| 5408 | + opt_level, |
| 5409 | + &artifacts.requested, |
| 5410 | + &selected_backend, |
| 5411 | + ) { |
| 5412 | + Ok(result) => artifacts.armfortas = Some(result), |
| 5413 | + Err(failure) => artifacts.armfortas_failure = Some(failure), |
| 5414 | + } |
| 5415 | + |
| 5416 | + let execution = match (&artifacts.armfortas, &artifacts.armfortas_failure) { |
| 5417 | + (Some(result), None) => { |
| 5418 | + if has_failure_expectation(case) { |
| 5419 | + Err(format!( |
| 5420 | + "expected armfortas to fail ({}) but compilation succeeded", |
| 5421 | + expected_failure_description(case) |
| 5422 | + )) |
| 5423 | + } else { |
| 5424 | + let observed = legacy_success_observed_program( |
| 5425 | + case, |
| 5426 | + &prepared.compiler_source, |
| 5427 | + opt_level, |
| 5428 | + result, |
| 5429 | + !artifacts.references.is_empty(), |
| 5430 | + &config.tools, |
| 5431 | + ); |
| 5432 | + artifacts.armfortas_observation = Some(observed.clone()); |
| 5433 | + let mut execution = evaluate_observation_expectations(case, &observed); |
| 5434 | + if execution.is_ok() && !artifacts.references.is_empty() { |
| 5435 | + let references = artifacts |
| 5436 | + .reference_observations |
| 5437 | + .iter() |
| 5438 | + .map(|observed| observed.observation.clone()) |
| 5439 | + .collect::<Vec<_>>(); |
| 5440 | + execution = compare_differential(&observed.observation, &references); |
| 5441 | + } |
| 5442 | + if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5443 | + artifacts.consistency_issues = |
| 5444 | + if legacy_case_uses_generic_consistency_checks(case) { |
| 5445 | + run_generic_consistency_checks( |
| 5446 | + &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5447 | + case, |
| 5448 | + &prepared.compiler_source, |
| 5449 | + opt_level, |
| 5450 | + &config.tools, |
| 5451 | + ) |
| 5452 | + } else { |
| 5453 | + run_consistency_checks( |
| 5454 | + case, |
| 5455 | + &prepared, |
| 5456 | + opt_level, |
| 5457 | + result, |
| 5458 | + &config.tools, |
| 5459 | + ) |
| 5460 | + }; |
| 5461 | + if !artifacts.consistency_issues.is_empty() { |
| 5462 | + execution = Err(format_consistency_issues(&artifacts.consistency_issues)); |
| 5463 | + } |
| 5464 | + } |
| 5465 | + execution |
| 5466 | + } |
| 5467 | + } |
| 5468 | + (None, Some(failure)) => { |
| 5469 | + let observed = |
| 5470 | + legacy_failure_observed_program(&prepared.compiler_source, case, failure); |
| 5471 | + artifacts.armfortas_observation = Some(observed.clone()); |
| 5472 | + let mut execution = |
| 5473 | + evaluate_failed_armfortas_with_observed(case, &artifacts, &observed); |
| 5474 | + if execution.is_ok() && !artifacts.references.is_empty() { |
| 5475 | + execution = |
| 5476 | + Err("differential comparison requires a successful armfortas run".to_string()); |
| 5477 | + } |
| 5478 | + execution |
| 5479 | + } |
| 5480 | + (Some(_), Some(_)) => Err("armfortas produced both a result and a failure".into()), |
| 5481 | + (None, None) => Err("armfortas produced neither a result nor a failure".into()), |
| 3830 | 5482 | }; |
| 3831 | | - format!( |
| 3832 | | - "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 3833 | | - signature.exit_code, stdout, stderr |
| 3834 | | - ) |
| 3835 | | -} |
| 3836 | 5483 | |
| 3837 | | -#[derive(Debug, Clone, PartialEq, Eq)] |
| 3838 | | -struct ObjectSnapshot { |
| 3839 | | - text: String, |
| 3840 | | - load_commands: String, |
| 3841 | | - relocations: String, |
| 3842 | | - symbols: String, |
| 3843 | | -} |
| 5484 | + let consistency_observations = artifacts |
| 5485 | + .consistency_issues |
| 5486 | + .iter() |
| 5487 | + .map(ConsistencyIssue::observation) |
| 5488 | + .collect::<Vec<_>>(); |
| 5489 | + let primary_backend = Some(PrimaryBackendReport::from_selected(&selected_backend)); |
| 3844 | 5490 | |
| 3845 | | -#[derive(Debug, Clone)] |
| 3846 | | -struct TextRun { |
| 3847 | | - label: String, |
| 3848 | | - command: String, |
| 3849 | | - normalized: String, |
| 3850 | | -} |
| 5491 | + let mut outcome = outcome_from_status_and_execution( |
| 5492 | + suite, |
| 5493 | + case, |
| 5494 | + opt_level, |
| 5495 | + effective_status, |
| 5496 | + execution, |
| 5497 | + primary_backend, |
| 5498 | + consistency_observations, |
| 5499 | + ); |
| 3851 | 5500 | |
| 3852 | | -#[derive(Debug, Clone)] |
| 3853 | | -struct BehaviorRun { |
| 3854 | | - label: String, |
| 3855 | | - command: String, |
| 3856 | | - signature: RunSignature, |
| 3857 | | - run: RunCapture, |
| 3858 | | -} |
| 5501 | + let should_bundle = matches!(outcome.kind, OutcomeKind::Fail | OutcomeKind::Xpass) |
| 5502 | + || (matches!(outcome.kind, OutcomeKind::Xfail) && !artifacts.consistency_issues.is_empty()); |
| 3859 | 5503 | |
| 3860 | | -#[derive(Debug, Clone)] |
| 3861 | | -struct ObjectRun { |
| 3862 | | - label: String, |
| 3863 | | - command: String, |
| 3864 | | - snapshot: ObjectSnapshot, |
| 3865 | | -} |
| 3866 | | - |
| 3867 | | -fn object_snapshot(path: &Path, tools: &ToolchainConfig) -> Result<ObjectSnapshot, String> { |
| 3868 | | - let text = normalize_tool_output(&tool_output( |
| 3869 | | - tools.otool_bin(), |
| 3870 | | - &["-t", path.to_str().unwrap()], |
| 3871 | | - )?); |
| 3872 | | - let load_commands = normalize_tool_output(&tool_output( |
| 3873 | | - tools.otool_bin(), |
| 3874 | | - &["-l", path.to_str().unwrap()], |
| 3875 | | - )?); |
| 3876 | | - let relocations = normalize_tool_output(&tool_output( |
| 3877 | | - tools.otool_bin(), |
| 3878 | | - &["-rv", path.to_str().unwrap()], |
| 3879 | | - )?); |
| 3880 | | - let symbols = normalize_tool_output(&tool_output( |
| 3881 | | - tools.nm_bin(), |
| 3882 | | - &["-m", path.to_str().unwrap()], |
| 3883 | | - )?); |
| 3884 | | - |
| 3885 | | - Ok(ObjectSnapshot { |
| 3886 | | - text, |
| 3887 | | - load_commands, |
| 3888 | | - relocations, |
| 3889 | | - symbols, |
| 3890 | | - }) |
| 3891 | | -} |
| 3892 | | - |
| 3893 | | -fn tool_output(tool: &str, args: &[&str]) -> Result<String, String> { |
| 3894 | | - let output = Command::new(tool) |
| 3895 | | - .args(args) |
| 3896 | | - .output() |
| 3897 | | - .map_err(|e| format!("cannot run {}: {}", tool, e))?; |
| 3898 | | - if output.status.success() { |
| 3899 | | - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) |
| 3900 | | - } else { |
| 3901 | | - Err(format!( |
| 3902 | | - "{} failed:\n{}", |
| 3903 | | - tool, |
| 3904 | | - String::from_utf8_lossy(&output.stderr) |
| 3905 | | - )) |
| 3906 | | - } |
| 3907 | | -} |
| 3908 | | - |
| 3909 | | -fn normalize_tool_output(text: &str) -> String { |
| 3910 | | - text.lines() |
| 3911 | | - .filter(|line| !line.trim_end().ends_with(".o:")) |
| 3912 | | - .map(str::trim_end) |
| 3913 | | - .collect::<Vec<_>>() |
| 3914 | | - .join("\n") |
| 3915 | | -} |
| 3916 | | - |
| 3917 | | -fn read_text_artifact(path: &Path) -> Result<String, String> { |
| 3918 | | - fs::read_to_string(path).map_err(|e| format!("cannot read '{}': {}", path.display(), e)) |
| 3919 | | -} |
| 3920 | | - |
| 3921 | | -fn normalize_text_artifact(text: &str) -> String { |
| 3922 | | - text.replace("\r\n", "\n") |
| 3923 | | - .lines() |
| 3924 | | - .map(str::trim_end) |
| 3925 | | - .collect::<Vec<_>>() |
| 3926 | | - .join("\n") |
| 3927 | | -} |
| 3928 | | - |
| 3929 | | -fn count_unique_strings<'a>(values: impl IntoIterator<Item = &'a str>) -> usize { |
| 3930 | | - values.into_iter().collect::<BTreeSet<_>>().len() |
| 3931 | | -} |
| 3932 | | - |
| 3933 | | -fn first_distinct_text_pair(runs: &[TextRun]) -> Option<(&TextRun, &TextRun)> { |
| 3934 | | - for left_index in 0..runs.len() { |
| 3935 | | - for right_index in (left_index + 1)..runs.len() { |
| 3936 | | - if runs[left_index].normalized != runs[right_index].normalized { |
| 3937 | | - return Some((&runs[left_index], &runs[right_index])); |
| 5504 | + if should_bundle { |
| 5505 | + match write_failure_bundle(suite, case, &prepared, &outcome, &artifacts) { |
| 5506 | + Ok(bundle) => outcome.bundle = Some(bundle), |
| 5507 | + Err(err) => { |
| 5508 | + if outcome.detail.is_empty() { |
| 5509 | + outcome.detail = format!("failed to write failure bundle: {}", err); |
| 5510 | + } else { |
| 5511 | + outcome.detail.push_str(&format!( |
| 5512 | + "\n\nwarning: failed to write failure bundle: {}", |
| 5513 | + err |
| 5514 | + )); |
| 5515 | + } |
| 3938 | 5516 | } |
| 3939 | 5517 | } |
| 3940 | 5518 | } |
| 3941 | | - None |
| 3942 | | -} |
| 3943 | 5519 | |
| 3944 | | -fn count_unique_run_signatures<'a>(values: impl IntoIterator<Item = &'a RunSignature>) -> usize { |
| 3945 | | - values.into_iter().collect::<BTreeSet<_>>().len() |
| 5520 | + cleanup_prepared_input(&prepared); |
| 5521 | + cleanup_consistency_issues(&artifacts.consistency_issues); |
| 5522 | + |
| 5523 | + Ok(outcome) |
| 3946 | 5524 | } |
| 3947 | 5525 | |
| 3948 | | -fn first_distinct_behavior_pair(runs: &[BehaviorRun]) -> Option<(&BehaviorRun, &BehaviorRun)> { |
| 3949 | | - for left_index in 0..runs.len() { |
| 3950 | | - for right_index in (left_index + 1)..runs.len() { |
| 3951 | | - if runs[left_index].signature != runs[right_index].signature { |
| 3952 | | - return Some((&runs[left_index], &runs[right_index])); |
| 3953 | | - } |
| 3954 | | - } |
| 5526 | +fn legacy_success_observed_program( |
| 5527 | + case: &CaseSpec, |
| 5528 | + program: &Path, |
| 5529 | + opt_level: OptLevel, |
| 5530 | + result: &CaptureResult, |
| 5531 | + has_references: bool, |
| 5532 | + tools: &ToolchainConfig, |
| 5533 | +) -> ObservedProgram { |
| 5534 | + let mut requested_artifacts = expected_artifacts_for_legacy_case(case); |
| 5535 | + if has_references { |
| 5536 | + requested_artifacts.extend(default_differential_artifacts()); |
| 3955 | 5537 | } |
| 3956 | | - None |
| 3957 | | -} |
| 3958 | 5538 | |
| 3959 | | -fn first_distinct_object_pair(runs: &[ObjectRun]) -> Option<(&ObjectRun, &ObjectRun)> { |
| 3960 | | - for left_index in 0..runs.len() { |
| 3961 | | - for right_index in (left_index + 1)..runs.len() { |
| 3962 | | - if runs[left_index].snapshot != runs[right_index].snapshot { |
| 3963 | | - return Some((&runs[left_index], &runs[right_index])); |
| 5539 | + if legacy_case_uses_generic_observation_execution(case, &case.requested) { |
| 5540 | + if let Ok(observation) = observe_compiler( |
| 5541 | + &CompilerSpec::Named(NamedCompiler::Armfortas), |
| 5542 | + program, |
| 5543 | + opt_level, |
| 5544 | + &requested_artifacts, |
| 5545 | + tools, |
| 5546 | + ) { |
| 5547 | + if observation.compile_exit_code == 0 { |
| 5548 | + return ObservedProgram { |
| 5549 | + observation, |
| 5550 | + requested_artifacts, |
| 5551 | + }; |
| 3964 | 5552 | } |
| 3965 | 5553 | } |
| 3966 | 5554 | } |
| 3967 | | - None |
| 3968 | | -} |
| 3969 | 5555 | |
| 3970 | | -fn render_object_snapshot(snapshot: &ObjectSnapshot) -> String { |
| 3971 | | - format!( |
| 3972 | | - "== text ==\n{}\n\n== load_commands ==\n{}\n\n== relocations ==\n{}\n\n== symbols ==\n{}", |
| 3973 | | - snapshot.text, snapshot.load_commands, snapshot.relocations, snapshot.symbols |
| 3974 | | - ) |
| 5556 | + observed_program_from_armfortas_capture(program, opt_level, requested_artifacts, result, None) |
| 3975 | 5557 | } |
| 3976 | 5558 | |
| 3977 | | -fn parse_object_snapshot_text(text: &str) -> Result<ObjectSnapshot, String> { |
| 3978 | | - let text = text |
| 3979 | | - .strip_prefix("== text ==\n") |
| 3980 | | - .ok_or_else(|| "object snapshot was missing the '== text ==' header".to_string())?; |
| 3981 | | - let (text, rest) = text |
| 3982 | | - .split_once("\n\n== load_commands ==\n") |
| 3983 | | - .ok_or_else(|| { |
| 3984 | | - "object snapshot was missing the '== load_commands ==' section".to_string() |
| 3985 | | - })?; |
| 3986 | | - let (load_commands, rest) = rest |
| 3987 | | - .split_once("\n\n== relocations ==\n") |
| 3988 | | - .ok_or_else(|| "object snapshot was missing the '== relocations ==' section".to_string())?; |
| 3989 | | - let (relocations, symbols) = rest |
| 3990 | | - .split_once("\n\n== symbols ==\n") |
| 3991 | | - .ok_or_else(|| "object snapshot was missing the '== symbols ==' section".to_string())?; |
| 3992 | | - |
| 3993 | | - Ok(ObjectSnapshot { |
| 3994 | | - text: text.to_string(), |
| 3995 | | - load_commands: load_commands.to_string(), |
| 3996 | | - relocations: relocations.to_string(), |
| 3997 | | - symbols: symbols.to_string(), |
| 3998 | | - }) |
| 5559 | +fn legacy_case_uses_generic_observation_execution( |
| 5560 | + case: &CaseSpec, |
| 5561 | + requested: &BTreeSet<Stage>, |
| 5562 | +) -> bool { |
| 5563 | + !has_failure_expectation(case) |
| 5564 | + && !requested.is_empty() |
| 5565 | + && requested |
| 5566 | + .iter() |
| 5567 | + .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)) |
| 3999 | 5568 | } |
| 4000 | 5569 | |
| 4001 | | -fn describe_text_difference( |
| 4002 | | - expected: &str, |
| 4003 | | - actual: &str, |
| 4004 | | - left_label: &str, |
| 4005 | | - right_label: &str, |
| 4006 | | -) -> String { |
| 4007 | | - let expected_lines: Vec<&str> = expected.lines().collect(); |
| 4008 | | - let actual_lines: Vec<&str> = actual.lines().collect(); |
| 4009 | | - let shared = expected_lines.len().min(actual_lines.len()); |
| 4010 | | - |
| 4011 | | - for index in 0..shared { |
| 4012 | | - if expected_lines[index] != actual_lines[index] { |
| 4013 | | - return format!( |
| 4014 | | - "first differing line: {}\n{}: {}\n{}: {}", |
| 4015 | | - index + 1, |
| 4016 | | - left_label, |
| 4017 | | - expected_lines[index], |
| 4018 | | - right_label, |
| 4019 | | - actual_lines[index] |
| 4020 | | - ); |
| 5570 | +fn execute_generic_compare_case_cell( |
| 5571 | + suite: &SuiteSpec, |
| 5572 | + case: &CaseSpec, |
| 5573 | + opt_level: OptLevel, |
| 5574 | + config: &RunConfig, |
| 5575 | +) -> Result<Outcome, String> { |
| 5576 | + let effective_status = status_for_opt(case, opt_level); |
| 5577 | + if let EffectiveStatus::Future(reason) = &effective_status { |
| 5578 | + if !config.include_future { |
| 5579 | + return Ok(Outcome { |
| 5580 | + suite: suite.name.clone(), |
| 5581 | + case: case.name.clone(), |
| 5582 | + opt_level, |
| 5583 | + kind: OutcomeKind::Future, |
| 5584 | + detail: reason.clone(), |
| 5585 | + bundle: None, |
| 5586 | + primary_backend: None, |
| 5587 | + consistency_observations: Vec::new(), |
| 5588 | + }); |
| 4021 | 5589 | } |
| 4022 | 5590 | } |
| 4023 | 5591 | |
| 4024 | | - format!( |
| 4025 | | - "snapshot length differs\n{} lines: {}\n{} lines: {}", |
| 4026 | | - left_label, |
| 4027 | | - expected_lines.len(), |
| 4028 | | - right_label, |
| 4029 | | - actual_lines.len() |
| 4030 | | - ) |
| 4031 | | -} |
| 5592 | + let generic = case |
| 5593 | + .generic_compare |
| 5594 | + .as_ref() |
| 5595 | + .ok_or_else(|| "missing generic compare case configuration".to_string())?; |
| 5596 | + let prepared = prepare_case_input(case, suite, opt_level)?; |
| 4032 | 5597 | |
| 4033 | | -fn describe_object_difference( |
| 4034 | | - expected: &ObjectSnapshot, |
| 4035 | | - actual: &ObjectSnapshot, |
| 4036 | | - left_label: &str, |
| 4037 | | - right_label: &str, |
| 4038 | | -) -> String { |
| 4039 | | - let mut differing = Vec::new(); |
| 4040 | | - if expected.text != actual.text { |
| 4041 | | - differing.push(("text", &expected.text, &actual.text)); |
| 4042 | | - } |
| 4043 | | - if expected.load_commands != actual.load_commands { |
| 4044 | | - differing.push(( |
| 4045 | | - "load_commands", |
| 4046 | | - &expected.load_commands, |
| 4047 | | - &actual.load_commands, |
| 4048 | | - )); |
| 4049 | | - } |
| 4050 | | - if expected.relocations != actual.relocations { |
| 4051 | | - differing.push(("relocations", &expected.relocations, &actual.relocations)); |
| 4052 | | - } |
| 4053 | | - if expected.symbols != actual.symbols { |
| 4054 | | - differing.push(("symbols", &expected.symbols, &actual.symbols)); |
| 5598 | + if let Some(detail) = compare_capability_issue( |
| 5599 | + &generic.left, |
| 5600 | + &generic.right, |
| 5601 | + &generic.artifacts, |
| 5602 | + &config.tools, |
| 5603 | + ) { |
| 5604 | + let mut outcome = outcome_from_status_and_execution( |
| 5605 | + suite, |
| 5606 | + case, |
| 5607 | + opt_level, |
| 5608 | + capability_effective_status(&effective_status, case), |
| 5609 | + Err(detail), |
| 5610 | + None, |
| 5611 | + Vec::new(), |
| 5612 | + ); |
| 5613 | + outcome.detail = outcome.detail.trim().to_string(); |
| 5614 | + cleanup_prepared_input(&prepared); |
| 5615 | + return Ok(outcome); |
| 4055 | 5616 | } |
| 4056 | 5617 | |
| 4057 | | - if differing.is_empty() { |
| 4058 | | - return "object snapshots matched".to_string(); |
| 5618 | + if config.verbose { |
| 5619 | + let artifacts = generic |
| 5620 | + .artifacts |
| 5621 | + .iter() |
| 5622 | + .map(ArtifactKey::as_str) |
| 5623 | + .collect::<Vec<_>>() |
| 5624 | + .join(", "); |
| 5625 | + println!(" source: {}", case.source_label()); |
| 5626 | + if case.is_graph() { |
| 5627 | + for file in &case.graph_files { |
| 5628 | + println!(" file: {}", file.display()); |
| 5629 | + } |
| 5630 | + println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5631 | + } |
| 5632 | + println!( |
| 5633 | + " compare: {} vs {}", |
| 5634 | + generic.left.display_name(), |
| 5635 | + generic.right.display_name() |
| 5636 | + ); |
| 5637 | + println!(" opt: {}", opt_level.as_str()); |
| 5638 | + println!(" artifacts: {}", artifacts); |
| 4059 | 5639 | } |
| 4060 | 5640 | |
| 4061 | | - let component_list = differing |
| 4062 | | - .iter() |
| 4063 | | - .map(|(name, _, _)| *name) |
| 4064 | | - .collect::<Vec<_>>() |
| 4065 | | - .join(", "); |
| 4066 | | - let (first_name, first_expected, first_actual) = differing[0]; |
| 5641 | + let result = run_compare(&CompareConfig { |
| 5642 | + left: generic.left.clone(), |
| 5643 | + right: generic.right.clone(), |
| 5644 | + program: prepared.compiler_source.clone(), |
| 5645 | + opt_level, |
| 5646 | + artifacts: generic.artifacts.clone(), |
| 5647 | + json_report: None, |
| 5648 | + markdown_report: None, |
| 5649 | + tools: config.tools.clone(), |
| 5650 | + }); |
| 4067 | 5651 | |
| 4068 | | - format!( |
| 4069 | | - "differing object components: {}\n{}\n{}", |
| 4070 | | - component_list, |
| 4071 | | - format!("first differing component: {}", first_name), |
| 4072 | | - describe_text_difference(first_expected, first_actual, left_label, right_label) |
| 4073 | | - ) |
| 4074 | | -} |
| 5652 | + let execution = if has_failure_expectation(case) { |
| 5653 | + Err("suite-v2 compare cases do not support expect-fail rules".to_string()) |
| 5654 | + } else if let Ok(result) = &result { |
| 5655 | + evaluate_compare_expectations(case, result) |
| 5656 | + } else { |
| 5657 | + Err(result.unwrap_err()) |
| 5658 | + }; |
| 4075 | 5659 | |
| 4076 | | -fn describe_run_difference( |
| 4077 | | - expected: &RunCapture, |
| 4078 | | - actual: &RunCapture, |
| 4079 | | - left_label: &str, |
| 4080 | | - right_label: &str, |
| 4081 | | -) -> String { |
| 4082 | | - let expected = normalize_run_signature(expected); |
| 4083 | | - let actual = normalize_run_signature(actual); |
| 4084 | | - let mut differing = Vec::new(); |
| 4085 | | - if expected.exit_code != actual.exit_code { |
| 4086 | | - differing.push("exit_code"); |
| 4087 | | - } |
| 4088 | | - if expected.stdout != actual.stdout { |
| 4089 | | - differing.push("stdout"); |
| 4090 | | - } |
| 4091 | | - if expected.stderr != actual.stderr { |
| 4092 | | - differing.push("stderr"); |
| 4093 | | - } |
| 4094 | | - |
| 4095 | | - if differing.is_empty() { |
| 4096 | | - return "runtime behavior matched".to_string(); |
| 4097 | | - } |
| 4098 | | - |
| 4099 | | - let component_list = differing.join(", "); |
| 4100 | | - match differing[0] { |
| 4101 | | - "exit_code" => format!( |
| 4102 | | - "differing runtime components: {}\nfirst differing component: exit_code\n{}: {}\n{}: {}", |
| 4103 | | - component_list, |
| 4104 | | - left_label, |
| 4105 | | - expected.exit_code, |
| 4106 | | - right_label, |
| 4107 | | - actual.exit_code |
| 4108 | | - ), |
| 4109 | | - "stdout" => format!( |
| 4110 | | - "differing runtime components: {}\nfirst differing component: stdout\n{}", |
| 4111 | | - component_list, |
| 4112 | | - describe_text_difference(&expected.stdout, &actual.stdout, left_label, right_label) |
| 4113 | | - ), |
| 4114 | | - "stderr" => format!( |
| 4115 | | - "differing runtime components: {}\nfirst differing component: stderr\n{}", |
| 4116 | | - component_list, |
| 4117 | | - describe_text_difference(&expected.stderr, &actual.stderr, left_label, right_label) |
| 4118 | | - ), |
| 4119 | | - _ => unreachable!("only known runtime components are compared"), |
| 4120 | | - } |
| 4121 | | -} |
| 4122 | | - |
| 4123 | | -fn varying_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 4124 | | - object_components_by_variation(snapshots, true) |
| 4125 | | -} |
| 5660 | + let mut outcome = match (effective_status, execution) { |
| 5661 | + (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5662 | + suite: suite.name.clone(), |
| 5663 | + case: case.name.clone(), |
| 5664 | + opt_level, |
| 5665 | + kind: OutcomeKind::Pass, |
| 5666 | + detail: String::new(), |
| 5667 | + bundle: None, |
| 5668 | + primary_backend: None, |
| 5669 | + consistency_observations: Vec::new(), |
| 5670 | + }, |
| 5671 | + (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5672 | + suite: suite.name.clone(), |
| 5673 | + case: case.name.clone(), |
| 5674 | + opt_level, |
| 5675 | + kind: OutcomeKind::Fail, |
| 5676 | + detail, |
| 5677 | + bundle: None, |
| 5678 | + primary_backend: None, |
| 5679 | + consistency_observations: Vec::new(), |
| 5680 | + }, |
| 5681 | + (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5682 | + suite: suite.name.clone(), |
| 5683 | + case: case.name.clone(), |
| 5684 | + opt_level, |
| 5685 | + kind: OutcomeKind::Xpass, |
| 5686 | + detail: reason, |
| 5687 | + bundle: None, |
| 5688 | + primary_backend: None, |
| 5689 | + consistency_observations: Vec::new(), |
| 5690 | + }, |
| 5691 | + (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5692 | + suite: suite.name.clone(), |
| 5693 | + case: case.name.clone(), |
| 5694 | + opt_level, |
| 5695 | + kind: OutcomeKind::Xfail, |
| 5696 | + detail: format!("{}\n{}", reason, detail), |
| 5697 | + bundle: None, |
| 5698 | + primary_backend: None, |
| 5699 | + consistency_observations: Vec::new(), |
| 5700 | + }, |
| 5701 | + (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5702 | + suite: suite.name.clone(), |
| 5703 | + case: case.name.clone(), |
| 5704 | + opt_level, |
| 5705 | + kind: OutcomeKind::Xpass, |
| 5706 | + detail: reason, |
| 5707 | + bundle: None, |
| 5708 | + primary_backend: None, |
| 5709 | + consistency_observations: Vec::new(), |
| 5710 | + }, |
| 5711 | + (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5712 | + suite: suite.name.clone(), |
| 5713 | + case: case.name.clone(), |
| 5714 | + opt_level, |
| 5715 | + kind: OutcomeKind::Future, |
| 5716 | + detail: format!("{}\n{}", reason, detail), |
| 5717 | + bundle: None, |
| 5718 | + primary_backend: None, |
| 5719 | + consistency_observations: Vec::new(), |
| 5720 | + }, |
| 5721 | + }; |
| 4126 | 5722 | |
| 4127 | | -fn stable_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 4128 | | - object_components_by_variation(snapshots, false) |
| 5723 | + outcome.detail = outcome.detail.trim().to_string(); |
| 5724 | + cleanup_prepared_input(&prepared); |
| 5725 | + Ok(outcome) |
| 4129 | 5726 | } |
| 4130 | 5727 | |
| 4131 | | -fn varying_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 4132 | | - run_components_by_variation(signatures, true) |
| 4133 | | -} |
| 5728 | +fn execute_generic_introspect_case_cell( |
| 5729 | + suite: &SuiteSpec, |
| 5730 | + case: &CaseSpec, |
| 5731 | + opt_level: OptLevel, |
| 5732 | + config: &RunConfig, |
| 5733 | +) -> Result<Outcome, String> { |
| 5734 | + let effective_status = status_for_opt(case, opt_level); |
| 5735 | + if let EffectiveStatus::Future(reason) = &effective_status { |
| 5736 | + if !config.include_future { |
| 5737 | + return Ok(Outcome { |
| 5738 | + suite: suite.name.clone(), |
| 5739 | + case: case.name.clone(), |
| 5740 | + opt_level, |
| 5741 | + kind: OutcomeKind::Future, |
| 5742 | + detail: reason.clone(), |
| 5743 | + bundle: None, |
| 5744 | + primary_backend: None, |
| 5745 | + consistency_observations: Vec::new(), |
| 5746 | + }); |
| 5747 | + } |
| 5748 | + } |
| 4134 | 5749 | |
| 4135 | | -fn stable_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 4136 | | - run_components_by_variation(signatures, false) |
| 4137 | | -} |
| 5750 | + let generic = case |
| 5751 | + .generic_introspect |
| 5752 | + .as_ref() |
| 5753 | + .ok_or_else(|| "missing generic introspection case configuration".to_string())?; |
| 5754 | + let prepared = prepare_case_input(case, suite, opt_level)?; |
| 4138 | 5755 | |
| 4139 | | -fn object_components_by_variation( |
| 4140 | | - snapshots: &[&ObjectSnapshot], |
| 4141 | | - want_varying: bool, |
| 4142 | | -) -> Vec<&'static str> { |
| 4143 | | - let components = [ |
| 4144 | | - ( |
| 4145 | | - "text", |
| 4146 | | - snapshots |
| 4147 | | - .iter() |
| 4148 | | - .map(|snapshot| snapshot.text.as_str()) |
| 4149 | | - .collect::<Vec<_>>(), |
| 4150 | | - ), |
| 4151 | | - ( |
| 4152 | | - "load_commands", |
| 4153 | | - snapshots |
| 4154 | | - .iter() |
| 4155 | | - .map(|snapshot| snapshot.load_commands.as_str()) |
| 4156 | | - .collect::<Vec<_>>(), |
| 4157 | | - ), |
| 4158 | | - ( |
| 4159 | | - "relocations", |
| 4160 | | - snapshots |
| 4161 | | - .iter() |
| 4162 | | - .map(|snapshot| snapshot.relocations.as_str()) |
| 4163 | | - .collect::<Vec<_>>(), |
| 4164 | | - ), |
| 4165 | | - ( |
| 4166 | | - "symbols", |
| 4167 | | - snapshots |
| 4168 | | - .iter() |
| 4169 | | - .map(|snapshot| snapshot.symbols.as_str()) |
| 4170 | | - .collect::<Vec<_>>(), |
| 4171 | | - ), |
| 4172 | | - ]; |
| 5756 | + if let Some(detail) = |
| 5757 | + capability_request_issue(&generic.compiler, &generic.artifacts, &config.tools) |
| 5758 | + { |
| 5759 | + let mut outcome = outcome_from_status_and_execution( |
| 5760 | + suite, |
| 5761 | + case, |
| 5762 | + opt_level, |
| 5763 | + capability_effective_status(&effective_status, case), |
| 5764 | + Err(detail), |
| 5765 | + None, |
| 5766 | + Vec::new(), |
| 5767 | + ); |
| 5768 | + outcome.detail = outcome.detail.trim().to_string(); |
| 5769 | + cleanup_prepared_input(&prepared); |
| 5770 | + return Ok(outcome); |
| 5771 | + } |
| 4173 | 5772 | |
| 4174 | | - components |
| 4175 | | - .into_iter() |
| 4176 | | - .filter_map(|(name, values)| { |
| 4177 | | - let varies = count_unique_strings(values) > 1; |
| 4178 | | - if varies == want_varying { |
| 4179 | | - Some(name) |
| 4180 | | - } else { |
| 4181 | | - None |
| 5773 | + if config.verbose { |
| 5774 | + let artifacts = generic |
| 5775 | + .artifacts |
| 5776 | + .iter() |
| 5777 | + .map(ArtifactKey::as_str) |
| 5778 | + .collect::<Vec<_>>() |
| 5779 | + .join(", "); |
| 5780 | + println!(" source: {}", case.source_label()); |
| 5781 | + if case.is_graph() { |
| 5782 | + for file in &case.graph_files { |
| 5783 | + println!(" file: {}", file.display()); |
| 4182 | 5784 | } |
| 4183 | | - }) |
| 4184 | | - .collect() |
| 4185 | | -} |
| 4186 | | - |
| 4187 | | -fn run_components_by_variation( |
| 4188 | | - signatures: &[&RunSignature], |
| 4189 | | - want_varying: bool, |
| 4190 | | -) -> Vec<&'static str> { |
| 4191 | | - let components = [ |
| 4192 | | - ( |
| 4193 | | - "exit_code", |
| 4194 | | - signatures |
| 4195 | | - .iter() |
| 4196 | | - .map(|signature| signature.exit_code.to_string()) |
| 4197 | | - .collect::<Vec<_>>(), |
| 4198 | | - ), |
| 4199 | | - ( |
| 4200 | | - "stdout", |
| 4201 | | - signatures |
| 4202 | | - .iter() |
| 4203 | | - .map(|signature| signature.stdout.clone()) |
| 4204 | | - .collect::<Vec<_>>(), |
| 4205 | | - ), |
| 4206 | | - ( |
| 4207 | | - "stderr", |
| 4208 | | - signatures |
| 4209 | | - .iter() |
| 4210 | | - .map(|signature| signature.stderr.clone()) |
| 4211 | | - .collect::<Vec<_>>(), |
| 4212 | | - ), |
| 4213 | | - ]; |
| 5785 | + println!(" compiled_as: {}", prepared.compiler_source.display()); |
| 5786 | + } |
| 5787 | + println!(" compiler: {}", generic.compiler.display_name()); |
| 5788 | + println!(" opt: {}", opt_level.as_str()); |
| 5789 | + println!(" artifacts: {}", artifacts); |
| 5790 | + if !case.reference_compilers.is_empty() { |
| 5791 | + println!( |
| 5792 | + " refs: {}", |
| 5793 | + case.reference_compilers |
| 5794 | + .iter() |
| 5795 | + .map(ReferenceCompiler::as_str) |
| 5796 | + .collect::<Vec<_>>() |
| 5797 | + .join(", ") |
| 5798 | + ); |
| 5799 | + } |
| 5800 | + if !case.consistency_checks.is_empty() { |
| 5801 | + println!( |
| 5802 | + " consistency: {}", |
| 5803 | + case.consistency_checks |
| 5804 | + .iter() |
| 5805 | + .map(ConsistencyCheck::as_str) |
| 5806 | + .collect::<Vec<_>>() |
| 5807 | + .join(", ") |
| 5808 | + ); |
| 5809 | + println!(" repeat: {}", case.repeat_count); |
| 5810 | + } |
| 5811 | + } |
| 4214 | 5812 | |
| 4215 | | - components |
| 4216 | | - .into_iter() |
| 4217 | | - .filter_map(|(name, values)| { |
| 4218 | | - let varies = count_unique_strings(values.iter().map(String::as_str)) > 1; |
| 4219 | | - if varies == want_varying { |
| 4220 | | - Some(name) |
| 4221 | | - } else { |
| 4222 | | - None |
| 4223 | | - } |
| 4224 | | - }) |
| 4225 | | - .collect() |
| 4226 | | -} |
| 5813 | + let observed = run_introspect(&IntrospectConfig { |
| 5814 | + compiler: generic.compiler.clone(), |
| 5815 | + program: prepared.compiler_source.clone(), |
| 5816 | + opt_level, |
| 5817 | + artifacts: generic.artifacts.clone(), |
| 5818 | + json_report: None, |
| 5819 | + markdown_report: None, |
| 5820 | + all_artifacts: false, |
| 5821 | + summary_only: false, |
| 5822 | + max_artifact_lines: None, |
| 5823 | + tools: config.tools.clone(), |
| 5824 | + })?; |
| 4227 | 5825 | |
| 4228 | | -fn join_or_none(values: &[&str]) -> String { |
| 4229 | | - if values.is_empty() { |
| 4230 | | - "none".to_string() |
| 5826 | + let mut execution = if observed.observation.compile_exit_code == 0 { |
| 5827 | + if has_failure_expectation(case) { |
| 5828 | + Err(format!( |
| 5829 | + "expected {} to fail ({}) but compilation succeeded", |
| 5830 | + generic.compiler.display_name(), |
| 5831 | + expected_failure_description(case) |
| 5832 | + )) |
| 5833 | + } else { |
| 5834 | + evaluate_observation_expectations(case, &observed) |
| 5835 | + } |
| 5836 | + } else if has_failure_expectation(case) { |
| 5837 | + evaluate_observation_failure_expectations(case, &observed.observation) |
| 4231 | 5838 | } else { |
| 4232 | | - values.join(", ") |
| 4233 | | - } |
| 4234 | | -} |
| 5839 | + Err(compose_observation_failure_detail(&observed.observation)) |
| 5840 | + }; |
| 4235 | 5841 | |
| 4236 | | -fn join_or_none_from_strings(values: &[String]) -> String { |
| 4237 | | - if values.is_empty() { |
| 4238 | | - "none".to_string() |
| 4239 | | - } else { |
| 4240 | | - values.join(", ") |
| 5842 | + if execution.is_ok() && !case.reference_compilers.is_empty() { |
| 5843 | + execution = run_generic_differential( |
| 5844 | + &generic.compiler, |
| 5845 | + &prepared.compiler_source, |
| 5846 | + opt_level, |
| 5847 | + &case.reference_compilers, |
| 5848 | + &config.tools, |
| 5849 | + ); |
| 4241 | 5850 | } |
| 4242 | | -} |
| 4243 | 5851 | |
| 4244 | | -fn join_usize_set(values: &BTreeSet<usize>) -> String { |
| 4245 | | - if values.is_empty() { |
| 4246 | | - "n/a".to_string() |
| 4247 | | - } else { |
| 4248 | | - values |
| 4249 | | - .iter() |
| 4250 | | - .map(|value| value.to_string()) |
| 4251 | | - .collect::<Vec<_>>() |
| 4252 | | - .join(", ") |
| 5852 | + let mut consistency_issues = Vec::new(); |
| 5853 | + if execution.is_ok() && !case.consistency_checks.is_empty() { |
| 5854 | + consistency_issues = run_generic_consistency_checks( |
| 5855 | + &generic.compiler, |
| 5856 | + case, |
| 5857 | + &prepared.compiler_source, |
| 5858 | + opt_level, |
| 5859 | + &config.tools, |
| 5860 | + ); |
| 5861 | + if !consistency_issues.is_empty() { |
| 5862 | + execution = Err(format_consistency_issues(&consistency_issues)); |
| 5863 | + } |
| 4253 | 5864 | } |
| 4254 | | -} |
| 4255 | 5865 | |
| 4256 | | -fn join_string_set(values: &BTreeSet<String>) -> String { |
| 4257 | | - if values.is_empty() { |
| 4258 | | - "none".to_string() |
| 4259 | | - } else { |
| 4260 | | - values.iter().cloned().collect::<Vec<_>>().join(", ") |
| 4261 | | - } |
| 5866 | + let consistency_observations = consistency_issues |
| 5867 | + .iter() |
| 5868 | + .map(ConsistencyIssue::observation) |
| 5869 | + .collect::<Vec<_>>(); |
| 5870 | + |
| 5871 | + let mut outcome = match (effective_status, execution) { |
| 5872 | + (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 5873 | + suite: suite.name.clone(), |
| 5874 | + case: case.name.clone(), |
| 5875 | + opt_level, |
| 5876 | + kind: OutcomeKind::Pass, |
| 5877 | + detail: String::new(), |
| 5878 | + bundle: None, |
| 5879 | + primary_backend: None, |
| 5880 | + consistency_observations: consistency_observations.clone(), |
| 5881 | + }, |
| 5882 | + (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 5883 | + suite: suite.name.clone(), |
| 5884 | + case: case.name.clone(), |
| 5885 | + opt_level, |
| 5886 | + kind: OutcomeKind::Fail, |
| 5887 | + detail, |
| 5888 | + bundle: None, |
| 5889 | + primary_backend: None, |
| 5890 | + consistency_observations: consistency_observations.clone(), |
| 5891 | + }, |
| 5892 | + (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 5893 | + suite: suite.name.clone(), |
| 5894 | + case: case.name.clone(), |
| 5895 | + opt_level, |
| 5896 | + kind: OutcomeKind::Xpass, |
| 5897 | + detail: reason, |
| 5898 | + bundle: None, |
| 5899 | + primary_backend: None, |
| 5900 | + consistency_observations: consistency_observations.clone(), |
| 5901 | + }, |
| 5902 | + (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 5903 | + suite: suite.name.clone(), |
| 5904 | + case: case.name.clone(), |
| 5905 | + opt_level, |
| 5906 | + kind: OutcomeKind::Xfail, |
| 5907 | + detail: format!("{}\n{}", reason, detail), |
| 5908 | + bundle: None, |
| 5909 | + primary_backend: None, |
| 5910 | + consistency_observations: consistency_observations.clone(), |
| 5911 | + }, |
| 5912 | + (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 5913 | + suite: suite.name.clone(), |
| 5914 | + case: case.name.clone(), |
| 5915 | + opt_level, |
| 5916 | + kind: OutcomeKind::Xpass, |
| 5917 | + detail: reason, |
| 5918 | + bundle: None, |
| 5919 | + primary_backend: None, |
| 5920 | + consistency_observations: consistency_observations.clone(), |
| 5921 | + }, |
| 5922 | + (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 5923 | + suite: suite.name.clone(), |
| 5924 | + case: case.name.clone(), |
| 5925 | + opt_level, |
| 5926 | + kind: OutcomeKind::Future, |
| 5927 | + detail: format!("{}\n{}", reason, detail), |
| 5928 | + bundle: None, |
| 5929 | + primary_backend: None, |
| 5930 | + consistency_observations, |
| 5931 | + }, |
| 5932 | + }; |
| 5933 | + |
| 5934 | + outcome.detail = outcome.detail.trim().to_string(); |
| 5935 | + cleanup_prepared_input(&prepared); |
| 5936 | + cleanup_consistency_issues(&consistency_issues); |
| 5937 | + Ok(outcome) |
| 4262 | 5938 | } |
| 4263 | 5939 | |
| 4264 | | -fn render_consistency_rollup(rollup: &ConsistencyRollup) -> String { |
| 4265 | | - let mut parts = vec![format!("{} cells", rollup.cells)]; |
| 4266 | | - if !rollup.repeat_counts.is_empty() { |
| 4267 | | - parts.push(format!( |
| 4268 | | - "repeat_count={}", |
| 4269 | | - join_usize_set(&rollup.repeat_counts) |
| 4270 | | - )); |
| 4271 | | - } |
| 4272 | | - if !rollup.unique_variant_counts.is_empty() { |
| 4273 | | - parts.push(format!( |
| 4274 | | - "unique_variants={}", |
| 4275 | | - join_usize_set(&rollup.unique_variant_counts) |
| 4276 | | - )); |
| 4277 | | - } |
| 4278 | | - if !rollup.varying_components.is_empty() { |
| 4279 | | - parts.push(format!( |
| 4280 | | - "varying={}", |
| 4281 | | - join_string_set(&rollup.varying_components) |
| 4282 | | - )); |
| 4283 | | - } |
| 4284 | | - if !rollup.stable_components.is_empty() { |
| 4285 | | - parts.push(format!( |
| 4286 | | - "stable={}", |
| 4287 | | - join_string_set(&rollup.stable_components) |
| 4288 | | - )); |
| 5940 | +fn prepare_case_input( |
| 5941 | + case: &CaseSpec, |
| 5942 | + suite: &SuiteSpec, |
| 5943 | + opt_level: OptLevel, |
| 5944 | +) -> Result<PreparedInput, String> { |
| 5945 | + if case.graph_files.is_empty() { |
| 5946 | + return Ok(PreparedInput { |
| 5947 | + compiler_source: case.source.clone(), |
| 5948 | + generated_source: None, |
| 5949 | + temp_root: None, |
| 5950 | + }); |
| 4289 | 5951 | } |
| 4290 | | - parts.join("; ") |
| 4291 | | -} |
| 4292 | 5952 | |
| 4293 | | -fn write_behavior_run_artifacts( |
| 4294 | | - root: &Path, |
| 4295 | | - prefix: &str, |
| 4296 | | - command: &str, |
| 4297 | | - run: &RunCapture, |
| 4298 | | -) -> Result<(), std::io::Error> { |
| 4299 | | - let signature = normalize_run_signature(run); |
| 4300 | | - fs::write(root.join(format!("{}.command.txt", prefix)), command)?; |
| 4301 | | - fs::write(root.join(format!("{}.stdout.txt", prefix)), &run.stdout)?; |
| 4302 | | - fs::write(root.join(format!("{}.stderr.txt", prefix)), &run.stderr)?; |
| 4303 | | - fs::write( |
| 4304 | | - root.join(format!("{}.exit_code.txt", prefix)), |
| 4305 | | - format!("{}\n", run.exit_code), |
| 4306 | | - )?; |
| 4307 | | - fs::write( |
| 4308 | | - root.join(format!("{}.normalized.txt", prefix)), |
| 4309 | | - format_run_signature(&signature), |
| 4310 | | - )?; |
| 4311 | | - Ok(()) |
| 4312 | | -} |
| 5953 | + let temp_root = default_report_root().join(".tmp").join(format!( |
| 5954 | + "graph_{}_{}_{}", |
| 5955 | + sanitize_component(&suite.name), |
| 5956 | + sanitize_component(&case.name), |
| 5957 | + next_report_suffix(opt_level) |
| 5958 | + )); |
| 5959 | + fs::create_dir_all(&temp_root).map_err(|e| { |
| 5960 | + format!( |
| 5961 | + "cannot create graph temp dir '{}': {}", |
| 5962 | + temp_root.display(), |
| 5963 | + e |
| 5964 | + ) |
| 5965 | + })?; |
| 4313 | 5966 | |
| 4314 | | -fn render_summary(summary: &Summary) -> String { |
| 4315 | | - let mut lines = vec![ |
| 4316 | | - "Summary".to_string(), |
| 4317 | | - format!(" passed: {}", summary.passed), |
| 4318 | | - format!(" failed: {}", summary.failed), |
| 4319 | | - format!(" xfailed: {}", summary.xfailed), |
| 4320 | | - format!(" xpassed: {}", summary.xpassed), |
| 4321 | | - format!(" future: {}", summary.future), |
| 4322 | | - ]; |
| 5967 | + let extension = case |
| 5968 | + .source |
| 5969 | + .extension() |
| 5970 | + .and_then(|ext| ext.to_str()) |
| 5971 | + .filter(|ext| !ext.is_empty()) |
| 5972 | + .unwrap_or("f90"); |
| 5973 | + let generated_source = temp_root.join(format!( |
| 5974 | + "{}_graph.{}", |
| 5975 | + sanitize_component(&case.name), |
| 5976 | + extension |
| 5977 | + )); |
| 4323 | 5978 | |
| 4324 | | - if !summary.consistency.is_empty() { |
| 4325 | | - lines.push(String::new()); |
| 4326 | | - lines.push("Consistency".to_string()); |
| 4327 | | - lines.push(format!(" affected_checks: {}", summary.consistency.len())); |
| 4328 | | - lines.push(format!( |
| 4329 | | - " cells_with_issues: {}", |
| 4330 | | - summary |
| 4331 | | - .consistency |
| 4332 | | - .values() |
| 4333 | | - .map(|rollup| rollup.cells) |
| 4334 | | - .sum::<usize>() |
| 4335 | | - )); |
| 4336 | | - for (check, rollup) in &summary.consistency { |
| 4337 | | - lines.push(format!( |
| 4338 | | - " {}: {}", |
| 4339 | | - check.as_str(), |
| 4340 | | - render_consistency_rollup(rollup) |
| 4341 | | - )); |
| 5979 | + let mut combined = String::new(); |
| 5980 | + for (index, file) in case.graph_files.iter().enumerate() { |
| 5981 | + let text = fs::read_to_string(file) |
| 5982 | + .map_err(|e| format!("cannot read graph file '{}': {}", file.display(), e))?; |
| 5983 | + if index > 0 { |
| 5984 | + combined.push('\n'); |
| 5985 | + } |
| 5986 | + combined.push_str(&text); |
| 5987 | + if !text.ends_with('\n') { |
| 5988 | + combined.push('\n'); |
| 4342 | 5989 | } |
| 4343 | 5990 | } |
| 4344 | 5991 | |
| 4345 | | - lines.join("\n") |
| 4346 | | -} |
| 4347 | | - |
| 4348 | | -fn write_failure_bundle( |
| 4349 | | - suite: &SuiteSpec, |
| 4350 | | - case: &CaseSpec, |
| 4351 | | - prepared: &PreparedInput, |
| 4352 | | - outcome: &Outcome, |
| 4353 | | - artifacts: &ExecutionArtifacts, |
| 4354 | | -) -> Result<PathBuf, String> { |
| 4355 | | - let bundle_root = default_report_root() |
| 4356 | | - .join(sanitize_component(&suite.name)) |
| 4357 | | - .join(sanitize_component(&case.name)) |
| 4358 | | - .join(next_report_suffix(outcome.opt_level)); |
| 4359 | | - fs::create_dir_all(&bundle_root).map_err(|e| { |
| 5992 | + fs::write(&generated_source, combined).map_err(|e| { |
| 4360 | 5993 | format!( |
| 4361 | | - "cannot create report bundle '{}': {}", |
| 4362 | | - bundle_root.display(), |
| 5994 | + "cannot write generated graph input '{}': {}", |
| 5995 | + generated_source.display(), |
| 4363 | 5996 | e |
| 4364 | 5997 | ) |
| 4365 | 5998 | })?; |
| 4366 | 5999 | |
| 4367 | | - let stage_list = artifacts |
| 4368 | | - .requested |
| 4369 | | - .iter() |
| 4370 | | - .map(Stage::as_str) |
| 4371 | | - .collect::<Vec<_>>() |
| 4372 | | - .join(", "); |
| 4373 | | - let refs = if case.reference_compilers.is_empty() { |
| 4374 | | - "none".to_string() |
| 4375 | | - } else { |
| 4376 | | - case.reference_compilers |
| 6000 | + Ok(PreparedInput { |
| 6001 | + compiler_source: generated_source.clone(), |
| 6002 | + generated_source: Some(generated_source), |
| 6003 | + temp_root: Some(temp_root), |
| 6004 | + }) |
| 6005 | +} |
| 6006 | + |
| 6007 | +fn cleanup_prepared_input(prepared: &PreparedInput) { |
| 6008 | + if let Some(temp_root) = &prepared.temp_root { |
| 6009 | + let _ = fs::remove_dir_all(temp_root); |
| 6010 | + } |
| 6011 | +} |
| 6012 | + |
| 6013 | +fn primary_backend_kind_for_case( |
| 6014 | + case: &CaseSpec, |
| 6015 | + requested: &BTreeSet<Stage>, |
| 6016 | + tools: &ToolchainConfig, |
| 6017 | +) -> PrimaryCaptureBackendKind { |
| 6018 | + let cli_observable_only = !requested.is_empty() |
| 6019 | + && requested |
| 4377 | 6020 | .iter() |
| 4378 | | - .map(ReferenceCompiler::as_str) |
| 4379 | | - .collect::<Vec<_>>() |
| 4380 | | - .join(", ") |
| 4381 | | - }; |
| 4382 | | - let consistency = if case.consistency_checks.is_empty() { |
| 4383 | | - "none".to_string() |
| 6021 | + .all(|stage| matches!(stage, Stage::Asm | Stage::Obj | Stage::Run)); |
| 6022 | + let supports_cli_primary = matches!( |
| 6023 | + tools.armfortas_adapters().cli(), |
| 6024 | + ArmfortasCliAdapter::External(_) |
| 6025 | + ); |
| 6026 | + let capture_checks_required = case |
| 6027 | + .consistency_checks |
| 6028 | + .iter() |
| 6029 | + .any(ConsistencyCheck::requires_capture_result); |
| 6030 | + |
| 6031 | + if supports_cli_primary |
| 6032 | + && cli_observable_only |
| 6033 | + && !has_failure_expectation(case) |
| 6034 | + && !capture_checks_required |
| 6035 | + { |
| 6036 | + PrimaryCaptureBackendKind::Observable |
| 4384 | 6037 | } else { |
| 4385 | | - case.consistency_checks |
| 4386 | | - .iter() |
| 4387 | | - .map(ConsistencyCheck::as_str) |
| 4388 | | - .collect::<Vec<_>>() |
| 4389 | | - .join(", ") |
| 4390 | | - }; |
| 4391 | | - let metadata = format!( |
| 4392 | | - "suite: {}\ncase: {}\noutcome: {:?}\nopt: {}\nsource: {}\nrequested_stages: {}\nrepeat_count: {}\nreference_compilers: {}\nconsistency_checks: {}\n", |
| 4393 | | - suite.name, |
| 4394 | | - case.name, |
| 4395 | | - outcome.kind, |
| 4396 | | - outcome.opt_level.as_str(), |
| 4397 | | - case.source_label(), |
| 4398 | | - stage_list, |
| 4399 | | - case.repeat_count, |
| 4400 | | - refs, |
| 4401 | | - consistency |
| 4402 | | - ); |
| 4403 | | - fs::write(bundle_root.join("metadata.txt"), metadata) |
| 4404 | | - .map_err(|e| format!("cannot write bundle metadata: {}", e))?; |
| 4405 | | - fs::write(bundle_root.join("detail.txt"), &outcome.detail) |
| 4406 | | - .map_err(|e| format!("cannot write bundle detail: {}", e))?; |
| 6038 | + PrimaryCaptureBackendKind::Full |
| 6039 | + } |
| 6040 | +} |
| 4407 | 6041 | |
| 4408 | | - write_case_sources_bundle(&bundle_root, case, prepared)?; |
| 6042 | +fn select_primary_capture_backend( |
| 6043 | + case: &CaseSpec, |
| 6044 | + requested: &BTreeSet<Stage>, |
| 6045 | + opt_level: OptLevel, |
| 6046 | + tools: &ToolchainConfig, |
| 6047 | +) -> SelectedPrimaryBackend { |
| 6048 | + let kind = primary_backend_kind_for_case(case, requested, tools); |
| 6049 | + let backend: Box<dyn CaptureBackend> = match kind { |
| 6050 | + PrimaryCaptureBackendKind::Full => Box::new(tools.armfortas_adapters()), |
| 6051 | + PrimaryCaptureBackendKind::Observable => { |
| 6052 | + Box::new(tools.cli_observable_capture_backend(next_primary_cli_temp_root(opt_level))) |
| 6053 | + } |
| 6054 | + }; |
| 6055 | + SelectedPrimaryBackend { kind, backend } |
| 6056 | +} |
| 4409 | 6057 | |
| 4410 | | - let armfortas_root = bundle_root.join("armfortas"); |
| 4411 | | - fs::create_dir_all(&armfortas_root) |
| 4412 | | - .map_err(|e| format!("cannot create armfortas bundle dir: {}", e))?; |
| 4413 | | - if let Some(result) = &artifacts.armfortas { |
| 4414 | | - write_capture_result(&armfortas_root, result)?; |
| 4415 | | - } |
| 4416 | | - if let Some(failure) = &artifacts.armfortas_failure { |
| 4417 | | - write_capture_result(&armfortas_root, &failure.partial_result())?; |
| 4418 | | - fs::write( |
| 4419 | | - armfortas_root.join("error.txt"), |
| 4420 | | - format!("stage: {}\n{}\n", failure.stage.as_str(), failure.detail), |
| 4421 | | - ) |
| 4422 | | - .map_err(|e| format!("cannot write armfortas error bundle: {}", e))?; |
| 4423 | | - } |
| 6058 | +fn execute_primary_armfortas( |
| 6059 | + prepared: &PreparedInput, |
| 6060 | + opt_level: OptLevel, |
| 6061 | + requested: &BTreeSet<Stage>, |
| 6062 | + selected: &SelectedPrimaryBackend, |
| 6063 | +) -> Result<CaptureResult, CaptureFailure> { |
| 6064 | + let request = CaptureRequest { |
| 6065 | + input: prepared.compiler_source.clone(), |
| 6066 | + requested: requested.clone(), |
| 6067 | + opt_level, |
| 6068 | + }; |
| 6069 | + selected.backend.capture(&request) |
| 6070 | +} |
| 4424 | 6071 | |
| 4425 | | - if !artifacts.references.is_empty() { |
| 4426 | | - let refs_root = bundle_root.join("references"); |
| 4427 | | - fs::create_dir_all(&refs_root) |
| 4428 | | - .map_err(|e| format!("cannot create references bundle dir: {}", e))?; |
| 4429 | | - for reference in &artifacts.references { |
| 4430 | | - write_reference_bundle(&refs_root, reference)?; |
| 6072 | +fn status_for_opt(case: &CaseSpec, opt_level: OptLevel) -> EffectiveStatus { |
| 6073 | + let mut status = EffectiveStatus::Normal; |
| 6074 | + for rule in &case.status_rules { |
| 6075 | + if rule.selector.matches(opt_level) { |
| 6076 | + status = match rule.kind { |
| 6077 | + StatusKind::Xfail => EffectiveStatus::Xfail(rule.reason.clone()), |
| 6078 | + StatusKind::Future => EffectiveStatus::Future(rule.reason.clone()), |
| 6079 | + }; |
| 4431 | 6080 | } |
| 4432 | 6081 | } |
| 6082 | + status |
| 6083 | +} |
| 4433 | 6084 | |
| 4434 | | - if !artifacts.consistency_issues.is_empty() { |
| 4435 | | - write_consistency_bundle(&bundle_root, &artifacts.consistency_issues)?; |
| 6085 | +fn capability_effective_status(base: &EffectiveStatus, case: &CaseSpec) -> EffectiveStatus { |
| 6086 | + match base { |
| 6087 | + EffectiveStatus::Normal => match &case.capability_policy { |
| 6088 | + Some(policy) => match policy.kind { |
| 6089 | + StatusKind::Xfail => EffectiveStatus::Xfail(policy.reason.clone()), |
| 6090 | + StatusKind::Future => EffectiveStatus::Future(policy.reason.clone()), |
| 6091 | + }, |
| 6092 | + None => EffectiveStatus::Normal, |
| 6093 | + }, |
| 6094 | + other => other.clone(), |
| 4436 | 6095 | } |
| 6096 | +} |
| 4437 | 6097 | |
| 4438 | | - Ok(bundle_root) |
| 6098 | +fn ensure_target_stage(expectation: &Expectation, requested: &mut BTreeSet<Stage>) { |
| 6099 | + match expectation { |
| 6100 | + Expectation::CheckComments(target) |
| 6101 | + | Expectation::Contains { target, .. } |
| 6102 | + | Expectation::NotContains { target, .. } |
| 6103 | + | Expectation::Equals { target, .. } |
| 6104 | + | Expectation::IntEquals { target, .. } => match target { |
| 6105 | + Target::Stage(stage) => { |
| 6106 | + requested.insert(*stage); |
| 6107 | + } |
| 6108 | + Target::Artifact(artifact) => ensure_artifact_stage(artifact, requested), |
| 6109 | + Target::CompareStatus |
| 6110 | + | Target::CompareClassification |
| 6111 | + | Target::CompareChangedArtifacts |
| 6112 | + | Target::CompareDifferenceCount |
| 6113 | + | Target::CompareBasis => {} |
| 6114 | + Target::RunStdout | Target::RunStderr | Target::RunExitCode => { |
| 6115 | + requested.insert(Stage::Run); |
| 6116 | + } |
| 6117 | + }, |
| 6118 | + Expectation::FailContains { .. } |
| 6119 | + | Expectation::FailEquals { .. } |
| 6120 | + | Expectation::FailSourceComments |
| 6121 | + | Expectation::FailCommentPatterns(_) => {} |
| 6122 | + } |
| 4439 | 6123 | } |
| 4440 | 6124 | |
| 4441 | | -fn write_case_sources_bundle( |
| 4442 | | - bundle_root: &Path, |
| 4443 | | - case: &CaseSpec, |
| 4444 | | - prepared: &PreparedInput, |
| 4445 | | -) -> Result<(), String> { |
| 4446 | | - if case.graph_files.is_empty() { |
| 4447 | | - let source_text = fs::read_to_string(&case.source) |
| 4448 | | - .map_err(|e| format!("cannot read case source '{}': {}", case.source.display(), e))?; |
| 4449 | | - fs::write(bundle_root.join("source.f90"), source_text) |
| 4450 | | - .map_err(|e| format!("cannot write bundle source copy: {}", e))?; |
| 4451 | | - return Ok(()); |
| 6125 | +fn ensure_consistency_stage(check: ConsistencyCheck, requested: &mut BTreeSet<Stage>) { |
| 6126 | + if let Some(stage) = check.required_stage() { |
| 6127 | + requested.insert(stage); |
| 4452 | 6128 | } |
| 6129 | +} |
| 4453 | 6130 | |
| 4454 | | - let generated_source = prepared.generated_source.as_ref().ok_or_else(|| { |
| 4455 | | - format!( |
| 4456 | | - "graph case '{}' was missing a generated compiler source", |
| 4457 | | - case.name |
| 4458 | | - ) |
| 4459 | | - })?; |
| 4460 | | - let generated_text = fs::read_to_string(generated_source).map_err(|e| { |
| 4461 | | - format!( |
| 4462 | | - "cannot read generated graph source '{}': {}", |
| 4463 | | - generated_source.display(), |
| 4464 | | - e |
| 4465 | | - ) |
| 4466 | | - })?; |
| 4467 | | - fs::write(bundle_root.join("source.f90"), generated_text) |
| 4468 | | - .map_err(|e| format!("cannot write generated bundle source copy: {}", e))?; |
| 6131 | +fn ensure_artifact_stage(artifact: &ArtifactKey, requested: &mut BTreeSet<Stage>) { |
| 6132 | + match artifact { |
| 6133 | + ArtifactKey::Asm => { |
| 6134 | + requested.insert(Stage::Asm); |
| 6135 | + } |
| 6136 | + ArtifactKey::Obj => { |
| 6137 | + requested.insert(Stage::Obj); |
| 6138 | + } |
| 6139 | + ArtifactKey::Runtime |
| 6140 | + | ArtifactKey::Stdout |
| 6141 | + | ArtifactKey::Stderr |
| 6142 | + | ArtifactKey::ExitCode => { |
| 6143 | + requested.insert(Stage::Run); |
| 6144 | + } |
| 6145 | + ArtifactKey::Extra(name) => { |
| 6146 | + if let Some(stage) = armfortas_extra_stage(name) { |
| 6147 | + requested.insert(stage); |
| 6148 | + } |
| 6149 | + } |
| 6150 | + ArtifactKey::Diagnostics | ArtifactKey::Executable => {} |
| 6151 | + } |
| 6152 | +} |
| 4469 | 6153 | |
| 4470 | | - let sources_root = bundle_root.join("sources"); |
| 4471 | | - fs::create_dir_all(&sources_root) |
| 4472 | | - .map_err(|e| format!("cannot create bundle sources dir: {}", e))?; |
| 4473 | | - for (index, file) in case.graph_files.iter().enumerate() { |
| 4474 | | - let text = fs::read_to_string(file) |
| 4475 | | - .map_err(|e| format!("cannot read graph source '{}': {}", file.display(), e))?; |
| 4476 | | - let file_name = file |
| 4477 | | - .file_name() |
| 4478 | | - .and_then(|name| name.to_str()) |
| 4479 | | - .unwrap_or("source.f90"); |
| 4480 | | - let target = sources_root.join(format!("{:02}_{}", index, file_name)); |
| 4481 | | - fs::write(target, text).map_err(|e| format!("cannot write bundle graph source: {}", e))?; |
| 6154 | +fn armfortas_extra_stage(name: &str) -> Option<Stage> { |
| 6155 | + let (namespace, suffix) = name.split_once('.')?; |
| 6156 | + if namespace.eq_ignore_ascii_case("armfortas") { |
| 6157 | + Stage::parse(suffix) |
| 6158 | + } else { |
| 6159 | + None |
| 4482 | 6160 | } |
| 6161 | +} |
| 4483 | 6162 | |
| 6163 | +fn evaluate_observation_expectations( |
| 6164 | + case: &CaseSpec, |
| 6165 | + observed: &ObservedProgram, |
| 6166 | +) -> Result<(), String> { |
| 6167 | + for expectation in &case.expectations { |
| 6168 | + match expectation { |
| 6169 | + Expectation::CheckComments(target) => { |
| 6170 | + let text = observation_target_text(&observed.observation, target)?; |
| 6171 | + let source = fs::read_to_string(&case.source) |
| 6172 | + .map_err(|e| format!("cannot read '{}': {}", case.source.display(), e))?; |
| 6173 | + let checks = if target_uses_ir_comment_checks(target) { |
| 6174 | + extract_ir_checks(&source) |
| 6175 | + } else { |
| 6176 | + extract_checks(&source) |
| 6177 | + }; |
| 6178 | + if checks.is_empty() { |
| 6179 | + let expected_label = if target_uses_ir_comment_checks(target) { |
| 6180 | + "! IR_CHECK: / ! IR_NOT:" |
| 6181 | + } else { |
| 6182 | + "! CHECK:" |
| 6183 | + }; |
| 6184 | + return Err(format!( |
| 6185 | + "case '{}' requested check-comments but '{}' has no {} lines", |
| 6186 | + case.name, |
| 6187 | + case.source.display(), |
| 6188 | + expected_label |
| 6189 | + )); |
| 6190 | + } |
| 6191 | + match_checks(&checks, text, &case.name)?; |
| 6192 | + } |
| 6193 | + Expectation::Contains { target, needle } => { |
| 6194 | + let text = observation_target_text(&observed.observation, target)?; |
| 6195 | + if !text.contains(needle) { |
| 6196 | + return Err(format!( |
| 6197 | + "expected {} to contain {:?}\nactual:\n{}", |
| 6198 | + target_name(target), |
| 6199 | + needle, |
| 6200 | + text |
| 6201 | + )); |
| 6202 | + } |
| 6203 | + } |
| 6204 | + Expectation::NotContains { target, needle } => { |
| 6205 | + let text = observation_target_text(&observed.observation, target)?; |
| 6206 | + if text.contains(needle) { |
| 6207 | + return Err(format!( |
| 6208 | + "expected {} to not contain {:?}\nactual:\n{}", |
| 6209 | + target_name(target), |
| 6210 | + needle, |
| 6211 | + text |
| 6212 | + )); |
| 6213 | + } |
| 6214 | + } |
| 6215 | + Expectation::Equals { target, value } => { |
| 6216 | + let text = observation_target_text(&observed.observation, target)?; |
| 6217 | + if text.trim_end() != value { |
| 6218 | + return Err(format!( |
| 6219 | + "expected {} to equal {:?}\nactual:\n{}", |
| 6220 | + target_name(target), |
| 6221 | + value, |
| 6222 | + text |
| 6223 | + )); |
| 6224 | + } |
| 6225 | + } |
| 6226 | + Expectation::IntEquals { target, value } => { |
| 6227 | + let actual = observation_target_int(&observed.observation, target)?; |
| 6228 | + if actual != *value { |
| 6229 | + return Err(format!( |
| 6230 | + "expected {} to equal {}\nactual: {}", |
| 6231 | + target_name(target), |
| 6232 | + value, |
| 6233 | + actual |
| 6234 | + )); |
| 6235 | + } |
| 6236 | + } |
| 6237 | + Expectation::FailContains { .. } |
| 6238 | + | Expectation::FailEquals { .. } |
| 6239 | + | Expectation::FailSourceComments |
| 6240 | + | Expectation::FailCommentPatterns(_) => {} |
| 6241 | + } |
| 6242 | + } |
| 4484 | 6243 | Ok(()) |
| 4485 | 6244 | } |
| 4486 | 6245 | |
| 4487 | | -fn write_capture_result(root: &Path, result: &CaptureResult) -> Result<(), String> { |
| 4488 | | - for (stage, captured) in &result.stages { |
| 4489 | | - match captured { |
| 4490 | | - CapturedStage::Text(text) => { |
| 4491 | | - fs::write(root.join(format!("{}.txt", stage.as_str())), text).map_err(|e| { |
| 4492 | | - format!("cannot write '{}' stage bundle: {}", stage.as_str(), e) |
| 4493 | | - })?; |
| 6246 | +fn evaluate_compare_expectations(case: &CaseSpec, result: &ComparisonResult) -> Result<(), String> { |
| 6247 | + for expectation in &case.expectations { |
| 6248 | + match expectation { |
| 6249 | + Expectation::CheckComments(_) => { |
| 6250 | + return Err("compare cases do not support check-comments expectations".into()) |
| 4494 | 6251 | } |
| 4495 | | - CapturedStage::Run(run) => { |
| 4496 | | - fs::write(root.join("run.stdout.txt"), &run.stdout) |
| 4497 | | - .map_err(|e| format!("cannot write run stdout bundle: {}", e))?; |
| 4498 | | - fs::write(root.join("run.stderr.txt"), &run.stderr) |
| 4499 | | - .map_err(|e| format!("cannot write run stderr bundle: {}", e))?; |
| 4500 | | - fs::write( |
| 4501 | | - root.join("run.exit_code.txt"), |
| 4502 | | - format!("{}\n", run.exit_code), |
| 4503 | | - ) |
| 4504 | | - .map_err(|e| format!("cannot write run exit-code bundle: {}", e))?; |
| 6252 | + Expectation::Contains { target, needle } => { |
| 6253 | + let text = compare_target_text(result, target)?; |
| 6254 | + if !text.contains(needle) { |
| 6255 | + return Err(format!( |
| 6256 | + "expected {} to contain {:?}\nactual:\n{}", |
| 6257 | + target_name(target), |
| 6258 | + needle, |
| 6259 | + text |
| 6260 | + )); |
| 6261 | + } |
| 6262 | + } |
| 6263 | + Expectation::NotContains { target, needle } => { |
| 6264 | + let text = compare_target_text(result, target)?; |
| 6265 | + if text.contains(needle) { |
| 6266 | + return Err(format!( |
| 6267 | + "expected {} to not contain {:?}\nactual:\n{}", |
| 6268 | + target_name(target), |
| 6269 | + needle, |
| 6270 | + text |
| 6271 | + )); |
| 6272 | + } |
| 6273 | + } |
| 6274 | + Expectation::Equals { target, value } => { |
| 6275 | + let text = compare_target_text(result, target)?; |
| 6276 | + if text.trim_end() != value { |
| 6277 | + return Err(format!( |
| 6278 | + "expected {} to equal {:?}\nactual:\n{}", |
| 6279 | + target_name(target), |
| 6280 | + value, |
| 6281 | + text |
| 6282 | + )); |
| 6283 | + } |
| 6284 | + } |
| 6285 | + Expectation::IntEquals { target, value } => { |
| 6286 | + let actual = compare_target_int(result, target)?; |
| 6287 | + if actual != *value { |
| 6288 | + return Err(format!( |
| 6289 | + "expected {} to equal {}\nactual: {}", |
| 6290 | + target_name(target), |
| 6291 | + value, |
| 6292 | + actual |
| 6293 | + )); |
| 6294 | + } |
| 4505 | 6295 | } |
| 6296 | + Expectation::FailContains { .. } |
| 6297 | + | Expectation::FailEquals { .. } |
| 6298 | + | Expectation::FailSourceComments |
| 6299 | + | Expectation::FailCommentPatterns(_) => {} |
| 4506 | 6300 | } |
| 4507 | 6301 | } |
| 4508 | 6302 | Ok(()) |
| 4509 | 6303 | } |
| 4510 | 6304 | |
| 4511 | | -fn write_reference_bundle(root: &Path, reference: &ReferenceResult) -> Result<(), String> { |
| 4512 | | - let ref_root = root.join(sanitize_component(reference.compiler.as_str())); |
| 4513 | | - fs::create_dir_all(&ref_root) |
| 4514 | | - .map_err(|e| format!("cannot create reference bundle dir: {}", e))?; |
| 4515 | | - fs::write(ref_root.join("command.txt"), &reference.compile_command) |
| 4516 | | - .map_err(|e| format!("cannot write reference command bundle: {}", e))?; |
| 4517 | | - fs::write( |
| 4518 | | - ref_root.join("compile.exit_code.txt"), |
| 4519 | | - format!("{}\n", reference.compile_exit_code), |
| 4520 | | - ) |
| 4521 | | - .map_err(|e| format!("cannot write reference compile exit-code bundle: {}", e))?; |
| 4522 | | - fs::write( |
| 4523 | | - ref_root.join("compile.stdout.txt"), |
| 4524 | | - &reference.compile_stdout, |
| 4525 | | - ) |
| 4526 | | - .map_err(|e| format!("cannot write reference compile stdout bundle: {}", e))?; |
| 4527 | | - fs::write( |
| 4528 | | - ref_root.join("compile.stderr.txt"), |
| 4529 | | - &reference.compile_stderr, |
| 4530 | | - ) |
| 4531 | | - .map_err(|e| format!("cannot write reference compile stderr bundle: {}", e))?; |
| 4532 | | - if let Some(run) = &reference.run { |
| 4533 | | - fs::write(ref_root.join("run.stdout.txt"), &run.stdout) |
| 4534 | | - .map_err(|e| format!("cannot write reference run stdout bundle: {}", e))?; |
| 4535 | | - fs::write(ref_root.join("run.stderr.txt"), &run.stderr) |
| 4536 | | - .map_err(|e| format!("cannot write reference run stderr bundle: {}", e))?; |
| 4537 | | - fs::write( |
| 4538 | | - ref_root.join("run.exit_code.txt"), |
| 4539 | | - format!("{}\n", run.exit_code), |
| 4540 | | - ) |
| 4541 | | - .map_err(|e| format!("cannot write reference run exit-code bundle: {}", e))?; |
| 4542 | | - } |
| 4543 | | - if let Some(err) = &reference.run_error { |
| 4544 | | - fs::write(ref_root.join("run.error.txt"), err) |
| 4545 | | - .map_err(|e| format!("cannot write reference run error bundle: {}", e))?; |
| 4546 | | - } |
| 4547 | | - Ok(()) |
| 4548 | | -} |
| 4549 | | - |
| 4550 | | -fn render_consistency_bundle_summary(issues: &[ConsistencyIssue]) -> String { |
| 4551 | | - let mut rollups = BTreeMap::new(); |
| 4552 | | - for issue in issues { |
| 4553 | | - rollups |
| 4554 | | - .entry(issue.check) |
| 4555 | | - .or_insert_with(ConsistencyRollup::default) |
| 4556 | | - .record(&issue.observation()); |
| 4557 | | - } |
| 4558 | | - |
| 4559 | | - let mut aggregate = ConsistencyRollup::default(); |
| 4560 | | - for issue in issues { |
| 4561 | | - aggregate.record(&issue.observation()); |
| 4562 | | - } |
| 4563 | | - |
| 4564 | | - let checks = if rollups.is_empty() { |
| 4565 | | - "none".to_string() |
| 4566 | | - } else { |
| 4567 | | - rollups |
| 4568 | | - .keys() |
| 4569 | | - .map(ConsistencyCheck::as_str) |
| 4570 | | - .collect::<Vec<_>>() |
| 4571 | | - .join(", ") |
| 4572 | | - }; |
| 4573 | | - |
| 4574 | | - let mut lines = vec![ |
| 4575 | | - format!("issue_count: {}", issues.len()), |
| 4576 | | - format!("checks: {}", checks), |
| 4577 | | - ]; |
| 4578 | | - |
| 4579 | | - if !aggregate.repeat_counts.is_empty() { |
| 4580 | | - lines.push(format!( |
| 4581 | | - "repeat_counts: {}", |
| 4582 | | - join_usize_set(&aggregate.repeat_counts) |
| 4583 | | - )); |
| 4584 | | - } |
| 4585 | | - if !aggregate.unique_variant_counts.is_empty() { |
| 4586 | | - lines.push(format!( |
| 4587 | | - "unique_variants: {}", |
| 4588 | | - join_usize_set(&aggregate.unique_variant_counts) |
| 4589 | | - )); |
| 4590 | | - } |
| 4591 | | - if !aggregate.varying_components.is_empty() { |
| 4592 | | - lines.push(format!( |
| 4593 | | - "varying_components: {}", |
| 4594 | | - join_string_set(&aggregate.varying_components) |
| 4595 | | - )); |
| 4596 | | - } |
| 4597 | | - if !aggregate.stable_components.is_empty() { |
| 4598 | | - lines.push(format!( |
| 4599 | | - "stable_components: {}", |
| 4600 | | - join_string_set(&aggregate.stable_components) |
| 4601 | | - )); |
| 4602 | | - } |
| 4603 | | - |
| 4604 | | - if !rollups.is_empty() { |
| 4605 | | - lines.push(String::new()); |
| 4606 | | - lines.push("per_check:".to_string()); |
| 4607 | | - for (check, rollup) in rollups { |
| 4608 | | - lines.push(format!( |
| 4609 | | - " {}: {}", |
| 4610 | | - check.as_str(), |
| 4611 | | - render_consistency_rollup(&rollup) |
| 4612 | | - )); |
| 6305 | +fn evaluate_observation_failure_expectations( |
| 6306 | + case: &CaseSpec, |
| 6307 | + observation: &CompilerObservation, |
| 6308 | +) -> Result<(), String> { |
| 6309 | + let mut saw_failure_expectation = false; |
| 6310 | + let diagnostics = observation_diagnostics_text(observation).unwrap_or_default(); |
| 6311 | + for expectation in &case.expectations { |
| 6312 | + match expectation { |
| 6313 | + Expectation::FailContains { stage, needle } => { |
| 6314 | + saw_failure_expectation = true; |
| 6315 | + let actual_stage = observation_failure_stage(observation); |
| 6316 | + if actual_stage != Some(*stage) { |
| 6317 | + let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6318 | + return Err(format!( |
| 6319 | + "expected failure stage {} but compiler failed in {}\n{}", |
| 6320 | + stage.as_str(), |
| 6321 | + actual, |
| 6322 | + diagnostics |
| 6323 | + )); |
| 6324 | + } |
| 6325 | + if !diagnostics.contains(needle) { |
| 6326 | + return Err(format!( |
| 6327 | + "expected failure detail at {} to contain {:?}\nactual:\n{}", |
| 6328 | + stage.as_str(), |
| 6329 | + needle, |
| 6330 | + diagnostics |
| 6331 | + )); |
| 6332 | + } |
| 6333 | + } |
| 6334 | + Expectation::FailEquals { stage, value } => { |
| 6335 | + saw_failure_expectation = true; |
| 6336 | + let actual_stage = observation_failure_stage(observation); |
| 6337 | + if actual_stage != Some(*stage) { |
| 6338 | + let actual = actual_stage.map(|stage| stage.as_str()).unwrap_or("none"); |
| 6339 | + return Err(format!( |
| 6340 | + "expected failure stage {} but compiler failed in {}\n{}", |
| 6341 | + stage.as_str(), |
| 6342 | + actual, |
| 6343 | + diagnostics |
| 6344 | + )); |
| 6345 | + } |
| 6346 | + if diagnostics.trim_end() != value { |
| 6347 | + return Err(format!( |
| 6348 | + "expected failure detail at {} to equal {:?}\nactual:\n{}", |
| 6349 | + stage.as_str(), |
| 6350 | + value, |
| 6351 | + diagnostics |
| 6352 | + )); |
| 6353 | + } |
| 6354 | + } |
| 6355 | + Expectation::FailCommentPatterns(patterns) => { |
| 6356 | + saw_failure_expectation = true; |
| 6357 | + for needle in patterns { |
| 6358 | + if !diagnostics.contains(needle) { |
| 6359 | + return Err(format!( |
| 6360 | + "expected failure detail to contain source comment {:?}\nactual:\n{}", |
| 6361 | + needle, diagnostics |
| 6362 | + )); |
| 6363 | + } |
| 6364 | + } |
| 6365 | + } |
| 6366 | + Expectation::CheckComments(_) |
| 6367 | + | Expectation::Contains { .. } |
| 6368 | + | Expectation::NotContains { .. } |
| 6369 | + | Expectation::Equals { .. } |
| 6370 | + | Expectation::IntEquals { .. } |
| 6371 | + | Expectation::FailSourceComments => {} |
| 4613 | 6372 | } |
| 4614 | 6373 | } |
| 4615 | 6374 | |
| 4616 | | - lines.push(String::new()); |
| 4617 | | - for issue in issues { |
| 4618 | | - lines.push(format!("check: {}", issue.check.as_str())); |
| 4619 | | - lines.push(format!("summary: {}", issue.summary)); |
| 4620 | | - if let Some(repeat_count) = issue.repeat_count { |
| 4621 | | - lines.push(format!("repeat_count: {}", repeat_count)); |
| 4622 | | - } |
| 4623 | | - if let Some(unique_variant_count) = issue.unique_variant_count { |
| 4624 | | - lines.push(format!("unique_variants: {}", unique_variant_count)); |
| 4625 | | - } |
| 4626 | | - if !issue.varying_components.is_empty() { |
| 4627 | | - lines.push(format!( |
| 4628 | | - "varying_components: {}", |
| 4629 | | - join_or_none_from_strings(&issue.varying_components) |
| 4630 | | - )); |
| 4631 | | - } |
| 4632 | | - if !issue.stable_components.is_empty() { |
| 4633 | | - lines.push(format!( |
| 4634 | | - "stable_components: {}", |
| 4635 | | - join_or_none_from_strings(&issue.stable_components) |
| 4636 | | - )); |
| 4637 | | - } |
| 4638 | | - lines.push(format!( |
| 4639 | | - "artifacts: {}", |
| 4640 | | - sanitize_component(issue.check.as_str()) |
| 6375 | + if !saw_failure_expectation { |
| 6376 | + return Err(format!( |
| 6377 | + "{} failed but the case did not declare an expect-fail rule\n{}", |
| 6378 | + observation.compiler.display_name(), |
| 6379 | + diagnostics |
| 4641 | 6380 | )); |
| 4642 | | - lines.push(String::new()); |
| 4643 | | - } |
| 4644 | | - |
| 4645 | | - lines.join("\n") |
| 4646 | | -} |
| 4647 | | - |
| 4648 | | -fn write_consistency_bundle(root: &Path, issues: &[ConsistencyIssue]) -> Result<(), String> { |
| 4649 | | - let consistency_root = root.join("consistency"); |
| 4650 | | - fs::create_dir_all(&consistency_root) |
| 4651 | | - .map_err(|e| format!("cannot create consistency bundle dir: {}", e))?; |
| 4652 | | - |
| 4653 | | - let summary = render_consistency_bundle_summary(issues); |
| 4654 | | - fs::write(consistency_root.join("summary.txt"), summary) |
| 4655 | | - .map_err(|e| format!("cannot write consistency summary bundle: {}", e))?; |
| 4656 | | - |
| 4657 | | - for issue in issues { |
| 4658 | | - let issue_root = consistency_root.join(sanitize_component(issue.check.as_str())); |
| 4659 | | - fs::create_dir_all(&issue_root) |
| 4660 | | - .map_err(|e| format!("cannot create consistency issue bundle dir: {}", e))?; |
| 4661 | | - fs::write(issue_root.join("summary.txt"), &issue.summary) |
| 4662 | | - .map_err(|e| format!("cannot write consistency issue summary bundle: {}", e))?; |
| 4663 | | - fs::write(issue_root.join("detail.txt"), &issue.detail) |
| 4664 | | - .map_err(|e| format!("cannot write consistency issue detail bundle: {}", e))?; |
| 4665 | | - let runs_root = issue_root.join("artifacts"); |
| 4666 | | - copy_directory_recursive(&issue.temp_root, &runs_root)?; |
| 4667 | 6381 | } |
| 4668 | 6382 | |
| 4669 | 6383 | Ok(()) |
| 4670 | 6384 | } |
| 4671 | 6385 | |
| 4672 | | -fn copy_directory_recursive(source: &Path, destination: &Path) -> Result<(), String> { |
| 4673 | | - fs::create_dir_all(destination).map_err(|e| { |
| 4674 | | - format!( |
| 4675 | | - "cannot create copied artifact dir '{}': {}", |
| 4676 | | - destination.display(), |
| 4677 | | - e |
| 4678 | | - ) |
| 4679 | | - })?; |
| 6386 | +#[cfg(test)] |
| 6387 | +fn evaluate_failed_armfortas( |
| 6388 | + case: &CaseSpec, |
| 6389 | + artifacts: &ExecutionArtifacts, |
| 6390 | + failure: &CaptureFailure, |
| 6391 | +) -> Result<(), String> { |
| 6392 | + let observed = legacy_failure_observed_program(&case.source, case, failure); |
| 6393 | + evaluate_failed_armfortas_with_observed(case, artifacts, &observed) |
| 6394 | +} |
| 4680 | 6395 | |
| 4681 | | - for entry in fs::read_dir(source) |
| 4682 | | - .map_err(|e| format!("cannot read artifact dir '{}': {}", source.display(), e))? |
| 4683 | | - { |
| 4684 | | - let entry = entry |
| 4685 | | - .map_err(|e| format!("cannot read artifact entry '{}': {}", source.display(), e))?; |
| 4686 | | - let source_path = entry.path(); |
| 4687 | | - let destination_path = destination.join(entry.file_name()); |
| 4688 | | - let file_type = entry.file_type().map_err(|e| { |
| 4689 | | - format!( |
| 4690 | | - "cannot read artifact type '{}': {}", |
| 4691 | | - source_path.display(), |
| 4692 | | - e |
| 4693 | | - ) |
| 4694 | | - })?; |
| 4695 | | - if file_type.is_dir() { |
| 4696 | | - copy_directory_recursive(&source_path, &destination_path)?; |
| 4697 | | - } else { |
| 4698 | | - fs::copy(&source_path, &destination_path).map_err(|e| { |
| 4699 | | - format!( |
| 4700 | | - "cannot copy artifact '{}' to '{}': {}", |
| 4701 | | - source_path.display(), |
| 4702 | | - destination_path.display(), |
| 4703 | | - e |
| 4704 | | - ) |
| 4705 | | - })?; |
| 6396 | +fn evaluate_failed_armfortas_with_observed( |
| 6397 | + case: &CaseSpec, |
| 6398 | + artifacts: &ExecutionArtifacts, |
| 6399 | + observed: &ObservedProgram, |
| 6400 | +) -> Result<(), String> { |
| 6401 | + if has_failure_expectation(case) { |
| 6402 | + evaluate_observation_failure_expectations(case, &observed.observation) |
| 6403 | + } else { |
| 6404 | + match evaluate_observation_expectations(case, observed) { |
| 6405 | + Ok(()) => Err(compose_armfortas_failure_detail(artifacts)), |
| 6406 | + Err(detail) if is_missing_stage_detail(&detail) => { |
| 6407 | + Err(compose_armfortas_failure_detail(artifacts)) |
| 6408 | + } |
| 6409 | + Err(detail) => Err(detail), |
| 4706 | 6410 | } |
| 4707 | 6411 | } |
| 6412 | +} |
| 4708 | 6413 | |
| 4709 | | - Ok(()) |
| 6414 | +fn legacy_failure_observed_program( |
| 6415 | + program: &Path, |
| 6416 | + case: &CaseSpec, |
| 6417 | + failure: &CaptureFailure, |
| 6418 | +) -> ObservedProgram { |
| 6419 | + let partial = failure.partial_result(); |
| 6420 | + observed_program_from_armfortas_capture( |
| 6421 | + program, |
| 6422 | + failure.opt_level, |
| 6423 | + expected_artifacts_for_legacy_case(case), |
| 6424 | + &partial, |
| 6425 | + Some(failure), |
| 6426 | + ) |
| 4710 | 6427 | } |
| 4711 | 6428 | |
| 4712 | | -fn render_command(binary: &str, args: &[String]) -> String { |
| 4713 | | - let mut rendered = vec![quote_arg(binary)]; |
| 4714 | | - rendered.extend(args.iter().map(|arg| quote_arg(arg))); |
| 4715 | | - rendered.join(" ") |
| 6429 | +fn has_failure_expectation(case: &CaseSpec) -> bool { |
| 6430 | + case.expectations.iter().any(|expectation| { |
| 6431 | + matches!( |
| 6432 | + expectation, |
| 6433 | + Expectation::FailContains { .. } |
| 6434 | + | Expectation::FailEquals { .. } |
| 6435 | + | Expectation::FailSourceComments |
| 6436 | + | Expectation::FailCommentPatterns(_) |
| 6437 | + ) |
| 6438 | + }) |
| 4716 | 6439 | } |
| 4717 | 6440 | |
| 4718 | | -fn quote_arg(arg: &str) -> String { |
| 4719 | | - if arg |
| 4720 | | - .chars() |
| 4721 | | - .all(|ch| ch.is_ascii_alphanumeric() || "-_./".contains(ch)) |
| 6441 | +fn legacy_unavailable_backend_detail( |
| 6442 | + case: &CaseSpec, |
| 6443 | + selected_backend: &SelectedPrimaryBackend, |
| 6444 | +) -> Option<String> { |
| 6445 | + if selected_backend.kind == PrimaryCaptureBackendKind::Full |
| 6446 | + && selected_backend.backend.mode_name() == "unavailable" |
| 4722 | 6447 | { |
| 4723 | | - arg.to_string() |
| 6448 | + Some(format!( |
| 6449 | + "case requires linked armfortas capture, but this build only provides the external-driver surface\nsource: {}\nrequired backend: {}\nuse scripts/bootstrap-linked-armfortas.sh for rich stages and legacy frontend/module suites, or run a generic suite-v2 / observable-only case instead", |
| 6450 | + case.source_label(), |
| 6451 | + selected_backend.backend.description() |
| 6452 | + )) |
| 4724 | 6453 | } else { |
| 4725 | | - format!("{:?}", arg) |
| 6454 | + None |
| 4726 | 6455 | } |
| 4727 | 6456 | } |
| 4728 | 6457 | |
| 4729 | | -fn sanitize_component(value: &str) -> String { |
| 4730 | | - let mut out = String::new(); |
| 4731 | | - for ch in value.chars() { |
| 4732 | | - if ch.is_ascii_alphanumeric() { |
| 4733 | | - out.push(ch.to_ascii_lowercase()); |
| 4734 | | - } else { |
| 4735 | | - out.push('_'); |
| 4736 | | - } |
| 4737 | | - } |
| 4738 | | - while out.contains("__") { |
| 4739 | | - out = out.replace("__", "_"); |
| 6458 | +fn outcome_from_status_and_execution( |
| 6459 | + suite: &SuiteSpec, |
| 6460 | + case: &CaseSpec, |
| 6461 | + opt_level: OptLevel, |
| 6462 | + effective_status: EffectiveStatus, |
| 6463 | + execution: Result<(), String>, |
| 6464 | + primary_backend: Option<PrimaryBackendReport>, |
| 6465 | + consistency_observations: Vec<ConsistencyObservation>, |
| 6466 | +) -> Outcome { |
| 6467 | + match (effective_status, execution) { |
| 6468 | + (EffectiveStatus::Normal, Ok(())) => Outcome { |
| 6469 | + suite: suite.name.clone(), |
| 6470 | + case: case.name.clone(), |
| 6471 | + opt_level, |
| 6472 | + kind: OutcomeKind::Pass, |
| 6473 | + detail: String::new(), |
| 6474 | + bundle: None, |
| 6475 | + primary_backend, |
| 6476 | + consistency_observations, |
| 6477 | + }, |
| 6478 | + (EffectiveStatus::Normal, Err(detail)) => Outcome { |
| 6479 | + suite: suite.name.clone(), |
| 6480 | + case: case.name.clone(), |
| 6481 | + opt_level, |
| 6482 | + kind: OutcomeKind::Fail, |
| 6483 | + detail, |
| 6484 | + bundle: None, |
| 6485 | + primary_backend, |
| 6486 | + consistency_observations, |
| 6487 | + }, |
| 6488 | + (EffectiveStatus::Xfail(reason), Ok(())) => Outcome { |
| 6489 | + suite: suite.name.clone(), |
| 6490 | + case: case.name.clone(), |
| 6491 | + opt_level, |
| 6492 | + kind: OutcomeKind::Xpass, |
| 6493 | + detail: reason, |
| 6494 | + bundle: None, |
| 6495 | + primary_backend, |
| 6496 | + consistency_observations, |
| 6497 | + }, |
| 6498 | + (EffectiveStatus::Xfail(reason), Err(detail)) => Outcome { |
| 6499 | + suite: suite.name.clone(), |
| 6500 | + case: case.name.clone(), |
| 6501 | + opt_level, |
| 6502 | + kind: OutcomeKind::Xfail, |
| 6503 | + detail: format!("{}\n{}", reason, detail), |
| 6504 | + bundle: None, |
| 6505 | + primary_backend, |
| 6506 | + consistency_observations, |
| 6507 | + }, |
| 6508 | + (EffectiveStatus::Future(reason), Ok(())) => Outcome { |
| 6509 | + suite: suite.name.clone(), |
| 6510 | + case: case.name.clone(), |
| 6511 | + opt_level, |
| 6512 | + kind: OutcomeKind::Xpass, |
| 6513 | + detail: reason, |
| 6514 | + bundle: None, |
| 6515 | + primary_backend, |
| 6516 | + consistency_observations, |
| 6517 | + }, |
| 6518 | + (EffectiveStatus::Future(reason), Err(detail)) => Outcome { |
| 6519 | + suite: suite.name.clone(), |
| 6520 | + case: case.name.clone(), |
| 6521 | + opt_level, |
| 6522 | + kind: OutcomeKind::Future, |
| 6523 | + detail: format!("{}\n{}", reason, detail), |
| 6524 | + bundle: None, |
| 6525 | + primary_backend, |
| 6526 | + consistency_observations, |
| 6527 | + }, |
| 6528 | + } |
| 6529 | +} |
| 6530 | + |
| 6531 | +fn expected_failure_description(case: &CaseSpec) -> String { |
| 6532 | + let mut items = Vec::new(); |
| 6533 | + for expectation in &case.expectations { |
| 6534 | + match expectation { |
| 6535 | + Expectation::FailContains { stage, needle } => { |
| 6536 | + items.push(format!("{} contains {:?}", stage.as_str(), needle)); |
| 6537 | + } |
| 6538 | + Expectation::FailEquals { stage, value } => { |
| 6539 | + items.push(format!("{} equals {:?}", stage.as_str(), value)); |
| 6540 | + } |
| 6541 | + Expectation::FailCommentPatterns(patterns) => { |
| 6542 | + for needle in patterns { |
| 6543 | + items.push(format!("comments contain {:?}", needle)); |
| 6544 | + } |
| 6545 | + } |
| 6546 | + _ => {} |
| 6547 | + } |
| 6548 | + } |
| 6549 | + if items.is_empty() { |
| 6550 | + "declared failure".to_string() |
| 6551 | + } else { |
| 6552 | + items.join(", ") |
| 6553 | + } |
| 6554 | +} |
| 6555 | + |
| 6556 | +fn is_missing_stage_detail(detail: &str) -> bool { |
| 6557 | + detail.starts_with("missing captured stage '") |
| 6558 | + || detail == "missing captured run stage" |
| 6559 | + || detail.starts_with("missing artifact '") |
| 6560 | +} |
| 6561 | + |
| 6562 | +fn target_name(target: &Target) -> String { |
| 6563 | + match target { |
| 6564 | + Target::Stage(stage) => stage.as_str().to_string(), |
| 6565 | + Target::Artifact(artifact) => artifact.as_str().to_string(), |
| 6566 | + Target::CompareStatus => "compare.status".to_string(), |
| 6567 | + Target::CompareClassification => "compare.classification".to_string(), |
| 6568 | + Target::CompareChangedArtifacts => "compare.changed_artifacts".to_string(), |
| 6569 | + Target::CompareDifferenceCount => "compare.difference_count".to_string(), |
| 6570 | + Target::CompareBasis => "compare.basis".to_string(), |
| 6571 | + Target::RunStdout => "run.stdout".to_string(), |
| 6572 | + Target::RunStderr => "run.stderr".to_string(), |
| 6573 | + Target::RunExitCode => "run.exit_code".to_string(), |
| 6574 | + } |
| 6575 | +} |
| 6576 | + |
| 6577 | +fn compare_target_text(result: &ComparisonResult, target: &Target) -> Result<String, String> { |
| 6578 | + match target { |
| 6579 | + Target::CompareStatus => Ok(compare_status(result).to_string()), |
| 6580 | + Target::CompareClassification => Ok(compare_classification(result).to_string()), |
| 6581 | + Target::CompareChangedArtifacts => { |
| 6582 | + let changed = compare_changed_artifacts(result); |
| 6583 | + if changed.is_empty() { |
| 6584 | + Ok("none".to_string()) |
| 6585 | + } else { |
| 6586 | + Ok(changed.join(", ")) |
| 6587 | + } |
| 6588 | + } |
| 6589 | + Target::CompareBasis => Ok(result.basis.clone()), |
| 6590 | + Target::CompareDifferenceCount => Err( |
| 6591 | + "compare.difference_count is numeric; use 'expect compare.difference_count equals <int>'" |
| 6592 | + .into(), |
| 6593 | + ), |
| 6594 | + _ => Err(format!( |
| 6595 | + "{} is not a compare text target", |
| 6596 | + target_name(target) |
| 6597 | + )), |
| 6598 | + } |
| 6599 | +} |
| 6600 | + |
| 6601 | +fn compare_target_int(result: &ComparisonResult, target: &Target) -> Result<i32, String> { |
| 6602 | + match target { |
| 6603 | + Target::CompareDifferenceCount => Ok(result.differences.len() as i32), |
| 6604 | + _ => Err(format!( |
| 6605 | + "{} is textual; use a string matcher instead", |
| 6606 | + target_name(target) |
| 6607 | + )), |
| 6608 | + } |
| 6609 | +} |
| 6610 | + |
| 6611 | +fn observation_target_text<'a>( |
| 6612 | + observation: &'a CompilerObservation, |
| 6613 | + target: &Target, |
| 6614 | +) -> Result<&'a str, String> { |
| 6615 | + match target { |
| 6616 | + Target::Stage(stage) => match stage { |
| 6617 | + Stage::Asm => observation_artifact_text(observation, &ArtifactKey::Asm), |
| 6618 | + Stage::Obj => observation_artifact_text(observation, &ArtifactKey::Obj), |
| 6619 | + Stage::Run => { |
| 6620 | + Err("run is structured; use run.stdout, run.stderr, or run.exit_code".into()) |
| 6621 | + } |
| 6622 | + other => observation_artifact_text( |
| 6623 | + observation, |
| 6624 | + &ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6625 | + ), |
| 6626 | + }, |
| 6627 | + Target::Artifact(artifact) => observation_artifact_text(observation, artifact), |
| 6628 | + Target::CompareStatus |
| 6629 | + | Target::CompareClassification |
| 6630 | + | Target::CompareChangedArtifacts |
| 6631 | + | Target::CompareDifferenceCount |
| 6632 | + | Target::CompareBasis => { |
| 6633 | + Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6634 | + } |
| 6635 | + Target::RunStdout => observation_run_stdout(observation), |
| 6636 | + Target::RunStderr => observation_run_stderr(observation), |
| 6637 | + Target::RunExitCode => { |
| 6638 | + Err("run.exit_code is numeric; use 'expect run.exit_code equals <int>'".into()) |
| 6639 | + } |
| 6640 | + } |
| 6641 | +} |
| 6642 | + |
| 6643 | +fn observation_target_int( |
| 6644 | + observation: &CompilerObservation, |
| 6645 | + target: &Target, |
| 6646 | +) -> Result<i32, String> { |
| 6647 | + match target { |
| 6648 | + Target::RunExitCode => observation_run_exit_code(observation), |
| 6649 | + Target::Artifact(ArtifactKey::ExitCode) => observation_run_exit_code(observation), |
| 6650 | + Target::CompareStatus |
| 6651 | + | Target::CompareClassification |
| 6652 | + | Target::CompareChangedArtifacts |
| 6653 | + | Target::CompareDifferenceCount |
| 6654 | + | Target::CompareBasis => { |
| 6655 | + Err("compare targets are only valid in compare suite-v2 cases".into()) |
| 6656 | + } |
| 6657 | + _ => Err(format!( |
| 6658 | + "{} is textual; use a string matcher instead", |
| 6659 | + target_name(target) |
| 6660 | + )), |
| 6661 | + } |
| 6662 | +} |
| 6663 | + |
| 6664 | +fn observation_artifact_text<'a>( |
| 6665 | + observation: &'a CompilerObservation, |
| 6666 | + artifact: &ArtifactKey, |
| 6667 | +) -> Result<&'a str, String> { |
| 6668 | + match observation.artifacts.get(artifact) { |
| 6669 | + Some(ArtifactValue::Text(text)) => Ok(text), |
| 6670 | + Some(ArtifactValue::Int(_)) => Err(format!( |
| 6671 | + "artifact '{}' is numeric; use an integer matcher instead", |
| 6672 | + artifact.as_str() |
| 6673 | + )), |
| 6674 | + Some(ArtifactValue::Run(_)) => Err(format!( |
| 6675 | + "artifact '{}' is structured runtime data; use run.stdout, run.stderr, or run.exit_code", |
| 6676 | + artifact.as_str() |
| 6677 | + )), |
| 6678 | + Some(ArtifactValue::Path(_)) => Err(format!( |
| 6679 | + "artifact '{}' is binary/path data, not text", |
| 6680 | + artifact.as_str() |
| 6681 | + )), |
| 6682 | + None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 6683 | + } |
| 6684 | +} |
| 6685 | + |
| 6686 | +fn observation_run_stdout(observation: &CompilerObservation) -> Result<&str, String> { |
| 6687 | + if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6688 | + Ok(&run.stdout) |
| 6689 | + } else { |
| 6690 | + observation_artifact_text(observation, &ArtifactKey::Stdout) |
| 6691 | + } |
| 6692 | +} |
| 6693 | + |
| 6694 | +fn observation_run_stderr(observation: &CompilerObservation) -> Result<&str, String> { |
| 6695 | + if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6696 | + Ok(&run.stderr) |
| 6697 | + } else { |
| 6698 | + observation_artifact_text(observation, &ArtifactKey::Stderr) |
| 6699 | + } |
| 6700 | +} |
| 6701 | + |
| 6702 | +fn observation_run_exit_code(observation: &CompilerObservation) -> Result<i32, String> { |
| 6703 | + if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 6704 | + Ok(run.exit_code) |
| 6705 | + } else { |
| 6706 | + match observation.artifacts.get(&ArtifactKey::ExitCode) { |
| 6707 | + Some(ArtifactValue::Int(value)) => Ok(*value), |
| 6708 | + Some(_) => Err("artifact 'exit-code' is not numeric".into()), |
| 6709 | + None => Err("missing artifact 'exit-code'".into()), |
| 6710 | + } |
| 6711 | + } |
| 6712 | +} |
| 6713 | + |
| 6714 | +fn observation_diagnostics_text(observation: &CompilerObservation) -> Option<&str> { |
| 6715 | + match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 6716 | + Some(ArtifactValue::Text(text)) => Some(text.as_str()), |
| 6717 | + _ => None, |
| 6718 | + } |
| 6719 | +} |
| 6720 | + |
| 6721 | +fn observation_failure_stage(observation: &CompilerObservation) -> Option<FailureStage> { |
| 6722 | + observation |
| 6723 | + .provenance |
| 6724 | + .failure_stage |
| 6725 | + .as_deref() |
| 6726 | + .and_then(FailureStage::parse) |
| 6727 | +} |
| 6728 | + |
| 6729 | +fn compare_differential( |
| 6730 | + armfortas: &CompilerObservation, |
| 6731 | + references: &[CompilerObservation], |
| 6732 | +) -> Result<(), String> { |
| 6733 | + let requested = default_differential_artifacts(); |
| 6734 | + let comparisons = references |
| 6735 | + .iter() |
| 6736 | + .cloned() |
| 6737 | + .map(|reference| compare_observations(armfortas.clone(), reference, &requested)) |
| 6738 | + .collect::<Vec<_>>(); |
| 6739 | + let matching_refs = comparisons |
| 6740 | + .iter() |
| 6741 | + .filter(|comparison| comparison.differences.is_empty()) |
| 6742 | + .count(); |
| 6743 | + |
| 6744 | + if matching_refs == comparisons.len() { |
| 6745 | + return Ok(()); |
| 6746 | + } |
| 6747 | + |
| 6748 | + let reference_disagreement = if references.len() > 1 { |
| 6749 | + let baseline = references[0].clone(); |
| 6750 | + references[1..].iter().cloned().any(|reference| { |
| 6751 | + !compare_observations(baseline.clone(), reference, &requested) |
| 6752 | + .differences |
| 6753 | + .is_empty() |
| 6754 | + }) |
| 6755 | + } else { |
| 6756 | + false |
| 6757 | + }; |
| 6758 | + |
| 6759 | + let classification = if matching_refs == 0 && !reference_disagreement { |
| 6760 | + "classification: armfortas-only divergence" |
| 6761 | + } else if reference_disagreement { |
| 6762 | + "classification: reference disagreement" |
| 6763 | + } else { |
| 6764 | + "classification: partial disagreement" |
| 6765 | + }; |
| 6766 | + |
| 6767 | + let detail = comparisons |
| 6768 | + .iter() |
| 6769 | + .filter(|comparison| !comparison.differences.is_empty()) |
| 6770 | + .map(render_compare_text) |
| 6771 | + .collect::<Vec<_>>(); |
| 6772 | + |
| 6773 | + Err(format!( |
| 6774 | + "behavior mismatch against reference compilers\n{}\n\n{}", |
| 6775 | + classification, |
| 6776 | + detail.join("\n\n") |
| 6777 | + )) |
| 6778 | +} |
| 6779 | + |
| 6780 | +fn compose_armfortas_failure_detail(artifacts: &ExecutionArtifacts) -> String { |
| 6781 | + let mut detail = String::new(); |
| 6782 | + if let Some(failure) = &artifacts.armfortas_failure { |
| 6783 | + detail.push_str(&format!( |
| 6784 | + "armfortas failed in {}\n{}", |
| 6785 | + failure.stage.as_str(), |
| 6786 | + failure.detail |
| 6787 | + )); |
| 6788 | + } else { |
| 6789 | + detail.push_str("armfortas failed without an error message"); |
| 6790 | + } |
| 6791 | + |
| 6792 | + if !artifacts.references.is_empty() { |
| 6793 | + detail.push_str("\n\nreference compilers\n"); |
| 6794 | + detail.push_str(&format_reference_summary(&artifacts.references)); |
| 6795 | + } |
| 6796 | + |
| 6797 | + detail |
| 6798 | +} |
| 6799 | + |
| 6800 | +fn compose_observation_failure_detail(observation: &CompilerObservation) -> String { |
| 6801 | + if observation.provenance.backend_mode == "unavailable" { |
| 6802 | + let mut detail = format!( |
| 6803 | + "{} unavailable for requested artifacts in this build", |
| 6804 | + observation.compiler.display_name() |
| 6805 | + ); |
| 6806 | + if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6807 | + detail.push('\n'); |
| 6808 | + detail.push_str(diagnostics); |
| 6809 | + } |
| 6810 | + return detail; |
| 6811 | + } |
| 6812 | + |
| 6813 | + if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6814 | + if diagnostics.contains("does not support requested artifacts in this adapter") { |
| 6815 | + return format!( |
| 6816 | + "{} does not support requested artifacts in this adapter\n{}", |
| 6817 | + observation.compiler.display_name(), |
| 6818 | + diagnostics |
| 6819 | + ); |
| 6820 | + } |
| 6821 | + } |
| 6822 | + |
| 6823 | + let mut detail = String::new(); |
| 6824 | + detail.push_str(&format!("{} failed", observation.compiler.display_name())); |
| 6825 | + if let Some(stage) = &observation.provenance.failure_stage { |
| 6826 | + detail.push_str(&format!(" in {}", stage)); |
| 6827 | + } |
| 6828 | + if let Some(diagnostics) = observation_diagnostics_text(observation) { |
| 6829 | + detail.push('\n'); |
| 6830 | + detail.push_str(diagnostics); |
| 6831 | + } |
| 6832 | + detail |
| 6833 | +} |
| 6834 | + |
| 6835 | +fn run_generic_differential( |
| 6836 | + compiler: &CompilerSpec, |
| 6837 | + program: &Path, |
| 6838 | + opt_level: OptLevel, |
| 6839 | + references: &[ReferenceCompiler], |
| 6840 | + tools: &ToolchainConfig, |
| 6841 | +) -> Result<(), String> { |
| 6842 | + let requested = default_differential_artifacts(); |
| 6843 | + let primary = observe_compiler(compiler, program, opt_level, &requested, tools)?; |
| 6844 | + let references = references |
| 6845 | + .iter() |
| 6846 | + .copied() |
| 6847 | + .map(reference_compiler_spec) |
| 6848 | + .map(|reference| observe_compiler(&reference, program, opt_level, &requested, tools)) |
| 6849 | + .collect::<Result<Vec<_>, _>>()?; |
| 6850 | + compare_differential(&primary, &references) |
| 6851 | +} |
| 6852 | + |
| 6853 | +fn expected_artifacts_for_legacy_case(case: &CaseSpec) -> BTreeSet<ArtifactKey> { |
| 6854 | + let mut requested = BTreeSet::new(); |
| 6855 | + for stage in &case.requested { |
| 6856 | + requested.insert(stage_to_artifact_key(*stage)); |
| 6857 | + } |
| 6858 | + for expectation in &case.expectations { |
| 6859 | + match expectation { |
| 6860 | + Expectation::CheckComments(target) |
| 6861 | + | Expectation::Contains { target, .. } |
| 6862 | + | Expectation::NotContains { target, .. } |
| 6863 | + | Expectation::Equals { target, .. } |
| 6864 | + | Expectation::IntEquals { target, .. } => match target { |
| 6865 | + Target::Stage(stage) => { |
| 6866 | + requested.insert(stage_to_artifact_key(*stage)); |
| 6867 | + } |
| 6868 | + Target::Artifact(artifact) => { |
| 6869 | + requested.insert(artifact.clone()); |
| 6870 | + } |
| 6871 | + Target::RunStdout => { |
| 6872 | + requested.insert(ArtifactKey::Stdout); |
| 6873 | + } |
| 6874 | + Target::RunStderr => { |
| 6875 | + requested.insert(ArtifactKey::Stderr); |
| 6876 | + } |
| 6877 | + Target::RunExitCode => { |
| 6878 | + requested.insert(ArtifactKey::ExitCode); |
| 6879 | + } |
| 6880 | + Target::CompareStatus |
| 6881 | + | Target::CompareClassification |
| 6882 | + | Target::CompareChangedArtifacts |
| 6883 | + | Target::CompareDifferenceCount |
| 6884 | + | Target::CompareBasis => {} |
| 6885 | + }, |
| 6886 | + Expectation::FailContains { .. } |
| 6887 | + | Expectation::FailEquals { .. } |
| 6888 | + | Expectation::FailSourceComments |
| 6889 | + | Expectation::FailCommentPatterns(_) => {} |
| 6890 | + } |
| 6891 | + } |
| 6892 | + requested |
| 6893 | +} |
| 6894 | + |
| 6895 | +fn legacy_case_uses_generic_consistency_checks(case: &CaseSpec) -> bool { |
| 6896 | + !case.consistency_checks.is_empty() |
| 6897 | + && case |
| 6898 | + .consistency_checks |
| 6899 | + .iter() |
| 6900 | + .copied() |
| 6901 | + .all(|check| check.supports_generic_introspect()) |
| 6902 | +} |
| 6903 | + |
| 6904 | +fn stage_to_artifact_key(stage: Stage) -> ArtifactKey { |
| 6905 | + match stage { |
| 6906 | + Stage::Asm => ArtifactKey::Asm, |
| 6907 | + Stage::Obj => ArtifactKey::Obj, |
| 6908 | + Stage::Run => ArtifactKey::Runtime, |
| 6909 | + other => ArtifactKey::Extra(format!("armfortas.{}", other.as_str())), |
| 6910 | + } |
| 6911 | +} |
| 6912 | + |
| 6913 | +fn observed_program_from_armfortas_capture( |
| 6914 | + program: &Path, |
| 6915 | + opt_level: OptLevel, |
| 6916 | + requested_artifacts: BTreeSet<ArtifactKey>, |
| 6917 | + result: &CaptureResult, |
| 6918 | + failure: Option<&CaptureFailure>, |
| 6919 | +) -> ObservedProgram { |
| 6920 | + let mut artifacts = BTreeMap::new(); |
| 6921 | + for (stage, captured) in &result.stages { |
| 6922 | + match (stage, captured) { |
| 6923 | + (Stage::Asm, CapturedStage::Text(text)) |
| 6924 | + if requested_artifacts.contains(&ArtifactKey::Asm) => |
| 6925 | + { |
| 6926 | + artifacts.insert(ArtifactKey::Asm, ArtifactValue::Text(text.clone())); |
| 6927 | + } |
| 6928 | + (Stage::Obj, CapturedStage::Text(text)) |
| 6929 | + if requested_artifacts.contains(&ArtifactKey::Obj) => |
| 6930 | + { |
| 6931 | + artifacts.insert(ArtifactKey::Obj, ArtifactValue::Text(text.clone())); |
| 6932 | + } |
| 6933 | + (Stage::Run, CapturedStage::Run(run)) => { |
| 6934 | + insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 6935 | + } |
| 6936 | + (stage, CapturedStage::Text(text)) => { |
| 6937 | + let key = ArtifactKey::Extra(format!("armfortas.{}", stage.as_str())); |
| 6938 | + if requested_artifacts.contains(&key) { |
| 6939 | + artifacts.insert(key, ArtifactValue::Text(text.clone())); |
| 6940 | + } |
| 6941 | + } |
| 6942 | + _ => {} |
| 6943 | + } |
| 6944 | + } |
| 6945 | + if let Some(failure) = failure { |
| 6946 | + if requested_artifacts.contains(&ArtifactKey::Diagnostics) |
| 6947 | + || !artifacts.contains_key(&ArtifactKey::Diagnostics) |
| 6948 | + { |
| 6949 | + artifacts.insert( |
| 6950 | + ArtifactKey::Diagnostics, |
| 6951 | + ArtifactValue::Text(failure.detail.clone()), |
| 6952 | + ); |
| 6953 | + } |
| 6954 | + } |
| 6955 | + let artifacts_captured = artifacts |
| 6956 | + .keys() |
| 6957 | + .map(|artifact| artifact.as_str().to_string()) |
| 6958 | + .collect::<Vec<_>>(); |
| 6959 | + ObservedProgram { |
| 6960 | + observation: CompilerObservation { |
| 6961 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 6962 | + program: program.to_path_buf(), |
| 6963 | + opt_level, |
| 6964 | + compile_exit_code: if failure.is_some() { 1 } else { 0 }, |
| 6965 | + artifacts, |
| 6966 | + provenance: ObservationProvenance { |
| 6967 | + compiler_identity: "armfortas".into(), |
| 6968 | + adapter_kind: "named".into(), |
| 6969 | + backend_mode: "suite-legacy-capture".into(), |
| 6970 | + backend_detail: "legacy suite cell capture converted into generic observation" |
| 6971 | + .into(), |
| 6972 | + artifacts_captured, |
| 6973 | + comparison_basis: None, |
| 6974 | + failure_stage: failure.map(|failure| failure.stage.as_str().to_string()), |
| 6975 | + }, |
| 6976 | + }, |
| 6977 | + requested_artifacts, |
| 6978 | + } |
| 6979 | +} |
| 6980 | + |
| 6981 | +fn observed_program_from_reference_result( |
| 6982 | + program: &Path, |
| 6983 | + opt_level: OptLevel, |
| 6984 | + requested_artifacts: BTreeSet<ArtifactKey>, |
| 6985 | + reference: &ReferenceResult, |
| 6986 | +) -> ObservedProgram { |
| 6987 | + let mut artifacts = BTreeMap::new(); |
| 6988 | + let diagnostics = [ |
| 6989 | + reference.compile_stdout.trim_end(), |
| 6990 | + reference.compile_stderr.trim_end(), |
| 6991 | + ] |
| 6992 | + .iter() |
| 6993 | + .filter(|part| !part.is_empty()) |
| 6994 | + .copied() |
| 6995 | + .collect::<Vec<_>>() |
| 6996 | + .join("\n"); |
| 6997 | + |
| 6998 | + if requested_artifacts.contains(&ArtifactKey::Diagnostics) && !diagnostics.is_empty() { |
| 6999 | + artifacts.insert(ArtifactKey::Diagnostics, ArtifactValue::Text(diagnostics)); |
| 7000 | + } |
| 7001 | + |
| 7002 | + if let Some(run) = &reference.run { |
| 7003 | + insert_run_artifacts(&requested_artifacts, run, &mut artifacts); |
| 7004 | + } else if let Some(run_error) = &reference.run_error { |
| 7005 | + let diagnostics = artifacts |
| 7006 | + .entry(ArtifactKey::Diagnostics) |
| 7007 | + .or_insert_with(|| ArtifactValue::Text(String::new())); |
| 7008 | + if let ArtifactValue::Text(text) = diagnostics { |
| 7009 | + if !text.is_empty() { |
| 7010 | + text.push('\n'); |
| 7011 | + } |
| 7012 | + text.push_str(&format!("run error: {}", run_error)); |
| 7013 | + } |
| 7014 | + } |
| 7015 | + |
| 7016 | + let artifacts_captured = artifacts |
| 7017 | + .keys() |
| 7018 | + .map(|artifact| artifact.as_str().to_string()) |
| 7019 | + .collect::<Vec<_>>(); |
| 7020 | + ObservedProgram { |
| 7021 | + observation: CompilerObservation { |
| 7022 | + compiler: match reference.compiler { |
| 7023 | + ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 7024 | + ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 7025 | + }, |
| 7026 | + program: program.to_path_buf(), |
| 7027 | + opt_level, |
| 7028 | + compile_exit_code: reference.compile_exit_code, |
| 7029 | + artifacts, |
| 7030 | + provenance: ObservationProvenance { |
| 7031 | + compiler_identity: reference.compiler.as_str().to_string(), |
| 7032 | + adapter_kind: "named".into(), |
| 7033 | + backend_mode: "legacy-reference".into(), |
| 7034 | + backend_detail: format!( |
| 7035 | + "legacy differential reference observation via {}", |
| 7036 | + reference.compile_command |
| 7037 | + ), |
| 7038 | + artifacts_captured, |
| 7039 | + comparison_basis: None, |
| 7040 | + failure_stage: None, |
| 7041 | + }, |
| 7042 | + }, |
| 7043 | + requested_artifacts, |
| 7044 | + } |
| 7045 | +} |
| 7046 | + |
| 7047 | +fn reference_compiler_spec(compiler: ReferenceCompiler) -> CompilerSpec { |
| 7048 | + match compiler { |
| 7049 | + ReferenceCompiler::Gfortran => CompilerSpec::Named(NamedCompiler::Gfortran), |
| 7050 | + ReferenceCompiler::FlangNew => CompilerSpec::Named(NamedCompiler::FlangNew), |
| 7051 | + } |
| 7052 | +} |
| 7053 | + |
| 7054 | +fn run_generic_consistency_checks( |
| 7055 | + compiler: &CompilerSpec, |
| 7056 | + case: &CaseSpec, |
| 7057 | + source: &Path, |
| 7058 | + opt_level: OptLevel, |
| 7059 | + tools: &ToolchainConfig, |
| 7060 | +) -> Vec<ConsistencyIssue> { |
| 7061 | + let mut failures = Vec::new(); |
| 7062 | + for check in &case.consistency_checks { |
| 7063 | + let issue = match check { |
| 7064 | + ConsistencyCheck::CliAsmReproducible => run_generic_cli_asm_reproducible( |
| 7065 | + compiler, |
| 7066 | + source, |
| 7067 | + opt_level, |
| 7068 | + case.repeat_count, |
| 7069 | + tools, |
| 7070 | + ), |
| 7071 | + ConsistencyCheck::CliObjReproducible => run_generic_cli_obj_reproducible( |
| 7072 | + compiler, |
| 7073 | + source, |
| 7074 | + opt_level, |
| 7075 | + case.repeat_count, |
| 7076 | + tools, |
| 7077 | + ), |
| 7078 | + ConsistencyCheck::CliRunReproducible => run_generic_cli_run_reproducible( |
| 7079 | + compiler, |
| 7080 | + source, |
| 7081 | + opt_level, |
| 7082 | + case.repeat_count, |
| 7083 | + tools, |
| 7084 | + ), |
| 7085 | + _ => Some(ConsistencyIssue { |
| 7086 | + check: *check, |
| 7087 | + summary: "unsupported generic consistency check".into(), |
| 7088 | + repeat_count: None, |
| 7089 | + unique_variant_count: None, |
| 7090 | + varying_components: Vec::new(), |
| 7091 | + stable_components: Vec::new(), |
| 7092 | + detail: format!( |
| 7093 | + "generic compiler cases do not support '{}' yet", |
| 7094 | + check.as_str() |
| 7095 | + ), |
| 7096 | + temp_root: next_consistency_temp_root(opt_level), |
| 7097 | + }), |
| 7098 | + }; |
| 7099 | + if let Some(issue) = issue { |
| 7100 | + failures.push(issue); |
| 7101 | + } |
| 7102 | + } |
| 7103 | + failures |
| 7104 | +} |
| 7105 | + |
| 7106 | +fn run_generic_cli_asm_reproducible( |
| 7107 | + compiler: &CompilerSpec, |
| 7108 | + source: &Path, |
| 7109 | + opt_level: OptLevel, |
| 7110 | + repeat_count: usize, |
| 7111 | + tools: &ToolchainConfig, |
| 7112 | +) -> Option<ConsistencyIssue> { |
| 7113 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7114 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7115 | + return Some(ConsistencyIssue { |
| 7116 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7117 | + summary: "could not create consistency temp dir".into(), |
| 7118 | + repeat_count: None, |
| 7119 | + unique_variant_count: None, |
| 7120 | + varying_components: Vec::new(), |
| 7121 | + stable_components: Vec::new(), |
| 7122 | + detail: format!( |
| 7123 | + "cannot create consistency temp dir '{}': {}", |
| 7124 | + temp_root.display(), |
| 7125 | + err |
| 7126 | + ), |
| 7127 | + temp_root, |
| 7128 | + }); |
| 7129 | + } |
| 7130 | + |
| 7131 | + let requested = BTreeSet::from([ArtifactKey::Asm]); |
| 7132 | + let mut runs = Vec::new(); |
| 7133 | + for index in 0..repeat_count { |
| 7134 | + let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7135 | + Ok(observation) => observation, |
| 7136 | + Err(detail) => { |
| 7137 | + return Some(ConsistencyIssue { |
| 7138 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7139 | + summary: "compiler observation failed during consistency check".into(), |
| 7140 | + repeat_count: Some(repeat_count), |
| 7141 | + unique_variant_count: None, |
| 7142 | + varying_components: Vec::new(), |
| 7143 | + stable_components: Vec::new(), |
| 7144 | + detail, |
| 7145 | + temp_root, |
| 7146 | + }) |
| 7147 | + } |
| 7148 | + }; |
| 7149 | + let asm = match observation_text_artifact(&observation, &ArtifactKey::Asm) { |
| 7150 | + Ok(asm) => asm, |
| 7151 | + Err(detail) => { |
| 7152 | + return Some(ConsistencyIssue { |
| 7153 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7154 | + summary: "missing asm artifact during consistency check".into(), |
| 7155 | + repeat_count: Some(repeat_count), |
| 7156 | + unique_variant_count: None, |
| 7157 | + varying_components: Vec::new(), |
| 7158 | + stable_components: Vec::new(), |
| 7159 | + detail, |
| 7160 | + temp_root, |
| 7161 | + }) |
| 7162 | + } |
| 7163 | + }; |
| 7164 | + runs.push(TextRun { |
| 7165 | + label: format!("run {}", index + 1), |
| 7166 | + command: observation_command_hint(&observation), |
| 7167 | + normalized: normalize_text_artifact(&asm), |
| 7168 | + }); |
| 7169 | + } |
| 7170 | + |
| 7171 | + let unique_variant_count = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 7172 | + if unique_variant_count > 1 { |
| 7173 | + let (left, right) = first_distinct_text_pair(&runs).unwrap(); |
| 7174 | + return Some(ConsistencyIssue { |
| 7175 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7176 | + summary: format!( |
| 7177 | + "repeat_count={} unique_variants={}", |
| 7178 | + repeat_count, unique_variant_count |
| 7179 | + ), |
| 7180 | + repeat_count: Some(repeat_count), |
| 7181 | + unique_variant_count: Some(unique_variant_count), |
| 7182 | + varying_components: Vec::new(), |
| 7183 | + stable_components: Vec::new(), |
| 7184 | + detail: format!( |
| 7185 | + "asm output was not reproducible for {}\n{}\n{}\n{}", |
| 7186 | + compiler.display_name(), |
| 7187 | + left.command, |
| 7188 | + right.command, |
| 7189 | + describe_text_difference( |
| 7190 | + &left.normalized, |
| 7191 | + &right.normalized, |
| 7192 | + &left.label, |
| 7193 | + &right.label |
| 7194 | + ) |
| 7195 | + ), |
| 7196 | + temp_root, |
| 7197 | + }); |
| 7198 | + } |
| 7199 | + |
| 7200 | + let _ = fs::remove_dir_all(&temp_root); |
| 7201 | + None |
| 7202 | +} |
| 7203 | + |
| 7204 | +fn run_generic_cli_obj_reproducible( |
| 7205 | + compiler: &CompilerSpec, |
| 7206 | + source: &Path, |
| 7207 | + opt_level: OptLevel, |
| 7208 | + repeat_count: usize, |
| 7209 | + tools: &ToolchainConfig, |
| 7210 | +) -> Option<ConsistencyIssue> { |
| 7211 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7212 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7213 | + return Some(ConsistencyIssue { |
| 7214 | + check: ConsistencyCheck::CliObjReproducible, |
| 7215 | + summary: "could not create consistency temp dir".into(), |
| 7216 | + repeat_count: None, |
| 7217 | + unique_variant_count: None, |
| 7218 | + varying_components: Vec::new(), |
| 7219 | + stable_components: Vec::new(), |
| 7220 | + detail: format!( |
| 7221 | + "cannot create consistency temp dir '{}': {}", |
| 7222 | + temp_root.display(), |
| 7223 | + err |
| 7224 | + ), |
| 7225 | + temp_root, |
| 7226 | + }); |
| 7227 | + } |
| 7228 | + |
| 7229 | + let requested = BTreeSet::from([ArtifactKey::Obj]); |
| 7230 | + let mut rendered_runs = Vec::new(); |
| 7231 | + let mut object_runs = Vec::new(); |
| 7232 | + let mut parseable = true; |
| 7233 | + for index in 0..repeat_count { |
| 7234 | + let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7235 | + Ok(observation) => observation, |
| 7236 | + Err(detail) => { |
| 7237 | + return Some(ConsistencyIssue { |
| 7238 | + check: ConsistencyCheck::CliObjReproducible, |
| 7239 | + summary: "compiler observation failed during consistency check".into(), |
| 7240 | + repeat_count: Some(repeat_count), |
| 7241 | + unique_variant_count: None, |
| 7242 | + varying_components: Vec::new(), |
| 7243 | + stable_components: Vec::new(), |
| 7244 | + detail, |
| 7245 | + temp_root, |
| 7246 | + }) |
| 7247 | + } |
| 7248 | + }; |
| 7249 | + let obj_text = match observation_text_artifact(&observation, &ArtifactKey::Obj) { |
| 7250 | + Ok(text) => text, |
| 7251 | + Err(detail) => { |
| 7252 | + return Some(ConsistencyIssue { |
| 7253 | + check: ConsistencyCheck::CliObjReproducible, |
| 7254 | + summary: "missing obj artifact during consistency check".into(), |
| 7255 | + repeat_count: Some(repeat_count), |
| 7256 | + unique_variant_count: None, |
| 7257 | + varying_components: Vec::new(), |
| 7258 | + stable_components: Vec::new(), |
| 7259 | + detail, |
| 7260 | + temp_root, |
| 7261 | + }) |
| 7262 | + } |
| 7263 | + }; |
| 7264 | + let label = format!("run {}", index + 1); |
| 7265 | + let command = observation_command_hint(&observation); |
| 7266 | + rendered_runs.push(TextRun { |
| 7267 | + label: label.clone(), |
| 7268 | + command: command.clone(), |
| 7269 | + normalized: normalize_text_artifact(&obj_text), |
| 7270 | + }); |
| 7271 | + match parse_object_snapshot_text(&obj_text) { |
| 7272 | + Ok(snapshot) => object_runs.push(ObjectRun { |
| 7273 | + label, |
| 7274 | + command, |
| 7275 | + snapshot, |
| 7276 | + }), |
| 7277 | + Err(_) => parseable = false, |
| 7278 | + } |
| 7279 | + } |
| 7280 | + |
| 7281 | + if parseable { |
| 7282 | + let rendered = object_runs |
| 7283 | + .iter() |
| 7284 | + .map(|run| render_object_snapshot(&run.snapshot)) |
| 7285 | + .collect::<Vec<_>>(); |
| 7286 | + let unique_variant_count = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7287 | + if unique_variant_count > 1 { |
| 7288 | + let (left, right) = first_distinct_object_pair(&object_runs).unwrap(); |
| 7289 | + let snapshots = object_runs |
| 7290 | + .iter() |
| 7291 | + .map(|run| &run.snapshot) |
| 7292 | + .collect::<Vec<_>>(); |
| 7293 | + let varying = varying_object_components(&snapshots) |
| 7294 | + .into_iter() |
| 7295 | + .map(str::to_string) |
| 7296 | + .collect::<Vec<_>>(); |
| 7297 | + let stable = stable_object_components(&snapshots) |
| 7298 | + .into_iter() |
| 7299 | + .map(str::to_string) |
| 7300 | + .collect::<Vec<_>>(); |
| 7301 | + return Some(ConsistencyIssue { |
| 7302 | + check: ConsistencyCheck::CliObjReproducible, |
| 7303 | + summary: format!( |
| 7304 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7305 | + repeat_count, |
| 7306 | + unique_variant_count, |
| 7307 | + join_or_none_from_strings(&varying), |
| 7308 | + join_or_none_from_strings(&stable) |
| 7309 | + ), |
| 7310 | + repeat_count: Some(repeat_count), |
| 7311 | + unique_variant_count: Some(unique_variant_count), |
| 7312 | + varying_components: varying, |
| 7313 | + stable_components: stable, |
| 7314 | + detail: format!( |
| 7315 | + "object output was not reproducible for {}\n{}\n{}\n{}", |
| 7316 | + compiler.display_name(), |
| 7317 | + left.command, |
| 7318 | + right.command, |
| 7319 | + describe_object_difference( |
| 7320 | + &left.snapshot, |
| 7321 | + &right.snapshot, |
| 7322 | + &left.label, |
| 7323 | + &right.label |
| 7324 | + ) |
| 7325 | + ), |
| 7326 | + temp_root, |
| 7327 | + }); |
| 7328 | + } |
| 7329 | + } else { |
| 7330 | + let unique_variant_count = |
| 7331 | + count_unique_strings(rendered_runs.iter().map(|run| run.normalized.as_str())); |
| 7332 | + if unique_variant_count > 1 { |
| 7333 | + let (left, right) = first_distinct_text_pair(&rendered_runs).unwrap(); |
| 7334 | + return Some(ConsistencyIssue { |
| 7335 | + check: ConsistencyCheck::CliObjReproducible, |
| 7336 | + summary: format!( |
| 7337 | + "repeat_count={} unique_variants={}", |
| 7338 | + repeat_count, unique_variant_count |
| 7339 | + ), |
| 7340 | + repeat_count: Some(repeat_count), |
| 7341 | + unique_variant_count: Some(unique_variant_count), |
| 7342 | + varying_components: Vec::new(), |
| 7343 | + stable_components: Vec::new(), |
| 7344 | + detail: format!( |
| 7345 | + "object artifact text was not reproducible for {}\n{}\n{}\n{}", |
| 7346 | + compiler.display_name(), |
| 7347 | + left.command, |
| 7348 | + right.command, |
| 7349 | + describe_text_difference( |
| 7350 | + &left.normalized, |
| 7351 | + &right.normalized, |
| 7352 | + &left.label, |
| 7353 | + &right.label |
| 7354 | + ) |
| 7355 | + ), |
| 7356 | + temp_root, |
| 7357 | + }); |
| 7358 | + } |
| 7359 | + } |
| 7360 | + |
| 7361 | + let _ = fs::remove_dir_all(&temp_root); |
| 7362 | + None |
| 7363 | +} |
| 7364 | + |
| 7365 | +fn run_generic_cli_run_reproducible( |
| 7366 | + compiler: &CompilerSpec, |
| 7367 | + source: &Path, |
| 7368 | + opt_level: OptLevel, |
| 7369 | + repeat_count: usize, |
| 7370 | + tools: &ToolchainConfig, |
| 7371 | +) -> Option<ConsistencyIssue> { |
| 7372 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7373 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7374 | + return Some(ConsistencyIssue { |
| 7375 | + check: ConsistencyCheck::CliRunReproducible, |
| 7376 | + summary: "could not create consistency temp dir".into(), |
| 7377 | + repeat_count: None, |
| 7378 | + unique_variant_count: None, |
| 7379 | + varying_components: Vec::new(), |
| 7380 | + stable_components: Vec::new(), |
| 7381 | + detail: format!( |
| 7382 | + "cannot create consistency temp dir '{}': {}", |
| 7383 | + temp_root.display(), |
| 7384 | + err |
| 7385 | + ), |
| 7386 | + temp_root, |
| 7387 | + }); |
| 7388 | + } |
| 7389 | + |
| 7390 | + let requested = BTreeSet::from([ArtifactKey::Runtime]); |
| 7391 | + let mut runs = Vec::new(); |
| 7392 | + for index in 0..repeat_count { |
| 7393 | + let observation = match observe_compiler(compiler, source, opt_level, &requested, tools) { |
| 7394 | + Ok(observation) => observation, |
| 7395 | + Err(detail) => { |
| 7396 | + return Some(ConsistencyIssue { |
| 7397 | + check: ConsistencyCheck::CliRunReproducible, |
| 7398 | + summary: "compiler observation failed during consistency check".into(), |
| 7399 | + repeat_count: Some(repeat_count), |
| 7400 | + unique_variant_count: None, |
| 7401 | + varying_components: Vec::new(), |
| 7402 | + stable_components: Vec::new(), |
| 7403 | + detail, |
| 7404 | + temp_root, |
| 7405 | + }) |
| 7406 | + } |
| 7407 | + }; |
| 7408 | + let run = match observation_run_capture(&observation) { |
| 7409 | + Ok(run) => run, |
| 7410 | + Err(detail) => { |
| 7411 | + return Some(ConsistencyIssue { |
| 7412 | + check: ConsistencyCheck::CliRunReproducible, |
| 7413 | + summary: "missing runtime artifact during consistency check".into(), |
| 7414 | + repeat_count: Some(repeat_count), |
| 7415 | + unique_variant_count: None, |
| 7416 | + varying_components: Vec::new(), |
| 7417 | + stable_components: Vec::new(), |
| 7418 | + detail, |
| 7419 | + temp_root, |
| 7420 | + }) |
| 7421 | + } |
| 7422 | + }; |
| 7423 | + runs.push(BehaviorRun { |
| 7424 | + label: format!("run {}", index + 1), |
| 7425 | + command: observation_command_hint(&observation), |
| 7426 | + signature: normalize_run_signature(&run), |
| 7427 | + run, |
| 7428 | + }); |
| 7429 | + } |
| 7430 | + |
| 7431 | + let unique_variant_count = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 7432 | + if unique_variant_count > 1 { |
| 7433 | + let (left, right) = first_distinct_behavior_pair(&runs).unwrap(); |
| 7434 | + let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 7435 | + let varying = varying_run_components(&signatures) |
| 7436 | + .into_iter() |
| 7437 | + .map(str::to_string) |
| 7438 | + .collect::<Vec<_>>(); |
| 7439 | + let stable = stable_run_components(&signatures) |
| 7440 | + .into_iter() |
| 7441 | + .map(str::to_string) |
| 7442 | + .collect::<Vec<_>>(); |
| 7443 | + return Some(ConsistencyIssue { |
| 7444 | + check: ConsistencyCheck::CliRunReproducible, |
| 7445 | + summary: format!( |
| 7446 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7447 | + repeat_count, |
| 7448 | + unique_variant_count, |
| 7449 | + join_or_none_from_strings(&varying), |
| 7450 | + join_or_none_from_strings(&stable) |
| 7451 | + ), |
| 7452 | + repeat_count: Some(repeat_count), |
| 7453 | + unique_variant_count: Some(unique_variant_count), |
| 7454 | + varying_components: varying, |
| 7455 | + stable_components: stable, |
| 7456 | + detail: format!( |
| 7457 | + "runtime behavior was not reproducible for {}\n{}\n{}\n{}", |
| 7458 | + compiler.display_name(), |
| 7459 | + left.command, |
| 7460 | + right.command, |
| 7461 | + describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 7462 | + ), |
| 7463 | + temp_root, |
| 7464 | + }); |
| 7465 | + } |
| 7466 | + |
| 7467 | + let _ = fs::remove_dir_all(&temp_root); |
| 7468 | + None |
| 7469 | +} |
| 7470 | + |
| 7471 | +fn observation_text_artifact( |
| 7472 | + observation: &CompilerObservation, |
| 7473 | + artifact: &ArtifactKey, |
| 7474 | +) -> Result<String, String> { |
| 7475 | + match observation.artifacts.get(artifact) { |
| 7476 | + Some(ArtifactValue::Text(text)) => Ok(text.clone()), |
| 7477 | + Some(ArtifactValue::Int(_)) => Err(format!( |
| 7478 | + "artifact '{}' is numeric, not text", |
| 7479 | + artifact.as_str() |
| 7480 | + )), |
| 7481 | + Some(ArtifactValue::Run(_)) => Err(format!( |
| 7482 | + "artifact '{}' is structured runtime data, not text", |
| 7483 | + artifact.as_str() |
| 7484 | + )), |
| 7485 | + Some(ArtifactValue::Path(path)) => Err(format!( |
| 7486 | + "artifact '{}' is path data ('{}'), not text", |
| 7487 | + artifact.as_str(), |
| 7488 | + path.display() |
| 7489 | + )), |
| 7490 | + None => Err(format!("missing artifact '{}'", artifact.as_str())), |
| 7491 | + } |
| 7492 | +} |
| 7493 | + |
| 7494 | +fn observation_run_capture(observation: &CompilerObservation) -> Result<RunCapture, String> { |
| 7495 | + if let Some(ArtifactValue::Run(run)) = observation.artifacts.get(&ArtifactKey::Runtime) { |
| 7496 | + return Ok(run.clone()); |
| 7497 | + } |
| 7498 | + |
| 7499 | + Ok(RunCapture { |
| 7500 | + exit_code: observation_run_exit_code(observation)?, |
| 7501 | + stdout: observation_run_stdout(observation)?.to_string(), |
| 7502 | + stderr: observation_run_stderr(observation)?.to_string(), |
| 7503 | + }) |
| 7504 | +} |
| 7505 | + |
| 7506 | +fn observation_command_hint(observation: &CompilerObservation) -> String { |
| 7507 | + format!( |
| 7508 | + "{} [{}; {}]", |
| 7509 | + observation.compiler.display_name(), |
| 7510 | + observation.provenance.backend_mode, |
| 7511 | + observation.provenance.backend_detail |
| 7512 | + ) |
| 7513 | +} |
| 7514 | + |
| 7515 | +fn run_consistency_checks( |
| 7516 | + case: &CaseSpec, |
| 7517 | + prepared: &PreparedInput, |
| 7518 | + opt_level: OptLevel, |
| 7519 | + capture_result: &CaptureResult, |
| 7520 | + tools: &ToolchainConfig, |
| 7521 | +) -> Vec<ConsistencyIssue> { |
| 7522 | + let mut failures = Vec::new(); |
| 7523 | + for check in &case.consistency_checks { |
| 7524 | + let issue = match check { |
| 7525 | + ConsistencyCheck::CliObjVsSystemAs => { |
| 7526 | + run_cli_obj_vs_system_as(&prepared.compiler_source, opt_level, tools) |
| 7527 | + } |
| 7528 | + ConsistencyCheck::CliAsmReproducible => run_cli_asm_reproducible( |
| 7529 | + &prepared.compiler_source, |
| 7530 | + opt_level, |
| 7531 | + case.repeat_count, |
| 7532 | + tools, |
| 7533 | + ), |
| 7534 | + ConsistencyCheck::CliObjReproducible => run_cli_obj_reproducible( |
| 7535 | + &prepared.compiler_source, |
| 7536 | + opt_level, |
| 7537 | + case.repeat_count, |
| 7538 | + tools, |
| 7539 | + ), |
| 7540 | + ConsistencyCheck::CliRunReproducible => run_cli_run_reproducible( |
| 7541 | + &prepared.compiler_source, |
| 7542 | + opt_level, |
| 7543 | + case.repeat_count, |
| 7544 | + tools, |
| 7545 | + ), |
| 7546 | + ConsistencyCheck::CaptureAsmVsCliAsm => run_capture_asm_vs_cli_asm( |
| 7547 | + &prepared.compiler_source, |
| 7548 | + opt_level, |
| 7549 | + case.repeat_count, |
| 7550 | + capture_result, |
| 7551 | + tools, |
| 7552 | + ), |
| 7553 | + ConsistencyCheck::CaptureObjVsCliObj => run_capture_obj_vs_cli_obj( |
| 7554 | + &prepared.compiler_source, |
| 7555 | + opt_level, |
| 7556 | + case.repeat_count, |
| 7557 | + capture_result, |
| 7558 | + tools, |
| 7559 | + ), |
| 7560 | + ConsistencyCheck::CaptureRunVsCliRun => run_capture_run_vs_cli_run( |
| 7561 | + &prepared.compiler_source, |
| 7562 | + opt_level, |
| 7563 | + case.repeat_count, |
| 7564 | + capture_result, |
| 7565 | + tools, |
| 7566 | + ), |
| 7567 | + ConsistencyCheck::CaptureAsmReproducible => run_capture_asm_reproducible( |
| 7568 | + &prepared.compiler_source, |
| 7569 | + opt_level, |
| 7570 | + case.repeat_count, |
| 7571 | + capture_result, |
| 7572 | + tools, |
| 7573 | + ), |
| 7574 | + ConsistencyCheck::CaptureObjReproducible => run_capture_obj_reproducible( |
| 7575 | + &prepared.compiler_source, |
| 7576 | + opt_level, |
| 7577 | + case.repeat_count, |
| 7578 | + capture_result, |
| 7579 | + tools, |
| 7580 | + ), |
| 7581 | + ConsistencyCheck::CaptureRunReproducible => run_capture_run_reproducible( |
| 7582 | + &prepared.compiler_source, |
| 7583 | + opt_level, |
| 7584 | + case.repeat_count, |
| 7585 | + capture_result, |
| 7586 | + tools, |
| 7587 | + ), |
| 7588 | + }; |
| 7589 | + if let Some(issue) = issue { |
| 7590 | + failures.push(issue); |
| 7591 | + } |
| 7592 | + } |
| 7593 | + failures |
| 7594 | +} |
| 7595 | + |
| 7596 | +fn format_consistency_issues(issues: &[ConsistencyIssue]) -> String { |
| 7597 | + issues |
| 7598 | + .iter() |
| 7599 | + .map(|issue| { |
| 7600 | + format!( |
| 7601 | + "consistency check '{}' failed\n{}", |
| 7602 | + issue.check.as_str(), |
| 7603 | + issue.detail |
| 7604 | + ) |
| 7605 | + }) |
| 7606 | + .collect::<Vec<_>>() |
| 7607 | + .join("\n\n") |
| 7608 | +} |
| 7609 | + |
| 7610 | +fn cleanup_consistency_issues(issues: &[ConsistencyIssue]) { |
| 7611 | + for issue in issues { |
| 7612 | + let _ = fs::remove_dir_all(&issue.temp_root); |
| 7613 | + } |
| 7614 | +} |
| 7615 | + |
| 7616 | +fn run_cli_obj_vs_system_as( |
| 7617 | + source: &Path, |
| 7618 | + opt_level: OptLevel, |
| 7619 | + tools: &ToolchainConfig, |
| 7620 | +) -> Option<ConsistencyIssue> { |
| 7621 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7622 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7623 | + return Some(ConsistencyIssue { |
| 7624 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7625 | + summary: "could not create consistency temp dir".into(), |
| 7626 | + repeat_count: None, |
| 7627 | + unique_variant_count: None, |
| 7628 | + varying_components: Vec::new(), |
| 7629 | + stable_components: Vec::new(), |
| 7630 | + detail: format!( |
| 7631 | + "cannot create consistency temp dir '{}': {}", |
| 7632 | + temp_root.display(), |
| 7633 | + err |
| 7634 | + ), |
| 7635 | + temp_root, |
| 7636 | + }); |
| 7637 | + } |
| 7638 | + |
| 7639 | + let asm_path = temp_root.join("from_cli.s"); |
| 7640 | + let asm_obj_path = temp_root.join("from_cli_asm.o"); |
| 7641 | + let obj_path = temp_root.join("from_cli_obj.o"); |
| 7642 | + |
| 7643 | + let asm_command = |
| 7644 | + match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7645 | + Ok(command) => command, |
| 7646 | + Err(detail) => { |
| 7647 | + return Some(ConsistencyIssue { |
| 7648 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7649 | + summary: "armfortas -S failed during consistency check".into(), |
| 7650 | + repeat_count: None, |
| 7651 | + unique_variant_count: None, |
| 7652 | + varying_components: Vec::new(), |
| 7653 | + stable_components: Vec::new(), |
| 7654 | + detail, |
| 7655 | + temp_root, |
| 7656 | + }) |
| 7657 | + } |
| 7658 | + }; |
| 7659 | + |
| 7660 | + let as_args = vec![ |
| 7661 | + "-o".to_string(), |
| 7662 | + asm_obj_path.display().to_string(), |
| 7663 | + asm_path.display().to_string(), |
| 7664 | + ]; |
| 7665 | + let as_command = render_command(tools.system_as_bin(), &as_args); |
| 7666 | + let as_output = match Command::new(tools.system_as_bin()) |
| 7667 | + .args([ |
| 7668 | + "-o", |
| 7669 | + asm_obj_path.to_str().unwrap(), |
| 7670 | + asm_path.to_str().unwrap(), |
| 7671 | + ]) |
| 7672 | + .output() |
| 7673 | + { |
| 7674 | + Ok(output) => output, |
| 7675 | + Err(err) => { |
| 7676 | + return Some(ConsistencyIssue { |
| 7677 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7678 | + summary: "system assembler invocation failed".into(), |
| 7679 | + repeat_count: None, |
| 7680 | + unique_variant_count: None, |
| 7681 | + varying_components: Vec::new(), |
| 7682 | + stable_components: Vec::new(), |
| 7683 | + detail: format!("{}\ncannot run assembler: {}", as_command, err), |
| 7684 | + temp_root, |
| 7685 | + }) |
| 7686 | + } |
| 7687 | + }; |
| 7688 | + if !as_output.status.success() { |
| 7689 | + let stderr = String::from_utf8_lossy(&as_output.stderr); |
| 7690 | + return Some(ConsistencyIssue { |
| 7691 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7692 | + summary: "system assembler rejected armfortas -S output".into(), |
| 7693 | + repeat_count: None, |
| 7694 | + unique_variant_count: None, |
| 7695 | + varying_components: Vec::new(), |
| 7696 | + stable_components: Vec::new(), |
| 7697 | + detail: format!("{}\nassembler failed:\n{}", as_command, stderr), |
| 7698 | + temp_root, |
| 7699 | + }); |
| 7700 | + } |
| 7701 | + |
| 7702 | + let obj_command = |
| 7703 | + match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7704 | + Ok(command) => command, |
| 7705 | + Err(detail) => { |
| 7706 | + return Some(ConsistencyIssue { |
| 7707 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7708 | + summary: "armfortas -c failed during consistency check".into(), |
| 7709 | + repeat_count: None, |
| 7710 | + unique_variant_count: None, |
| 7711 | + varying_components: Vec::new(), |
| 7712 | + stable_components: Vec::new(), |
| 7713 | + detail, |
| 7714 | + temp_root, |
| 7715 | + }) |
| 7716 | + } |
| 7717 | + }; |
| 7718 | + |
| 7719 | + let asm_snapshot = match object_snapshot(&asm_obj_path, tools) { |
| 7720 | + Ok(snapshot) => snapshot, |
| 7721 | + Err(detail) => { |
| 7722 | + return Some(ConsistencyIssue { |
| 7723 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7724 | + summary: "could not snapshot object assembled from -S output".into(), |
| 7725 | + repeat_count: None, |
| 7726 | + unique_variant_count: None, |
| 7727 | + varying_components: Vec::new(), |
| 7728 | + stable_components: Vec::new(), |
| 7729 | + detail: format!("{}\n{}", as_command, detail), |
| 7730 | + temp_root, |
| 7731 | + }) |
| 7732 | + } |
| 7733 | + }; |
| 7734 | + let obj_snapshot = match object_snapshot(&obj_path, tools) { |
| 7735 | + Ok(snapshot) => snapshot, |
| 7736 | + Err(detail) => { |
| 7737 | + return Some(ConsistencyIssue { |
| 7738 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7739 | + summary: "could not snapshot object from armfortas -c".into(), |
| 7740 | + repeat_count: None, |
| 7741 | + unique_variant_count: None, |
| 7742 | + varying_components: Vec::new(), |
| 7743 | + stable_components: Vec::new(), |
| 7744 | + detail: format!("{}\n{}", obj_command, detail), |
| 7745 | + temp_root, |
| 7746 | + }) |
| 7747 | + } |
| 7748 | + }; |
| 7749 | + |
| 7750 | + if asm_snapshot != obj_snapshot { |
| 7751 | + let snapshots = [&asm_snapshot, &obj_snapshot]; |
| 7752 | + let varying = varying_object_components(&snapshots) |
| 7753 | + .into_iter() |
| 7754 | + .map(str::to_string) |
| 7755 | + .collect::<Vec<_>>(); |
| 7756 | + let stable = stable_object_components(&snapshots) |
| 7757 | + .into_iter() |
| 7758 | + .map(str::to_string) |
| 7759 | + .collect::<Vec<_>>(); |
| 7760 | + return Some(ConsistencyIssue { |
| 7761 | + check: ConsistencyCheck::CliObjVsSystemAs, |
| 7762 | + summary: format!( |
| 7763 | + "varying_components={} stable_components={}", |
| 7764 | + join_or_none_from_strings(&varying), |
| 7765 | + join_or_none_from_strings(&stable) |
| 7766 | + ), |
| 7767 | + repeat_count: None, |
| 7768 | + unique_variant_count: None, |
| 7769 | + varying_components: varying, |
| 7770 | + stable_components: stable, |
| 7771 | + detail: format!( |
| 7772 | + "object snapshot mismatch between armfortas -S | as and armfortas -c\n{}\n{}\n{}\n{}", |
| 7773 | + asm_command, |
| 7774 | + as_command, |
| 7775 | + obj_command, |
| 7776 | + describe_object_difference(&asm_snapshot, &obj_snapshot, "-S | as", "-c") |
| 7777 | + ), |
| 7778 | + temp_root, |
| 7779 | + }); |
| 7780 | + } |
| 7781 | + |
| 7782 | + let _ = fs::remove_dir_all(&temp_root); |
| 7783 | + None |
| 7784 | +} |
| 7785 | + |
| 7786 | +fn run_cli_asm_reproducible( |
| 7787 | + source: &Path, |
| 7788 | + opt_level: OptLevel, |
| 7789 | + repeat_count: usize, |
| 7790 | + tools: &ToolchainConfig, |
| 7791 | +) -> Option<ConsistencyIssue> { |
| 7792 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7793 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7794 | + return Some(ConsistencyIssue { |
| 7795 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7796 | + summary: "could not create consistency temp dir".into(), |
| 7797 | + repeat_count: None, |
| 7798 | + unique_variant_count: None, |
| 7799 | + varying_components: Vec::new(), |
| 7800 | + stable_components: Vec::new(), |
| 7801 | + detail: format!( |
| 7802 | + "cannot create consistency temp dir '{}': {}", |
| 7803 | + temp_root.display(), |
| 7804 | + err |
| 7805 | + ), |
| 7806 | + temp_root, |
| 7807 | + }); |
| 7808 | + } |
| 7809 | + |
| 7810 | + let mut runs = Vec::new(); |
| 7811 | + for index in 0..repeat_count { |
| 7812 | + let asm_path = temp_root.join(format!("run_{:02}.s", index)); |
| 7813 | + let command = |
| 7814 | + match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 7815 | + Ok(command) => command, |
| 7816 | + Err(detail) => { |
| 7817 | + return Some(ConsistencyIssue { |
| 7818 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7819 | + summary: "armfortas -S failed during reproducibility check".into(), |
| 7820 | + repeat_count: None, |
| 7821 | + unique_variant_count: None, |
| 7822 | + varying_components: Vec::new(), |
| 7823 | + stable_components: Vec::new(), |
| 7824 | + detail, |
| 7825 | + temp_root, |
| 7826 | + }) |
| 7827 | + } |
| 7828 | + }; |
| 7829 | + let text = match read_text_artifact(&asm_path) { |
| 7830 | + Ok(text) => text, |
| 7831 | + Err(detail) => { |
| 7832 | + return Some(ConsistencyIssue { |
| 7833 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7834 | + summary: "could not read emitted assembly during reproducibility check".into(), |
| 7835 | + repeat_count: None, |
| 7836 | + unique_variant_count: None, |
| 7837 | + varying_components: Vec::new(), |
| 7838 | + stable_components: Vec::new(), |
| 7839 | + detail, |
| 7840 | + temp_root, |
| 7841 | + }) |
| 7842 | + } |
| 7843 | + }; |
| 7844 | + runs.push(TextRun { |
| 7845 | + label: format!("run {} (-S)", index + 1), |
| 7846 | + command, |
| 7847 | + normalized: normalize_text_artifact(&text), |
| 7848 | + }); |
| 7849 | + } |
| 7850 | + |
| 7851 | + let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 7852 | + if unique_variants > 1 { |
| 7853 | + let (left, right) = |
| 7854 | + first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7855 | + return Some(ConsistencyIssue { |
| 7856 | + check: ConsistencyCheck::CliAsmReproducible, |
| 7857 | + summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 7858 | + repeat_count: Some(repeat_count), |
| 7859 | + unique_variant_count: Some(unique_variants), |
| 7860 | + varying_components: Vec::new(), |
| 7861 | + stable_components: Vec::new(), |
| 7862 | + detail: format!( |
| 7863 | + "assembly output is not reproducible across repeated armfortas -S runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 7864 | + repeat_count, |
| 7865 | + unique_variants, |
| 7866 | + left.command, |
| 7867 | + right.command, |
| 7868 | + describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 7869 | + ), |
| 7870 | + temp_root, |
| 7871 | + }); |
| 7872 | + } |
| 7873 | + |
| 7874 | + let _ = fs::remove_dir_all(&temp_root); |
| 7875 | + None |
| 7876 | +} |
| 7877 | + |
| 7878 | +fn run_cli_obj_reproducible( |
| 7879 | + source: &Path, |
| 7880 | + opt_level: OptLevel, |
| 7881 | + repeat_count: usize, |
| 7882 | + tools: &ToolchainConfig, |
| 7883 | +) -> Option<ConsistencyIssue> { |
| 7884 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7885 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7886 | + return Some(ConsistencyIssue { |
| 7887 | + check: ConsistencyCheck::CliObjReproducible, |
| 7888 | + summary: "could not create consistency temp dir".into(), |
| 7889 | + repeat_count: None, |
| 7890 | + unique_variant_count: None, |
| 7891 | + varying_components: Vec::new(), |
| 7892 | + stable_components: Vec::new(), |
| 7893 | + detail: format!( |
| 7894 | + "cannot create consistency temp dir '{}': {}", |
| 7895 | + temp_root.display(), |
| 7896 | + err |
| 7897 | + ), |
| 7898 | + temp_root, |
| 7899 | + }); |
| 7900 | + } |
| 7901 | + |
| 7902 | + let mut runs = Vec::new(); |
| 7903 | + for index in 0..repeat_count { |
| 7904 | + let obj_path = temp_root.join(format!("run_{:02}.o", index)); |
| 7905 | + let command = |
| 7906 | + match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 7907 | + Ok(command) => command, |
| 7908 | + Err(detail) => { |
| 7909 | + return Some(ConsistencyIssue { |
| 7910 | + check: ConsistencyCheck::CliObjReproducible, |
| 7911 | + summary: "armfortas -c failed during reproducibility check".into(), |
| 7912 | + repeat_count: None, |
| 7913 | + unique_variant_count: None, |
| 7914 | + varying_components: Vec::new(), |
| 7915 | + stable_components: Vec::new(), |
| 7916 | + detail, |
| 7917 | + temp_root, |
| 7918 | + }) |
| 7919 | + } |
| 7920 | + }; |
| 7921 | + let snapshot = match object_snapshot(&obj_path, tools) { |
| 7922 | + Ok(snapshot) => snapshot, |
| 7923 | + Err(detail) => { |
| 7924 | + return Some(ConsistencyIssue { |
| 7925 | + check: ConsistencyCheck::CliObjReproducible, |
| 7926 | + summary: "could not snapshot object during reproducibility check".into(), |
| 7927 | + repeat_count: None, |
| 7928 | + unique_variant_count: None, |
| 7929 | + varying_components: Vec::new(), |
| 7930 | + stable_components: Vec::new(), |
| 7931 | + detail: format!("{}\n{}", command, detail), |
| 7932 | + temp_root, |
| 7933 | + }) |
| 7934 | + } |
| 7935 | + }; |
| 7936 | + runs.push(ObjectRun { |
| 7937 | + label: format!("run {} (-c)", index + 1), |
| 7938 | + command, |
| 7939 | + snapshot, |
| 7940 | + }); |
| 7941 | + } |
| 7942 | + |
| 7943 | + let rendered = runs |
| 7944 | + .iter() |
| 7945 | + .map(|run| render_object_snapshot(&run.snapshot)) |
| 7946 | + .collect::<Vec<_>>(); |
| 7947 | + let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 7948 | + if unique_variants > 1 { |
| 7949 | + let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 7950 | + let (left, right) = |
| 7951 | + first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 7952 | + let varying = join_or_none(&varying_object_components(&snapshots)); |
| 7953 | + let stable = join_or_none(&stable_object_components(&snapshots)); |
| 7954 | + let varying_components = varying_object_components(&snapshots) |
| 7955 | + .into_iter() |
| 7956 | + .map(str::to_string) |
| 7957 | + .collect::<Vec<_>>(); |
| 7958 | + let stable_components = stable_object_components(&snapshots) |
| 7959 | + .into_iter() |
| 7960 | + .map(str::to_string) |
| 7961 | + .collect::<Vec<_>>(); |
| 7962 | + return Some(ConsistencyIssue { |
| 7963 | + check: ConsistencyCheck::CliObjReproducible, |
| 7964 | + summary: format!( |
| 7965 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 7966 | + repeat_count, unique_variants, varying, stable |
| 7967 | + ), |
| 7968 | + repeat_count: Some(repeat_count), |
| 7969 | + unique_variant_count: Some(unique_variants), |
| 7970 | + varying_components, |
| 7971 | + stable_components, |
| 7972 | + detail: format!( |
| 7973 | + "object output is not reproducible across repeated armfortas -c runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 7974 | + repeat_count, |
| 7975 | + unique_variants, |
| 7976 | + varying, |
| 7977 | + stable, |
| 7978 | + left.command, |
| 7979 | + right.command, |
| 7980 | + describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 7981 | + ), |
| 7982 | + temp_root, |
| 7983 | + }); |
| 7984 | + } |
| 7985 | + |
| 7986 | + let _ = fs::remove_dir_all(&temp_root); |
| 7987 | + None |
| 7988 | +} |
| 7989 | + |
| 7990 | +fn run_cli_run_reproducible( |
| 7991 | + source: &Path, |
| 7992 | + opt_level: OptLevel, |
| 7993 | + repeat_count: usize, |
| 7994 | + tools: &ToolchainConfig, |
| 7995 | +) -> Option<ConsistencyIssue> { |
| 7996 | + let temp_root = next_consistency_temp_root(opt_level); |
| 7997 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 7998 | + return Some(ConsistencyIssue { |
| 7999 | + check: ConsistencyCheck::CliRunReproducible, |
| 8000 | + summary: "could not create consistency temp dir".into(), |
| 8001 | + repeat_count: None, |
| 8002 | + unique_variant_count: None, |
| 8003 | + varying_components: Vec::new(), |
| 8004 | + stable_components: Vec::new(), |
| 8005 | + detail: format!( |
| 8006 | + "cannot create consistency temp dir '{}': {}", |
| 8007 | + temp_root.display(), |
| 8008 | + err |
| 8009 | + ), |
| 8010 | + temp_root, |
| 8011 | + }); |
| 8012 | + } |
| 8013 | + |
| 8014 | + let mut runs = Vec::new(); |
| 8015 | + for index in 0..repeat_count { |
| 8016 | + let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 8017 | + let build_command = match compile_with_driver( |
| 8018 | + source, |
| 8019 | + opt_level, |
| 8020 | + DriverEmitMode::Binary, |
| 8021 | + &binary_path, |
| 8022 | + tools, |
| 8023 | + ) { |
| 8024 | + Ok(command) => command, |
| 8025 | + Err(detail) => { |
| 8026 | + return Some(ConsistencyIssue { |
| 8027 | + check: ConsistencyCheck::CliRunReproducible, |
| 8028 | + summary: "armfortas binary build failed during runtime reproducibility check" |
| 8029 | + .into(), |
| 8030 | + repeat_count: None, |
| 8031 | + unique_variant_count: None, |
| 8032 | + varying_components: Vec::new(), |
| 8033 | + stable_components: Vec::new(), |
| 8034 | + detail, |
| 8035 | + temp_root, |
| 8036 | + }) |
| 8037 | + } |
| 8038 | + }; |
| 8039 | + let run_command = render_binary_run_command(&binary_path); |
| 8040 | + let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 8041 | + Ok(run) => run, |
| 8042 | + Err(detail) => { |
| 8043 | + return Some(ConsistencyIssue { |
| 8044 | + check: ConsistencyCheck::CliRunReproducible, |
| 8045 | + summary: "armfortas binary could not run during runtime reproducibility check" |
| 8046 | + .into(), |
| 8047 | + repeat_count: None, |
| 8048 | + unique_variant_count: None, |
| 8049 | + varying_components: Vec::new(), |
| 8050 | + stable_components: Vec::new(), |
| 8051 | + detail, |
| 8052 | + temp_root, |
| 8053 | + }) |
| 8054 | + } |
| 8055 | + }; |
| 8056 | + let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 8057 | + if let Err(err) = write_behavior_run_artifacts( |
| 8058 | + &temp_root, |
| 8059 | + &format!("cli_run_{:02}", index), |
| 8060 | + &command, |
| 8061 | + &run, |
| 8062 | + ) { |
| 8063 | + return Some(ConsistencyIssue { |
| 8064 | + check: ConsistencyCheck::CliRunReproducible, |
| 8065 | + summary: "could not write cli runtime artifact".into(), |
| 8066 | + repeat_count: None, |
| 8067 | + unique_variant_count: None, |
| 8068 | + varying_components: Vec::new(), |
| 8069 | + stable_components: Vec::new(), |
| 8070 | + detail: format!("cannot write cli runtime artifact: {}", err), |
| 8071 | + temp_root, |
| 8072 | + }); |
| 8073 | + } |
| 8074 | + runs.push(BehaviorRun { |
| 8075 | + label: format!("cli run {}", index + 1), |
| 8076 | + command, |
| 8077 | + signature: normalize_run_signature(&run), |
| 8078 | + run, |
| 8079 | + }); |
| 8080 | + } |
| 8081 | + |
| 8082 | + let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 8083 | + if unique_variants > 1 { |
| 8084 | + let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 8085 | + let varying = varying_run_components(&signatures); |
| 8086 | + let stable = stable_run_components(&signatures); |
| 8087 | + let (left, right) = first_distinct_behavior_pair(&runs) |
| 8088 | + .expect("unique variants > 1 implies a distinct pair"); |
| 8089 | + return Some(ConsistencyIssue { |
| 8090 | + check: ConsistencyCheck::CliRunReproducible, |
| 8091 | + summary: format!( |
| 8092 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8093 | + repeat_count, |
| 8094 | + unique_variants, |
| 8095 | + join_or_none(&varying), |
| 8096 | + join_or_none(&stable) |
| 8097 | + ), |
| 8098 | + repeat_count: Some(repeat_count), |
| 8099 | + unique_variant_count: Some(unique_variants), |
| 8100 | + varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8101 | + stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8102 | + detail: format!( |
| 8103 | + "armfortas runtime behavior is not reproducible across repeated full CLI builds\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8104 | + repeat_count, |
| 8105 | + unique_variants, |
| 8106 | + join_or_none(&varying), |
| 8107 | + join_or_none(&stable), |
| 8108 | + left.command, |
| 8109 | + right.command, |
| 8110 | + describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 8111 | + ), |
| 8112 | + temp_root, |
| 8113 | + }); |
| 8114 | + } |
| 8115 | + |
| 8116 | + let _ = fs::remove_dir_all(&temp_root); |
| 8117 | + None |
| 8118 | +} |
| 8119 | + |
| 8120 | +fn run_capture_asm_vs_cli_asm( |
| 8121 | + source: &Path, |
| 8122 | + opt_level: OptLevel, |
| 8123 | + repeat_count: usize, |
| 8124 | + capture_result: &CaptureResult, |
| 8125 | + tools: &ToolchainConfig, |
| 8126 | +) -> Option<ConsistencyIssue> { |
| 8127 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8128 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8129 | + return Some(ConsistencyIssue { |
| 8130 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8131 | + summary: "could not create consistency temp dir".into(), |
| 8132 | + repeat_count: None, |
| 8133 | + unique_variant_count: None, |
| 8134 | + varying_components: Vec::new(), |
| 8135 | + stable_components: Vec::new(), |
| 8136 | + detail: format!( |
| 8137 | + "cannot create consistency temp dir '{}': {}", |
| 8138 | + temp_root.display(), |
| 8139 | + err |
| 8140 | + ), |
| 8141 | + temp_root, |
| 8142 | + }); |
| 8143 | + } |
| 8144 | + |
| 8145 | + let capture_command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 8146 | + let capture_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 8147 | + Ok(text) => text, |
| 8148 | + Err(detail) => { |
| 8149 | + return Some(ConsistencyIssue { |
| 8150 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8151 | + summary: "capture result did not include assembly text".into(), |
| 8152 | + repeat_count: None, |
| 8153 | + unique_variant_count: None, |
| 8154 | + varying_components: Vec::new(), |
| 8155 | + stable_components: Vec::new(), |
| 8156 | + detail, |
| 8157 | + temp_root, |
| 8158 | + }) |
| 8159 | + } |
| 8160 | + }; |
| 8161 | + if let Err(err) = fs::write(temp_root.join("from_capture.s"), capture_text) { |
| 8162 | + return Some(ConsistencyIssue { |
| 8163 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8164 | + summary: "could not write captured assembly artifact".into(), |
| 8165 | + repeat_count: None, |
| 8166 | + unique_variant_count: None, |
| 8167 | + varying_components: Vec::new(), |
| 8168 | + stable_components: Vec::new(), |
| 8169 | + detail: format!("cannot write captured assembly artifact: {}", err), |
| 8170 | + temp_root, |
| 8171 | + }); |
| 8172 | + } |
| 8173 | + let capture_normalized = normalize_text_artifact(capture_text); |
| 8174 | + |
| 8175 | + let mut cli_runs = Vec::new(); |
| 8176 | + let mut mismatch_indices = Vec::new(); |
| 8177 | + for index in 0..repeat_count { |
| 8178 | + let asm_path = temp_root.join(format!("cli_run_{:02}.s", index)); |
| 8179 | + let command = |
| 8180 | + match compile_with_driver(source, opt_level, DriverEmitMode::Asm, &asm_path, tools) { |
| 8181 | + Ok(command) => command, |
| 8182 | + Err(detail) => { |
| 8183 | + return Some(ConsistencyIssue { |
| 8184 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8185 | + summary: "armfortas -S failed during capture-vs-cli consistency check" |
| 8186 | + .into(), |
| 8187 | + repeat_count: None, |
| 8188 | + unique_variant_count: None, |
| 8189 | + varying_components: Vec::new(), |
| 8190 | + stable_components: Vec::new(), |
| 8191 | + detail, |
| 8192 | + temp_root, |
| 8193 | + }) |
| 8194 | + } |
| 8195 | + }; |
| 8196 | + let text = match read_text_artifact(&asm_path) { |
| 8197 | + Ok(text) => text, |
| 8198 | + Err(detail) => { |
| 8199 | + return Some(ConsistencyIssue { |
| 8200 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8201 | + summary: "could not read cli assembly artifact".into(), |
| 8202 | + repeat_count: None, |
| 8203 | + unique_variant_count: None, |
| 8204 | + varying_components: Vec::new(), |
| 8205 | + stable_components: Vec::new(), |
| 8206 | + detail, |
| 8207 | + temp_root, |
| 8208 | + }) |
| 8209 | + } |
| 8210 | + }; |
| 8211 | + let normalized = normalize_text_artifact(&text); |
| 8212 | + if normalized != capture_normalized { |
| 8213 | + mismatch_indices.push(index); |
| 8214 | + } |
| 8215 | + cli_runs.push(TextRun { |
| 8216 | + label: format!("cli run {} (-S)", index + 1), |
| 8217 | + command, |
| 8218 | + normalized, |
| 8219 | + }); |
| 8220 | + } |
| 8221 | + |
| 8222 | + if !mismatch_indices.is_empty() { |
| 8223 | + let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8224 | + let unique_cli_variants = |
| 8225 | + count_unique_strings(cli_runs.iter().map(|run| run.normalized.as_str())); |
| 8226 | + let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8227 | + return Some(ConsistencyIssue { |
| 8228 | + check: ConsistencyCheck::CaptureAsmVsCliAsm, |
| 8229 | + summary: format!( |
| 8230 | + "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={}", |
| 8231 | + repeat_count, |
| 8232 | + matching_runs, |
| 8233 | + mismatch_indices.len(), |
| 8234 | + unique_cli_variants |
| 8235 | + ), |
| 8236 | + repeat_count: Some(repeat_count), |
| 8237 | + unique_variant_count: Some(unique_cli_variants), |
| 8238 | + varying_components: Vec::new(), |
| 8239 | + stable_components: Vec::new(), |
| 8240 | + detail: format!( |
| 8241 | + "captured assembly does not match repeated armfortas -S runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8242 | + repeat_count, |
| 8243 | + matching_runs, |
| 8244 | + mismatch_indices.len(), |
| 8245 | + unique_cli_variants, |
| 8246 | + capture_command, |
| 8247 | + first_mismatch.command, |
| 8248 | + describe_text_difference( |
| 8249 | + &capture_normalized, |
| 8250 | + &first_mismatch.normalized, |
| 8251 | + "capture asm", |
| 8252 | + &first_mismatch.label |
| 8253 | + ) |
| 8254 | + ), |
| 8255 | + temp_root, |
| 8256 | + }); |
| 8257 | + } |
| 8258 | + |
| 8259 | + let _ = fs::remove_dir_all(&temp_root); |
| 8260 | + None |
| 8261 | +} |
| 8262 | + |
| 8263 | +fn run_capture_obj_vs_cli_obj( |
| 8264 | + source: &Path, |
| 8265 | + opt_level: OptLevel, |
| 8266 | + repeat_count: usize, |
| 8267 | + capture_result: &CaptureResult, |
| 8268 | + tools: &ToolchainConfig, |
| 8269 | +) -> Option<ConsistencyIssue> { |
| 8270 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8271 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8272 | + return Some(ConsistencyIssue { |
| 8273 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8274 | + summary: "could not create consistency temp dir".into(), |
| 8275 | + repeat_count: None, |
| 8276 | + unique_variant_count: None, |
| 8277 | + varying_components: Vec::new(), |
| 8278 | + stable_components: Vec::new(), |
| 8279 | + detail: format!( |
| 8280 | + "cannot create consistency temp dir '{}': {}", |
| 8281 | + temp_root.display(), |
| 8282 | + err |
| 8283 | + ), |
| 8284 | + temp_root, |
| 8285 | + }); |
| 8286 | + } |
| 8287 | + |
| 8288 | + let capture_command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8289 | + let capture_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8290 | + Ok(text) => text, |
| 8291 | + Err(detail) => { |
| 8292 | + return Some(ConsistencyIssue { |
| 8293 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8294 | + summary: "capture result did not include object snapshot text".into(), |
| 8295 | + repeat_count: None, |
| 8296 | + unique_variant_count: None, |
| 8297 | + varying_components: Vec::new(), |
| 8298 | + stable_components: Vec::new(), |
| 8299 | + detail, |
| 8300 | + temp_root, |
| 8301 | + }) |
| 8302 | + } |
| 8303 | + }; |
| 8304 | + if let Err(err) = fs::write(temp_root.join("from_capture.obj.txt"), capture_text) { |
| 8305 | + return Some(ConsistencyIssue { |
| 8306 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8307 | + summary: "could not write captured object snapshot artifact".into(), |
| 8308 | + repeat_count: None, |
| 8309 | + unique_variant_count: None, |
| 8310 | + varying_components: Vec::new(), |
| 8311 | + stable_components: Vec::new(), |
| 8312 | + detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8313 | + temp_root, |
| 8314 | + }); |
| 8315 | + } |
| 8316 | + let capture_snapshot = match parse_object_snapshot_text(capture_text) { |
| 8317 | + Ok(snapshot) => snapshot, |
| 8318 | + Err(detail) => { |
| 8319 | + return Some(ConsistencyIssue { |
| 8320 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8321 | + summary: "captured object snapshot had an unexpected format".into(), |
| 8322 | + repeat_count: None, |
| 8323 | + unique_variant_count: None, |
| 8324 | + varying_components: Vec::new(), |
| 8325 | + stable_components: Vec::new(), |
| 8326 | + detail, |
| 8327 | + temp_root, |
| 8328 | + }) |
| 8329 | + } |
| 8330 | + }; |
| 8331 | + |
| 8332 | + let mut cli_runs = Vec::new(); |
| 8333 | + let mut mismatch_indices = Vec::new(); |
| 8334 | + for index in 0..repeat_count { |
| 8335 | + let obj_path = temp_root.join(format!("cli_run_{:02}.o", index)); |
| 8336 | + let command = |
| 8337 | + match compile_with_driver(source, opt_level, DriverEmitMode::Obj, &obj_path, tools) { |
| 8338 | + Ok(command) => command, |
| 8339 | + Err(detail) => { |
| 8340 | + return Some(ConsistencyIssue { |
| 8341 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8342 | + summary: "armfortas -c failed during capture-vs-cli consistency check" |
| 8343 | + .into(), |
| 8344 | + repeat_count: None, |
| 8345 | + unique_variant_count: None, |
| 8346 | + varying_components: Vec::new(), |
| 8347 | + stable_components: Vec::new(), |
| 8348 | + detail, |
| 8349 | + temp_root, |
| 8350 | + }) |
| 8351 | + } |
| 8352 | + }; |
| 8353 | + let snapshot = match object_snapshot(&obj_path, tools) { |
| 8354 | + Ok(snapshot) => snapshot, |
| 8355 | + Err(detail) => { |
| 8356 | + return Some(ConsistencyIssue { |
| 8357 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8358 | + summary: "could not snapshot cli object artifact".into(), |
| 8359 | + repeat_count: None, |
| 8360 | + unique_variant_count: None, |
| 8361 | + varying_components: Vec::new(), |
| 8362 | + stable_components: Vec::new(), |
| 8363 | + detail: format!("{}\n{}", command, detail), |
| 8364 | + temp_root, |
| 8365 | + }) |
| 8366 | + } |
| 8367 | + }; |
| 8368 | + if let Err(err) = fs::write( |
| 8369 | + temp_root.join(format!("cli_run_{:02}.obj.txt", index)), |
| 8370 | + render_object_snapshot(&snapshot), |
| 8371 | + ) { |
| 8372 | + return Some(ConsistencyIssue { |
| 8373 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8374 | + summary: "could not write cli object snapshot artifact".into(), |
| 8375 | + repeat_count: None, |
| 8376 | + unique_variant_count: None, |
| 8377 | + varying_components: Vec::new(), |
| 8378 | + stable_components: Vec::new(), |
| 8379 | + detail: format!("cannot write cli object snapshot artifact: {}", err), |
| 8380 | + temp_root, |
| 8381 | + }); |
| 8382 | + } |
| 8383 | + if snapshot != capture_snapshot { |
| 8384 | + mismatch_indices.push(index); |
| 8385 | + } |
| 8386 | + cli_runs.push(ObjectRun { |
| 8387 | + label: format!("cli run {} (-c)", index + 1), |
| 8388 | + command, |
| 8389 | + snapshot, |
| 8390 | + }); |
| 8391 | + } |
| 8392 | + |
| 8393 | + if !mismatch_indices.is_empty() { |
| 8394 | + let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8395 | + let rendered = cli_runs |
| 8396 | + .iter() |
| 8397 | + .map(|run| render_object_snapshot(&run.snapshot)) |
| 8398 | + .collect::<Vec<_>>(); |
| 8399 | + let unique_cli_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8400 | + let mismatch_snapshots = mismatch_indices |
| 8401 | + .iter() |
| 8402 | + .map(|index| &cli_runs[*index].snapshot) |
| 8403 | + .collect::<Vec<_>>(); |
| 8404 | + let mut summary_snapshots = vec![&capture_snapshot]; |
| 8405 | + summary_snapshots.extend(mismatch_snapshots.iter().copied()); |
| 8406 | + let varying = varying_object_components(&summary_snapshots) |
| 8407 | + .into_iter() |
| 8408 | + .map(str::to_string) |
| 8409 | + .collect::<Vec<_>>(); |
| 8410 | + let stable = stable_object_components(&summary_snapshots) |
| 8411 | + .into_iter() |
| 8412 | + .map(str::to_string) |
| 8413 | + .collect::<Vec<_>>(); |
| 8414 | + let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8415 | + return Some(ConsistencyIssue { |
| 8416 | + check: ConsistencyCheck::CaptureObjVsCliObj, |
| 8417 | + summary: format!( |
| 8418 | + "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8419 | + repeat_count, |
| 8420 | + matching_runs, |
| 8421 | + mismatch_indices.len(), |
| 8422 | + unique_cli_variants, |
| 8423 | + join_or_none_from_strings(&varying), |
| 8424 | + join_or_none_from_strings(&stable) |
| 8425 | + ), |
| 8426 | + repeat_count: Some(repeat_count), |
| 8427 | + unique_variant_count: Some(unique_cli_variants), |
| 8428 | + varying_components: varying, |
| 8429 | + stable_components: stable, |
| 8430 | + detail: format!( |
| 8431 | + "captured object snapshot does not match repeated armfortas -c runs\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8432 | + repeat_count, |
| 8433 | + matching_runs, |
| 8434 | + mismatch_indices.len(), |
| 8435 | + unique_cli_variants, |
| 8436 | + capture_command, |
| 8437 | + first_mismatch.command, |
| 8438 | + describe_object_difference( |
| 8439 | + &capture_snapshot, |
| 8440 | + &first_mismatch.snapshot, |
| 8441 | + "capture obj", |
| 8442 | + &first_mismatch.label |
| 8443 | + ) |
| 8444 | + ), |
| 8445 | + temp_root, |
| 8446 | + }); |
| 8447 | + } |
| 8448 | + |
| 8449 | + let _ = fs::remove_dir_all(&temp_root); |
| 8450 | + None |
| 8451 | +} |
| 8452 | + |
| 8453 | +fn run_capture_run_vs_cli_run( |
| 8454 | + source: &Path, |
| 8455 | + opt_level: OptLevel, |
| 8456 | + repeat_count: usize, |
| 8457 | + capture_result: &CaptureResult, |
| 8458 | + tools: &ToolchainConfig, |
| 8459 | +) -> Option<ConsistencyIssue> { |
| 8460 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8461 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8462 | + return Some(ConsistencyIssue { |
| 8463 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8464 | + summary: "could not create consistency temp dir".into(), |
| 8465 | + repeat_count: None, |
| 8466 | + unique_variant_count: None, |
| 8467 | + varying_components: Vec::new(), |
| 8468 | + stable_components: Vec::new(), |
| 8469 | + detail: format!( |
| 8470 | + "cannot create consistency temp dir '{}': {}", |
| 8471 | + temp_root.display(), |
| 8472 | + err |
| 8473 | + ), |
| 8474 | + temp_root, |
| 8475 | + }); |
| 8476 | + } |
| 8477 | + |
| 8478 | + let capture_command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8479 | + let capture_run = match capture_run_stage(capture_result) { |
| 8480 | + Ok(run) => run.clone(), |
| 8481 | + Err(detail) => { |
| 8482 | + return Some(ConsistencyIssue { |
| 8483 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8484 | + summary: "capture result did not include runtime behavior".into(), |
| 8485 | + repeat_count: None, |
| 8486 | + unique_variant_count: None, |
| 8487 | + varying_components: Vec::new(), |
| 8488 | + stable_components: Vec::new(), |
| 8489 | + detail, |
| 8490 | + temp_root, |
| 8491 | + }) |
| 8492 | + } |
| 8493 | + }; |
| 8494 | + if let Err(err) = |
| 8495 | + write_behavior_run_artifacts(&temp_root, "from_capture", &capture_command, &capture_run) |
| 8496 | + { |
| 8497 | + return Some(ConsistencyIssue { |
| 8498 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8499 | + summary: "could not write captured runtime artifact".into(), |
| 8500 | + repeat_count: None, |
| 8501 | + unique_variant_count: None, |
| 8502 | + varying_components: Vec::new(), |
| 8503 | + stable_components: Vec::new(), |
| 8504 | + detail: format!("cannot write captured runtime artifact: {}", err), |
| 8505 | + temp_root, |
| 8506 | + }); |
| 8507 | + } |
| 8508 | + let capture_signature = normalize_run_signature(&capture_run); |
| 8509 | + |
| 8510 | + let mut cli_runs = Vec::new(); |
| 8511 | + let mut mismatch_indices = Vec::new(); |
| 8512 | + for index in 0..repeat_count { |
| 8513 | + let binary_path = temp_root.join(format!("cli_run_{:02}.out", index)); |
| 8514 | + let build_command = match compile_with_driver( |
| 8515 | + source, |
| 8516 | + opt_level, |
| 8517 | + DriverEmitMode::Binary, |
| 8518 | + &binary_path, |
| 8519 | + tools, |
| 8520 | + ) { |
| 8521 | + Ok(command) => command, |
| 8522 | + Err(detail) => { |
| 8523 | + return Some(ConsistencyIssue { |
| 8524 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8525 | + summary: "armfortas binary build failed during capture-vs-cli runtime check" |
| 8526 | + .into(), |
| 8527 | + repeat_count: None, |
| 8528 | + unique_variant_count: None, |
| 8529 | + varying_components: Vec::new(), |
| 8530 | + stable_components: Vec::new(), |
| 8531 | + detail, |
| 8532 | + temp_root, |
| 8533 | + }) |
| 8534 | + } |
| 8535 | + }; |
| 8536 | + let run_command = render_binary_run_command(&binary_path); |
| 8537 | + let run = match run_binary_capture(&binary_path, &temp_root, &run_command) { |
| 8538 | + Ok(run) => run, |
| 8539 | + Err(detail) => { |
| 8540 | + return Some(ConsistencyIssue { |
| 8541 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8542 | + summary: "armfortas binary could not run during capture-vs-cli runtime check" |
| 8543 | + .into(), |
| 8544 | + repeat_count: None, |
| 8545 | + unique_variant_count: None, |
| 8546 | + varying_components: Vec::new(), |
| 8547 | + stable_components: Vec::new(), |
| 8548 | + detail, |
| 8549 | + temp_root, |
| 8550 | + }) |
| 8551 | + } |
| 8552 | + }; |
| 8553 | + let command = format!("build: {}\nrun: {}", build_command, run_command); |
| 8554 | + if let Err(err) = write_behavior_run_artifacts( |
| 8555 | + &temp_root, |
| 8556 | + &format!("cli_run_{:02}", index), |
| 8557 | + &command, |
| 8558 | + &run, |
| 8559 | + ) { |
| 8560 | + return Some(ConsistencyIssue { |
| 8561 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8562 | + summary: "could not write cli runtime artifact".into(), |
| 8563 | + repeat_count: None, |
| 8564 | + unique_variant_count: None, |
| 8565 | + varying_components: Vec::new(), |
| 8566 | + stable_components: Vec::new(), |
| 8567 | + detail: format!("cannot write cli runtime artifact: {}", err), |
| 8568 | + temp_root, |
| 8569 | + }); |
| 8570 | + } |
| 8571 | + if normalize_run_signature(&run) != capture_signature { |
| 8572 | + mismatch_indices.push(index); |
| 8573 | + } |
| 8574 | + cli_runs.push(BehaviorRun { |
| 8575 | + label: format!("cli run {}", index + 1), |
| 8576 | + command, |
| 8577 | + signature: normalize_run_signature(&run), |
| 8578 | + run, |
| 8579 | + }); |
| 8580 | + } |
| 8581 | + |
| 8582 | + if !mismatch_indices.is_empty() { |
| 8583 | + let matching_runs = repeat_count.saturating_sub(mismatch_indices.len()); |
| 8584 | + let unique_cli_variants = |
| 8585 | + count_unique_run_signatures(cli_runs.iter().map(|run| &run.signature)); |
| 8586 | + let mismatch_signatures = mismatch_indices |
| 8587 | + .iter() |
| 8588 | + .map(|index| &cli_runs[*index].signature) |
| 8589 | + .collect::<Vec<_>>(); |
| 8590 | + let mut summary_signatures = vec![&capture_signature]; |
| 8591 | + summary_signatures.extend(mismatch_signatures.iter().copied()); |
| 8592 | + let varying = varying_run_components(&summary_signatures) |
| 8593 | + .into_iter() |
| 8594 | + .map(str::to_string) |
| 8595 | + .collect::<Vec<_>>(); |
| 8596 | + let stable = stable_run_components(&summary_signatures) |
| 8597 | + .into_iter() |
| 8598 | + .map(str::to_string) |
| 8599 | + .collect::<Vec<_>>(); |
| 8600 | + let first_mismatch = &cli_runs[mismatch_indices[0]]; |
| 8601 | + return Some(ConsistencyIssue { |
| 8602 | + check: ConsistencyCheck::CaptureRunVsCliRun, |
| 8603 | + summary: format!( |
| 8604 | + "repeat_count={} matching_runs={} mismatching_runs={} unique_cli_variants={} varying_components={} stable_components={}", |
| 8605 | + repeat_count, |
| 8606 | + matching_runs, |
| 8607 | + mismatch_indices.len(), |
| 8608 | + unique_cli_variants, |
| 8609 | + join_or_none_from_strings(&varying), |
| 8610 | + join_or_none_from_strings(&stable) |
| 8611 | + ), |
| 8612 | + repeat_count: Some(repeat_count), |
| 8613 | + unique_variant_count: Some(unique_cli_variants), |
| 8614 | + varying_components: varying, |
| 8615 | + stable_components: stable, |
| 8616 | + detail: format!( |
| 8617 | + "captured runtime behavior does not match repeated full CLI builds\nrepeat count: {}\nmatching runs: {}\nmismatching runs: {}\nunique cli variants: {}\n{}\n{}\n{}", |
| 8618 | + repeat_count, |
| 8619 | + matching_runs, |
| 8620 | + mismatch_indices.len(), |
| 8621 | + unique_cli_variants, |
| 8622 | + capture_command, |
| 8623 | + first_mismatch.command, |
| 8624 | + describe_run_difference( |
| 8625 | + &capture_run, |
| 8626 | + &first_mismatch.run, |
| 8627 | + "capture run", |
| 8628 | + &first_mismatch.label |
| 8629 | + ) |
| 8630 | + ), |
| 8631 | + temp_root, |
| 8632 | + }); |
| 8633 | + } |
| 8634 | + |
| 8635 | + let _ = fs::remove_dir_all(&temp_root); |
| 8636 | + None |
| 8637 | +} |
| 8638 | + |
| 8639 | +fn run_capture_asm_reproducible( |
| 8640 | + source: &Path, |
| 8641 | + opt_level: OptLevel, |
| 8642 | + repeat_count: usize, |
| 8643 | + capture_result: &CaptureResult, |
| 8644 | + tools: &ToolchainConfig, |
| 8645 | +) -> Option<ConsistencyIssue> { |
| 8646 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8647 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8648 | + return Some(ConsistencyIssue { |
| 8649 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8650 | + summary: "could not create consistency temp dir".into(), |
| 8651 | + repeat_count: None, |
| 8652 | + unique_variant_count: None, |
| 8653 | + varying_components: Vec::new(), |
| 8654 | + stable_components: Vec::new(), |
| 8655 | + detail: format!( |
| 8656 | + "cannot create consistency temp dir '{}': {}", |
| 8657 | + temp_root.display(), |
| 8658 | + err |
| 8659 | + ), |
| 8660 | + temp_root, |
| 8661 | + }); |
| 8662 | + } |
| 8663 | + |
| 8664 | + let mut runs = Vec::new(); |
| 8665 | + let command = render_capture_command(source, opt_level, Stage::Asm, tools); |
| 8666 | + let initial_text = match capture_text_stage(capture_result, Stage::Asm) { |
| 8667 | + Ok(text) => text, |
| 8668 | + Err(detail) => { |
| 8669 | + return Some(ConsistencyIssue { |
| 8670 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8671 | + summary: "initial capture result did not include assembly text".into(), |
| 8672 | + repeat_count: None, |
| 8673 | + unique_variant_count: None, |
| 8674 | + varying_components: Vec::new(), |
| 8675 | + stable_components: Vec::new(), |
| 8676 | + detail, |
| 8677 | + temp_root, |
| 8678 | + }) |
| 8679 | + } |
| 8680 | + }; |
| 8681 | + if let Err(err) = fs::write(temp_root.join("capture_run_00.s"), initial_text) { |
| 8682 | + return Some(ConsistencyIssue { |
| 8683 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8684 | + summary: "could not write captured assembly artifact".into(), |
| 8685 | + repeat_count: None, |
| 8686 | + unique_variant_count: None, |
| 8687 | + varying_components: Vec::new(), |
| 8688 | + stable_components: Vec::new(), |
| 8689 | + detail: format!("cannot write captured assembly artifact: {}", err), |
| 8690 | + temp_root, |
| 8691 | + }); |
| 8692 | + } |
| 8693 | + runs.push(TextRun { |
| 8694 | + label: "capture run 1".into(), |
| 8695 | + command: command.clone(), |
| 8696 | + normalized: normalize_text_artifact(initial_text), |
| 8697 | + }); |
| 8698 | + |
| 8699 | + for index in 1..repeat_count { |
| 8700 | + let text = match capture_text_from_testing(source, opt_level, Stage::Asm, tools) { |
| 8701 | + Ok(text) => text, |
| 8702 | + Err(detail) => { |
| 8703 | + return Some(ConsistencyIssue { |
| 8704 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8705 | + summary: "armfortas::testing capture failed during asm reproducibility check" |
| 8706 | + .into(), |
| 8707 | + repeat_count: None, |
| 8708 | + unique_variant_count: None, |
| 8709 | + varying_components: Vec::new(), |
| 8710 | + stable_components: Vec::new(), |
| 8711 | + detail, |
| 8712 | + temp_root, |
| 8713 | + }) |
| 8714 | + } |
| 8715 | + }; |
| 8716 | + if let Err(err) = fs::write(temp_root.join(format!("capture_run_{:02}.s", index)), &text) { |
| 8717 | + return Some(ConsistencyIssue { |
| 8718 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8719 | + summary: "could not write captured assembly artifact".into(), |
| 8720 | + repeat_count: None, |
| 8721 | + unique_variant_count: None, |
| 8722 | + varying_components: Vec::new(), |
| 8723 | + stable_components: Vec::new(), |
| 8724 | + detail: format!("cannot write captured assembly artifact: {}", err), |
| 8725 | + temp_root, |
| 8726 | + }); |
| 8727 | + } |
| 8728 | + runs.push(TextRun { |
| 8729 | + label: format!("capture run {}", index + 1), |
| 8730 | + command: command.clone(), |
| 8731 | + normalized: normalize_text_artifact(&text), |
| 8732 | + }); |
| 8733 | + } |
| 8734 | + |
| 8735 | + let unique_variants = count_unique_strings(runs.iter().map(|run| run.normalized.as_str())); |
| 8736 | + if unique_variants > 1 { |
| 8737 | + let (left, right) = |
| 8738 | + first_distinct_text_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8739 | + return Some(ConsistencyIssue { |
| 8740 | + check: ConsistencyCheck::CaptureAsmReproducible, |
| 8741 | + summary: format!("repeat_count={} unique_variants={}", repeat_count, unique_variants), |
| 8742 | + repeat_count: Some(repeat_count), |
| 8743 | + unique_variant_count: Some(unique_variants), |
| 8744 | + varying_components: Vec::new(), |
| 8745 | + stable_components: Vec::new(), |
| 8746 | + detail: format!( |
| 8747 | + "captured assembly is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\n{}\n{}\n{}", |
| 8748 | + repeat_count, |
| 8749 | + unique_variants, |
| 8750 | + left.command, |
| 8751 | + right.command, |
| 8752 | + describe_text_difference(&left.normalized, &right.normalized, &left.label, &right.label) |
| 8753 | + ), |
| 8754 | + temp_root, |
| 8755 | + }); |
| 8756 | + } |
| 8757 | + |
| 8758 | + let _ = fs::remove_dir_all(&temp_root); |
| 8759 | + None |
| 8760 | +} |
| 8761 | + |
| 8762 | +fn run_capture_obj_reproducible( |
| 8763 | + source: &Path, |
| 8764 | + opt_level: OptLevel, |
| 8765 | + repeat_count: usize, |
| 8766 | + capture_result: &CaptureResult, |
| 8767 | + tools: &ToolchainConfig, |
| 8768 | +) -> Option<ConsistencyIssue> { |
| 8769 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8770 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8771 | + return Some(ConsistencyIssue { |
| 8772 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8773 | + summary: "could not create consistency temp dir".into(), |
| 8774 | + repeat_count: None, |
| 8775 | + unique_variant_count: None, |
| 8776 | + varying_components: Vec::new(), |
| 8777 | + stable_components: Vec::new(), |
| 8778 | + detail: format!( |
| 8779 | + "cannot create consistency temp dir '{}': {}", |
| 8780 | + temp_root.display(), |
| 8781 | + err |
| 8782 | + ), |
| 8783 | + temp_root, |
| 8784 | + }); |
| 8785 | + } |
| 8786 | + |
| 8787 | + let command = render_capture_command(source, opt_level, Stage::Obj, tools); |
| 8788 | + let initial_text = match capture_text_stage(capture_result, Stage::Obj) { |
| 8789 | + Ok(text) => text, |
| 8790 | + Err(detail) => { |
| 8791 | + return Some(ConsistencyIssue { |
| 8792 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8793 | + summary: "initial capture result did not include object snapshot text".into(), |
| 8794 | + repeat_count: None, |
| 8795 | + unique_variant_count: None, |
| 8796 | + varying_components: Vec::new(), |
| 8797 | + stable_components: Vec::new(), |
| 8798 | + detail, |
| 8799 | + temp_root, |
| 8800 | + }) |
| 8801 | + } |
| 8802 | + }; |
| 8803 | + let initial_snapshot = match parse_object_snapshot_text(initial_text) { |
| 8804 | + Ok(snapshot) => snapshot, |
| 8805 | + Err(detail) => { |
| 8806 | + return Some(ConsistencyIssue { |
| 8807 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8808 | + summary: "captured object snapshot had an unexpected format".into(), |
| 8809 | + repeat_count: None, |
| 8810 | + unique_variant_count: None, |
| 8811 | + varying_components: Vec::new(), |
| 8812 | + stable_components: Vec::new(), |
| 8813 | + detail, |
| 8814 | + temp_root, |
| 8815 | + }) |
| 8816 | + } |
| 8817 | + }; |
| 8818 | + if let Err(err) = fs::write(temp_root.join("capture_run_00.obj.txt"), initial_text) { |
| 8819 | + return Some(ConsistencyIssue { |
| 8820 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8821 | + summary: "could not write captured object snapshot artifact".into(), |
| 8822 | + repeat_count: None, |
| 8823 | + unique_variant_count: None, |
| 8824 | + varying_components: Vec::new(), |
| 8825 | + stable_components: Vec::new(), |
| 8826 | + detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8827 | + temp_root, |
| 8828 | + }); |
| 8829 | + } |
| 8830 | + |
| 8831 | + let mut runs = vec![ObjectRun { |
| 8832 | + label: "capture run 1".into(), |
| 8833 | + command: command.clone(), |
| 8834 | + snapshot: initial_snapshot, |
| 8835 | + }]; |
| 8836 | + |
| 8837 | + for index in 1..repeat_count { |
| 8838 | + let text = match capture_text_from_testing(source, opt_level, Stage::Obj, tools) { |
| 8839 | + Ok(text) => text, |
| 8840 | + Err(detail) => { |
| 8841 | + return Some(ConsistencyIssue { |
| 8842 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8843 | + summary: "armfortas::testing capture failed during obj reproducibility check" |
| 8844 | + .into(), |
| 8845 | + repeat_count: None, |
| 8846 | + unique_variant_count: None, |
| 8847 | + varying_components: Vec::new(), |
| 8848 | + stable_components: Vec::new(), |
| 8849 | + detail, |
| 8850 | + temp_root, |
| 8851 | + }) |
| 8852 | + } |
| 8853 | + }; |
| 8854 | + if let Err(err) = fs::write( |
| 8855 | + temp_root.join(format!("capture_run_{:02}.obj.txt", index)), |
| 8856 | + &text, |
| 8857 | + ) { |
| 8858 | + return Some(ConsistencyIssue { |
| 8859 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8860 | + summary: "could not write captured object snapshot artifact".into(), |
| 8861 | + repeat_count: None, |
| 8862 | + unique_variant_count: None, |
| 8863 | + varying_components: Vec::new(), |
| 8864 | + stable_components: Vec::new(), |
| 8865 | + detail: format!("cannot write captured object snapshot artifact: {}", err), |
| 8866 | + temp_root, |
| 8867 | + }); |
| 8868 | + } |
| 8869 | + let snapshot = match parse_object_snapshot_text(&text) { |
| 8870 | + Ok(snapshot) => snapshot, |
| 8871 | + Err(detail) => { |
| 8872 | + return Some(ConsistencyIssue { |
| 8873 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8874 | + summary: "captured object snapshot had an unexpected format".into(), |
| 8875 | + repeat_count: None, |
| 8876 | + unique_variant_count: None, |
| 8877 | + varying_components: Vec::new(), |
| 8878 | + stable_components: Vec::new(), |
| 8879 | + detail, |
| 8880 | + temp_root, |
| 8881 | + }) |
| 8882 | + } |
| 8883 | + }; |
| 8884 | + runs.push(ObjectRun { |
| 8885 | + label: format!("capture run {}", index + 1), |
| 8886 | + command: command.clone(), |
| 8887 | + snapshot, |
| 8888 | + }); |
| 8889 | + } |
| 8890 | + |
| 8891 | + let rendered = runs |
| 8892 | + .iter() |
| 8893 | + .map(|run| render_object_snapshot(&run.snapshot)) |
| 8894 | + .collect::<Vec<_>>(); |
| 8895 | + let unique_variants = count_unique_strings(rendered.iter().map(String::as_str)); |
| 8896 | + if unique_variants > 1 { |
| 8897 | + let snapshots = runs.iter().map(|run| &run.snapshot).collect::<Vec<_>>(); |
| 8898 | + let (left, right) = |
| 8899 | + first_distinct_object_pair(&runs).expect("unique variants > 1 implies a distinct pair"); |
| 8900 | + let varying = varying_object_components(&snapshots); |
| 8901 | + let stable = stable_object_components(&snapshots); |
| 8902 | + return Some(ConsistencyIssue { |
| 8903 | + check: ConsistencyCheck::CaptureObjReproducible, |
| 8904 | + summary: format!( |
| 8905 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 8906 | + repeat_count, |
| 8907 | + unique_variants, |
| 8908 | + join_or_none(&varying), |
| 8909 | + join_or_none(&stable) |
| 8910 | + ), |
| 8911 | + repeat_count: Some(repeat_count), |
| 8912 | + unique_variant_count: Some(unique_variants), |
| 8913 | + varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 8914 | + stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 8915 | + detail: format!( |
| 8916 | + "captured object snapshots are not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 8917 | + repeat_count, |
| 8918 | + unique_variants, |
| 8919 | + join_or_none(&varying), |
| 8920 | + join_or_none(&stable), |
| 8921 | + left.command, |
| 8922 | + right.command, |
| 8923 | + describe_object_difference(&left.snapshot, &right.snapshot, &left.label, &right.label) |
| 8924 | + ), |
| 8925 | + temp_root, |
| 8926 | + }); |
| 8927 | + } |
| 8928 | + |
| 8929 | + let _ = fs::remove_dir_all(&temp_root); |
| 8930 | + None |
| 8931 | +} |
| 8932 | + |
| 8933 | +fn run_capture_run_reproducible( |
| 8934 | + source: &Path, |
| 8935 | + opt_level: OptLevel, |
| 8936 | + repeat_count: usize, |
| 8937 | + capture_result: &CaptureResult, |
| 8938 | + tools: &ToolchainConfig, |
| 8939 | +) -> Option<ConsistencyIssue> { |
| 8940 | + let temp_root = next_consistency_temp_root(opt_level); |
| 8941 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 8942 | + return Some(ConsistencyIssue { |
| 8943 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 8944 | + summary: "could not create consistency temp dir".into(), |
| 8945 | + repeat_count: None, |
| 8946 | + unique_variant_count: None, |
| 8947 | + varying_components: Vec::new(), |
| 8948 | + stable_components: Vec::new(), |
| 8949 | + detail: format!( |
| 8950 | + "cannot create consistency temp dir '{}': {}", |
| 8951 | + temp_root.display(), |
| 8952 | + err |
| 8953 | + ), |
| 8954 | + temp_root, |
| 8955 | + }); |
| 8956 | + } |
| 8957 | + |
| 8958 | + let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 8959 | + let initial_run = match capture_run_stage(capture_result) { |
| 8960 | + Ok(run) => run.clone(), |
| 8961 | + Err(detail) => { |
| 8962 | + return Some(ConsistencyIssue { |
| 8963 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 8964 | + summary: "initial capture result did not include runtime behavior".into(), |
| 8965 | + repeat_count: None, |
| 8966 | + unique_variant_count: None, |
| 8967 | + varying_components: Vec::new(), |
| 8968 | + stable_components: Vec::new(), |
| 8969 | + detail, |
| 8970 | + temp_root, |
| 8971 | + }) |
| 8972 | + } |
| 8973 | + }; |
| 8974 | + if let Err(err) = |
| 8975 | + write_behavior_run_artifacts(&temp_root, "capture_run_00", &command, &initial_run) |
| 8976 | + { |
| 8977 | + return Some(ConsistencyIssue { |
| 8978 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 8979 | + summary: "could not write captured runtime artifact".into(), |
| 8980 | + repeat_count: None, |
| 8981 | + unique_variant_count: None, |
| 8982 | + varying_components: Vec::new(), |
| 8983 | + stable_components: Vec::new(), |
| 8984 | + detail: format!("cannot write captured runtime artifact: {}", err), |
| 8985 | + temp_root, |
| 8986 | + }); |
| 8987 | + } |
| 8988 | + let mut runs = vec![BehaviorRun { |
| 8989 | + label: "capture run 1".into(), |
| 8990 | + command: command.clone(), |
| 8991 | + signature: normalize_run_signature(&initial_run), |
| 8992 | + run: initial_run, |
| 8993 | + }]; |
| 8994 | + |
| 8995 | + for index in 1..repeat_count { |
| 8996 | + let run = match capture_run_from_testing(source, opt_level, tools) { |
| 8997 | + Ok(run) => run, |
| 8998 | + Err(detail) => { |
| 8999 | + return Some(ConsistencyIssue { |
| 9000 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 9001 | + summary: |
| 9002 | + "armfortas::testing capture failed during runtime reproducibility check" |
| 9003 | + .into(), |
| 9004 | + repeat_count: None, |
| 9005 | + unique_variant_count: None, |
| 9006 | + varying_components: Vec::new(), |
| 9007 | + stable_components: Vec::new(), |
| 9008 | + detail, |
| 9009 | + temp_root, |
| 9010 | + }) |
| 9011 | + } |
| 9012 | + }; |
| 9013 | + if let Err(err) = write_behavior_run_artifacts( |
| 9014 | + &temp_root, |
| 9015 | + &format!("capture_run_{:02}", index), |
| 9016 | + &command, |
| 9017 | + &run, |
| 9018 | + ) { |
| 9019 | + return Some(ConsistencyIssue { |
| 9020 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 9021 | + summary: "could not write captured runtime artifact".into(), |
| 9022 | + repeat_count: None, |
| 9023 | + unique_variant_count: None, |
| 9024 | + varying_components: Vec::new(), |
| 9025 | + stable_components: Vec::new(), |
| 9026 | + detail: format!("cannot write captured runtime artifact: {}", err), |
| 9027 | + temp_root, |
| 9028 | + }); |
| 9029 | + } |
| 9030 | + runs.push(BehaviorRun { |
| 9031 | + label: format!("capture run {}", index + 1), |
| 9032 | + command: command.clone(), |
| 9033 | + signature: normalize_run_signature(&run), |
| 9034 | + run, |
| 9035 | + }); |
| 9036 | + } |
| 9037 | + |
| 9038 | + let unique_variants = count_unique_run_signatures(runs.iter().map(|run| &run.signature)); |
| 9039 | + if unique_variants > 1 { |
| 9040 | + let signatures = runs.iter().map(|run| &run.signature).collect::<Vec<_>>(); |
| 9041 | + let varying = varying_run_components(&signatures); |
| 9042 | + let stable = stable_run_components(&signatures); |
| 9043 | + let (left, right) = first_distinct_behavior_pair(&runs) |
| 9044 | + .expect("unique variants > 1 implies a distinct pair"); |
| 9045 | + return Some(ConsistencyIssue { |
| 9046 | + check: ConsistencyCheck::CaptureRunReproducible, |
| 9047 | + summary: format!( |
| 9048 | + "repeat_count={} unique_variants={} varying_components={} stable_components={}", |
| 9049 | + repeat_count, |
| 9050 | + unique_variants, |
| 9051 | + join_or_none(&varying), |
| 9052 | + join_or_none(&stable) |
| 9053 | + ), |
| 9054 | + repeat_count: Some(repeat_count), |
| 9055 | + unique_variant_count: Some(unique_variants), |
| 9056 | + varying_components: varying.iter().map(|value| (*value).to_string()).collect(), |
| 9057 | + stable_components: stable.iter().map(|value| (*value).to_string()).collect(), |
| 9058 | + detail: format!( |
| 9059 | + "captured runtime behavior is not reproducible across repeated armfortas::testing runs\nrepeat count: {}\nunique variants: {}\nvarying components across repeats: {}\nstable components across repeats: {}\n{}\n{}\n{}", |
| 9060 | + repeat_count, |
| 9061 | + unique_variants, |
| 9062 | + join_or_none(&varying), |
| 9063 | + join_or_none(&stable), |
| 9064 | + left.command, |
| 9065 | + right.command, |
| 9066 | + describe_run_difference(&left.run, &right.run, &left.label, &right.label) |
| 9067 | + ), |
| 9068 | + temp_root, |
| 9069 | + }); |
| 9070 | + } |
| 9071 | + |
| 9072 | + let _ = fs::remove_dir_all(&temp_root); |
| 9073 | + None |
| 9074 | +} |
| 9075 | + |
| 9076 | +fn run_reference_compilers( |
| 9077 | + prepared: &PreparedInput, |
| 9078 | + case: &CaseSpec, |
| 9079 | + opt_level: OptLevel, |
| 9080 | + tools: &ToolchainConfig, |
| 9081 | +) -> Vec<ReferenceResult> { |
| 9082 | + case.reference_compilers |
| 9083 | + .iter() |
| 9084 | + .copied() |
| 9085 | + .map(|compiler| run_reference_case(&prepared.compiler_source, opt_level, compiler, tools)) |
| 9086 | + .collect() |
| 9087 | +} |
| 9088 | + |
| 9089 | +fn run_reference_case( |
| 9090 | + source: &Path, |
| 9091 | + opt_level: OptLevel, |
| 9092 | + compiler: ReferenceCompiler, |
| 9093 | + tools: &ToolchainConfig, |
| 9094 | +) -> ReferenceResult { |
| 9095 | + let temp_root = next_report_temp_root(compiler, opt_level); |
| 9096 | + let binary = temp_root.join("reference.out"); |
| 9097 | + let uses_cpp = source_uses_cpp(source); |
| 9098 | + |
| 9099 | + let mut args = vec![opt_level.as_flag().to_string()]; |
| 9100 | + if uses_cpp { |
| 9101 | + args.push("-cpp".to_string()); |
| 9102 | + } |
| 9103 | + args.push(source.display().to_string()); |
| 9104 | + args.push("-o".to_string()); |
| 9105 | + args.push(binary.display().to_string()); |
| 9106 | + |
| 9107 | + let compiler_bin = tools.reference_binary(compiler); |
| 9108 | + let command_string = render_command(compiler_bin, &args); |
| 9109 | + |
| 9110 | + if let Err(err) = fs::create_dir_all(&temp_root) { |
| 9111 | + return ReferenceResult::infrastructure_error( |
| 9112 | + compiler, |
| 9113 | + command_string, |
| 9114 | + format!("cannot create temp dir '{}': {}", temp_root.display(), err), |
| 9115 | + ); |
| 9116 | + } |
| 9117 | + |
| 9118 | + let compile = match Command::new(compiler_bin) |
| 9119 | + .current_dir(&temp_root) |
| 9120 | + .args(&args) |
| 9121 | + .output() |
| 9122 | + { |
| 9123 | + Ok(output) => output, |
| 9124 | + Err(err) => { |
| 9125 | + return ReferenceResult::infrastructure_error( |
| 9126 | + compiler, |
| 9127 | + command_string, |
| 9128 | + format!("cannot run {}: {}", compiler_bin, err), |
| 9129 | + ); |
| 9130 | + } |
| 9131 | + }; |
| 9132 | + |
| 9133 | + let mut result = ReferenceResult { |
| 9134 | + compiler, |
| 9135 | + compile_command: command_string, |
| 9136 | + compile_exit_code: compile.status.code().unwrap_or(-1), |
| 9137 | + compile_stdout: String::from_utf8_lossy(&compile.stdout).into_owned(), |
| 9138 | + compile_stderr: String::from_utf8_lossy(&compile.stderr).into_owned(), |
| 9139 | + run: None, |
| 9140 | + run_error: None, |
| 9141 | + }; |
| 9142 | + |
| 9143 | + if compile.status.success() { |
| 9144 | + match Command::new(&binary).current_dir(&temp_root).output() { |
| 9145 | + Ok(output) => { |
| 9146 | + result.run = Some(RunCapture { |
| 9147 | + exit_code: output.status.code().unwrap_or(-1), |
| 9148 | + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 9149 | + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 9150 | + }); |
| 9151 | + } |
| 9152 | + Err(err) => { |
| 9153 | + result.run_error = Some(format!("cannot run '{}': {}", binary.display(), err)); |
| 9154 | + } |
| 9155 | + } |
| 9156 | + } |
| 9157 | + |
| 9158 | + let _ = fs::remove_dir_all(&temp_root); |
| 9159 | + result |
| 9160 | +} |
| 9161 | + |
| 9162 | +fn source_uses_cpp(source: &Path) -> bool { |
| 9163 | + fs::read_to_string(source) |
| 9164 | + .map(|text| text.lines().any(|line| line.trim_start().starts_with('#'))) |
| 9165 | + .unwrap_or(false) |
| 9166 | +} |
| 9167 | + |
| 9168 | +#[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 9169 | +enum DriverEmitMode { |
| 9170 | + Asm, |
| 9171 | + Obj, |
| 9172 | + Binary, |
| 9173 | +} |
| 9174 | + |
| 9175 | +fn compile_with_driver( |
| 9176 | + source: &Path, |
| 9177 | + opt_level: OptLevel, |
| 9178 | + mode: DriverEmitMode, |
| 9179 | + output: &Path, |
| 9180 | + tools: &ToolchainConfig, |
| 9181 | +) -> Result<String, String> { |
| 9182 | + let command = render_armfortas_command(source, opt_level, mode, output, tools); |
| 9183 | + let emit_mode = match mode { |
| 9184 | + DriverEmitMode::Asm => EmitMode::Asm, |
| 9185 | + DriverEmitMode::Obj => EmitMode::Obj, |
| 9186 | + DriverEmitMode::Binary => EmitMode::Binary, |
| 9187 | + }; |
| 9188 | + tools |
| 9189 | + .armfortas_adapters() |
| 9190 | + .compile_output(source, opt_level, emit_mode, output) |
| 9191 | + .map_err(|detail| format!("{} failed:\n{}", command, detail))?; |
| 9192 | + Ok(command) |
| 9193 | +} |
| 9194 | + |
| 9195 | +fn render_armfortas_command( |
| 9196 | + source: &Path, |
| 9197 | + opt_level: OptLevel, |
| 9198 | + mode: DriverEmitMode, |
| 9199 | + output: &Path, |
| 9200 | + tools: &ToolchainConfig, |
| 9201 | +) -> String { |
| 9202 | + let armfortas = tools.armfortas_adapters(); |
| 9203 | + let mut args = vec![opt_level.as_flag().to_string()]; |
| 9204 | + match mode { |
| 9205 | + DriverEmitMode::Asm => args.push("-S".to_string()), |
| 9206 | + DriverEmitMode::Obj => args.push("-c".to_string()), |
| 9207 | + DriverEmitMode::Binary => {} |
| 9208 | + } |
| 9209 | + args.push(source.display().to_string()); |
| 9210 | + args.push("-o".to_string()); |
| 9211 | + args.push(output.display().to_string()); |
| 9212 | + render_command(armfortas.cli_command_name(), &args) |
| 9213 | +} |
| 9214 | + |
| 9215 | +fn render_binary_run_command(binary: &Path) -> String { |
| 9216 | + render_command(&binary.display().to_string(), &[]) |
| 9217 | +} |
| 9218 | + |
| 9219 | +fn render_capture_command( |
| 9220 | + source: &Path, |
| 9221 | + opt_level: OptLevel, |
| 9222 | + stage: Stage, |
| 9223 | + tools: &ToolchainConfig, |
| 9224 | +) -> String { |
| 9225 | + let armfortas = tools.armfortas_adapters(); |
| 9226 | + format!( |
| 9227 | + "{} {} --stage {} {}", |
| 9228 | + armfortas.capture_command_name(), |
| 9229 | + opt_level.as_flag(), |
| 9230 | + stage.as_str(), |
| 9231 | + quote_arg(&source.display().to_string()), |
| 9232 | + ) |
| 9233 | +} |
| 9234 | + |
| 9235 | +fn capture_text_from_testing( |
| 9236 | + source: &Path, |
| 9237 | + opt_level: OptLevel, |
| 9238 | + stage: Stage, |
| 9239 | + tools: &ToolchainConfig, |
| 9240 | +) -> Result<String, String> { |
| 9241 | + let command = render_capture_command(source, opt_level, stage, tools); |
| 9242 | + let request = CaptureRequest { |
| 9243 | + input: source.to_path_buf(), |
| 9244 | + requested: BTreeSet::from([stage]), |
| 9245 | + opt_level, |
| 9246 | + }; |
| 9247 | + let result = tools |
| 9248 | + .armfortas_adapters() |
| 9249 | + .capture(&request) |
| 9250 | + .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9251 | + capture_text_stage(&result, stage).map(str::to_string) |
| 9252 | +} |
| 9253 | + |
| 9254 | +fn capture_run_from_testing( |
| 9255 | + source: &Path, |
| 9256 | + opt_level: OptLevel, |
| 9257 | + tools: &ToolchainConfig, |
| 9258 | +) -> Result<RunCapture, String> { |
| 9259 | + let command = render_capture_command(source, opt_level, Stage::Run, tools); |
| 9260 | + let request = CaptureRequest { |
| 9261 | + input: source.to_path_buf(), |
| 9262 | + requested: BTreeSet::from([Stage::Run]), |
| 9263 | + opt_level, |
| 9264 | + }; |
| 9265 | + let result = tools |
| 9266 | + .armfortas_adapters() |
| 9267 | + .capture(&request) |
| 9268 | + .map_err(|failure| format!("{} failed:\n{}", command, failure))?; |
| 9269 | + capture_run_stage(&result).cloned() |
| 9270 | +} |
| 9271 | + |
| 9272 | +fn capture_text_stage<'a>(result: &'a CaptureResult, stage: Stage) -> Result<&'a str, String> { |
| 9273 | + match result.get(stage) { |
| 9274 | + Some(CapturedStage::Text(text)) => Ok(text), |
| 9275 | + Some(CapturedStage::Run(_)) => Err(format!( |
| 9276 | + "capture result contained non-text data for stage '{}'", |
| 9277 | + stage.as_str() |
| 9278 | + )), |
| 9279 | + None => Err(format!( |
| 9280 | + "capture result was missing requested stage '{}'", |
| 9281 | + stage.as_str() |
| 9282 | + )), |
| 9283 | + } |
| 9284 | +} |
| 9285 | + |
| 9286 | +fn capture_run_stage(result: &CaptureResult) -> Result<&RunCapture, String> { |
| 9287 | + match result.get(Stage::Run) { |
| 9288 | + Some(CapturedStage::Run(run)) => Ok(run), |
| 9289 | + Some(CapturedStage::Text(_)) => { |
| 9290 | + Err("capture result contained text data for the run stage".into()) |
| 9291 | + } |
| 9292 | + None => Err("capture result was missing requested stage 'run'".into()), |
| 9293 | + } |
| 9294 | +} |
| 9295 | + |
| 9296 | +fn run_binary_capture( |
| 9297 | + binary: &Path, |
| 9298 | + current_dir: &Path, |
| 9299 | + command: &str, |
| 9300 | +) -> Result<RunCapture, String> { |
| 9301 | + let output = Command::new(binary) |
| 9302 | + .current_dir(current_dir) |
| 9303 | + .output() |
| 9304 | + .map_err(|err| { |
| 9305 | + format!( |
| 9306 | + "{} failed:\ncannot run '{}': {}", |
| 9307 | + command, |
| 9308 | + binary.display(), |
| 9309 | + err |
| 9310 | + ) |
| 9311 | + })?; |
| 9312 | + Ok(RunCapture { |
| 9313 | + exit_code: output.status.code().unwrap_or(-1), |
| 9314 | + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), |
| 9315 | + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), |
| 9316 | + }) |
| 9317 | +} |
| 9318 | + |
| 9319 | +fn normalize_run_signature(run: &RunCapture) -> RunSignature { |
| 9320 | + RunSignature { |
| 9321 | + exit_code: run.exit_code, |
| 9322 | + stdout: normalize_behavior_text(&run.stdout), |
| 9323 | + stderr: normalize_behavior_text(&run.stderr), |
| 9324 | + } |
| 9325 | +} |
| 9326 | + |
| 9327 | +fn normalize_behavior_text(text: &str) -> String { |
| 9328 | + text.replace("\r\n", "\n") |
| 9329 | + .lines() |
| 9330 | + .map(normalize_behavior_line) |
| 9331 | + .collect::<Vec<_>>() |
| 9332 | + .join("\n") |
| 9333 | + .trim() |
| 9334 | + .to_string() |
| 9335 | +} |
| 9336 | + |
| 9337 | +fn normalize_behavior_line(line: &str) -> String { |
| 9338 | + line.split_whitespace() |
| 9339 | + .map(normalize_behavior_token) |
| 9340 | + .collect::<Vec<_>>() |
| 9341 | + .join(" ") |
| 9342 | +} |
| 9343 | + |
| 9344 | +fn normalize_behavior_token(token: &str) -> String { |
| 9345 | + if let Some(number) = parse_numeric_token(token) { |
| 9346 | + format!("num:{:.6e}", number) |
| 9347 | + } else { |
| 9348 | + token.to_string() |
| 9349 | + } |
| 9350 | +} |
| 9351 | + |
| 9352 | +fn parse_numeric_token(token: &str) -> Option<f64> { |
| 9353 | + if token.is_empty() { |
| 9354 | + return None; |
| 9355 | + } |
| 9356 | + |
| 9357 | + let normalized = token |
| 9358 | + .trim() |
| 9359 | + .trim_end_matches(',') |
| 9360 | + .trim_end_matches(';') |
| 9361 | + .replace('D', "E") |
| 9362 | + .replace('d', "e"); |
| 9363 | + |
| 9364 | + normalized.parse::<f64>().ok() |
| 9365 | +} |
| 9366 | + |
| 9367 | +fn format_reference_summary(references: &[ReferenceResult]) -> String { |
| 9368 | + references |
| 9369 | + .iter() |
| 9370 | + .map(format_reference_result) |
| 9371 | + .collect::<Vec<_>>() |
| 9372 | + .join("\n\n") |
| 9373 | +} |
| 9374 | + |
| 9375 | +fn format_reference_result(reference: &ReferenceResult) -> String { |
| 9376 | + let mut lines = Vec::new(); |
| 9377 | + lines.push(reference.compiler.as_str().to_string()); |
| 9378 | + lines.push(format!("command: {}", reference.compile_command)); |
| 9379 | + lines.push(format!("compile exit: {}", reference.compile_exit_code)); |
| 9380 | + if !reference.compile_stdout.trim().is_empty() { |
| 9381 | + lines.push(format!( |
| 9382 | + "compile stdout:\n{}", |
| 9383 | + reference.compile_stdout.trim_end() |
| 9384 | + )); |
| 9385 | + } |
| 9386 | + if !reference.compile_stderr.trim().is_empty() { |
| 9387 | + lines.push(format!( |
| 9388 | + "compile stderr:\n{}", |
| 9389 | + reference.compile_stderr.trim_end() |
| 9390 | + )); |
| 9391 | + } |
| 9392 | + match (&reference.run, &reference.run_error) { |
| 9393 | + (Some(run), _) => { |
| 9394 | + lines.push(format!("run\n{}", format_run_capture(run))); |
| 9395 | + } |
| 9396 | + (None, Some(err)) => { |
| 9397 | + lines.push(format!("run error: {}", err)); |
| 9398 | + } |
| 9399 | + (None, None) => {} |
| 9400 | + } |
| 9401 | + lines.join("\n") |
| 9402 | +} |
| 9403 | + |
| 9404 | +fn format_run_capture(run: &RunCapture) -> String { |
| 9405 | + let stdout = if run.stdout.is_empty() { |
| 9406 | + "<empty>".to_string() |
| 9407 | + } else { |
| 9408 | + run.stdout.trim_end().to_string() |
| 9409 | + }; |
| 9410 | + let stderr = if run.stderr.is_empty() { |
| 9411 | + "<empty>".to_string() |
| 9412 | + } else { |
| 9413 | + run.stderr.trim_end().to_string() |
| 9414 | + }; |
| 9415 | + format!( |
| 9416 | + "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9417 | + run.exit_code, stdout, stderr |
| 9418 | + ) |
| 9419 | +} |
| 9420 | + |
| 9421 | +fn format_run_signature(signature: &RunSignature) -> String { |
| 9422 | + let stdout = if signature.stdout.is_empty() { |
| 9423 | + "<empty>".to_string() |
| 9424 | + } else { |
| 9425 | + signature.stdout.clone() |
| 9426 | + }; |
| 9427 | + let stderr = if signature.stderr.is_empty() { |
| 9428 | + "<empty>".to_string() |
| 9429 | + } else { |
| 9430 | + signature.stderr.clone() |
| 9431 | + }; |
| 9432 | + format!( |
| 9433 | + "exit: {}\nstdout:\n{}\nstderr:\n{}", |
| 9434 | + signature.exit_code, stdout, stderr |
| 9435 | + ) |
| 9436 | +} |
| 9437 | + |
| 9438 | +#[derive(Debug, Clone, PartialEq, Eq)] |
| 9439 | +struct ObjectSnapshot { |
| 9440 | + text: String, |
| 9441 | + load_commands: String, |
| 9442 | + relocations: String, |
| 9443 | + symbols: String, |
| 9444 | +} |
| 9445 | + |
| 9446 | +#[derive(Debug, Clone)] |
| 9447 | +struct TextRun { |
| 9448 | + label: String, |
| 9449 | + command: String, |
| 9450 | + normalized: String, |
| 9451 | +} |
| 9452 | + |
| 9453 | +#[derive(Debug, Clone)] |
| 9454 | +struct BehaviorRun { |
| 9455 | + label: String, |
| 9456 | + command: String, |
| 9457 | + signature: RunSignature, |
| 9458 | + run: RunCapture, |
| 9459 | +} |
| 9460 | + |
| 9461 | +#[derive(Debug, Clone)] |
| 9462 | +struct ObjectRun { |
| 9463 | + label: String, |
| 9464 | + command: String, |
| 9465 | + snapshot: ObjectSnapshot, |
| 9466 | +} |
| 9467 | + |
| 9468 | +fn object_snapshot(path: &Path, tools: &ToolchainConfig) -> Result<ObjectSnapshot, String> { |
| 9469 | + let text = normalize_tool_output(&tool_output( |
| 9470 | + tools.otool_bin(), |
| 9471 | + &["-t", path.to_str().unwrap()], |
| 9472 | + )?); |
| 9473 | + let load_commands = normalize_tool_output(&tool_output( |
| 9474 | + tools.otool_bin(), |
| 9475 | + &["-l", path.to_str().unwrap()], |
| 9476 | + )?); |
| 9477 | + let relocations = normalize_tool_output(&tool_output( |
| 9478 | + tools.otool_bin(), |
| 9479 | + &["-rv", path.to_str().unwrap()], |
| 9480 | + )?); |
| 9481 | + let symbols = normalize_tool_output(&tool_output( |
| 9482 | + tools.nm_bin(), |
| 9483 | + &["-m", path.to_str().unwrap()], |
| 9484 | + )?); |
| 9485 | + |
| 9486 | + Ok(ObjectSnapshot { |
| 9487 | + text, |
| 9488 | + load_commands, |
| 9489 | + relocations, |
| 9490 | + symbols, |
| 9491 | + }) |
| 9492 | +} |
| 9493 | + |
| 9494 | +fn tool_output(tool: &str, args: &[&str]) -> Result<String, String> { |
| 9495 | + let output = Command::new(tool) |
| 9496 | + .args(args) |
| 9497 | + .output() |
| 9498 | + .map_err(|e| format!("cannot run {}: {}", tool, e))?; |
| 9499 | + if output.status.success() { |
| 9500 | + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) |
| 9501 | + } else { |
| 9502 | + Err(format!( |
| 9503 | + "{} failed:\n{}", |
| 9504 | + tool, |
| 9505 | + String::from_utf8_lossy(&output.stderr) |
| 9506 | + )) |
| 9507 | + } |
| 9508 | +} |
| 9509 | + |
| 9510 | +fn normalize_tool_output(text: &str) -> String { |
| 9511 | + text.lines() |
| 9512 | + .filter(|line| !line.trim_end().ends_with(".o:")) |
| 9513 | + .map(str::trim_end) |
| 9514 | + .collect::<Vec<_>>() |
| 9515 | + .join("\n") |
| 9516 | +} |
| 9517 | + |
| 9518 | +fn read_text_artifact(path: &Path) -> Result<String, String> { |
| 9519 | + fs::read_to_string(path).map_err(|e| format!("cannot read '{}': {}", path.display(), e)) |
| 9520 | +} |
| 9521 | + |
| 9522 | +fn normalize_text_artifact(text: &str) -> String { |
| 9523 | + text.replace("\r\n", "\n") |
| 9524 | + .lines() |
| 9525 | + .map(str::trim_end) |
| 9526 | + .collect::<Vec<_>>() |
| 9527 | + .join("\n") |
| 9528 | +} |
| 9529 | + |
| 9530 | +fn count_unique_strings<'a>(values: impl IntoIterator<Item = &'a str>) -> usize { |
| 9531 | + values.into_iter().collect::<BTreeSet<_>>().len() |
| 9532 | +} |
| 9533 | + |
| 9534 | +fn first_distinct_text_pair(runs: &[TextRun]) -> Option<(&TextRun, &TextRun)> { |
| 9535 | + for left_index in 0..runs.len() { |
| 9536 | + for right_index in (left_index + 1)..runs.len() { |
| 9537 | + if runs[left_index].normalized != runs[right_index].normalized { |
| 9538 | + return Some((&runs[left_index], &runs[right_index])); |
| 9539 | + } |
| 9540 | + } |
| 9541 | + } |
| 9542 | + None |
| 9543 | +} |
| 9544 | + |
| 9545 | +fn count_unique_run_signatures<'a>(values: impl IntoIterator<Item = &'a RunSignature>) -> usize { |
| 9546 | + values.into_iter().collect::<BTreeSet<_>>().len() |
| 9547 | +} |
| 9548 | + |
| 9549 | +fn first_distinct_behavior_pair(runs: &[BehaviorRun]) -> Option<(&BehaviorRun, &BehaviorRun)> { |
| 9550 | + for left_index in 0..runs.len() { |
| 9551 | + for right_index in (left_index + 1)..runs.len() { |
| 9552 | + if runs[left_index].signature != runs[right_index].signature { |
| 9553 | + return Some((&runs[left_index], &runs[right_index])); |
| 9554 | + } |
| 9555 | + } |
| 9556 | + } |
| 9557 | + None |
| 9558 | +} |
| 9559 | + |
| 9560 | +fn first_distinct_object_pair(runs: &[ObjectRun]) -> Option<(&ObjectRun, &ObjectRun)> { |
| 9561 | + for left_index in 0..runs.len() { |
| 9562 | + for right_index in (left_index + 1)..runs.len() { |
| 9563 | + if runs[left_index].snapshot != runs[right_index].snapshot { |
| 9564 | + return Some((&runs[left_index], &runs[right_index])); |
| 9565 | + } |
| 9566 | + } |
| 9567 | + } |
| 9568 | + None |
| 9569 | +} |
| 9570 | + |
| 9571 | +fn render_object_snapshot(snapshot: &ObjectSnapshot) -> String { |
| 9572 | + format!( |
| 9573 | + "== text ==\n{}\n\n== load_commands ==\n{}\n\n== relocations ==\n{}\n\n== symbols ==\n{}", |
| 9574 | + snapshot.text, snapshot.load_commands, snapshot.relocations, snapshot.symbols |
| 9575 | + ) |
| 9576 | +} |
| 9577 | + |
| 9578 | +fn parse_object_snapshot_text(text: &str) -> Result<ObjectSnapshot, String> { |
| 9579 | + let text = text |
| 9580 | + .strip_prefix("== text ==\n") |
| 9581 | + .ok_or_else(|| "object snapshot was missing the '== text ==' header".to_string())?; |
| 9582 | + let (text, rest) = text |
| 9583 | + .split_once("\n\n== load_commands ==\n") |
| 9584 | + .ok_or_else(|| { |
| 9585 | + "object snapshot was missing the '== load_commands ==' section".to_string() |
| 9586 | + })?; |
| 9587 | + let (load_commands, rest) = rest |
| 9588 | + .split_once("\n\n== relocations ==\n") |
| 9589 | + .ok_or_else(|| "object snapshot was missing the '== relocations ==' section".to_string())?; |
| 9590 | + let (relocations, symbols) = rest |
| 9591 | + .split_once("\n\n== symbols ==\n") |
| 9592 | + .ok_or_else(|| "object snapshot was missing the '== symbols ==' section".to_string())?; |
| 9593 | + |
| 9594 | + Ok(ObjectSnapshot { |
| 9595 | + text: text.to_string(), |
| 9596 | + load_commands: load_commands.to_string(), |
| 9597 | + relocations: relocations.to_string(), |
| 9598 | + symbols: symbols.to_string(), |
| 9599 | + }) |
| 9600 | +} |
| 9601 | + |
| 9602 | +fn describe_text_difference( |
| 9603 | + expected: &str, |
| 9604 | + actual: &str, |
| 9605 | + left_label: &str, |
| 9606 | + right_label: &str, |
| 9607 | +) -> String { |
| 9608 | + let expected_lines: Vec<&str> = expected.lines().collect(); |
| 9609 | + let actual_lines: Vec<&str> = actual.lines().collect(); |
| 9610 | + let shared = expected_lines.len().min(actual_lines.len()); |
| 9611 | + |
| 9612 | + for index in 0..shared { |
| 9613 | + if expected_lines[index] != actual_lines[index] { |
| 9614 | + return format!( |
| 9615 | + "first differing line: {}\n{}: {}\n{}: {}", |
| 9616 | + index + 1, |
| 9617 | + left_label, |
| 9618 | + expected_lines[index], |
| 9619 | + right_label, |
| 9620 | + actual_lines[index] |
| 9621 | + ); |
| 9622 | + } |
| 9623 | + } |
| 9624 | + |
| 9625 | + let mut detail = format!( |
| 9626 | + "snapshot length differs\n{} lines: {}\n{} lines: {}", |
| 9627 | + left_label, |
| 9628 | + expected_lines.len(), |
| 9629 | + right_label, |
| 9630 | + actual_lines.len() |
| 9631 | + ); |
| 9632 | + if let Some(extra) = expected_lines.get(shared) { |
| 9633 | + detail.push_str(&format!( |
| 9634 | + "\nfirst extra line: {}\n{}: {}", |
| 9635 | + shared + 1, |
| 9636 | + left_label, |
| 9637 | + extra |
| 9638 | + )); |
| 9639 | + } else if let Some(extra) = actual_lines.get(shared) { |
| 9640 | + detail.push_str(&format!( |
| 9641 | + "\nfirst extra line: {}\n{}: {}", |
| 9642 | + shared + 1, |
| 9643 | + right_label, |
| 9644 | + extra |
| 9645 | + )); |
| 9646 | + } |
| 9647 | + detail |
| 9648 | +} |
| 9649 | + |
| 9650 | +fn describe_object_difference( |
| 9651 | + expected: &ObjectSnapshot, |
| 9652 | + actual: &ObjectSnapshot, |
| 9653 | + left_label: &str, |
| 9654 | + right_label: &str, |
| 9655 | +) -> String { |
| 9656 | + let mut differing = Vec::new(); |
| 9657 | + if expected.text != actual.text { |
| 9658 | + differing.push(("text", &expected.text, &actual.text)); |
| 9659 | + } |
| 9660 | + if expected.load_commands != actual.load_commands { |
| 9661 | + differing.push(( |
| 9662 | + "load_commands", |
| 9663 | + &expected.load_commands, |
| 9664 | + &actual.load_commands, |
| 9665 | + )); |
| 9666 | + } |
| 9667 | + if expected.relocations != actual.relocations { |
| 9668 | + differing.push(("relocations", &expected.relocations, &actual.relocations)); |
| 9669 | + } |
| 9670 | + if expected.symbols != actual.symbols { |
| 9671 | + differing.push(("symbols", &expected.symbols, &actual.symbols)); |
| 9672 | + } |
| 9673 | + |
| 9674 | + if differing.is_empty() { |
| 9675 | + return "object snapshots matched".to_string(); |
| 9676 | + } |
| 9677 | + |
| 9678 | + let component_list = differing |
| 9679 | + .iter() |
| 9680 | + .map(|(name, _, _)| *name) |
| 9681 | + .collect::<Vec<_>>() |
| 9682 | + .join(", "); |
| 9683 | + let (first_name, first_expected, first_actual) = differing[0]; |
| 9684 | + |
| 9685 | + format!( |
| 9686 | + "differing object components: {}\n{}\n{}", |
| 9687 | + component_list, |
| 9688 | + format!("first differing component: {}", first_name), |
| 9689 | + describe_text_difference(first_expected, first_actual, left_label, right_label) |
| 9690 | + ) |
| 9691 | +} |
| 9692 | + |
| 9693 | +fn describe_run_difference( |
| 9694 | + expected: &RunCapture, |
| 9695 | + actual: &RunCapture, |
| 9696 | + left_label: &str, |
| 9697 | + right_label: &str, |
| 9698 | +) -> String { |
| 9699 | + let expected = normalize_run_signature(expected); |
| 9700 | + let actual = normalize_run_signature(actual); |
| 9701 | + let mut differing = Vec::new(); |
| 9702 | + if expected.exit_code != actual.exit_code { |
| 9703 | + differing.push("exit_code"); |
| 9704 | + } |
| 9705 | + if expected.stdout != actual.stdout { |
| 9706 | + differing.push("stdout"); |
| 9707 | + } |
| 9708 | + if expected.stderr != actual.stderr { |
| 9709 | + differing.push("stderr"); |
| 9710 | + } |
| 9711 | + |
| 9712 | + if differing.is_empty() { |
| 9713 | + return "runtime behavior matched".to_string(); |
| 9714 | + } |
| 9715 | + |
| 9716 | + let component_list = differing.join(", "); |
| 9717 | + match differing[0] { |
| 9718 | + "exit_code" => format!( |
| 9719 | + "differing runtime components: {}\nfirst differing component: exit_code\n{}: {}\n{}: {}", |
| 9720 | + component_list, |
| 9721 | + left_label, |
| 9722 | + expected.exit_code, |
| 9723 | + right_label, |
| 9724 | + actual.exit_code |
| 9725 | + ), |
| 9726 | + "stdout" => format!( |
| 9727 | + "differing runtime components: {}\nfirst differing component: stdout\n{}", |
| 9728 | + component_list, |
| 9729 | + describe_text_difference(&expected.stdout, &actual.stdout, left_label, right_label) |
| 9730 | + ), |
| 9731 | + "stderr" => format!( |
| 9732 | + "differing runtime components: {}\nfirst differing component: stderr\n{}", |
| 9733 | + component_list, |
| 9734 | + describe_text_difference(&expected.stderr, &actual.stderr, left_label, right_label) |
| 9735 | + ), |
| 9736 | + _ => unreachable!("only known runtime components are compared"), |
| 9737 | + } |
| 9738 | +} |
| 9739 | + |
| 9740 | +fn varying_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9741 | + object_components_by_variation(snapshots, true) |
| 9742 | +} |
| 9743 | + |
| 9744 | +fn stable_object_components(snapshots: &[&ObjectSnapshot]) -> Vec<&'static str> { |
| 9745 | + object_components_by_variation(snapshots, false) |
| 9746 | +} |
| 9747 | + |
| 9748 | +fn varying_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9749 | + run_components_by_variation(signatures, true) |
| 9750 | +} |
| 9751 | + |
| 9752 | +fn stable_run_components(signatures: &[&RunSignature]) -> Vec<&'static str> { |
| 9753 | + run_components_by_variation(signatures, false) |
| 9754 | +} |
| 9755 | + |
| 9756 | +fn object_components_by_variation( |
| 9757 | + snapshots: &[&ObjectSnapshot], |
| 9758 | + want_varying: bool, |
| 9759 | +) -> Vec<&'static str> { |
| 9760 | + let components = [ |
| 9761 | + ( |
| 9762 | + "text", |
| 9763 | + snapshots |
| 9764 | + .iter() |
| 9765 | + .map(|snapshot| snapshot.text.as_str()) |
| 9766 | + .collect::<Vec<_>>(), |
| 9767 | + ), |
| 9768 | + ( |
| 9769 | + "load_commands", |
| 9770 | + snapshots |
| 9771 | + .iter() |
| 9772 | + .map(|snapshot| snapshot.load_commands.as_str()) |
| 9773 | + .collect::<Vec<_>>(), |
| 9774 | + ), |
| 9775 | + ( |
| 9776 | + "relocations", |
| 9777 | + snapshots |
| 9778 | + .iter() |
| 9779 | + .map(|snapshot| snapshot.relocations.as_str()) |
| 9780 | + .collect::<Vec<_>>(), |
| 9781 | + ), |
| 9782 | + ( |
| 9783 | + "symbols", |
| 9784 | + snapshots |
| 9785 | + .iter() |
| 9786 | + .map(|snapshot| snapshot.symbols.as_str()) |
| 9787 | + .collect::<Vec<_>>(), |
| 9788 | + ), |
| 9789 | + ]; |
| 9790 | + |
| 9791 | + components |
| 9792 | + .into_iter() |
| 9793 | + .filter_map(|(name, values)| { |
| 9794 | + let varies = count_unique_strings(values) > 1; |
| 9795 | + if varies == want_varying { |
| 9796 | + Some(name) |
| 9797 | + } else { |
| 9798 | + None |
| 9799 | + } |
| 9800 | + }) |
| 9801 | + .collect() |
| 9802 | +} |
| 9803 | + |
| 9804 | +fn run_components_by_variation( |
| 9805 | + signatures: &[&RunSignature], |
| 9806 | + want_varying: bool, |
| 9807 | +) -> Vec<&'static str> { |
| 9808 | + let components = [ |
| 9809 | + ( |
| 9810 | + "exit_code", |
| 9811 | + signatures |
| 9812 | + .iter() |
| 9813 | + .map(|signature| signature.exit_code.to_string()) |
| 9814 | + .collect::<Vec<_>>(), |
| 9815 | + ), |
| 9816 | + ( |
| 9817 | + "stdout", |
| 9818 | + signatures |
| 9819 | + .iter() |
| 9820 | + .map(|signature| signature.stdout.clone()) |
| 9821 | + .collect::<Vec<_>>(), |
| 9822 | + ), |
| 9823 | + ( |
| 9824 | + "stderr", |
| 9825 | + signatures |
| 9826 | + .iter() |
| 9827 | + .map(|signature| signature.stderr.clone()) |
| 9828 | + .collect::<Vec<_>>(), |
| 9829 | + ), |
| 9830 | + ]; |
| 9831 | + |
| 9832 | + components |
| 9833 | + .into_iter() |
| 9834 | + .filter_map(|(name, values)| { |
| 9835 | + let varies = count_unique_strings(values.iter().map(String::as_str)) > 1; |
| 9836 | + if varies == want_varying { |
| 9837 | + Some(name) |
| 9838 | + } else { |
| 9839 | + None |
| 9840 | + } |
| 9841 | + }) |
| 9842 | + .collect() |
| 9843 | +} |
| 9844 | + |
| 9845 | +fn join_or_none(values: &[&str]) -> String { |
| 9846 | + if values.is_empty() { |
| 9847 | + "none".to_string() |
| 9848 | + } else { |
| 9849 | + values.join(", ") |
| 9850 | + } |
| 9851 | +} |
| 9852 | + |
| 9853 | +fn join_or_none_from_strings(values: &[String]) -> String { |
| 9854 | + if values.is_empty() { |
| 9855 | + "none".to_string() |
| 9856 | + } else { |
| 9857 | + values.join(", ") |
| 9858 | + } |
| 9859 | +} |
| 9860 | + |
| 9861 | +fn join_usize_set(values: &BTreeSet<usize>) -> String { |
| 9862 | + if values.is_empty() { |
| 9863 | + "n/a".to_string() |
| 9864 | + } else { |
| 9865 | + values |
| 9866 | + .iter() |
| 9867 | + .map(|value| value.to_string()) |
| 9868 | + .collect::<Vec<_>>() |
| 9869 | + .join(", ") |
| 9870 | + } |
| 9871 | +} |
| 9872 | + |
| 9873 | +fn join_string_set(values: &BTreeSet<String>) -> String { |
| 9874 | + if values.is_empty() { |
| 9875 | + "none".to_string() |
| 9876 | + } else { |
| 9877 | + values.iter().cloned().collect::<Vec<_>>().join(", ") |
| 9878 | + } |
| 9879 | +} |
| 9880 | + |
| 9881 | +fn render_consistency_rollup(rollup: &ConsistencyRollup) -> String { |
| 9882 | + let mut parts = vec![format!("{} cells", rollup.cells)]; |
| 9883 | + if !rollup.repeat_counts.is_empty() { |
| 9884 | + parts.push(format!( |
| 9885 | + "repeat_count={}", |
| 9886 | + join_usize_set(&rollup.repeat_counts) |
| 9887 | + )); |
| 9888 | + } |
| 9889 | + if !rollup.unique_variant_counts.is_empty() { |
| 9890 | + parts.push(format!( |
| 9891 | + "unique_variants={}", |
| 9892 | + join_usize_set(&rollup.unique_variant_counts) |
| 9893 | + )); |
| 9894 | + } |
| 9895 | + if !rollup.varying_components.is_empty() { |
| 9896 | + parts.push(format!( |
| 9897 | + "varying={}", |
| 9898 | + join_string_set(&rollup.varying_components) |
| 9899 | + )); |
| 9900 | + } |
| 9901 | + if !rollup.stable_components.is_empty() { |
| 9902 | + parts.push(format!( |
| 9903 | + "stable={}", |
| 9904 | + join_string_set(&rollup.stable_components) |
| 9905 | + )); |
| 9906 | + } |
| 9907 | + parts.join("; ") |
| 9908 | +} |
| 9909 | + |
| 9910 | +fn write_behavior_run_artifacts( |
| 9911 | + root: &Path, |
| 9912 | + prefix: &str, |
| 9913 | + command: &str, |
| 9914 | + run: &RunCapture, |
| 9915 | +) -> Result<(), std::io::Error> { |
| 9916 | + let signature = normalize_run_signature(run); |
| 9917 | + fs::write(root.join(format!("{}.command.txt", prefix)), command)?; |
| 9918 | + fs::write(root.join(format!("{}.stdout.txt", prefix)), &run.stdout)?; |
| 9919 | + fs::write(root.join(format!("{}.stderr.txt", prefix)), &run.stderr)?; |
| 9920 | + fs::write( |
| 9921 | + root.join(format!("{}.exit_code.txt", prefix)), |
| 9922 | + format!("{}\n", run.exit_code), |
| 9923 | + )?; |
| 9924 | + fs::write( |
| 9925 | + root.join(format!("{}.normalized.txt", prefix)), |
| 9926 | + format_run_signature(&signature), |
| 9927 | + )?; |
| 9928 | + Ok(()) |
| 9929 | +} |
| 9930 | + |
| 9931 | +fn render_summary(summary: &Summary) -> String { |
| 9932 | + let mut lines = vec![ |
| 9933 | + "Summary".to_string(), |
| 9934 | + format!(" passed: {}", summary.passed), |
| 9935 | + format!(" failed: {}", summary.failed), |
| 9936 | + format!(" xfailed: {}", summary.xfailed), |
| 9937 | + format!(" xpassed: {}", summary.xpassed), |
| 9938 | + format!(" future: {}", summary.future), |
| 9939 | + ]; |
| 9940 | + |
| 9941 | + if !summary.consistency.is_empty() { |
| 9942 | + lines.push(String::new()); |
| 9943 | + lines.push("Consistency".to_string()); |
| 9944 | + lines.push(format!(" affected_checks: {}", summary.consistency.len())); |
| 9945 | + lines.push(format!( |
| 9946 | + " cells_with_issues: {}", |
| 9947 | + summary |
| 9948 | + .consistency |
| 9949 | + .values() |
| 9950 | + .map(|rollup| rollup.cells) |
| 9951 | + .sum::<usize>() |
| 9952 | + )); |
| 9953 | + for (check, rollup) in &summary.consistency { |
| 9954 | + lines.push(format!( |
| 9955 | + " {}: {}", |
| 9956 | + check.as_str(), |
| 9957 | + render_consistency_rollup(rollup) |
| 9958 | + )); |
| 9959 | + } |
| 9960 | + } |
| 9961 | + |
| 9962 | + lines.join("\n") |
| 9963 | +} |
| 9964 | + |
| 9965 | +fn write_requested_reports(config: &RunConfig, summary: &Summary) -> Result<(), String> { |
| 9966 | + if let Some(path) = &config.json_report { |
| 9967 | + write_report(path, &render_json_report(summary), "json report")?; |
| 9968 | + println!("json report: {}", path.display()); |
| 9969 | + } |
| 9970 | + if let Some(path) = &config.markdown_report { |
| 9971 | + write_report(path, &render_markdown_report(summary), "markdown report")?; |
| 9972 | + println!("markdown report: {}", path.display()); |
| 9973 | + } |
| 9974 | + Ok(()) |
| 9975 | +} |
| 9976 | + |
| 9977 | +fn write_report(path: &Path, content: &str, label: &str) -> Result<(), String> { |
| 9978 | + if let Some(parent) = path.parent() { |
| 9979 | + fs::create_dir_all(parent).map_err(|e| { |
| 9980 | + format!( |
| 9981 | + "cannot create parent directory for {} '{}': {}", |
| 9982 | + label, |
| 9983 | + path.display(), |
| 9984 | + e |
| 9985 | + ) |
| 9986 | + })?; |
| 9987 | + } |
| 9988 | + fs::write(path, content) |
| 9989 | + .map_err(|e| format!("cannot write {} '{}': {}", label, path.display(), e)) |
| 9990 | +} |
| 9991 | + |
| 9992 | +fn render_json_report(summary: &Summary) -> String { |
| 9993 | + let mut lines = vec![ |
| 9994 | + "{".to_string(), |
| 9995 | + format!(" \"passed\": {},", summary.passed), |
| 9996 | + format!(" \"failed\": {},", summary.failed), |
| 9997 | + format!(" \"xfailed\": {},", summary.xfailed), |
| 9998 | + format!(" \"xpassed\": {},", summary.xpassed), |
| 9999 | + format!(" \"future\": {},", summary.future), |
| 10000 | + " \"outcomes\": [".to_string(), |
| 10001 | + ]; |
| 10002 | + |
| 10003 | + for (index, outcome) in summary.outcomes.iter().enumerate() { |
| 10004 | + lines.push(" {".to_string()); |
| 10005 | + lines.push(format!( |
| 10006 | + " \"suite\": \"{}\",", |
| 10007 | + json_escape(&outcome.suite) |
| 10008 | + )); |
| 10009 | + lines.push(format!( |
| 10010 | + " \"case\": \"{}\",", |
| 10011 | + json_escape(&outcome.case) |
| 10012 | + )); |
| 10013 | + lines.push(format!( |
| 10014 | + " \"opt\": \"{}\",", |
| 10015 | + outcome.opt_level.as_str() |
| 10016 | + )); |
| 10017 | + lines.push(format!( |
| 10018 | + " \"kind\": \"{}\",", |
| 10019 | + outcome_kind_name(outcome.kind) |
| 10020 | + )); |
| 10021 | + match &outcome.primary_backend { |
| 10022 | + Some(backend) => { |
| 10023 | + lines.push(" \"primary_backend\": {".to_string()); |
| 10024 | + lines.push(format!( |
| 10025 | + " \"kind\": \"{}\",", |
| 10026 | + json_escape(&backend.kind) |
| 10027 | + )); |
| 10028 | + lines.push(format!( |
| 10029 | + " \"mode\": \"{}\",", |
| 10030 | + json_escape(&backend.mode) |
| 10031 | + )); |
| 10032 | + lines.push(format!( |
| 10033 | + " \"detail\": \"{}\"", |
| 10034 | + json_escape(&backend.detail) |
| 10035 | + )); |
| 10036 | + lines.push(" },".to_string()); |
| 10037 | + } |
| 10038 | + None => lines.push(" \"primary_backend\": null,".to_string()), |
| 10039 | + } |
| 10040 | + lines.push(format!( |
| 10041 | + " \"detail\": \"{}\",", |
| 10042 | + json_escape(&outcome.detail) |
| 10043 | + )); |
| 10044 | + match &outcome.bundle { |
| 10045 | + Some(bundle) => lines.push(format!( |
| 10046 | + " \"bundle\": \"{}\",", |
| 10047 | + json_escape(&bundle.display().to_string()) |
| 10048 | + )), |
| 10049 | + None => lines.push(" \"bundle\": null,".to_string()), |
| 10050 | + } |
| 10051 | + lines.push(format!( |
| 10052 | + " \"consistency\": {}", |
| 10053 | + render_json_consistency_observations(&outcome.consistency_observations) |
| 10054 | + )); |
| 10055 | + lines.push(if index + 1 == summary.outcomes.len() { |
| 10056 | + " }".to_string() |
| 10057 | + } else { |
| 10058 | + " },".to_string() |
| 10059 | + }); |
| 10060 | + } |
| 10061 | + |
| 10062 | + lines.push(" ],".to_string()); |
| 10063 | + lines.push(" \"consistency\": {".to_string()); |
| 10064 | + for (index, (check, rollup)) in summary.consistency.iter().enumerate() { |
| 10065 | + lines.push(format!(" \"{}\": {{", check.as_str())); |
| 10066 | + lines.push(format!(" \"cells\": {},", rollup.cells)); |
| 10067 | + lines.push(format!( |
| 10068 | + " \"repeat_counts\": {},", |
| 10069 | + json_usize_set(&rollup.repeat_counts) |
| 10070 | + )); |
| 10071 | + lines.push(format!( |
| 10072 | + " \"unique_variant_counts\": {},", |
| 10073 | + json_usize_set(&rollup.unique_variant_counts) |
| 10074 | + )); |
| 10075 | + lines.push(format!( |
| 10076 | + " \"varying_components\": {},", |
| 10077 | + json_string_iter(rollup.varying_components.iter().map(|value| value.as_str())) |
| 10078 | + )); |
| 10079 | + lines.push(format!( |
| 10080 | + " \"stable_components\": {}", |
| 10081 | + json_string_iter(rollup.stable_components.iter().map(|value| value.as_str())) |
| 10082 | + )); |
| 10083 | + lines.push(if index + 1 == summary.consistency.len() { |
| 10084 | + " }".to_string() |
| 10085 | + } else { |
| 10086 | + " },".to_string() |
| 10087 | + }); |
| 10088 | + } |
| 10089 | + lines.push(" }".to_string()); |
| 10090 | + lines.push("}".to_string()); |
| 10091 | + lines.join("\n") + "\n" |
| 10092 | +} |
| 10093 | + |
| 10094 | +fn render_json_consistency_observations(observations: &[ConsistencyObservation]) -> String { |
| 10095 | + let mut rendered = String::from("["); |
| 10096 | + for (index, observation) in observations.iter().enumerate() { |
| 10097 | + if index > 0 { |
| 10098 | + rendered.push_str(", "); |
| 10099 | + } |
| 10100 | + rendered.push('{'); |
| 10101 | + rendered.push_str(&format!( |
| 10102 | + "\"check\":\"{}\",\"summary\":\"{}\",", |
| 10103 | + observation.check.as_str(), |
| 10104 | + json_escape(&observation.summary) |
| 10105 | + )); |
| 10106 | + match observation.repeat_count { |
| 10107 | + Some(count) => rendered.push_str(&format!("\"repeat_count\":{},", count)), |
| 10108 | + None => rendered.push_str("\"repeat_count\":null,"), |
| 10109 | + } |
| 10110 | + match observation.unique_variant_count { |
| 10111 | + Some(count) => rendered.push_str(&format!("\"unique_variant_count\":{},", count)), |
| 10112 | + None => rendered.push_str("\"unique_variant_count\":null,"), |
| 10113 | + } |
| 10114 | + rendered.push_str(&format!( |
| 10115 | + "\"varying_components\":{},\"stable_components\":{}", |
| 10116 | + json_string_array(&observation.varying_components), |
| 10117 | + json_string_array(&observation.stable_components) |
| 10118 | + )); |
| 10119 | + rendered.push('}'); |
| 10120 | + } |
| 10121 | + rendered.push(']'); |
| 10122 | + rendered |
| 10123 | +} |
| 10124 | + |
| 10125 | +fn render_markdown_report(summary: &Summary) -> String { |
| 10126 | + let mut lines = vec![ |
| 10127 | + "# afs-tests report".to_string(), |
| 10128 | + String::new(), |
| 10129 | + "## Summary".to_string(), |
| 10130 | + String::new(), |
| 10131 | + "| kind | count |".to_string(), |
| 10132 | + "| --- | ---: |".to_string(), |
| 10133 | + format!("| passed | {} |", summary.passed), |
| 10134 | + format!("| failed | {} |", summary.failed), |
| 10135 | + format!("| xfailed | {} |", summary.xfailed), |
| 10136 | + format!("| xpassed | {} |", summary.xpassed), |
| 10137 | + format!("| future | {} |", summary.future), |
| 10138 | + ]; |
| 10139 | + |
| 10140 | + if !summary.consistency.is_empty() { |
| 10141 | + lines.push(String::new()); |
| 10142 | + lines.push("## Consistency".to_string()); |
| 10143 | + lines.push(String::new()); |
| 10144 | + lines.push("| check | cells | repeats | unique variants | varying | stable |".to_string()); |
| 10145 | + lines.push("| --- | ---: | --- | --- | --- | --- |".to_string()); |
| 10146 | + for (check, rollup) in &summary.consistency { |
| 10147 | + lines.push(format!( |
| 10148 | + "| `{}` | {} | {} | {} | {} | {} |", |
| 10149 | + check.as_str(), |
| 10150 | + rollup.cells, |
| 10151 | + join_usize_set(&rollup.repeat_counts), |
| 10152 | + join_usize_set(&rollup.unique_variant_counts), |
| 10153 | + join_string_set(&rollup.varying_components), |
| 10154 | + join_string_set(&rollup.stable_components), |
| 10155 | + )); |
| 10156 | + } |
| 10157 | + } |
| 10158 | + |
| 10159 | + lines.push(String::new()); |
| 10160 | + lines.push("## Outcomes".to_string()); |
| 10161 | + for outcome in &summary.outcomes { |
| 10162 | + lines.push(String::new()); |
| 10163 | + lines.push(format!( |
| 10164 | + "### `{}` / `{}` / `{}` / `{}`", |
| 10165 | + outcome.suite, |
| 10166 | + outcome.case, |
| 10167 | + outcome.opt_level.as_str(), |
| 10168 | + outcome_kind_name(outcome.kind) |
| 10169 | + )); |
| 10170 | + if let Some(backend) = &outcome.primary_backend { |
| 10171 | + lines.push(format!( |
| 10172 | + "primary_backend: `{}` (`{}`)", |
| 10173 | + backend.kind, backend.mode |
| 10174 | + )); |
| 10175 | + lines.push(format!("primary_backend_detail: {}", backend.detail)); |
| 10176 | + } |
| 10177 | + if let Some(bundle) = &outcome.bundle { |
| 10178 | + lines.push(format!("bundle: `{}`", bundle.display())); |
| 10179 | + } |
| 10180 | + if !outcome.detail.trim().is_empty() { |
| 10181 | + lines.push(String::new()); |
| 10182 | + lines.push("```text".to_string()); |
| 10183 | + lines.extend( |
| 10184 | + outcome |
| 10185 | + .detail |
| 10186 | + .trim_end() |
| 10187 | + .lines() |
| 10188 | + .map(|line| line.to_string()), |
| 10189 | + ); |
| 10190 | + lines.push("```".to_string()); |
| 10191 | + } |
| 10192 | + } |
| 10193 | + |
| 10194 | + lines.join("\n") + "\n" |
| 10195 | +} |
| 10196 | + |
| 10197 | +fn outcome_kind_name(kind: OutcomeKind) -> &'static str { |
| 10198 | + match kind { |
| 10199 | + OutcomeKind::Pass => "pass", |
| 10200 | + OutcomeKind::Fail => "fail", |
| 10201 | + OutcomeKind::Xfail => "xfail", |
| 10202 | + OutcomeKind::Xpass => "xpass", |
| 10203 | + OutcomeKind::Future => "future", |
| 10204 | + } |
| 10205 | +} |
| 10206 | + |
| 10207 | +fn json_escape(text: &str) -> String { |
| 10208 | + let mut escaped = String::new(); |
| 10209 | + for ch in text.chars() { |
| 10210 | + match ch { |
| 10211 | + '\\' => escaped.push_str("\\\\"), |
| 10212 | + '"' => escaped.push_str("\\\""), |
| 10213 | + '\n' => escaped.push_str("\\n"), |
| 10214 | + '\r' => escaped.push_str("\\r"), |
| 10215 | + '\t' => escaped.push_str("\\t"), |
| 10216 | + c if c.is_control() => escaped.push_str(&format!("\\u{:04x}", c as u32)), |
| 10217 | + c => escaped.push(c), |
| 10218 | + } |
| 10219 | + } |
| 10220 | + escaped |
| 10221 | +} |
| 10222 | + |
| 10223 | +fn json_string_array(items: &[String]) -> String { |
| 10224 | + json_string_iter(items.iter().map(|item| item.as_str())) |
| 10225 | +} |
| 10226 | + |
| 10227 | +fn json_string_iter<'a>(items: impl Iterator<Item = &'a str>) -> String { |
| 10228 | + let mut rendered = String::from("["); |
| 10229 | + for (index, item) in items.enumerate() { |
| 10230 | + if index > 0 { |
| 10231 | + rendered.push_str(", "); |
| 10232 | + } |
| 10233 | + rendered.push('"'); |
| 10234 | + rendered.push_str(&json_escape(item)); |
| 10235 | + rendered.push('"'); |
| 10236 | + } |
| 10237 | + rendered.push(']'); |
| 10238 | + rendered |
| 10239 | +} |
| 10240 | + |
| 10241 | +fn json_usize_set(items: &BTreeSet<usize>) -> String { |
| 10242 | + let mut rendered = String::from("["); |
| 10243 | + for (index, item) in items.iter().enumerate() { |
| 10244 | + if index > 0 { |
| 10245 | + rendered.push_str(", "); |
| 10246 | + } |
| 10247 | + rendered.push_str(&item.to_string()); |
| 10248 | + } |
| 10249 | + rendered.push(']'); |
| 10250 | + rendered |
| 10251 | +} |
| 10252 | + |
| 10253 | +fn write_failure_bundle( |
| 10254 | + suite: &SuiteSpec, |
| 10255 | + case: &CaseSpec, |
| 10256 | + prepared: &PreparedInput, |
| 10257 | + outcome: &Outcome, |
| 10258 | + artifacts: &ExecutionArtifacts, |
| 10259 | +) -> Result<PathBuf, String> { |
| 10260 | + let bundle_root = default_report_root() |
| 10261 | + .join(sanitize_component(&suite.name)) |
| 10262 | + .join(sanitize_component(&case.name)) |
| 10263 | + .join(next_report_suffix(outcome.opt_level)); |
| 10264 | + fs::create_dir_all(&bundle_root).map_err(|e| { |
| 10265 | + format!( |
| 10266 | + "cannot create report bundle '{}': {}", |
| 10267 | + bundle_root.display(), |
| 10268 | + e |
| 10269 | + ) |
| 10270 | + })?; |
| 10271 | + |
| 10272 | + let stage_list = artifacts |
| 10273 | + .requested |
| 10274 | + .iter() |
| 10275 | + .map(Stage::as_str) |
| 10276 | + .collect::<Vec<_>>() |
| 10277 | + .join(", "); |
| 10278 | + let refs = if case.reference_compilers.is_empty() { |
| 10279 | + "none".to_string() |
| 10280 | + } else { |
| 10281 | + case.reference_compilers |
| 10282 | + .iter() |
| 10283 | + .map(ReferenceCompiler::as_str) |
| 10284 | + .collect::<Vec<_>>() |
| 10285 | + .join(", ") |
| 10286 | + }; |
| 10287 | + let consistency = if case.consistency_checks.is_empty() { |
| 10288 | + "none".to_string() |
| 10289 | + } else { |
| 10290 | + case.consistency_checks |
| 10291 | + .iter() |
| 10292 | + .map(ConsistencyCheck::as_str) |
| 10293 | + .collect::<Vec<_>>() |
| 10294 | + .join(", ") |
| 10295 | + }; |
| 10296 | + let primary_backend_kind = outcome |
| 10297 | + .primary_backend |
| 10298 | + .as_ref() |
| 10299 | + .map(|backend| backend.kind.as_str()) |
| 10300 | + .unwrap_or("none"); |
| 10301 | + let primary_backend_mode = outcome |
| 10302 | + .primary_backend |
| 10303 | + .as_ref() |
| 10304 | + .map(|backend| backend.mode.as_str()) |
| 10305 | + .unwrap_or("none"); |
| 10306 | + let primary_backend_detail = outcome |
| 10307 | + .primary_backend |
| 10308 | + .as_ref() |
| 10309 | + .map(|backend| backend.detail.as_str()) |
| 10310 | + .unwrap_or("none"); |
| 10311 | + let metadata = format!( |
| 10312 | + "suite: {}\ncase: {}\noutcome: {:?}\nopt: {}\nsource: {}\nrequested_stages: {}\nrepeat_count: {}\nreference_compilers: {}\nconsistency_checks: {}\nprimary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\n", |
| 10313 | + suite.name, |
| 10314 | + case.name, |
| 10315 | + outcome.kind, |
| 10316 | + outcome.opt_level.as_str(), |
| 10317 | + case.source_label(), |
| 10318 | + stage_list, |
| 10319 | + case.repeat_count, |
| 10320 | + refs, |
| 10321 | + consistency, |
| 10322 | + primary_backend_kind, |
| 10323 | + primary_backend_mode, |
| 10324 | + primary_backend_detail |
| 10325 | + ); |
| 10326 | + fs::write(bundle_root.join("metadata.txt"), metadata) |
| 10327 | + .map_err(|e| format!("cannot write bundle metadata: {}", e))?; |
| 10328 | + fs::write(bundle_root.join("detail.txt"), &outcome.detail) |
| 10329 | + .map_err(|e| format!("cannot write bundle detail: {}", e))?; |
| 10330 | + |
| 10331 | + write_case_sources_bundle(&bundle_root, case, prepared)?; |
| 10332 | + |
| 10333 | + let armfortas_root = bundle_root.join("armfortas"); |
| 10334 | + fs::create_dir_all(&armfortas_root) |
| 10335 | + .map_err(|e| format!("cannot create armfortas bundle dir: {}", e))?; |
| 10336 | + write_armfortas_bundle_metadata(&armfortas_root, outcome, artifacts)?; |
| 10337 | + if let Some(result) = &artifacts.armfortas { |
| 10338 | + write_capture_result(&armfortas_root, result)?; |
| 10339 | + } |
| 10340 | + if let Some(failure) = &artifacts.armfortas_failure { |
| 10341 | + write_capture_result(&armfortas_root, &failure.partial_result())?; |
| 10342 | + fs::write( |
| 10343 | + armfortas_root.join("error.txt"), |
| 10344 | + format!("stage: {}\n{}\n", failure.stage.as_str(), failure.detail), |
| 10345 | + ) |
| 10346 | + .map_err(|e| format!("cannot write armfortas error bundle: {}", e))?; |
| 10347 | + } |
| 10348 | + write_armfortas_observation_bundle(&armfortas_root, prepared, artifacts)?; |
| 10349 | + |
| 10350 | + if !artifacts.references.is_empty() { |
| 10351 | + let refs_root = bundle_root.join("references"); |
| 10352 | + fs::create_dir_all(&refs_root) |
| 10353 | + .map_err(|e| format!("cannot create references bundle dir: {}", e))?; |
| 10354 | + let reference_observations = reference_observations_for_bundle( |
| 10355 | + &prepared.compiler_source, |
| 10356 | + outcome.opt_level, |
| 10357 | + artifacts, |
| 10358 | + ); |
| 10359 | + write_reference_summary_bundle(&refs_root, &artifacts.references, &reference_observations)?; |
| 10360 | + for (index, reference) in artifacts.references.iter().enumerate() { |
| 10361 | + write_reference_bundle( |
| 10362 | + &refs_root, |
| 10363 | + &prepared.compiler_source, |
| 10364 | + outcome.opt_level, |
| 10365 | + reference, |
| 10366 | + reference_observations.get(index), |
| 10367 | + )?; |
| 10368 | + } |
| 10369 | + } |
| 10370 | + |
| 10371 | + if !artifacts.consistency_issues.is_empty() { |
| 10372 | + write_consistency_bundle(&bundle_root, &artifacts.consistency_issues)?; |
| 10373 | + } |
| 10374 | + |
| 10375 | + Ok(bundle_root) |
| 10376 | +} |
| 10377 | + |
| 10378 | +fn write_armfortas_bundle_metadata( |
| 10379 | + armfortas_root: &Path, |
| 10380 | + outcome: &Outcome, |
| 10381 | + artifacts: &ExecutionArtifacts, |
| 10382 | +) -> Result<(), String> { |
| 10383 | + let primary_backend_kind = outcome |
| 10384 | + .primary_backend |
| 10385 | + .as_ref() |
| 10386 | + .map(|backend| backend.kind.as_str()) |
| 10387 | + .unwrap_or("none"); |
| 10388 | + let primary_backend_mode = outcome |
| 10389 | + .primary_backend |
| 10390 | + .as_ref() |
| 10391 | + .map(|backend| backend.mode.as_str()) |
| 10392 | + .unwrap_or("none"); |
| 10393 | + let primary_backend_detail = outcome |
| 10394 | + .primary_backend |
| 10395 | + .as_ref() |
| 10396 | + .map(|backend| backend.detail.as_str()) |
| 10397 | + .unwrap_or("none"); |
| 10398 | + let captured_stages = if let Some(result) = &artifacts.armfortas { |
| 10399 | + join_or_none(&result.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10400 | + } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10401 | + join_or_none(&failure.stages.keys().map(Stage::as_str).collect::<Vec<_>>()) |
| 10402 | + } else { |
| 10403 | + "none".to_string() |
| 10404 | + }; |
| 10405 | + let error_stage = artifacts |
| 10406 | + .armfortas_failure |
| 10407 | + .as_ref() |
| 10408 | + .map(|failure| failure.stage.as_str()) |
| 10409 | + .unwrap_or("none"); |
| 10410 | + let metadata = format!( |
| 10411 | + "primary_backend_kind: {}\nprimary_backend_mode: {}\nprimary_backend_detail: {}\ncaptured_stages: {}\nerror_stage: {}\n", |
| 10412 | + primary_backend_kind, |
| 10413 | + primary_backend_mode, |
| 10414 | + primary_backend_detail, |
| 10415 | + captured_stages, |
| 10416 | + error_stage |
| 10417 | + ); |
| 10418 | + fs::write(armfortas_root.join("metadata.txt"), metadata) |
| 10419 | + .map_err(|e| format!("cannot write armfortas bundle metadata: {}", e)) |
| 10420 | +} |
| 10421 | + |
| 10422 | +fn write_case_sources_bundle( |
| 10423 | + bundle_root: &Path, |
| 10424 | + case: &CaseSpec, |
| 10425 | + prepared: &PreparedInput, |
| 10426 | +) -> Result<(), String> { |
| 10427 | + if case.graph_files.is_empty() { |
| 10428 | + let source_text = fs::read_to_string(&case.source) |
| 10429 | + .map_err(|e| format!("cannot read case source '{}': {}", case.source.display(), e))?; |
| 10430 | + fs::write(bundle_root.join("source.f90"), source_text) |
| 10431 | + .map_err(|e| format!("cannot write bundle source copy: {}", e))?; |
| 10432 | + return Ok(()); |
| 10433 | + } |
| 10434 | + |
| 10435 | + let generated_source = prepared.generated_source.as_ref().ok_or_else(|| { |
| 10436 | + format!( |
| 10437 | + "graph case '{}' was missing a generated compiler source", |
| 10438 | + case.name |
| 10439 | + ) |
| 10440 | + })?; |
| 10441 | + let generated_text = fs::read_to_string(generated_source).map_err(|e| { |
| 10442 | + format!( |
| 10443 | + "cannot read generated graph source '{}': {}", |
| 10444 | + generated_source.display(), |
| 10445 | + e |
| 10446 | + ) |
| 10447 | + })?; |
| 10448 | + fs::write(bundle_root.join("source.f90"), generated_text) |
| 10449 | + .map_err(|e| format!("cannot write generated bundle source copy: {}", e))?; |
| 10450 | + |
| 10451 | + let sources_root = bundle_root.join("sources"); |
| 10452 | + fs::create_dir_all(&sources_root) |
| 10453 | + .map_err(|e| format!("cannot create bundle sources dir: {}", e))?; |
| 10454 | + for (index, file) in case.graph_files.iter().enumerate() { |
| 10455 | + let text = fs::read_to_string(file) |
| 10456 | + .map_err(|e| format!("cannot read graph source '{}': {}", file.display(), e))?; |
| 10457 | + let file_name = file |
| 10458 | + .file_name() |
| 10459 | + .and_then(|name| name.to_str()) |
| 10460 | + .unwrap_or("source.f90"); |
| 10461 | + let target = sources_root.join(format!("{:02}_{}", index, file_name)); |
| 10462 | + fs::write(target, text).map_err(|e| format!("cannot write bundle graph source: {}", e))?; |
| 10463 | + } |
| 10464 | + |
| 10465 | + Ok(()) |
| 10466 | +} |
| 10467 | + |
| 10468 | +fn write_capture_result(root: &Path, result: &CaptureResult) -> Result<(), String> { |
| 10469 | + for (stage, captured) in &result.stages { |
| 10470 | + match captured { |
| 10471 | + CapturedStage::Text(text) => { |
| 10472 | + fs::write(root.join(format!("{}.txt", stage.as_str())), text).map_err(|e| { |
| 10473 | + format!("cannot write '{}' stage bundle: {}", stage.as_str(), e) |
| 10474 | + })?; |
| 10475 | + } |
| 10476 | + CapturedStage::Run(run) => { |
| 10477 | + fs::write(root.join("run.stdout.txt"), &run.stdout) |
| 10478 | + .map_err(|e| format!("cannot write run stdout bundle: {}", e))?; |
| 10479 | + fs::write(root.join("run.stderr.txt"), &run.stderr) |
| 10480 | + .map_err(|e| format!("cannot write run stderr bundle: {}", e))?; |
| 10481 | + fs::write( |
| 10482 | + root.join("run.exit_code.txt"), |
| 10483 | + format!("{}\n", run.exit_code), |
| 10484 | + ) |
| 10485 | + .map_err(|e| format!("cannot write run exit-code bundle: {}", e))?; |
| 10486 | + } |
| 10487 | + } |
| 10488 | + } |
| 10489 | + Ok(()) |
| 10490 | +} |
| 10491 | + |
| 10492 | +fn write_armfortas_observation_bundle( |
| 10493 | + armfortas_root: &Path, |
| 10494 | + prepared: &PreparedInput, |
| 10495 | + artifacts: &ExecutionArtifacts, |
| 10496 | +) -> Result<(), String> { |
| 10497 | + let observed = match observed_program_for_armfortas_bundle(prepared, artifacts) { |
| 10498 | + Some(observed) => observed, |
| 10499 | + None => return Ok(()), |
| 10500 | + }; |
| 10501 | + let render_config = IntrospectionRenderConfig { |
| 10502 | + summary_only: false, |
| 10503 | + max_artifact_lines: None, |
| 10504 | + }; |
| 10505 | + fs::write( |
| 10506 | + armfortas_root.join("observation.txt"), |
| 10507 | + render_introspection_text(&observed, render_config), |
| 10508 | + ) |
| 10509 | + .map_err(|e| format!("cannot write armfortas observation text bundle: {}", e))?; |
| 10510 | + fs::write( |
| 10511 | + armfortas_root.join("observation.json"), |
| 10512 | + render_introspection_json(&observed), |
| 10513 | + ) |
| 10514 | + .map_err(|e| format!("cannot write armfortas observation json bundle: {}", e))?; |
| 10515 | + fs::write( |
| 10516 | + armfortas_root.join("observation.md"), |
| 10517 | + render_introspection_markdown(&observed, render_config), |
| 10518 | + ) |
| 10519 | + .map_err(|e| format!("cannot write armfortas observation markdown bundle: {}", e))?; |
| 10520 | + Ok(()) |
| 10521 | +} |
| 10522 | + |
| 10523 | +fn observed_program_for_armfortas_bundle( |
| 10524 | + prepared: &PreparedInput, |
| 10525 | + artifacts: &ExecutionArtifacts, |
| 10526 | +) -> Option<ObservedProgram> { |
| 10527 | + if let Some(observed) = &artifacts.armfortas_observation { |
| 10528 | + Some(observed.clone()) |
| 10529 | + } else if let Some(result) = &artifacts.armfortas { |
| 10530 | + Some(observed_program_from_armfortas_capture( |
| 10531 | + &prepared.compiler_source, |
| 10532 | + result.opt_level, |
| 10533 | + bundle_artifacts_for_capture_result(result), |
| 10534 | + result, |
| 10535 | + None, |
| 10536 | + )) |
| 10537 | + } else if let Some(failure) = &artifacts.armfortas_failure { |
| 10538 | + let partial = failure.partial_result(); |
| 10539 | + Some(observed_program_from_armfortas_capture( |
| 10540 | + &prepared.compiler_source, |
| 10541 | + failure.opt_level, |
| 10542 | + bundle_artifacts_for_capture_failure(failure), |
| 10543 | + &partial, |
| 10544 | + Some(failure), |
| 10545 | + )) |
| 10546 | + } else { |
| 10547 | + None |
| 10548 | + } |
| 10549 | +} |
| 10550 | + |
| 10551 | +fn bundle_artifacts_for_capture_result(result: &CaptureResult) -> BTreeSet<ArtifactKey> { |
| 10552 | + bundle_artifacts_for_stages(&result.stages) |
| 10553 | +} |
| 10554 | + |
| 10555 | +fn bundle_artifacts_for_capture_failure(failure: &CaptureFailure) -> BTreeSet<ArtifactKey> { |
| 10556 | + let mut requested = bundle_artifacts_for_stages(&failure.stages); |
| 10557 | + requested.insert(ArtifactKey::Diagnostics); |
| 10558 | + requested |
| 10559 | +} |
| 10560 | + |
| 10561 | +fn bundle_artifacts_for_stages(stages: &BTreeMap<Stage, CapturedStage>) -> BTreeSet<ArtifactKey> { |
| 10562 | + let mut requested = BTreeSet::new(); |
| 10563 | + for (stage, captured) in stages { |
| 10564 | + match (stage, captured) { |
| 10565 | + (Stage::Asm, CapturedStage::Text(_)) => { |
| 10566 | + requested.insert(ArtifactKey::Asm); |
| 10567 | + } |
| 10568 | + (Stage::Obj, CapturedStage::Text(_)) => { |
| 10569 | + requested.insert(ArtifactKey::Obj); |
| 10570 | + } |
| 10571 | + (Stage::Run, CapturedStage::Run(_)) => { |
| 10572 | + requested.insert(ArtifactKey::Runtime); |
| 10573 | + } |
| 10574 | + (stage, CapturedStage::Text(_)) => { |
| 10575 | + requested.insert(ArtifactKey::Extra(format!("armfortas.{}", stage.as_str()))); |
| 10576 | + } |
| 10577 | + _ => {} |
| 10578 | + } |
| 10579 | + } |
| 10580 | + requested |
| 10581 | +} |
| 10582 | + |
| 10583 | +fn reference_observations_for_bundle( |
| 10584 | + program: &Path, |
| 10585 | + opt_level: OptLevel, |
| 10586 | + artifacts: &ExecutionArtifacts, |
| 10587 | +) -> Vec<ObservedProgram> { |
| 10588 | + if artifacts.reference_observations.len() == artifacts.references.len() { |
| 10589 | + artifacts.reference_observations.clone() |
| 10590 | + } else { |
| 10591 | + artifacts |
| 10592 | + .references |
| 10593 | + .iter() |
| 10594 | + .map(|reference| { |
| 10595 | + observed_program_from_reference_result( |
| 10596 | + program, |
| 10597 | + opt_level, |
| 10598 | + default_differential_artifacts(), |
| 10599 | + reference, |
| 10600 | + ) |
| 10601 | + }) |
| 10602 | + .collect() |
| 10603 | + } |
| 10604 | +} |
| 10605 | + |
| 10606 | +fn write_reference_summary_bundle( |
| 10607 | + refs_root: &Path, |
| 10608 | + references: &[ReferenceResult], |
| 10609 | + observations: &[ObservedProgram], |
| 10610 | +) -> Result<(), String> { |
| 10611 | + let summary = render_reference_bundle_summary(references, observations); |
| 10612 | + fs::write(refs_root.join("summary.txt"), summary) |
| 10613 | + .map_err(|e| format!("cannot write reference summary bundle: {}", e)) |
| 10614 | +} |
| 10615 | + |
| 10616 | +fn render_reference_bundle_summary( |
| 10617 | + references: &[ReferenceResult], |
| 10618 | + observations: &[ObservedProgram], |
| 10619 | +) -> String { |
| 10620 | + let mut lines = vec![ |
| 10621 | + format!("reference_count: {}", references.len()), |
| 10622 | + format!( |
| 10623 | + "compilers: {}", |
| 10624 | + if references.is_empty() { |
| 10625 | + "none".to_string() |
| 10626 | + } else { |
| 10627 | + references |
| 10628 | + .iter() |
| 10629 | + .map(|reference| reference.compiler.as_str()) |
| 10630 | + .collect::<Vec<_>>() |
| 10631 | + .join(", ") |
| 10632 | + } |
| 10633 | + ), |
| 10634 | + ]; |
| 10635 | + |
| 10636 | + for (reference, observed) in references.iter().zip(observations.iter()) { |
| 10637 | + let observation = &observed.observation; |
| 10638 | + lines.push(String::new()); |
| 10639 | + lines.push(format!("compiler: {}", reference.compiler.as_str())); |
| 10640 | + lines.push(format!("status: {}", introspection_status(observation))); |
| 10641 | + lines.push(format!( |
| 10642 | + "compile_exit_code: {}", |
| 10643 | + observation.compile_exit_code |
| 10644 | + )); |
| 10645 | + lines.push(format!("command: {}", reference.compile_command)); |
| 10646 | + lines.push(format!( |
| 10647 | + "generic_artifacts: {}", |
| 10648 | + join_or_none_from_strings( |
| 10649 | + &observation_generic_artifacts(observation) |
| 10650 | + .into_iter() |
| 10651 | + .map(|(name, _)| name) |
| 10652 | + .collect::<Vec<_>>() |
| 10653 | + ) |
| 10654 | + )); |
| 10655 | + lines.push(format!( |
| 10656 | + "adapter_extras: {}", |
| 10657 | + format_adapter_extra_summary(&observation_adapter_extras(observation)) |
| 10658 | + )); |
| 10659 | + } |
| 10660 | + |
| 10661 | + lines.join("\n") + "\n" |
| 10662 | +} |
| 10663 | + |
| 10664 | +fn write_reference_bundle( |
| 10665 | + root: &Path, |
| 10666 | + program: &Path, |
| 10667 | + opt_level: OptLevel, |
| 10668 | + reference: &ReferenceResult, |
| 10669 | + observed: Option<&ObservedProgram>, |
| 10670 | +) -> Result<(), String> { |
| 10671 | + let ref_root = root.join(sanitize_component(reference.compiler.as_str())); |
| 10672 | + fs::create_dir_all(&ref_root) |
| 10673 | + .map_err(|e| format!("cannot create reference bundle dir: {}", e))?; |
| 10674 | + fs::write(ref_root.join("command.txt"), &reference.compile_command) |
| 10675 | + .map_err(|e| format!("cannot write reference command bundle: {}", e))?; |
| 10676 | + fs::write( |
| 10677 | + ref_root.join("compile.exit_code.txt"), |
| 10678 | + format!("{}\n", reference.compile_exit_code), |
| 10679 | + ) |
| 10680 | + .map_err(|e| format!("cannot write reference compile exit-code bundle: {}", e))?; |
| 10681 | + fs::write( |
| 10682 | + ref_root.join("compile.stdout.txt"), |
| 10683 | + &reference.compile_stdout, |
| 10684 | + ) |
| 10685 | + .map_err(|e| format!("cannot write reference compile stdout bundle: {}", e))?; |
| 10686 | + fs::write( |
| 10687 | + ref_root.join("compile.stderr.txt"), |
| 10688 | + &reference.compile_stderr, |
| 10689 | + ) |
| 10690 | + .map_err(|e| format!("cannot write reference compile stderr bundle: {}", e))?; |
| 10691 | + if let Some(run) = &reference.run { |
| 10692 | + fs::write(ref_root.join("run.stdout.txt"), &run.stdout) |
| 10693 | + .map_err(|e| format!("cannot write reference run stdout bundle: {}", e))?; |
| 10694 | + fs::write(ref_root.join("run.stderr.txt"), &run.stderr) |
| 10695 | + .map_err(|e| format!("cannot write reference run stderr bundle: {}", e))?; |
| 10696 | + fs::write( |
| 10697 | + ref_root.join("run.exit_code.txt"), |
| 10698 | + format!("{}\n", run.exit_code), |
| 10699 | + ) |
| 10700 | + .map_err(|e| format!("cannot write reference run exit-code bundle: {}", e))?; |
| 10701 | + } |
| 10702 | + if let Some(err) = &reference.run_error { |
| 10703 | + fs::write(ref_root.join("run.error.txt"), err) |
| 10704 | + .map_err(|e| format!("cannot write reference run error bundle: {}", e))?; |
| 10705 | + } |
| 10706 | + write_reference_observation_bundle(&ref_root, program, opt_level, reference, observed)?; |
| 10707 | + Ok(()) |
| 10708 | +} |
| 10709 | + |
| 10710 | +fn write_reference_observation_bundle( |
| 10711 | + ref_root: &Path, |
| 10712 | + program: &Path, |
| 10713 | + opt_level: OptLevel, |
| 10714 | + reference: &ReferenceResult, |
| 10715 | + observed: Option<&ObservedProgram>, |
| 10716 | +) -> Result<(), String> { |
| 10717 | + let observed = observed.cloned().unwrap_or_else(|| { |
| 10718 | + observed_program_from_reference_result( |
| 10719 | + program, |
| 10720 | + opt_level, |
| 10721 | + default_differential_artifacts(), |
| 10722 | + reference, |
| 10723 | + ) |
| 10724 | + }); |
| 10725 | + let render_config = IntrospectionRenderConfig { |
| 10726 | + summary_only: false, |
| 10727 | + max_artifact_lines: None, |
| 10728 | + }; |
| 10729 | + fs::write( |
| 10730 | + ref_root.join("observation.txt"), |
| 10731 | + render_introspection_text(&observed, render_config), |
| 10732 | + ) |
| 10733 | + .map_err(|e| format!("cannot write reference observation text bundle: {}", e))?; |
| 10734 | + fs::write( |
| 10735 | + ref_root.join("observation.json"), |
| 10736 | + render_introspection_json(&observed), |
| 10737 | + ) |
| 10738 | + .map_err(|e| format!("cannot write reference observation json bundle: {}", e))?; |
| 10739 | + fs::write( |
| 10740 | + ref_root.join("observation.md"), |
| 10741 | + render_introspection_markdown(&observed, render_config), |
| 10742 | + ) |
| 10743 | + .map_err(|e| format!("cannot write reference observation markdown bundle: {}", e))?; |
| 10744 | + Ok(()) |
| 10745 | +} |
| 10746 | + |
| 10747 | +fn render_consistency_bundle_summary(issues: &[ConsistencyIssue]) -> String { |
| 10748 | + let mut rollups = BTreeMap::new(); |
| 10749 | + for issue in issues { |
| 10750 | + rollups |
| 10751 | + .entry(issue.check) |
| 10752 | + .or_insert_with(ConsistencyRollup::default) |
| 10753 | + .record(&issue.observation()); |
| 10754 | + } |
| 10755 | + |
| 10756 | + let mut aggregate = ConsistencyRollup::default(); |
| 10757 | + for issue in issues { |
| 10758 | + aggregate.record(&issue.observation()); |
| 10759 | + } |
| 10760 | + |
| 10761 | + let checks = if rollups.is_empty() { |
| 10762 | + "none".to_string() |
| 10763 | + } else { |
| 10764 | + rollups |
| 10765 | + .keys() |
| 10766 | + .map(ConsistencyCheck::as_str) |
| 10767 | + .collect::<Vec<_>>() |
| 10768 | + .join(", ") |
| 10769 | + }; |
| 10770 | + |
| 10771 | + let mut lines = vec![ |
| 10772 | + format!("issue_count: {}", issues.len()), |
| 10773 | + format!("checks: {}", checks), |
| 10774 | + ]; |
| 10775 | + |
| 10776 | + if !aggregate.repeat_counts.is_empty() { |
| 10777 | + lines.push(format!( |
| 10778 | + "repeat_counts: {}", |
| 10779 | + join_usize_set(&aggregate.repeat_counts) |
| 10780 | + )); |
| 10781 | + } |
| 10782 | + if !aggregate.unique_variant_counts.is_empty() { |
| 10783 | + lines.push(format!( |
| 10784 | + "unique_variants: {}", |
| 10785 | + join_usize_set(&aggregate.unique_variant_counts) |
| 10786 | + )); |
| 10787 | + } |
| 10788 | + if !aggregate.varying_components.is_empty() { |
| 10789 | + lines.push(format!( |
| 10790 | + "varying_components: {}", |
| 10791 | + join_string_set(&aggregate.varying_components) |
| 10792 | + )); |
| 10793 | + } |
| 10794 | + if !aggregate.stable_components.is_empty() { |
| 10795 | + lines.push(format!( |
| 10796 | + "stable_components: {}", |
| 10797 | + join_string_set(&aggregate.stable_components) |
| 10798 | + )); |
| 10799 | + } |
| 10800 | + |
| 10801 | + if !rollups.is_empty() { |
| 10802 | + lines.push(String::new()); |
| 10803 | + lines.push("per_check:".to_string()); |
| 10804 | + for (check, rollup) in rollups { |
| 10805 | + lines.push(format!( |
| 10806 | + " {}: {}", |
| 10807 | + check.as_str(), |
| 10808 | + render_consistency_rollup(&rollup) |
| 10809 | + )); |
| 10810 | + } |
| 10811 | + } |
| 10812 | + |
| 10813 | + lines.push(String::new()); |
| 10814 | + for issue in issues { |
| 10815 | + lines.push(format!("check: {}", issue.check.as_str())); |
| 10816 | + lines.push(format!("summary: {}", issue.summary)); |
| 10817 | + if let Some(repeat_count) = issue.repeat_count { |
| 10818 | + lines.push(format!("repeat_count: {}", repeat_count)); |
| 10819 | + } |
| 10820 | + if let Some(unique_variant_count) = issue.unique_variant_count { |
| 10821 | + lines.push(format!("unique_variants: {}", unique_variant_count)); |
| 10822 | + } |
| 10823 | + if !issue.varying_components.is_empty() { |
| 10824 | + lines.push(format!( |
| 10825 | + "varying_components: {}", |
| 10826 | + join_or_none_from_strings(&issue.varying_components) |
| 10827 | + )); |
| 10828 | + } |
| 10829 | + if !issue.stable_components.is_empty() { |
| 10830 | + lines.push(format!( |
| 10831 | + "stable_components: {}", |
| 10832 | + join_or_none_from_strings(&issue.stable_components) |
| 10833 | + )); |
| 10834 | + } |
| 10835 | + lines.push(format!( |
| 10836 | + "artifacts: {}", |
| 10837 | + sanitize_component(issue.check.as_str()) |
| 10838 | + )); |
| 10839 | + lines.push(String::new()); |
| 10840 | + } |
| 10841 | + |
| 10842 | + lines.join("\n") |
| 10843 | +} |
| 10844 | + |
| 10845 | +fn write_consistency_bundle(root: &Path, issues: &[ConsistencyIssue]) -> Result<(), String> { |
| 10846 | + let consistency_root = root.join("consistency"); |
| 10847 | + fs::create_dir_all(&consistency_root) |
| 10848 | + .map_err(|e| format!("cannot create consistency bundle dir: {}", e))?; |
| 10849 | + |
| 10850 | + let summary = render_consistency_bundle_summary(issues); |
| 10851 | + fs::write(consistency_root.join("summary.txt"), summary) |
| 10852 | + .map_err(|e| format!("cannot write consistency summary bundle: {}", e))?; |
| 10853 | + |
| 10854 | + for issue in issues { |
| 10855 | + let issue_root = consistency_root.join(sanitize_component(issue.check.as_str())); |
| 10856 | + fs::create_dir_all(&issue_root) |
| 10857 | + .map_err(|e| format!("cannot create consistency issue bundle dir: {}", e))?; |
| 10858 | + fs::write(issue_root.join("summary.txt"), &issue.summary) |
| 10859 | + .map_err(|e| format!("cannot write consistency issue summary bundle: {}", e))?; |
| 10860 | + fs::write(issue_root.join("detail.txt"), &issue.detail) |
| 10861 | + .map_err(|e| format!("cannot write consistency issue detail bundle: {}", e))?; |
| 10862 | + let runs_root = issue_root.join("artifacts"); |
| 10863 | + copy_directory_recursive(&issue.temp_root, &runs_root)?; |
| 10864 | + } |
| 10865 | + |
| 10866 | + Ok(()) |
| 10867 | +} |
| 10868 | + |
| 10869 | +fn copy_directory_recursive(source: &Path, destination: &Path) -> Result<(), String> { |
| 10870 | + fs::create_dir_all(destination).map_err(|e| { |
| 10871 | + format!( |
| 10872 | + "cannot create copied artifact dir '{}': {}", |
| 10873 | + destination.display(), |
| 10874 | + e |
| 10875 | + ) |
| 10876 | + })?; |
| 10877 | + |
| 10878 | + for entry in fs::read_dir(source) |
| 10879 | + .map_err(|e| format!("cannot read artifact dir '{}': {}", source.display(), e))? |
| 10880 | + { |
| 10881 | + let entry = entry |
| 10882 | + .map_err(|e| format!("cannot read artifact entry '{}': {}", source.display(), e))?; |
| 10883 | + let source_path = entry.path(); |
| 10884 | + let destination_path = destination.join(entry.file_name()); |
| 10885 | + let file_type = entry.file_type().map_err(|e| { |
| 10886 | + format!( |
| 10887 | + "cannot read artifact type '{}': {}", |
| 10888 | + source_path.display(), |
| 10889 | + e |
| 10890 | + ) |
| 10891 | + })?; |
| 10892 | + if file_type.is_dir() { |
| 10893 | + copy_directory_recursive(&source_path, &destination_path)?; |
| 10894 | + } else { |
| 10895 | + fs::copy(&source_path, &destination_path).map_err(|e| { |
| 10896 | + format!( |
| 10897 | + "cannot copy artifact '{}' to '{}': {}", |
| 10898 | + source_path.display(), |
| 10899 | + destination_path.display(), |
| 10900 | + e |
| 10901 | + ) |
| 10902 | + })?; |
| 10903 | + } |
| 10904 | + } |
| 10905 | + |
| 10906 | + Ok(()) |
| 10907 | +} |
| 10908 | + |
| 10909 | +fn render_command(binary: &str, args: &[String]) -> String { |
| 10910 | + let mut rendered = vec![quote_arg(binary)]; |
| 10911 | + rendered.extend(args.iter().map(|arg| quote_arg(arg))); |
| 10912 | + rendered.join(" ") |
| 10913 | +} |
| 10914 | + |
| 10915 | +fn quote_arg(arg: &str) -> String { |
| 10916 | + if arg |
| 10917 | + .chars() |
| 10918 | + .all(|ch| ch.is_ascii_alphanumeric() || "-_./".contains(ch)) |
| 10919 | + { |
| 10920 | + arg.to_string() |
| 10921 | + } else { |
| 10922 | + format!("{:?}", arg) |
| 10923 | + } |
| 10924 | +} |
| 10925 | + |
| 10926 | +fn sanitize_component(value: &str) -> String { |
| 10927 | + let mut out = String::new(); |
| 10928 | + for ch in value.chars() { |
| 10929 | + if ch.is_ascii_alphanumeric() { |
| 10930 | + out.push(ch.to_ascii_lowercase()); |
| 10931 | + } else { |
| 10932 | + out.push('_'); |
| 10933 | + } |
| 10934 | + } |
| 10935 | + while out.contains("__") { |
| 10936 | + out = out.replace("__", "_"); |
| 10937 | + } |
| 10938 | + out.trim_matches('_').to_string() |
| 10939 | +} |
| 10940 | + |
| 10941 | +fn next_report_temp_root(compiler: ReferenceCompiler, opt_level: OptLevel) -> PathBuf { |
| 10942 | + default_report_root().join(".tmp").join(format!( |
| 10943 | + "{}_{}_{}", |
| 10944 | + sanitize_component(compiler.as_str()), |
| 10945 | + opt_level.as_str().to_ascii_lowercase(), |
| 10946 | + next_report_suffix(opt_level) |
| 10947 | + )) |
| 10948 | +} |
| 10949 | + |
| 10950 | +fn next_primary_cli_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10951 | + default_report_root().join(".tmp").join(format!( |
| 10952 | + "primary_cli_{}_{}", |
| 10953 | + opt_level.as_str().to_ascii_lowercase(), |
| 10954 | + next_report_suffix(opt_level) |
| 10955 | + )) |
| 10956 | +} |
| 10957 | + |
| 10958 | +fn next_consistency_temp_root(opt_level: OptLevel) -> PathBuf { |
| 10959 | + default_report_root().join(".tmp").join(format!( |
| 10960 | + "consistency_{}_{}", |
| 10961 | + opt_level.as_str().to_ascii_lowercase(), |
| 10962 | + next_report_suffix(opt_level) |
| 10963 | + )) |
| 10964 | +} |
| 10965 | + |
| 10966 | +fn next_report_suffix(opt_level: OptLevel) -> String { |
| 10967 | + format!( |
| 10968 | + "{}-{}-{:04}", |
| 10969 | + opt_level.as_str().to_ascii_lowercase(), |
| 10970 | + std::process::id(), |
| 10971 | + REPORT_COUNTER.fetch_add(1, Ordering::Relaxed) |
| 10972 | + ) |
| 10973 | +} |
| 10974 | + |
| 10975 | +fn print_outcome(outcome: &Outcome) { |
| 10976 | + let label = format!( |
| 10977 | + "{}::{}[{}]", |
| 10978 | + outcome.suite, |
| 10979 | + outcome.case, |
| 10980 | + outcome.opt_level.as_str() |
| 10981 | + ); |
| 10982 | + match outcome.kind { |
| 10983 | + OutcomeKind::Pass => println!("PASS {}", label), |
| 10984 | + OutcomeKind::Fail => { |
| 10985 | + println!("FAIL {}", label); |
| 10986 | + if !outcome.detail.is_empty() { |
| 10987 | + println!("{}", outcome.detail); |
| 10988 | + } |
| 10989 | + } |
| 10990 | + OutcomeKind::Xfail => { |
| 10991 | + println!("XFAIL {}", label); |
| 10992 | + if !outcome.detail.is_empty() { |
| 10993 | + println!("{}", outcome.detail); |
| 10994 | + } |
| 10995 | + } |
| 10996 | + OutcomeKind::Xpass => { |
| 10997 | + println!("XPASS {}", label); |
| 10998 | + if !outcome.detail.is_empty() { |
| 10999 | + println!("{}", outcome.detail); |
| 11000 | + } |
| 11001 | + } |
| 11002 | + OutcomeKind::Future => { |
| 11003 | + println!("FUTURE {}", label); |
| 11004 | + if !outcome.detail.is_empty() { |
| 11005 | + println!("{}", outcome.detail); |
| 11006 | + } |
| 11007 | + } |
| 11008 | + } |
| 11009 | + if let Some(bundle) = &outcome.bundle { |
| 11010 | + println!("bundle: {}", bundle.display()); |
| 11011 | + } |
| 11012 | +} |
| 11013 | + |
| 11014 | +fn print_summary(summary: &Summary) { |
| 11015 | + println!(); |
| 11016 | + println!("{}", render_summary(summary)); |
| 11017 | +} |
| 11018 | + |
| 11019 | +#[derive(Debug, Clone)] |
| 11020 | +struct Check { |
| 11021 | + line_num: usize, |
| 11022 | + pattern: String, |
| 11023 | + negative: bool, |
| 11024 | + kind: &'static str, |
| 11025 | +} |
| 11026 | + |
| 11027 | +fn extract_checks(source: &str) -> Vec<Check> { |
| 11028 | + source |
| 11029 | + .lines() |
| 11030 | + .enumerate() |
| 11031 | + .filter_map(|(i, line)| { |
| 11032 | + let trimmed = line.trim(); |
| 11033 | + trimmed.strip_prefix("! CHECK:").map(|rest| Check { |
| 11034 | + line_num: i + 1, |
| 11035 | + pattern: rest.trim().to_string(), |
| 11036 | + negative: false, |
| 11037 | + kind: "CHECK", |
| 11038 | + }) |
| 11039 | + }) |
| 11040 | + .collect() |
| 11041 | +} |
| 11042 | + |
| 11043 | +fn extract_xfail_reason(source: &str) -> Option<String> { |
| 11044 | + source.lines().find_map(|line| { |
| 11045 | + line.trim() |
| 11046 | + .strip_prefix("! XFAIL:") |
| 11047 | + .map(|rest| rest.trim().to_string()) |
| 11048 | + }) |
| 11049 | +} |
| 11050 | + |
| 11051 | +fn extract_error_expected_patterns(source: &str) -> Vec<String> { |
| 11052 | + source |
| 11053 | + .lines() |
| 11054 | + .filter_map(|line| { |
| 11055 | + line.trim() |
| 11056 | + .strip_prefix("! ERROR_EXPECTED:") |
| 11057 | + .map(|rest| rest.trim().to_string()) |
| 11058 | + }) |
| 11059 | + .collect() |
| 11060 | +} |
| 11061 | + |
| 11062 | +fn extract_ir_checks(source: &str) -> Vec<Check> { |
| 11063 | + source |
| 11064 | + .lines() |
| 11065 | + .enumerate() |
| 11066 | + .filter_map(|(i, line)| { |
| 11067 | + let trimmed = line.trim(); |
| 11068 | + if let Some(rest) = trimmed.strip_prefix("! IR_CHECK:") { |
| 11069 | + Some(Check { |
| 11070 | + line_num: i + 1, |
| 11071 | + pattern: rest.trim().to_string(), |
| 11072 | + negative: false, |
| 11073 | + kind: "IR_CHECK", |
| 11074 | + }) |
| 11075 | + } else { |
| 11076 | + trimmed.strip_prefix("! IR_NOT:").map(|rest| Check { |
| 11077 | + line_num: i + 1, |
| 11078 | + pattern: rest.trim().to_string(), |
| 11079 | + negative: true, |
| 11080 | + kind: "IR_NOT", |
| 11081 | + }) |
| 11082 | + } |
| 11083 | + }) |
| 11084 | + .collect() |
| 11085 | +} |
| 11086 | + |
| 11087 | +fn match_checks(checks: &[Check], output: &str, case_name: &str) -> Result<(), String> { |
| 11088 | + let output_lines: Vec<&str> = output.lines().collect(); |
| 11089 | + let mut output_idx = 0; |
| 11090 | + |
| 11091 | + for check in checks { |
| 11092 | + if check.negative { |
| 11093 | + if output.contains(&check.pattern) { |
| 11094 | + return Err(format!( |
| 11095 | + "{}:{}: {} failed: substring '{}' appears in output\nfull output:\n{}", |
| 11096 | + case_name, check.line_num, check.kind, check.pattern, output |
| 11097 | + )); |
| 11098 | + } |
| 11099 | + continue; |
| 11100 | + } |
| 11101 | + |
| 11102 | + let mut found = false; |
| 11103 | + while output_idx < output_lines.len() { |
| 11104 | + if output_lines[output_idx].trim().contains(&check.pattern) { |
| 11105 | + found = true; |
| 11106 | + output_idx += 1; |
| 11107 | + break; |
| 11108 | + } |
| 11109 | + output_idx += 1; |
| 11110 | + } |
| 11111 | + if !found { |
| 11112 | + return Err(format!( |
| 11113 | + "{}:{}: {} failed: expected '{}' not found in remaining output\nfull output:\n{}", |
| 11114 | + case_name, check.line_num, check.kind, check.pattern, output |
| 11115 | + )); |
| 11116 | + } |
| 11117 | + } |
| 11118 | + |
| 11119 | + Ok(()) |
| 11120 | +} |
| 11121 | + |
| 11122 | +fn target_uses_ir_comment_checks(target: &Target) -> bool { |
| 11123 | + match target { |
| 11124 | + Target::Stage(Stage::Ir) => true, |
| 11125 | + Target::Artifact(ArtifactKey::Extra(name)) => name == "armfortas.ir", |
| 11126 | + _ => false, |
| 11127 | + } |
| 11128 | +} |
| 11129 | + |
| 11130 | +#[cfg(test)] |
| 11131 | +mod tests { |
| 11132 | + use super::*; |
| 11133 | + |
| 11134 | + struct DummyBackend { |
| 11135 | + mode: &'static str, |
| 11136 | + detail: &'static str, |
| 11137 | + } |
| 11138 | + |
| 11139 | + impl CaptureBackend for DummyBackend { |
| 11140 | + fn mode_name(&self) -> &'static str { |
| 11141 | + self.mode |
| 11142 | + } |
| 11143 | + |
| 11144 | + fn description(&self) -> &'static str { |
| 11145 | + self.detail |
| 11146 | + } |
| 11147 | + |
| 11148 | + fn capture(&self, request: &CaptureRequest) -> Result<CaptureResult, CaptureFailure> { |
| 11149 | + Err(CaptureFailure { |
| 11150 | + input: request.input.clone(), |
| 11151 | + opt_level: request.opt_level, |
| 11152 | + stage: FailureStage::Ir, |
| 11153 | + detail: self.detail.to_string(), |
| 11154 | + stages: BTreeMap::new(), |
| 11155 | + }) |
| 11156 | + } |
| 11157 | + } |
| 11158 | + |
| 11159 | + #[cfg(unix)] |
| 11160 | + fn bencch_repo_root() -> PathBuf { |
| 11161 | + PathBuf::from(env!("CARGO_MANIFEST_DIR")) |
| 11162 | + .parent() |
| 11163 | + .unwrap() |
| 11164 | + .to_path_buf() |
| 11165 | + } |
| 11166 | + |
| 11167 | + #[cfg(unix)] |
| 11168 | + fn fake_compiler_fixture(name: &str) -> PathBuf { |
| 11169 | + bencch_repo_root() |
| 11170 | + .join("fixtures") |
| 11171 | + .join("fake_compilers") |
| 11172 | + .join(name) |
| 11173 | + } |
| 11174 | + |
| 11175 | + #[cfg(unix)] |
| 11176 | + fn runtime_fixture(name: &str) -> PathBuf { |
| 11177 | + bencch_repo_root() |
| 11178 | + .join("fixtures") |
| 11179 | + .join("runtime") |
| 11180 | + .join(name) |
| 11181 | + } |
| 11182 | + |
| 11183 | + fn full_introspection_render_config() -> IntrospectionRenderConfig { |
| 11184 | + IntrospectionRenderConfig { |
| 11185 | + summary_only: false, |
| 11186 | + max_artifact_lines: None, |
| 11187 | + } |
| 11188 | + } |
| 11189 | + |
| 11190 | + #[cfg(unix)] |
| 11191 | + fn invalid_fixture(name: &str) -> PathBuf { |
| 11192 | + bencch_repo_root() |
| 11193 | + .join("fixtures") |
| 11194 | + .join("invalid") |
| 11195 | + .join(name) |
| 11196 | + } |
| 11197 | + |
| 11198 | + #[cfg(unix)] |
| 11199 | + fn ensure_fixture_executable(path: &Path) { |
| 11200 | + let mut perms = fs::metadata(path).unwrap().permissions(); |
| 11201 | + perms.set_mode(0o755); |
| 11202 | + fs::set_permissions(path, perms).unwrap(); |
| 11203 | + } |
| 11204 | + |
| 11205 | + #[cfg(unix)] |
| 11206 | + fn write_probe_script(path: &Path, banner: &str) { |
| 11207 | + fs::write(path, format!("#!/bin/sh\nprintf '%s\\n' {:?}\n", banner)).unwrap(); |
| 11208 | + let mut perms = fs::metadata(path).unwrap().permissions(); |
| 11209 | + perms.set_mode(0o755); |
| 11210 | + fs::set_permissions(path, perms).unwrap(); |
| 11211 | + } |
| 11212 | + |
| 11213 | + #[cfg(unix)] |
| 11214 | + fn command_is_available(name: &str) -> bool { |
| 11215 | + Command::new("which") |
| 11216 | + .arg(name) |
| 11217 | + .output() |
| 11218 | + .map(|output| output.status.success()) |
| 11219 | + .unwrap_or(false) |
| 11220 | + } |
| 11221 | + |
| 11222 | + #[cfg(unix)] |
| 11223 | + fn armfortas_smoke_binary() -> Option<PathBuf> { |
| 11224 | + if let Some(path) = std::env::var_os("BENCCH_ARMFORTAS_SMOKE_BIN") { |
| 11225 | + let path = PathBuf::from(path); |
| 11226 | + if path.is_file() { |
| 11227 | + return Some(path); |
| 11228 | + } |
| 11229 | + } |
| 11230 | + |
| 11231 | + let candidate = bencch_repo_root() |
| 11232 | + .parent()? |
| 11233 | + .join("target") |
| 11234 | + .join("debug") |
| 11235 | + .join("armfortas"); |
| 11236 | + if candidate.is_file() { |
| 11237 | + Some(candidate) |
| 11238 | + } else { |
| 11239 | + None |
| 11240 | + } |
| 11241 | + } |
| 11242 | + |
| 11243 | + #[cfg(unix)] |
| 11244 | + fn stable_runtime_compare_corpus() -> Vec<PathBuf> { |
| 11245 | + [ |
| 11246 | + "allocatable.f90", |
| 11247 | + "do_while.f90", |
| 11248 | + "exit_cycle.f90", |
| 11249 | + "nested_loops.f90", |
| 11250 | + "subroutine_call.f90", |
| 11251 | + "string_fixed.f90", |
| 11252 | + "if_else.f90", |
| 11253 | + "mixed_types.f90", |
| 11254 | + "select_case.f90", |
| 11255 | + "function_call.f90", |
| 11256 | + "real_function.f90", |
| 11257 | + "where_construct.f90", |
| 11258 | + ] |
| 11259 | + .into_iter() |
| 11260 | + .map(runtime_fixture) |
| 11261 | + .collect() |
| 11262 | + } |
| 11263 | + |
| 11264 | + fn stable_runtime_compare_opt_levels() -> Vec<OptLevel> { |
| 11265 | + vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 11266 | + } |
| 11267 | + use crate::compiler::test_support::{ |
| 11268 | + verify_module, BlockParam, FloatWidth, Function, Inst, InstKind, IntWidth, IrType, Module, |
| 11269 | + Position, Span, Terminator, ValueId, |
| 11270 | + }; |
| 11271 | + #[cfg(unix)] |
| 11272 | + use std::os::unix::fs::PermissionsExt; |
| 11273 | + |
| 11274 | + fn dummy_span() -> Span { |
| 11275 | + Span { |
| 11276 | + file_id: 0, |
| 11277 | + start: Position { line: 1, col: 1 }, |
| 11278 | + end: Position { line: 1, col: 1 }, |
| 11279 | + } |
| 11280 | + } |
| 11281 | + |
| 11282 | + #[test] |
| 11283 | + fn primary_backend_selection_uses_observable_backend_for_external_cases() { |
| 11284 | + let case = CaseSpec { |
| 11285 | + name: "runtime_case".into(), |
| 11286 | + source: PathBuf::from("demo.f90"), |
| 11287 | + graph_files: Vec::new(), |
| 11288 | + requested: BTreeSet::from([Stage::Run]), |
| 11289 | + generic_introspect: None, |
| 11290 | + generic_compare: None, |
| 11291 | + opt_levels: vec![OptLevel::O0], |
| 11292 | + repeat_count: 3, |
| 11293 | + reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 11294 | + consistency_checks: vec![ConsistencyCheck::CliRunReproducible], |
| 11295 | + expectations: vec![Expectation::Contains { |
| 11296 | + target: Target::RunStdout, |
| 11297 | + needle: "42".into(), |
| 11298 | + }], |
| 11299 | + status_rules: Vec::new(), |
| 11300 | + capability_policy: None, |
| 11301 | + }; |
| 11302 | + let requested = BTreeSet::from([Stage::Run]); |
| 11303 | + let external_tools = ToolchainConfig { |
| 11304 | + armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 11305 | + gfortran: "gfortran".into(), |
| 11306 | + flang_new: "flang-new".into(), |
| 11307 | + lfortran: "lfortran".into(), |
| 11308 | + ifort: "ifort".into(), |
| 11309 | + ifx: "ifx".into(), |
| 11310 | + nvfortran: "nvfortran".into(), |
| 11311 | + system_as: "as".into(), |
| 11312 | + otool: "otool".into(), |
| 11313 | + nm: "nm".into(), |
| 11314 | + }; |
| 11315 | + |
| 11316 | + assert_eq!( |
| 11317 | + primary_backend_kind_for_case(&case, &requested, &external_tools), |
| 11318 | + PrimaryCaptureBackendKind::Observable |
| 11319 | + ); |
| 11320 | + let selected = |
| 11321 | + select_primary_capture_backend(&case, &requested, OptLevel::O0, &external_tools); |
| 11322 | + assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11323 | + |
| 11324 | + let linked_tools = ToolchainConfig { |
| 11325 | + armfortas: ArmfortasCliAdapter::Linked, |
| 11326 | + ..external_tools.clone() |
| 11327 | + }; |
| 11328 | + assert_eq!( |
| 11329 | + primary_backend_kind_for_case(&case, &requested, &linked_tools), |
| 11330 | + PrimaryCaptureBackendKind::Full |
| 11331 | + ); |
| 11332 | + |
| 11333 | + let mut capture_check_case = case.clone(); |
| 11334 | + capture_check_case.consistency_checks = vec![ConsistencyCheck::CaptureRunVsCliRun]; |
| 11335 | + assert_eq!( |
| 11336 | + primary_backend_kind_for_case(&capture_check_case, &requested, &external_tools), |
| 11337 | + PrimaryCaptureBackendKind::Full |
| 11338 | + ); |
| 11339 | + |
| 11340 | + let mut failure_case = case.clone(); |
| 11341 | + failure_case.expectations.push(Expectation::FailContains { |
| 11342 | + stage: FailureStage::Run, |
| 11343 | + needle: "broken".into(), |
| 11344 | + }); |
| 11345 | + assert_eq!( |
| 11346 | + primary_backend_kind_for_case(&failure_case, &requested, &external_tools), |
| 11347 | + PrimaryCaptureBackendKind::Full |
| 11348 | + ); |
| 11349 | + |
| 11350 | + let richer_request = BTreeSet::from([Stage::Run, Stage::Asm]); |
| 11351 | + assert_eq!( |
| 11352 | + primary_backend_kind_for_case(&case, &richer_request, &external_tools), |
| 11353 | + PrimaryCaptureBackendKind::Observable |
| 11354 | + ); |
| 11355 | + |
| 11356 | + let asm_only_request = BTreeSet::from([Stage::Asm]); |
| 11357 | + assert_eq!( |
| 11358 | + primary_backend_kind_for_case(&case, &asm_only_request, &external_tools), |
| 11359 | + PrimaryCaptureBackendKind::Observable |
| 11360 | + ); |
| 11361 | + } |
| 11362 | + |
| 11363 | + #[test] |
| 11364 | + fn legacy_unavailable_backend_detail_is_explicit() { |
| 11365 | + let case = CaseSpec { |
| 11366 | + name: "frontend_case".into(), |
| 11367 | + source: PathBuf::from("frontend.f90"), |
| 11368 | + graph_files: Vec::new(), |
| 11369 | + requested: BTreeSet::from([Stage::Tokens]), |
| 11370 | + generic_introspect: None, |
| 11371 | + generic_compare: None, |
| 11372 | + opt_levels: vec![OptLevel::O0], |
| 11373 | + repeat_count: 1, |
| 11374 | + reference_compilers: Vec::new(), |
| 11375 | + consistency_checks: Vec::new(), |
| 11376 | + expectations: vec![Expectation::Contains { |
| 11377 | + target: Target::Stage(Stage::Tokens), |
| 11378 | + needle: "program".into(), |
| 11379 | + }], |
| 11380 | + status_rules: Vec::new(), |
| 11381 | + capability_policy: None, |
| 11382 | + }; |
| 11383 | + let backend = SelectedPrimaryBackend { |
| 11384 | + kind: PrimaryCaptureBackendKind::Full, |
| 11385 | + backend: Box::new(DummyBackend { |
| 11386 | + mode: "unavailable", |
| 11387 | + detail: "unavailable without linked-armfortas feature", |
| 11388 | + }), |
| 11389 | + }; |
| 11390 | + |
| 11391 | + let detail = legacy_unavailable_backend_detail(&case, &backend).unwrap(); |
| 11392 | + assert!(detail.contains("case requires linked armfortas capture")); |
| 11393 | + assert!(detail.contains("scripts/bootstrap-linked-armfortas.sh")); |
| 11394 | + } |
| 11395 | + |
| 11396 | + #[test] |
| 11397 | + fn legacy_cli_consistency_cases_use_generic_observation_path() { |
| 11398 | + let cli_only_case = CaseSpec { |
| 11399 | + name: "cli-consistency".into(), |
| 11400 | + source: PathBuf::from("demo.f90"), |
| 11401 | + graph_files: Vec::new(), |
| 11402 | + requested: BTreeSet::from([Stage::Run]), |
| 11403 | + generic_introspect: None, |
| 11404 | + generic_compare: None, |
| 11405 | + opt_levels: vec![OptLevel::O0], |
| 11406 | + repeat_count: 3, |
| 11407 | + reference_compilers: Vec::new(), |
| 11408 | + consistency_checks: vec![ |
| 11409 | + ConsistencyCheck::CliAsmReproducible, |
| 11410 | + ConsistencyCheck::CliRunReproducible, |
| 11411 | + ], |
| 11412 | + expectations: vec![Expectation::Contains { |
| 11413 | + target: Target::RunStdout, |
| 11414 | + needle: "42".into(), |
| 11415 | + }], |
| 11416 | + status_rules: Vec::new(), |
| 11417 | + capability_policy: None, |
| 11418 | + }; |
| 11419 | + assert!(legacy_case_uses_generic_consistency_checks(&cli_only_case)); |
| 11420 | + |
| 11421 | + let mixed_case = CaseSpec { |
| 11422 | + consistency_checks: vec![ |
| 11423 | + ConsistencyCheck::CliRunReproducible, |
| 11424 | + ConsistencyCheck::CaptureRunReproducible, |
| 11425 | + ], |
| 11426 | + ..cli_only_case.clone() |
| 11427 | + }; |
| 11428 | + assert!(!legacy_case_uses_generic_consistency_checks(&mixed_case)); |
| 11429 | + } |
| 11430 | + |
| 11431 | + #[test] |
| 11432 | + fn legacy_observable_cases_use_generic_observation_execution() { |
| 11433 | + let observable_case = CaseSpec { |
| 11434 | + name: "observable".into(), |
| 11435 | + source: PathBuf::from("demo.f90"), |
| 11436 | + graph_files: Vec::new(), |
| 11437 | + requested: BTreeSet::from([Stage::Run]), |
| 11438 | + generic_introspect: None, |
| 11439 | + generic_compare: None, |
| 11440 | + opt_levels: vec![OptLevel::O0], |
| 11441 | + repeat_count: 3, |
| 11442 | + reference_compilers: Vec::new(), |
| 11443 | + consistency_checks: Vec::new(), |
| 11444 | + expectations: vec![Expectation::Contains { |
| 11445 | + target: Target::RunStdout, |
| 11446 | + needle: "42".into(), |
| 11447 | + }], |
| 11448 | + status_rules: Vec::new(), |
| 11449 | + capability_policy: None, |
| 11450 | + }; |
| 11451 | + assert!(legacy_case_uses_generic_observation_execution( |
| 11452 | + &observable_case, |
| 11453 | + &observable_case.requested |
| 11454 | + )); |
| 11455 | + |
| 11456 | + let richer_case = CaseSpec { |
| 11457 | + requested: BTreeSet::from([Stage::Run, Stage::Ir]), |
| 11458 | + ..observable_case.clone() |
| 11459 | + }; |
| 11460 | + assert!(!legacy_case_uses_generic_observation_execution( |
| 11461 | + &richer_case, |
| 11462 | + &richer_case.requested |
| 11463 | + )); |
| 11464 | + |
| 11465 | + let failure_case = CaseSpec { |
| 11466 | + expectations: vec![Expectation::FailContains { |
| 11467 | + stage: FailureStage::Run, |
| 11468 | + needle: "boom".into(), |
| 11469 | + }], |
| 11470 | + ..observable_case |
| 11471 | + }; |
| 11472 | + assert!(!legacy_case_uses_generic_observation_execution( |
| 11473 | + &failure_case, |
| 11474 | + &failure_case.requested |
| 11475 | + )); |
| 11476 | + } |
| 11477 | + |
| 11478 | + #[cfg(unix)] |
| 11479 | + #[test] |
| 11480 | + fn external_cli_primary_execution_returns_observable_stages() { |
| 11481 | + let root = std::env::temp_dir().join("afs_tests_external_cli_primary"); |
| 11482 | + let _ = fs::remove_dir_all(&root); |
| 11483 | + fs::create_dir_all(&root).unwrap(); |
| 11484 | + |
| 11485 | + let source = root.join("demo.f90"); |
| 11486 | + fs::write(&source, "program demo\nprint *, 42\nend program\n").unwrap(); |
| 11487 | + |
| 11488 | + let compiler = root.join("fake-armfortas"); |
| 11489 | + fs::write( |
| 11490 | + &compiler, |
| 11491 | + "#!/bin/sh\nmode=bin\nout=\"\"\nwhile [ $# -gt 0 ]; do\n case \"$1\" in\n -S)\n mode=asm\n shift\n ;;\n -c)\n mode=obj\n shift\n ;;\n -o)\n out=\"$2\"\n shift 2\n ;;\n *)\n shift\n ;;\n esac\ndone\nif [ \"$mode\" = asm ]; then\n cat > \"$out\" <<'EOF'\n.globl _main\n_main:\n ret\nEOF\nelif [ \"$mode\" = obj ]; then\n printf 'fake object\\n' > \"$out\"\nelse\n cat > \"$out\" <<'EOF'\n#!/bin/sh\nprintf '42\\n'\nEOF\n chmod +x \"$out\"\nfi\n", |
| 11492 | + ) |
| 11493 | + .unwrap(); |
| 11494 | + let mut perms = fs::metadata(&compiler).unwrap().permissions(); |
| 11495 | + perms.set_mode(0o755); |
| 11496 | + fs::set_permissions(&compiler, perms).unwrap(); |
| 11497 | + |
| 11498 | + let case = CaseSpec { |
| 11499 | + name: "runtime_case".into(), |
| 11500 | + source: source.clone(), |
| 11501 | + graph_files: Vec::new(), |
| 11502 | + requested: BTreeSet::from([Stage::Asm, Stage::Run]), |
| 11503 | + generic_introspect: None, |
| 11504 | + generic_compare: None, |
| 11505 | + opt_levels: vec![OptLevel::O0], |
| 11506 | + repeat_count: 3, |
| 11507 | + reference_compilers: Vec::new(), |
| 11508 | + consistency_checks: Vec::new(), |
| 11509 | + expectations: vec![ |
| 11510 | + Expectation::Contains { |
| 11511 | + target: Target::Stage(Stage::Asm), |
| 11512 | + needle: ".globl _main".into(), |
| 11513 | + }, |
| 11514 | + Expectation::Contains { |
| 11515 | + target: Target::RunStdout, |
| 11516 | + needle: "42".into(), |
| 11517 | + }, |
| 11518 | + ], |
| 11519 | + status_rules: Vec::new(), |
| 11520 | + capability_policy: None, |
| 11521 | + }; |
| 11522 | + let prepared = PreparedInput { |
| 11523 | + compiler_source: source.clone(), |
| 11524 | + generated_source: None, |
| 11525 | + temp_root: None, |
| 11526 | + }; |
| 11527 | + let tools = ToolchainConfig { |
| 11528 | + armfortas: ArmfortasCliAdapter::External(compiler.display().to_string()), |
| 11529 | + gfortran: "gfortran".into(), |
| 11530 | + flang_new: "flang-new".into(), |
| 11531 | + lfortran: "lfortran".into(), |
| 11532 | + ifort: "ifort".into(), |
| 11533 | + ifx: "ifx".into(), |
| 11534 | + nvfortran: "nvfortran".into(), |
| 11535 | + system_as: "as".into(), |
| 11536 | + otool: "otool".into(), |
| 11537 | + nm: "nm".into(), |
| 11538 | + }; |
| 11539 | + let requested = BTreeSet::from([Stage::Asm, Stage::Run]); |
| 11540 | + let selected = select_primary_capture_backend(&case, &requested, OptLevel::O0, &tools); |
| 11541 | + assert_eq!(selected.kind, PrimaryCaptureBackendKind::Observable); |
| 11542 | + assert_eq!(selected.backend.mode_name(), "cli-observable"); |
| 11543 | + |
| 11544 | + let result = |
| 11545 | + execute_primary_armfortas(&prepared, OptLevel::O0, &requested, &selected).unwrap(); |
| 11546 | + let asm = capture_text_stage(&result, Stage::Asm).unwrap(); |
| 11547 | + let run = capture_run_stage(&result).unwrap(); |
| 11548 | + assert!(asm.contains(".globl _main")); |
| 11549 | + assert_eq!(run.exit_code, 0); |
| 11550 | + assert_eq!(run.stdout, "42\n"); |
| 11551 | + assert!(run.stderr.is_empty()); |
| 11552 | + assert_eq!(result.stages.len(), 2); |
| 11553 | + |
| 11554 | + let _ = fs::remove_dir_all(&root); |
| 11555 | + } |
| 11556 | + |
| 11557 | + #[cfg(unix)] |
| 11558 | + #[test] |
| 11559 | + fn compare_uses_generic_external_driver_observations() { |
| 11560 | + let compiler_a = fake_compiler_fixture("match_42_a.sh"); |
| 11561 | + let compiler_b = fake_compiler_fixture("runtime_41.sh"); |
| 11562 | + let source = runtime_fixture("mixed_types.f90"); |
| 11563 | + ensure_fixture_executable(&compiler_a); |
| 11564 | + ensure_fixture_executable(&compiler_b); |
| 11565 | + |
| 11566 | + let config = CompareConfig { |
| 11567 | + left: CompilerSpec::Binary(compiler_a.clone()), |
| 11568 | + right: CompilerSpec::Binary(compiler_b.clone()), |
| 11569 | + program: source.clone(), |
| 11570 | + opt_level: OptLevel::O0, |
| 11571 | + artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 11572 | + json_report: None, |
| 11573 | + markdown_report: None, |
| 11574 | + tools: ToolchainConfig::from_env(), |
| 11575 | + }; |
| 11576 | + |
| 11577 | + let result = run_compare(&config).unwrap(); |
| 11578 | + assert_eq!(result.left.provenance.backend_mode, "external-driver"); |
| 11579 | + assert_eq!(result.right.provenance.backend_mode, "external-driver"); |
| 11580 | + assert!(result |
| 11581 | + .differences |
| 11582 | + .iter() |
| 11583 | + .any(|difference| difference.artifact == "runtime")); |
| 11584 | + assert!(result |
| 11585 | + .differences |
| 11586 | + .iter() |
| 11587 | + .any(|difference| difference.artifact == "asm")); |
| 11588 | + let rendered = render_compare_text(&result); |
| 11589 | + assert!(rendered.contains("status: diff")); |
| 11590 | + assert!(rendered.contains("classification: mixed divergence")); |
| 11591 | + assert!(rendered.contains("difference_count: 2")); |
| 11592 | + } |
| 11593 | + |
| 11594 | + #[cfg(unix)] |
| 11595 | + #[test] |
| 11596 | + fn compare_fixture_compilers_report_compile_failures() { |
| 11597 | + let source = runtime_fixture("mixed_types.f90"); |
| 11598 | + let compiler_fail = fake_compiler_fixture("compile_fail.sh"); |
| 11599 | + let compiler_ok = fake_compiler_fixture("match_42_a.sh"); |
| 11600 | + ensure_fixture_executable(&compiler_fail); |
| 11601 | + ensure_fixture_executable(&compiler_ok); |
| 11602 | + |
| 11603 | + let config = CompareConfig { |
| 11604 | + left: CompilerSpec::Binary(compiler_fail), |
| 11605 | + right: CompilerSpec::Binary(compiler_ok), |
| 11606 | + program: source, |
| 11607 | + opt_level: OptLevel::O0, |
| 11608 | + artifacts: BTreeSet::new(), |
| 11609 | + json_report: None, |
| 11610 | + markdown_report: None, |
| 11611 | + tools: ToolchainConfig::from_env(), |
| 11612 | + }; |
| 11613 | + |
| 11614 | + let result = run_compare(&config).unwrap(); |
| 11615 | + assert_eq!(compare_status(&result), "diff"); |
| 11616 | + assert_eq!(compare_classification(&result), "compile divergence"); |
| 11617 | + assert!(result |
| 11618 | + .differences |
| 11619 | + .iter() |
| 11620 | + .any(|difference| difference.artifact == "compile-exit-code")); |
| 11621 | + let diagnostics = result |
| 11622 | + .differences |
| 11623 | + .iter() |
| 11624 | + .find(|difference| difference.artifact == "diagnostics") |
| 11625 | + .unwrap(); |
| 11626 | + assert!(diagnostics |
| 11627 | + .detail |
| 11628 | + .contains("fake compiler failure: missing lowering pass")); |
| 11629 | + } |
| 11630 | + |
| 11631 | + #[test] |
| 11632 | + fn compare_rejects_capability_mismatch_for_namespaced_artifacts() { |
| 11633 | + let config = CompareConfig { |
| 11634 | + left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11635 | + right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11636 | + program: runtime_fixture("if_else.f90"), |
| 11637 | + opt_level: OptLevel::O0, |
| 11638 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 11639 | + json_report: None, |
| 11640 | + markdown_report: None, |
| 11641 | + tools: ToolchainConfig::from_env(), |
| 11642 | + }; |
| 11643 | + |
| 11644 | + let err = run_compare(&config).unwrap_err(); |
| 11645 | + assert!(err.contains("compare request is not supported")); |
| 11646 | + assert!(err.contains("right:")); |
| 11647 | + assert!(err.contains("gfortran does not support requested artifacts")); |
| 11648 | + assert!(err.contains("armfortas.ir")); |
| 11649 | + } |
| 11650 | + |
| 11651 | + #[test] |
| 11652 | + fn compare_executable_artifact_uses_file_contents_not_paths() { |
| 11653 | + let root = std::env::temp_dir().join("bencch_compare_executable_paths"); |
| 11654 | + let _ = fs::remove_dir_all(&root); |
| 11655 | + fs::create_dir_all(&root).unwrap(); |
| 11656 | + |
| 11657 | + let left_exe = root.join("left.out"); |
| 11658 | + let right_exe = root.join("right.out"); |
| 11659 | + fs::write(&left_exe, b"same executable bytes").unwrap(); |
| 11660 | + fs::write(&right_exe, b"same executable bytes").unwrap(); |
| 11661 | + |
| 11662 | + let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11663 | + let left = CompilerObservation { |
| 11664 | + compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11665 | + program: PathBuf::from("demo.f90"), |
| 11666 | + opt_level: OptLevel::O0, |
| 11667 | + compile_exit_code: 0, |
| 11668 | + artifacts: BTreeMap::from([( |
| 11669 | + ArtifactKey::Executable, |
| 11670 | + ArtifactValue::Path(left_exe.clone()), |
| 11671 | + )]), |
| 11672 | + provenance: ObservationProvenance { |
| 11673 | + compiler_identity: "left".into(), |
| 11674 | + adapter_kind: "explicit-path".into(), |
| 11675 | + backend_mode: "external-driver".into(), |
| 11676 | + backend_detail: "left detail".into(), |
| 11677 | + artifacts_captured: vec!["executable".into()], |
| 11678 | + comparison_basis: None, |
| 11679 | + failure_stage: None, |
| 11680 | + }, |
| 11681 | + }; |
| 11682 | + let right = CompilerObservation { |
| 11683 | + compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11684 | + program: PathBuf::from("demo.f90"), |
| 11685 | + opt_level: OptLevel::O0, |
| 11686 | + compile_exit_code: 0, |
| 11687 | + artifacts: BTreeMap::from([( |
| 11688 | + ArtifactKey::Executable, |
| 11689 | + ArtifactValue::Path(right_exe.clone()), |
| 11690 | + )]), |
| 11691 | + provenance: ObservationProvenance { |
| 11692 | + compiler_identity: "right".into(), |
| 11693 | + adapter_kind: "explicit-path".into(), |
| 11694 | + backend_mode: "external-driver".into(), |
| 11695 | + backend_detail: "right detail".into(), |
| 11696 | + artifacts_captured: vec!["executable".into()], |
| 11697 | + comparison_basis: None, |
| 11698 | + failure_stage: None, |
| 11699 | + }, |
| 11700 | + }; |
| 11701 | + |
| 11702 | + let result = compare_observations(left, right, &requested); |
| 11703 | + assert!(result.differences.is_empty()); |
| 11704 | + |
| 11705 | + let _ = fs::remove_dir_all(&root); |
| 11706 | + } |
| 11707 | + |
| 11708 | + #[test] |
| 11709 | + fn compare_executable_artifact_reports_binary_difference() { |
| 11710 | + let root = std::env::temp_dir().join("bencch_compare_executable_bytes"); |
| 11711 | + let _ = fs::remove_dir_all(&root); |
| 11712 | + fs::create_dir_all(&root).unwrap(); |
| 11713 | + |
| 11714 | + let left_exe = root.join("left.out"); |
| 11715 | + let right_exe = root.join("right.out"); |
| 11716 | + fs::write(&left_exe, b"abc").unwrap(); |
| 11717 | + fs::write(&right_exe, b"axc").unwrap(); |
| 11718 | + |
| 11719 | + let requested = BTreeSet::from([ArtifactKey::Executable]); |
| 11720 | + let left = CompilerObservation { |
| 11721 | + compiler: CompilerSpec::Binary(PathBuf::from("/tmp/left-compiler")), |
| 11722 | + program: PathBuf::from("demo.f90"), |
| 11723 | + opt_level: OptLevel::O0, |
| 11724 | + compile_exit_code: 0, |
| 11725 | + artifacts: BTreeMap::from([( |
| 11726 | + ArtifactKey::Executable, |
| 11727 | + ArtifactValue::Path(left_exe.clone()), |
| 11728 | + )]), |
| 11729 | + provenance: ObservationProvenance { |
| 11730 | + compiler_identity: "left".into(), |
| 11731 | + adapter_kind: "explicit-path".into(), |
| 11732 | + backend_mode: "external-driver".into(), |
| 11733 | + backend_detail: "left detail".into(), |
| 11734 | + artifacts_captured: vec!["executable".into()], |
| 11735 | + comparison_basis: None, |
| 11736 | + failure_stage: None, |
| 11737 | + }, |
| 11738 | + }; |
| 11739 | + let right = CompilerObservation { |
| 11740 | + compiler: CompilerSpec::Binary(PathBuf::from("/tmp/right-compiler")), |
| 11741 | + program: PathBuf::from("demo.f90"), |
| 11742 | + opt_level: OptLevel::O0, |
| 11743 | + compile_exit_code: 0, |
| 11744 | + artifacts: BTreeMap::from([( |
| 11745 | + ArtifactKey::Executable, |
| 11746 | + ArtifactValue::Path(right_exe.clone()), |
| 11747 | + )]), |
| 11748 | + provenance: ObservationProvenance { |
| 11749 | + compiler_identity: "right".into(), |
| 11750 | + adapter_kind: "explicit-path".into(), |
| 11751 | + backend_mode: "external-driver".into(), |
| 11752 | + backend_detail: "right detail".into(), |
| 11753 | + artifacts_captured: vec!["executable".into()], |
| 11754 | + comparison_basis: None, |
| 11755 | + failure_stage: None, |
| 11756 | + }, |
| 11757 | + }; |
| 11758 | + |
| 11759 | + let result = compare_observations(left, right, &requested); |
| 11760 | + assert_eq!(result.differences.len(), 1); |
| 11761 | + assert_eq!(result.differences[0].artifact, "executable"); |
| 11762 | + assert!(result.differences[0] |
| 11763 | + .detail |
| 11764 | + .contains("first differing byte: 1")); |
| 11765 | + |
| 11766 | + let _ = fs::remove_dir_all(&root); |
| 11767 | + } |
| 11768 | + |
| 11769 | + #[cfg(unix)] |
| 11770 | + #[test] |
| 11771 | + fn compare_cli_with_fixture_compilers_writes_match_reports() { |
| 11772 | + let left = fake_compiler_fixture("match_42_a.sh"); |
| 11773 | + let right = fake_compiler_fixture("match_42_b.sh"); |
| 11774 | + let source = runtime_fixture("mixed_types.f90"); |
| 11775 | + ensure_fixture_executable(&left); |
| 11776 | + ensure_fixture_executable(&right); |
| 11777 | + |
| 11778 | + let root = std::env::temp_dir().join("bencch_compare_fixture_reports"); |
| 11779 | + let _ = fs::remove_dir_all(&root); |
| 11780 | + fs::create_dir_all(&root).unwrap(); |
| 11781 | + let json_report = root.join("compare.json"); |
| 11782 | + let markdown_report = root.join("compare.md"); |
| 11783 | + |
| 11784 | + let args = vec![ |
| 11785 | + "compare".to_string(), |
| 11786 | + left.display().to_string(), |
| 11787 | + right.display().to_string(), |
| 11788 | + "--program".to_string(), |
| 11789 | + source.display().to_string(), |
| 11790 | + "--artifact".to_string(), |
| 11791 | + "asm,obj".to_string(), |
| 11792 | + "--json-report".to_string(), |
| 11793 | + json_report.display().to_string(), |
| 11794 | + "--markdown-report".to_string(), |
| 11795 | + markdown_report.display().to_string(), |
| 11796 | + ]; |
| 11797 | + |
| 11798 | + let exit = run_cli_named("bencch", &args); |
| 11799 | + assert_eq!(exit, 0); |
| 11800 | + |
| 11801 | + let json = fs::read_to_string(&json_report).unwrap(); |
| 11802 | + assert!(json.contains("\"status\": \"match\"")); |
| 11803 | + assert!(json.contains("\"classification\": \"match\"")); |
| 11804 | + assert!(json.contains("\"difference_count\": 0")); |
| 11805 | + assert!(json.contains("\"changed_artifacts\": []")); |
| 11806 | + |
| 11807 | + let markdown = fs::read_to_string(&markdown_report).unwrap(); |
| 11808 | + assert!(markdown.contains("status: match")); |
| 11809 | + assert!(markdown.contains("classification: match")); |
| 11810 | + assert!(markdown.contains("difference_count: 0")); |
| 11811 | + assert!(markdown.contains("changed_artifacts: none")); |
| 11812 | + |
| 11813 | + let _ = fs::remove_dir_all(&root); |
| 11814 | + } |
| 11815 | + |
| 11816 | + #[cfg(unix)] |
| 11817 | + #[test] |
| 11818 | + fn compare_named_real_compilers_match_on_runtime_corpus() { |
| 11819 | + if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 11820 | + return; |
| 11821 | + } |
| 11822 | + |
| 11823 | + for opt_level in stable_runtime_compare_opt_levels() { |
| 11824 | + for program in stable_runtime_compare_corpus() { |
| 11825 | + let config = CompareConfig { |
| 11826 | + left: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11827 | + right: CompilerSpec::Named(NamedCompiler::FlangNew), |
| 11828 | + program, |
| 11829 | + opt_level, |
| 11830 | + artifacts: BTreeSet::new(), |
| 11831 | + json_report: None, |
| 11832 | + markdown_report: None, |
| 11833 | + tools: ToolchainConfig::from_env(), |
| 11834 | + }; |
| 11835 | + |
| 11836 | + let result = run_compare(&config).unwrap(); |
| 11837 | + assert_eq!(compare_status(&result), "match"); |
| 11838 | + assert_eq!(compare_classification(&result), "match"); |
| 11839 | + assert!(result.differences.is_empty()); |
| 11840 | + assert_eq!(result.left.provenance.adapter_kind, "named"); |
| 11841 | + assert_eq!(result.right.provenance.adapter_kind, "named"); |
| 11842 | + assert_eq!(result.left.opt_level, opt_level); |
| 11843 | + assert_eq!(result.right.opt_level, opt_level); |
| 11844 | + } |
| 11845 | + } |
| 11846 | + } |
| 11847 | + |
| 11848 | + #[cfg(unix)] |
| 11849 | + #[test] |
| 11850 | + fn compare_armfortas_and_gfortran_match_on_runtime_corpus_when_available() { |
| 11851 | + if !command_is_available("gfortran") { |
| 11852 | + return; |
| 11853 | + } |
| 11854 | + let Some(armfortas_bin) = armfortas_smoke_binary() else { |
| 11855 | + return; |
| 11856 | + }; |
| 11857 | + |
| 11858 | + let mut tools = ToolchainConfig::from_env(); |
| 11859 | + tools.armfortas = ArmfortasCliAdapter::External(armfortas_bin.display().to_string()); |
| 11860 | + |
| 11861 | + for opt_level in stable_runtime_compare_opt_levels() { |
| 11862 | + for program in stable_runtime_compare_corpus() { |
| 11863 | + let config = CompareConfig { |
| 11864 | + left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11865 | + right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 11866 | + program, |
| 11867 | + opt_level, |
| 11868 | + artifacts: BTreeSet::new(), |
| 11869 | + json_report: None, |
| 11870 | + markdown_report: None, |
| 11871 | + tools: tools.clone(), |
| 11872 | + }; |
| 11873 | + |
| 11874 | + let result = run_compare(&config).unwrap(); |
| 11875 | + assert_eq!(compare_status(&result), "match"); |
| 11876 | + assert_eq!(compare_classification(&result), "match"); |
| 11877 | + assert!(result.differences.is_empty()); |
| 11878 | + assert_eq!(result.left.provenance.backend_mode, "cli-observable"); |
| 11879 | + assert_eq!(result.left.opt_level, opt_level); |
| 11880 | + assert_eq!(result.right.opt_level, opt_level); |
| 11881 | + } |
| 11882 | + } |
| 11883 | + } |
| 11884 | + |
| 11885 | + #[cfg(unix)] |
| 11886 | + #[test] |
| 11887 | + fn introspect_armfortas_rich_artifacts_on_runtime_fixture() { |
| 11888 | + let config = IntrospectConfig { |
| 11889 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11890 | + program: runtime_fixture("if_else.f90"), |
| 11891 | + opt_level: OptLevel::O0, |
| 11892 | + artifacts: BTreeSet::from([ |
| 11893 | + ArtifactKey::Asm, |
| 11894 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 11895 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 11896 | + ]), |
| 11897 | + json_report: None, |
| 11898 | + markdown_report: None, |
| 11899 | + all_artifacts: false, |
| 11900 | + summary_only: false, |
| 11901 | + max_artifact_lines: None, |
| 11902 | + tools: ToolchainConfig::from_env(), |
| 11903 | + }; |
| 11904 | + |
| 11905 | + let observed = run_introspect(&config).unwrap(); |
| 11906 | + let observation = &observed.observation; |
| 11907 | + assert_eq!(observation.compile_exit_code, 0); |
| 11908 | + assert_eq!(observation.provenance.backend_mode, "linked"); |
| 11909 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 11910 | + assert!(observation |
| 11911 | + .artifacts |
| 11912 | + .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 11913 | + assert!(observation |
| 11914 | + .artifacts |
| 11915 | + .contains_key(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 11916 | + |
| 11917 | + let ir = match observation |
| 11918 | + .artifacts |
| 11919 | + .get(&ArtifactKey::Extra("armfortas.ir".into())) |
| 11920 | + .unwrap() |
| 11921 | + { |
| 11922 | + ArtifactValue::Text(text) => text, |
| 11923 | + other => panic!("expected text ir artifact, got {:?}", other), |
| 11924 | + }; |
| 11925 | + assert!(ir.contains("func") || ir.contains("module")); |
| 11926 | + |
| 11927 | + let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 11928 | + assert!(rendered.contains("Generic artifacts")); |
| 11929 | + assert!(rendered.contains("Adapter extras")); |
| 11930 | + assert!(rendered.contains("-- armfortas --")); |
| 11931 | + assert!(rendered.contains("== ir ==")); |
| 11932 | + } |
| 11933 | + |
| 11934 | + #[cfg(unix)] |
| 11935 | + #[test] |
| 11936 | + fn introspect_armfortas_all_artifacts_includes_stage_extras() { |
| 11937 | + let config = IntrospectConfig { |
| 11938 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11939 | + program: runtime_fixture("if_else.f90"), |
| 11940 | + opt_level: OptLevel::O0, |
| 11941 | + artifacts: BTreeSet::new(), |
| 11942 | + json_report: None, |
| 11943 | + markdown_report: None, |
| 11944 | + all_artifacts: true, |
| 11945 | + summary_only: false, |
| 11946 | + max_artifact_lines: None, |
| 11947 | + tools: ToolchainConfig::from_env(), |
| 11948 | + }; |
| 11949 | + |
| 11950 | + let observed = run_introspect(&config).unwrap(); |
| 11951 | + let observation = &observed.observation; |
| 11952 | + for artifact in [ |
| 11953 | + ArtifactKey::Asm, |
| 11954 | + ArtifactKey::Obj, |
| 11955 | + ArtifactKey::Runtime, |
| 11956 | + ArtifactKey::Extra("armfortas.preprocess".into()), |
| 11957 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 11958 | + ArtifactKey::Extra("armfortas.ast".into()), |
| 11959 | + ArtifactKey::Extra("armfortas.sema".into()), |
| 11960 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 11961 | + ArtifactKey::Extra("armfortas.optir".into()), |
| 11962 | + ArtifactKey::Extra("armfortas.mir".into()), |
| 11963 | + ArtifactKey::Extra("armfortas.regalloc".into()), |
| 11964 | + ] { |
| 11965 | + assert!( |
| 11966 | + observation.artifacts.contains_key(&artifact), |
| 11967 | + "missing artifact {}", |
| 11968 | + artifact.as_str() |
| 11969 | + ); |
| 11970 | + } |
| 11971 | + assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 11972 | + } |
| 11973 | + |
| 11974 | + #[cfg(unix)] |
| 11975 | + #[test] |
| 11976 | + fn introspect_armfortas_failure_reports_stage_and_partial_capture() { |
| 11977 | + let config = IntrospectConfig { |
| 11978 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 11979 | + program: invalid_fixture("parse_error.f90"), |
| 11980 | + opt_level: OptLevel::O0, |
| 11981 | + artifacts: BTreeSet::from([ |
| 11982 | + ArtifactKey::Asm, |
| 11983 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 11984 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 11985 | + ]), |
| 11986 | + json_report: None, |
| 11987 | + markdown_report: None, |
| 11988 | + all_artifacts: false, |
| 11989 | + summary_only: false, |
| 11990 | + max_artifact_lines: None, |
| 11991 | + tools: ToolchainConfig::from_env(), |
| 11992 | + }; |
| 11993 | + |
| 11994 | + let observed = run_introspect(&config).unwrap(); |
| 11995 | + let observation = &observed.observation; |
| 11996 | + assert_eq!(observation.compile_exit_code, 1); |
| 11997 | + assert_eq!( |
| 11998 | + observation.provenance.failure_stage.as_deref(), |
| 11999 | + Some("parser") |
| 12000 | + ); |
| 12001 | + assert!(observation |
| 12002 | + .artifacts |
| 12003 | + .contains_key(&ArtifactKey::Diagnostics)); |
| 12004 | + assert!(observation |
| 12005 | + .artifacts |
| 12006 | + .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 12007 | + assert!(missing_introspection_artifact_names(&observed).contains(&"asm".to_string())); |
| 12008 | + assert!( |
| 12009 | + missing_introspection_artifact_names(&observed).contains(&"armfortas.ir".to_string()) |
| 12010 | + ); |
| 12011 | + |
| 12012 | + let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 12013 | + assert!(rendered.contains("status: compile failed")); |
| 12014 | + assert!(rendered.contains("failure_stage: parser")); |
| 12015 | + assert!(rendered.contains("diagnostic_excerpt:")); |
| 12016 | + } |
| 12017 | + |
| 12018 | + #[cfg(unix)] |
| 12019 | + #[test] |
| 12020 | + fn introspect_named_external_compiler_reports_generic_artifacts_when_available() { |
| 12021 | + if !command_is_available("gfortran") { |
| 12022 | + return; |
| 12023 | + } |
| 12024 | + |
| 12025 | + let config = IntrospectConfig { |
| 12026 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12027 | + program: runtime_fixture("if_else.f90"), |
| 12028 | + opt_level: OptLevel::O0, |
| 12029 | + artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12030 | + json_report: None, |
| 12031 | + markdown_report: None, |
| 12032 | + all_artifacts: false, |
| 12033 | + summary_only: false, |
| 12034 | + max_artifact_lines: None, |
| 12035 | + tools: ToolchainConfig::from_env(), |
| 12036 | + }; |
| 12037 | + |
| 12038 | + let observed = run_introspect(&config).unwrap(); |
| 12039 | + let observation = &observed.observation; |
| 12040 | + assert_eq!(observation.compile_exit_code, 0); |
| 12041 | + assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12042 | + assert_eq!(observation.provenance.adapter_kind, "named"); |
| 12043 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 12044 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 12045 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 12046 | + assert!(observation_adapter_extras(observation).is_empty()); |
| 12047 | + assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 12048 | + } |
| 12049 | + |
| 12050 | + #[cfg(unix)] |
| 12051 | + #[test] |
| 12052 | + fn introspect_explicit_path_compiler_reports_generic_artifacts_when_available() { |
| 12053 | + let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12054 | + ensure_fixture_executable(&compiler); |
| 12055 | + |
| 12056 | + let config = IntrospectConfig { |
| 12057 | + compiler: CompilerSpec::Binary(compiler.clone()), |
| 12058 | + program: runtime_fixture("if_else.f90"), |
| 12059 | + opt_level: OptLevel::O0, |
| 12060 | + artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12061 | + json_report: None, |
| 12062 | + markdown_report: None, |
| 12063 | + all_artifacts: false, |
| 12064 | + summary_only: false, |
| 12065 | + max_artifact_lines: None, |
| 12066 | + tools: ToolchainConfig::from_env(), |
| 12067 | + }; |
| 12068 | + |
| 12069 | + let observed = run_introspect(&config).unwrap(); |
| 12070 | + let observation = &observed.observation; |
| 12071 | + assert_eq!(observation.compile_exit_code, 0); |
| 12072 | + assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12073 | + assert_eq!(observation.provenance.adapter_kind, "explicit-path"); |
| 12074 | + assert!(observation |
| 12075 | + .provenance |
| 12076 | + .backend_detail |
| 12077 | + .contains("match_42_a.sh")); |
| 12078 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Asm)); |
| 12079 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Obj)); |
| 12080 | + assert!(observation.artifacts.contains_key(&ArtifactKey::Runtime)); |
| 12081 | + assert!(observation_adapter_extras(observation).is_empty()); |
| 12082 | + assert!(missing_introspection_artifact_names(&observed).is_empty()); |
| 12083 | + } |
| 12084 | + |
| 12085 | + #[cfg(unix)] |
| 12086 | + #[test] |
| 12087 | + fn introspect_external_failure_reports_missing_requested_artifacts() { |
| 12088 | + let compiler = fake_compiler_fixture("compile_fail.sh"); |
| 12089 | + ensure_fixture_executable(&compiler); |
| 12090 | + |
| 12091 | + let config = IntrospectConfig { |
| 12092 | + compiler: CompilerSpec::Binary(compiler), |
| 12093 | + program: runtime_fixture("if_else.f90"), |
| 12094 | + opt_level: OptLevel::O0, |
| 12095 | + artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Obj, ArtifactKey::Runtime]), |
| 12096 | + json_report: None, |
| 12097 | + markdown_report: None, |
| 12098 | + all_artifacts: false, |
| 12099 | + summary_only: false, |
| 12100 | + max_artifact_lines: None, |
| 12101 | + tools: ToolchainConfig::from_env(), |
| 12102 | + }; |
| 12103 | + |
| 12104 | + let observed = run_introspect(&config).unwrap(); |
| 12105 | + let observation = &observed.observation; |
| 12106 | + assert_eq!(observation.compile_exit_code, 1); |
| 12107 | + assert_eq!(observation.provenance.failure_stage, None); |
| 12108 | + assert!(observation |
| 12109 | + .artifacts |
| 12110 | + .contains_key(&ArtifactKey::Diagnostics)); |
| 12111 | + assert_eq!( |
| 12112 | + missing_introspection_artifact_names(&observed), |
| 12113 | + vec!["asm".to_string(), "obj".to_string(), "runtime".to_string()] |
| 12114 | + ); |
| 12115 | + |
| 12116 | + let rendered = render_introspection_text(&observed, full_introspection_render_config()); |
| 12117 | + assert!(rendered.contains("status: compile failed")); |
| 12118 | + assert!(rendered.contains("failure_stage: none")); |
| 12119 | + } |
| 12120 | + |
| 12121 | + #[test] |
| 12122 | + fn introspect_named_external_compiler_rejects_namespaced_artifacts() { |
| 12123 | + let config = IntrospectConfig { |
| 12124 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12125 | + program: runtime_fixture("if_else.f90"), |
| 12126 | + opt_level: OptLevel::O0, |
| 12127 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12128 | + json_report: None, |
| 12129 | + markdown_report: None, |
| 12130 | + all_artifacts: false, |
| 12131 | + summary_only: false, |
| 12132 | + max_artifact_lines: None, |
| 12133 | + tools: ToolchainConfig::from_env(), |
| 12134 | + }; |
| 12135 | + |
| 12136 | + let observed = run_introspect(&config).unwrap(); |
| 12137 | + let observation = &observed.observation; |
| 12138 | + assert_eq!(observation.compile_exit_code, 1); |
| 12139 | + assert_eq!(observation.provenance.backend_mode, "external-driver"); |
| 12140 | + assert_eq!(observation.provenance.failure_stage, None); |
| 12141 | + let diagnostics = match observation.artifacts.get(&ArtifactKey::Diagnostics) { |
| 12142 | + Some(ArtifactValue::Text(text)) => text, |
| 12143 | + other => panic!("expected text diagnostics, got {:?}", other), |
| 12144 | + }; |
| 12145 | + assert!(diagnostics.contains("does not support requested artifacts")); |
| 12146 | + assert!(diagnostics.contains("armfortas.ir")); |
| 12147 | + } |
| 12148 | + |
| 12149 | + #[test] |
| 12150 | + fn compose_observation_failure_detail_uses_unavailable_wording() { |
| 12151 | + let observation = CompilerObservation { |
| 12152 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12153 | + program: PathBuf::from("demo.f90"), |
| 12154 | + opt_level: OptLevel::O0, |
| 12155 | + compile_exit_code: 1, |
| 12156 | + artifacts: BTreeMap::from([( |
| 12157 | + ArtifactKey::Diagnostics, |
| 12158 | + ArtifactValue::Text("linked armfortas capture is unavailable in this build".into()), |
| 12159 | + )]), |
| 12160 | + provenance: ObservationProvenance { |
| 12161 | + compiler_identity: "armfortas".into(), |
| 12162 | + adapter_kind: "named".into(), |
| 12163 | + backend_mode: "unavailable".into(), |
| 12164 | + backend_detail: "unavailable without linked-armfortas feature".into(), |
| 12165 | + artifacts_captured: vec!["diagnostics".into()], |
| 12166 | + comparison_basis: None, |
| 12167 | + failure_stage: None, |
| 12168 | + }, |
| 12169 | + }; |
| 12170 | + |
| 12171 | + let detail = compose_observation_failure_detail(&observation); |
| 12172 | + assert!(detail.contains("armfortas unavailable for requested artifacts in this build")); |
| 12173 | + assert!(!detail.contains("failed in")); |
| 12174 | + } |
| 12175 | + |
| 12176 | + #[test] |
| 12177 | + fn compose_observation_failure_detail_uses_unsupported_wording() { |
| 12178 | + let observation = CompilerObservation { |
| 12179 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12180 | + program: PathBuf::from("demo.f90"), |
| 12181 | + opt_level: OptLevel::O0, |
| 12182 | + compile_exit_code: 1, |
| 12183 | + artifacts: BTreeMap::from([( |
| 12184 | + ArtifactKey::Diagnostics, |
| 12185 | + ArtifactValue::Text( |
| 12186 | + "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 12187 | + .into(), |
| 12188 | + ), |
| 12189 | + )]), |
| 12190 | + provenance: ObservationProvenance { |
| 12191 | + compiler_identity: "gfortran".into(), |
| 12192 | + adapter_kind: "named".into(), |
| 12193 | + backend_mode: "external-driver".into(), |
| 12194 | + backend_detail: "generic external driver adapter using gfortran".into(), |
| 12195 | + artifacts_captured: vec!["diagnostics".into()], |
| 12196 | + comparison_basis: None, |
| 12197 | + failure_stage: None, |
| 12198 | + }, |
| 12199 | + }; |
| 12200 | + |
| 12201 | + let detail = compose_observation_failure_detail(&observation); |
| 12202 | + assert!(detail.contains("gfortran does not support requested artifacts in this adapter")); |
| 12203 | + assert!(!detail.contains("gfortran failed")); |
| 12204 | + } |
| 12205 | + |
| 12206 | + #[test] |
| 12207 | + fn execute_generic_introspect_case_reports_capability_mismatch_clearly() { |
| 12208 | + let suite = SuiteSpec { |
| 12209 | + name: "v2/generic-introspect".into(), |
| 12210 | + path: PathBuf::from("suite.afs"), |
| 12211 | + cases: Vec::new(), |
| 12212 | + }; |
| 12213 | + let case = CaseSpec { |
| 12214 | + name: "gfortran-armfortas-ir".into(), |
| 12215 | + source: runtime_fixture("if_else.f90"), |
| 12216 | + graph_files: Vec::new(), |
| 12217 | + requested: BTreeSet::new(), |
| 12218 | + generic_introspect: Some(GenericIntrospectCase { |
| 12219 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12220 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12221 | + }), |
| 12222 | + generic_compare: None, |
| 12223 | + opt_levels: vec![OptLevel::O0], |
| 12224 | + repeat_count: 2, |
| 12225 | + reference_compilers: Vec::new(), |
| 12226 | + consistency_checks: Vec::new(), |
| 12227 | + expectations: vec![Expectation::Contains { |
| 12228 | + target: Target::Artifact(ArtifactKey::Extra("armfortas.ir".into())), |
| 12229 | + needle: "func".into(), |
| 12230 | + }], |
| 12231 | + status_rules: Vec::new(), |
| 12232 | + capability_policy: None, |
| 12233 | + }; |
| 12234 | + let config = RunConfig { |
| 12235 | + suite_filter: None, |
| 12236 | + case_filter: None, |
| 12237 | + opt_filter: None, |
| 12238 | + verbose: false, |
| 12239 | + fail_fast: false, |
| 12240 | + include_future: false, |
| 12241 | + all_stages: false, |
| 12242 | + json_report: None, |
| 12243 | + markdown_report: None, |
| 12244 | + tools: ToolchainConfig::from_env(), |
| 12245 | + }; |
| 12246 | + |
| 12247 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12248 | + assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12249 | + assert!(outcome |
| 12250 | + .detail |
| 12251 | + .contains("gfortran does not support requested artifacts in this adapter")); |
| 12252 | + assert!(!outcome.detail.contains("gfortran failed")); |
| 12253 | + } |
| 12254 | + |
| 12255 | + #[test] |
| 12256 | + fn execute_generic_introspect_case_applies_future_capability_policy() { |
| 12257 | + let suite = SuiteSpec { |
| 12258 | + name: "v2/capability-policy".into(), |
| 12259 | + path: PathBuf::from("suite.afs"), |
| 12260 | + cases: Vec::new(), |
| 12261 | + }; |
| 12262 | + let case = CaseSpec { |
| 12263 | + name: "gfortran-armfortas-ir".into(), |
| 12264 | + source: runtime_fixture("if_else.f90"), |
| 12265 | + graph_files: Vec::new(), |
| 12266 | + requested: BTreeSet::new(), |
| 12267 | + generic_introspect: Some(GenericIntrospectCase { |
| 12268 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12269 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12270 | + }), |
| 12271 | + generic_compare: None, |
| 12272 | + opt_levels: vec![OptLevel::O0], |
| 12273 | + repeat_count: 2, |
| 12274 | + reference_compilers: Vec::new(), |
| 12275 | + consistency_checks: Vec::new(), |
| 12276 | + expectations: Vec::new(), |
| 12277 | + status_rules: Vec::new(), |
| 12278 | + capability_policy: Some(CapabilityPolicy { |
| 12279 | + kind: StatusKind::Future, |
| 12280 | + reason: "generic gfortran surface has no armfortas extras".into(), |
| 12281 | + }), |
| 12282 | + }; |
| 12283 | + let config = RunConfig { |
| 12284 | + suite_filter: None, |
| 12285 | + case_filter: None, |
| 12286 | + opt_filter: None, |
| 12287 | + verbose: false, |
| 12288 | + fail_fast: false, |
| 12289 | + include_future: false, |
| 12290 | + all_stages: false, |
| 12291 | + json_report: None, |
| 12292 | + markdown_report: None, |
| 12293 | + tools: ToolchainConfig::from_env(), |
| 12294 | + }; |
| 12295 | + |
| 12296 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12297 | + assert_eq!(outcome.kind, OutcomeKind::Future); |
| 12298 | + assert!(outcome |
| 12299 | + .detail |
| 12300 | + .contains("generic gfortran surface has no armfortas extras")); |
| 12301 | + assert!(outcome.detail.contains("armfortas.ir")); |
| 12302 | + } |
| 12303 | + |
| 12304 | + #[cfg(unix)] |
| 12305 | + #[test] |
| 12306 | + fn execute_generic_suite_case_uses_introspect_engine() { |
| 12307 | + let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12308 | + ensure_fixture_executable(&compiler); |
| 12309 | + |
| 12310 | + let suite = SuiteSpec { |
| 12311 | + name: "v2/generic-introspect".into(), |
| 12312 | + path: PathBuf::from("suite.afs"), |
| 12313 | + cases: Vec::new(), |
| 12314 | + }; |
| 12315 | + let case = CaseSpec { |
| 12316 | + name: "fake-runtime".into(), |
| 12317 | + source: runtime_fixture("if_else.f90"), |
| 12318 | + graph_files: Vec::new(), |
| 12319 | + requested: BTreeSet::new(), |
| 12320 | + generic_introspect: Some(GenericIntrospectCase { |
| 12321 | + compiler: CompilerSpec::Binary(compiler), |
| 12322 | + artifacts: BTreeSet::from([ |
| 12323 | + ArtifactKey::Asm, |
| 12324 | + ArtifactKey::Obj, |
| 12325 | + ArtifactKey::Runtime, |
| 12326 | + ]), |
| 12327 | + }), |
| 12328 | + generic_compare: None, |
| 12329 | + opt_levels: vec![OptLevel::O0], |
| 12330 | + repeat_count: 2, |
| 12331 | + reference_compilers: Vec::new(), |
| 12332 | + consistency_checks: Vec::new(), |
| 12333 | + expectations: vec![ |
| 12334 | + Expectation::Contains { |
| 12335 | + target: Target::Artifact(ArtifactKey::Asm), |
| 12336 | + needle: ".globl _main".into(), |
| 12337 | + }, |
| 12338 | + Expectation::Contains { |
| 12339 | + target: Target::RunStdout, |
| 12340 | + needle: "42".into(), |
| 12341 | + }, |
| 12342 | + ], |
| 12343 | + status_rules: Vec::new(), |
| 12344 | + capability_policy: None, |
| 12345 | + }; |
| 12346 | + let config = RunConfig { |
| 12347 | + suite_filter: None, |
| 12348 | + case_filter: None, |
| 12349 | + opt_filter: None, |
| 12350 | + verbose: false, |
| 12351 | + fail_fast: false, |
| 12352 | + include_future: false, |
| 12353 | + all_stages: false, |
| 12354 | + json_report: None, |
| 12355 | + markdown_report: None, |
| 12356 | + tools: ToolchainConfig::from_env(), |
| 12357 | + }; |
| 12358 | + |
| 12359 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12360 | + assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12361 | + assert!(outcome.detail.is_empty()); |
| 12362 | + assert!(outcome.bundle.is_none()); |
| 12363 | + } |
| 12364 | + |
| 12365 | + #[cfg(unix)] |
| 12366 | + #[test] |
| 12367 | + fn execute_generic_suite_case_supports_cli_consistency() { |
| 12368 | + let compiler = fake_compiler_fixture("match_42_a.sh"); |
| 12369 | + ensure_fixture_executable(&compiler); |
| 12370 | + |
| 12371 | + let suite = SuiteSpec { |
| 12372 | + name: "v2/generic-consistency".into(), |
| 12373 | + path: PathBuf::from("suite.afs"), |
| 12374 | + cases: Vec::new(), |
| 12375 | + }; |
| 12376 | + let case = CaseSpec { |
| 12377 | + name: "fake-runtime-consistency".into(), |
| 12378 | + source: runtime_fixture("if_else.f90"), |
| 12379 | + graph_files: Vec::new(), |
| 12380 | + requested: BTreeSet::new(), |
| 12381 | + generic_introspect: Some(GenericIntrospectCase { |
| 12382 | + compiler: CompilerSpec::Binary(compiler), |
| 12383 | + artifacts: BTreeSet::from([ArtifactKey::Asm, ArtifactKey::Runtime]), |
| 12384 | + }), |
| 12385 | + generic_compare: None, |
| 12386 | + opt_levels: vec![OptLevel::O0], |
| 12387 | + repeat_count: 3, |
| 12388 | + reference_compilers: Vec::new(), |
| 12389 | + consistency_checks: vec![ |
| 12390 | + ConsistencyCheck::CliAsmReproducible, |
| 12391 | + ConsistencyCheck::CliRunReproducible, |
| 12392 | + ], |
| 12393 | + expectations: vec![ |
| 12394 | + Expectation::Contains { |
| 12395 | + target: Target::Artifact(ArtifactKey::Asm), |
| 12396 | + needle: ".globl _main".into(), |
| 12397 | + }, |
| 12398 | + Expectation::Contains { |
| 12399 | + target: Target::RunStdout, |
| 12400 | + needle: "42".into(), |
| 12401 | + }, |
| 12402 | + ], |
| 12403 | + status_rules: Vec::new(), |
| 12404 | + capability_policy: None, |
| 12405 | + }; |
| 12406 | + let config = RunConfig { |
| 12407 | + suite_filter: None, |
| 12408 | + case_filter: None, |
| 12409 | + opt_filter: None, |
| 12410 | + verbose: false, |
| 12411 | + fail_fast: false, |
| 12412 | + include_future: false, |
| 12413 | + all_stages: false, |
| 12414 | + json_report: None, |
| 12415 | + markdown_report: None, |
| 12416 | + tools: ToolchainConfig::from_env(), |
| 12417 | + }; |
| 12418 | + |
| 12419 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12420 | + assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12421 | + assert!(outcome.detail.is_empty()); |
| 12422 | + assert!(outcome.consistency_observations.is_empty()); |
| 12423 | + } |
| 12424 | + |
| 12425 | + #[cfg(unix)] |
| 12426 | + #[test] |
| 12427 | + fn execute_generic_suite_case_supports_differential_when_available() { |
| 12428 | + if !command_is_available("gfortran") || !command_is_available("flang-new") { |
| 12429 | + return; |
| 12430 | + } |
| 12431 | + |
| 12432 | + let suite = SuiteSpec { |
| 12433 | + name: "v2/generic-differential".into(), |
| 12434 | + path: PathBuf::from("suite.afs"), |
| 12435 | + cases: Vec::new(), |
| 12436 | + }; |
| 12437 | + let case = CaseSpec { |
| 12438 | + name: "gfortran-vs-flang".into(), |
| 12439 | + source: runtime_fixture("if_else.f90"), |
| 12440 | + graph_files: Vec::new(), |
| 12441 | + requested: BTreeSet::new(), |
| 12442 | + generic_introspect: Some(GenericIntrospectCase { |
| 12443 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12444 | + artifacts: BTreeSet::from([ArtifactKey::Runtime]), |
| 12445 | + }), |
| 12446 | + generic_compare: None, |
| 12447 | + opt_levels: vec![OptLevel::O0], |
| 12448 | + repeat_count: 2, |
| 12449 | + reference_compilers: vec![ReferenceCompiler::FlangNew], |
| 12450 | + consistency_checks: Vec::new(), |
| 12451 | + expectations: vec![ |
| 12452 | + Expectation::Contains { |
| 12453 | + target: Target::RunStdout, |
| 12454 | + needle: "positive".into(), |
| 12455 | + }, |
| 12456 | + Expectation::IntEquals { |
| 12457 | + target: Target::RunExitCode, |
| 12458 | + value: 0, |
| 12459 | + }, |
| 12460 | + ], |
| 12461 | + status_rules: Vec::new(), |
| 12462 | + capability_policy: None, |
| 12463 | + }; |
| 12464 | + let config = RunConfig { |
| 12465 | + suite_filter: None, |
| 12466 | + case_filter: None, |
| 12467 | + opt_filter: None, |
| 12468 | + verbose: false, |
| 12469 | + fail_fast: false, |
| 12470 | + include_future: false, |
| 12471 | + all_stages: false, |
| 12472 | + json_report: None, |
| 12473 | + markdown_report: None, |
| 12474 | + tools: ToolchainConfig::from_env(), |
| 12475 | + }; |
| 12476 | + |
| 12477 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12478 | + assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12479 | + assert!(outcome.detail.is_empty()); |
| 12480 | + } |
| 12481 | + |
| 12482 | + #[cfg(unix)] |
| 12483 | + #[test] |
| 12484 | + fn execute_generic_compare_suite_case_uses_compare_engine() { |
| 12485 | + let left = fake_compiler_fixture("match_42_a.sh"); |
| 12486 | + let right = fake_compiler_fixture("runtime_41.sh"); |
| 12487 | + ensure_fixture_executable(&left); |
| 12488 | + ensure_fixture_executable(&right); |
| 12489 | + |
| 12490 | + let suite = SuiteSpec { |
| 12491 | + name: "v2/generic-compare".into(), |
| 12492 | + path: PathBuf::from("suite.afs"), |
| 12493 | + cases: Vec::new(), |
| 12494 | + }; |
| 12495 | + let case = CaseSpec { |
| 12496 | + name: "fake-divergence".into(), |
| 12497 | + source: runtime_fixture("if_else.f90"), |
| 12498 | + graph_files: Vec::new(), |
| 12499 | + requested: BTreeSet::new(), |
| 12500 | + generic_introspect: None, |
| 12501 | + generic_compare: Some(GenericCompareCase { |
| 12502 | + left: CompilerSpec::Binary(left), |
| 12503 | + right: CompilerSpec::Binary(right), |
| 12504 | + artifacts: BTreeSet::from([ |
| 12505 | + ArtifactKey::Diagnostics, |
| 12506 | + ArtifactKey::Runtime, |
| 12507 | + ArtifactKey::Asm, |
| 12508 | + ]), |
| 12509 | + }), |
| 12510 | + opt_levels: vec![OptLevel::O0], |
| 12511 | + repeat_count: 2, |
| 12512 | + reference_compilers: Vec::new(), |
| 12513 | + consistency_checks: Vec::new(), |
| 12514 | + expectations: vec![ |
| 12515 | + Expectation::Equals { |
| 12516 | + target: Target::CompareStatus, |
| 12517 | + value: "diff".into(), |
| 12518 | + }, |
| 12519 | + Expectation::Equals { |
| 12520 | + target: Target::CompareClassification, |
| 12521 | + value: "mixed divergence".into(), |
| 12522 | + }, |
| 12523 | + Expectation::Contains { |
| 12524 | + target: Target::CompareChangedArtifacts, |
| 12525 | + needle: "asm".into(), |
| 12526 | + }, |
| 12527 | + Expectation::Contains { |
| 12528 | + target: Target::CompareChangedArtifacts, |
| 12529 | + needle: "runtime".into(), |
| 12530 | + }, |
| 12531 | + Expectation::IntEquals { |
| 12532 | + target: Target::CompareDifferenceCount, |
| 12533 | + value: 2, |
| 12534 | + }, |
| 12535 | + ], |
| 12536 | + status_rules: Vec::new(), |
| 12537 | + capability_policy: None, |
| 12538 | + }; |
| 12539 | + let config = RunConfig { |
| 12540 | + suite_filter: None, |
| 12541 | + case_filter: None, |
| 12542 | + opt_filter: None, |
| 12543 | + verbose: false, |
| 12544 | + fail_fast: false, |
| 12545 | + include_future: false, |
| 12546 | + all_stages: false, |
| 12547 | + json_report: None, |
| 12548 | + markdown_report: None, |
| 12549 | + tools: ToolchainConfig::from_env(), |
| 12550 | + }; |
| 12551 | + |
| 12552 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12553 | + assert_eq!(outcome.kind, OutcomeKind::Pass); |
| 12554 | + } |
| 12555 | + |
| 12556 | + #[test] |
| 12557 | + fn execute_generic_compare_suite_case_reports_capability_mismatch() { |
| 12558 | + let suite = SuiteSpec { |
| 12559 | + name: "v2/generic-compare".into(), |
| 12560 | + path: PathBuf::from("suite.afs"), |
| 12561 | + cases: Vec::new(), |
| 12562 | + }; |
| 12563 | + let case = CaseSpec { |
| 12564 | + name: "armfortas-ir-vs-gfortran".into(), |
| 12565 | + source: runtime_fixture("if_else.f90"), |
| 12566 | + graph_files: Vec::new(), |
| 12567 | + requested: BTreeSet::new(), |
| 12568 | + generic_introspect: None, |
| 12569 | + generic_compare: Some(GenericCompareCase { |
| 12570 | + left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12571 | + right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12572 | + artifacts: BTreeSet::from([ |
| 12573 | + ArtifactKey::Diagnostics, |
| 12574 | + ArtifactKey::Runtime, |
| 12575 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 12576 | + ]), |
| 12577 | + }), |
| 12578 | + opt_levels: vec![OptLevel::O0], |
| 12579 | + repeat_count: 2, |
| 12580 | + reference_compilers: Vec::new(), |
| 12581 | + consistency_checks: Vec::new(), |
| 12582 | + expectations: vec![Expectation::Equals { |
| 12583 | + target: Target::CompareStatus, |
| 12584 | + value: "match".into(), |
| 12585 | + }], |
| 12586 | + status_rules: Vec::new(), |
| 12587 | + capability_policy: None, |
| 12588 | + }; |
| 12589 | + let config = RunConfig { |
| 12590 | + suite_filter: None, |
| 12591 | + case_filter: None, |
| 12592 | + opt_filter: None, |
| 12593 | + verbose: false, |
| 12594 | + fail_fast: false, |
| 12595 | + include_future: false, |
| 12596 | + all_stages: false, |
| 12597 | + json_report: None, |
| 12598 | + markdown_report: None, |
| 12599 | + tools: ToolchainConfig::from_env(), |
| 12600 | + }; |
| 12601 | + |
| 12602 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12603 | + assert_eq!(outcome.kind, OutcomeKind::Fail); |
| 12604 | + assert!(outcome.detail.contains("compare request is not supported")); |
| 12605 | + assert!(outcome.detail.contains("armfortas.ir")); |
| 12606 | + } |
| 12607 | + |
| 12608 | + #[test] |
| 12609 | + fn execute_generic_compare_suite_case_applies_xfail_capability_policy() { |
| 12610 | + let suite = SuiteSpec { |
| 12611 | + name: "v2/capability-policy".into(), |
| 12612 | + path: PathBuf::from("suite.afs"), |
| 12613 | + cases: Vec::new(), |
| 12614 | + }; |
| 12615 | + let case = CaseSpec { |
| 12616 | + name: "armfortas-ir-vs-gfortran".into(), |
| 12617 | + source: runtime_fixture("if_else.f90"), |
| 12618 | + graph_files: Vec::new(), |
| 12619 | + requested: BTreeSet::new(), |
| 12620 | + generic_introspect: None, |
| 12621 | + generic_compare: Some(GenericCompareCase { |
| 12622 | + left: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 12623 | + right: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 12624 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 12625 | + }), |
| 12626 | + opt_levels: vec![OptLevel::O0], |
| 12627 | + repeat_count: 2, |
| 12628 | + reference_compilers: Vec::new(), |
| 12629 | + consistency_checks: Vec::new(), |
| 12630 | + expectations: Vec::new(), |
| 12631 | + status_rules: Vec::new(), |
| 12632 | + capability_policy: Some(CapabilityPolicy { |
| 12633 | + kind: StatusKind::Xfail, |
| 12634 | + reason: "mixed-surface namespaced compare stays soft for now".into(), |
| 12635 | + }), |
| 12636 | + }; |
| 12637 | + let config = RunConfig { |
| 12638 | + suite_filter: None, |
| 12639 | + case_filter: None, |
| 12640 | + opt_filter: None, |
| 12641 | + verbose: false, |
| 12642 | + fail_fast: false, |
| 12643 | + include_future: false, |
| 12644 | + all_stages: false, |
| 12645 | + json_report: None, |
| 12646 | + markdown_report: None, |
| 12647 | + tools: ToolchainConfig::from_env(), |
| 12648 | + }; |
| 12649 | + |
| 12650 | + let outcome = execute_case_cell(&suite, &case, OptLevel::O0, &config).unwrap(); |
| 12651 | + assert_eq!(outcome.kind, OutcomeKind::Xfail); |
| 12652 | + assert!(outcome |
| 12653 | + .detail |
| 12654 | + .contains("mixed-surface namespaced compare stays soft for now")); |
| 12655 | + assert!(outcome.detail.contains("armfortas.ir")); |
| 12656 | + } |
| 12657 | + |
| 12658 | + #[test] |
| 12659 | + fn parses_suite_and_case() { |
| 12660 | + let root = std::env::temp_dir().join("afs_tests_parser_spec.afs"); |
| 12661 | + fs::write( |
| 12662 | + &root, |
| 12663 | + r#"suite "runtime/smoke" |
| 12664 | + |
| 12665 | +case "hello" |
| 12666 | +source "../../../test_programs/hello.f90" |
| 12667 | +armfortas => run, ir |
| 12668 | +expect run.stdout check-comments |
| 12669 | +expect ir contains "module main" |
| 12670 | +expect asm not-contains "x18" |
| 12671 | +end |
| 12672 | +"#, |
| 12673 | + ) |
| 12674 | + .unwrap(); |
| 12675 | + |
| 12676 | + let suite = parse_suite_file(&root).unwrap(); |
| 12677 | + assert_eq!(suite.name, "runtime/smoke"); |
| 12678 | + assert_eq!(suite.cases.len(), 1); |
| 12679 | + assert!(suite.cases[0].requested.contains(&Stage::Run)); |
| 12680 | + assert!(suite.cases[0].requested.contains(&Stage::Ir)); |
| 12681 | + assert!(suite.cases[0].generic_introspect.is_none()); |
| 12682 | + assert!(matches!( |
| 12683 | + suite.cases[0].expectations[2], |
| 12684 | + Expectation::NotContains { |
| 12685 | + target: Target::Artifact(ArtifactKey::Asm), |
| 12686 | + .. |
| 12687 | + } |
| 12688 | + )); |
| 12689 | + assert_eq!(suite.cases[0].opt_levels, vec![OptLevel::O0]); |
| 12690 | + let _ = fs::remove_file(&root); |
| 12691 | + } |
| 12692 | + |
| 12693 | + #[test] |
| 12694 | + fn parses_generic_compiler_case() { |
| 12695 | + let root = std::env::temp_dir().join("bencch_generic_parser_spec.afs"); |
| 12696 | + fs::write( |
| 12697 | + &root, |
| 12698 | + r#"suite "v2/generic-introspect" |
| 12699 | + |
| 12700 | +case "fake-runtime" |
| 12701 | +source "../../fixtures/runtime/if_else.f90" |
| 12702 | +compiler gfortran => asm, obj, runtime |
| 12703 | +expect asm contains ".globl _main" |
| 12704 | +expect run.stdout contains "42" |
| 12705 | +end |
| 12706 | +"#, |
| 12707 | + ) |
| 12708 | + .unwrap(); |
| 12709 | + |
| 12710 | + let suite = parse_suite_file(&root).unwrap(); |
| 12711 | + let case = &suite.cases[0]; |
| 12712 | + let generic = case.generic_introspect.as_ref().unwrap(); |
| 12713 | + assert_eq!( |
| 12714 | + generic.compiler, |
| 12715 | + CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12716 | + ); |
| 12717 | + assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12718 | + assert!(generic.artifacts.contains(&ArtifactKey::Obj)); |
| 12719 | + assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12720 | + assert!(case.requested.is_empty()); |
| 12721 | + let _ = fs::remove_file(&root); |
| 12722 | + } |
| 12723 | + |
| 12724 | + #[test] |
| 12725 | + fn parses_generic_compiler_case_with_differential_and_cli_consistency() { |
| 12726 | + let root = std::env::temp_dir().join("bencch_generic_differential_parser_spec.afs"); |
| 12727 | + fs::write( |
| 12728 | + &root, |
| 12729 | + r#"suite "v2/generic-differential" |
| 12730 | + |
| 12731 | +case "gfortran_runtime_matrix" |
| 12732 | +source "../../fixtures/runtime/if_else.f90" |
| 12733 | +opts => O0, O1, O2 |
| 12734 | +repeat => 3 |
| 12735 | +compiler gfortran => runtime, asm |
| 12736 | +differential => flang-new |
| 12737 | +consistency => cli_asm_reproducible, cli_run_reproducible |
| 12738 | +expect run.stdout check-comments |
| 12739 | +expect run.exit_code equals 0 |
| 12740 | +end |
| 12741 | +"#, |
| 12742 | + ) |
| 12743 | + .unwrap(); |
| 12744 | + |
| 12745 | + let suite = parse_suite_file(&root).unwrap(); |
| 12746 | + let case = &suite.cases[0]; |
| 12747 | + let generic = case.generic_introspect.as_ref().unwrap(); |
| 12748 | + assert_eq!( |
| 12749 | + generic.compiler, |
| 12750 | + CompilerSpec::Named(NamedCompiler::Gfortran) |
| 12751 | + ); |
| 12752 | + assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12753 | + assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12754 | + assert_eq!(case.reference_compilers, vec![ReferenceCompiler::FlangNew]); |
| 12755 | + assert_eq!( |
| 12756 | + case.consistency_checks, |
| 12757 | + vec![ |
| 12758 | + ConsistencyCheck::CliAsmReproducible, |
| 12759 | + ConsistencyCheck::CliRunReproducible, |
| 12760 | + ] |
| 12761 | + ); |
| 12762 | + let _ = fs::remove_file(&root); |
| 12763 | + } |
| 12764 | + |
| 12765 | + #[test] |
| 12766 | + fn rejects_capture_consistency_on_generic_compiler_case() { |
| 12767 | + let root = std::env::temp_dir().join("bencch_generic_capture_consistency_parser_spec.afs"); |
| 12768 | + fs::write( |
| 12769 | + &root, |
| 12770 | + r#"suite "v2/generic-consistency" |
| 12771 | + |
| 12772 | +case "armfortas_capture_run" |
| 12773 | +source "../../fixtures/runtime/if_else.f90" |
| 12774 | +compiler armfortas => runtime |
| 12775 | +consistency => capture_run_reproducible |
| 12776 | +expect run.exit_code equals 0 |
| 12777 | +end |
| 12778 | +"#, |
| 12779 | + ) |
| 12780 | + .unwrap(); |
| 12781 | + |
| 12782 | + let err = parse_suite_file(&root).unwrap_err(); |
| 12783 | + assert!(err.contains("generic compiler cases only support")); |
| 12784 | + assert!(err.contains("capture_run_reproducible")); |
| 12785 | + let _ = fs::remove_file(&root); |
| 12786 | + } |
| 12787 | + |
| 12788 | + #[test] |
| 12789 | + fn parses_generic_compare_case() { |
| 12790 | + let root = std::env::temp_dir().join("bencch_generic_compare_parser_spec.afs"); |
| 12791 | + fs::write( |
| 12792 | + &root, |
| 12793 | + r#"suite "v2/generic-compare" |
| 12794 | + |
| 12795 | +case "fake-match" |
| 12796 | +source "../../fixtures/runtime/if_else.f90" |
| 12797 | +opts => O0, O1, O2 |
| 12798 | +compare gfortran flang-new => asm |
| 12799 | +expect compare.status equals "match" |
| 12800 | +expect compare.difference_count equals 0 |
| 12801 | +end |
| 12802 | +"#, |
| 12803 | + ) |
| 12804 | + .unwrap(); |
| 12805 | + |
| 12806 | + let suite = parse_suite_file(&root).unwrap(); |
| 12807 | + let case = &suite.cases[0]; |
| 12808 | + let generic = case.generic_compare.as_ref().unwrap(); |
| 12809 | + assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Gfortran)); |
| 12810 | + assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::FlangNew)); |
| 12811 | + assert!(generic.artifacts.contains(&ArtifactKey::Asm)); |
| 12812 | + assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12813 | + assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12814 | + assert_eq!( |
| 12815 | + case.opt_levels, |
| 12816 | + vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12817 | + ); |
| 12818 | + let _ = fs::remove_file(&root); |
| 12819 | + } |
| 12820 | + |
| 12821 | + #[test] |
| 12822 | + fn parses_generic_compare_case_with_namespaced_artifact() { |
| 12823 | + let root = std::env::temp_dir().join("bencch_generic_compare_namespaced_spec.afs"); |
| 12824 | + fs::write( |
| 12825 | + &root, |
| 12826 | + r#"suite "v2/generic-compare" |
| 12827 | + |
| 12828 | +case "armfortas-ir" |
| 12829 | +source "../../fixtures/runtime/if_else.f90" |
| 12830 | +compare armfortas armfortas => armfortas.ir |
| 12831 | +expect compare.status equals "match" |
| 12832 | +end |
| 12833 | +"#, |
| 12834 | + ) |
| 12835 | + .unwrap(); |
| 12836 | + |
| 12837 | + let suite = parse_suite_file(&root).unwrap(); |
| 12838 | + let case = &suite.cases[0]; |
| 12839 | + let generic = case.generic_compare.as_ref().unwrap(); |
| 12840 | + assert_eq!(generic.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12841 | + assert_eq!(generic.right, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 12842 | + assert!(generic |
| 12843 | + .artifacts |
| 12844 | + .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 12845 | + assert!(generic.artifacts.contains(&ArtifactKey::Diagnostics)); |
| 12846 | + assert!(generic.artifacts.contains(&ArtifactKey::Runtime)); |
| 12847 | + |
| 12848 | + let _ = fs::remove_file(&root); |
| 12849 | + } |
| 12850 | + |
| 12851 | + #[test] |
| 12852 | + fn parses_capability_policy_for_generic_case() { |
| 12853 | + let root = std::env::temp_dir().join("bencch_generic_capability_policy_spec.afs"); |
| 12854 | + fs::write( |
| 12855 | + &root, |
| 12856 | + r#"suite "v2/capability-policy" |
| 12857 | + |
| 12858 | +case "gfortran_armfortas_ir" |
| 12859 | +source "../../fixtures/runtime/if_else.f90" |
| 12860 | +compiler gfortran => armfortas.ir |
| 12861 | +future capability "generic gfortran surface has no armfortas extras" |
| 12862 | +end |
| 12863 | +"#, |
| 12864 | + ) |
| 12865 | + .unwrap(); |
| 12866 | + |
| 12867 | + let suite = parse_suite_file(&root).unwrap(); |
| 12868 | + let case = &suite.cases[0]; |
| 12869 | + let policy = case.capability_policy.as_ref().unwrap(); |
| 12870 | + assert!(matches!(policy.kind, StatusKind::Future)); |
| 12871 | + assert_eq!( |
| 12872 | + policy.reason, |
| 12873 | + "generic gfortran surface has no armfortas extras" |
| 12874 | + ); |
| 12875 | + let _ = fs::remove_file(&root); |
| 12876 | + } |
| 12877 | + |
| 12878 | + #[test] |
| 12879 | + fn parses_matrix_status_and_differential() { |
| 12880 | + let root = std::env::temp_dir().join("afs_tests_matrix_spec.afs"); |
| 12881 | + fs::write( |
| 12882 | + &root, |
| 12883 | + r#"suite "runtime/matrix" |
| 12884 | + |
| 12885 | +case "hello" |
| 12886 | +source "../../../test_programs/hello.f90" |
| 12887 | +opts => O0, O1, O2 |
| 12888 | +armfortas => run |
| 12889 | +differential => gfortran, flang-new |
| 12890 | +expect run.exit_code equals 0 |
| 12891 | +xfail when O1, O2 because "known issue" |
| 12892 | +end |
| 12893 | +"#, |
| 12894 | + ) |
| 12895 | + .unwrap(); |
| 12896 | + |
| 12897 | + let suite = parse_suite_file(&root).unwrap(); |
| 12898 | + let case = &suite.cases[0]; |
| 12899 | + assert_eq!( |
| 12900 | + case.opt_levels, |
| 12901 | + vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 12902 | + ); |
| 12903 | + assert_eq!( |
| 12904 | + case.reference_compilers, |
| 12905 | + vec![ReferenceCompiler::Gfortran, ReferenceCompiler::FlangNew] |
| 12906 | + ); |
| 12907 | + assert!(matches!( |
| 12908 | + status_for_opt(case, OptLevel::O0), |
| 12909 | + EffectiveStatus::Normal |
| 12910 | + )); |
| 12911 | + assert!(matches!( |
| 12912 | + status_for_opt(case, OptLevel::O1), |
| 12913 | + EffectiveStatus::Xfail(_) |
| 12914 | + )); |
| 12915 | + let _ = fs::remove_file(&root); |
| 12916 | + } |
| 12917 | + |
| 12918 | + #[test] |
| 12919 | + fn parses_consistency_checks() { |
| 12920 | + let root = std::env::temp_dir().join("afs_tests_consistency_spec.afs"); |
| 12921 | + fs::write( |
| 12922 | + &root, |
| 12923 | + r#"suite "consistency/object" |
| 12924 | + |
| 12925 | +case "driver_paths" |
| 12926 | +source "../../fixtures/backend/runtime_calls.f90" |
| 12927 | +armfortas => asm, obj |
| 12928 | +repeat => 5 |
| 12929 | +consistency => cli_obj_vs_system_as, cli-obj-vs-system-as, cli_asm_reproducible, cli-obj-reproducible, cli_run_reproducible, capture_asm_vs_cli_asm, capture-obj-vs-cli-obj, capture_run_vs_cli_run, capture_asm_reproducible, capture-obj-reproducible, capture_run_reproducible |
| 12930 | +expect obj contains "_main" |
| 12931 | +end |
| 12932 | +"#, |
| 12933 | + ) |
| 12934 | + .unwrap(); |
| 12935 | + |
| 12936 | + let suite = parse_suite_file(&root).unwrap(); |
| 12937 | + let case = &suite.cases[0]; |
| 12938 | + assert_eq!( |
| 12939 | + case.consistency_checks, |
| 12940 | + vec![ |
| 12941 | + ConsistencyCheck::CliObjVsSystemAs, |
| 12942 | + ConsistencyCheck::CliAsmReproducible, |
| 12943 | + ConsistencyCheck::CliObjReproducible, |
| 12944 | + ConsistencyCheck::CliRunReproducible, |
| 12945 | + ConsistencyCheck::CaptureAsmVsCliAsm, |
| 12946 | + ConsistencyCheck::CaptureObjVsCliObj, |
| 12947 | + ConsistencyCheck::CaptureRunVsCliRun, |
| 12948 | + ConsistencyCheck::CaptureAsmReproducible, |
| 12949 | + ConsistencyCheck::CaptureObjReproducible, |
| 12950 | + ConsistencyCheck::CaptureRunReproducible, |
| 12951 | + ] |
| 12952 | + ); |
| 12953 | + assert_eq!(case.repeat_count, 5); |
| 12954 | + let _ = fs::remove_file(&root); |
| 12955 | + } |
| 12956 | + |
| 12957 | + #[test] |
| 12958 | + fn parses_graph_case() { |
| 12959 | + let root = std::env::temp_dir().join("afs_tests_graph_spec"); |
| 12960 | + let _ = fs::remove_dir_all(&root); |
| 12961 | + fs::create_dir_all(&root).unwrap(); |
| 12962 | + fs::write( |
| 12963 | + root.join("math_values.f90"), |
| 12964 | + "module math_values\nend module\n", |
| 12965 | + ) |
| 12966 | + .unwrap(); |
| 12967 | + fs::write(root.join("main.f90"), "program main\nend program\n").unwrap(); |
| 12968 | + fs::write( |
| 12969 | + root.join("graph.afs"), |
| 12970 | + r#"suite "modules/graph" |
| 12971 | + |
| 12972 | +case "basic_use" |
| 12973 | +entry "main.f90" |
| 12974 | +file "math_values.f90" |
| 12975 | +file "main.f90" |
| 12976 | +armfortas => run |
| 12977 | +expect run.exit_code equals 0 |
| 12978 | +end |
| 12979 | +"#, |
| 12980 | + ) |
| 12981 | + .unwrap(); |
| 12982 | + |
| 12983 | + let suite = parse_suite_file(&root.join("graph.afs")).unwrap(); |
| 12984 | + let case = &suite.cases[0]; |
| 12985 | + assert_eq!(case.source, root.join("main.f90")); |
| 12986 | + assert_eq!( |
| 12987 | + case.graph_files, |
| 12988 | + vec![root.join("math_values.f90"), root.join("main.f90")] |
| 12989 | + ); |
| 12990 | + |
| 12991 | + let _ = fs::remove_dir_all(&root); |
| 12992 | + } |
| 12993 | + |
| 12994 | + #[test] |
| 12995 | + fn parse_cli_collects_tool_overrides() { |
| 12996 | + let args = vec![ |
| 12997 | + "run".to_string(), |
| 12998 | + "--suite".to_string(), |
| 12999 | + "consistency/runtime".to_string(), |
| 13000 | + "--json-report".to_string(), |
| 13001 | + "/tmp/report.json".to_string(), |
| 13002 | + "--markdown-report".to_string(), |
| 13003 | + "/tmp/report.md".to_string(), |
| 13004 | + "--armfortas-bin".to_string(), |
| 13005 | + "/tmp/armfortas".to_string(), |
| 13006 | + "--gfortran-bin".to_string(), |
| 13007 | + "/tmp/gfortran".to_string(), |
| 13008 | + "--flang-bin".to_string(), |
| 13009 | + "/tmp/flang-new".to_string(), |
| 13010 | + "--lfortran-bin".to_string(), |
| 13011 | + "/tmp/lfortran".to_string(), |
| 13012 | + "--ifort-bin".to_string(), |
| 13013 | + "/tmp/ifort".to_string(), |
| 13014 | + "--ifx-bin".to_string(), |
| 13015 | + "/tmp/ifx".to_string(), |
| 13016 | + "--nvfortran-bin".to_string(), |
| 13017 | + "/tmp/nvfortran".to_string(), |
| 13018 | + "--as-bin".to_string(), |
| 13019 | + "/tmp/as".to_string(), |
| 13020 | + "--otool-bin".to_string(), |
| 13021 | + "/tmp/otool".to_string(), |
| 13022 | + "--nm-bin".to_string(), |
| 13023 | + "/tmp/nm".to_string(), |
| 13024 | + ]; |
| 13025 | + |
| 13026 | + let command = parse_cli(&args).unwrap(); |
| 13027 | + let config = match command { |
| 13028 | + CommandKind::Run(config) => config, |
| 13029 | + other => panic!( |
| 13030 | + "expected run command, got {:?}", |
| 13031 | + std::mem::discriminant(&other) |
| 13032 | + ), |
| 13033 | + }; |
| 13034 | + |
| 13035 | + assert_eq!(config.suite_filter.as_deref(), Some("consistency/runtime")); |
| 13036 | + assert_eq!( |
| 13037 | + config.json_report.as_deref(), |
| 13038 | + Some(Path::new("/tmp/report.json")) |
| 13039 | + ); |
| 13040 | + assert_eq!( |
| 13041 | + config.markdown_report.as_deref(), |
| 13042 | + Some(Path::new("/tmp/report.md")) |
| 13043 | + ); |
| 13044 | + assert_eq!( |
| 13045 | + config.tools.armfortas, |
| 13046 | + ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13047 | + ); |
| 13048 | + assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 13049 | + assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 13050 | + assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 13051 | + assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 13052 | + assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 13053 | + assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 13054 | + assert_eq!(config.tools.system_as, "/tmp/as"); |
| 13055 | + assert_eq!(config.tools.otool, "/tmp/otool"); |
| 13056 | + assert_eq!(config.tools.nm, "/tmp/nm"); |
| 13057 | + } |
| 13058 | + |
| 13059 | + #[test] |
| 13060 | + fn parse_cli_collects_list_config() { |
| 13061 | + let args = vec![ |
| 13062 | + "list".to_string(), |
| 13063 | + "--suite".to_string(), |
| 13064 | + "v2/generic".to_string(), |
| 13065 | + "--verbose".to_string(), |
| 13066 | + "--armfortas-bin".to_string(), |
| 13067 | + "/tmp/armfortas".to_string(), |
| 13068 | + ]; |
| 13069 | + |
| 13070 | + let command = parse_cli(&args).unwrap(); |
| 13071 | + let config = match command { |
| 13072 | + CommandKind::List(config) => config, |
| 13073 | + other => panic!( |
| 13074 | + "expected list command, got {:?}", |
| 13075 | + std::mem::discriminant(&other) |
| 13076 | + ), |
| 13077 | + }; |
| 13078 | + |
| 13079 | + assert_eq!(config.suite_filter.as_deref(), Some("v2/generic")); |
| 13080 | + assert!(config.verbose); |
| 13081 | + assert_eq!( |
| 13082 | + config.tools.armfortas, |
| 13083 | + ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13084 | + ); |
| 13085 | + } |
| 13086 | + |
| 13087 | + #[test] |
| 13088 | + fn parse_cli_collects_doctor_tool_overrides() { |
| 13089 | + let args = vec![ |
| 13090 | + "doctor".to_string(), |
| 13091 | + "--json-report".to_string(), |
| 13092 | + "/tmp/doctor.json".to_string(), |
| 13093 | + "--markdown-report".to_string(), |
| 13094 | + "/tmp/doctor.md".to_string(), |
| 13095 | + "--armfortas-bin".to_string(), |
| 13096 | + "/tmp/armfortas".to_string(), |
| 13097 | + "--gfortran-bin".to_string(), |
| 13098 | + "/tmp/gfortran".to_string(), |
| 13099 | + "--flang-bin".to_string(), |
| 13100 | + "/tmp/flang-new".to_string(), |
| 13101 | + "--lfortran-bin".to_string(), |
| 13102 | + "/tmp/lfortran".to_string(), |
| 13103 | + "--ifort-bin".to_string(), |
| 13104 | + "/tmp/ifort".to_string(), |
| 13105 | + "--ifx-bin".to_string(), |
| 13106 | + "/tmp/ifx".to_string(), |
| 13107 | + "--nvfortran-bin".to_string(), |
| 13108 | + "/tmp/nvfortran".to_string(), |
| 13109 | + "--as-bin".to_string(), |
| 13110 | + "/tmp/as".to_string(), |
| 13111 | + "--otool-bin".to_string(), |
| 13112 | + "/tmp/otool".to_string(), |
| 13113 | + "--nm-bin".to_string(), |
| 13114 | + "/tmp/nm".to_string(), |
| 13115 | + ]; |
| 13116 | + |
| 13117 | + let command = parse_cli(&args).unwrap(); |
| 13118 | + let config = match command { |
| 13119 | + CommandKind::Doctor(config) => config, |
| 13120 | + other => panic!( |
| 13121 | + "expected doctor command, got {:?}", |
| 13122 | + std::mem::discriminant(&other) |
| 13123 | + ), |
| 13124 | + }; |
| 13125 | + |
| 13126 | + assert_eq!( |
| 13127 | + config.tools.armfortas, |
| 13128 | + ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 13129 | + ); |
| 13130 | + assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 13131 | + assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 13132 | + assert_eq!(config.tools.lfortran, "/tmp/lfortran"); |
| 13133 | + assert_eq!(config.tools.ifort, "/tmp/ifort"); |
| 13134 | + assert_eq!(config.tools.ifx, "/tmp/ifx"); |
| 13135 | + assert_eq!(config.tools.nvfortran, "/tmp/nvfortran"); |
| 13136 | + assert_eq!(config.tools.system_as, "/tmp/as"); |
| 13137 | + assert_eq!(config.tools.otool, "/tmp/otool"); |
| 13138 | + assert_eq!(config.tools.nm, "/tmp/nm"); |
| 13139 | + assert_eq!( |
| 13140 | + config.json_report.as_deref(), |
| 13141 | + Some(Path::new("/tmp/doctor.json")) |
| 13142 | + ); |
| 13143 | + assert_eq!( |
| 13144 | + config.markdown_report.as_deref(), |
| 13145 | + Some(Path::new("/tmp/doctor.md")) |
| 13146 | + ); |
| 13147 | + } |
| 13148 | + |
| 13149 | + #[test] |
| 13150 | + fn parse_cli_collects_compare_config() { |
| 13151 | + let args = vec![ |
| 13152 | + "compare".to_string(), |
| 13153 | + "armfortas".to_string(), |
| 13154 | + "/tmp/other-compiler".to_string(), |
| 13155 | + "--program".to_string(), |
| 13156 | + "/tmp/demo.f90".to_string(), |
| 13157 | + "--opt".to_string(), |
| 13158 | + "O2".to_string(), |
| 13159 | + "--artifact".to_string(), |
| 13160 | + "asm,obj,armfortas.ir".to_string(), |
| 13161 | + "--json-report".to_string(), |
| 13162 | + "/tmp/compare.json".to_string(), |
| 13163 | + "--markdown-report".to_string(), |
| 13164 | + "/tmp/compare.md".to_string(), |
| 13165 | + ]; |
| 13166 | + |
| 13167 | + let command = parse_cli(&args).unwrap(); |
| 13168 | + let config = match command { |
| 13169 | + CommandKind::Compare(config) => config, |
| 13170 | + other => panic!( |
| 13171 | + "expected compare command, got {:?}", |
| 13172 | + std::mem::discriminant(&other) |
| 13173 | + ), |
| 13174 | + }; |
| 13175 | + |
| 13176 | + assert_eq!(config.left, CompilerSpec::Named(NamedCompiler::Armfortas)); |
| 13177 | + assert_eq!( |
| 13178 | + config.right, |
| 13179 | + CompilerSpec::Binary(PathBuf::from("/tmp/other-compiler")) |
| 13180 | + ); |
| 13181 | + assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 13182 | + assert_eq!(config.opt_level, OptLevel::O2); |
| 13183 | + assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 13184 | + assert!(config.artifacts.contains(&ArtifactKey::Obj)); |
| 13185 | + assert!(config |
| 13186 | + .artifacts |
| 13187 | + .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 13188 | + assert_eq!( |
| 13189 | + config.json_report.as_deref(), |
| 13190 | + Some(Path::new("/tmp/compare.json")) |
| 13191 | + ); |
| 13192 | + assert_eq!( |
| 13193 | + config.markdown_report.as_deref(), |
| 13194 | + Some(Path::new("/tmp/compare.md")) |
| 13195 | + ); |
| 13196 | + } |
| 13197 | + |
| 13198 | + #[test] |
| 13199 | + fn parse_cli_collects_introspect_config() { |
| 13200 | + let args = vec![ |
| 13201 | + "introspect".to_string(), |
| 13202 | + "armfortas".to_string(), |
| 13203 | + "/tmp/demo.f90".to_string(), |
| 13204 | + "--artifact".to_string(), |
| 13205 | + "armfortas.ir,asm".to_string(), |
| 13206 | + "--all".to_string(), |
| 13207 | + "--summary-only".to_string(), |
| 13208 | + "--max-artifact-lines".to_string(), |
| 13209 | + "12".to_string(), |
| 13210 | + "--json-report".to_string(), |
| 13211 | + "/tmp/introspect.json".to_string(), |
| 13212 | + ]; |
| 13213 | + |
| 13214 | + let command = parse_cli(&args).unwrap(); |
| 13215 | + let config = match command { |
| 13216 | + CommandKind::Introspect(config) => config, |
| 13217 | + other => panic!( |
| 13218 | + "expected introspect command, got {:?}", |
| 13219 | + std::mem::discriminant(&other) |
| 13220 | + ), |
| 13221 | + }; |
| 13222 | + |
| 13223 | + assert_eq!( |
| 13224 | + config.compiler, |
| 13225 | + CompilerSpec::Named(NamedCompiler::Armfortas) |
| 13226 | + ); |
| 13227 | + assert_eq!(config.program, PathBuf::from("/tmp/demo.f90")); |
| 13228 | + assert!(config.artifacts.contains(&ArtifactKey::Asm)); |
| 13229 | + assert!(config |
| 13230 | + .artifacts |
| 13231 | + .contains(&ArtifactKey::Extra("armfortas.ir".into()))); |
| 13232 | + assert!(config.all_artifacts); |
| 13233 | + assert!(config.summary_only); |
| 13234 | + assert_eq!(config.max_artifact_lines, Some(12)); |
| 13235 | + assert_eq!( |
| 13236 | + config.json_report.as_deref(), |
| 13237 | + Some(Path::new("/tmp/introspect.json")) |
| 13238 | + ); |
| 13239 | + } |
| 13240 | + |
| 13241 | + #[test] |
| 13242 | + fn parses_failure_expectation() { |
| 13243 | + let root = std::env::temp_dir().join("afs_tests_failure_spec.afs"); |
| 13244 | + fs::write( |
| 13245 | + &root, |
| 13246 | + r#"suite "frontend/parser" |
| 13247 | + |
| 13248 | +case "missing_then" |
| 13249 | +source "../../fixtures/frontend/parser/missing_then.f90" |
| 13250 | +armfortas => tokens |
| 13251 | +expect tokens contains "if" |
| 13252 | +expect-fail parser contains "expected" |
| 13253 | +end |
| 13254 | +"#, |
| 13255 | + ) |
| 13256 | + .unwrap(); |
| 13257 | + |
| 13258 | + let suite = parse_suite_file(&root).unwrap(); |
| 13259 | + assert_eq!(suite.cases.len(), 1); |
| 13260 | + assert!(has_failure_expectation(&suite.cases[0])); |
| 13261 | + let _ = fs::remove_file(&root); |
| 13262 | + } |
| 13263 | + |
| 13264 | + #[test] |
| 13265 | + fn parses_failure_expectation_from_source_comments() { |
| 13266 | + let root = std::env::temp_dir().join("afs_tests_failure_comment_spec"); |
| 13267 | + let _ = fs::remove_dir_all(&root); |
| 13268 | + fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13269 | + fs::create_dir_all(root.join("suites")).unwrap(); |
| 13270 | + |
| 13271 | + let source = root.join("fixtures/error_expected.f90"); |
| 13272 | + fs::write( |
| 13273 | + &source, |
| 13274 | + "! ERROR_EXPECTED: hidden\nprogram error_expected\n print *, hidden\nend program\n", |
| 13275 | + ) |
| 13276 | + .unwrap(); |
| 13277 | + |
| 13278 | + let suite_path = root.join("suites/spec.afs"); |
| 13279 | + fs::write( |
| 13280 | + &suite_path, |
| 13281 | + r#"suite "v2/comment-failure" |
| 13282 | + |
| 13283 | +case "error_expected_comments" |
| 13284 | +source "../fixtures/error_expected.f90" |
| 13285 | +compiler armfortas => diagnostics |
| 13286 | +expect-fail comments |
| 13287 | +end |
| 13288 | +"#, |
| 13289 | + ) |
| 13290 | + .unwrap(); |
| 13291 | + |
| 13292 | + let suite = parse_suite_file(&suite_path).unwrap(); |
| 13293 | + match &suite.cases[0].expectations[0] { |
| 13294 | + Expectation::FailCommentPatterns(patterns) => { |
| 13295 | + assert_eq!(patterns, &vec!["hidden".to_string()]); |
| 13296 | + } |
| 13297 | + other => panic!("expected source-comment failure expectation, got {other:?}"), |
| 13298 | + } |
| 13299 | + |
| 13300 | + let _ = fs::remove_dir_all(&root); |
| 13301 | + } |
| 13302 | + |
| 13303 | + #[test] |
| 13304 | + fn resolves_xfail_comments_from_source() { |
| 13305 | + let root = std::env::temp_dir().join("afs_tests_xfail_comment_spec"); |
| 13306 | + let _ = fs::remove_dir_all(&root); |
| 13307 | + fs::create_dir_all(root.join("fixtures")).unwrap(); |
| 13308 | + fs::create_dir_all(root.join("suites")).unwrap(); |
| 13309 | + |
| 13310 | + let source = root.join("fixtures/xfail_case.f90"); |
| 13311 | + fs::write( |
| 13312 | + &source, |
| 13313 | + "! XFAIL: audit BLOCKING-1 (demo)\nprogram xfail_case\nend program\n", |
| 13314 | + ) |
| 13315 | + .unwrap(); |
| 13316 | + |
| 13317 | + let suite_path = root.join("suites/spec.afs"); |
| 13318 | + fs::write( |
| 13319 | + &suite_path, |
| 13320 | + r#"suite "v2/comment-xfail" |
| 13321 | + |
| 13322 | +case "xfail_comments" |
| 13323 | +source "../fixtures/xfail_case.f90" |
| 13324 | +compiler armfortas => runtime |
| 13325 | +xfail comments |
| 13326 | +end |
| 13327 | +"#, |
| 13328 | + ) |
| 13329 | + .unwrap(); |
| 13330 | + |
| 13331 | + let suite = parse_suite_file(&suite_path).unwrap(); |
| 13332 | + match status_for_opt(&suite.cases[0], OptLevel::O0) { |
| 13333 | + EffectiveStatus::Xfail(reason) => { |
| 13334 | + assert_eq!(reason, "audit BLOCKING-1 (demo)"); |
| 13335 | + } |
| 13336 | + other => panic!("expected xfail status, got {other:?}"), |
| 13337 | + } |
| 13338 | + |
| 13339 | + let _ = fs::remove_dir_all(&root); |
| 13340 | + } |
| 13341 | + |
| 13342 | + #[test] |
| 13343 | + fn check_matching_preserves_order() { |
| 13344 | + let checks = vec![ |
| 13345 | + Check { |
| 13346 | + line_num: 1, |
| 13347 | + pattern: "alpha".into(), |
| 13348 | + negative: false, |
| 13349 | + kind: "CHECK", |
| 13350 | + }, |
| 13351 | + Check { |
| 13352 | + line_num: 2, |
| 13353 | + pattern: "omega".into(), |
| 13354 | + negative: false, |
| 13355 | + kind: "CHECK", |
| 13356 | + }, |
| 13357 | + ]; |
| 13358 | + assert!(match_checks(&checks, "alpha\nmiddle\nomega\n", "demo").is_ok()); |
| 13359 | + assert!(match_checks(&checks, "omega\nalpha\n", "demo").is_err()); |
| 13360 | + } |
| 13361 | + |
| 13362 | + #[test] |
| 13363 | + fn ir_check_matching_supports_negative_patterns() { |
| 13364 | + let checks = vec![ |
| 13365 | + Check { |
| 13366 | + line_num: 1, |
| 13367 | + pattern: "func @demo".into(), |
| 13368 | + negative: false, |
| 13369 | + kind: "IR_CHECK", |
| 13370 | + }, |
| 13371 | + Check { |
| 13372 | + line_num: 2, |
| 13373 | + pattern: "zeroinit".into(), |
| 13374 | + negative: true, |
| 13375 | + kind: "IR_NOT", |
| 13376 | + }, |
| 13377 | + ]; |
| 13378 | + let ok_ir = "module main\n\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13379 | + assert!(match_checks(&checks, ok_ir, "demo").is_ok()); |
| 13380 | + |
| 13381 | + let bad_ir = "module main\n global @value: i32 = zeroinit\n func @demo() -> void {\n entry():\n ret void\n }\n"; |
| 13382 | + let err = match_checks(&checks, bad_ir, "demo").unwrap_err(); |
| 13383 | + assert!(err.contains("IR_NOT failed")); |
| 13384 | + } |
| 13385 | + |
| 13386 | + fn run_only_result(stdout: &str, stderr: &str, exit_code: i32) -> CaptureResult { |
| 13387 | + CaptureResult { |
| 13388 | + input: PathBuf::from("demo.f90"), |
| 13389 | + opt_level: OptLevel::O0, |
| 13390 | + stages: std::collections::BTreeMap::from([( |
| 13391 | + Stage::Run, |
| 13392 | + CapturedStage::Run(RunCapture { |
| 13393 | + exit_code, |
| 13394 | + stdout: stdout.into(), |
| 13395 | + stderr: stderr.into(), |
| 13396 | + }), |
| 13397 | + )]), |
| 13398 | + } |
| 13399 | + } |
| 13400 | + |
| 13401 | + fn reference_run( |
| 13402 | + compiler: ReferenceCompiler, |
| 13403 | + stdout: &str, |
| 13404 | + stderr: &str, |
| 13405 | + exit_code: i32, |
| 13406 | + ) -> ReferenceResult { |
| 13407 | + ReferenceResult { |
| 13408 | + compiler, |
| 13409 | + compile_command: format!("{} demo.f90 -o demo", compiler.as_str()), |
| 13410 | + compile_exit_code: 0, |
| 13411 | + compile_stdout: String::new(), |
| 13412 | + compile_stderr: String::new(), |
| 13413 | + run: Some(RunCapture { |
| 13414 | + exit_code, |
| 13415 | + stdout: stdout.into(), |
| 13416 | + stderr: stderr.into(), |
| 13417 | + }), |
| 13418 | + run_error: None, |
| 13419 | + } |
| 13420 | + } |
| 13421 | + |
| 13422 | + fn differential_armfortas_observation( |
| 13423 | + stdout: &str, |
| 13424 | + stderr: &str, |
| 13425 | + exit_code: i32, |
| 13426 | + ) -> CompilerObservation { |
| 13427 | + observed_program_from_armfortas_capture( |
| 13428 | + Path::new("demo.f90"), |
| 13429 | + OptLevel::O0, |
| 13430 | + default_differential_artifacts(), |
| 13431 | + &run_only_result(stdout, stderr, exit_code), |
| 13432 | + None, |
| 13433 | + ) |
| 13434 | + .observation |
| 13435 | + } |
| 13436 | + |
| 13437 | + fn differential_reference_observation( |
| 13438 | + compiler: ReferenceCompiler, |
| 13439 | + stdout: &str, |
| 13440 | + stderr: &str, |
| 13441 | + exit_code: i32, |
| 13442 | + ) -> CompilerObservation { |
| 13443 | + observed_program_from_reference_result( |
| 13444 | + Path::new("demo.f90"), |
| 13445 | + OptLevel::O0, |
| 13446 | + default_differential_artifacts(), |
| 13447 | + &reference_run(compiler, stdout, stderr, exit_code), |
| 13448 | + ) |
| 13449 | + .observation |
| 13450 | + } |
| 13451 | + |
| 13452 | + #[test] |
| 13453 | + fn not_contains_expectation_checks_text_absence() { |
| 13454 | + let case = CaseSpec { |
| 13455 | + name: "no_reserved_register".into(), |
| 13456 | + source: PathBuf::from("demo.f90"), |
| 13457 | + graph_files: Vec::new(), |
| 13458 | + requested: BTreeSet::from([Stage::Asm]), |
| 13459 | + generic_introspect: None, |
| 13460 | + generic_compare: None, |
| 13461 | + opt_levels: vec![OptLevel::O0], |
| 13462 | + repeat_count: 2, |
| 13463 | + reference_compilers: Vec::new(), |
| 13464 | + consistency_checks: Vec::new(), |
| 13465 | + expectations: vec![Expectation::NotContains { |
| 13466 | + target: Target::Stage(Stage::Asm), |
| 13467 | + needle: "x18".into(), |
| 13468 | + }], |
| 13469 | + status_rules: Vec::new(), |
| 13470 | + capability_policy: None, |
| 13471 | + }; |
| 13472 | + let result = CaptureResult { |
| 13473 | + input: PathBuf::from("demo.f90"), |
| 13474 | + opt_level: OptLevel::O0, |
| 13475 | + stages: std::collections::BTreeMap::from([( |
| 13476 | + Stage::Asm, |
| 13477 | + CapturedStage::Text("mov x19, x0\nret\n".into()), |
| 13478 | + )]), |
| 13479 | + }; |
| 13480 | + let observed = observed_program_from_armfortas_capture( |
| 13481 | + Path::new("demo.f90"), |
| 13482 | + OptLevel::O0, |
| 13483 | + expected_artifacts_for_legacy_case(&case), |
| 13484 | + &result, |
| 13485 | + None, |
| 13486 | + ); |
| 13487 | + assert!(evaluate_observation_expectations(&case, &observed).is_ok()); |
| 13488 | + |
| 13489 | + let bad = CaptureResult { |
| 13490 | + input: PathBuf::from("demo.f90"), |
| 13491 | + opt_level: OptLevel::O0, |
| 13492 | + stages: std::collections::BTreeMap::from([( |
| 13493 | + Stage::Asm, |
| 13494 | + CapturedStage::Text("mov x18, x0\nret\n".into()), |
| 13495 | + )]), |
| 13496 | + }; |
| 13497 | + let observed = observed_program_from_armfortas_capture( |
| 13498 | + Path::new("demo.f90"), |
| 13499 | + OptLevel::O0, |
| 13500 | + expected_artifacts_for_legacy_case(&case), |
| 13501 | + &bad, |
| 13502 | + None, |
| 13503 | + ); |
| 13504 | + let err = evaluate_observation_expectations(&case, &observed).unwrap_err(); |
| 13505 | + assert!(err.contains("expected asm to not contain")); |
| 13506 | + } |
| 13507 | + |
| 13508 | + #[test] |
| 13509 | + fn writes_failure_bundle_with_artifacts() { |
| 13510 | + let source = std::env::temp_dir().join("afs_tests_bundle_source.f90"); |
| 13511 | + fs::write(&source, "program hello\nprint *, 'hello'\nend program\n").unwrap(); |
| 13512 | + |
| 13513 | + let suite = SuiteSpec { |
| 13514 | + name: "runtime/bundles".into(), |
| 13515 | + path: PathBuf::from("/tmp/runtime/bundles.afs"), |
| 13516 | + cases: Vec::new(), |
| 13517 | + }; |
| 13518 | + let case = CaseSpec { |
| 13519 | + name: "hello_bundle".into(), |
| 13520 | + source: source.clone(), |
| 13521 | + graph_files: Vec::new(), |
| 13522 | + requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13523 | + generic_introspect: None, |
| 13524 | + generic_compare: None, |
| 13525 | + opt_levels: vec![OptLevel::O0], |
| 13526 | + repeat_count: 3, |
| 13527 | + reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 13528 | + consistency_checks: vec![ConsistencyCheck::CliObjVsSystemAs], |
| 13529 | + expectations: Vec::new(), |
| 13530 | + status_rules: Vec::new(), |
| 13531 | + capability_policy: None, |
| 13532 | + }; |
| 13533 | + let mut stages = std::collections::BTreeMap::new(); |
| 13534 | + stages.insert(Stage::Ir, CapturedStage::Text("module main".into())); |
| 13535 | + stages.insert( |
| 13536 | + Stage::Run, |
| 13537 | + CapturedStage::Run(RunCapture { |
| 13538 | + exit_code: 1, |
| 13539 | + stdout: "oops\n".into(), |
| 13540 | + stderr: "broken\n".into(), |
| 13541 | + }), |
| 13542 | + ); |
| 13543 | + let artifacts = ExecutionArtifacts { |
| 13544 | + requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 13545 | + armfortas: None, |
| 13546 | + armfortas_failure: Some(CaptureFailure { |
| 13547 | + input: source.clone(), |
| 13548 | + opt_level: OptLevel::O0, |
| 13549 | + stage: FailureStage::Sema, |
| 13550 | + detail: "compiler failed".into(), |
| 13551 | + stages, |
| 13552 | + }), |
| 13553 | + armfortas_observation: Some(ObservedProgram { |
| 13554 | + observation: CompilerObservation { |
| 13555 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13556 | + program: source.clone(), |
| 13557 | + opt_level: OptLevel::O0, |
| 13558 | + compile_exit_code: 1, |
| 13559 | + artifacts: BTreeMap::from([ |
| 13560 | + ( |
| 13561 | + ArtifactKey::Diagnostics, |
| 13562 | + ArtifactValue::Text("cached observation failure".into()), |
| 13563 | + ), |
| 13564 | + ( |
| 13565 | + ArtifactKey::Extra("armfortas.ast".into()), |
| 13566 | + ArtifactValue::Text("program hello".into()), |
| 13567 | + ), |
| 13568 | + ]), |
| 13569 | + provenance: ObservationProvenance { |
| 13570 | + compiler_identity: "armfortas".into(), |
| 13571 | + adapter_kind: "named".into(), |
| 13572 | + backend_mode: "linked".into(), |
| 13573 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13574 | + artifacts_captured: vec!["diagnostics".into(), "armfortas.ast".into()], |
| 13575 | + comparison_basis: None, |
| 13576 | + failure_stage: Some("sema".into()), |
| 13577 | + }, |
| 13578 | + }, |
| 13579 | + requested_artifacts: BTreeSet::from([ |
| 13580 | + ArtifactKey::Diagnostics, |
| 13581 | + ArtifactKey::Extra("armfortas.ast".into()), |
| 13582 | + ]), |
| 13583 | + }), |
| 13584 | + references: vec![ReferenceResult { |
| 13585 | + compiler: ReferenceCompiler::Gfortran, |
| 13586 | + compile_command: "gfortran hello.f90 -o hello".into(), |
| 13587 | + compile_exit_code: 0, |
| 13588 | + compile_stdout: String::new(), |
| 13589 | + compile_stderr: String::new(), |
| 13590 | + run: Some(RunCapture { |
| 13591 | + exit_code: 0, |
| 13592 | + stdout: "hello\n".into(), |
| 13593 | + stderr: String::new(), |
| 13594 | + }), |
| 13595 | + run_error: None, |
| 13596 | + }], |
| 13597 | + reference_observations: vec![ObservedProgram { |
| 13598 | + observation: CompilerObservation { |
| 13599 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 13600 | + program: source.clone(), |
| 13601 | + opt_level: OptLevel::O0, |
| 13602 | + compile_exit_code: 0, |
| 13603 | + artifacts: BTreeMap::from([( |
| 13604 | + ArtifactKey::Asm, |
| 13605 | + ArtifactValue::Text(".globl _main".into()), |
| 13606 | + )]), |
| 13607 | + provenance: ObservationProvenance { |
| 13608 | + compiler_identity: "gfortran".into(), |
| 13609 | + adapter_kind: "named".into(), |
| 13610 | + backend_mode: "legacy-reference".into(), |
| 13611 | + backend_detail: "cached reference observation".into(), |
| 13612 | + artifacts_captured: vec!["asm".into()], |
| 13613 | + comparison_basis: None, |
| 13614 | + failure_stage: None, |
| 13615 | + }, |
| 13616 | + }, |
| 13617 | + requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 13618 | + }], |
| 13619 | + consistency_issues: { |
| 13620 | + let asm_temp_root = |
| 13621 | + std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm"); |
| 13622 | + fs::create_dir_all(&asm_temp_root).unwrap(); |
| 13623 | + fs::write(asm_temp_root.join("run_00.s"), "mov x19, x0\n").unwrap(); |
| 13624 | + |
| 13625 | + let obj_temp_root = |
| 13626 | + std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj"); |
| 13627 | + fs::create_dir_all(&obj_temp_root).unwrap(); |
| 13628 | + fs::write(obj_temp_root.join("run_00.o"), "fake object bytes\n").unwrap(); |
| 13629 | + |
| 13630 | + vec![ |
| 13631 | + ConsistencyIssue { |
| 13632 | + check: ConsistencyCheck::CliAsmReproducible, |
| 13633 | + summary: "repeat_count=3 unique_variants=3".into(), |
| 13634 | + repeat_count: Some(3), |
| 13635 | + unique_variant_count: Some(3), |
| 13636 | + varying_components: Vec::new(), |
| 13637 | + stable_components: Vec::new(), |
| 13638 | + detail: "assembly output is not reproducible".into(), |
| 13639 | + temp_root: asm_temp_root, |
| 13640 | + }, |
| 13641 | + ConsistencyIssue { |
| 13642 | + check: ConsistencyCheck::CliObjReproducible, |
| 13643 | + summary: "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols".into(), |
| 13644 | + repeat_count: Some(3), |
| 13645 | + unique_variant_count: Some(2), |
| 13646 | + varying_components: vec!["text".into()], |
| 13647 | + stable_components: vec![ |
| 13648 | + "load_commands".into(), |
| 13649 | + "relocations".into(), |
| 13650 | + "symbols".into(), |
| 13651 | + ], |
| 13652 | + detail: "object output is not reproducible".into(), |
| 13653 | + temp_root: obj_temp_root, |
| 13654 | + }, |
| 13655 | + ] |
| 13656 | + }, |
| 13657 | + }; |
| 13658 | + let outcome = Outcome { |
| 13659 | + suite: suite.name.clone(), |
| 13660 | + case: case.name.clone(), |
| 13661 | + opt_level: OptLevel::O0, |
| 13662 | + kind: OutcomeKind::Fail, |
| 13663 | + detail: "boom".into(), |
| 13664 | + bundle: None, |
| 13665 | + primary_backend: Some(PrimaryBackendReport { |
| 13666 | + kind: "full".into(), |
| 13667 | + mode: "linked".into(), |
| 13668 | + detail: "linked armfortas::testing capture adapter".into(), |
| 13669 | + }), |
| 13670 | + consistency_observations: Vec::new(), |
| 13671 | + }; |
| 13672 | + let prepared = PreparedInput { |
| 13673 | + compiler_source: source.clone(), |
| 13674 | + generated_source: None, |
| 13675 | + temp_root: None, |
| 13676 | + }; |
| 13677 | + |
| 13678 | + let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13679 | + assert!(bundle.join("metadata.txt").exists()); |
| 13680 | + assert!(bundle.join("detail.txt").exists()); |
| 13681 | + assert!(bundle.join("source.f90").exists()); |
| 13682 | + assert!(bundle.join("armfortas").join("ir.txt").exists()); |
| 13683 | + assert!(bundle.join("armfortas").join("metadata.txt").exists()); |
| 13684 | + assert!(bundle.join("armfortas").join("observation.txt").exists()); |
| 13685 | + assert!(bundle.join("armfortas").join("observation.json").exists()); |
| 13686 | + assert!(bundle.join("armfortas").join("observation.md").exists()); |
| 13687 | + assert!(bundle.join("armfortas").join("run.stdout.txt").exists()); |
| 13688 | + assert!(bundle.join("armfortas").join("error.txt").exists()); |
| 13689 | + assert!(bundle |
| 13690 | + .join("references") |
| 13691 | + .join("gfortran") |
| 13692 | + .join("observation.txt") |
| 13693 | + .exists()); |
| 13694 | + assert!(bundle |
| 13695 | + .join("references") |
| 13696 | + .join("gfortran") |
| 13697 | + .join("observation.json") |
| 13698 | + .exists()); |
| 13699 | + assert!(bundle |
| 13700 | + .join("references") |
| 13701 | + .join("gfortran") |
| 13702 | + .join("observation.md") |
| 13703 | + .exists()); |
| 13704 | + assert!(bundle.join("references").join("summary.txt").exists()); |
| 13705 | + assert!(bundle |
| 13706 | + .join("references") |
| 13707 | + .join("gfortran") |
| 13708 | + .join("run.stdout.txt") |
| 13709 | + .exists()); |
| 13710 | + assert!(bundle.join("consistency").join("summary.txt").exists()); |
| 13711 | + let metadata = fs::read_to_string(bundle.join("metadata.txt")).unwrap(); |
| 13712 | + assert!(metadata.contains("primary_backend_kind: full")); |
| 13713 | + assert!(metadata.contains("primary_backend_mode: linked")); |
| 13714 | + assert!( |
| 13715 | + metadata.contains("primary_backend_detail: linked armfortas::testing capture adapter") |
| 13716 | + ); |
| 13717 | + let armfortas_metadata = |
| 13718 | + fs::read_to_string(bundle.join("armfortas").join("metadata.txt")).unwrap(); |
| 13719 | + assert!(armfortas_metadata.contains("primary_backend_kind: full")); |
| 13720 | + assert!(armfortas_metadata.contains("primary_backend_mode: linked")); |
| 13721 | + assert!(armfortas_metadata |
| 13722 | + .contains("primary_backend_detail: linked armfortas::testing capture adapter")); |
| 13723 | + assert!(armfortas_metadata.contains("captured_stages: ir, run")); |
| 13724 | + assert!(armfortas_metadata.contains("error_stage: sema")); |
| 13725 | + let observation = |
| 13726 | + fs::read_to_string(bundle.join("armfortas").join("observation.txt")).unwrap(); |
| 13727 | + assert!(observation.contains("Introspect")); |
| 13728 | + assert!(observation.contains("compiler: armfortas")); |
| 13729 | + assert!(observation.contains("failure_stage: sema")); |
| 13730 | + assert!(observation.contains("generic_artifacts: diagnostics")); |
| 13731 | + assert!(observation.contains("adapter_extras: armfortas(ast)")); |
| 13732 | + assert!(observation.contains("cached observation failure")); |
| 13733 | + let reference_observation = fs::read_to_string( |
| 13734 | + bundle |
| 13735 | + .join("references") |
| 13736 | + .join("gfortran") |
| 13737 | + .join("observation.txt"), |
| 13738 | + ) |
| 13739 | + .unwrap(); |
| 13740 | + assert!(reference_observation.contains("Introspect")); |
| 13741 | + assert!(reference_observation.contains("compiler: gfortran")); |
| 13742 | + assert!(reference_observation.contains("status: compile ok")); |
| 13743 | + assert!(reference_observation.contains("requested_artifacts: asm")); |
| 13744 | + assert!(reference_observation.contains("generic_artifacts: asm")); |
| 13745 | + assert!(reference_observation.contains("cached reference observation")); |
| 13746 | + let reference_summary = |
| 13747 | + fs::read_to_string(bundle.join("references").join("summary.txt")).unwrap(); |
| 13748 | + assert!(reference_summary.contains("reference_count: 1")); |
| 13749 | + assert!(reference_summary.contains("compilers: gfortran")); |
| 13750 | + assert!(reference_summary.contains("compiler: gfortran")); |
| 13751 | + assert!(reference_summary.contains("status: compile ok")); |
| 13752 | + assert!(reference_summary.contains("compile_exit_code: 0")); |
| 13753 | + assert!(reference_summary.contains("command: gfortran hello.f90 -o hello")); |
| 13754 | + assert!(reference_summary.contains("generic_artifacts: asm")); |
| 13755 | + assert!(reference_summary.contains("adapter_extras: none")); |
| 13756 | + let consistency_summary = |
| 13757 | + fs::read_to_string(bundle.join("consistency").join("summary.txt")).unwrap(); |
| 13758 | + assert!(consistency_summary.contains("issue_count: 2")); |
| 13759 | + assert!(consistency_summary.contains("checks: cli_asm_reproducible, cli_obj_reproducible")); |
| 13760 | + assert!(consistency_summary.contains("repeat_counts: 3")); |
| 13761 | + assert!(consistency_summary.contains("unique_variants: 2, 3")); |
| 13762 | + assert!(consistency_summary.contains("varying_components: text")); |
| 13763 | + assert!( |
| 13764 | + consistency_summary.contains("stable_components: load_commands, relocations, symbols") |
| 13765 | + ); |
| 13766 | + assert!(bundle |
| 13767 | + .join("consistency") |
| 13768 | + .join("cli_asm_reproducible") |
| 13769 | + .join("summary.txt") |
| 13770 | + .exists()); |
| 13771 | + assert!(bundle |
| 13772 | + .join("consistency") |
| 13773 | + .join("cli_asm_reproducible") |
| 13774 | + .join("detail.txt") |
| 13775 | + .exists()); |
| 13776 | + assert!(bundle |
| 13777 | + .join("consistency") |
| 13778 | + .join("cli_asm_reproducible") |
| 13779 | + .join("artifacts") |
| 13780 | + .join("run_00.s") |
| 13781 | + .exists()); |
| 13782 | + assert!(bundle |
| 13783 | + .join("consistency") |
| 13784 | + .join("cli_obj_reproducible") |
| 13785 | + .join("artifacts") |
| 13786 | + .join("run_00.o") |
| 13787 | + .exists()); |
| 13788 | + |
| 13789 | + let _ = fs::remove_dir_all(bundle); |
| 13790 | + let _ = |
| 13791 | + fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm")); |
| 13792 | + let _ = |
| 13793 | + fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj")); |
| 13794 | + let _ = fs::remove_file(source); |
| 4740 | 13795 | } |
| 4741 | | - out.trim_matches('_').to_string() |
| 4742 | | -} |
| 4743 | 13796 | |
| 4744 | | -fn next_report_temp_root(compiler: ReferenceCompiler, opt_level: OptLevel) -> PathBuf { |
| 4745 | | - default_report_root().join(".tmp").join(format!( |
| 4746 | | - "{}_{}_{}", |
| 4747 | | - sanitize_component(compiler.as_str()), |
| 4748 | | - opt_level.as_str().to_ascii_lowercase(), |
| 4749 | | - next_report_suffix(opt_level) |
| 4750 | | - )) |
| 4751 | | -} |
| 13797 | + #[test] |
| 13798 | + fn materializes_graph_input_in_declared_file_order() { |
| 13799 | + let root = std::env::temp_dir().join("afs_tests_graph_materialize"); |
| 13800 | + let _ = fs::remove_dir_all(&root); |
| 13801 | + fs::create_dir_all(&root).unwrap(); |
| 13802 | + let module = root.join("math_values.f90"); |
| 13803 | + let main = root.join("main.f90"); |
| 13804 | + fs::write(&module, "module math_values\ncontains\nend module\n").unwrap(); |
| 13805 | + fs::write(&main, "program main\nuse math_values\nend program\n").unwrap(); |
| 4752 | 13806 | |
| 4753 | | -fn next_consistency_temp_root(opt_level: OptLevel) -> PathBuf { |
| 4754 | | - default_report_root().join(".tmp").join(format!( |
| 4755 | | - "consistency_{}_{}", |
| 4756 | | - opt_level.as_str().to_ascii_lowercase(), |
| 4757 | | - next_report_suffix(opt_level) |
| 4758 | | - )) |
| 4759 | | -} |
| 13807 | + let suite = SuiteSpec { |
| 13808 | + name: "modules/graph".into(), |
| 13809 | + path: root.join("graph.afs"), |
| 13810 | + cases: Vec::new(), |
| 13811 | + }; |
| 13812 | + let case = CaseSpec { |
| 13813 | + name: "basic_use".into(), |
| 13814 | + source: main.clone(), |
| 13815 | + graph_files: vec![module.clone(), main.clone()], |
| 13816 | + requested: BTreeSet::from([Stage::Run]), |
| 13817 | + generic_introspect: None, |
| 13818 | + generic_compare: None, |
| 13819 | + opt_levels: vec![OptLevel::O0], |
| 13820 | + repeat_count: 2, |
| 13821 | + reference_compilers: Vec::new(), |
| 13822 | + consistency_checks: Vec::new(), |
| 13823 | + expectations: Vec::new(), |
| 13824 | + status_rules: Vec::new(), |
| 13825 | + capability_policy: None, |
| 13826 | + }; |
| 4760 | 13827 | |
| 4761 | | -fn next_report_suffix(opt_level: OptLevel) -> String { |
| 4762 | | - format!( |
| 4763 | | - "{}-{}-{:04}", |
| 4764 | | - opt_level.as_str().to_ascii_lowercase(), |
| 4765 | | - std::process::id(), |
| 4766 | | - REPORT_COUNTER.fetch_add(1, Ordering::Relaxed) |
| 4767 | | - ) |
| 4768 | | -} |
| 13828 | + let prepared = prepare_case_input(&case, &suite, OptLevel::O0).unwrap(); |
| 13829 | + let generated = fs::read_to_string(&prepared.compiler_source).unwrap(); |
| 13830 | + assert!(generated.contains("module math_values")); |
| 13831 | + assert!(generated.contains("program main")); |
| 13832 | + assert!( |
| 13833 | + generated.find("module math_values").unwrap() < generated.find("program main").unwrap() |
| 13834 | + ); |
| 4769 | 13835 | |
| 4770 | | -fn print_outcome(outcome: &Outcome) { |
| 4771 | | - let label = format!( |
| 4772 | | - "{}::{}[{}]", |
| 4773 | | - outcome.suite, |
| 4774 | | - outcome.case, |
| 4775 | | - outcome.opt_level.as_str() |
| 4776 | | - ); |
| 4777 | | - match outcome.kind { |
| 4778 | | - OutcomeKind::Pass => println!("PASS {}", label), |
| 4779 | | - OutcomeKind::Fail => { |
| 4780 | | - println!("FAIL {}", label); |
| 4781 | | - if !outcome.detail.is_empty() { |
| 4782 | | - println!("{}", outcome.detail); |
| 4783 | | - } |
| 4784 | | - } |
| 4785 | | - OutcomeKind::Xfail => { |
| 4786 | | - println!("XFAIL {}", label); |
| 4787 | | - if !outcome.detail.is_empty() { |
| 4788 | | - println!("{}", outcome.detail); |
| 4789 | | - } |
| 4790 | | - } |
| 4791 | | - OutcomeKind::Xpass => { |
| 4792 | | - println!("XPASS {}", label); |
| 4793 | | - if !outcome.detail.is_empty() { |
| 4794 | | - println!("{}", outcome.detail); |
| 4795 | | - } |
| 4796 | | - } |
| 4797 | | - OutcomeKind::Future => { |
| 4798 | | - println!("FUTURE {}", label); |
| 4799 | | - if !outcome.detail.is_empty() { |
| 4800 | | - println!("{}", outcome.detail); |
| 4801 | | - } |
| 4802 | | - } |
| 4803 | | - } |
| 4804 | | - if let Some(bundle) = &outcome.bundle { |
| 4805 | | - println!("bundle: {}", bundle.display()); |
| 13836 | + cleanup_prepared_input(&prepared); |
| 13837 | + let _ = fs::remove_dir_all(&root); |
| 4806 | 13838 | } |
| 4807 | | -} |
| 4808 | | - |
| 4809 | | -fn print_summary(summary: &Summary) { |
| 4810 | | - println!(); |
| 4811 | | - println!("{}", render_summary(summary)); |
| 4812 | | -} |
| 4813 | 13839 | |
| 4814 | | -#[derive(Debug, Clone)] |
| 4815 | | -struct Check { |
| 4816 | | - line_num: usize, |
| 4817 | | - pattern: String, |
| 4818 | | -} |
| 13840 | + #[test] |
| 13841 | + fn graph_failure_bundle_writes_authored_sources() { |
| 13842 | + let root = std::env::temp_dir().join("afs_tests_graph_bundle"); |
| 13843 | + let _ = fs::remove_dir_all(&root); |
| 13844 | + fs::create_dir_all(&root).unwrap(); |
| 13845 | + let module = root.join("math_values.f90"); |
| 13846 | + let main = root.join("main.f90"); |
| 13847 | + let generated = root.join("generated.f90"); |
| 13848 | + fs::write( |
| 13849 | + &module, |
| 13850 | + "module math_values\n integer :: answer = 42\nend module\n", |
| 13851 | + ) |
| 13852 | + .unwrap(); |
| 13853 | + fs::write( |
| 13854 | + &main, |
| 13855 | + "program main\n use math_values\n print *, answer\nend program\n", |
| 13856 | + ) |
| 13857 | + .unwrap(); |
| 13858 | + fs::write(&generated, "module math_values\n integer :: answer = 42\nend module\n\nprogram main\n use math_values\n print *, answer\nend program\n").unwrap(); |
| 4819 | 13859 | |
| 4820 | | -fn extract_checks(source: &str) -> Vec<Check> { |
| 4821 | | - source |
| 4822 | | - .lines() |
| 4823 | | - .enumerate() |
| 4824 | | - .filter_map(|(i, line)| { |
| 4825 | | - let trimmed = line.trim(); |
| 4826 | | - trimmed.strip_prefix("! CHECK:").map(|rest| Check { |
| 4827 | | - line_num: i + 1, |
| 4828 | | - pattern: rest.trim().to_string(), |
| 4829 | | - }) |
| 4830 | | - }) |
| 4831 | | - .collect() |
| 4832 | | -} |
| 13860 | + let suite = SuiteSpec { |
| 13861 | + name: "modules/bundles".into(), |
| 13862 | + path: root.join("bundle.afs"), |
| 13863 | + cases: Vec::new(), |
| 13864 | + }; |
| 13865 | + let case = CaseSpec { |
| 13866 | + name: "graph_bundle".into(), |
| 13867 | + source: main.clone(), |
| 13868 | + graph_files: vec![module.clone(), main.clone()], |
| 13869 | + requested: BTreeSet::from([Stage::Run]), |
| 13870 | + generic_introspect: None, |
| 13871 | + generic_compare: None, |
| 13872 | + opt_levels: vec![OptLevel::O0], |
| 13873 | + repeat_count: 2, |
| 13874 | + reference_compilers: Vec::new(), |
| 13875 | + consistency_checks: Vec::new(), |
| 13876 | + expectations: Vec::new(), |
| 13877 | + status_rules: Vec::new(), |
| 13878 | + capability_policy: None, |
| 13879 | + }; |
| 13880 | + let outcome = Outcome { |
| 13881 | + suite: suite.name.clone(), |
| 13882 | + case: case.name.clone(), |
| 13883 | + opt_level: OptLevel::O0, |
| 13884 | + kind: OutcomeKind::Fail, |
| 13885 | + detail: "boom".into(), |
| 13886 | + bundle: None, |
| 13887 | + primary_backend: Some(PrimaryBackendReport { |
| 13888 | + kind: "full".into(), |
| 13889 | + mode: "linked".into(), |
| 13890 | + detail: "linked armfortas::testing capture adapter".into(), |
| 13891 | + }), |
| 13892 | + consistency_observations: Vec::new(), |
| 13893 | + }; |
| 13894 | + let artifacts = ExecutionArtifacts { |
| 13895 | + requested: BTreeSet::from([Stage::Run]), |
| 13896 | + armfortas: Some(run_only_result("42\n", "", 0)), |
| 13897 | + armfortas_failure: None, |
| 13898 | + armfortas_observation: None, |
| 13899 | + references: Vec::new(), |
| 13900 | + reference_observations: Vec::new(), |
| 13901 | + consistency_issues: Vec::new(), |
| 13902 | + }; |
| 13903 | + let prepared = PreparedInput { |
| 13904 | + compiler_source: generated.clone(), |
| 13905 | + generated_source: Some(generated.clone()), |
| 13906 | + temp_root: None, |
| 13907 | + }; |
| 4833 | 13908 | |
| 4834 | | -fn match_checks(checks: &[Check], output: &str, case_name: &str) -> Result<(), String> { |
| 4835 | | - let output_lines: Vec<&str> = output.lines().collect(); |
| 4836 | | - let mut output_idx = 0; |
| 13909 | + let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 13910 | + assert!(bundle.join("source.f90").exists()); |
| 13911 | + assert!(bundle.join("sources").join("00_math_values.f90").exists()); |
| 13912 | + assert!(bundle.join("sources").join("01_main.f90").exists()); |
| 4837 | 13913 | |
| 4838 | | - for check in checks { |
| 4839 | | - let mut found = false; |
| 4840 | | - while output_idx < output_lines.len() { |
| 4841 | | - if output_lines[output_idx].trim().contains(&check.pattern) { |
| 4842 | | - found = true; |
| 4843 | | - output_idx += 1; |
| 4844 | | - break; |
| 4845 | | - } |
| 4846 | | - output_idx += 1; |
| 4847 | | - } |
| 4848 | | - if !found { |
| 4849 | | - return Err(format!( |
| 4850 | | - "{}:{}: CHECK failed: expected '{}' not found in remaining output\nfull output:\n{}", |
| 4851 | | - case_name, check.line_num, check.pattern, output |
| 4852 | | - )); |
| 4853 | | - } |
| 13914 | + let _ = fs::remove_dir_all(bundle); |
| 13915 | + let _ = fs::remove_dir_all(&root); |
| 4854 | 13916 | } |
| 4855 | 13917 | |
| 4856 | | - Ok(()) |
| 4857 | | -} |
| 13918 | + #[test] |
| 13919 | + fn armfortas_bundle_observation_prefers_cached_observation() { |
| 13920 | + let prepared = PreparedInput { |
| 13921 | + compiler_source: PathBuf::from("demo.f90"), |
| 13922 | + generated_source: None, |
| 13923 | + temp_root: None, |
| 13924 | + }; |
| 13925 | + let artifacts = ExecutionArtifacts { |
| 13926 | + requested: BTreeSet::from([Stage::Run]), |
| 13927 | + armfortas: Some(run_only_result("42\n", "", 0)), |
| 13928 | + armfortas_failure: None, |
| 13929 | + armfortas_observation: Some(ObservedProgram { |
| 13930 | + observation: CompilerObservation { |
| 13931 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 13932 | + program: PathBuf::from("demo.f90"), |
| 13933 | + opt_level: OptLevel::O0, |
| 13934 | + compile_exit_code: 0, |
| 13935 | + artifacts: BTreeMap::from([( |
| 13936 | + ArtifactKey::Extra("armfortas.sema".into()), |
| 13937 | + ArtifactValue::Text("ok".into()), |
| 13938 | + )]), |
| 13939 | + provenance: ObservationProvenance { |
| 13940 | + compiler_identity: "armfortas".into(), |
| 13941 | + adapter_kind: "named".into(), |
| 13942 | + backend_mode: "linked".into(), |
| 13943 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 13944 | + artifacts_captured: vec!["armfortas.sema".into()], |
| 13945 | + comparison_basis: None, |
| 13946 | + failure_stage: None, |
| 13947 | + }, |
| 13948 | + }, |
| 13949 | + requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.sema".into())]), |
| 13950 | + }), |
| 13951 | + references: Vec::new(), |
| 13952 | + reference_observations: Vec::new(), |
| 13953 | + consistency_issues: Vec::new(), |
| 13954 | + }; |
| 4858 | 13955 | |
| 4859 | | -#[cfg(test)] |
| 4860 | | -mod tests { |
| 4861 | | - use super::*; |
| 4862 | | - use crate::compiler::test_support::{ |
| 4863 | | - verify_module, BlockParam, FloatWidth, Function, Inst, InstKind, IntWidth, IrType, Module, |
| 4864 | | - Position, Span, Terminator, ValueId, |
| 4865 | | - }; |
| 13956 | + let observed = observed_program_for_armfortas_bundle(&prepared, &artifacts).unwrap(); |
| 13957 | + assert!(observed |
| 13958 | + .observation |
| 13959 | + .artifacts |
| 13960 | + .contains_key(&ArtifactKey::Extra("armfortas.sema".into()))); |
| 13961 | + assert!(!observed |
| 13962 | + .observation |
| 13963 | + .artifacts |
| 13964 | + .contains_key(&ArtifactKey::Runtime)); |
| 13965 | + } |
| 13966 | + |
| 13967 | + #[test] |
| 13968 | + fn render_summary_includes_consistency_rollups() { |
| 13969 | + let mut summary = Summary::default(); |
| 13970 | + summary.record_consistency(&[ |
| 13971 | + ConsistencyObservation { |
| 13972 | + check: ConsistencyCheck::CliAsmReproducible, |
| 13973 | + summary: "repeat_count=3 unique_variants=3".into(), |
| 13974 | + repeat_count: Some(3), |
| 13975 | + unique_variant_count: Some(3), |
| 13976 | + varying_components: Vec::new(), |
| 13977 | + stable_components: Vec::new(), |
| 13978 | + }, |
| 13979 | + ConsistencyObservation { |
| 13980 | + check: ConsistencyCheck::CliObjReproducible, |
| 13981 | + summary: |
| 13982 | + "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols" |
| 13983 | + .into(), |
| 13984 | + repeat_count: Some(3), |
| 13985 | + unique_variant_count: Some(2), |
| 13986 | + varying_components: vec!["text".into()], |
| 13987 | + stable_components: vec![ |
| 13988 | + "load_commands".into(), |
| 13989 | + "relocations".into(), |
| 13990 | + "symbols".into(), |
| 13991 | + ], |
| 13992 | + }, |
| 13993 | + ]); |
| 4866 | 13994 | |
| 4867 | | - fn dummy_span() -> Span { |
| 4868 | | - Span { |
| 4869 | | - file_id: 0, |
| 4870 | | - start: Position { line: 1, col: 1 }, |
| 4871 | | - end: Position { line: 1, col: 1 }, |
| 4872 | | - } |
| 13995 | + let rendered = render_summary(&summary); |
| 13996 | + assert!(rendered.contains("Consistency")); |
| 13997 | + assert!(rendered.contains("affected_checks: 2")); |
| 13998 | + assert!(rendered.contains("cells_with_issues: 2")); |
| 13999 | + assert!( |
| 14000 | + rendered.contains("cli_asm_reproducible: 1 cells; repeat_count=3; unique_variants=3") |
| 14001 | + ); |
| 14002 | + assert!(rendered.contains( |
| 14003 | + "cli_obj_reproducible: 1 cells; repeat_count=3; unique_variants=2; varying=text; stable=load_commands, relocations, symbols" |
| 14004 | + )); |
| 4873 | 14005 | } |
| 4874 | 14006 | |
| 4875 | 14007 | #[test] |
| 4876 | | - fn parses_suite_and_case() { |
| 4877 | | - let root = std::env::temp_dir().join("afs_tests_parser_spec.afs"); |
| 4878 | | - fs::write( |
| 4879 | | - &root, |
| 4880 | | - r#"suite "runtime/smoke" |
| 14008 | + fn render_reports_include_outcomes() { |
| 14009 | + let mut summary = Summary::default(); |
| 14010 | + summary.record_outcome(&Outcome { |
| 14011 | + suite: "modules/runtime-graphs".into(), |
| 14012 | + case: "module_chain_runtime".into(), |
| 14013 | + opt_level: OptLevel::O0, |
| 14014 | + kind: OutcomeKind::Xfail, |
| 14015 | + detail: "expected 42, got 0".into(), |
| 14016 | + bundle: Some(PathBuf::from("/tmp/bundle")), |
| 14017 | + primary_backend: Some(PrimaryBackendReport { |
| 14018 | + kind: "observable".into(), |
| 14019 | + mode: "cli-observable".into(), |
| 14020 | + detail: "cli-observable armfortas driver capture adapter".into(), |
| 14021 | + }), |
| 14022 | + consistency_observations: vec![ConsistencyObservation { |
| 14023 | + check: ConsistencyCheck::CliRunReproducible, |
| 14024 | + summary: "repeat_count=3 unique_variants=1".into(), |
| 14025 | + repeat_count: Some(3), |
| 14026 | + unique_variant_count: Some(1), |
| 14027 | + varying_components: Vec::new(), |
| 14028 | + stable_components: vec!["exit_code".into(), "stdout".into(), "stderr".into()], |
| 14029 | + }], |
| 14030 | + }); |
| 4881 | 14031 | |
| 4882 | | -case "hello" |
| 4883 | | -source "../../../test_programs/hello.f90" |
| 4884 | | -armfortas => run, ir |
| 4885 | | -expect run.stdout check-comments |
| 4886 | | -expect ir contains "module main" |
| 4887 | | -expect asm not-contains "x18" |
| 4888 | | -end |
| 4889 | | -"#, |
| 4890 | | - ) |
| 4891 | | - .unwrap(); |
| 14032 | + let json = render_json_report(&summary); |
| 14033 | + assert!(json.contains("\"outcomes\": [")); |
| 14034 | + assert!(json.contains("\"suite\": \"modules/runtime-graphs\"")); |
| 14035 | + assert!(json.contains("\"bundle\": \"/tmp/bundle\"")); |
| 14036 | + assert!(json.contains("\"primary_backend\": {")); |
| 14037 | + assert!(json.contains("\"mode\": \"cli-observable\"")); |
| 4892 | 14038 | |
| 4893 | | - let suite = parse_suite_file(&root).unwrap(); |
| 4894 | | - assert_eq!(suite.name, "runtime/smoke"); |
| 4895 | | - assert_eq!(suite.cases.len(), 1); |
| 4896 | | - assert!(suite.cases[0].requested.contains(&Stage::Run)); |
| 4897 | | - assert!(suite.cases[0].requested.contains(&Stage::Ir)); |
| 4898 | | - assert!(matches!( |
| 4899 | | - suite.cases[0].expectations[2], |
| 4900 | | - Expectation::NotContains { |
| 4901 | | - target: Target::Stage(Stage::Asm), |
| 4902 | | - .. |
| 4903 | | - } |
| 4904 | | - )); |
| 4905 | | - assert_eq!(suite.cases[0].opt_levels, vec![OptLevel::O0]); |
| 4906 | | - let _ = fs::remove_file(&root); |
| 14039 | + let markdown = render_markdown_report(&summary); |
| 14040 | + assert!(markdown.contains("# afs-tests report")); |
| 14041 | + assert!(markdown |
| 14042 | + .contains("### `modules/runtime-graphs` / `module_chain_runtime` / `O0` / `xfail`")); |
| 14043 | + assert!(markdown.contains("primary_backend: `observable` (`cli-observable`)")); |
| 14044 | + assert!(markdown.contains("bundle: `/tmp/bundle`")); |
| 14045 | + assert!(markdown.contains("expected 42, got 0")); |
| 4907 | 14046 | } |
| 4908 | 14047 | |
| 4909 | 14048 | #[test] |
| 4910 | | - fn parses_matrix_status_and_differential() { |
| 4911 | | - let root = std::env::temp_dir().join("afs_tests_matrix_spec.afs"); |
| 4912 | | - fs::write( |
| 4913 | | - &root, |
| 4914 | | - r#"suite "runtime/matrix" |
| 14049 | + fn render_generic_reports_include_provenance() { |
| 14050 | + let observation = CompilerObservation { |
| 14051 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14052 | + program: PathBuf::from("demo.f90"), |
| 14053 | + opt_level: OptLevel::O0, |
| 14054 | + compile_exit_code: 0, |
| 14055 | + artifacts: BTreeMap::from([ |
| 14056 | + ( |
| 14057 | + ArtifactKey::Asm, |
| 14058 | + ArtifactValue::Text(".globl _main\n".into()), |
| 14059 | + ), |
| 14060 | + ( |
| 14061 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 14062 | + ArtifactValue::Text("module main".into()), |
| 14063 | + ), |
| 14064 | + ]), |
| 14065 | + provenance: ObservationProvenance { |
| 14066 | + compiler_identity: "armfortas".into(), |
| 14067 | + adapter_kind: "named".into(), |
| 14068 | + backend_mode: "linked".into(), |
| 14069 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14070 | + artifacts_captured: vec!["asm".into(), "armfortas.ir".into()], |
| 14071 | + comparison_basis: None, |
| 14072 | + failure_stage: None, |
| 14073 | + }, |
| 14074 | + }; |
| 14075 | + let compare = ComparisonResult { |
| 14076 | + left: observation.clone(), |
| 14077 | + right: CompilerObservation { |
| 14078 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14079 | + program: PathBuf::from("demo.f90"), |
| 14080 | + opt_level: OptLevel::O0, |
| 14081 | + compile_exit_code: 0, |
| 14082 | + artifacts: BTreeMap::from([( |
| 14083 | + ArtifactKey::Asm, |
| 14084 | + ArtifactValue::Text(".arch armv8.5-a".into()), |
| 14085 | + )]), |
| 14086 | + provenance: ObservationProvenance { |
| 14087 | + compiler_identity: "gfortran".into(), |
| 14088 | + adapter_kind: "named".into(), |
| 14089 | + backend_mode: "external-driver".into(), |
| 14090 | + backend_detail: "generic external driver adapter using gfortran".into(), |
| 14091 | + artifacts_captured: vec!["asm".into()], |
| 14092 | + comparison_basis: Some("compile-status, diagnostics, runtime, asm".into()), |
| 14093 | + failure_stage: None, |
| 14094 | + }, |
| 14095 | + }, |
| 14096 | + basis: "compile-status, diagnostics, runtime, asm".into(), |
| 14097 | + differences: vec![ArtifactDifference { |
| 14098 | + artifact: "asm".into(), |
| 14099 | + detail: "first differing line: 1".into(), |
| 14100 | + }], |
| 14101 | + }; |
| 4915 | 14102 | |
| 4916 | | -case "hello" |
| 4917 | | -source "../../../test_programs/hello.f90" |
| 4918 | | -opts => O0, O1, O2 |
| 4919 | | -armfortas => run |
| 4920 | | -differential => gfortran, flang-new |
| 4921 | | -expect run.exit_code equals 0 |
| 4922 | | -xfail when O1, O2 because "known issue" |
| 4923 | | -end |
| 4924 | | -"#, |
| 4925 | | - ) |
| 4926 | | - .unwrap(); |
| 14103 | + let observed = ObservedProgram { |
| 14104 | + observation: observation.clone(), |
| 14105 | + requested_artifacts: BTreeSet::from([ |
| 14106 | + ArtifactKey::Asm, |
| 14107 | + ArtifactKey::Extra("armfortas.ir".into()), |
| 14108 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 14109 | + ]), |
| 14110 | + }; |
| 4927 | 14111 | |
| 4928 | | - let suite = parse_suite_file(&root).unwrap(); |
| 4929 | | - let case = &suite.cases[0]; |
| 4930 | | - assert_eq!( |
| 4931 | | - case.opt_levels, |
| 4932 | | - vec![OptLevel::O0, OptLevel::O1, OptLevel::O2] |
| 4933 | | - ); |
| 4934 | | - assert_eq!( |
| 4935 | | - case.reference_compilers, |
| 4936 | | - vec![ReferenceCompiler::Gfortran, ReferenceCompiler::FlangNew] |
| 14112 | + let introspection_text = |
| 14113 | + render_introspection_text(&observed, full_introspection_render_config()); |
| 14114 | + assert!(introspection_text.contains("status: compile ok")); |
| 14115 | + assert!(introspection_text.contains("artifact_count: 2")); |
| 14116 | + assert!( |
| 14117 | + introspection_text.contains("requested_artifacts: asm, armfortas.ir, armfortas.tokens") |
| 4937 | 14118 | ); |
| 4938 | | - assert!(matches!( |
| 4939 | | - status_for_opt(case, OptLevel::O0), |
| 4940 | | - EffectiveStatus::Normal |
| 4941 | | - )); |
| 4942 | | - assert!(matches!( |
| 4943 | | - status_for_opt(case, OptLevel::O1), |
| 4944 | | - EffectiveStatus::Xfail(_) |
| 14119 | + assert!(introspection_text.contains("missing_artifacts: armfortas.tokens")); |
| 14120 | + assert!(introspection_text.contains("generic_artifacts: asm")); |
| 14121 | + assert!(introspection_text.contains("adapter_extras: armfortas(ir)")); |
| 14122 | + assert!(introspection_text.contains("Generic artifacts")); |
| 14123 | + assert!(introspection_text.contains("Adapter extras")); |
| 14124 | + |
| 14125 | + let introspection_json = render_introspection_json(&observed); |
| 14126 | + assert!(introspection_json.contains("\"status\": \"compile ok\"")); |
| 14127 | + assert!(introspection_json.contains("\"artifact_count\": 2")); |
| 14128 | + assert!(introspection_json.contains( |
| 14129 | + "\"requested_artifacts\": [\"asm\", \"armfortas.ir\", \"armfortas.tokens\"]" |
| 4945 | 14130 | )); |
| 4946 | | - let _ = fs::remove_file(&root); |
| 14131 | + assert!(introspection_json.contains("\"missing_artifacts\": [\"armfortas.tokens\"]")); |
| 14132 | + assert!(introspection_json.contains("\"artifact_summaries\":")); |
| 14133 | + assert!(introspection_json.contains("\"asm\": {\"kind\":\"text\"")); |
| 14134 | + assert!(introspection_json.contains("\"line_count\":1")); |
| 14135 | + assert!(introspection_json.contains("\"generic_artifacts\": [\"asm\"]")); |
| 14136 | + assert!(introspection_json.contains("\"adapter_extras\": {\"armfortas\": [\"ir\"]}")); |
| 14137 | + assert!(introspection_json.contains("\"backend_mode\": \"linked\"")); |
| 14138 | + assert!(introspection_json.contains("\"armfortas.ir\"")); |
| 14139 | + |
| 14140 | + let introspection_markdown = |
| 14141 | + render_introspection_markdown(&observed, full_introspection_render_config()); |
| 14142 | + assert!(introspection_markdown.contains("# bencch introspect report")); |
| 14143 | + assert!(introspection_markdown.contains("status: compile ok")); |
| 14144 | + assert!(introspection_markdown.contains("failure_stage: `none`")); |
| 14145 | + assert!(introspection_markdown.contains("artifact_count: 2")); |
| 14146 | + assert!(introspection_markdown |
| 14147 | + .contains("requested_artifacts: `asm`, `armfortas.ir`, `armfortas.tokens`")); |
| 14148 | + assert!(introspection_markdown.contains("missing_artifacts: `armfortas.tokens`")); |
| 14149 | + assert!(introspection_markdown.contains("## Generic artifacts")); |
| 14150 | + assert!(introspection_markdown.contains("## Adapter extras")); |
| 14151 | + assert!(introspection_markdown.contains("### `armfortas`")); |
| 14152 | + assert!(introspection_markdown.contains("#### `ir`")); |
| 14153 | + |
| 14154 | + let compare_markdown = render_compare_markdown(&compare); |
| 14155 | + assert!(compare_markdown.contains("# bencch compare report")); |
| 14156 | + assert!(compare_markdown.contains("status: diff")); |
| 14157 | + assert!(compare_markdown.contains("classification: artifact divergence")); |
| 14158 | + assert!(compare_markdown.contains("difference_count: 1")); |
| 14159 | + assert!(compare_markdown.contains("changed_artifacts: asm")); |
| 14160 | + assert!(compare_markdown.contains("backend_mode: `external-driver`")); |
| 14161 | + assert!(compare_markdown.contains("### `asm`")); |
| 14162 | + |
| 14163 | + let compare_json = render_compare_json(&compare); |
| 14164 | + assert!(compare_json.contains("\"status\": \"diff\"")); |
| 14165 | + assert!(compare_json.contains("\"classification\": \"artifact divergence\"")); |
| 14166 | + assert!(compare_json.contains("\"difference_count\": 1")); |
| 14167 | + assert!(compare_json.contains("\"changed_artifacts\": [\"asm\"]")); |
| 4947 | 14168 | } |
| 4948 | 14169 | |
| 4949 | 14170 | #[test] |
| 4950 | | - fn parses_consistency_checks() { |
| 4951 | | - let root = std::env::temp_dir().join("afs_tests_consistency_spec.afs"); |
| 4952 | | - fs::write( |
| 4953 | | - &root, |
| 4954 | | - r#"suite "consistency/object" |
| 14171 | + fn render_failure_introspection_reports_stage_and_excerpt() { |
| 14172 | + let observed = ObservedProgram { |
| 14173 | + observation: CompilerObservation { |
| 14174 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14175 | + program: PathBuf::from("broken.f90"), |
| 14176 | + opt_level: OptLevel::O0, |
| 14177 | + compile_exit_code: 1, |
| 14178 | + artifacts: BTreeMap::from([ |
| 14179 | + ( |
| 14180 | + ArtifactKey::Diagnostics, |
| 14181 | + ArtifactValue::Text("undefined symbol: missing_value\nmore detail".into()), |
| 14182 | + ), |
| 14183 | + ( |
| 14184 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 14185 | + ArtifactValue::Text("token stream".into()), |
| 14186 | + ), |
| 14187 | + ]), |
| 14188 | + provenance: ObservationProvenance { |
| 14189 | + compiler_identity: "armfortas".into(), |
| 14190 | + adapter_kind: "named".into(), |
| 14191 | + backend_mode: "linked".into(), |
| 14192 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14193 | + artifacts_captured: vec!["diagnostics".into(), "armfortas.tokens".into()], |
| 14194 | + comparison_basis: None, |
| 14195 | + failure_stage: Some("sema".into()), |
| 14196 | + }, |
| 14197 | + }, |
| 14198 | + requested_artifacts: BTreeSet::from([ |
| 14199 | + ArtifactKey::Asm, |
| 14200 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 14201 | + ]), |
| 14202 | + }; |
| 4955 | 14203 | |
| 4956 | | -case "driver_paths" |
| 4957 | | -source "../../fixtures/backend/runtime_calls.f90" |
| 4958 | | -armfortas => asm, obj |
| 4959 | | -repeat => 5 |
| 4960 | | -consistency => cli_obj_vs_system_as, cli-obj-vs-system-as, cli_asm_reproducible, cli-obj-reproducible, cli_run_reproducible, capture_asm_vs_cli_asm, capture-obj-vs-cli-obj, capture_run_vs_cli_run, capture_asm_reproducible, capture-obj-reproducible, capture_run_reproducible |
| 4961 | | -expect obj contains "_main" |
| 4962 | | -end |
| 4963 | | -"#, |
| 4964 | | - ) |
| 4965 | | - .unwrap(); |
| 14204 | + let text = render_introspection_text(&observed, full_introspection_render_config()); |
| 14205 | + assert!(text.contains("status: compile failed")); |
| 14206 | + assert!(text.contains("failure_stage: sema")); |
| 14207 | + assert!(text.contains("diagnostic_excerpt: undefined symbol: missing_value")); |
| 14208 | + assert!(text.contains("missing_artifacts: asm")); |
| 4966 | 14209 | |
| 4967 | | - let suite = parse_suite_file(&root).unwrap(); |
| 4968 | | - let case = &suite.cases[0]; |
| 4969 | | - assert_eq!( |
| 4970 | | - case.consistency_checks, |
| 4971 | | - vec![ |
| 4972 | | - ConsistencyCheck::CliObjVsSystemAs, |
| 4973 | | - ConsistencyCheck::CliAsmReproducible, |
| 4974 | | - ConsistencyCheck::CliObjReproducible, |
| 4975 | | - ConsistencyCheck::CliRunReproducible, |
| 4976 | | - ConsistencyCheck::CaptureAsmVsCliAsm, |
| 4977 | | - ConsistencyCheck::CaptureObjVsCliObj, |
| 4978 | | - ConsistencyCheck::CaptureRunVsCliRun, |
| 4979 | | - ConsistencyCheck::CaptureAsmReproducible, |
| 4980 | | - ConsistencyCheck::CaptureObjReproducible, |
| 4981 | | - ConsistencyCheck::CaptureRunReproducible, |
| 4982 | | - ] |
| 4983 | | - ); |
| 4984 | | - assert_eq!(case.repeat_count, 5); |
| 4985 | | - let _ = fs::remove_file(&root); |
| 14210 | + let json = render_introspection_json(&observed); |
| 14211 | + assert!(json.contains("\"stage\": \"sema\"")); |
| 14212 | + assert!(json.contains("\"diagnostic_excerpt\": \"undefined symbol: missing_value\"")); |
| 14213 | + assert!(json.contains("\"failure_stage\": \"sema\"")); |
| 14214 | + assert!(json.contains("\"diagnostics\": {\"kind\":\"text\"")); |
| 14215 | + assert!(json.contains("\"summary\":\"text, 2 lines, ")); |
| 14216 | + assert!(json.contains("\"line_count\":2")); |
| 14217 | + |
| 14218 | + let markdown = render_introspection_markdown(&observed, full_introspection_render_config()); |
| 14219 | + assert!(markdown.contains("status: compile failed")); |
| 14220 | + assert!(markdown.contains("failure_stage: `sema`")); |
| 14221 | + assert!(markdown.contains("diagnostic_excerpt: `undefined symbol: missing_value`")); |
| 4986 | 14222 | } |
| 4987 | 14223 | |
| 4988 | 14224 | #[test] |
| 4989 | | - fn parses_graph_case() { |
| 4990 | | - let root = std::env::temp_dir().join("afs_tests_graph_spec"); |
| 4991 | | - let _ = fs::remove_dir_all(&root); |
| 4992 | | - fs::create_dir_all(&root).unwrap(); |
| 4993 | | - fs::write( |
| 4994 | | - root.join("math_values.f90"), |
| 4995 | | - "module math_values\nend module\n", |
| 4996 | | - ) |
| 4997 | | - .unwrap(); |
| 4998 | | - fs::write(root.join("main.f90"), "program main\nend program\n").unwrap(); |
| 4999 | | - fs::write( |
| 5000 | | - root.join("graph.afs"), |
| 5001 | | - r#"suite "modules/graph" |
| 5002 | | - |
| 5003 | | -case "basic_use" |
| 5004 | | -entry "main.f90" |
| 5005 | | -file "math_values.f90" |
| 5006 | | -file "main.f90" |
| 5007 | | -armfortas => run |
| 5008 | | -expect run.exit_code equals 0 |
| 5009 | | -end |
| 5010 | | -"#, |
| 5011 | | - ) |
| 5012 | | - .unwrap(); |
| 14225 | + fn render_introspection_summary_only_omits_artifact_bodies() { |
| 14226 | + let observed = ObservedProgram { |
| 14227 | + observation: CompilerObservation { |
| 14228 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14229 | + program: PathBuf::from("demo.f90"), |
| 14230 | + opt_level: OptLevel::O0, |
| 14231 | + compile_exit_code: 0, |
| 14232 | + artifacts: BTreeMap::from([( |
| 14233 | + ArtifactKey::Extra("armfortas.tokens".into()), |
| 14234 | + ArtifactValue::Text("line1\nline2\nline3".into()), |
| 14235 | + )]), |
| 14236 | + provenance: ObservationProvenance { |
| 14237 | + compiler_identity: "armfortas".into(), |
| 14238 | + adapter_kind: "named".into(), |
| 14239 | + backend_mode: "linked".into(), |
| 14240 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14241 | + artifacts_captured: vec!["armfortas.tokens".into()], |
| 14242 | + comparison_basis: None, |
| 14243 | + failure_stage: None, |
| 14244 | + }, |
| 14245 | + }, |
| 14246 | + requested_artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.tokens".into())]), |
| 14247 | + }; |
| 5013 | 14248 | |
| 5014 | | - let suite = parse_suite_file(&root.join("graph.afs")).unwrap(); |
| 5015 | | - let case = &suite.cases[0]; |
| 5016 | | - assert_eq!(case.source, root.join("main.f90")); |
| 5017 | | - assert_eq!( |
| 5018 | | - case.graph_files, |
| 5019 | | - vec![root.join("math_values.f90"), root.join("main.f90")] |
| 5020 | | - ); |
| 14249 | + let config = IntrospectionRenderConfig { |
| 14250 | + summary_only: true, |
| 14251 | + max_artifact_lines: Some(1), |
| 14252 | + }; |
| 14253 | + let text = render_introspection_text(&observed, config); |
| 14254 | + assert!(text.contains("content_mode: summary-only")); |
| 14255 | + assert!(text.contains("summary: text, 3 lines, 17 chars")); |
| 14256 | + assert!(text.contains("[content omitted by --summary-only]")); |
| 14257 | + assert!(!text.contains("line2")); |
| 5021 | 14258 | |
| 5022 | | - let _ = fs::remove_dir_all(&root); |
| 14259 | + let markdown = render_introspection_markdown(&observed, config); |
| 14260 | + assert!(markdown.contains("content_mode: `summary-only`")); |
| 14261 | + assert!(markdown.contains("[content omitted by --summary-only]")); |
| 5023 | 14262 | } |
| 5024 | 14263 | |
| 5025 | 14264 | #[test] |
| 5026 | | - fn parse_cli_collects_tool_overrides() { |
| 5027 | | - let args = vec![ |
| 5028 | | - "run".to_string(), |
| 5029 | | - "--suite".to_string(), |
| 5030 | | - "consistency/runtime".to_string(), |
| 5031 | | - "--armfortas-bin".to_string(), |
| 5032 | | - "/tmp/armfortas".to_string(), |
| 5033 | | - "--gfortran-bin".to_string(), |
| 5034 | | - "/tmp/gfortran".to_string(), |
| 5035 | | - "--flang-bin".to_string(), |
| 5036 | | - "/tmp/flang-new".to_string(), |
| 5037 | | - "--as-bin".to_string(), |
| 5038 | | - "/tmp/as".to_string(), |
| 5039 | | - "--otool-bin".to_string(), |
| 5040 | | - "/tmp/otool".to_string(), |
| 5041 | | - "--nm-bin".to_string(), |
| 5042 | | - "/tmp/nm".to_string(), |
| 5043 | | - ]; |
| 14265 | + fn render_introspection_truncates_large_artifacts() { |
| 14266 | + let observed = ObservedProgram { |
| 14267 | + observation: CompilerObservation { |
| 14268 | + compiler: CompilerSpec::Named(NamedCompiler::Armfortas), |
| 14269 | + program: PathBuf::from("demo.f90"), |
| 14270 | + opt_level: OptLevel::O0, |
| 14271 | + compile_exit_code: 0, |
| 14272 | + artifacts: BTreeMap::from([( |
| 14273 | + ArtifactKey::Asm, |
| 14274 | + ArtifactValue::Text("a\nb\nc\nd".into()), |
| 14275 | + )]), |
| 14276 | + provenance: ObservationProvenance { |
| 14277 | + compiler_identity: "armfortas".into(), |
| 14278 | + adapter_kind: "named".into(), |
| 14279 | + backend_mode: "linked".into(), |
| 14280 | + backend_detail: "linked armfortas::testing capture adapter".into(), |
| 14281 | + artifacts_captured: vec!["asm".into()], |
| 14282 | + comparison_basis: None, |
| 14283 | + failure_stage: None, |
| 14284 | + }, |
| 14285 | + }, |
| 14286 | + requested_artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 14287 | + }; |
| 5044 | 14288 | |
| 5045 | | - let command = parse_cli(&args).unwrap(); |
| 5046 | | - let config = match command { |
| 5047 | | - CommandKind::Run(config) => config, |
| 5048 | | - other => panic!( |
| 5049 | | - "expected run command, got {:?}", |
| 5050 | | - std::mem::discriminant(&other) |
| 5051 | | - ), |
| 14289 | + let config = IntrospectionRenderConfig { |
| 14290 | + summary_only: false, |
| 14291 | + max_artifact_lines: Some(2), |
| 5052 | 14292 | }; |
| 14293 | + let text = render_introspection_text(&observed, config); |
| 14294 | + assert!(text.contains("content_mode: first 2 lines per artifact")); |
| 14295 | + assert!(text.contains("a\nb\n... (truncated; showing first 2 of 4 lines)")); |
| 14296 | + assert!(!text.contains("\nc\nd")); |
| 5053 | 14297 | |
| 5054 | | - assert_eq!(config.suite_filter.as_deref(), Some("consistency/runtime")); |
| 5055 | | - assert_eq!( |
| 5056 | | - config.tools.armfortas, |
| 5057 | | - ArmfortasCliAdapter::External("/tmp/armfortas".into()) |
| 5058 | | - ); |
| 5059 | | - assert_eq!(config.tools.gfortran, "/tmp/gfortran"); |
| 5060 | | - assert_eq!(config.tools.flang_new, "/tmp/flang-new"); |
| 5061 | | - assert_eq!(config.tools.system_as, "/tmp/as"); |
| 5062 | | - assert_eq!(config.tools.otool, "/tmp/otool"); |
| 5063 | | - assert_eq!(config.tools.nm, "/tmp/nm"); |
| 14298 | + let markdown = render_introspection_markdown(&observed, config); |
| 14299 | + assert!(markdown.contains("content_mode: `first 2 lines per artifact`")); |
| 14300 | + assert!(markdown.contains("... (truncated; showing first 2 of 4 lines)")); |
| 5064 | 14301 | } |
| 5065 | 14302 | |
| 5066 | 14303 | #[test] |
| 5067 | | - fn parses_failure_expectation() { |
| 5068 | | - let root = std::env::temp_dir().join("afs_tests_failure_spec.afs"); |
| 5069 | | - fs::write( |
| 5070 | | - &root, |
| 5071 | | - r#"suite "frontend/parser" |
| 14304 | + fn write_requested_reports_emits_files() { |
| 14305 | + let root = std::env::temp_dir().join("afs_tests_report_output"); |
| 14306 | + let _ = fs::remove_dir_all(&root); |
| 14307 | + let json_path = root.join("result.json"); |
| 14308 | + let markdown_path = root.join("result.md"); |
| 14309 | + let config = RunConfig { |
| 14310 | + suite_filter: None, |
| 14311 | + case_filter: None, |
| 14312 | + opt_filter: None, |
| 14313 | + verbose: false, |
| 14314 | + fail_fast: false, |
| 14315 | + include_future: false, |
| 14316 | + all_stages: false, |
| 14317 | + json_report: Some(json_path.clone()), |
| 14318 | + markdown_report: Some(markdown_path.clone()), |
| 14319 | + tools: ToolchainConfig::from_env(), |
| 14320 | + }; |
| 14321 | + let mut summary = Summary::default(); |
| 14322 | + summary.record_outcome(&Outcome { |
| 14323 | + suite: "frontend/parser".into(), |
| 14324 | + case: "where_construct".into(), |
| 14325 | + opt_level: OptLevel::O0, |
| 14326 | + kind: OutcomeKind::Pass, |
| 14327 | + detail: String::new(), |
| 14328 | + bundle: None, |
| 14329 | + primary_backend: Some(PrimaryBackendReport { |
| 14330 | + kind: "full".into(), |
| 14331 | + mode: "linked".into(), |
| 14332 | + detail: "linked armfortas::testing capture adapter".into(), |
| 14333 | + }), |
| 14334 | + consistency_observations: Vec::new(), |
| 14335 | + }); |
| 5072 | 14336 | |
| 5073 | | -case "missing_then" |
| 5074 | | -source "../../fixtures/frontend/parser/missing_then.f90" |
| 5075 | | -armfortas => tokens |
| 5076 | | -expect tokens contains "if" |
| 5077 | | -expect-fail parser contains "expected" |
| 5078 | | -end |
| 5079 | | -"#, |
| 5080 | | - ) |
| 5081 | | - .unwrap(); |
| 14337 | + write_requested_reports(&config, &summary).unwrap(); |
| 5082 | 14338 | |
| 5083 | | - let suite = parse_suite_file(&root).unwrap(); |
| 5084 | | - assert_eq!(suite.cases.len(), 1); |
| 5085 | | - assert!(has_failure_expectation(&suite.cases[0])); |
| 5086 | | - let _ = fs::remove_file(&root); |
| 14339 | + let json = fs::read_to_string(&json_path).unwrap(); |
| 14340 | + let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14341 | + assert!(json.contains("\"passed\": 1")); |
| 14342 | + assert!(markdown.contains("| passed | 1 |")); |
| 14343 | + |
| 14344 | + let _ = fs::remove_dir_all(&root); |
| 5087 | 14345 | } |
| 5088 | 14346 | |
| 5089 | 14347 | #[test] |
| 5090 | | - fn check_matching_preserves_order() { |
| 5091 | | - let checks = vec![ |
| 5092 | | - Check { |
| 5093 | | - line_num: 1, |
| 5094 | | - pattern: "alpha".into(), |
| 5095 | | - }, |
| 5096 | | - Check { |
| 5097 | | - line_num: 2, |
| 5098 | | - pattern: "omega".into(), |
| 14348 | + fn render_doctor_report_includes_tool_status() { |
| 14349 | + let root = std::env::temp_dir().join("afs_tests_doctor_paths"); |
| 14350 | + let _ = fs::remove_dir_all(&root); |
| 14351 | + fs::create_dir_all(&root).unwrap(); |
| 14352 | + let armfortas_bin = root.join("armfortas"); |
| 14353 | + let gfortran_bin = root.join("gfortran"); |
| 14354 | + write_probe_script(&armfortas_bin, "armfortas dev build"); |
| 14355 | + write_probe_script(&gfortran_bin, "GNU Fortran 99.1"); |
| 14356 | + |
| 14357 | + let config = DoctorConfig { |
| 14358 | + tools: ToolchainConfig { |
| 14359 | + armfortas: ArmfortasCliAdapter::External(armfortas_bin.display().to_string()), |
| 14360 | + gfortran: gfortran_bin.display().to_string(), |
| 14361 | + flang_new: "/tmp/does-not-exist-flang".into(), |
| 14362 | + lfortran: "/tmp/does-not-exist-lfortran".into(), |
| 14363 | + ifort: "/tmp/does-not-exist-ifort".into(), |
| 14364 | + ifx: "/tmp/does-not-exist-ifx".into(), |
| 14365 | + nvfortran: "/tmp/does-not-exist-nvfortran".into(), |
| 14366 | + system_as: "/tmp/does-not-exist-as".into(), |
| 14367 | + otool: "/tmp/does-not-exist-otool".into(), |
| 14368 | + nm: "/tmp/does-not-exist-nm".into(), |
| 5099 | 14369 | }, |
| 5100 | | - ]; |
| 5101 | | - assert!(match_checks(&checks, "alpha\nmiddle\nomega\n", "demo").is_ok()); |
| 5102 | | - assert!(match_checks(&checks, "omega\nalpha\n", "demo").is_err()); |
| 5103 | | - } |
| 14370 | + json_report: None, |
| 14371 | + markdown_report: None, |
| 14372 | + }; |
| 5104 | 14373 | |
| 5105 | | - fn run_only_result(stdout: &str, stderr: &str, exit_code: i32) -> CaptureResult { |
| 5106 | | - CaptureResult { |
| 5107 | | - input: PathBuf::from("demo.f90"), |
| 5108 | | - opt_level: OptLevel::O0, |
| 5109 | | - stages: std::collections::BTreeMap::from([( |
| 5110 | | - Stage::Run, |
| 5111 | | - CapturedStage::Run(RunCapture { |
| 5112 | | - exit_code, |
| 5113 | | - stdout: stdout.into(), |
| 5114 | | - stderr: stderr.into(), |
| 5115 | | - }), |
| 5116 | | - )]), |
| 5117 | | - } |
| 14374 | + let rendered = render_doctor_report(&config); |
| 14375 | + assert!(rendered.contains("Doctor")); |
| 14376 | + assert!(rendered.contains("armfortas_cli_adapter: external armfortas binary adapter")); |
| 14377 | + assert!(rendered.contains("armfortas_cli_mode: external")); |
| 14378 | + assert!(rendered |
| 14379 | + .contains("armfortas_capture_adapter: linked armfortas::testing capture adapter")); |
| 14380 | + assert!(rendered.contains("armfortas_capture_mode: linked")); |
| 14381 | + assert!(rendered.contains("armfortas_capture_manifest:")); |
| 14382 | + assert!( |
| 14383 | + rendered.contains("primary_backend_full: linked armfortas::testing capture adapter") |
| 14384 | + ); |
| 14385 | + assert!(rendered.contains( |
| 14386 | + "linked_mode_surface: rich armfortas stages, legacy frontend/module suites, capture consistency" |
| 14387 | + )); |
| 14388 | + assert!(rendered.contains( |
| 14389 | + "primary_backend_observable: cli-observable armfortas driver capture adapter" |
| 14390 | + )); |
| 14391 | + assert!(rendered.contains( |
| 14392 | + "primary_backend_selection: observable backend is selected for asm/obj/run-only cells" |
| 14393 | + )); |
| 14394 | + assert!(rendered.contains("named_compiler.armfortas: cli=external capture=linked")); |
| 14395 | + assert!(rendered.contains( |
| 14396 | + "named_compiler.armfortas.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14397 | + )); |
| 14398 | + assert!(rendered.contains("named_compiler.armfortas.adapter_extras: armfortas(")); |
| 14399 | + assert!(rendered.contains("named_compiler.armfortas.unavailable_artifacts: none")); |
| 14400 | + assert!(rendered.contains("named_compiler.gfortran:")); |
| 14401 | + assert!(rendered.contains( |
| 14402 | + "named_compiler.gfortran.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14403 | + )); |
| 14404 | + assert!(rendered.contains("named_compiler.gfortran.adapter_extras: none")); |
| 14405 | + assert!(rendered.contains("named_compiler.lfortran:")); |
| 14406 | + assert!(rendered.contains("named_compiler.lfortran.accepted_names: lfortran")); |
| 14407 | + assert!(rendered.contains("named_compiler.lfortran.candidate_binaries: lfortran")); |
| 14408 | + assert!(rendered.contains("named_compiler.ifx.accepted_names: ifx")); |
| 14409 | + assert!(rendered.contains("named_compiler.nvfortran.accepted_names: nvfortran, pgfortran")); |
| 14410 | + assert!(rendered.contains("named_compiler.armfortas.probe_status: invokable")); |
| 14411 | + assert!(rendered.contains("named_compiler.armfortas.probe_banner: armfortas dev build")); |
| 14412 | + assert!(rendered.contains("named_compiler.gfortran.probe_status: invokable")); |
| 14413 | + assert!(rendered.contains("named_compiler.gfortran.probe_banner: GNU Fortran 99.1")); |
| 14414 | + assert!(rendered.contains( |
| 14415 | + "explicit_compiler_path: any filesystem path passed to compare/introspect uses the generic external-driver adapter" |
| 14416 | + )); |
| 14417 | + assert!(rendered.contains( |
| 14418 | + "explicit_compiler_path.generic_artifacts: diagnostics, exit-code, stdout, stderr, asm, obj, executable, runtime" |
| 14419 | + )); |
| 14420 | + assert!(rendered.contains(&format!( |
| 14421 | + "configured={} resolved={}", |
| 14422 | + armfortas_bin.display(), |
| 14423 | + armfortas_bin.display() |
| 14424 | + ))); |
| 14425 | + assert!(rendered.contains(&format!( |
| 14426 | + "configured={} resolved={}", |
| 14427 | + gfortran_bin.display(), |
| 14428 | + gfortran_bin.display() |
| 14429 | + ))); |
| 14430 | + assert!(rendered.contains("configured=/tmp/does-not-exist-flang resolved=missing")); |
| 14431 | + let rendered_json = render_doctor_json(&config); |
| 14432 | + let rendered_markdown = render_doctor_markdown(&config); |
| 14433 | + assert!(rendered_json.contains("\"command\": \"doctor\"")); |
| 14434 | + assert!(rendered_json.contains("\"workspace\": {")); |
| 14435 | + assert!(rendered_json.contains("\"named_compilers\": {")); |
| 14436 | + assert!(rendered_json.contains("\"tools\": {")); |
| 14437 | + assert!(rendered_json.contains("\"lfortran\": {")); |
| 14438 | + assert!(rendered_json.contains("\"named_compiler.armfortas.adapter_extras\"")); |
| 14439 | + assert!(rendered_json.contains("\"probe\": {")); |
| 14440 | + assert!(rendered_markdown.contains("# bencch doctor report")); |
| 14441 | + assert!(rendered_markdown.contains("| `named_compiler.armfortas` |")); |
| 14442 | + |
| 14443 | + let _ = fs::remove_dir_all(&root); |
| 5118 | 14444 | } |
| 5119 | 14445 | |
| 5120 | | - fn reference_run( |
| 5121 | | - compiler: ReferenceCompiler, |
| 5122 | | - stdout: &str, |
| 5123 | | - stderr: &str, |
| 5124 | | - exit_code: i32, |
| 5125 | | - ) -> ReferenceResult { |
| 5126 | | - ReferenceResult { |
| 5127 | | - compiler, |
| 5128 | | - compile_command: format!("{} demo.f90 -o demo", compiler.as_str()), |
| 5129 | | - compile_exit_code: 0, |
| 5130 | | - compile_stdout: String::new(), |
| 5131 | | - compile_stderr: String::new(), |
| 5132 | | - run: Some(RunCapture { |
| 5133 | | - exit_code, |
| 5134 | | - stdout: stdout.into(), |
| 5135 | | - stderr: stderr.into(), |
| 5136 | | - }), |
| 5137 | | - run_error: None, |
| 5138 | | - } |
| 14446 | + #[cfg(unix)] |
| 14447 | + #[test] |
| 14448 | + fn tool_probe_reads_banner_from_executable() { |
| 14449 | + let root = std::env::temp_dir().join("bencch_tool_probe_banner"); |
| 14450 | + let _ = fs::remove_dir_all(&root); |
| 14451 | + fs::create_dir_all(&root).unwrap(); |
| 14452 | + let probe_bin = root.join("fakefortran"); |
| 14453 | + write_probe_script(&probe_bin, "Fake Fortran 1.2.3"); |
| 14454 | + |
| 14455 | + let probe = tool_probe(&probe_bin.display().to_string(), true); |
| 14456 | + assert_eq!(probe.status, "invokable"); |
| 14457 | + assert_eq!(probe.banner.as_deref(), Some("Fake Fortran 1.2.3")); |
| 14458 | + assert!(probe |
| 14459 | + .detail |
| 14460 | + .as_deref() |
| 14461 | + .unwrap_or_default() |
| 14462 | + .contains("--version")); |
| 14463 | + |
| 14464 | + let _ = fs::remove_dir_all(&root); |
| 5139 | 14465 | } |
| 5140 | 14466 | |
| 5141 | 14467 | #[test] |
| 5142 | | - fn not_contains_expectation_checks_text_absence() { |
| 14468 | + fn case_discovery_lines_report_capability_block_for_generic_introspect() { |
| 5143 | 14469 | let case = CaseSpec { |
| 5144 | | - name: "no_reserved_register".into(), |
| 14470 | + name: "unsupported_extra".into(), |
| 5145 | 14471 | source: PathBuf::from("demo.f90"), |
| 5146 | 14472 | graph_files: Vec::new(), |
| 5147 | | - requested: BTreeSet::from([Stage::Asm]), |
| 14473 | + requested: BTreeSet::new(), |
| 14474 | + generic_introspect: Some(GenericIntrospectCase { |
| 14475 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14476 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 14477 | + }), |
| 14478 | + generic_compare: None, |
| 5148 | 14479 | opt_levels: vec![OptLevel::O0], |
| 5149 | 14480 | repeat_count: 2, |
| 5150 | 14481 | reference_compilers: Vec::new(), |
| 5151 | 14482 | consistency_checks: Vec::new(), |
| 5152 | | - expectations: vec![Expectation::NotContains { |
| 5153 | | - target: Target::Stage(Stage::Asm), |
| 5154 | | - needle: "x18".into(), |
| 5155 | | - }], |
| 14483 | + expectations: Vec::new(), |
| 5156 | 14484 | status_rules: Vec::new(), |
| 14485 | + capability_policy: None, |
| 5157 | 14486 | }; |
| 5158 | | - let result = CaptureResult { |
| 5159 | | - input: PathBuf::from("demo.f90"), |
| 5160 | | - opt_level: OptLevel::O0, |
| 5161 | | - stages: std::collections::BTreeMap::from([( |
| 5162 | | - Stage::Asm, |
| 5163 | | - CapturedStage::Text("mov x19, x0\nret\n".into()), |
| 5164 | | - )]), |
| 5165 | | - }; |
| 5166 | | - assert!(evaluate_positive_expectations(&case, &result).is_ok()); |
| 5167 | 14487 | |
| 5168 | | - let bad = CaptureResult { |
| 5169 | | - input: PathBuf::from("demo.f90"), |
| 5170 | | - opt_level: OptLevel::O0, |
| 5171 | | - stages: std::collections::BTreeMap::from([( |
| 5172 | | - Stage::Asm, |
| 5173 | | - CapturedStage::Text("mov x18, x0\nret\n".into()), |
| 5174 | | - )]), |
| 5175 | | - }; |
| 5176 | | - let err = evaluate_positive_expectations(&case, &bad).unwrap_err(); |
| 5177 | | - assert!(err.contains("expected asm to not contain")); |
| 14488 | + let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14489 | + assert!(lines.contains(&"capability_status: blocked".to_string())); |
| 14490 | + assert!(lines.iter().any(|line| line.contains( |
| 14491 | + "gfortran does not support requested artifacts in this adapter: armfortas.ir" |
| 14492 | + ))); |
| 5178 | 14493 | } |
| 5179 | 14494 | |
| 5180 | 14495 | #[test] |
| 5181 | | - fn writes_failure_bundle_with_artifacts() { |
| 5182 | | - let source = std::env::temp_dir().join("afs_tests_bundle_source.f90"); |
| 5183 | | - fs::write(&source, "program hello\nprint *, 'hello'\nend program\n").unwrap(); |
| 5184 | | - |
| 5185 | | - let suite = SuiteSpec { |
| 5186 | | - name: "runtime/bundles".into(), |
| 5187 | | - path: PathBuf::from("/tmp/runtime/bundles.afs"), |
| 5188 | | - cases: Vec::new(), |
| 5189 | | - }; |
| 14496 | + fn case_discovery_lines_report_capability_policy_as_deferred() { |
| 5190 | 14497 | let case = CaseSpec { |
| 5191 | | - name: "hello_bundle".into(), |
| 5192 | | - source: source.clone(), |
| 14498 | + name: "unsupported_extra".into(), |
| 14499 | + source: PathBuf::from("demo.f90"), |
| 5193 | 14500 | graph_files: Vec::new(), |
| 5194 | | - requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 5195 | | - opt_levels: vec![OptLevel::O0], |
| 5196 | | - repeat_count: 3, |
| 5197 | | - reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 5198 | | - consistency_checks: vec![ConsistencyCheck::CliObjVsSystemAs], |
| 5199 | | - expectations: Vec::new(), |
| 5200 | | - status_rules: Vec::new(), |
| 5201 | | - }; |
| 5202 | | - let mut stages = std::collections::BTreeMap::new(); |
| 5203 | | - stages.insert(Stage::Ir, CapturedStage::Text("module main".into())); |
| 5204 | | - stages.insert( |
| 5205 | | - Stage::Run, |
| 5206 | | - CapturedStage::Run(RunCapture { |
| 5207 | | - exit_code: 1, |
| 5208 | | - stdout: "oops\n".into(), |
| 5209 | | - stderr: "broken\n".into(), |
| 5210 | | - }), |
| 5211 | | - ); |
| 5212 | | - let artifacts = ExecutionArtifacts { |
| 5213 | | - requested: BTreeSet::from([Stage::Ir, Stage::Run]), |
| 5214 | | - armfortas: None, |
| 5215 | | - armfortas_failure: Some(CaptureFailure { |
| 5216 | | - input: source.clone(), |
| 5217 | | - opt_level: OptLevel::O0, |
| 5218 | | - stage: FailureStage::Sema, |
| 5219 | | - detail: "compiler failed".into(), |
| 5220 | | - stages, |
| 14501 | + requested: BTreeSet::new(), |
| 14502 | + generic_introspect: Some(GenericIntrospectCase { |
| 14503 | + compiler: CompilerSpec::Named(NamedCompiler::Gfortran), |
| 14504 | + artifacts: BTreeSet::from([ArtifactKey::Extra("armfortas.ir".into())]), |
| 5221 | 14505 | }), |
| 5222 | | - references: vec![ReferenceResult { |
| 5223 | | - compiler: ReferenceCompiler::Gfortran, |
| 5224 | | - compile_command: "gfortran hello.f90 -o hello".into(), |
| 5225 | | - compile_exit_code: 0, |
| 5226 | | - compile_stdout: String::new(), |
| 5227 | | - compile_stderr: String::new(), |
| 5228 | | - run: Some(RunCapture { |
| 5229 | | - exit_code: 0, |
| 5230 | | - stdout: "hello\n".into(), |
| 5231 | | - stderr: String::new(), |
| 5232 | | - }), |
| 5233 | | - run_error: None, |
| 5234 | | - }], |
| 5235 | | - consistency_issues: { |
| 5236 | | - let asm_temp_root = |
| 5237 | | - std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm"); |
| 5238 | | - fs::create_dir_all(&asm_temp_root).unwrap(); |
| 5239 | | - fs::write(asm_temp_root.join("run_00.s"), "mov x19, x0\n").unwrap(); |
| 5240 | | - |
| 5241 | | - let obj_temp_root = |
| 5242 | | - std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj"); |
| 5243 | | - fs::create_dir_all(&obj_temp_root).unwrap(); |
| 5244 | | - fs::write(obj_temp_root.join("run_00.o"), "fake object bytes\n").unwrap(); |
| 5245 | | - |
| 5246 | | - vec![ |
| 5247 | | - ConsistencyIssue { |
| 5248 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 5249 | | - summary: "repeat_count=3 unique_variants=3".into(), |
| 5250 | | - repeat_count: Some(3), |
| 5251 | | - unique_variant_count: Some(3), |
| 5252 | | - varying_components: Vec::new(), |
| 5253 | | - stable_components: Vec::new(), |
| 5254 | | - detail: "assembly output is not reproducible".into(), |
| 5255 | | - temp_root: asm_temp_root, |
| 5256 | | - }, |
| 5257 | | - ConsistencyIssue { |
| 5258 | | - check: ConsistencyCheck::CliObjReproducible, |
| 5259 | | - summary: "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols".into(), |
| 5260 | | - repeat_count: Some(3), |
| 5261 | | - unique_variant_count: Some(2), |
| 5262 | | - varying_components: vec!["text".into()], |
| 5263 | | - stable_components: vec![ |
| 5264 | | - "load_commands".into(), |
| 5265 | | - "relocations".into(), |
| 5266 | | - "symbols".into(), |
| 5267 | | - ], |
| 5268 | | - detail: "object output is not reproducible".into(), |
| 5269 | | - temp_root: obj_temp_root, |
| 5270 | | - }, |
| 5271 | | - ] |
| 5272 | | - }, |
| 5273 | | - }; |
| 5274 | | - let outcome = Outcome { |
| 5275 | | - suite: suite.name.clone(), |
| 5276 | | - case: case.name.clone(), |
| 5277 | | - opt_level: OptLevel::O0, |
| 5278 | | - kind: OutcomeKind::Fail, |
| 5279 | | - detail: "boom".into(), |
| 5280 | | - bundle: None, |
| 5281 | | - consistency_observations: Vec::new(), |
| 5282 | | - }; |
| 5283 | | - let prepared = PreparedInput { |
| 5284 | | - compiler_source: source.clone(), |
| 5285 | | - generated_source: None, |
| 5286 | | - temp_root: None, |
| 5287 | | - }; |
| 5288 | | - |
| 5289 | | - let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 5290 | | - assert!(bundle.join("metadata.txt").exists()); |
| 5291 | | - assert!(bundle.join("detail.txt").exists()); |
| 5292 | | - assert!(bundle.join("source.f90").exists()); |
| 5293 | | - assert!(bundle.join("armfortas").join("ir.txt").exists()); |
| 5294 | | - assert!(bundle.join("armfortas").join("run.stdout.txt").exists()); |
| 5295 | | - assert!(bundle.join("armfortas").join("error.txt").exists()); |
| 5296 | | - assert!(bundle |
| 5297 | | - .join("references") |
| 5298 | | - .join("gfortran") |
| 5299 | | - .join("run.stdout.txt") |
| 5300 | | - .exists()); |
| 5301 | | - assert!(bundle.join("consistency").join("summary.txt").exists()); |
| 5302 | | - let consistency_summary = |
| 5303 | | - fs::read_to_string(bundle.join("consistency").join("summary.txt")).unwrap(); |
| 5304 | | - assert!(consistency_summary.contains("issue_count: 2")); |
| 5305 | | - assert!(consistency_summary.contains("checks: cli_asm_reproducible, cli_obj_reproducible")); |
| 5306 | | - assert!(consistency_summary.contains("repeat_counts: 3")); |
| 5307 | | - assert!(consistency_summary.contains("unique_variants: 2, 3")); |
| 5308 | | - assert!(consistency_summary.contains("varying_components: text")); |
| 5309 | | - assert!( |
| 5310 | | - consistency_summary.contains("stable_components: load_commands, relocations, symbols") |
| 5311 | | - ); |
| 5312 | | - assert!(bundle |
| 5313 | | - .join("consistency") |
| 5314 | | - .join("cli_asm_reproducible") |
| 5315 | | - .join("summary.txt") |
| 5316 | | - .exists()); |
| 5317 | | - assert!(bundle |
| 5318 | | - .join("consistency") |
| 5319 | | - .join("cli_asm_reproducible") |
| 5320 | | - .join("detail.txt") |
| 5321 | | - .exists()); |
| 5322 | | - assert!(bundle |
| 5323 | | - .join("consistency") |
| 5324 | | - .join("cli_asm_reproducible") |
| 5325 | | - .join("artifacts") |
| 5326 | | - .join("run_00.s") |
| 5327 | | - .exists()); |
| 5328 | | - assert!(bundle |
| 5329 | | - .join("consistency") |
| 5330 | | - .join("cli_obj_reproducible") |
| 5331 | | - .join("artifacts") |
| 5332 | | - .join("run_00.o") |
| 5333 | | - .exists()); |
| 14506 | + generic_compare: None, |
| 14507 | + opt_levels: vec![OptLevel::O0], |
| 14508 | + repeat_count: 2, |
| 14509 | + reference_compilers: Vec::new(), |
| 14510 | + consistency_checks: Vec::new(), |
| 14511 | + expectations: Vec::new(), |
| 14512 | + status_rules: Vec::new(), |
| 14513 | + capability_policy: Some(CapabilityPolicy { |
| 14514 | + kind: StatusKind::Future, |
| 14515 | + reason: "generic gfortran surface has no armfortas extras".into(), |
| 14516 | + }), |
| 14517 | + }; |
| 5334 | 14518 | |
| 5335 | | - let _ = fs::remove_dir_all(bundle); |
| 5336 | | - let _ = |
| 5337 | | - fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_asm")); |
| 5338 | | - let _ = |
| 5339 | | - fs::remove_dir_all(std::env::temp_dir().join("afs_tests_consistency_bundle_issue_obj")); |
| 5340 | | - let _ = fs::remove_file(source); |
| 14519 | + let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14520 | + assert!(lines.contains(&"capability_status: deferred".to_string())); |
| 14521 | + assert!(lines.contains( |
| 14522 | + &"capability_policy: future when blocked (generic gfortran surface has no armfortas extras)" |
| 14523 | + .to_string() |
| 14524 | + )); |
| 5341 | 14525 | } |
| 5342 | 14526 | |
| 14527 | + #[cfg(unix)] |
| 5343 | 14528 | #[test] |
| 5344 | | - fn materializes_graph_input_in_declared_file_order() { |
| 5345 | | - let root = std::env::temp_dir().join("afs_tests_graph_materialize"); |
| 14529 | + fn case_discovery_lines_include_compiler_probe_banner() { |
| 14530 | + let root = std::env::temp_dir().join("bencch_case_discovery_probe"); |
| 5346 | 14531 | let _ = fs::remove_dir_all(&root); |
| 5347 | 14532 | fs::create_dir_all(&root).unwrap(); |
| 5348 | | - let module = root.join("math_values.f90"); |
| 5349 | | - let main = root.join("main.f90"); |
| 5350 | | - fs::write(&module, "module math_values\ncontains\nend module\n").unwrap(); |
| 5351 | | - fs::write(&main, "program main\nuse math_values\nend program\n").unwrap(); |
| 14533 | + let compiler = root.join("probe-compiler"); |
| 14534 | + write_probe_script(&compiler, "Probe Compiler 7.4"); |
| 5352 | 14535 | |
| 5353 | | - let suite = SuiteSpec { |
| 5354 | | - name: "modules/graph".into(), |
| 5355 | | - path: root.join("graph.afs"), |
| 5356 | | - cases: Vec::new(), |
| 5357 | | - }; |
| 5358 | 14536 | let case = CaseSpec { |
| 5359 | | - name: "basic_use".into(), |
| 5360 | | - source: main.clone(), |
| 5361 | | - graph_files: vec![module.clone(), main.clone()], |
| 5362 | | - requested: BTreeSet::from([Stage::Run]), |
| 14537 | + name: "probe".into(), |
| 14538 | + source: PathBuf::from("demo.f90"), |
| 14539 | + graph_files: Vec::new(), |
| 14540 | + requested: BTreeSet::new(), |
| 14541 | + generic_introspect: Some(GenericIntrospectCase { |
| 14542 | + compiler: CompilerSpec::Binary(compiler.clone()), |
| 14543 | + artifacts: BTreeSet::from([ArtifactKey::Asm]), |
| 14544 | + }), |
| 14545 | + generic_compare: None, |
| 5363 | 14546 | opt_levels: vec![OptLevel::O0], |
| 5364 | 14547 | repeat_count: 2, |
| 5365 | 14548 | reference_compilers: Vec::new(), |
| 5366 | 14549 | consistency_checks: Vec::new(), |
| 5367 | 14550 | expectations: Vec::new(), |
| 5368 | 14551 | status_rules: Vec::new(), |
| 14552 | + capability_policy: None, |
| 5369 | 14553 | }; |
| 5370 | 14554 | |
| 5371 | | - let prepared = prepare_case_input(&case, &suite, OptLevel::O0).unwrap(); |
| 5372 | | - let generated = fs::read_to_string(&prepared.compiler_source).unwrap(); |
| 5373 | | - assert!(generated.contains("module math_values")); |
| 5374 | | - assert!(generated.contains("program main")); |
| 5375 | | - assert!( |
| 5376 | | - generated.find("module math_values").unwrap() < generated.find("program main").unwrap() |
| 5377 | | - ); |
| 14555 | + let lines = case_discovery_lines(&case, &ToolchainConfig::from_env()); |
| 14556 | + assert!(lines.contains(&"compiler_probe_status: invokable".to_string())); |
| 14557 | + assert!(lines |
| 14558 | + .iter() |
| 14559 | + .any(|line| line.contains("compiler_probe_banner: Probe Compiler 7.4"))); |
| 5378 | 14560 | |
| 5379 | | - cleanup_prepared_input(&prepared); |
| 5380 | 14561 | let _ = fs::remove_dir_all(&root); |
| 5381 | 14562 | } |
| 5382 | 14563 | |
| 5383 | 14564 | #[test] |
| 5384 | | - fn graph_failure_bundle_writes_authored_sources() { |
| 5385 | | - let root = std::env::temp_dir().join("afs_tests_graph_bundle"); |
| 5386 | | - let _ = fs::remove_dir_all(&root); |
| 5387 | | - fs::create_dir_all(&root).unwrap(); |
| 5388 | | - let module = root.join("math_values.f90"); |
| 5389 | | - let main = root.join("main.f90"); |
| 5390 | | - let generated = root.join("generated.f90"); |
| 5391 | | - fs::write( |
| 5392 | | - &module, |
| 5393 | | - "module math_values\n integer :: answer = 42\nend module\n", |
| 5394 | | - ) |
| 5395 | | - .unwrap(); |
| 5396 | | - fs::write( |
| 5397 | | - &main, |
| 5398 | | - "program main\n use math_values\n print *, answer\nend program\n", |
| 5399 | | - ) |
| 5400 | | - .unwrap(); |
| 5401 | | - fs::write(&generated, "module math_values\n integer :: answer = 42\nend module\n\nprogram main\n use math_values\n print *, answer\nend program\n").unwrap(); |
| 5402 | | - |
| 5403 | | - let suite = SuiteSpec { |
| 5404 | | - name: "modules/bundles".into(), |
| 5405 | | - path: root.join("bundle.afs"), |
| 5406 | | - cases: Vec::new(), |
| 5407 | | - }; |
| 5408 | | - let case = CaseSpec { |
| 5409 | | - name: "graph_bundle".into(), |
| 5410 | | - source: main.clone(), |
| 5411 | | - graph_files: vec![module.clone(), main.clone()], |
| 14565 | + fn case_discovery_lines_distinguish_legacy_surfaces() { |
| 14566 | + let observable_case = CaseSpec { |
| 14567 | + name: "observable".into(), |
| 14568 | + source: PathBuf::from("demo.f90"), |
| 14569 | + graph_files: Vec::new(), |
| 5412 | 14570 | requested: BTreeSet::from([Stage::Run]), |
| 14571 | + generic_introspect: None, |
| 14572 | + generic_compare: None, |
| 5413 | 14573 | opt_levels: vec![OptLevel::O0], |
| 5414 | 14574 | repeat_count: 2, |
| 5415 | 14575 | reference_compilers: Vec::new(), |
| 5416 | 14576 | consistency_checks: Vec::new(), |
| 5417 | 14577 | expectations: Vec::new(), |
| 5418 | 14578 | status_rules: Vec::new(), |
| 14579 | + capability_policy: None, |
| 5419 | 14580 | }; |
| 5420 | | - let outcome = Outcome { |
| 5421 | | - suite: suite.name.clone(), |
| 5422 | | - case: case.name.clone(), |
| 5423 | | - opt_level: OptLevel::O0, |
| 5424 | | - kind: OutcomeKind::Fail, |
| 5425 | | - detail: "boom".into(), |
| 5426 | | - bundle: None, |
| 5427 | | - consistency_observations: Vec::new(), |
| 5428 | | - }; |
| 5429 | | - let artifacts = ExecutionArtifacts { |
| 5430 | | - requested: BTreeSet::from([Stage::Run]), |
| 5431 | | - armfortas: Some(run_only_result("42\n", "", 0)), |
| 5432 | | - armfortas_failure: None, |
| 5433 | | - references: Vec::new(), |
| 5434 | | - consistency_issues: Vec::new(), |
| 5435 | | - }; |
| 5436 | | - let prepared = PreparedInput { |
| 5437 | | - compiler_source: generated.clone(), |
| 5438 | | - generated_source: Some(generated.clone()), |
| 5439 | | - temp_root: None, |
| 14581 | + let linked_case = CaseSpec { |
| 14582 | + name: "linked".into(), |
| 14583 | + source: PathBuf::from("demo.f90"), |
| 14584 | + graph_files: Vec::new(), |
| 14585 | + requested: BTreeSet::from([Stage::Tokens]), |
| 14586 | + generic_introspect: None, |
| 14587 | + generic_compare: None, |
| 14588 | + opt_levels: vec![OptLevel::O0, OptLevel::O1], |
| 14589 | + repeat_count: 2, |
| 14590 | + reference_compilers: vec![ReferenceCompiler::Gfortran], |
| 14591 | + consistency_checks: vec![ConsistencyCheck::CaptureAsmReproducible], |
| 14592 | + expectations: Vec::new(), |
| 14593 | + status_rules: Vec::new(), |
| 14594 | + capability_policy: None, |
| 5440 | 14595 | }; |
| 5441 | 14596 | |
| 5442 | | - let bundle = write_failure_bundle(&suite, &case, &prepared, &outcome, &artifacts).unwrap(); |
| 5443 | | - assert!(bundle.join("source.f90").exists()); |
| 5444 | | - assert!(bundle.join("sources").join("00_math_values.f90").exists()); |
| 5445 | | - assert!(bundle.join("sources").join("01_main.f90").exists()); |
| 14597 | + let tools = ToolchainConfig { |
| 14598 | + armfortas: ArmfortasCliAdapter::External("/tmp/armfortas".into()), |
| 14599 | + ..ToolchainConfig::from_env() |
| 14600 | + }; |
| 14601 | + let observable_lines = case_discovery_lines(&observable_case, &tools); |
| 14602 | + let linked_lines = case_discovery_lines(&linked_case, &tools); |
| 5446 | 14603 | |
| 5447 | | - let _ = fs::remove_dir_all(bundle); |
| 5448 | | - let _ = fs::remove_dir_all(&root); |
| 14604 | + assert!(observable_lines.contains(&"surface: observable-only legacy path".to_string())); |
| 14605 | + assert!(observable_lines.contains(&"capability_status: ready".to_string())); |
| 14606 | + assert!(linked_lines.contains(&"surface: linked armfortas capture".to_string())); |
| 14607 | + assert!(linked_lines |
| 14608 | + .iter() |
| 14609 | + .any(|line| line.contains("differential: gfortran"))); |
| 14610 | + assert!(linked_lines |
| 14611 | + .iter() |
| 14612 | + .any(|line| line.contains("consistency: capture_asm_reproducible"))); |
| 5449 | 14613 | } |
| 5450 | 14614 | |
| 5451 | 14615 | #[test] |
| 5452 | | - fn render_summary_includes_consistency_rollups() { |
| 5453 | | - let mut summary = Summary::default(); |
| 5454 | | - summary.record_consistency(&[ |
| 5455 | | - ConsistencyObservation { |
| 5456 | | - check: ConsistencyCheck::CliAsmReproducible, |
| 5457 | | - summary: "repeat_count=3 unique_variants=3".into(), |
| 5458 | | - repeat_count: Some(3), |
| 5459 | | - unique_variant_count: Some(3), |
| 5460 | | - varying_components: Vec::new(), |
| 5461 | | - stable_components: Vec::new(), |
| 5462 | | - }, |
| 5463 | | - ConsistencyObservation { |
| 5464 | | - check: ConsistencyCheck::CliObjReproducible, |
| 5465 | | - summary: |
| 5466 | | - "repeat_count=3 unique_variants=2 varying_components=text stable_components=load_commands, relocations, symbols" |
| 5467 | | - .into(), |
| 5468 | | - repeat_count: Some(3), |
| 5469 | | - unique_variant_count: Some(2), |
| 5470 | | - varying_components: vec!["text".into()], |
| 5471 | | - stable_components: vec![ |
| 5472 | | - "load_commands".into(), |
| 5473 | | - "relocations".into(), |
| 5474 | | - "symbols".into(), |
| 5475 | | - ], |
| 5476 | | - }, |
| 5477 | | - ]); |
| 14616 | + fn write_doctor_reports_emits_files() { |
| 14617 | + let root = std::env::temp_dir().join("afs_tests_doctor_report_output"); |
| 14618 | + let _ = fs::remove_dir_all(&root); |
| 14619 | + fs::create_dir_all(&root).unwrap(); |
| 14620 | + let json_path = root.join("doctor.json"); |
| 14621 | + let markdown_path = root.join("doctor.md"); |
| 14622 | + let config = DoctorConfig { |
| 14623 | + tools: ToolchainConfig::from_env(), |
| 14624 | + json_report: Some(json_path.clone()), |
| 14625 | + markdown_report: Some(markdown_path.clone()), |
| 14626 | + }; |
| 5478 | 14627 | |
| 5479 | | - let rendered = render_summary(&summary); |
| 5480 | | - assert!(rendered.contains("Consistency")); |
| 5481 | | - assert!(rendered.contains("affected_checks: 2")); |
| 5482 | | - assert!(rendered.contains("cells_with_issues: 2")); |
| 5483 | | - assert!( |
| 5484 | | - rendered.contains("cli_asm_reproducible: 1 cells; repeat_count=3; unique_variants=3") |
| 5485 | | - ); |
| 5486 | | - assert!(rendered.contains( |
| 5487 | | - "cli_obj_reproducible: 1 cells; repeat_count=3; unique_variants=2; varying=text; stable=load_commands, relocations, symbols" |
| 5488 | | - )); |
| 14628 | + write_doctor_reports(&config).unwrap(); |
| 14629 | + |
| 14630 | + let json = fs::read_to_string(&json_path).unwrap(); |
| 14631 | + let markdown = fs::read_to_string(&markdown_path).unwrap(); |
| 14632 | + assert!(json.contains("\"command\": \"doctor\"")); |
| 14633 | + assert!(json.contains("\"workspace_root\"")); |
| 14634 | + assert!(markdown.contains("# bencch doctor report")); |
| 14635 | + assert!(markdown.contains("| `workspace_root` |")); |
| 14636 | + |
| 14637 | + let _ = fs::remove_dir_all(&root); |
| 5489 | 14638 | } |
| 5490 | 14639 | |
| 5491 | 14640 | #[test] |
| 5492 | 14641 | fn differential_reports_armfortas_only_divergence() { |
| 5493 | | - let result = run_only_result("0\n", "", 0); |
| 14642 | + let armfortas = differential_armfortas_observation("0\n", "", 0); |
| 5494 | 14643 | let refs = vec![ |
| 5495 | | - reference_run(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 5496 | | - reference_run(ReferenceCompiler::FlangNew, "42\n", "", 0), |
| 14644 | + differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14645 | + differential_reference_observation(ReferenceCompiler::FlangNew, "42\n", "", 0), |
| 5497 | 14646 | ]; |
| 5498 | 14647 | |
| 5499 | | - let err = compare_differential(&result, &refs).unwrap_err(); |
| 14648 | + let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 5500 | 14649 | assert!(err.contains("classification: armfortas-only divergence")); |
| 14650 | + assert!(err.contains("basis: compile-status, diagnostics, runtime")); |
| 5501 | 14651 | } |
| 5502 | 14652 | |
| 5503 | 14653 | #[test] |
| 5504 | 14654 | fn differential_reports_reference_disagreement() { |
| 5505 | | - let result = run_only_result("42\n", "", 0); |
| 14655 | + let armfortas = differential_armfortas_observation("42\n", "", 0); |
| 5506 | 14656 | let refs = vec![ |
| 5507 | | - reference_run(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 5508 | | - reference_run(ReferenceCompiler::FlangNew, "99\n", "", 0), |
| 14657 | + differential_reference_observation(ReferenceCompiler::Gfortran, "42\n", "", 0), |
| 14658 | + differential_reference_observation(ReferenceCompiler::FlangNew, "99\n", "", 0), |
| 5509 | 14659 | ]; |
| 5510 | 14660 | |
| 5511 | | - let err = compare_differential(&result, &refs).unwrap_err(); |
| 14661 | + let err = compare_differential(&armfortas, &refs).unwrap_err(); |
| 5512 | 14662 | assert!(err.contains("classification: reference disagreement")); |
| 5513 | 14663 | } |
| 5514 | 14664 | |
| 5515 | 14665 | #[test] |
| 5516 | 14666 | fn differential_tolerates_numeric_formatting_differences() { |
| 5517 | | - let result = run_only_result(" 5.5000000E0\n", "", 0); |
| 14667 | + let armfortas = differential_armfortas_observation(" 5.5000000E0\n", "", 0); |
| 5518 | 14668 | let refs = vec![ |
| 5519 | | - reference_run(ReferenceCompiler::Gfortran, " 5.50000000\n", "", 0), |
| 5520 | | - reference_run(ReferenceCompiler::FlangNew, " 5.5\n", "", 0), |
| 14669 | + differential_reference_observation( |
| 14670 | + ReferenceCompiler::Gfortran, |
| 14671 | + " 5.50000000\n", |
| 14672 | + "", |
| 14673 | + 0, |
| 14674 | + ), |
| 14675 | + differential_reference_observation(ReferenceCompiler::FlangNew, " 5.5\n", "", 0), |
| 5521 | 14676 | ]; |
| 5522 | 14677 | |
| 5523 | | - assert!(compare_differential(&result, &refs).is_ok()); |
| 14678 | + assert!(compare_differential(&armfortas, &refs).is_ok()); |
| 5524 | 14679 | } |
| 5525 | 14680 | |
| 5526 | 14681 | #[test] |
@@ -5531,6 +14686,14 @@ end |
| 5531 | 14686 | assert!(detail.contains("right: gamma")); |
| 5532 | 14687 | } |
| 5533 | 14688 | |
| 14689 | + #[test] |
| 14690 | + fn consistency_diff_reports_first_extra_line() { |
| 14691 | + let detail = describe_text_difference("alpha\nbeta\n", "", "left", "right"); |
| 14692 | + assert!(detail.contains("snapshot length differs")); |
| 14693 | + assert!(detail.contains("first extra line: 1")); |
| 14694 | + assert!(detail.contains("left: alpha")); |
| 14695 | + } |
| 14696 | + |
| 5534 | 14697 | #[test] |
| 5535 | 14698 | fn object_diff_reports_changed_components() { |
| 5536 | 14699 | let expected = ObjectSnapshot { |
@@ -5721,4 +14884,217 @@ end |
| 5721 | 14884 | errors |
| 5722 | 14885 | ); |
| 5723 | 14886 | } |
| 14887 | + |
| 14888 | + #[test] |
| 14889 | + fn failure_expectation_precedes_partial_stage_checks() { |
| 14890 | + let case = CaseSpec { |
| 14891 | + name: "missing_then".into(), |
| 14892 | + source: PathBuf::from("demo.f90"), |
| 14893 | + graph_files: Vec::new(), |
| 14894 | + requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14895 | + generic_introspect: None, |
| 14896 | + generic_compare: None, |
| 14897 | + opt_levels: vec![OptLevel::O0], |
| 14898 | + repeat_count: 3, |
| 14899 | + reference_compilers: Vec::new(), |
| 14900 | + consistency_checks: Vec::new(), |
| 14901 | + expectations: vec![ |
| 14902 | + Expectation::Contains { |
| 14903 | + target: Target::RunStdout, |
| 14904 | + needle: "42".into(), |
| 14905 | + }, |
| 14906 | + Expectation::FailContains { |
| 14907 | + stage: FailureStage::Parser, |
| 14908 | + needle: "expected 'then'".into(), |
| 14909 | + }, |
| 14910 | + ], |
| 14911 | + status_rules: Vec::new(), |
| 14912 | + capability_policy: None, |
| 14913 | + }; |
| 14914 | + let artifacts = ExecutionArtifacts { |
| 14915 | + requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14916 | + armfortas: None, |
| 14917 | + armfortas_failure: None, |
| 14918 | + armfortas_observation: None, |
| 14919 | + references: Vec::new(), |
| 14920 | + reference_observations: Vec::new(), |
| 14921 | + consistency_issues: Vec::new(), |
| 14922 | + }; |
| 14923 | + let failure = CaptureFailure { |
| 14924 | + input: PathBuf::from("demo.f90"), |
| 14925 | + opt_level: OptLevel::O0, |
| 14926 | + stage: FailureStage::Parser, |
| 14927 | + detail: "expected 'then'".into(), |
| 14928 | + stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14929 | + }; |
| 14930 | + |
| 14931 | + assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14932 | + } |
| 14933 | + |
| 14934 | + #[test] |
| 14935 | + fn legacy_capture_failures_use_observation_failure_semantics() { |
| 14936 | + let case = CaseSpec { |
| 14937 | + name: "hidden_use_only".into(), |
| 14938 | + source: PathBuf::from("demo.f90"), |
| 14939 | + graph_files: Vec::new(), |
| 14940 | + requested: BTreeSet::from([Stage::Run]), |
| 14941 | + generic_introspect: None, |
| 14942 | + generic_compare: None, |
| 14943 | + opt_levels: vec![OptLevel::O0], |
| 14944 | + repeat_count: 3, |
| 14945 | + reference_compilers: Vec::new(), |
| 14946 | + consistency_checks: Vec::new(), |
| 14947 | + expectations: vec![Expectation::FailCommentPatterns(vec!["hidden".into()])], |
| 14948 | + status_rules: Vec::new(), |
| 14949 | + capability_policy: None, |
| 14950 | + }; |
| 14951 | + let artifacts = ExecutionArtifacts { |
| 14952 | + requested: BTreeSet::from([Stage::Run]), |
| 14953 | + armfortas: None, |
| 14954 | + armfortas_failure: None, |
| 14955 | + armfortas_observation: None, |
| 14956 | + references: Vec::new(), |
| 14957 | + reference_observations: Vec::new(), |
| 14958 | + consistency_issues: Vec::new(), |
| 14959 | + }; |
| 14960 | + let failure = CaptureFailure { |
| 14961 | + input: PathBuf::from("demo.f90"), |
| 14962 | + opt_level: OptLevel::O0, |
| 14963 | + stage: FailureStage::Sema, |
| 14964 | + detail: "demo.f90:4:3: semantic error: hidden".into(), |
| 14965 | + stages: BTreeMap::new(), |
| 14966 | + }; |
| 14967 | + |
| 14968 | + assert!(evaluate_failed_armfortas(&case, &artifacts, &failure).is_ok()); |
| 14969 | + } |
| 14970 | + |
| 14971 | + #[test] |
| 14972 | + fn legacy_failure_observed_program_uses_prepared_source_and_partial_stages() { |
| 14973 | + let case = CaseSpec { |
| 14974 | + name: "missing_then".into(), |
| 14975 | + source: PathBuf::from("authored.f90"), |
| 14976 | + graph_files: Vec::new(), |
| 14977 | + requested: BTreeSet::from([Stage::Tokens, Stage::Run]), |
| 14978 | + generic_introspect: None, |
| 14979 | + generic_compare: None, |
| 14980 | + opt_levels: vec![OptLevel::O0], |
| 14981 | + repeat_count: 3, |
| 14982 | + reference_compilers: Vec::new(), |
| 14983 | + consistency_checks: Vec::new(), |
| 14984 | + expectations: vec![Expectation::FailContains { |
| 14985 | + stage: FailureStage::Parser, |
| 14986 | + needle: "expected 'then'".into(), |
| 14987 | + }], |
| 14988 | + status_rules: Vec::new(), |
| 14989 | + capability_policy: None, |
| 14990 | + }; |
| 14991 | + let failure = CaptureFailure { |
| 14992 | + input: PathBuf::from("generated.f90"), |
| 14993 | + opt_level: OptLevel::O0, |
| 14994 | + stage: FailureStage::Parser, |
| 14995 | + detail: "expected 'then'".into(), |
| 14996 | + stages: BTreeMap::from([(Stage::Tokens, CapturedStage::Text("if\n".into()))]), |
| 14997 | + }; |
| 14998 | + |
| 14999 | + let observed = legacy_failure_observed_program(Path::new("prepared.f90"), &case, &failure); |
| 15000 | + assert_eq!(observed.observation.program, PathBuf::from("prepared.f90")); |
| 15001 | + assert_eq!(observed.observation.compile_exit_code, 1); |
| 15002 | + assert_eq!( |
| 15003 | + observed.observation.provenance.failure_stage.as_deref(), |
| 15004 | + Some("parser") |
| 15005 | + ); |
| 15006 | + assert!(observed |
| 15007 | + .observation |
| 15008 | + .artifacts |
| 15009 | + .contains_key(&ArtifactKey::Diagnostics)); |
| 15010 | + assert!(observed |
| 15011 | + .observation |
| 15012 | + .artifacts |
| 15013 | + .contains_key(&ArtifactKey::Extra("armfortas.tokens".into()))); |
| 15014 | + } |
| 15015 | + |
| 15016 | + #[test] |
| 15017 | + fn unexpected_capture_failure_reports_compiler_failure_detail() { |
| 15018 | + let case = CaseSpec { |
| 15019 | + name: "module_procedure_runtime".into(), |
| 15020 | + source: PathBuf::from("graph.f90"), |
| 15021 | + graph_files: Vec::new(), |
| 15022 | + requested: BTreeSet::from([Stage::Run]), |
| 15023 | + generic_introspect: None, |
| 15024 | + generic_compare: None, |
| 15025 | + opt_levels: vec![OptLevel::O0], |
| 15026 | + repeat_count: 3, |
| 15027 | + reference_compilers: Vec::new(), |
| 15028 | + consistency_checks: Vec::new(), |
| 15029 | + expectations: vec![Expectation::Contains { |
| 15030 | + target: Target::RunStdout, |
| 15031 | + needle: "42".into(), |
| 15032 | + }], |
| 15033 | + status_rules: Vec::new(), |
| 15034 | + capability_policy: None, |
| 15035 | + }; |
| 15036 | + let failure = CaptureFailure { |
| 15037 | + input: PathBuf::from("graph.f90"), |
| 15038 | + opt_level: OptLevel::O0, |
| 15039 | + stage: FailureStage::Run, |
| 15040 | + detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 15041 | + stages: BTreeMap::new(), |
| 15042 | + }; |
| 15043 | + let artifacts = ExecutionArtifacts { |
| 15044 | + requested: BTreeSet::from([Stage::Run]), |
| 15045 | + armfortas: None, |
| 15046 | + armfortas_failure: Some(failure.clone()), |
| 15047 | + armfortas_observation: None, |
| 15048 | + references: Vec::new(), |
| 15049 | + reference_observations: Vec::new(), |
| 15050 | + consistency_issues: Vec::new(), |
| 15051 | + }; |
| 15052 | + |
| 15053 | + let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 15054 | + assert!(err.contains("armfortas failed in run")); |
| 15055 | + assert!(err.contains("_add_one")); |
| 15056 | + assert!(!err.contains("missing captured run stage")); |
| 15057 | + } |
| 15058 | + |
| 15059 | + #[test] |
| 15060 | + fn partial_stage_expectation_failure_is_preserved_on_capture_failure() { |
| 15061 | + let case = CaseSpec { |
| 15062 | + name: "module_procedure_backend".into(), |
| 15063 | + source: PathBuf::from("graph.f90"), |
| 15064 | + graph_files: Vec::new(), |
| 15065 | + requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 15066 | + generic_introspect: None, |
| 15067 | + generic_compare: None, |
| 15068 | + opt_levels: vec![OptLevel::O0], |
| 15069 | + repeat_count: 3, |
| 15070 | + reference_compilers: Vec::new(), |
| 15071 | + consistency_checks: Vec::new(), |
| 15072 | + expectations: vec![Expectation::Contains { |
| 15073 | + target: Target::Stage(Stage::Asm), |
| 15074 | + needle: ".globl _add_one".into(), |
| 15075 | + }], |
| 15076 | + status_rules: Vec::new(), |
| 15077 | + capability_policy: None, |
| 15078 | + }; |
| 15079 | + let failure = CaptureFailure { |
| 15080 | + input: PathBuf::from("graph.f90"), |
| 15081 | + opt_level: OptLevel::O0, |
| 15082 | + stage: FailureStage::Run, |
| 15083 | + detail: "Undefined symbols for architecture arm64:\n \"_add_one\"".into(), |
| 15084 | + stages: BTreeMap::from([(Stage::Asm, CapturedStage::Text(".globl _main\n".into()))]), |
| 15085 | + }; |
| 15086 | + let artifacts = ExecutionArtifacts { |
| 15087 | + requested: BTreeSet::from([Stage::Asm, Stage::Obj, Stage::Run]), |
| 15088 | + armfortas: None, |
| 15089 | + armfortas_failure: Some(failure.clone()), |
| 15090 | + armfortas_observation: None, |
| 15091 | + references: Vec::new(), |
| 15092 | + reference_observations: Vec::new(), |
| 15093 | + consistency_issues: Vec::new(), |
| 15094 | + }; |
| 15095 | + |
| 15096 | + let err = evaluate_failed_armfortas(&case, &artifacts, &failure).unwrap_err(); |
| 15097 | + assert!(err.contains("expected asm to contain")); |
| 15098 | + assert!(!err.contains("armfortas failed in run")); |
| 15099 | + } |
| 5724 | 15100 | } |