`afcd90f`

Report parity matrix timings

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 1 week ago

SHA: afcd90fd76b1c6845c9b8d72e92d2cd087ab55c4
Parents: e41e4b6
Tree: 49b10ac

2 changed files

Status	File	+	-
M	`tests/common/harness.rs`	16	14
M	`tests/parity_matrix.rs`	159	58

tests/common/harness.rsmodified


 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum ArtifactKind {
+    Dylib,
+    Archive,
+    ReexportDylib,
 }
 
+type SymbolPartitions = (Vec<String>, Vec<String>, Vec<String>);
+
 pub struct LinkOutputs {
     pub ours: Vec<u8>,
     pub theirs: Vec<u8>,

             .map_err(|e| format!("read artifact src {}: {e}", src.display()))?;
         let out = work_dir.join(&artifact.out_name);
         match artifact.kind {
+            ArtifactKind::Dylib => compile_dylib_c(&src_contents, &out)?,
+            ArtifactKind::Archive => compile_archive_c(&src_contents, &out)?,
+            ArtifactKind::ReexportDylib => {
                 let dep_name = artifact.dep_name.as_ref().ok_or_else(|| {
                     format!(
                         "missing reexport dependency for artifact {}",

                         path.display()
                     ));
                 }
+                (ArtifactKind::Dylib, None)
             }
             "clang_archive" => {
                 if dep_name.is_some() {

                         path.display()
                     ));
                 }
+                (ArtifactKind::Archive, None)
             }
             "clang_reexport_dylib" => {
                 let dep_name = dep_name.ok_or_else(|| {

                         path.display()
                     )
                 })?;
+                (ArtifactKind::ReexportDylib, Some(dep_name))
             }
             other => return Err(format!("unknown artifact kind `{other}`")),
         };

                 }
             }
             LC_ID_DYLIB | LC_LOAD_DYLIB | LC_LOAD_WEAK_DYLIB | LC_REEXPORT_DYLIB
+            | LC_LOAD_UPWARD_DYLIB
+                if cmdsize >= 16 =>
+            {
+                mark_range(&mut mask, cursor + 12, cursor + 16, "dylib timestamp");
             }
             _ => {}
         }

     Ok(out)
 }
 
+fn symbol_partition_names(bytes: &[u8]) -> Result<SymbolPartitions, String> {
     let (symtab, dysymtab) = symtab_and_dysymtab(bytes)?;
     let symbols =
         parse_nlist_table(bytes, symtab.symoff, symtab.nsyms).map_err(|e| e.to_string())?;

tests/parity_matrix.rsmodified

+         }
          case_reports.push((case, report));
+     }
++    print_timing_summary(started.elapsed(), &case_reports);
      if let Some(dir) = artifact_dir.as_ref() {
          write_index_artifact(dir, &case_reports).expect("write parity index");
  #[derive(Debug)]
  struct CaseStep {
      name: &'static str,
++    duration: Duration,
      error: Option<String>,
+ }
  #[derive(Debug, Default)]
  struct CaseReport {
      steps: Vec<CaseStep>,
++    elapsed: Duration,
+ }
  impl CaseReport {
      fn push(&mut self, name: &'static str, result: Result<(), String>) -> bool {
++        self.push_timed(name, Duration::ZERO, result)
++    }
++
++    fn push_timed(
++        &mut self,
++        name: &'static str,
++        duration: Duration,
++        result: Result<(), String>,
++    ) -> bool {
          match result {
              Ok(()) => {
--                self.steps.push(CaseStep { name, error: None });
++                self.steps.push(CaseStep {
++                    name,
++                    duration,
++                    error: None,
++                });
                  true
+             }
              Err(error) => {
                  self.steps.push(CaseStep {
                      name,
++                    duration,
                      error: Some(error),
                  });
                  false
+         }
+     }
++    fn measure<F>(&mut self, name: &'static str, action: F) -> bool
++    where
++        F: FnOnce() -> Result<(), String>,
++    {
++        let started = Instant::now();
++        let result = action();
++        self.push_timed(name, started.elapsed(), result)
++    }
++
++    fn finish(&mut self, elapsed: Duration) {
++        self.elapsed = elapsed;
++    }
++
      fn passed(&self) -> bool {
          self.steps.iter().all(|step| step.error.is_none())
+     }
++    fn slowest_step(&self) -> Option<&CaseStep> {
++        self.steps.iter().max_by_key(|step| step.duration)
++    }
++
      fn error_message(&self, case_name: &str) -> Option<String> {
          self.steps.iter().find_map(|step| {
--            step.error.as_ref().map(|error| {
++            step.error
--                format!("[{case_name}] {} failed:\n{}", step.name, error)
++                .as_ref()
--            })
++                .map(|error| format!("[{case_name}] {} failed:\n{}", step.name, error))
          })
+     }
+ }
+ }
  fn run_case(case: &LinkCase) -> CaseReport {
++    let case_started = Instant::now();
      let mut report = CaseReport::default();
++    let link_started = Instant::now();
      let outputs = match link_both(case) {
          Ok(outputs) => {
--            report.push("link", Ok(()));
++            report.push_timed("link", link_started.elapsed(), Ok(()));
              outputs
+         }
          Err(error) => {
--            report.push(
++            report.push_timed(
                  "link",
++                link_started.elapsed(),
                  Err(format!(
                      "failed to link parity case from {}:\n{}",
                      case.dir.display(),
                      error
                  )),
              );
--            return report;
++            return finish_case(report, case_started);
+         }
      };
--    if !report.push(
++    if !report.measure("load-command ids", || {
--        "load-command ids",
++        compare_command_ids(&outputs.ours, &outputs.theirs, &case.ignored_load_commands)
--        compare_command_ids(&outputs.ours, &outputs.theirs, &case.ignored_load_commands),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("command details", || {
--        "command details",
++        compare_command_details(&outputs.ours, &outputs.theirs, &case.command_checks)
--        compare_command_details(&outputs.ours, &outputs.theirs, &case.command_checks),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("afs-ld absent commands", || {
--        "afs-ld absent commands",
++        ensure_absent_load_commands(&outputs.ours, &case.absent_load_commands, "afs-ld")
--        ensure_absent_load_commands(&outputs.ours, &case.absent_load_commands, "afs-ld"),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("Apple absent commands", || {
--        "Apple absent commands",
++        ensure_absent_load_commands(&outputs.theirs, &case.absent_load_commands, "Apple ld")
--        ensure_absent_load_commands(&outputs.theirs, &case.absent_load_commands, "Apple ld"),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("afs-ld absent sections", || {
--        "afs-ld absent sections",
++        ensure_absent_sections(&outputs.ours, &case.absent_sections, "afs-ld")
--        ensure_absent_sections(&outputs.ours, &case.absent_sections, "afs-ld"),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("Apple absent sections", || {
--        "Apple absent sections",
++        ensure_absent_sections(&outputs.theirs, &case.absent_sections, "Apple ld")
--        ensure_absent_sections(&outputs.theirs, &case.absent_sections, "Apple ld"),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
--    if !report.push(
++    if !report.measure("section parity", || {
--        "section parity",
          compare_sections(
              &outputs.ours,
              &outputs.theirs,
              &case.section_checks,
              &case.case_tolerances,
--        ),
++        )
--    ) {
++    }) {
--        return report;
++        return finish_case(report, case_started);
+     }
--    if !report.push(
++    if !report.measure("page-ref parity", || {
--        "page-ref parity",
++        compare_page_refs(&outputs.ours, &outputs.theirs, &case.page_ref_checks)
--        compare_page_refs(&outputs.ours, &outputs.theirs, &case.page_ref_checks),
++    }) {
--    ) {
++        return finish_case(report, case_started);
--        return report;
+     }
      if !case.runtime_args.is_empty() || case.dir.join("runtime.txt").exists() {
--        report.push(
++        report.measure("runtime parity", || {
--            "runtime parity",
++            compare_runtime(&outputs.our_path, &outputs.their_path, &case.runtime_args)
--            compare_runtime(&outputs.our_path, &outputs.their_path, &case.runtime_args),
++        });
--        );
+     }
++    finish_case(report, case_started)
++}
++
++fn finish_case(mut report: CaseReport, started: Instant) -> CaseReport {
++    report.finish(started.elapsed());
      report
+ }
          if report.passed() { "ok" } else { "fail" },
          if report.passed() { "PASS" } else { "FAIL" }
      ));
++    html.push_str(&format!(
++        "<p>Total: <strong>{}</strong></p>",
++        format_duration(report.elapsed)
++    ));
      html.push_str("<h2>Steps</h2><ul>");
      for step in &report.steps {
          match &step.error {
              None => html.push_str(&format!(
--                "<li><span class=\"ok\">PASS</span> {}</li>",
++                "<li><span class=\"ok\">PASS</span> {} <span class=\"time\">{}</span></li>",
--                escape_html(step.name)
++                escape_html(step.name),
++                format_duration(step.duration)
              )),
              Some(error) => html.push_str(&format!(
--                "<li><span class=\"fail\">FAIL</span> {}<pre>{}</pre></li>",
++                "<li><span class=\"fail\">FAIL</span> {} <span class=\"time\">{}</span><pre>{}</pre></li>",
                  escape_html(step.name),
++                format_duration(step.duration),
                  escape_html(error)
              )),
+         }
  fn write_index_artifact(dir: &Path, cases: &[(LinkCase, CaseReport)]) -> Result<(), String> {
      let mut html = String::new();
      html.push_str("<!doctype html><html><head><meta charset=\"utf-8\">");
--    html.push_str("<title>Parity Matrix</title><style>body{font-family:ui-monospace,Menlo,monospace;padding:2rem;} .ok{color:#0a0;} .fail{color:#a00;}</style></head><body>");
++    html.push_str("<title>Parity Matrix</title><style>body{font-family:ui-monospace,Menlo,monospace;padding:2rem;} .ok{color:#0a0;} .fail{color:#a00;} .time{color:#57606a;} table{border-collapse:collapse;margin:1rem 0;} td,th{border:1px solid #d0d7de;padding:.35rem .6rem;text-align:left;}</style></head><body>");
--    html.push_str("<h1>Parity Matrix</h1><ul>");
++    html.push_str("<h1>Parity Matrix</h1>");
++    html.push_str("<h2>Slowest Cases</h2><table><thead><tr><th>Case</th><th>Total</th><th>Slowest Step</th></tr></thead><tbody>");
++    for (case, report) in slowest_cases(cases, 10) {
++        let slowest = report
++            .slowest_step()
++            .map(|step| format!("{} {}", step.name, format_duration(step.duration)))
++            .unwrap_or_else(|| "n/a".to_string());
++        html.push_str(&format!(
++            "<tr><td><a href=\"{}.html\">{}</a></td><td>{}</td><td>{}</td></tr>",
++            slug(&case.name),
++            escape_html(&case.name),
++            format_duration(report.elapsed),
++            escape_html(&slowest)
++        ));
++    }
++    html.push_str("</tbody></table><h2>Cases</h2><ul>");
      for (case, report) in cases {
          let slug = slug(&case.name);
          html.push_str(&format!(
--            "<li><a href=\"{}.html\">{}</a> <strong class=\"{}\">{}</strong></li>",
++            "<li><a href=\"{}.html\">{}</a> <strong class=\"{}\">{}</strong> <span class=\"time\">{}</span></li>",
              slug,
              escape_html(&case.name),
              if report.passed() { "ok" } else { "fail" },
--            if report.passed() { "PASS" } else { "FAIL" }
++            if report.passed() { "PASS" } else { "FAIL" },
++            format_duration(report.elapsed)
          ));
+     }
      html.push_str("</ul></body></html>");
      fs::write(&path, html).map_err(|e| format!("write {}: {e}", path.display()))
+ }
++fn print_timing_summary(elapsed: Duration, cases: &[(LinkCase, CaseReport)]) {
++    eprintln!(
++        "parity matrix timing: {} case(s) in {}",
++        cases.len(),
++        format_duration(elapsed)
++    );
++    for (case, report) in slowest_cases(cases, 10) {
++        let slowest = report
++            .slowest_step()
++            .map(|step| {
++                format!(
++                    "; slowest step: {} {}",
++                    step.name,
++                    format_duration(step.duration)
++                )
++            })
++            .unwrap_or_default();
++        eprintln!(
++            "  {:>9} {}{}",
++            format_duration(report.elapsed),
++            case.name,
++            slowest
++        );
++    }
++}
++
++fn slowest_cases(cases: &[(LinkCase, CaseReport)], limit: usize) -> Vec<(&LinkCase, &CaseReport)> {
++    let mut timed: Vec<_> = cases.iter().map(|(case, report)| (case, report)).collect();
++    timed.sort_by(|a, b| {
++        b.1.elapsed
++            .cmp(&a.1.elapsed)
++            .then_with(|| a.0.name.cmp(&b.0.name))
++    });
++    timed.truncate(limit);
++    timed
++}
++
  fn slug(name: &str) -> String {
      name.chars()
          .map(|ch| {
      Some(Duration::from_secs(seconds))
+ }
++fn format_duration(duration: Duration) -> String {
++    let millis = duration.as_secs_f64() * 1000.0;
++    if millis >= 1000.0 {
++        format!("{:.2}s", duration.as_secs_f64())
++    } else {
++        format!("{millis:.1}ms")
++    }
++}
++
  fn escape_html(text: &str) -> String {
      text.replace('&', "&amp;")
          .replace('<', "&lt;")

`@@ -116,11 +116,13 @@` struct ArtifactSpec {
116		116
117	#[derive(Debug, Clone, Copy, PartialEq, Eq)]	117	#[derive(Debug, Clone, Copy, PartialEq, Eq)]
118	enum ArtifactKind {	118	enum ArtifactKind {
119	- ClangDylib,	119	+ Dylib,
120	- ClangArchive,	120	+ Archive,
121	- ClangReexportDylib,	121	+ ReexportDylib,
122	}	122	}
123		123
		124	+type SymbolPartitions = (Vec<String>, Vec<String>, Vec<String>);
		125	+
124	pub struct LinkOutputs {	126	pub struct LinkOutputs {
125	pub ours: Vec<u8>,	127	pub ours: Vec<u8>,
126	pub theirs: Vec<u8>,	128	pub theirs: Vec<u8>,
`@@ -509,9 +511,9 @@` pub fn link_both(case: &LinkCase) -> Result<LinkOutputs, String> {
509	.map_err(\|e\| format!("read artifact src {}: {e}", src.display()))?;	511	.map_err(\|e\| format!("read artifact src {}: {e}", src.display()))?;
510	let out = work_dir.join(&artifact.out_name);	512	let out = work_dir.join(&artifact.out_name);
511	match artifact.kind {	513	match artifact.kind {
512	- ArtifactKind::ClangDylib => compile_dylib_c(&src_contents, &out)?,	514	+ ArtifactKind::Dylib => compile_dylib_c(&src_contents, &out)?,
513	- ArtifactKind::ClangArchive => compile_archive_c(&src_contents, &out)?,	515	+ ArtifactKind::Archive => compile_archive_c(&src_contents, &out)?,
514	- ArtifactKind::ClangReexportDylib => {	516	+ ArtifactKind::ReexportDylib => {
515	let dep_name = artifact.dep_name.as_ref().ok_or_else(\|\| {	517	let dep_name = artifact.dep_name.as_ref().ok_or_else(\|\| {
516	format!(	518	format!(
517	"missing reexport dependency for artifact {}",	519	"missing reexport dependency for artifact {}",
`@@ -1390,7 +1392,7 @@` fn read_artifacts(path: &Path) -> Result<Vec<ArtifactSpec>, String> {
1390	path.display()	1392	path.display()
1391	));	1393	));
1392	}	1394	}
1393	- (ArtifactKind::ClangDylib, None)	1395	+ (ArtifactKind::Dylib, None)
1394	}	1396	}
1395	"clang_archive" => {	1397	"clang_archive" => {
1396	if dep_name.is_some() {	1398	if dep_name.is_some() {
`@@ -1399,7 +1401,7 @@` fn read_artifacts(path: &Path) -> Result<Vec<ArtifactSpec>, String> {
1399	path.display()	1401	path.display()
1400	));	1402	));
1401	}	1403	}
1402	- (ArtifactKind::ClangArchive, None)	1404	+ (ArtifactKind::Archive, None)
1403	}	1405	}
1404	"clang_reexport_dylib" => {	1406	"clang_reexport_dylib" => {
1405	let dep_name = dep_name.ok_or_else(\|\| {	1407	let dep_name = dep_name.ok_or_else(\|\| {
`@@ -1408,7 +1410,7 @@` fn read_artifacts(path: &Path) -> Result<Vec<ArtifactSpec>, String> {
1408	path.display()	1410	path.display()
1409	)	1411	)
1410	})?;	1412	})?;
1411	- (ArtifactKind::ClangReexportDylib, Some(dep_name))	1413	+ (ArtifactKind::ReexportDylib, Some(dep_name))
1412	}	1414	}
1413	other => return Err(format!("unknown artifact kind `{other}`")),	1415	other => return Err(format!("unknown artifact kind `{other}`")),
1414	};	1416	};
`@@ -1592,10 +1594,10 @@` fn tolerated_mask(bytes: &[u8]) -> Vec<Option<&'static str>> {
1592	}	1594	}
1593	}	1595	}
1594	LC_ID_DYLIB \| LC_LOAD_DYLIB \| LC_LOAD_WEAK_DYLIB \| LC_REEXPORT_DYLIB	1596	LC_ID_DYLIB \| LC_LOAD_DYLIB \| LC_LOAD_WEAK_DYLIB \| LC_REEXPORT_DYLIB
1595	- \| LC_LOAD_UPWARD_DYLIB => {	1597	+ \| LC_LOAD_UPWARD_DYLIB
1596	- if cmdsize >= 16 {	1598	+ if cmdsize >= 16 =>
1597	- mark_range(&mut mask, cursor + 12, cursor + 16, "dylib timestamp");	1599	+ {
1598	- }	1600	+ mark_range(&mut mask, cursor + 12, cursor + 16, "dylib timestamp");
1599	}	1601	}
1600	_ => {}	1602	_ => {}
1601	}	1603	}
`@@ -1765,7 +1767,7 @@` fn canonical_export_records(bytes: &[u8]) -> Result<Vec<CanonicalExportRecord>,
1765	Ok(out)	1767	Ok(out)
1766	}	1768	}
1767		1769
1768	-fn symbol_partition_names(bytes: &[u8]) -> Result<(Vec<String>, Vec<String>, Vec<String>), String> {	1770	+fn symbol_partition_names(bytes: &[u8]) -> Result<SymbolPartitions, String> {
1769	let (symtab, dysymtab) = symtab_and_dysymtab(bytes)?;	1771	let (symtab, dysymtab) = symtab_and_dysymtab(bytes)?;
1770	let symbols =	1772	let symbols =
1771	parse_nlist_table(bytes, symtab.symoff, symtab.nsyms).map_err(\|e\| e.to_string())?;	1773	parse_nlist_table(bytes, symtab.symoff, symtab.nsyms).map_err(\|e\| e.to_string())?;