Increase benchmark duration for better results

client/build.rs

@@ -70,7 +70,7 @@ fn main() {
         benchmark_functions.push(function_name);
     }
 
-    writeln!(generated_code, "#[allow(non_camel_case_types)] pub enum Benchmark {{").unwrap();
+    writeln!(generated_code, "#[allow(non_camel_case_types)] #[derive(Clone, Copy)] pub enum Benchmark {{").unwrap();
     for benchmark_name in &benchmark_functions {
         writeln!(generated_code, "    {benchmark_name},").unwrap();
     }

client/src/lib.rs

@@ -14,10 +14,12 @@ fn ptwrite(val: u32) {
 
 pub fn program(benchmark: embench_sys::Benchmark) {
     let start = std::time::Instant::now();
+    for _ in std::hint::black_box(0..100) {
         ptwrite(42);
         unsafe {
             embench_sys::run(benchmark);
         }
         ptwrite(43);
+    }
     println!("{}", start.elapsed().as_secs_f64());
 }

src/benchmark.rs

@@ -1,9 +1,10 @@
+use std::fmt::Debug;
 use std::io::Write;
 
 pub trait Benchmark {
     const TITLE: &'static str;
 
-    type BenchmarkResult: ToCsv;
+    type BenchmarkResult: ToCsv + Debug;
 
     fn execute_once(&mut self) -> Self::BenchmarkResult;
 }

src/benchmark_nyx_pt.rs

@@ -4,6 +4,7 @@ use csv::Writer;
 use libnyx::{NyxConfig, NyxProcess, NyxProcessRole, NyxReturnValue};
 use pt_dump_decoder::{AnalyzeData, analyze_dump};
 use std::error::Error;
+use std::io;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
@@ -13,7 +14,11 @@ pub struct NyxRunner(NyxProcess);
 
 impl NyxRunner {
     pub fn setup(client_bin: &Path) -> Result<Self, Box<dyn Error>> {
-        let _ = std::fs::remove_dir_all(WORKDIR_PATH);
+        if let Err(e) = std::fs::remove_dir_all(WORKDIR_PATH)
+            && !matches!(e.kind(), io::ErrorKind::NotFound)
+        {
+            panic!("Could not remove working directory at {WORKDIR_PATH}");
+        }
         let sharedir = nyx::setup_nyx(&PathBuf::from("nyx_data"), client_bin)?;
         let sharedir = sharedir.to_str().expect("Expected unicode path");
 
@@ -28,6 +33,7 @@ impl NyxRunner {
         let mut nyx_runner = NyxProcess::new(&mut nyx_config, 0)?;
 
         nyx_runner.option_set_trace_mode(true);
+        nyx_runner.option_set_reload_mode(false);
         nyx_runner.option_set_timeout(10, 0);
         nyx_runner.option_apply();
 

src/main.rs

@@ -6,12 +6,13 @@ mod benchmark_nyx_pt;
 mod nyx;
 
 use crate::benchmark::{Benchmark, ToCsv};
-use crate::benchmark_baseline::Baseline;
+use crate::benchmark_nyx_pt::NyxRunner;
 use clap::Parser;
 use std::error::Error;
+use std::fmt::Debug;
 use std::path::{Path, PathBuf};
 use std::time::{Duration, Instant};
-use crate::benchmark_nyx_pt::NyxRunner;
+use crate::benchmark_baseline::Baseline;
 
 #[derive(Debug, Parser)]
 #[command(about = "Tool to execute a program with nyx and trace it with intel pt")]
@@ -26,6 +27,7 @@ struct Args {
 fn main() -> Result<(), Box<dyn Error>> {
     let args = Args::parse();
     let nyx_runner = NyxRunner::setup(&args.client)?;
+    // dbg!({nyx_runner}.execute_once());
     let binary_name = args.client.file_name().unwrap().to_str().unwrap();
     benchmark(binary_name, &args.output_dir, nyx_runner)?;
     benchmark(binary_name, &args.output_dir, Baseline(&args.client))?;
@@ -47,7 +49,7 @@ fn benchmark<B: Benchmark>(
 
 /// Warm up to make sure caches are initialized, etc.
 fn warmup(benchmark: &mut impl Benchmark) {
-    let warmup_duration = Duration::from_secs(60);
+    let warmup_duration = Duration::from_secs(5);
     println!("Warming up for {warmup_duration:?}");
     let mut iterations = 0;
     let start = Instant::now();
@@ -59,10 +61,10 @@ fn warmup(benchmark: &mut impl Benchmark) {
 }
 
 fn benchmark_loop<B: Benchmark>(benchmark: &mut B) -> Vec<B::BenchmarkResult> {
-    let n = 500;
+    let n = 50;
     println!("Perform {n} iterations...");
     (0..n)
-        .map(|_| std::hint::black_box(benchmark.execute_once()))
+        .map(|_| dbg!(std::hint::black_box(benchmark.execute_once())))
         .collect::<Vec<_>>()
 }
 

src/nyx.rs

@@ -95,7 +95,7 @@ fn create_nyx_workdir(out_dir: &Path, client_bin: &Path) -> Result<PathBuf, io::
     shell(
         out_dir,
         &format!(
-            "python3 packer/packer/nyx_packer.py {client_bin} build afl processor_trace --fast_reload_mode --purge"
+            "python3 packer/packer/nyx_packer.py {client_bin} build afl processor_trace --purge"
         ),
     )?;