Add benches and plot generation

Cargo.toml

@@ -25,6 +25,13 @@ assert_fs = "1.0.12"
 predicates = "3.0.1"
 rstest = "0.16.0"
 glob = "0.3.1"
+tempfile = "3.8.1"
+json = "0.12.4"
+plotters = "0.3.5"
 
 [[bin]]
 name = "rs-aggregate"
+
+[[bench]]
+name = "perf"
+harness = false

benches/perf.rs

@@ -0,0 +1,221 @@
+use json::JsonValue;
+use plotters::backend::BitMapBackend;
+use plotters::chart::ChartBuilder;
+use plotters::coord::ranged1d::{IntoSegmentedCoord, SegmentValue, SegmentedCoord};
+use plotters::drawing::IntoDrawingArea;
+use plotters::element::{Circle, EmptyElement, Text};
+use plotters::series::{Histogram, PointSeries};
+use plotters::style::full_palette::{BLUEGREY, GREY};
+use plotters::style::{Color, IntoFont, RGBAColor, RGBColor, ShapeStyle, BLACK, WHITE};
+use std::ffi::OsStr;
+use std::io::Read;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use tempfile::NamedTempFile;
+
+const BAR_COLOUR: RGBColor = RGBColor(66, 133, 244);
+
+#[derive(Clone, Debug)]
+struct TestDefinition {
+    cmd: String,
+    name: String, // including version
+}
+
+#[derive(Clone, Debug)]
+struct TestResult {
+    mean: f64,
+    stddev: f64,
+    median: f64,
+    min: f64,
+    max: f64,
+}
+
+impl From<JsonValue> for TestResult {
+    fn from(value: JsonValue) -> Self {
+        Self {
+            mean: value["mean"].as_f64().unwrap(),
+            stddev: value["stddev"].as_f64().unwrap(),
+            median: value["median"].as_f64().unwrap(),
+            min: value["min"].as_f64().unwrap(),
+            max: value["max"].as_f64().unwrap(),
+        }
+    }
+}
+
+fn make_tests(input_path: &str) -> Vec<TestDefinition> {
+    let our_version = format!("rs-aggregate {}", env!("CARGO_PKG_VERSION"));
+    let our_path = env!("CARGO_BIN_EXE_rs-aggregate");
+
+    let python_version_raw = std::process::Command::new("python3")
+        .arg("--version")
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("Unable to run python3")
+        .wait_with_output()
+        .expect("Couldn't get python3 output")
+        .stdout;
+    let python_version = String::from_utf8_lossy(&python_version_raw);
+
+    let agg6_version_raw = std::process::Command::new("python3")
+        .arg("-m")
+        .arg("aggregate6")
+        .arg("-V")
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("Unable to run aggregate6")
+        .wait_with_output()
+        .expect("Couldn't get aggregate6 output")
+        .stdout;
+    let agg6_version = String::from_utf8_lossy(&agg6_version_raw);
+
+    vec![
+        TestDefinition {
+            cmd: format!("{} {}", our_path, input_path),
+            name: our_version.into(),
+        },
+        TestDefinition {
+            cmd: format!("aggregate6 {}", input_path),
+            // cmd: "sleep 0.25".into(),
+            name: format!("{} ({})", agg6_version.trim(), python_version.trim()),
+        },
+    ]
+}
+
+fn make_v4_tests(input_path: &str) -> Vec<TestDefinition> {
+    let mut all_tests = make_tests(input_path);
+
+    let iprange_version_raw = std::process::Command::new("iprange")
+        .arg("--version")
+        .stdout(Stdio::piped())
+        .spawn()
+        .expect("Unable to run iprange")
+        .wait_with_output()
+        .expect("Couldn't get iprange output")
+        .stdout;
+    let iprange_version = String::from_utf8_lossy(&iprange_version_raw);
+
+    all_tests.push(TestDefinition {
+        cmd: format!("iprange --optimize {}", input_path),
+        name: iprange_version.lines().nth(0).unwrap().into(),
+    });
+
+    all_tests
+}
+
+fn hyperfine_harness<S>(cmd: S) -> Result<TestResult, Box<dyn std::error::Error>>
+where
+    S: AsRef<OsStr>,
+{
+    let resultfile = NamedTempFile::new().expect("Unable to create tempfile");
+
+    let mut process = std::process::Command::new("hyperfine")
+        .arg("--export-json")
+        .arg(resultfile.path())
+        .arg("--min-runs")
+        .arg("2")
+        .arg("--")
+        .arg(cmd)
+        .stdout(Stdio::null())
+        .spawn()
+        .expect("unable to run command");
+    let rc = process.wait().expect("unable to wait on process");
+
+    let mut raw_result_buf = Vec::new();
+    resultfile
+        .as_file()
+        .read_to_end(&mut raw_result_buf)
+        .expect("Can't read results");
+    resultfile.close().unwrap();
+
+    let hf_result = json::parse(&String::from_utf8_lossy(&raw_result_buf))
+        .expect("Can't parse hyperfine json results");
+
+    let final_result = &hf_result["results"][0];
+
+    Ok((final_result.clone()).into())
+}
+
+fn plot_results(
+    results: &Vec<(TestDefinition, TestResult)>,
+    caption: &str,
+    outfile: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    // Second result is our baseline
+    let norm_numerator = results[1].1.mean;
+    let max_result = norm_numerator / results.iter().map(|x| x.1.mean).reduce(f64::min).unwrap();
+
+    let drawing = BitMapBackend::new(outfile, (640, 480)).into_drawing_area();
+    drawing.fill(&WHITE)?;
+    let mut chart = ChartBuilder::on(&drawing)
+        .x_label_area_size(40)
+        .y_label_area_size(40)
+        .caption(caption, ("Roboto", 24).into_font())
+        .build_cartesian_2d((0..results.len() - 1).into_segmented(), 0.0..max_result)?;
+
+    chart
+        .configure_mesh()
+        .y_desc("Speedup vs aggregate6")
+        .y_labels(5)
+        .y_label_formatter(&|x| std::fmt::format(format_args!("{:.0}", *x)))
+        .light_line_style(WHITE)
+        .bold_line_style(GREY)
+        .disable_x_mesh()
+        .x_label_style(("Roboto", 18).into_font())
+        .x_label_formatter(&|x| match x {
+            SegmentValue::Exact(val) => results[*val].0.name.clone(),
+            SegmentValue::CenterOf(val) => results[*val].0.name.clone(),
+            SegmentValue::Last => String::new(),
+        })
+        .draw()?;
+
+    chart.draw_series(
+        Histogram::vertical(&chart)
+            .style(BAR_COLOUR.filled())
+            .margin(10)
+            .data(
+                results
+                    .iter()
+                    .enumerate()
+                    .map(|(x, y)| (x, norm_numerator / y.1.mean)),
+            ),
+    )?;
+
+    chart.draw_series(PointSeries::of_element(
+        results
+            .iter()
+            .enumerate()
+            .map(|(x, y)| (SegmentValue::CenterOf(x), norm_numerator / y.1.mean)),
+        5,
+        ShapeStyle::from(&BLACK).filled(),
+        &|coord, size, style| {
+            EmptyElement::at(coord.clone())
+                + Text::new(
+                    format!("{:.1} x", coord.1),
+                    (0, 10),
+                    ("Roboto", 18).into_font().color(&WHITE),
+                )
+        },
+    ))?;
+
+    Ok(())
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let mut results: Vec<(TestDefinition, TestResult)> = Vec::new();
+    for test in make_tests("test-data/dfz_combined/input") {
+        results.push((test.clone(), hyperfine_harness(&test.cmd)?));
+    }
+    plot_results(
+        &results,
+        "IPv4 & IPv6 Full DFZ Prefixes",
+        "doc/perfcomp_all.png",
+    )?;
+
+    let mut results = Vec::new();
+    for test in make_v4_tests("test-data/dfz_v4/input") {
+        results.push((test.clone(), hyperfine_harness(&test.cmd)?));
+    }
+    plot_results(&results, "IPv4 Full DFZ Prefixes", "doc/perfcomp_v4.png")?;
+
+    Ok(())
+}