diff --git a/Cargo.lock b/Cargo.lock
index c15043a..1bf054d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -965,6 +965,12 @@ dependencies = [
  "miniz_oxide",
 ]
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
 [[package]]
 name = "predicates"
 version = "3.0.4"
@@ -1014,6 +1020,36 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.4.1"
@@ -1076,6 +1112,8 @@ dependencies = [
  "json",
  "plotters",
  "predicates",
+ "rand",
+ "rand_chacha",
  "rstest",
  "tempfile",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 1f4adcd..caf6357 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,6 +28,8 @@ glob = "0.3.1"
 tempfile = "3.8.1"
 json = "0.12.4"
 plotters = "0.3.5"
+rand_chacha = "0.3.1"
+rand = "0.8.5"
 
 [[bin]]
 name = "rs-aggregate"
diff --git a/benches/perf.rs b/benches/perf.rs
index 1cff82c..68fbff5 100644
--- a/benches/perf.rs
+++ b/benches/perf.rs
@@ -1,3 +1,4 @@
+use ipnet::Ipv4Net;
 use json::JsonValue;
 use plotters::backend::BitMapBackend;
 use plotters::chart::ChartBuilder;
@@ -8,8 +9,10 @@ use plotters::series::{Histogram, PointSeries};
 use plotters::style::full_palette::GREY;
 use plotters::style::text_anchor::{HPos, Pos, VPos};
 use plotters::style::{Color, IntoFont, RGBColor, ShapeStyle, BLACK, WHITE};
+use rand::prelude::*;
+use rand_chacha::ChaChaRng;
 use std::ffi::OsStr;
-use std::io::Read;
+use std::io::{Read, Write};
 
 use std::process::Stdio;
 use tempfile::NamedTempFile;
@@ -102,6 +105,33 @@ fn make_v4_tests(input_path: &str) -> Vec<TestDefinition> {
     all_tests
 }
 
+// We don't really care if aggregation will actually be possible, but we'll only
+// generate prefixes with length 8->24 so some should be possible.
+fn make_random_prefix(rng: &mut impl Rng) -> Ipv4Net {
+    let prefix_len: u8 = rng.gen_range(8..25);
+    let netaddr: u32 = rng.gen_range(0..(1 << prefix_len)) << 32 - prefix_len;
+
+    Ipv4Net::new(netaddr.into(), prefix_len).unwrap()
+}
+
+// Generate 1024 random v4 addresses as a startup time test
+fn make_startup_tests() -> (NamedTempFile, Vec<TestDefinition>) {
+    let mut rng = ChaChaRng::seed_from_u64(0); // use a repeatable rng with custom seed
+    let addresses = std::iter::repeat_with(|| make_random_prefix(&mut rng)).take(1024);
+
+    let mut outfile = NamedTempFile::new().unwrap();
+    let mut outfile_f = outfile.as_file();
+    for addr in addresses {
+        outfile_f.write_fmt(format_args!("{}\n", addr)).unwrap();
+    }
+    outfile.flush().unwrap();
+
+    let outpath = outfile.path().as_os_str().to_string_lossy().to_string();
+
+    // outfile needs to live on so destructor doesn't delete it before we run the benches
+    (outfile, make_v4_tests(outpath.as_str()))
+}
+
 fn hyperfine_harness<S>(cmd: S) -> Result<TestResult, Box<dyn std::error::Error>>
 where
     S: AsRef<OsStr>,
@@ -113,6 +143,7 @@ where
         .arg(resultfile.path())
         .arg("--min-runs")
         .arg("10")
+        .arg("-N")
         .arg("--")
         .arg(&cmd)
         .stdout(Stdio::null())
@@ -214,21 +245,38 @@ fn plot_results(
 }
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let mut results: Vec<(TestDefinition, TestResult)> = Vec::new();
-    for test in make_tests("test-data/dfz_combined/input") {
-        results.push((test.clone(), hyperfine_harness(&test.cmd)?));
-    }
-    plot_results(
-        &results,
-        "IPv4 & IPv6 Full DFZ Prefixes",
+    run_and_plot(
+        make_tests("test-data/dfz_combined/input"),
         "doc/perfcomp_all.png",
+        "IPv4 & IPv6 Full DFZ",
+    )?;
+    run_and_plot(
+        make_v4_tests("test-data/dfz_v4/input"),
+        "doc/perfcomp_v4.png",
+        "IPv4 Full DFZ",
     )?;
 
-    let mut results = Vec::new();
-    for test in make_v4_tests("test-data/dfz_v4/input") {
-        results.push((test.clone(), hyperfine_harness(&test.cmd)?));
-    }
-    plot_results(&results, "IPv4 Full DFZ Prefixes", "doc/perfcomp_v4.png")?;
+    // Need to hold on to tmpfile so it doesn't get deleted before we can bench
+    let (_tmpfile, tests) = make_startup_tests();
+    run_and_plot(
+        tests,
+        "doc/perfcomp_startup.png",
+        "1024 Random IPv4 Prefixes",
+    )?;
 
     Ok(())
 }
+
+fn run_and_plot(
+    tests: Vec<TestDefinition>,
+    filename: &str,
+    caption: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let mut results: Vec<(TestDefinition, TestResult)> = Vec::new();
+    for test in tests {
+        println!("Running bench: {:?}", test);
+        results.push((test.clone(), hyperfine_harness(&test.cmd)?));
+    }
+    plot_results(&results, caption, filename)?;
+    Ok(())
+}