add rust performance runner

performance/README.md

@@ -0,0 +1,13 @@
+# Performance Testing
+This directory includes dbt project setups to test on and a test runner written in Rust which runs specific dbt commands on each of the projects. Orchestration is done via the GitHub Action workflow in `/.github/workflows/performance.yml`. The workflow is scheduled to run every night, but it can also be triggered manually.
+
+## Adding a new project
+Just make a new directory under projects. It will automatically be picked up by the tests.
+
+## Adding a new dbt command
+In `runner/src/main.rs` add a metric to the `metrics` Vec in the main function. The Github Action will handle recompilation.
+
+## Future work
+- add more projects to test different configurations that have been known bottlenecks
+- add more dbt commands to measure
+- consider storing these results so they can be graphed over time

performance/results/.gitignore

@@ -0,0 +1,5 @@
+# all files here are generated results
+*
+
+# except this one
+!.gitignore

performance/runner/.gitignore

@@ -0,0 +1,2 @@
+target/
+projects/*/logs

performance/runner/Cargo.lock

@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "runner"
+version = "0.1.0"

performance/runner/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "runner"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]

performance/runner/src/main.rs

@@ -0,0 +1,70 @@
+use std::{env, fs, io};
+use std::process::{Command, ExitStatus, exit};
+
+
+#[derive(Debug, Clone)]
+struct Metric<'a> {
+    name: &'a str,
+    prepare: &'a str,
+    cmd: &'a str,
+}
+
+impl Metric<'_> {
+    fn outfile(&self, project: &str) -> String {
+        [self.name, "_", project, ".json"].join("")
+    }
+}
+
+fn main() {
+    // TODO args lib
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 2 {
+        println!("please provide the git project directory root");
+        exit(1);
+    }
+    let git_directory = &args[1];
+
+    // to add a new metric to the test suite, simply define it in this list:
+    let metrics: Vec<Metric> = vec![
+        Metric { name:"parse", prepare: "rm -rf target/", cmd: "dbt parse --no-version-check" },
+    ];
+
+    // list out all projects
+    let project_entries = fs::read_dir([&git_directory, "performance/projects"].join("")).unwrap();
+
+    let results: Result<Vec<ExitStatus>, io::Error > = project_entries.map(|entry| {
+        metrics.clone().into_iter().map(|metric| {
+            let path = entry.as_ref().unwrap().path();
+            let project_name = &path.file_name().and_then(|x| x.to_str()).unwrap();
+
+            Command::new("hyperfine")
+                .current_dir(&path)
+                .arg("--prepare")
+                .arg(metric.prepare)
+                .arg(metric.cmd)
+                .arg("--export-json")
+                .arg([&git_directory, "/performance/results/", &metric.outfile(project_name)].join(""))
+                // this prevents capture dbt output. Noisy, but good for debugging when tests fail.
+                .arg("--show-output")
+                .status() // use spawn() here instead for more information
+        }).collect::<Vec<Result<ExitStatus, io::Error>>>()
+    }).flatten().collect();
+
+    // only exit with status code 0 if everything ran as expected
+    match results {
+        // if dispatch of any of the commands failed, panic with that error
+        Err(e) => panic!("{}", e),
+        Ok(statuses) => {
+            for status in statuses {
+                match status.code() {
+                    None => (),
+                    Some(0) => (),
+                    // if any child command exited with a non zero status code, exit with the same one here.
+                    Some(nonzero) => exit(nonzero),
+                }
+            }
+            // everything ran as expected
+            exit(0);
+        },
+    }
+}