optimize load spec; update logging; parallelize output

Cargo.lock

@@ -2136,6 +2136,7 @@ dependencies = [
  "clap",
  "criterion",
  "flamegraph",
+ "rayon",
  "regex",
  "serde",
  "serde_json",

Cargo.toml

@@ -10,6 +10,7 @@ debug = true
 [dependencies]
 anyhow = "1"
 clap = { version = "4.5" }
+rayon = "1"
 regex = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

README.md

@@ -167,6 +167,5 @@ cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devi
 Using Demo:
 
 ``` bash
-process_specs           time:   [1.6096 ms 1.6205 ms 1.6327 ms]
+process_specs           time:   [1.2339 ms 1.2399 ms 1.2463 ms]
-from_spec               time:   [3.3230 ms 3.3397 ms 3.3575 ms]
 ```

benches/benchmark.rs

@@ -16,31 +16,14 @@ fn benchmark(c: &mut Criterion) {
     LOG_LEVEL.set(LogLevel::Warning).ok();
     let args = Args {
         devices: regex::Regex::new(".*").unwrap(),
-        classic: false,
         env: env(),
     };
     c.bench_function("process_specs", |b| {
         b.iter(|| {
-            DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap();
+            DeviceConfigBundle::process_specs(Specification::compile(&args).unwrap()).unwrap();
         });
     });
 }
 
-fn benchmark_classic(c: &mut Criterion) {
+criterion_group!(benches, benchmark);
-    LOG_LEVEL.set(LogLevel::Warning).ok();
-    let args = Args {
-        devices: regex::Regex::new(".*").unwrap(),
-        classic: true,
-        env: env(),
-    };
-    c.bench_function("from_spec", |b| {
-        b.iter(|| {
-            for spec in Specification::compile(&args) {
-                DeviceConfigBundle::from_spec(spec).unwrap();
-            }
-        });
-    });
-}
-
-criterion_group!(benches, benchmark, benchmark_classic);
 criterion_main!(benches);

src/cli.rs

@@ -15,7 +15,6 @@ const ENV_MSG: &str = r#"Environment:
 #[derive(Debug)]
 pub struct Args {
     pub devices: Regex,
-    pub classic: bool,
     pub env: EnvVars,
 }
 
@@ -39,7 +38,6 @@ impl Args {
 
         Args {
             devices,
-            classic: matches.get_flag("classic"),
             env: EnvVars::parse(),
         }
     }
@@ -79,14 +77,6 @@ impl Args {
                     .action(ArgAction::SetTrue)
                     .required(false),
             )
-            .arg(
-                Arg::new("classic")
-                    .short('c')
-                    .long("classic")
-                    .help("creates a new renderer for each device")
-                    .action(ArgAction::SetTrue)
-                    .required(false),
-            )
             .group(
                 ArgGroup::new("loglevel")
                     .args(&["debug", "verbose", "silent"])

src/error.rs

@@ -0,0 +1,33 @@
+pub trait ErrorExt {
+    fn chain_msg(&self) -> String;
+}
+
+impl<E: std::error::Error> ErrorExt for E {
+    fn chain_msg(&self) -> String {
+        let mut out = self.to_string();
+        let mut source = self.source();
+        while let Some(s) = source {
+            out.push_str(&format!("\n > {}", s));
+            source = s.source();
+        }
+        out
+    }
+}
+
+#[derive(Debug)]
+pub struct ContextError {
+    pub message: String,
+    pub source: Box<dyn std::error::Error>,
+}
+
+impl std::fmt::Display for ContextError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl std::error::Error for ContextError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(&*self.source)
+    }
+}

src/lib.rs

@@ -1,4 +1,5 @@
 pub mod cli;
+pub mod error;
 pub mod log;
 pub mod render;
 pub mod spec;

src/log.rs

@@ -25,7 +25,7 @@ impl std::fmt::Display for LogLevel {
 #[macro_export]
 macro_rules! info {
     ($($msg:expr),*) => {
-        if LogLevel::Info >= $crate::log_level() {
+        if $crate::log::LogLevel::Info >= $crate::log_level() {
             println!($($msg),*);
         }
     };
@@ -34,7 +34,7 @@ macro_rules! info {
 #[macro_export]
 macro_rules! verb {
     ($($msg:expr),*) => {
-        if LogLevel::Verbose >= $crate::log_level() {
+        if $crate::log::LogLevel::Verbose >= $crate::log_level() {
             println!($($msg),*);
         }
     };
@@ -43,7 +43,7 @@ macro_rules! verb {
 #[macro_export]
 macro_rules! dbug {
     ($($msg:expr),*) => {
-        if LogLevel::Debug >= $crate::log_level() {
+        if $crate::log::LogLevel::Debug >= $crate::log_level() {
             println!($($msg),*);
         }
     };
@@ -52,7 +52,7 @@ macro_rules! dbug {
 #[macro_export]
 macro_rules! warn {
     ($($msg:expr),*) => {
-        if LogLevel::Warning >= $crate::log_level() {
+        if $crate::log::LogLevel::Warning >= $crate::log_level() {
             eprintln!($($msg),*);
         }
     };
@@ -61,7 +61,7 @@ macro_rules! warn {
 #[macro_export]
 macro_rules! crit {
     ($($msg:expr),*) => {
-        if LogLevel::Critical >= $crate::log_level() {
+        if $crate::log::LogLevel::Critical >= $crate::log_level() {
             eprintln!($($msg),*);
         }
     };

src/main.rs

@@ -1,3 +1,4 @@
+use rayon::prelude::*;
 use skyforge::{Args, DeviceConfigBundle, Specification};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -5,15 +6,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     args.use_tmp();
 
-    if args.classic {
+    let specifications = Specification::compile(&args)?;
-        for spec in Specification::compile(&args) {
+    DeviceConfigBundle::process_specs(specifications)?
-            DeviceConfigBundle::from_spec(spec)?.output_artifacts();
+        .into_par_iter()
-        }
+        .for_each(|c| c.output_artifacts());
-    } else {
-        DeviceConfigBundle::process_specs(Specification::compile(&args))?
-            .into_iter()
-            .for_each(|c| c.output_artifacts());
-    }
 
     println!("Completed Successfully");
     Ok(())

src/render.rs

@@ -1,8 +1,8 @@
-use crate::log::LogLevel;
+use crate::error::{ContextError, ErrorExt};
 use crate::spec::Specification;
 use crate::tmpl;
 use crate::{crit, dbug, info, verb};
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::error::Error;
 use std::io::Write;
 use std::path::PathBuf;
@@ -27,79 +27,47 @@ pub struct Configuration {
 
 pub struct Renderer {
     pub engine: Tera,
-    pub from: PathBuf,
+    pub structure: tmpl::Structure,
 }
 
 impl DeviceConfigBundle {
-    pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn Error>> {
-        let structure =
-            tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| {
-                crit!("{}", e);
-                e
-            })?;
-        let mut renderer = Tera::default();
-        renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
-        Self::render_configs(spec, &structure, &renderer)
-    }
-
     pub fn process_specs(specs: Vec<Specification>) -> Result<Vec<Self>, Box<dyn Error>> {
         let mut failed = false;
-        let renderers: Vec<Renderer> = specs
+        let renderers: HashMap<PathBuf, Renderer> = specs
             .iter()
-            .map(|s| s.tmplpath.clone())
+            .map(|s| &s.tmplpath)
-            .collect::<HashSet<PathBuf>>()
+            .collect::<HashSet<&PathBuf>>()
             .into_iter()
             .filter_map(|p| {
-                let structure = tmpl::Structure::from_file(&p.join("structure.yaml"))
+                Self::get_renderer(&p)
                     .map_err(|e| {
-                        crit!("[!] {}", e);
+                        crit!("{}", e.chain_msg());
                         failed = true;
                     })
-                    .ok()?;
+                    .ok()
-                let mut t = Tera::default();
-                t.add_raw_templates(structure.load_template_data(&p))
-                    .map_err(|e| {
-                        crit!("While loading {}: {}", &p.display(), e.chain_msg());
-                        failed = true;
-                    })
-                    .ok()?;
-                Some(Renderer { engine: t, from: p })
             })
             .collect();
-
         if failed {
             return Err("Failed to load templates.".into());
         }
 
         info!("Rendering configs:");
-        let results: Vec<Result<Self, _>> = specs
+        let mut failed = false;
+        let successes: Vec<Self> = specs
             .into_iter()
-            .map(|s| {
+            .filter_map(|s| {
-                let renderer =
+                let renderer = renderers
-                    renderers
+                    .get(&s.tmplpath)
-                        .iter()
+                    .expect("renderer exists for all tmplaths");
-                        .find(|r| r.from == s.tmplpath)
+                Self::render_configs(s, &renderer.structure, &renderer.engine)
-                        .ok_or_else(|| {
-                            format!(
-                                "Unexpected. Missing Tera renderer: {}",
-                                s.tmplpath.display()
-                            )
-                        })?;
-                let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml"))
                     .map_err(|e| {
                         crit!("{}", e);
-                        e
+                        failed = true;
-                    })?;
+                    })
-                Self::render_configs(s, &structure, &renderer.engine)
+                    .ok()
             })
             .collect();
 
-        failed = results.iter().any(|r| r.is_err());
-        let successes: Vec<Self> = results
-            .into_iter()
-            .filter_map(|r| r.map_err(|e| crit!("{}", e)).ok())
-            .collect();
-
         if failed {
             return Err("One or more specs failed to process.".into());
         }
@@ -107,6 +75,23 @@ impl DeviceConfigBundle {
         Ok(successes)
     }
 
+    fn get_renderer(tmplpath: &PathBuf) -> Result<(PathBuf, Renderer), ContextError> {
+        let structure = Self::get_structure(&tmplpath)?;
+        let mut t = Tera::default();
+        t.add_raw_templates(structure.load_template_data(&tmplpath))
+            .map_err(|e| ContextError {
+                message: format!("While loading {}", tmplpath.display()),
+                source: Box::new(e),
+            })?;
+        Ok((
+            tmplpath.to_owned(),
+            Renderer {
+                engine: t,
+                structure,
+            },
+        ))
+    }
+
     fn render_configs(
         spec: Specification,
         structure: &tmpl::Structure,
@@ -125,7 +110,7 @@ impl DeviceConfigBundle {
                 dbug!(" |\t{template_name}.{variant}");
                 match renderer.render(template_name, &context) {
                     Ok(data) => cfgs.push(Configuration {
-                        name: String::from(template_name),
+                        name: template_name.to_owned(),
                         data,
                     }),
                     Err(e) => {
@@ -135,7 +120,7 @@ impl DeviceConfigBundle {
                 }
             }
             configs.push(ConfigVariant {
-                name: String::from(variant),
+                name: variant.to_owned(),
                 configs: cfgs,
             });
             context.remove(variant);
@@ -149,7 +134,15 @@ impl DeviceConfigBundle {
             hostname,
             configs,
             spec,
-            platform: structure.platform.clone(),
+            platform: structure.platform.to_owned(),
+        })
+    }
+
+    fn get_structure(tmplpath: &PathBuf) -> Result<tmpl::Structure, ContextError> {
+        let structure_path = &tmplpath.join("structure.yaml");
+        tmpl::Structure::from_file(structure_path).map_err(|e| ContextError {
+            message: format!("While loading {}", structure_path.display()),
+            source: e,
         })
     }
 
@@ -218,25 +211,10 @@ impl DeviceConfigBundle {
         verb!(" | {}", &compiled_spec_path.display());
         if let Err(e) = std::fs::write(
             &compiled_spec_path,
-            yaml_serde::to_string(&self.spec.compiled).unwrap(),
+            yaml_serde::to_string(&self.spec.compiled)
+                .expect("spec was successfully compiled and must be serializable"),
         ) {
             crit!("Failed to write {}: {}", compiled_spec_path.display(), e);
         }
     }
 }
-
-trait ErrorExt {
-    fn chain_msg(&self) -> String;
-}
-
-impl ErrorExt for tera::Error {
-    fn chain_msg(&self) -> String {
-        let mut out = self.to_string();
-        let mut source = self.source();
-        while let Some(s) = source {
-            out.push_str(&format!("\n > {}", s));
-            source = s.source();
-        }
-        out
-    }
-}

src/spec.rs

@@ -1,6 +1,8 @@
 use crate::cli::Args;
-use crate::{LogLevel, dbug, info, verb};
+use crate::error::*;
+use crate::{crit, dbug, info, verb};
 use serde_json::Value;
+use std::collections::HashMap;
 use std::path::PathBuf;
 
 #[derive(Debug)]
@@ -23,24 +25,92 @@ trait Pipe: Sized {
 impl<T> Pipe for T {}
 
 impl Specification {
-    pub fn build(
+    /// compiles specifications from regex and paths provided by Args
+    pub fn compile(args: &Args) -> Result<Vec<Specification>, Box<dyn std::error::Error>> {
+        let mut file_cache: HashMap<PathBuf, Value> = HashMap::new();
+        let mut failed = false;
+        let specifications: Vec<Specification> = Self::get_specs(&args.env.spec_path)
+            .pipe(|p| {
+                info!("Skyforge found {} renderable devices", p.len());
+                Self::filter_specs(&args.devices, p)
+            })
+            .tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices))
+            .into_iter()
+            .filter_map(|spec| {
+                verb!(" | {}", spec.display());
+                let common = Self::get_common(&spec);
+                let regional = Self::get_regional(&common);
+                Self::build_cached(
+                    Components {
+                        partitional: Self::get_partitional(&regional),
+                        regional,
+                        common,
+                        zonal: Self::get_zonal(&spec),
+                        device: spec.to_owned(),
+                    },
+                    args.env.out_path.to_owned(),
+                    args.env.tmpl_path.to_owned(),
+                    &mut file_cache,
+                )
+                .map_err(|e| {
+                    crit!(
+                        "Failed to build Specification for {}:\n{}",
+                        spec.display(),
+                        e.chain_msg()
+                    );
+                    failed = true;
+                })
+                .ok()
+            })
+            .collect();
+
+        if failed {
+            return Err("One or more specifications failed to compile.".into());
+        }
+
+        Ok(specifications)
+    }
+
+    fn build_cached(
         components: Components,
         outpath: PathBuf,
         tmplpath: PathBuf,
-    ) -> Result<Self, Box<dyn std::error::Error>> {
+        cache: &mut HashMap<PathBuf, Value>,
+    ) -> Result<Self, ContextError> {
         dbug!("{:#?}", components);
         let merged_map: serde_json::Map<String, Value> = [
-            &components.partitional,
+            components.partitional.as_ref(),
-            &components.regional,
+            Some(&components.regional),
-            &components.common,
+            Some(&components.common),
-            &components.zonal,
+            Some(&components.zonal),
-            &components.device,
+            Some(&components.device),
         ]
-        .iter()
-        .filter_map(|path| std::fs::read_to_string(path).ok())
-        .map(|content| yaml_serde::from_str::<Value>(&content))
-        .collect::<Result<Vec<Value>, _>>()?
         .into_iter()
+        .flatten()
+        .map(|path| -> Result<Option<Value>, ContextError> {
+            if let Some(cached) = cache.get(path) {
+                return Ok(Some(cached.clone()));
+            }
+            match std::fs::read_to_string(path) {
+                Ok(content) => {
+                    let parsed =
+                        yaml_serde::from_str::<Value>(&content).map_err(|e| ContextError {
+                            message: format!("Failed to parse {}", path.display()),
+                            source: Box::new(e),
+                        })?;
+                    cache.insert(path.clone(), parsed.clone());
+                    Ok(Some(parsed))
+                }
+                Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
+                Err(e) => Err(ContextError {
+                    message: format!("Failed to read {}", path.display()),
+                    source: Box::new(e),
+                }),
+            }
+        })
+        .collect::<Result<Vec<Option<Value>>, _>>()?
+        .into_iter()
+        .flatten()
         .filter_map(|v| v.as_object().cloned())
         .flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
         .flatten()
@@ -68,36 +138,7 @@ impl Specification {
         )
     }
 
-    /// compiles specifications from regex and paths provided by Args
+    fn get_partitional(regional: &PathBuf) -> Option<PathBuf> {
-    pub fn compile(args: &Args) -> Vec<Specification> {
-        Self::get_specs(&args.env.spec_path)
-            .pipe(|p| {
-                info!("Skyforge found {} renderable devices", p.len());
-                Self::filter_specs(&args.devices, p)
-            })
-            .tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices))
-            .into_iter()
-            .map(|spec| {
-                verb!(" | {}", spec.display());
-                let common = Self::get_common(&spec);
-                let regional = Self::get_regional(&common);
-                Self::build(
-                    Components {
-                        partitional: Self::get_partitional(&regional),
-                        regional,
-                        common,
-                        zonal: Self::get_zonal(&spec),
-                        device: spec,
-                    },
-                    args.env.out_path.clone(),
-                    args.env.tmpl_path.clone(),
-                )
-                .expect("failed to build Specification")
-            })
-            .collect()
-    }
-
-    fn get_partitional(regional: &PathBuf) -> PathBuf {
         let basedir = regional.ancestors().nth(3).unwrap().join("common");
         match yaml_serde::from_str::<yaml_serde::Value>(
             &std::fs::read_to_string(regional).unwrap_or_default(),
@@ -105,10 +146,10 @@ impl Specification {
         .unwrap_or_default()["regional"]["partition"]
             .as_str()
         {
-            Some(partition) => basedir.join(format!("{partition}.yaml")),
+            Some(partition) => Some(basedir.join(format!("{partition}.yaml"))),
             None => {
                 verb!("[?] {} missing regional.partition", regional.display());
-                basedir.join("default.yaml")
+                None
             }
         }
     }
@@ -159,20 +200,20 @@ impl Specification {
     }
 
     fn get_specs(spec_path: &PathBuf) -> Vec<PathBuf> {
-        let mut result = Vec::new();
         let root = spec_path.as_path();
-        for entry in walkdir::WalkDir::new(root)
+        walkdir::WalkDir::new(root)
             .into_iter()
-            .filter_map(|e| e.ok())
+            .filter_map(|e| {
-        {
+                let entry = e.ok()?;
-            if let Ok(path) = entry.path().strip_prefix(root) {
+                let path = entry.path().strip_prefix(root).ok()?;
-                let path_str = path.to_string_lossy().to_string();
+                let path_str = path.to_string_lossy();
                 if !path_str.contains("common") && path_str.ends_with("yaml") {
-                    result.push(root.join(path));
+                    Some(root.join(path))
+                } else {
+                    None
                 }
-            }
+            })
-        }
+            .collect()
-        result
     }
 }
 
@@ -188,7 +229,7 @@ impl std::fmt::Display for Specification {
 
 #[derive(Debug)]
 pub struct Components {
-    partitional: PathBuf,
+    partitional: Option<PathBuf>,
     regional: PathBuf,
     common: PathBuf,
     zonal: PathBuf,

src/tmpl.rs

@@ -1,4 +1,4 @@
-use crate::{LogLevel, crit, dbug};
+use crate::{crit, dbug};
 use serde::Deserialize;
 use std::error::Error;
 use std::path::PathBuf;