From 92b36e09b7038d59fefeb34b33293fef7bedeb53 Mon Sep 17 00:00:00 2001 From: lost Date: Fri, 27 Feb 2026 05:00:28 -0700 Subject: [PATCH] optimize load spec; update logging; parallelize output --- Cargo.lock | 1 + Cargo.toml | 1 + README.md | 3 +- benches/benchmark.rs | 21 +----- src/cli.rs | 10 --- src/error.rs | 33 ++++++++++ src/lib.rs | 1 + src/log.rs | 10 +-- src/main.rs | 14 ++-- src/render.rs | 120 ++++++++++++++-------------------- src/spec.rs | 151 +++++++++++++++++++++++++++---------------- src/tmpl.rs | 2 +- 12 files changed, 195 insertions(+), 172 deletions(-) create mode 100644 src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 95abb88..c379e77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2136,6 +2136,7 @@ dependencies = [ "clap", "criterion", "flamegraph", + "rayon", "regex", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index cb6f999..df1b1ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ debug = true [dependencies] anyhow = "1" clap = { version = "4.5" } +rayon = "1" regex = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/README.md b/README.md index 3d1b62c..8b2d51f 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,5 @@ cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devi Using Demo: ``` bash -process_specs time: [1.6096 ms 1.6205 ms 1.6327 ms] -from_spec time: [3.3230 ms 3.3397 ms 3.3575 ms] +process_specs time: [1.2339 ms 1.2399 ms 1.2463 ms] ``` diff --git a/benches/benchmark.rs b/benches/benchmark.rs index 777d298..3aafc3a 100644 --- a/benches/benchmark.rs +++ b/benches/benchmark.rs @@ -16,31 +16,14 @@ fn benchmark(c: &mut Criterion) { LOG_LEVEL.set(LogLevel::Warning).ok(); let args = Args { devices: regex::Regex::new(".*").unwrap(), - classic: false, env: env(), }; c.bench_function("process_specs", |b| { b.iter(|| { - DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap(); + DeviceConfigBundle::process_specs(Specification::compile(&args).unwrap()).unwrap(); }); }); } -fn benchmark_classic(c: &mut Criterion) { - LOG_LEVEL.set(LogLevel::Warning).ok(); - let args = Args { - devices: regex::Regex::new(".*").unwrap(), - classic: true, - env: env(), - }; - c.bench_function("from_spec", |b| { - b.iter(|| { - for spec in Specification::compile(&args) { - DeviceConfigBundle::from_spec(spec).unwrap(); - } - }); - }); -} - -criterion_group!(benches, benchmark, benchmark_classic); +criterion_group!(benches, benchmark); criterion_main!(benches); diff --git a/src/cli.rs b/src/cli.rs index eed7bd4..174f6b3 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -15,7 +15,6 @@ const ENV_MSG: &str = r#"Environment: #[derive(Debug)] pub struct Args { pub devices: Regex, - pub classic: bool, pub env: EnvVars, } @@ -39,7 +38,6 @@ impl Args { Args { devices, - classic: matches.get_flag("classic"), env: EnvVars::parse(), } } @@ -79,14 +77,6 @@ impl Args { .action(ArgAction::SetTrue) .required(false), ) - .arg( - Arg::new("classic") - .short('c') - .long("classic") - .help("creates a new renderer for each device") - .action(ArgAction::SetTrue) - .required(false), - ) .group( ArgGroup::new("loglevel") .args(&["debug", "verbose", "silent"]) diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..263eae0 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,33 @@ +pub trait ErrorExt { + fn chain_msg(&self) -> String; +} + +impl ErrorExt for E { + fn chain_msg(&self) -> String { + let mut out = self.to_string(); + let mut source = self.source(); + while let Some(s) = source { + out.push_str(&format!("\n > {}", s)); + source = s.source(); + } + out + } +} + +#[derive(Debug)] +pub struct ContextError { + pub message: String, + pub source: Box, +} + +impl std::fmt::Display for ContextError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ContextError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&*self.source) + } +} diff --git a/src/lib.rs b/src/lib.rs index 8cdac83..94c0827 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ pub mod cli; +pub mod error; pub mod log; pub mod render; pub mod spec; diff --git a/src/log.rs b/src/log.rs index 13ed4ab..b279094 100644 --- a/src/log.rs +++ b/src/log.rs @@ -25,7 +25,7 @@ impl std::fmt::Display for LogLevel { #[macro_export] macro_rules! info { ($($msg:expr),*) => { - if LogLevel::Info >= $crate::log_level() { + if $crate::log::LogLevel::Info >= $crate::log_level() { println!($($msg),*); } }; @@ -34,7 +34,7 @@ macro_rules! info { #[macro_export] macro_rules! verb { ($($msg:expr),*) => { - if LogLevel::Verbose >= $crate::log_level() { + if $crate::log::LogLevel::Verbose >= $crate::log_level() { println!($($msg),*); } }; @@ -43,7 +43,7 @@ macro_rules! verb { #[macro_export] macro_rules! dbug { ($($msg:expr),*) => { - if LogLevel::Debug >= $crate::log_level() { + if $crate::log::LogLevel::Debug >= $crate::log_level() { println!($($msg),*); } }; @@ -52,7 +52,7 @@ macro_rules! dbug { #[macro_export] macro_rules! warn { ($($msg:expr),*) => { - if LogLevel::Warning >= $crate::log_level() { + if $crate::log::LogLevel::Warning >= $crate::log_level() { eprintln!($($msg),*); } }; @@ -61,7 +61,7 @@ macro_rules! warn { #[macro_export] macro_rules! crit { ($($msg:expr),*) => { - if LogLevel::Critical >= $crate::log_level() { + if $crate::log::LogLevel::Critical >= $crate::log_level() { eprintln!($($msg),*); } }; diff --git a/src/main.rs b/src/main.rs index 28d5aa7..baca105 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use rayon::prelude::*; use skyforge::{Args, DeviceConfigBundle, Specification}; fn main() -> Result<(), Box> { @@ -5,15 +6,10 @@ fn main() -> Result<(), Box> { args.use_tmp(); - if args.classic { - for spec in Specification::compile(&args) { - DeviceConfigBundle::from_spec(spec)?.output_artifacts(); - } - } else { - DeviceConfigBundle::process_specs(Specification::compile(&args))? - .into_iter() - .for_each(|c| c.output_artifacts()); - } + let specifications = Specification::compile(&args)?; + DeviceConfigBundle::process_specs(specifications)? + .into_par_iter() + .for_each(|c| c.output_artifacts()); println!("Completed Successfully"); Ok(()) diff --git a/src/render.rs b/src/render.rs index 9fcbe21..9d20800 100644 --- a/src/render.rs +++ b/src/render.rs @@ -1,8 +1,8 @@ -use crate::log::LogLevel; +use crate::error::{ContextError, ErrorExt}; use crate::spec::Specification; use crate::tmpl; use crate::{crit, dbug, info, verb}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::error::Error; use std::io::Write; use std::path::PathBuf; @@ -27,79 +27,47 @@ pub struct Configuration { pub struct Renderer { pub engine: Tera, - pub from: PathBuf, + pub structure: tmpl::Structure, } impl DeviceConfigBundle { - pub fn from_spec(spec: Specification) -> Result> { - let structure = - tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| { - crit!("{}", e); - e - })?; - let mut renderer = Tera::default(); - renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?; - Self::render_configs(spec, &structure, &renderer) - } - pub fn process_specs(specs: Vec) -> Result, Box> { let mut failed = false; - let renderers: Vec = specs + let renderers: HashMap = specs .iter() - .map(|s| s.tmplpath.clone()) - .collect::>() + .map(|s| &s.tmplpath) + .collect::>() .into_iter() .filter_map(|p| { - let structure = tmpl::Structure::from_file(&p.join("structure.yaml")) + Self::get_renderer(&p) .map_err(|e| { - crit!("[!] {}", e); + crit!("{}", e.chain_msg()); failed = true; }) - .ok()?; - let mut t = Tera::default(); - t.add_raw_templates(structure.load_template_data(&p)) - .map_err(|e| { - crit!("While loading {}: {}", &p.display(), e.chain_msg()); - failed = true; - }) - .ok()?; - Some(Renderer { engine: t, from: p }) + .ok() }) .collect(); - if failed { return Err("Failed to load templates.".into()); } info!("Rendering configs:"); - let results: Vec> = specs + let mut failed = false; + let successes: Vec = specs .into_iter() - .map(|s| { - let renderer = - renderers - .iter() - .find(|r| r.from == s.tmplpath) - .ok_or_else(|| { - format!( - "Unexpected. Missing Tera renderer: {}", - s.tmplpath.display() - ) - })?; - let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml")) + .filter_map(|s| { + let renderer = renderers + .get(&s.tmplpath) + .expect("renderer exists for all tmplaths"); + Self::render_configs(s, &renderer.structure, &renderer.engine) .map_err(|e| { crit!("{}", e); - e - })?; - Self::render_configs(s, &structure, &renderer.engine) + failed = true; + }) + .ok() }) .collect(); - failed = results.iter().any(|r| r.is_err()); - let successes: Vec = results - .into_iter() - .filter_map(|r| r.map_err(|e| crit!("{}", e)).ok()) - .collect(); - if failed { return Err("One or more specs failed to process.".into()); } @@ -107,6 +75,23 @@ impl DeviceConfigBundle { Ok(successes) } + fn get_renderer(tmplpath: &PathBuf) -> Result<(PathBuf, Renderer), ContextError> { + let structure = Self::get_structure(&tmplpath)?; + let mut t = Tera::default(); + t.add_raw_templates(structure.load_template_data(&tmplpath)) + .map_err(|e| ContextError { + message: format!("While loading {}", tmplpath.display()), + source: Box::new(e), + })?; + Ok(( + tmplpath.to_owned(), + Renderer { + engine: t, + structure, + }, + )) + } + fn render_configs( spec: Specification, structure: &tmpl::Structure, @@ -125,7 +110,7 @@ impl DeviceConfigBundle { dbug!(" |\t{template_name}.{variant}"); match renderer.render(template_name, &context) { Ok(data) => cfgs.push(Configuration { - name: String::from(template_name), + name: template_name.to_owned(), data, }), Err(e) => { @@ -135,7 +120,7 @@ impl DeviceConfigBundle { } } configs.push(ConfigVariant { - name: String::from(variant), + name: variant.to_owned(), configs: cfgs, }); context.remove(variant); @@ -149,7 +134,15 @@ impl DeviceConfigBundle { hostname, configs, spec, - platform: structure.platform.clone(), + platform: structure.platform.to_owned(), + }) + } + + fn get_structure(tmplpath: &PathBuf) -> Result { + let structure_path = &tmplpath.join("structure.yaml"); + tmpl::Structure::from_file(structure_path).map_err(|e| ContextError { + message: format!("While loading {}", structure_path.display()), + source: e, }) } @@ -218,25 +211,10 @@ impl DeviceConfigBundle { verb!(" | {}", &compiled_spec_path.display()); if let Err(e) = std::fs::write( &compiled_spec_path, - yaml_serde::to_string(&self.spec.compiled).unwrap(), + yaml_serde::to_string(&self.spec.compiled) + .expect("spec was successfully compiled and must be serializable"), ) { crit!("Failed to write {}: {}", compiled_spec_path.display(), e); } } } - -trait ErrorExt { - fn chain_msg(&self) -> String; -} - -impl ErrorExt for tera::Error { - fn chain_msg(&self) -> String { - let mut out = self.to_string(); - let mut source = self.source(); - while let Some(s) = source { - out.push_str(&format!("\n > {}", s)); - source = s.source(); - } - out - } -} diff --git a/src/spec.rs b/src/spec.rs index 5dc698b..e514c7e 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -1,6 +1,8 @@ use crate::cli::Args; -use crate::{LogLevel, dbug, info, verb}; +use crate::error::*; +use crate::{crit, dbug, info, verb}; use serde_json::Value; +use std::collections::HashMap; use std::path::PathBuf; #[derive(Debug)] @@ -23,24 +25,92 @@ trait Pipe: Sized { impl Pipe for T {} impl Specification { - pub fn build( + /// compiles specifications from regex and paths provided by Args + pub fn compile(args: &Args) -> Result, Box> { + let mut file_cache: HashMap = HashMap::new(); + let mut failed = false; + let specifications: Vec = Self::get_specs(&args.env.spec_path) + .pipe(|p| { + info!("Skyforge found {} renderable devices", p.len()); + Self::filter_specs(&args.devices, p) + }) + .tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices)) + .into_iter() + .filter_map(|spec| { + verb!(" | {}", spec.display()); + let common = Self::get_common(&spec); + let regional = Self::get_regional(&common); + Self::build_cached( + Components { + partitional: Self::get_partitional(®ional), + regional, + common, + zonal: Self::get_zonal(&spec), + device: spec.to_owned(), + }, + args.env.out_path.to_owned(), + args.env.tmpl_path.to_owned(), + &mut file_cache, + ) + .map_err(|e| { + crit!( + "Failed to build Specification for {}:\n{}", + spec.display(), + e.chain_msg() + ); + failed = true; + }) + .ok() + }) + .collect(); + + if failed { + return Err("One or more specifications failed to compile.".into()); + } + + Ok(specifications) + } + + fn build_cached( components: Components, outpath: PathBuf, tmplpath: PathBuf, - ) -> Result> { + cache: &mut HashMap, + ) -> Result { dbug!("{:#?}", components); let merged_map: serde_json::Map = [ - &components.partitional, - &components.regional, - &components.common, - &components.zonal, - &components.device, + components.partitional.as_ref(), + Some(&components.regional), + Some(&components.common), + Some(&components.zonal), + Some(&components.device), ] - .iter() - .filter_map(|path| std::fs::read_to_string(path).ok()) - .map(|content| yaml_serde::from_str::(&content)) - .collect::, _>>()? .into_iter() + .flatten() + .map(|path| -> Result, ContextError> { + if let Some(cached) = cache.get(path) { + return Ok(Some(cached.clone())); + } + match std::fs::read_to_string(path) { + Ok(content) => { + let parsed = + yaml_serde::from_str::(&content).map_err(|e| ContextError { + message: format!("Failed to parse {}", path.display()), + source: Box::new(e), + })?; + cache.insert(path.clone(), parsed.clone()); + Ok(Some(parsed)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(ContextError { + message: format!("Failed to read {}", path.display()), + source: Box::new(e), + }), + } + }) + .collect::>, _>>()? + .into_iter() + .flatten() .filter_map(|v| v.as_object().cloned()) .flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned())) .flatten() @@ -68,36 +138,7 @@ impl Specification { ) } - /// compiles specifications from regex and paths provided by Args - pub fn compile(args: &Args) -> Vec { - Self::get_specs(&args.env.spec_path) - .pipe(|p| { - info!("Skyforge found {} renderable devices", p.len()); - Self::filter_specs(&args.devices, p) - }) - .tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices)) - .into_iter() - .map(|spec| { - verb!(" | {}", spec.display()); - let common = Self::get_common(&spec); - let regional = Self::get_regional(&common); - Self::build( - Components { - partitional: Self::get_partitional(®ional), - regional, - common, - zonal: Self::get_zonal(&spec), - device: spec, - }, - args.env.out_path.clone(), - args.env.tmpl_path.clone(), - ) - .expect("failed to build Specification") - }) - .collect() - } - - fn get_partitional(regional: &PathBuf) -> PathBuf { + fn get_partitional(regional: &PathBuf) -> Option { let basedir = regional.ancestors().nth(3).unwrap().join("common"); match yaml_serde::from_str::( &std::fs::read_to_string(regional).unwrap_or_default(), @@ -105,10 +146,10 @@ impl Specification { .unwrap_or_default()["regional"]["partition"] .as_str() { - Some(partition) => basedir.join(format!("{partition}.yaml")), + Some(partition) => Some(basedir.join(format!("{partition}.yaml"))), None => { verb!("[?] {} missing regional.partition", regional.display()); - basedir.join("default.yaml") + None } } } @@ -159,20 +200,20 @@ impl Specification { } fn get_specs(spec_path: &PathBuf) -> Vec { - let mut result = Vec::new(); let root = spec_path.as_path(); - for entry in walkdir::WalkDir::new(root) + walkdir::WalkDir::new(root) .into_iter() - .filter_map(|e| e.ok()) - { - if let Ok(path) = entry.path().strip_prefix(root) { - let path_str = path.to_string_lossy().to_string(); + .filter_map(|e| { + let entry = e.ok()?; + let path = entry.path().strip_prefix(root).ok()?; + let path_str = path.to_string_lossy(); if !path_str.contains("common") && path_str.ends_with("yaml") { - result.push(root.join(path)); + Some(root.join(path)) + } else { + None } - } - } - result + }) + .collect() } } @@ -188,7 +229,7 @@ impl std::fmt::Display for Specification { #[derive(Debug)] pub struct Components { - partitional: PathBuf, + partitional: Option, regional: PathBuf, common: PathBuf, zonal: PathBuf, diff --git a/src/tmpl.rs b/src/tmpl.rs index 459907c..3cabec4 100644 --- a/src/tmpl.rs +++ b/src/tmpl.rs @@ -1,4 +1,4 @@ -use crate::{LogLevel, crit, dbug}; +use crate::{crit, dbug}; use serde::Deserialize; use std::error::Error; use std::path::PathBuf;