From d8134095aba72fd27283fbff55db03ade73de791 Mon Sep 17 00:00:00 2001 From: lost Date: Thu, 26 Feb 2026 02:00:11 -0700 Subject: [PATCH] feat dedup Tera renderer; update logging --- README.md | 57 +++++++----- benches/benchmark.rs | 48 ++++++---- src/cli.rs | 70 ++++++++++++++- src/log.rs | 1 - src/main.rs | 24 +++-- src/render.rs | 210 +++++++++++++++++++++++++++++++------------ src/tmpl.rs | 4 +- 7 files changed, 303 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 3966a24..3d1b62c 100644 --- a/README.md +++ b/README.md @@ -60,25 +60,24 @@ Environment: $ skyforge -d xyz1-ex-edge-r101 Skyforge found 9 renderable devices Matched 1 devices against 'xyz1-ex-edge-r101' -Processing templates for 'xyz1-ex-edge-r101' +Rendering configs: + | xyz1-ex-edge-r101 Writing Output: | ./out/xyz1-ex-edge-r101/all.live.junos | ./out/xyz1-ex-edge-r101/all.shifted.junos | ./out/xyz1-ex-edge-r101/all.init.junos +Completed Successfully ``` ### Verbose ``` bash -$ skyforge -d xyz1-ex-edge-r101 --verbose +$ skyforge -d xyz1-ex-edge-r101 -v Skyforge found 9 renderable devices Matched 1 devices against 'xyz1-ex-edge-r101' | ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml -Processing templates for 'xyz1-ex-edge-r101' - | ./tmpl/ex-edge-r/system.tera - | ./tmpl/ex-edge-r/interfaces.tera - | ./tmpl/ex-edge-r/protocols.tera - | ./tmpl/ex-edge-r/policy-options.tera +Rendering configs: + | xyz1-ex-edge-r101 Writing Output: | ./out/xyz1-ex-edge-r101/live/system.junos | ./out/xyz1-ex-edge-r101/live/interfaces.junos @@ -96,12 +95,16 @@ Writing Output: | ./out/xyz1-ex-edge-r101/init/policy-options.junos | ./out/xyz1-ex-edge-r101/all.init.junos | ./out/xyz1-ex-edge-r101/context.yaml +Completed Successfully ``` ### Debug ``` bash -$ skyforge -d xyz1-ex-edge-r101 --debug +$ skyforge -d xyz1-ex-edge-r101 -D +Using tmp dir: /tmp/skyforge-1772096013345 +Removing existing output path: ./out +Output symlinked: ./out -> /tmp/skyforge-1772096013345 Skyforge found 9 renderable devices Matched 1 devices against 'xyz1-ex-edge-r101' | ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml @@ -112,24 +115,24 @@ Components { zonal: "./spec/xyz/ex-edge-r/xyz1.common.yaml", device: "./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml", } -Processing templates for 'xyz1-ex-edge-r101' | ./tmpl/ex-edge-r/system.tera | ./tmpl/ex-edge-r/interfaces.tera | ./tmpl/ex-edge-r/protocols.tera | ./tmpl/ex-edge-r/policy-options.tera -Rendering templates for xyz1-ex-edge-r101 - | system.live - | interfaces.live - | protocols.live - | policy-options.live - | system.shifted - | interfaces.shifted - | protocols.shifted - | policy-options.shifted - | system.init - | interfaces.init - | protocols.init - | policy-options.init +Rendering configs: + | xyz1-ex-edge-r101 + | system.live + | interfaces.live + | protocols.live + | policy-options.live + | system.shifted + | interfaces.shifted + | protocols.shifted + | policy-options.shifted + | system.init + | interfaces.init + | protocols.init + | policy-options.init Writing Output: | ./out/xyz1-ex-edge-r101/live/system.junos | ./out/xyz1-ex-edge-r101/live/interfaces.junos @@ -147,6 +150,7 @@ Writing Output: | ./out/xyz1-ex-edge-r101/init/policy-options.junos | ./out/xyz1-ex-edge-r101/all.init.junos | ./out/xyz1-ex-edge-r101/context.yaml +Completed Successfully ``` ## Flamegraph @@ -157,3 +161,12 @@ Assume flamelens is installed, otherwise `cargo install flamelens` cd demo cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devices ".*" ``` + +## Benchmark + +Using Demo: + +``` bash +process_specs time: [1.6096 ms 1.6205 ms 1.6327 ms] +from_spec time: [3.3230 ms 3.3397 ms 3.3575 ms] +``` diff --git a/benches/benchmark.rs b/benches/benchmark.rs index 863f7dd..777d298 100644 --- a/benches/benchmark.rs +++ b/benches/benchmark.rs @@ -1,28 +1,46 @@ use criterion::{Criterion, criterion_group, criterion_main}; +use skyforge::log::{LOG_LEVEL, LogLevel}; use skyforge::{Args, DeviceConfigBundle, Specification}; use std::path::PathBuf; +fn env() -> skyforge::cli::EnvVars { + skyforge::cli::EnvVars { + spec_path: PathBuf::from("./demo/spec"), + tmpl_path: PathBuf::from("./demo/tmpl"), + out_path: PathBuf::from("./demo/out"), + log_path: PathBuf::from("./demo/log"), + } +} + fn benchmark(c: &mut Criterion) { + LOG_LEVEL.set(LogLevel::Warning).ok(); let args = Args { devices: regex::Regex::new(".*").unwrap(), - env: skyforge::cli::EnvVars { - spec_path: PathBuf::from("./demo/spec"), - tmpl_path: PathBuf::from("./demo/tmpl"), - out_path: PathBuf::from("./demo/out"), - log_path: PathBuf::from("./demo/log"), - }, + classic: false, + env: env(), }; - - c.bench_function("compile", |b| { + c.bench_function("process_specs", |b| { b.iter(|| { - for spec in Specification::compile(&args) { - DeviceConfigBundle::from_spec(spec) - .unwrap() - .output_artifacts(); - } - }) + DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap(); + }); }); } -criterion_group!(benches, benchmark); +fn benchmark_classic(c: &mut Criterion) { + LOG_LEVEL.set(LogLevel::Warning).ok(); + let args = Args { + devices: regex::Regex::new(".*").unwrap(), + classic: true, + env: env(), + }; + c.bench_function("from_spec", |b| { + b.iter(|| { + for spec in Specification::compile(&args) { + DeviceConfigBundle::from_spec(spec).unwrap(); + } + }); + }); +} + +criterion_group!(benches, benchmark, benchmark_classic); criterion_main!(benches); diff --git a/src/cli.rs b/src/cli.rs index 1a5b91d..eed7bd4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,4 +1,5 @@ use crate::log::{LOG_LEVEL, LogLevel}; +use crate::{crit, dbug}; use clap::{Arg, ArgAction, ArgGroup, Command}; use regex::Regex; use std::path::PathBuf; @@ -14,6 +15,7 @@ const ENV_MSG: &str = r#"Environment: #[derive(Debug)] pub struct Args { pub devices: Regex, + pub classic: bool, pub env: EnvVars, } @@ -30,11 +32,14 @@ impl Args { loglevel = LogLevel::Debug; } else if matches.get_flag("verbose") { loglevel = LogLevel::Verbose; + } else if matches.get_flag("silent") { + loglevel = LogLevel::Warning; } LOG_LEVEL.set(loglevel).ok(); Args { devices, + classic: matches.get_flag("classic"), env: EnvVars::parse(), } } @@ -52,6 +57,7 @@ impl Args { ) .arg( Arg::new("debug") + .short('D') .long("debug") .help("Print debug information") .action(ArgAction::SetTrue) @@ -65,16 +71,76 @@ impl Args { .action(ArgAction::SetTrue) .required(false), ) + .arg( + Arg::new("silent") + .short('s') + .long("silent") + .help("Sets loglevel to Warn") + .action(ArgAction::SetTrue) + .required(false), + ) + .arg( + Arg::new("classic") + .short('c') + .long("classic") + .help("creates a new renderer for each device") + .action(ArgAction::SetTrue) + .required(false), + ) .group( ArgGroup::new("loglevel") - .args(&["debug", "verbose"]) + .args(&["debug", "verbose", "silent"]) .required(false), ) .after_help(ENV_MSG) } + + pub fn use_tmp(self: &Self) { + let exec_id = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + + let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id)); + dbug!("Using tmp dir: {}", tmp_out.display()); + + if let Err(e) = std::fs::create_dir_all(&tmp_out) { + crit!("Failed to create tmp dir {}: {}", tmp_out.display(), e); + return; + } + + if self.env.out_path.exists() { + dbug!( + "Removing existing output path: {}", + self.env.out_path.display() + ); + if let Err(e) = std::fs::remove_file(&self.env.out_path) + .or_else(|_| std::fs::remove_dir_all(&self.env.out_path)) + { + crit!("Failed to remove {}: {}", self.env.out_path.display(), e); + return; + } + } + + if let Err(e) = std::os::unix::fs::symlink(&tmp_out, &self.env.out_path) { + crit!( + "Failed to symlink {} -> {}: {}", + tmp_out.display(), + self.env.out_path.display(), + e + ); + return; + } + + dbug!( + "Output symlinked: {} -> {}", + self.env.out_path.display(), + tmp_out.display() + ); + } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct EnvVars { pub spec_path: PathBuf, pub tmpl_path: PathBuf, diff --git a/src/log.rs b/src/log.rs index c844cfb..13ed4ab 100644 --- a/src/log.rs +++ b/src/log.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] use std::sync::OnceLock; #[derive(Debug, PartialEq, PartialOrd, Clone, Copy)] diff --git a/src/main.rs b/src/main.rs index 34350b8..28d5aa7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,22 +3,18 @@ use skyforge::{Args, DeviceConfigBundle, Specification}; fn main() -> Result<(), Box> { let args = Args::parse(); - let exec_id = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis(); + args.use_tmp(); - let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id)); - std::fs::create_dir_all(&tmp_out)?; - if args.env.out_path.exists() { - std::fs::remove_file(&args.env.out_path) - .or_else(|_| std::fs::remove_dir_all(&args.env.out_path))?; - } - std::os::unix::fs::symlink(&tmp_out, &args.env.out_path)?; - - for spec in Specification::compile(&args) { - DeviceConfigBundle::from_spec(spec)?.output_artifacts(); + if args.classic { + for spec in Specification::compile(&args) { + DeviceConfigBundle::from_spec(spec)?.output_artifacts(); + } + } else { + DeviceConfigBundle::process_specs(Specification::compile(&args))? + .into_iter() + .for_each(|c| c.output_artifacts()); } + println!("Completed Successfully"); Ok(()) } diff --git a/src/render.rs b/src/render.rs index 6711fbd..9fcbe21 100644 --- a/src/render.rs +++ b/src/render.rs @@ -2,8 +2,11 @@ use crate::log::LogLevel; use crate::spec::Specification; use crate::tmpl; use crate::{crit, dbug, info, verb}; +use std::collections::HashSet; use std::error::Error; use std::io::Write; +use std::path::PathBuf; +use tera::Tera; pub struct DeviceConfigBundle { pub hostname: String, @@ -22,20 +25,96 @@ pub struct Configuration { pub data: String, } +pub struct Renderer { + pub engine: Tera, + pub from: PathBuf, +} + impl DeviceConfigBundle { - pub fn from_spec(spec: Specification) -> Result> { - let mut context = Self::merge_context(&spec); - - let hostname = Self::get_hostname(&context, &spec); - info!("Processing templates for '{hostname}'"); - - let structure_path = spec.tmplpath.join("structure.yaml"); - let structure = tmpl::Structure::from_file(&structure_path) - .map_err(|e| format!("Failed to parse {:?}: {e}", structure_path))?; - - let mut renderer = tera::Tera::default(); + pub fn from_spec(spec: Specification) -> Result> { + let structure = + tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| { + crit!("{}", e); + e + })?; + let mut renderer = Tera::default(); renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?; - dbug!("Rendering templates for {hostname}"); + Self::render_configs(spec, &structure, &renderer) + } + + pub fn process_specs(specs: Vec) -> Result, Box> { + let mut failed = false; + let renderers: Vec = specs + .iter() + .map(|s| s.tmplpath.clone()) + .collect::>() + .into_iter() + .filter_map(|p| { + let structure = tmpl::Structure::from_file(&p.join("structure.yaml")) + .map_err(|e| { + crit!("[!] {}", e); + failed = true; + }) + .ok()?; + let mut t = Tera::default(); + t.add_raw_templates(structure.load_template_data(&p)) + .map_err(|e| { + crit!("While loading {}: {}", &p.display(), e.chain_msg()); + failed = true; + }) + .ok()?; + Some(Renderer { engine: t, from: p }) + }) + .collect(); + + if failed { + return Err("Failed to load templates.".into()); + } + + info!("Rendering configs:"); + let results: Vec> = specs + .into_iter() + .map(|s| { + let renderer = + renderers + .iter() + .find(|r| r.from == s.tmplpath) + .ok_or_else(|| { + format!( + "Unexpected. Missing Tera renderer: {}", + s.tmplpath.display() + ) + })?; + let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml")) + .map_err(|e| { + crit!("{}", e); + e + })?; + Self::render_configs(s, &structure, &renderer.engine) + }) + .collect(); + + failed = results.iter().any(|r| r.is_err()); + let successes: Vec = results + .into_iter() + .filter_map(|r| r.map_err(|e| crit!("{}", e)).ok()) + .collect(); + + if failed { + return Err("One or more specs failed to process.".into()); + } + + Ok(successes) + } + + fn render_configs( + spec: Specification, + structure: &tmpl::Structure, + renderer: &Tera, + ) -> Result> { + let mut context = Self::merge_context(&spec); + let hostname = spec.components.get_hostname(); + info!(" | {hostname}"); let mut failures = false; let mut configs = Vec::new(); @@ -43,37 +122,35 @@ impl DeviceConfigBundle { context.insert(variant, &true); let mut cfgs = Vec::new(); for template_name in &structure.files { - dbug!(" | {template_name}.{variant}"); + dbug!(" |\t{template_name}.{variant}"); match renderer.render(template_name, &context) { - Ok(data) => cfgs.push(Configuration::new(template_name, data)), + Ok(data) => cfgs.push(Configuration { + name: String::from(template_name), + data, + }), Err(e) => { - crit!("[!] {}", Error::source(&e).unwrap_or_else(|| &e)); - renderer.add_raw_template(template_name, "")?; + crit!("{}", e.chain_msg()); failures = true; } } } - configs.push(ConfigVariant::new(variant, cfgs)); + configs.push(ConfigVariant { + name: String::from(variant), + configs: cfgs, + }); context.remove(variant); } - match failures { - true => Err(Box::::from("Rendering failed.")), - false => Ok(Self { - hostname, - configs, - spec, - platform: structure.platform, - }), + if failures { + return Err(Box::::from("Rendering failed.")); } - } - fn get_hostname(context: &tera::Context, spec: &Specification) -> String { - context - .get("hostname") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .unwrap_or_else(|| spec.components.get_hostname()) + Ok(Self { + hostname, + configs, + spec, + platform: structure.platform.clone(), + }) } fn merge_context(spec: &Specification) -> tera::Context { @@ -86,57 +163,80 @@ impl DeviceConfigBundle { context } - pub fn output_artifacts(self: Self) { + pub fn output_artifacts(self) { info!("Writing Output:"); let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname); - std::fs::create_dir_all(&device_outpath).ok(); + if let Err(e) = std::fs::create_dir_all(&device_outpath) { + crit!( + "Failed to create output directory {}: {}", + device_outpath.display(), + e + ); + return; + } for variant_configs in self.configs { let out_path = device_outpath.join(&variant_configs.name); - std::fs::create_dir_all(&out_path).ok(); + if let Err(e) = std::fs::create_dir_all(&out_path) { + crit!( + "Failed to create variant directory {}: {}", + out_path.display(), + e + ); + continue; + } let merged_config_path = device_outpath.join(format!("all.{}.{}", variant_configs.name, self.platform)); - let mut all_file: std::fs::File = std::fs::OpenOptions::new() + let mut all_file = match std::fs::OpenOptions::new() .create(true) .append(true) .open(&merged_config_path) - .expect("unable to open"); + { + Ok(f) => f, + Err(e) => { + crit!("Failed to open {}: {}", merged_config_path.display(), e); + continue; + } + }; for config in &variant_configs.configs { let config_outpath = out_path.join(format!("{}.{}", config.name, self.platform)); verb!(" | {}", &config_outpath.display()); - std::fs::write(&config_outpath, &config.data).ok(); - all_file.write_all(&config.data.as_bytes()).ok(); + if let Err(e) = std::fs::write(&config_outpath, &config.data) { + crit!("Failed to write {}: {}", config_outpath.display(), e); + } + if let Err(e) = all_file.write_all(config.data.as_bytes()) { + crit!("Failed to write to {}: {}", merged_config_path.display(), e); + } } info!(" | {} ", &merged_config_path.display()); } let compiled_spec_path = device_outpath.join("context.yaml"); verb!(" | {}", &compiled_spec_path.display()); - std::fs::write( + if let Err(e) = std::fs::write( &compiled_spec_path, yaml_serde::to_string(&self.spec.compiled).unwrap(), - ) - .ok(); - } -} - -impl ConfigVariant { - pub fn new(name: &str, configs: Vec) -> Self { - Self { - name: String::from(name), - configs, + ) { + crit!("Failed to write {}: {}", compiled_spec_path.display(), e); } } } -impl Configuration { - pub fn new(name: &str, data: String) -> Self { - Self { - name: String::from(name), - data, +trait ErrorExt { + fn chain_msg(&self) -> String; +} + +impl ErrorExt for tera::Error { + fn chain_msg(&self) -> String { + let mut out = self.to_string(); + let mut source = self.source(); + while let Some(s) = source { + out.push_str(&format!("\n > {}", s)); + source = s.source(); } + out } } diff --git a/src/tmpl.rs b/src/tmpl.rs index 31ee0fb..459907c 100644 --- a/src/tmpl.rs +++ b/src/tmpl.rs @@ -1,4 +1,4 @@ -use crate::{LogLevel, crit, verb}; +use crate::{LogLevel, crit, dbug}; use serde::Deserialize; use std::error::Error; use std::path::PathBuf; @@ -24,7 +24,7 @@ impl Structure { let template = tmpl_dir.join(format!("{}.tera", name)); match std::fs::read_to_string(&template) { Ok(content) => { - verb!(" | {}", template.display()); + dbug!(" | {}", template.display()); Some((name.clone(), content)) } Err(e) => {