feat dedup Tera renderer; update logging

README.md

@@ -60,25 +60,24 @@ Environment:
 $ skyforge -d xyz1-ex-edge-r101
 Skyforge found 9 renderable devices
 Matched 1 devices against 'xyz1-ex-edge-r101'
-Processing templates for 'xyz1-ex-edge-r101'
+Rendering configs:
+ | xyz1-ex-edge-r101
 Writing Output:
  | ./out/xyz1-ex-edge-r101/all.live.junos
  | ./out/xyz1-ex-edge-r101/all.shifted.junos
  | ./out/xyz1-ex-edge-r101/all.init.junos
+Completed Successfully
 ```
 
 ### Verbose
 
 ``` bash
-$ skyforge -d xyz1-ex-edge-r101 --verbose
+$ skyforge -d xyz1-ex-edge-r101 -v
 Skyforge found 9 renderable devices
 Matched 1 devices against 'xyz1-ex-edge-r101'
  | ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
-Processing templates for 'xyz1-ex-edge-r101'
+Rendering configs:
- | ./tmpl/ex-edge-r/system.tera
+ | xyz1-ex-edge-r101
- | ./tmpl/ex-edge-r/interfaces.tera
- | ./tmpl/ex-edge-r/protocols.tera
- | ./tmpl/ex-edge-r/policy-options.tera
 Writing Output:
  | ./out/xyz1-ex-edge-r101/live/system.junos
  | ./out/xyz1-ex-edge-r101/live/interfaces.junos
@@ -96,12 +95,16 @@ Writing Output:
  | ./out/xyz1-ex-edge-r101/init/policy-options.junos
  | ./out/xyz1-ex-edge-r101/all.init.junos
  | ./out/xyz1-ex-edge-r101/context.yaml
+Completed Successfully
 ```
 
 ### Debug
 
 ``` bash
-$ skyforge -d xyz1-ex-edge-r101 --debug
+$ skyforge -d xyz1-ex-edge-r101 -D
+Using tmp dir: /tmp/skyforge-1772096013345
+Removing existing output path: ./out
+Output symlinked: ./out -> /tmp/skyforge-1772096013345
 Skyforge found 9 renderable devices
 Matched 1 devices against 'xyz1-ex-edge-r101'
  | ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
@@ -112,12 +115,12 @@ Components {
     zonal: "./spec/xyz/ex-edge-r/xyz1.common.yaml",
     device: "./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml",
 }
-Processing templates for 'xyz1-ex-edge-r101'
  | ./tmpl/ex-edge-r/system.tera
  | ./tmpl/ex-edge-r/interfaces.tera
  | ./tmpl/ex-edge-r/protocols.tera
  | ./tmpl/ex-edge-r/policy-options.tera
-Rendering templates for xyz1-ex-edge-r101
+Rendering configs:
+ | xyz1-ex-edge-r101
  |	system.live
  |	interfaces.live
  |	protocols.live
@@ -147,6 +150,7 @@ Writing Output:
  | ./out/xyz1-ex-edge-r101/init/policy-options.junos
  | ./out/xyz1-ex-edge-r101/all.init.junos
  | ./out/xyz1-ex-edge-r101/context.yaml
+Completed Successfully
 ```
 
 ## Flamegraph
@@ -157,3 +161,12 @@ Assume flamelens is installed, otherwise `cargo install flamelens`
 cd demo
 cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devices ".*"
 ```
+
+## Benchmark
+
+Using Demo:
+
+``` bash
+process_specs           time:   [1.6096 ms 1.6205 ms 1.6327 ms]
+from_spec               time:   [3.3230 ms 3.3397 ms 3.3575 ms]
+```

benches/benchmark.rs

@@ -1,28 +1,46 @@
 use criterion::{Criterion, criterion_group, criterion_main};
+use skyforge::log::{LOG_LEVEL, LogLevel};
 use skyforge::{Args, DeviceConfigBundle, Specification};
 use std::path::PathBuf;
 
-fn benchmark(c: &mut Criterion) {
+fn env() -> skyforge::cli::EnvVars {
-    let args = Args {
+    skyforge::cli::EnvVars {
-        devices: regex::Regex::new(".*").unwrap(),
-        env: skyforge::cli::EnvVars {
         spec_path: PathBuf::from("./demo/spec"),
         tmpl_path: PathBuf::from("./demo/tmpl"),
         out_path: PathBuf::from("./demo/out"),
         log_path: PathBuf::from("./demo/log"),
-        },
-    };
-
-    c.bench_function("compile", |b| {
-        b.iter(|| {
-            for spec in Specification::compile(&args) {
-                DeviceConfigBundle::from_spec(spec)
-                    .unwrap()
-                    .output_artifacts();
     }
-        })
+}
+
+fn benchmark(c: &mut Criterion) {
+    LOG_LEVEL.set(LogLevel::Warning).ok();
+    let args = Args {
+        devices: regex::Regex::new(".*").unwrap(),
+        classic: false,
+        env: env(),
+    };
+    c.bench_function("process_specs", |b| {
+        b.iter(|| {
+            DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap();
+        });
     });
 }
 
-criterion_group!(benches, benchmark);
+fn benchmark_classic(c: &mut Criterion) {
+    LOG_LEVEL.set(LogLevel::Warning).ok();
+    let args = Args {
+        devices: regex::Regex::new(".*").unwrap(),
+        classic: true,
+        env: env(),
+    };
+    c.bench_function("from_spec", |b| {
+        b.iter(|| {
+            for spec in Specification::compile(&args) {
+                DeviceConfigBundle::from_spec(spec).unwrap();
+            }
+        });
+    });
+}
+
+criterion_group!(benches, benchmark, benchmark_classic);
 criterion_main!(benches);

src/cli.rs

@@ -1,4 +1,5 @@
 use crate::log::{LOG_LEVEL, LogLevel};
+use crate::{crit, dbug};
 use clap::{Arg, ArgAction, ArgGroup, Command};
 use regex::Regex;
 use std::path::PathBuf;
@@ -14,6 +15,7 @@ const ENV_MSG: &str = r#"Environment:
 #[derive(Debug)]
 pub struct Args {
     pub devices: Regex,
+    pub classic: bool,
     pub env: EnvVars,
 }
 
@@ -30,11 +32,14 @@ impl Args {
             loglevel = LogLevel::Debug;
         } else if matches.get_flag("verbose") {
             loglevel = LogLevel::Verbose;
+        } else if matches.get_flag("silent") {
+            loglevel = LogLevel::Warning;
         }
         LOG_LEVEL.set(loglevel).ok();
 
         Args {
             devices,
+            classic: matches.get_flag("classic"),
             env: EnvVars::parse(),
         }
     }
@@ -52,6 +57,7 @@ impl Args {
             )
             .arg(
                 Arg::new("debug")
+                    .short('D')
                     .long("debug")
                     .help("Print debug information")
                     .action(ArgAction::SetTrue)
@@ -65,16 +71,76 @@ impl Args {
                     .action(ArgAction::SetTrue)
                     .required(false),
             )
+            .arg(
+                Arg::new("silent")
+                    .short('s')
+                    .long("silent")
+                    .help("Sets loglevel to Warn")
+                    .action(ArgAction::SetTrue)
+                    .required(false),
+            )
+            .arg(
+                Arg::new("classic")
+                    .short('c')
+                    .long("classic")
+                    .help("creates a new renderer for each device")
+                    .action(ArgAction::SetTrue)
+                    .required(false),
+            )
             .group(
                 ArgGroup::new("loglevel")
-                    .args(&["debug", "verbose"])
+                    .args(&["debug", "verbose", "silent"])
                     .required(false),
             )
             .after_help(ENV_MSG)
     }
+
+    pub fn use_tmp(self: &Self) {
+        let exec_id = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis();
+
+        let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
+        dbug!("Using tmp dir: {}", tmp_out.display());
+
+        if let Err(e) = std::fs::create_dir_all(&tmp_out) {
+            crit!("Failed to create tmp dir {}: {}", tmp_out.display(), e);
+            return;
         }
 
-#[derive(Debug)]
+        if self.env.out_path.exists() {
+            dbug!(
+                "Removing existing output path: {}",
+                self.env.out_path.display()
+            );
+            if let Err(e) = std::fs::remove_file(&self.env.out_path)
+                .or_else(|_| std::fs::remove_dir_all(&self.env.out_path))
+            {
+                crit!("Failed to remove {}: {}", self.env.out_path.display(), e);
+                return;
+            }
+        }
+
+        if let Err(e) = std::os::unix::fs::symlink(&tmp_out, &self.env.out_path) {
+            crit!(
+                "Failed to symlink {} -> {}: {}",
+                tmp_out.display(),
+                self.env.out_path.display(),
+                e
+            );
+            return;
+        }
+
+        dbug!(
+            "Output symlinked: {} -> {}",
+            self.env.out_path.display(),
+            tmp_out.display()
+        );
+    }
+}
+
+#[derive(Clone, Debug)]
 pub struct EnvVars {
     pub spec_path: PathBuf,
     pub tmpl_path: PathBuf,

src/log.rs

@@ -1,4 +1,3 @@
-#![allow(dead_code)]
 use std::sync::OnceLock;
 
 #[derive(Debug, PartialEq, PartialOrd, Clone, Copy)]

src/main.rs

@@ -3,22 +3,18 @@ use skyforge::{Args, DeviceConfigBundle, Specification};
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let args = Args::parse();
 
-    let exec_id = std::time::SystemTime::now()
+    args.use_tmp();
-        .duration_since(std::time::UNIX_EPOCH)
-        .unwrap()
-        .as_millis();
-
-    let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
-    std::fs::create_dir_all(&tmp_out)?;
-    if args.env.out_path.exists() {
-        std::fs::remove_file(&args.env.out_path)
-            .or_else(|_| std::fs::remove_dir_all(&args.env.out_path))?;
-    }
-    std::os::unix::fs::symlink(&tmp_out, &args.env.out_path)?;
 
+    if args.classic {
         for spec in Specification::compile(&args) {
             DeviceConfigBundle::from_spec(spec)?.output_artifacts();
         }
+    } else {
+        DeviceConfigBundle::process_specs(Specification::compile(&args))?
+            .into_iter()
+            .for_each(|c| c.output_artifacts());
+    }
 
+    println!("Completed Successfully");
     Ok(())
 }

src/render.rs

@@ -2,8 +2,11 @@ use crate::log::LogLevel;
 use crate::spec::Specification;
 use crate::tmpl;
 use crate::{crit, dbug, info, verb};
+use std::collections::HashSet;
 use std::error::Error;
 use std::io::Write;
+use std::path::PathBuf;
+use tera::Tera;
 
 pub struct DeviceConfigBundle {
     pub hostname: String,
@@ -22,20 +25,96 @@ pub struct Configuration {
     pub data: String,
 }
 
+pub struct Renderer {
+    pub engine: Tera,
+    pub from: PathBuf,
+}
+
 impl DeviceConfigBundle {
-    pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn std::error::Error>> {
+    pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn Error>> {
-        let mut context = Self::merge_context(&spec);
+        let structure =
-
+            tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| {
-        let hostname = Self::get_hostname(&context, &spec);
+                crit!("{}", e);
-        info!("Processing templates for '{hostname}'");
+                e
-
+            })?;
-        let structure_path = spec.tmplpath.join("structure.yaml");
+        let mut renderer = Tera::default();
-        let structure = tmpl::Structure::from_file(&structure_path)
-            .map_err(|e| format!("Failed to parse {:?}: {e}", structure_path))?;
-
-        let mut renderer = tera::Tera::default();
         renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
-        dbug!("Rendering templates for {hostname}");
+        Self::render_configs(spec, &structure, &renderer)
+    }
+
+    pub fn process_specs(specs: Vec<Specification>) -> Result<Vec<Self>, Box<dyn Error>> {
+        let mut failed = false;
+        let renderers: Vec<Renderer> = specs
+            .iter()
+            .map(|s| s.tmplpath.clone())
+            .collect::<HashSet<PathBuf>>()
+            .into_iter()
+            .filter_map(|p| {
+                let structure = tmpl::Structure::from_file(&p.join("structure.yaml"))
+                    .map_err(|e| {
+                        crit!("[!] {}", e);
+                        failed = true;
+                    })
+                    .ok()?;
+                let mut t = Tera::default();
+                t.add_raw_templates(structure.load_template_data(&p))
+                    .map_err(|e| {
+                        crit!("While loading {}: {}", &p.display(), e.chain_msg());
+                        failed = true;
+                    })
+                    .ok()?;
+                Some(Renderer { engine: t, from: p })
+            })
+            .collect();
+
+        if failed {
+            return Err("Failed to load templates.".into());
+        }
+
+        info!("Rendering configs:");
+        let results: Vec<Result<Self, _>> = specs
+            .into_iter()
+            .map(|s| {
+                let renderer =
+                    renderers
+                        .iter()
+                        .find(|r| r.from == s.tmplpath)
+                        .ok_or_else(|| {
+                            format!(
+                                "Unexpected. Missing Tera renderer: {}",
+                                s.tmplpath.display()
+                            )
+                        })?;
+                let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml"))
+                    .map_err(|e| {
+                        crit!("{}", e);
+                        e
+                    })?;
+                Self::render_configs(s, &structure, &renderer.engine)
+            })
+            .collect();
+
+        failed = results.iter().any(|r| r.is_err());
+        let successes: Vec<Self> = results
+            .into_iter()
+            .filter_map(|r| r.map_err(|e| crit!("{}", e)).ok())
+            .collect();
+
+        if failed {
+            return Err("One or more specs failed to process.".into());
+        }
+
+        Ok(successes)
+    }
+
+    fn render_configs(
+        spec: Specification,
+        structure: &tmpl::Structure,
+        renderer: &Tera,
+    ) -> Result<Self, Box<dyn Error>> {
+        let mut context = Self::merge_context(&spec);
+        let hostname = spec.components.get_hostname();
+        info!(" | {hostname}");
 
         let mut failures = false;
         let mut configs = Vec::new();
@@ -43,37 +122,35 @@ impl DeviceConfigBundle {
             context.insert(variant, &true);
             let mut cfgs = Vec::new();
             for template_name in &structure.files {
-                dbug!(" | {template_name}.{variant}");
+                dbug!(" |\t{template_name}.{variant}");
                 match renderer.render(template_name, &context) {
-                    Ok(data) => cfgs.push(Configuration::new(template_name, data)),
+                    Ok(data) => cfgs.push(Configuration {
+                        name: String::from(template_name),
+                        data,
+                    }),
                     Err(e) => {
-                        crit!("[!] {}", Error::source(&e).unwrap_or_else(|| &e));
+                        crit!("{}", e.chain_msg());
-                        renderer.add_raw_template(template_name, "")?;
                         failures = true;
                     }
                 }
             }
-            configs.push(ConfigVariant::new(variant, cfgs));
+            configs.push(ConfigVariant {
+                name: String::from(variant),
+                configs: cfgs,
+            });
             context.remove(variant);
         }
 
-        match failures {
+        if failures {
-            true => Err(Box::<dyn Error>::from("Rendering failed.")),
+            return Err(Box::<dyn Error>::from("Rendering failed."));
-            false => Ok(Self {
+        }
+
+        Ok(Self {
             hostname,
             configs,
             spec,
-                platform: structure.platform,
+            platform: structure.platform.clone(),
-            }),
+        })
-        }
-    }
-
-    fn get_hostname(context: &tera::Context, spec: &Specification) -> String {
-        context
-            .get("hostname")
-            .and_then(|v| v.as_str())
-            .map(|s| s.to_string())
-            .unwrap_or_else(|| spec.components.get_hostname())
     }
 
     fn merge_context(spec: &Specification) -> tera::Context {
@@ -86,57 +163,80 @@ impl DeviceConfigBundle {
         context
     }
 
-    pub fn output_artifacts(self: Self) {
+    pub fn output_artifacts(self) {
         info!("Writing Output:");
 
         let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
-        std::fs::create_dir_all(&device_outpath).ok();
+        if let Err(e) = std::fs::create_dir_all(&device_outpath) {
+            crit!(
+                "Failed to create output directory {}: {}",
+                device_outpath.display(),
+                e
+            );
+            return;
+        }
 
         for variant_configs in self.configs {
             let out_path = device_outpath.join(&variant_configs.name);
-            std::fs::create_dir_all(&out_path).ok();
+            if let Err(e) = std::fs::create_dir_all(&out_path) {
+                crit!(
+                    "Failed to create variant directory {}: {}",
+                    out_path.display(),
+                    e
+                );
+                continue;
+            }
 
             let merged_config_path =
                 device_outpath.join(format!("all.{}.{}", variant_configs.name, self.platform));
-            let mut all_file: std::fs::File = std::fs::OpenOptions::new()
+            let mut all_file = match std::fs::OpenOptions::new()
                 .create(true)
                 .append(true)
                 .open(&merged_config_path)
-                .expect("unable to open");
+            {
+                Ok(f) => f,
+                Err(e) => {
+                    crit!("Failed to open {}: {}", merged_config_path.display(), e);
+                    continue;
+                }
+            };
 
             for config in &variant_configs.configs {
                 let config_outpath = out_path.join(format!("{}.{}", config.name, self.platform));
                 verb!(" | {}", &config_outpath.display());
-                std::fs::write(&config_outpath, &config.data).ok();
+                if let Err(e) = std::fs::write(&config_outpath, &config.data) {
-                all_file.write_all(&config.data.as_bytes()).ok();
+                    crit!("Failed to write {}: {}", config_outpath.display(), e);
+                }
+                if let Err(e) = all_file.write_all(config.data.as_bytes()) {
+                    crit!("Failed to write to {}: {}", merged_config_path.display(), e);
+                }
             }
             info!(" | {} ", &merged_config_path.display());
         }
 
         let compiled_spec_path = device_outpath.join("context.yaml");
         verb!(" | {}", &compiled_spec_path.display());
-        std::fs::write(
+        if let Err(e) = std::fs::write(
             &compiled_spec_path,
             yaml_serde::to_string(&self.spec.compiled).unwrap(),
-        )
+        ) {
-        .ok();
+            crit!("Failed to write {}: {}", compiled_spec_path.display(), e);
-    }
-}
-
-impl ConfigVariant {
-    pub fn new(name: &str, configs: Vec<Configuration>) -> Self {
-        Self {
-            name: String::from(name),
-            configs,
         }
     }
 }
 
-impl Configuration {
+trait ErrorExt {
-    pub fn new(name: &str, data: String) -> Self {
+    fn chain_msg(&self) -> String;
-        Self {
+}
-            name: String::from(name),
+
-            data,
+impl ErrorExt for tera::Error {
-        }
+    fn chain_msg(&self) -> String {
+        let mut out = self.to_string();
+        let mut source = self.source();
+        while let Some(s) = source {
+            out.push_str(&format!("\n > {}", s));
+            source = s.source();
+        }
+        out
     }
 }

src/tmpl.rs

@@ -1,4 +1,4 @@
-use crate::{LogLevel, crit, verb};
+use crate::{LogLevel, crit, dbug};
 use serde::Deserialize;
 use std::error::Error;
 use std::path::PathBuf;
@@ -24,7 +24,7 @@ impl Structure {
                 let template = tmpl_dir.join(format!("{}.tera", name));
                 match std::fs::read_to_string(&template) {
                     Ok(content) => {
-                        verb!(" | {}", template.display());
+                        dbug!(" | {}", template.display());
                         Some((name.clone(), content))
                     }
                     Err(e) => {