lost/skyforge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat dedup Tera renderer; update logging

Browse Source
This commit is contained in:
lost
2026-02-26 02:00:11 -07:00
parent f8ced2196c
commit d8134095ab
7 changed files with 303 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
README.md
View File

@@ -60,25 +60,24 @@ Environment:
$ skyforge -d xyz1-ex-edge-r101
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
Processing templates for 'xyz1-ex-edge-r101'
Rendering configs:
| xyz1-ex-edge-r101
Writing Output:
| ./out/xyz1-ex-edge-r101/all.live.junos
| ./out/xyz1-ex-edge-r101/all.shifted.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
Completed Successfully
```
### Verbose
``` bash
$ skyforge -d xyz1-ex-edge-r101 --verbose
$ skyforge -d xyz1-ex-edge-r101 -v
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
Processing templates for 'xyz1-ex-edge-r101'
| ./tmpl/ex-edge-r/system.tera
| ./tmpl/ex-edge-r/interfaces.tera
| ./tmpl/ex-edge-r/protocols.tera
| ./tmpl/ex-edge-r/policy-options.tera
Rendering configs:
| xyz1-ex-edge-r101
Writing Output:
| ./out/xyz1-ex-edge-r101/live/system.junos
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
@@ -96,12 +95,16 @@ Writing Output:
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
| ./out/xyz1-ex-edge-r101/context.yaml
Completed Successfully
```
### Debug
``` bash
$ skyforge -d xyz1-ex-edge-r101 --debug
$ skyforge -d xyz1-ex-edge-r101 -D
Using tmp dir: /tmp/skyforge-1772096013345
Removing existing output path: ./out
Output symlinked: ./out -> /tmp/skyforge-1772096013345
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
@@ -112,24 +115,24 @@ Components {
zonal: "./spec/xyz/ex-edge-r/xyz1.common.yaml",
device: "./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml",
}
Processing templates for 'xyz1-ex-edge-r101'
| ./tmpl/ex-edge-r/system.tera
| ./tmpl/ex-edge-r/interfaces.tera
| ./tmpl/ex-edge-r/protocols.tera
| ./tmpl/ex-edge-r/policy-options.tera
Rendering templates for xyz1-ex-edge-r101
| system.live
| interfaces.live
| protocols.live
| policy-options.live
| system.shifted
| interfaces.shifted
| protocols.shifted
| policy-options.shifted
| system.init
| interfaces.init
| protocols.init
| policy-options.init
Rendering configs:
| xyz1-ex-edge-r101
| system.live
| interfaces.live
| protocols.live
| policy-options.live
| system.shifted
| interfaces.shifted
| protocols.shifted
| policy-options.shifted
| system.init
| interfaces.init
| protocols.init
| policy-options.init
Writing Output:
| ./out/xyz1-ex-edge-r101/live/system.junos
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
@@ -147,6 +150,7 @@ Writing Output:
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
| ./out/xyz1-ex-edge-r101/context.yaml
Completed Successfully
```
## Flamegraph
@@ -157,3 +161,12 @@ Assume flamelens is installed, otherwise `cargo install flamelens`
cd demo
cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devices ".*"
```
## Benchmark
Using Demo:
``` bash
process_specs time: [1.6096 ms 1.6205 ms 1.6327 ms]
from_spec time: [3.3230 ms 3.3397 ms 3.3575 ms]
```

48
benches/benchmark.rs
View File

@@ -1,28 +1,46 @@
use criterion::{Criterion, criterion_group, criterion_main};
use skyforge::log::{LOG_LEVEL, LogLevel};
use skyforge::{Args, DeviceConfigBundle, Specification};
use std::path::PathBuf;
fn env() -> skyforge::cli::EnvVars {
skyforge::cli::EnvVars {
spec_path: PathBuf::from("./demo/spec"),
tmpl_path: PathBuf::from("./demo/tmpl"),
out_path: PathBuf::from("./demo/out"),
log_path: PathBuf::from("./demo/log"),
}
}
fn benchmark(c: &mut Criterion) {
LOG_LEVEL.set(LogLevel::Warning).ok();
let args = Args {
devices: regex::Regex::new(".*").unwrap(),
env: skyforge::cli::EnvVars {
spec_path: PathBuf::from("./demo/spec"),
tmpl_path: PathBuf::from("./demo/tmpl"),
out_path: PathBuf::from("./demo/out"),
log_path: PathBuf::from("./demo/log"),
},
classic: false,
env: env(),
};
c.bench_function("compile", |b| {
c.bench_function("process_specs", |b| {
b.iter(|| {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)
.unwrap()
.output_artifacts();
}
})
DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap();
});
});
}
criterion_group!(benches, benchmark);
fn benchmark_classic(c: &mut Criterion) {
LOG_LEVEL.set(LogLevel::Warning).ok();
let args = Args {
devices: regex::Regex::new(".*").unwrap(),
classic: true,
env: env(),
};
c.bench_function("from_spec", |b| {
b.iter(|| {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec).unwrap();
}
});
});
}
criterion_group!(benches, benchmark, benchmark_classic);
criterion_main!(benches);

70
src/cli.rs
View File

@@ -1,4 +1,5 @@
use crate::log::{LOG_LEVEL, LogLevel};
use crate::{crit, dbug};
use clap::{Arg, ArgAction, ArgGroup, Command};
use regex::Regex;
use std::path::PathBuf;
@@ -14,6 +15,7 @@ const ENV_MSG: &str = r#"Environment:
#[derive(Debug)]
pub struct Args {
pub devices: Regex,
pub classic: bool,
pub env: EnvVars,
}
@@ -30,11 +32,14 @@ impl Args {
loglevel = LogLevel::Debug;
} else if matches.get_flag("verbose") {
loglevel = LogLevel::Verbose;
} else if matches.get_flag("silent") {
loglevel = LogLevel::Warning;
}
LOG_LEVEL.set(loglevel).ok();
Args {
devices,
classic: matches.get_flag("classic"),
env: EnvVars::parse(),
}
}
@@ -52,6 +57,7 @@ impl Args {
)
.arg(
Arg::new("debug")
.short('D')
.long("debug")
.help("Print debug information")
.action(ArgAction::SetTrue)
@@ -65,16 +71,76 @@ impl Args {
.action(ArgAction::SetTrue)
.required(false),
)
.arg(
Arg::new("silent")
.short('s')
.long("silent")
.help("Sets loglevel to Warn")
.action(ArgAction::SetTrue)
.required(false),
)
.arg(
Arg::new("classic")
.short('c')
.long("classic")
.help("creates a new renderer for each device")
.action(ArgAction::SetTrue)
.required(false),
)
.group(
ArgGroup::new("loglevel")
.args(&["debug", "verbose"])
.args(&["debug", "verbose", "silent"])
.required(false),
)
.after_help(ENV_MSG)
}
pub fn use_tmp(self: &Self) {
let exec_id = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis();
let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
dbug!("Using tmp dir: {}", tmp_out.display());
if let Err(e) = std::fs::create_dir_all(&tmp_out) {
crit!("Failed to create tmp dir {}: {}", tmp_out.display(), e);
return;
}
if self.env.out_path.exists() {
dbug!(
"Removing existing output path: {}",
self.env.out_path.display()
);
if let Err(e) = std::fs::remove_file(&self.env.out_path)
.or_else(|_| std::fs::remove_dir_all(&self.env.out_path))
{
crit!("Failed to remove {}: {}", self.env.out_path.display(), e);
return;
}
}
if let Err(e) = std::os::unix::fs::symlink(&tmp_out, &self.env.out_path) {
crit!(
"Failed to symlink {} -> {}: {}",
tmp_out.display(),
self.env.out_path.display(),
e
);
return;
}
dbug!(
"Output symlinked: {} -> {}",
self.env.out_path.display(),
tmp_out.display()
);
}
}
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct EnvVars {
pub spec_path: PathBuf,
pub tmpl_path: PathBuf,

1
src/log.rs
View File

@@ -1,4 +1,3 @@
#![allow(dead_code)]
use std::sync::OnceLock;
#[derive(Debug, PartialEq, PartialOrd, Clone, Copy)]

24
src/main.rs
View File

@@ -3,22 +3,18 @@ use skyforge::{Args, DeviceConfigBundle, Specification};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
let exec_id = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis();
args.use_tmp();
let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
std::fs::create_dir_all(&tmp_out)?;
if args.env.out_path.exists() {
std::fs::remove_file(&args.env.out_path)
.or_else(|_| std::fs::remove_dir_all(&args.env.out_path))?;
}
std::os::unix::fs::symlink(&tmp_out, &args.env.out_path)?;
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
if args.classic {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
}
} else {
DeviceConfigBundle::process_specs(Specification::compile(&args))?
.into_iter()
.for_each(|c| c.output_artifacts());
}
println!("Completed Successfully");
Ok(())
}

210
src/render.rs
View File

@@ -2,8 +2,11 @@ use crate::log::LogLevel;
use crate::spec::Specification;
use crate::tmpl;
use crate::{crit, dbug, info, verb};
use std::collections::HashSet;
use std::error::Error;
use std::io::Write;
use std::path::PathBuf;
use tera::Tera;
pub struct DeviceConfigBundle {
pub hostname: String,
@@ -22,20 +25,96 @@ pub struct Configuration {
pub data: String,
}
pub struct Renderer {
pub engine: Tera,
pub from: PathBuf,
}
impl DeviceConfigBundle {
pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn std::error::Error>> {
let mut context = Self::merge_context(&spec);
let hostname = Self::get_hostname(&context, &spec);
info!("Processing templates for '{hostname}'");
let structure_path = spec.tmplpath.join("structure.yaml");
let structure = tmpl::Structure::from_file(&structure_path)
.map_err(|e| format!("Failed to parse {:?}: {e}", structure_path))?;
let mut renderer = tera::Tera::default();
pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn Error>> {
let structure =
tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| {
crit!("{}", e);
e
})?;
let mut renderer = Tera::default();
renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
dbug!("Rendering templates for {hostname}");
Self::render_configs(spec, &structure, &renderer)
}
pub fn process_specs(specs: Vec<Specification>) -> Result<Vec<Self>, Box<dyn Error>> {
let mut failed = false;
let renderers: Vec<Renderer> = specs
.iter()
.map(|s| s.tmplpath.clone())
.collect::<HashSet<PathBuf>>()
.into_iter()
.filter_map(|p| {
let structure = tmpl::Structure::from_file(&p.join("structure.yaml"))
.map_err(|e| {
crit!("[!] {}", e);
failed = true;
})
.ok()?;
let mut t = Tera::default();
t.add_raw_templates(structure.load_template_data(&p))
.map_err(|e| {
crit!("While loading {}: {}", &p.display(), e.chain_msg());
failed = true;
})
.ok()?;
Some(Renderer { engine: t, from: p })
})
.collect();
if failed {
return Err("Failed to load templates.".into());
}
info!("Rendering configs:");
let results: Vec<Result<Self, _>> = specs
.into_iter()
.map(|s| {
let renderer =
renderers
.iter()
.find(|r| r.from == s.tmplpath)
.ok_or_else(|| {
format!(
"Unexpected. Missing Tera renderer: {}",
s.tmplpath.display()
)
})?;
let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml"))
.map_err(|e| {
crit!("{}", e);
e
})?;
Self::render_configs(s, &structure, &renderer.engine)
})
.collect();
failed = results.iter().any(|r| r.is_err());
let successes: Vec<Self> = results
.into_iter()
.filter_map(|r| r.map_err(|e| crit!("{}", e)).ok())
.collect();
if failed {
return Err("One or more specs failed to process.".into());
}
Ok(successes)
}
fn render_configs(
spec: Specification,
structure: &tmpl::Structure,
renderer: &Tera,
) -> Result<Self, Box<dyn Error>> {
let mut context = Self::merge_context(&spec);
let hostname = spec.components.get_hostname();
info!(" | {hostname}");
let mut failures = false;
let mut configs = Vec::new();
@@ -43,37 +122,35 @@ impl DeviceConfigBundle {
context.insert(variant, &true);
let mut cfgs = Vec::new();
for template_name in &structure.files {
dbug!(" | {template_name}.{variant}");
dbug!(" |\t{template_name}.{variant}");
match renderer.render(template_name, &context) {
Ok(data) => cfgs.push(Configuration::new(template_name, data)),
Ok(data) => cfgs.push(Configuration {
name: String::from(template_name),
data,
}),
Err(e) => {
crit!("[!] {}", Error::source(&e).unwrap_or_else(|| &e));
renderer.add_raw_template(template_name, "")?;
crit!("{}", e.chain_msg());
failures = true;
}
}
}
configs.push(ConfigVariant::new(variant, cfgs));
configs.push(ConfigVariant {
name: String::from(variant),
configs: cfgs,
});
context.remove(variant);
}
match failures {
true => Err(Box::<dyn Error>::from("Rendering failed.")),
false => Ok(Self {
hostname,
configs,
spec,
platform: structure.platform,
}),
if failures {
return Err(Box::<dyn Error>::from("Rendering failed."));
}
}
fn get_hostname(context: &tera::Context, spec: &Specification) -> String {
context
.get("hostname")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.unwrap_or_else(|| spec.components.get_hostname())
Ok(Self {
hostname,
configs,
spec,
platform: structure.platform.clone(),
})
}
fn merge_context(spec: &Specification) -> tera::Context {
@@ -86,57 +163,80 @@ impl DeviceConfigBundle {
context
}
pub fn output_artifacts(self: Self) {
pub fn output_artifacts(self) {
info!("Writing Output:");
let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
std::fs::create_dir_all(&device_outpath).ok();
if let Err(e) = std::fs::create_dir_all(&device_outpath) {
crit!(
"Failed to create output directory {}: {}",
device_outpath.display(),
e
);
return;
}
for variant_configs in self.configs {
let out_path = device_outpath.join(&variant_configs.name);
std::fs::create_dir_all(&out_path).ok();
if let Err(e) = std::fs::create_dir_all(&out_path) {
crit!(
"Failed to create variant directory {}: {}",
out_path.display(),
e
);
continue;
}
let merged_config_path =
device_outpath.join(format!("all.{}.{}", variant_configs.name, self.platform));
let mut all_file: std::fs::File = std::fs::OpenOptions::new()
let mut all_file = match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&merged_config_path)
.expect("unable to open");
{
Ok(f) => f,
Err(e) => {
crit!("Failed to open {}: {}", merged_config_path.display(), e);
continue;
}
};
for config in &variant_configs.configs {
let config_outpath = out_path.join(format!("{}.{}", config.name, self.platform));
verb!(" | {}", &config_outpath.display());
std::fs::write(&config_outpath, &config.data).ok();
all_file.write_all(&config.data.as_bytes()).ok();
if let Err(e) = std::fs::write(&config_outpath, &config.data) {
crit!("Failed to write {}: {}", config_outpath.display(), e);
}
if let Err(e) = all_file.write_all(config.data.as_bytes()) {
crit!("Failed to write to {}: {}", merged_config_path.display(), e);
}
}
info!(" | {} ", &merged_config_path.display());
}
let compiled_spec_path = device_outpath.join("context.yaml");
verb!(" | {}", &compiled_spec_path.display());
std::fs::write(
if let Err(e) = std::fs::write(
&compiled_spec_path,
yaml_serde::to_string(&self.spec.compiled).unwrap(),
)
.ok();
}
}
impl ConfigVariant {
pub fn new(name: &str, configs: Vec<Configuration>) -> Self {
Self {
name: String::from(name),
configs,
) {
crit!("Failed to write {}: {}", compiled_spec_path.display(), e);
}
}
}
impl Configuration {
pub fn new(name: &str, data: String) -> Self {
Self {
name: String::from(name),
data,
trait ErrorExt {
fn chain_msg(&self) -> String;
}
impl ErrorExt for tera::Error {
fn chain_msg(&self) -> String {
let mut out = self.to_string();
let mut source = self.source();
while let Some(s) = source {
out.push_str(&format!("\n > {}", s));
source = s.source();
}
out
}
}

4
src/tmpl.rs
View File

@@ -1,4 +1,4 @@
use crate::{LogLevel, crit, verb};
use crate::{LogLevel, crit, dbug};
use serde::Deserialize;
use std::error::Error;
use std::path::PathBuf;
@@ -24,7 +24,7 @@ impl Structure {
let template = tmpl_dir.join(format!("{}.tera", name));
match std::fs::read_to_string(&template) {
Ok(content) => {
verb!(" | {}", template.display());
dbug!(" | {}", template.display());
Some((name.clone(), content))
}
Err(e) => {