feat output artifacts to tmp; add bench & profiling; minor fixes

This commit is contained in:
2026-02-23 02:51:39 -07:00
parent 9649961580
commit f8ced2196c
13 changed files with 2404 additions and 221 deletions

1
.gitignore vendored
View File

@@ -1,2 +1 @@
/target
/demo/out

2185
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -3,6 +3,10 @@ name = "skyforge"
version = "0.1.112"
edition = "2024"
[profile.profiling]
inherits = "release"
debug = true
[dependencies]
anyhow = "1"
clap = { version = "4.5" }
@@ -13,3 +17,11 @@ yaml_serde = "0"
tera = "1"
thiserror = "1"
walkdir = "2"
[dev-dependencies]
criterion = "0.5"
flamegraph = "0.6"
[[bench]]
name = "benchmark"
harness = false

141
README.md
View File

@@ -9,6 +9,7 @@ skyforge was designed to assist in rendering thousands of device configurations
- Partitions are groups of regions
- Regions are groups of zones
- Zones are groups of devices
- Fabrics are groups of layers
- Layers are groups of common devices and facilitate template mapping
## Functionality
@@ -20,13 +21,13 @@ Skyforge takes a user provided regex pattern, performs a walk on a `./spec` dir,
For each "device" matched, skyforge then maps to all consituent files:
- Layer - from the `common.yaml` file in parent dir and maps the region
- Zonal - from the first group of chars in filename up to a `-` which is expected to be region and zone
- Regional - from the `<region>/common/<network>.yaml` of containing folder where network matches the layer info
- Partitional - from either layer (common.yaml) or regional yaml
- Zonal - from the first group of chars in filename up to a `-`
- Regional - from value of `common.fabric` field in `Layer` file
- Partitional - from value of `regional.partition` field in `Regional` file
Once all files are found, a compiled specifcation is built.
This spec is then passed to Tera as context.
Tera then loads the template files for that layer and renders the configuration files.
Tera loads the template files for that layer and renders the configuration files.
## Usage
@@ -47,68 +48,112 @@ Options:
-V, --version Print version
Environment:
SF_SPEC_PATH Directory containing templates
SF_TMPL_PATH Directory containing specifications
SF_OUT_PATH Directory for command output
SF_LOG_PATH Directory for log output
SKYFORGE_SPECDIR Directory containing templates
SKYFORGE_TMPLDIR Directory containing specifications
SKYFORGE_OUTDIR Directory for command output
SKYFORGE_LOGDIR Directory for log output
```
### Standard
``` bash
$ skyforge -d xyz1-ex-edge-r101
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
Rendering xyz1-ex-edge-r101
Writing Output
| out/xyz1-ex-edge-r101/all.conf
Processing templates for 'xyz1-ex-edge-r101'
Writing Output:
| ./out/xyz1-ex-edge-r101/all.live.junos
| ./out/xyz1-ex-edge-r101/all.shifted.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
```
### Verbose
``` bash
$ skyforge -d xyz1-ex-edge-r101 -v
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
$ skyforge -d xyz1-ex-edge-r101 --verbose
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
Rendering xyz1-ex-edge-r101
| ./tmpl/ex-edge-r/system.tmpl
| ./tmpl/ex-edge-r/chassis.tmpl
| ./tmpl/ex-edge-r/interfaces.tmpl
| ./tmpl/ex-edge-r/protocols.tmpl
Writing Output
| out/xyz1-ex-edge-r101/system.tmpl
| out/xyz1-ex-edge-r101/chassis.tmpl
| out/xyz1-ex-edge-r101/interfaces.tmpl
| out/xyz1-ex-edge-r101/protocols.tmpl
| out/xyz1-ex-edge-r101/compiled.spec
| out/xyz1-ex-edge-r101/all.conf
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
Processing templates for 'xyz1-ex-edge-r101'
| ./tmpl/ex-edge-r/system.tera
| ./tmpl/ex-edge-r/interfaces.tera
| ./tmpl/ex-edge-r/protocols.tera
| ./tmpl/ex-edge-r/policy-options.tera
Writing Output:
| ./out/xyz1-ex-edge-r101/live/system.junos
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
| ./out/xyz1-ex-edge-r101/live/protocols.junos
| ./out/xyz1-ex-edge-r101/live/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.live.junos
| ./out/xyz1-ex-edge-r101/shifted/system.junos
| ./out/xyz1-ex-edge-r101/shifted/interfaces.junos
| ./out/xyz1-ex-edge-r101/shifted/protocols.junos
| ./out/xyz1-ex-edge-r101/shifted/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.shifted.junos
| ./out/xyz1-ex-edge-r101/init/system.junos
| ./out/xyz1-ex-edge-r101/init/interfaces.junos
| ./out/xyz1-ex-edge-r101/init/protocols.junos
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
| ./out/xyz1-ex-edge-r101/context.yaml
```
### Debug
``` bash
$ skyforge -d xyz1-ex-edge-r101 --debug
devices: xyz1-ex-edge-r101, loglevel: Debug, env: spec_path: ./spec, tmpl_path: ./tmpl, out_path: ./out, log_path: ./log
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
Skyforge found 9 renderable devices
Matched 1 devices against 'xyz1-ex-edge-r101'
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
Compiled Spec for 'xyz1-ex-edge-r101.yaml'
| ./spec/common/us.yaml
| ./spec/xyz/common/ex.yaml
| ./spec/xyz/ex-edge-r1/common.yaml
| ./spec/xyz/ex-edge-r1/xyz1.common.yaml
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
Rendering xyz1-ex-edge-r101
| ./tmpl/ex-edge-r/system.tmpl
| ./tmpl/ex-edge-r/chassis.tmpl
| ./tmpl/ex-edge-r/interfaces.tmpl
| ./tmpl/ex-edge-r/protocols.tmpl
Writing Output
| out/xyz1-ex-edge-r101/system.tmpl
| out/xyz1-ex-edge-r101/chassis.tmpl
| out/xyz1-ex-edge-r101/interfaces.tmpl
| out/xyz1-ex-edge-r101/protocols.tmpl
| out/xyz1-ex-edge-r101/compiled.spec
| out/xyz1-ex-edge-r101/all.conf
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
Components {
partitional: "./spec/common/us.yaml",
regional: "./spec/xyz/common/ex.yaml",
common: "./spec/xyz/ex-edge-r/common.yaml",
zonal: "./spec/xyz/ex-edge-r/xyz1.common.yaml",
device: "./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml",
}
Processing templates for 'xyz1-ex-edge-r101'
| ./tmpl/ex-edge-r/system.tera
| ./tmpl/ex-edge-r/interfaces.tera
| ./tmpl/ex-edge-r/protocols.tera
| ./tmpl/ex-edge-r/policy-options.tera
Rendering templates for xyz1-ex-edge-r101
| system.live
| interfaces.live
| protocols.live
| policy-options.live
| system.shifted
| interfaces.shifted
| protocols.shifted
| policy-options.shifted
| system.init
| interfaces.init
| protocols.init
| policy-options.init
Writing Output:
| ./out/xyz1-ex-edge-r101/live/system.junos
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
| ./out/xyz1-ex-edge-r101/live/protocols.junos
| ./out/xyz1-ex-edge-r101/live/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.live.junos
| ./out/xyz1-ex-edge-r101/shifted/system.junos
| ./out/xyz1-ex-edge-r101/shifted/interfaces.junos
| ./out/xyz1-ex-edge-r101/shifted/protocols.junos
| ./out/xyz1-ex-edge-r101/shifted/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.shifted.junos
| ./out/xyz1-ex-edge-r101/init/system.junos
| ./out/xyz1-ex-edge-r101/init/interfaces.junos
| ./out/xyz1-ex-edge-r101/init/protocols.junos
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
| ./out/xyz1-ex-edge-r101/all.init.junos
| ./out/xyz1-ex-edge-r101/context.yaml
```
## Flamegraph
Assume flamelens is installed, otherwise `cargo install flamelens`
``` bash
cd demo
cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devices ".*"
```

28
benches/benchmark.rs Normal file
View File

@@ -0,0 +1,28 @@
use criterion::{Criterion, criterion_group, criterion_main};
use skyforge::{Args, DeviceConfigBundle, Specification};
use std::path::PathBuf;
fn benchmark(c: &mut Criterion) {
let args = Args {
devices: regex::Regex::new(".*").unwrap(),
env: skyforge::cli::EnvVars {
spec_path: PathBuf::from("./demo/spec"),
tmpl_path: PathBuf::from("./demo/tmpl"),
out_path: PathBuf::from("./demo/out"),
log_path: PathBuf::from("./demo/log"),
},
};
c.bench_function("compile", |b| {
b.iter(|| {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)
.unwrap()
.output_artifacts();
}
})
});
}
criterion_group!(benches, benchmark);
criterion_main!(benches);

3
demo/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
flamegraph*
perf*
out

View File

@@ -1,5 +1,5 @@
common:
partition: us
fabric: ex
layer: ex-core-r
protocol: ospf
uplink: et-0/0/36

View File

@@ -1,4 +1,5 @@
common:
fabric: ex
layer: ex-edge-r
protocol: bgp
uplink: et-0/0/0

View File

@@ -5,16 +5,15 @@ use std::path::PathBuf;
const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
const ENV_MSG: &str = r#"Environment:
SF_SPEC_PATH Directory containing templates
SF_TMPL_PATH Directory containing specifications
SF_OUT_PATH Directory for command output
SF_LOG_PATH Directory for log output
SKYFORGE_SPECDIR Directory containing templates
SKYFORGE_TMPLDIR Directory containing specifications
SKYFORGE_OUTDIR Directory for command output
SKYFORGE_LOGDIR Directory for log output
"#;
#[derive(Debug)]
pub struct Args {
pub devices: Regex,
pub loglevel: LogLevel,
pub env: EnvVars,
}
@@ -36,7 +35,6 @@ impl Args {
Args {
devices,
loglevel,
env: EnvVars::parse(),
}
}
@@ -87,16 +85,16 @@ pub struct EnvVars {
impl EnvVars {
pub fn parse() -> Self {
Self {
spec_path: std::env::var("SF_SPEC_PATH")
spec_path: std::env::var("SKYFORGE_SPECDIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("./spec")),
tmpl_path: std::env::var("SF_TMPL_PATH")
tmpl_path: std::env::var("SKYFORGE_TMPLDIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("./tmpl")),
out_path: std::env::var("SF_OUT_PATH")
out_path: std::env::var("SKYFORGE_OUTDIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("./out")),
log_path: std::env::var("SF_LOG_PATH")
log_path: std::env::var("SKYFORGE_LOGDIR")
.map(PathBuf::from)
.unwrap_or_else(|_| PathBuf::from("./log")),
}

View File

@@ -3,6 +3,19 @@ use skyforge::{Args, DeviceConfigBundle, Specification};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
let exec_id = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_millis();
let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
std::fs::create_dir_all(&tmp_out)?;
if args.env.out_path.exists() {
std::fs::remove_file(&args.env.out_path)
.or_else(|_| std::fs::remove_dir_all(&args.env.out_path))?;
}
std::os::unix::fs::symlink(&tmp_out, &args.env.out_path)?;
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
}

View File

@@ -27,16 +27,15 @@ impl DeviceConfigBundle {
let mut context = Self::merge_context(&spec);
let hostname = Self::get_hostname(&context, &spec);
let base_dir = std::path::Path::new("./tmpl").join(spec.get_layer());
info!("Loading templates for '{hostname}':");
info!("Processing templates for '{hostname}'");
let structure_path = base_dir.join("structure.yaml");
let structure_path = spec.tmplpath.join("structure.yaml");
let structure = tmpl::Structure::from_file(&structure_path)
.map_err(|e| format!("Failed to parse {}: {e}", structure_path.display()))?;
.map_err(|e| format!("Failed to parse {:?}: {e}", structure_path))?;
let mut renderer = tera::Tera::default();
renderer.add_raw_templates(structure.load_template_data(base_dir))?;
dbug!("Processing templates for {hostname}");
renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
dbug!("Rendering templates for {hostname}");
let mut failures = false;
let mut configs = Vec::new();
@@ -91,9 +90,6 @@ impl DeviceConfigBundle {
info!("Writing Output:");
let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
if device_outpath.exists() {
std::fs::remove_dir_all(&device_outpath).ok();
}
std::fs::create_dir_all(&device_outpath).ok();
for variant_configs in self.configs {

View File

@@ -1,16 +1,14 @@
use crate::cli::Args;
use crate::{LogLevel, info, verb};
use regex::Regex;
use serde_json::{Map, Value, json};
use crate::{LogLevel, dbug, info, verb};
use serde_json::Value;
use std::path::PathBuf;
use walkdir::WalkDir;
use yaml_serde;
#[derive(Debug)]
pub struct Specification {
pub compiled: Value,
pub components: Components,
pub outpath: PathBuf,
pub tmplpath: PathBuf,
}
trait Pipe: Sized {
@@ -26,59 +24,55 @@ impl<T> Pipe for T {}
impl Specification {
pub fn build(
partitional: PathBuf,
regional: PathBuf,
common: PathBuf,
zonal: PathBuf,
device: PathBuf,
components: Components,
outpath: PathBuf,
tmplpath: PathBuf,
) -> Result<Self, Box<dyn std::error::Error>> {
let json_values: Vec<Value> = [&partitional, &regional, &common, &zonal, &device]
dbug!("{:#?}", components);
let merged_map: serde_json::Map<String, Value> = [
&components.partitional,
&components.regional,
&components.common,
&components.zonal,
&components.device,
]
.iter()
.filter_map(|path| std::fs::read_to_string(path).ok())
.map(|content| yaml_serde::from_str::<Value>(&content))
.collect::<Result<Vec<Value>, _>>()?;
let merged_map: Map<String, Value> = json_values
.collect::<Result<Vec<Value>, _>>()?
.into_iter()
.filter_map(|v| v.as_object().cloned())
.flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
.flatten()
.collect();
let compiled = json!({"data": merged_map});
let tmplpath = Self::get_templates(&components.device, &tmplpath);
Ok(Self {
compiled,
components: Components {
partitional,
regional,
common,
zonal,
device,
},
compiled: serde_json::json!({"data": merged_map}),
components,
outpath,
tmplpath,
})
}
pub fn get_layer(&self) -> String {
self.components
.device
fn get_templates(device: &PathBuf, tmpldir: &PathBuf) -> PathBuf {
tmpldir.join(
device
.parent()
.unwrap()
.file_name()
.and_then(|f| f.file_name())
.unwrap()
.to_string_lossy()
.chars()
.filter(|c| !c.is_numeric())
.collect::<String>()
.collect::<String>(),
)
}
/// compiles specifications from regex and paths provided by Args
pub fn compile(args: &Args) -> Vec<Specification> {
Self::get_specs(&args.env.spec_path)
.pipe(|p| {
info!(" Skyforge found {} renderable devices", p.len());
info!("Skyforge found {} renderable devices", p.len());
Self::filter_specs(&args.devices, p)
})
.tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices))
@@ -86,52 +80,56 @@ impl Specification {
.map(|spec| {
verb!(" | {}", spec.display());
let common = Self::get_common(&spec);
let regional = Self::get_regional(&spec);
let regional = Self::get_regional(&common);
Self::build(
Self::get_partitional(&common, &regional),
Components {
partitional: Self::get_partitional(&regional),
regional,
common,
Self::get_zonal(&spec),
spec,
zonal: Self::get_zonal(&spec),
device: spec,
},
args.env.out_path.clone(),
args.env.tmpl_path.clone(),
)
.expect("failed to build Specification")
})
.collect()
}
fn get_partitional(common: &PathBuf, regional: &PathBuf) -> PathBuf {
let re = Regex::new(r"partition: ([^\n\r$]+)").expect("failed");
[common, regional]
.iter()
.find_map(|path| {
std::fs::read_to_string(path).ok().and_then(|contents| {
re.captures(&contents).and_then(|captures| {
captures.get(1).map(|matched| {
PathBuf::from(format!("./spec/common/{}.yaml", matched.as_str().trim()))
})
})
})
})
.unwrap_or_else(|| PathBuf::from("./spec/common/default.yaml"))
fn get_partitional(regional: &PathBuf) -> PathBuf {
let basedir = regional.ancestors().nth(3).unwrap().join("common");
match yaml_serde::from_str::<yaml_serde::Value>(
&std::fs::read_to_string(regional).unwrap_or_default(),
)
.unwrap_or_default()["regional"]["partition"]
.as_str()
{
Some(partition) => basedir.join(format!("{partition}.yaml")),
None => {
verb!("[?] {} missing regional.partition", regional.display());
basedir.join("default.yaml")
}
}
}
fn get_regional(path: &PathBuf) -> PathBuf {
let dir_name = path
.parent()
.unwrap()
.file_name()
.unwrap()
.to_string_lossy()
.into_owned();
path.parent()
.unwrap()
.parent()
.unwrap()
.join("common")
.join(format!("{}.yaml", &dir_name[0..2]))
fn get_regional(common: &PathBuf) -> PathBuf {
let basedir = common.ancestors().nth(2).unwrap().join("common");
match yaml_serde::from_str::<yaml_serde::Value>(
&std::fs::read_to_string(common).unwrap_or_default(),
)
.unwrap_or_default()["common"]["fabric"]
.as_str()
{
Some(fabric) => basedir.join(format!("{fabric}.yaml")),
None => {
verb!(
"[?] {} missing common.fabric, falling back to regional default",
common.display()
);
basedir.join("default.yaml")
}
}
}
fn get_common(spec: &PathBuf) -> PathBuf {
@@ -139,17 +137,17 @@ impl Specification {
}
fn get_zonal(spec: &PathBuf) -> PathBuf {
let zone = spec
.file_name()
spec.parent().unwrap().join(format!(
"{}.common.yaml",
spec.file_name()
.and_then(|f| f.to_str())
.and_then(|f| f.split_once('-'))
.map(|(zone, _)| spec.with_file_name(zone))
.unwrap_or_else(|| spec.with_file_name("common.yaml"));
PathBuf::from(format!("{}.common.yaml", zone.to_str().unwrap()))
.map(|(zone, _)| zone)
.unwrap_or("none")
))
}
fn filter_specs(pattern: &Regex, spec_list: Vec<PathBuf>) -> Vec<PathBuf> {
fn filter_specs(pattern: &regex::Regex, spec_list: Vec<PathBuf>) -> Vec<PathBuf> {
spec_list
.into_iter()
.filter(|spec| {
@@ -163,7 +161,10 @@ impl Specification {
fn get_specs(spec_path: &PathBuf) -> Vec<PathBuf> {
let mut result = Vec::new();
let root = spec_path.as_path();
for entry in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
for entry in walkdir::WalkDir::new(root)
.into_iter()
.filter_map(|e| e.ok())
{
if let Ok(path) = entry.path().strip_prefix(root) {
let path_str = path.to_string_lossy().to_string();
if !path_str.contains("common") && path_str.ends_with("yaml") {
@@ -179,8 +180,8 @@ impl std::fmt::Display for Specification {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
"Compiled Specification: {}\nComponents: {}",
self.compiled, self.components,
"| Compiled Specification: {}\n| Components: {:#?}\n| Outpath: {:?}\n",
self.compiled, self.components, self.outpath,
)
}
}
@@ -203,17 +204,3 @@ impl Components {
.to_string()
}
}
impl std::fmt::Display for Components {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
" | Partitional: {}\n | Regional: {}\n | Common: {}\n | Zonal: {}\n | Device: {}\n",
self.partitional.display(),
self.regional.display(),
self.common.display(),
self.zonal.display(),
self.device.display(),
)
}
}

View File

@@ -1,4 +1,4 @@
use crate::{LogLevel, crit, info};
use crate::{LogLevel, crit, verb};
use serde::Deserialize;
use std::error::Error;
use std::path::PathBuf;
@@ -17,18 +17,18 @@ impl Structure {
Ok(yaml_serde::from_str(&data)?)
}
pub fn load_template_data(&self, tmpl_dir: PathBuf) -> Vec<(String, String)> {
pub fn load_template_data(&self, tmpl_dir: &PathBuf) -> Vec<(String, String)> {
self.files
.iter()
.filter_map(|name| {
let template = tmpl_dir.join(format!("{}.tera", name));
match std::fs::read_to_string(&template) {
Ok(content) => {
info!(" | {}", template.display());
verb!(" | {}", template.display());
Some((name.clone(), content))
}
Err(e) => {
crit!("[!] {}: {e}", template.display());
crit!("[!] {:?}: {e}", template);
None
}
}