additional error handling; use path buffers; improve logic

This commit is contained in:
lost
2026-02-16 02:03:04 -07:00
parent 31311ee99e
commit 4177df0f4a
37 changed files with 178 additions and 267 deletions

136
Cargo.lock generated
View File

@@ -76,6 +76,12 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "anyhow"
version = "1.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea"
[[package]]
name = "autocfg"
version = "1.4.0"
@@ -269,25 +275,9 @@ dependencies = [
[[package]]
name = "equivalent"
version = "1.0.1"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "generic-array"
@@ -307,19 +297,7 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
dependencies = [
"cfg-if",
"libc",
"wasi 0.13.3+wasi-0.2.2",
"windows-targets",
"wasi",
]
[[package]]
@@ -348,9 +326,9 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.15.2"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "humansize"
@@ -402,9 +380,9 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.7.1"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown",
@@ -450,18 +428,6 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
[[package]]
name = "libyml"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e281a65eeba3d4503a2839252f86374528f9ceafe6fed97c1d3b52e1fb625c1"
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "log"
version = "0.4.25"
@@ -641,7 +607,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
"getrandom",
]
[[package]]
@@ -673,19 +639,6 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustversion"
version = "1.0.19"
@@ -739,23 +692,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_yml"
version = "0.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ce6afeda22f0b55dde2c34897bce76a629587348480384231205c14b59a01f"
dependencies = [
"indexmap",
"itoa",
"libyml",
"log",
"memchr",
"ryu",
"serde",
"serde_json",
"tempfile",
]
[[package]]
name = "sha2"
version = "0.10.8"
@@ -783,14 +719,15 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
name = "skyforge"
version = "0.1.0"
dependencies = [
"anyhow",
"clap",
"regex",
"serde",
"serde_json",
"serde_yml",
"tera",
"thiserror 1.0.69",
"walkdir",
"yaml_serde",
]
[[package]]
@@ -820,20 +757,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91"
dependencies = [
"cfg-if",
"fastrand",
"getrandom 0.3.1",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "tera"
version = "1.20.0"
@@ -964,6 +887,12 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034"
[[package]]
name = "unsafe-libyaml"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
[[package]]
name = "utf8parse"
version = "0.2.2"
@@ -992,15 +921,6 @@ version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.13.3+wasi-0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
dependencies = [
"wit-bindgen-rt",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
@@ -1151,12 +1071,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen-rt"
version = "0.33.0"
name = "yaml_serde"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
checksum = "4a7f5270edc6fab0529a772a772b3e505dfd883a8de5cc5b464e35fabe586411"
dependencies = [
"bitflags",
"indexmap",
"itoa",
"ryu",
"serde",
"unsafe-libyaml",
]
[[package]]

View File

@@ -1,14 +1,15 @@
[package]
name = "skyforge"
version = "0.1.0"
edition = "2021"
edition = "2024"
[dependencies]
clap = { version = "4.5.11" }
regex = "1.10.5"
serde = { version = "1.0", features = ["derive"] }
serde_yml = "0.0.10"
serde_json = "1.0.112"
tera = "1.20.0"
thiserror = "1.0"
walkdir = "2.3"
anyhow = "1"
clap = { version = "4.5" }
regex = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
yaml_serde = "0"
tera = "1"
thiserror = "1"
walkdir = "2"

View File

@@ -0,0 +1,5 @@
common:
partition: us
layer: ex-core-r
protocol: ospf
uplink: et-0/0/36

View File

@@ -0,0 +1,2 @@
zonal:
zone: 1

View File

@@ -0,0 +1,2 @@
zonal:
zone: 2

View File

@@ -0,0 +1,4 @@
common:
layer: ex-edge-r
protocol: bgp
uplink: et-0/0/0

View File

@@ -0,0 +1,2 @@
zonal:
zone: 1

View File

@@ -0,0 +1,2 @@
zonal:
zone: 2

View File

@@ -0,0 +1,8 @@
device:
zone: 5
hostname: xyz5-ex-edge-r101
uplinks:
- xe-0/0/1
- xe-0/0/2
- xe-0/0/3
- xe-0/0/4

View File

@@ -1,4 +0,0 @@
common:
layer: ex-edge-r
protocol: bgp
uplink: et-0/0/0

View File

@@ -1,2 +0,0 @@
zonal:
zone: 1

View File

@@ -1,2 +0,0 @@
zonal:
zone: 2

View File

@@ -1,5 +0,0 @@
common:
partition: eu
layer: ex-edge-r
protocol: ospf
uplink: et-0/0/36

View File

@@ -1,2 +0,0 @@
zonal:
az: 1

View File

@@ -1,2 +0,0 @@
zonal:
az: 2

1
demo/tmpl/ex-core-r Symbolic link
View File

@@ -0,0 +1 @@
ex-edge-r/

View File

@@ -1,5 +0,0 @@
chassis {
users {
{{ username }};
}
}

View File

@@ -0,0 +1,8 @@
interfaces {
{%- set interfaces = uplinks | default(value=[uplink]) %}
{%- for interace in interfaces %}
{{ interace }} {
unit 0 family inet dhcp;
}
{%- endfor %}
}

View File

@@ -1,4 +0,0 @@
interfaces {
{{ uplink }} {
}
}

View File

@@ -0,0 +1,3 @@
protocols {
{{ protocol }};
}

View File

@@ -1,4 +0,0 @@
protocols {
{{ protocol }} {
}
}

View File

@@ -1,5 +1,4 @@
files:
- system
- chassis
- interfaces
- protocols

View File

@@ -1,4 +1,8 @@
system {
hostname {{ hostname }};
location "{{ location }}";
zone {{ zone }};
users {
{{ username }};
}
}

View File

@@ -1 +0,0 @@
../common/ex/system.tmpl

View File

@@ -41,29 +41,21 @@ impl fmt::Display for EnvVars {
/// loads `command line arguments` and `environment variables` using custom `clap`
pub fn parse_args() -> Args {
let matches: clap::ArgMatches = build().get_matches();
let raw_devices = matches.get_one::<String>("devices").unwrap().to_string();
let devices: Regex =
Regex::new(&raw_devices).expect("Invalid regex pattern provided for `devices`");
let loglevel: LogLevel = match matches.get_flag("debug") {
true => LogLevel::Debug,
false => match matches.get_flag("verbose") {
true => LogLevel::Verbose,
false => LogLevel::Info,
},
};
let env: EnvVars = parse_env();
let raw_devices = matches.get_one::<String>("devices").unwrap();
Args {
devices,
loglevel,
env,
devices: Regex::new(&raw_devices).expect("Invalid regex pattern provided for `devices`"),
loglevel: if matches.get_flag("debug") {
LogLevel::Debug
} else if matches.get_flag("verbose") {
LogLevel::Verbose
} else {
LogLevel::Info
},
env: parse_env(),
}
}
/// loads only environment variables
/// loads environment variables
pub fn parse_env() -> EnvVars {
EnvVars {
spec_path: match std::env::var("SF_SPEC_PATH") {
@@ -87,10 +79,10 @@ pub fn parse_env() -> EnvVars {
const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
const ENV_MSG: &str = r#"Environment:
SF_SPEC_PATH Path to the directory containing templates. Defaults to "./spec".
SF_TMPL_PATH Path to the directory containing specifications. Defaults to "./tmpl".
SF_OUT_PATH Path to the directory for command output. Defaults to "./out".
SF_LOG_PATH Path to the directory for log output. Defaults to "./log".
SF_SPEC_PATH Directory containing templates
SF_TMPL_PATH Directory containing specifications
SF_OUT_PATH Directory for command output
SF_LOG_PATH Directory for log output
"#;
/// builds a custom command line argument parser

View File

@@ -4,57 +4,51 @@ mod specs;
mod tmpls;
use log::LogLevel;
use serde_yml;
use std::fs::{self, create_dir_all, write, OpenOptions};
use std::fs::{self, File, OpenOptions, create_dir_all, write};
use std::io::Write;
use std::path::Path;
use tmpls::RenderedConfig;
use yaml_serde;
fn main() {
let args: cli::Args = cli::parse_args();
let dbg: LogLevel = args.loglevel;
dbug!(dbg, "{}", &args);
dbug!(dbg, "{:#?}", &args);
let specifications: Vec<specs::Specification> =
specs::compile(&args.devices, &args.env.spec_path, dbg);
for spec in specifications {
let result = tmpls::process_templates(&spec, dbg).ok().unwrap();
output_rendered_configs(result, dbg)
output_rendered_configs(result, &args.env.out_path, dbg)
}
}
fn output_rendered_configs(rendered_config: RenderedConfig, dbg: LogLevel) {
fn output_rendered_configs(rendered_config: RenderedConfig, outdir: &str, dbg: LogLevel) {
info!(dbg, "Writing Output");
let path: String = format!("out/{}", rendered_config.hostname);
if Path::new(&path).exists() {
fs::remove_dir_all(&path).ok();
let out_path = Path::new(outdir).join(rendered_config.hostname);
if out_path.exists() {
fs::remove_dir_all(&out_path).ok();
}
create_dir_all(&path).ok();
let rendered_config_path = format!("{}/all.conf", path);
create_dir_all(&out_path).ok();
let merged_config_path = out_path.join("all.conf");
let mut all_file: File = OpenOptions::new()
.create(true)
.append(true)
.open(&merged_config_path)
.expect("unable to open");
for rendered_template in rendered_config.configs {
let outpath = format!("{}/{}.tmpl", path, rendered_template.0);
verb!(dbg, " | {}", &outpath);
write(&outpath, &rendered_template.1).ok();
append_to_file(&rendered_config_path, rendered_template.1).ok();
let template_path = out_path.join(rendered_template.0 + ".tera");
verb!(dbg, " | {}", &template_path.display());
write(&template_path, &rendered_template.1).ok();
all_file.write_all(&rendered_template.1.as_bytes()).ok();
}
let outpath = format!("{}/compiled.spec", path);
let spec: String = match serde_yml::to_string(&rendered_config.spec) {
Ok(yaml) => yaml,
Err(e) => {
eprintln!("Failed to convert to YAML: {}", e);
String::new()
}
};
verb!(dbg, " | {}", &outpath);
write(&outpath, spec).ok();
info!(dbg, " | {} ", &rendered_config_path);
}
fn append_to_file(path: &str, content: String) -> std::io::Result<()> {
let mut file = OpenOptions::new().create(true).append(true).open(path)?;
write!(file, "{}", content)?;
Ok(())
let spec: String = yaml_serde::to_string(&rendered_config.spec).unwrap();
let compiled_spec_path = out_path.join("compiled-spec.yaml");
verb!(dbg, " | {}", &compiled_spec_path.display());
write(&compiled_spec_path, spec).ok();
info!(dbg, " | {} ", &merged_config_path.display());
}

View File

@@ -1,11 +1,11 @@
use crate::{dbug, info, verb, LogLevel};
use crate::{LogLevel, info, verb};
use regex::Regex;
use serde_json::{json, Map, Value};
use serde_yml;
use serde_json::{Map, Value, json};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::{env, fmt, fs};
use walkdir::WalkDir;
use yaml_serde;
#[derive(Debug)]
pub struct Specification {
@@ -19,37 +19,25 @@ pub struct Specification {
impl Specification {
pub fn build(
partitional: &str,
regional: &str,
common: &str,
zonal: &str,
device: &str,
partitional: String,
regional: String,
common: String,
zonal: String,
device: String,
) -> Result<Self, Box<dyn std::error::Error>> {
// Read all YAML files
let yaml_contents: Vec<String> = [partitional, regional, common, zonal, device]
let json_values: Vec<Value> = [&partitional, &regional, &common, &zonal, &device]
.iter()
.filter_map(|path| fs::read_to_string(path).ok())
.collect();
// Convert each YAML to JSON Value and collect them
let json_values: Vec<Value> = yaml_contents
.iter()
.map(|content| serde_yml::from_str(content))
.map(|content| yaml_serde::from_str::<Value>(&content))
.collect::<Result<Vec<Value>, _>>()?;
// Merge all objects under a single key
let mut merged_map = Map::new();
for value in json_values {
if let Some(obj) = value.as_object() {
for (_key, value) in obj {
if let Some(inner_obj) = value.as_object() {
for (k, v) in inner_obj {
merged_map.insert(k.clone(), v.clone());
}
}
}
}
}
let merged_map: Map<String, Value> = json_values
.into_iter()
.filter_map(|v| v.as_object().cloned())
.flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
.flatten()
.collect();
// Create the final merged object
let compiled = json!({"data": merged_map});
@@ -75,27 +63,13 @@ impl Specification {
.filter(|c| !c.is_numeric())
.collect::<String>()
}
pub fn get_hostname(&self) -> String {
PathBuf::from(&self.device)
.file_name()
.unwrap()
.to_string_lossy()
.into_owned()
}
}
impl fmt::Display for Specification {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"Specification:\n\
Partitional: {}\n\
Regional: {}\n\
Common: {}\n\
Zonal: {}\n\
Device: {}\n\
Compiled:\n{}",
"Specification:\nPartitional: {}\nRegional: {}\nCommon: {}\nZonal: {}\nDevice: {}\nCompiled:\n{}",
self.partitional, self.regional, self.common, self.zonal, self.device, self.compiled
)
}
@@ -119,20 +93,8 @@ pub fn compile(pattern: &Regex, spec_path: &String, dbg: LogLevel) -> Vec<Specif
let regional: String = get_regional(&spec);
let partitional: String = get_partional(&common, &regional);
specifications.push(
match Specification::build(&partitional, &regional, &common, &zonal, &spec) {
Ok(compiled_spec) => {
dbug!(
dbg,
"Compiled Spec for '{}'\n | {}\n | {}\n | {}\n | {}\n | {}",
compiled_spec.get_hostname(),
compiled_spec.partitional,
compiled_spec.regional,
compiled_spec.common,
compiled_spec.zonal,
compiled_spec.device
);
compiled_spec
}
match Specification::build(partitional, regional, common, zonal, spec) {
Ok(compiled_spec) => compiled_spec,
Err(e) => panic!("failed to build Specification: {}", e),
},
)

View File

@@ -1,9 +1,9 @@
use crate::specs::Specification;
use crate::{info, verb, LogLevel};
use crate::{LogLevel, info, verb};
use serde_json::Value;
use serde_yml;
use std::fs;
use tera::{Context, Tera};
use yaml_serde;
#[derive(serde::Deserialize)]
struct TemplateConfig {
@@ -40,8 +40,17 @@ pub fn process_templates(
// Read structure.yaml
let structure_path = format!("{}/structure.yaml", base_dir);
let structure_content = fs::read_to_string(&structure_path)?;
let config: TemplateConfig = serde_yml::from_str(&structure_content)?;
let structure_content: String = match fs::read_to_string(&structure_path) {
Ok(content) => content,
Err(e) => {
eprintln!(
"while attempting to read {}: {}, skipping",
&structure_path, e
);
String::new()
}
};
let config: TemplateConfig = yaml_serde::from_str(&structure_content)?;
// Initialize Tera with the specific directory
let mut tera = Tera::default();
@@ -51,15 +60,35 @@ pub fn process_templates(
let mut configs = Vec::new();
for template_name in config.files {
// Read the template file directly
let template_path = format!("{}/{}.tmpl", base_dir, template_name);
let template_path = format!("{}/{}.tera", base_dir, template_name);
verb!(dbg, " | {}", &template_path);
let template_content = fs::read_to_string(&template_path)?;
let template_content = match fs::read_to_string(&template_path) {
Ok(content) => content,
Err(e) => {
eprintln!(
"while attempting to read {}: {}, skipping",
&template_path, e
);
String::new()
}
};
// Add this specific template to Tera
tera.add_raw_template(&template_name, &template_content)?;
// Render the template
let rendered = tera.render(&template_name, &context)?;
let rendered = tera
.render(&template_name, &context)
.inspect_err(|e| {
let mut chain = format!("{}", e);
let mut next_source = std::error::Error::source(e);
while let Some(source) = next_source {
chain.push_str(&format!(" -> {}", source));
next_source = source.source();
}
eprintln!("[tera] {}", chain);
})
.unwrap_or_default();
configs.push((String::from(&template_name), rendered));
}