refactor: modularize codebase and improve type safety

Cargo.lock

@@ -717,7 +717,7 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
 
 [[package]]
 name = "skyforge"
-version = "0.1.0"
+version = "0.1.112"
 dependencies = [
  "anyhow",
  "clap",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "skyforge"
-version = "0.1.0"
+version = "0.1.112"
 edition = "2024"
 
 [dependencies]

README.md

@@ -47,10 +47,10 @@ Options:
   -V, --version            Print version
 
 Environment:
-    SF_SPEC_PATH    Path to the directory containing templates.        Defaults to "./spec".
+    SF_SPEC_PATH    Directory containing templates
-    SF_TMPL_PATH    Path to the directory containing specifications.   Defaults to "./tmpl".
+    SF_TMPL_PATH    Directory containing specifications
-    SF_OUT_PATH     Path to the directory for command output.          Defaults to "./out".
+    SF_OUT_PATH     Directory for command output
-    SF_LOG_PATH     Path to the directory for log output.              Defaults to "./log".
+    SF_LOG_PATH     Directory for log output
 ```
 
 ### Standard

demo/spec/xyz/ex-core-r/xyz2-ex-core-r102.yaml

@@ -1,2 +1,2 @@
 device:
-  hostname: xyz2-ex-core-r202
+  hostname: xyz2-ex-core-r102

src/cli.rs

@@ -1,81 +1,7 @@
 use crate::log::LogLevel;
 use clap::{Arg, ArgAction, ArgGroup, Command};
 use regex::Regex;
-use std::fmt;
+use std::path::PathBuf;
-
-#[derive(Debug)]
-pub struct Args {
-    pub devices: Regex,
-    pub loglevel: LogLevel,
-    pub env: EnvVars,
-}
-
-impl fmt::Display for Args {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "devices: {}, loglevel: {}, env: {}",
-            self.devices, self.loglevel, self.env
-        )
-    }
-}
-
-#[derive(Debug)]
-pub struct EnvVars {
-    pub spec_path: String,
-    pub tmpl_path: String,
-    pub out_path: String,
-    pub log_path: String,
-}
-
-impl fmt::Display for EnvVars {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "spec_path: {}, tmpl_path: {}, out_path: {}, log_path: {}",
-            self.spec_path, self.tmpl_path, self.out_path, self.log_path
-        )
-    }
-}
-
-/// loads `command line arguments` and `environment variables` using custom `clap`
-pub fn parse_args() -> Args {
-    let matches: clap::ArgMatches = build().get_matches();
-    let raw_devices = matches.get_one::<String>("devices").unwrap();
-    Args {
-        devices: Regex::new(&raw_devices).expect("Invalid regex pattern provided for `devices`"),
-        loglevel: if matches.get_flag("debug") {
-            LogLevel::Debug
-        } else if matches.get_flag("verbose") {
-            LogLevel::Verbose
-        } else {
-            LogLevel::Info
-        },
-        env: parse_env(),
-    }
-}
-
-/// loads environment variables
-pub fn parse_env() -> EnvVars {
-    EnvVars {
-        spec_path: match std::env::var("SF_SPEC_PATH") {
-            Ok(path) => path.trim_end_matches('/').to_string(),
-            Err(_) => String::from("./spec"),
-        },
-        tmpl_path: match std::env::var("SF_TMPL_PATH") {
-            Ok(path) => path.trim_end_matches('/').to_string(),
-            Err(_) => String::from("./tmpl"),
-        },
-        out_path: match std::env::var("SF_OUT_PATH") {
-            Ok(path) => path.trim_end_matches('/').to_string(),
-            Err(_) => String::from("./out"),
-        },
-        log_path: match std::env::var("SF_LOG_PATH") {
-            Ok(path) => path.trim_end_matches('/').to_string(),
-            Err(_) => String::from("./log"),
-        },
-    }
-}
 
 const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
 const ENV_MSG: &str = r#"Environment:
@@ -85,9 +11,33 @@ const ENV_MSG: &str = r#"Environment:
     SF_LOG_PATH     Directory for log output
 "#;
 
-/// builds a custom command line argument parser
+#[derive(Debug)]
-fn build() -> Command {
+pub struct Args {
-    Command::new("app")
+    pub devices: Regex,
+    pub loglevel: LogLevel,
+    pub env: EnvVars,
+}
+
+impl Args {
+    /// loads command line arguments and environment variables
+    pub fn parse() -> Args {
+        let matches: clap::ArgMatches = Self::parser().get_matches();
+        let raw_devices = matches.get_one::<String>("devices").unwrap();
+        Args {
+            devices: Regex::new(&raw_devices)
+                .expect("Invalid regex pattern provided for `devices`"),
+            loglevel: if matches.get_flag("debug") {
+                LogLevel::Debug
+            } else if matches.get_flag("verbose") {
+                LogLevel::Verbose
+            } else {
+                LogLevel::Info
+            },
+            env: EnvVars::parse(),
+        }
+    }
+    fn parser() -> Command {
+        Command::new("skyforge")
             .about(ABOUT_MSG)
             .version(env!("CARGO_PKG_VERSION"))
             .author("rskntroot")
@@ -120,3 +70,31 @@ fn build() -> Command {
             )
             .after_help(ENV_MSG)
     }
+}
+
+#[derive(Debug)]
+pub struct EnvVars {
+    pub spec_path: PathBuf,
+    pub tmpl_path: PathBuf,
+    pub out_path: PathBuf,
+    pub log_path: PathBuf,
+}
+
+impl EnvVars {
+    pub fn parse() -> Self {
+        Self {
+            spec_path: std::env::var("SF_SPEC_PATH")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| PathBuf::from("./spec")),
+            tmpl_path: std::env::var("SF_TMPL_PATH")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| PathBuf::from("./tmpl")),
+            out_path: std::env::var("SF_OUT_PATH")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| PathBuf::from("./out")),
+            log_path: std::env::var("SF_LOG_PATH")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| PathBuf::from("./log")),
+        }
+    }
+}

src/lib.rs

@@ -0,0 +1,10 @@
+pub mod cli;
+pub mod log;
+pub mod render;
+pub mod spec;
+pub mod tmpl;
+
+pub use cli::Args;
+pub use log::LogLevel;
+pub use render::DeviceConfigBundle;
+pub use spec::Specification;

src/main.rs

@@ -1,27 +1,11 @@
-mod cli;
+use skyforge::{Args, DeviceConfigBundle, Specification};
-mod log;
-mod render;
-mod spec;
-mod tmpl;
 
-use log::LogLevel;
+fn main() -> Result<(), Box<dyn std::error::Error>> {
-use render::RenderedConfig;
+    let args = Args::parse();
 
-fn main() {
+    for spec in Specification::compile(&args) {
-    let args: cli::Args = cli::parse_args();
+        DeviceConfigBundle::from_spec(spec)?.output_artifacts();
-    let dbg: LogLevel = args.loglevel;
-
-    dbug!(dbg, "{:#?}", &args);
-
-    let specifications: Vec<spec::Specification> =
-        spec::compile(&args.devices, &args.env.spec_path, dbg);
-
-    for spec in specifications {
-        RenderedConfig::from_spec(&spec, dbg)
-            .unwrap_or_else(|e| {
-                eprintln!("{}", e);
-                std::process::exit(1);
-            })
-            .output(&args.env.out_path, dbg);
     }
+
+    Ok(())
 }

src/render.rs

@@ -1,13 +1,19 @@
+use crate::log::LogLevel;
 use crate::spec::Specification;
 use crate::tmpl;
-use crate::{LogLevel, info, verb};
+use crate::{dbug, info, verb};
-use serde_json::Value;
 use std::io::Write;
 
-pub struct RenderedConfig {
+pub struct DeviceConfigBundle {
     pub hostname: String,
+    pub configs: Vec<ConfigVariant>,
+    pub spec: Specification,
+    pub platform: String,
+}
+
+pub struct ConfigVariant {
+    pub name: String,
     pub configs: Vec<Configuration>,
-    pub spec: Value,
 }
 
 pub struct Configuration {
@@ -15,15 +21,12 @@ pub struct Configuration {
     pub data: String,
 }
 
-impl RenderedConfig {
+impl DeviceConfigBundle {
-    pub fn from_spec(
+    pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn std::error::Error>> {
-        spec: &Specification,
+        let mut context = Self::get_context(&spec);
-        dbg: LogLevel,
-    ) -> Result<Self, Box<dyn std::error::Error>> {
-        let context = Self::get_context(&spec);
 
         let hostname = Self::get_hostname(&context, &spec);
-        info!(dbg, "Rendering {}", hostname);
+        info!(&spec.loglevel, "Rendering {}", hostname);
 
         let base_dir = std::path::Path::new("./tmpl").join(spec.get_layer());
         let structure_path = base_dir.join("structure.yaml");
@@ -31,28 +34,38 @@ impl RenderedConfig {
             .map_err(|e| format!("Failed to parse {}: {}", structure_path.display(), e))?;
 
         let mut renderer = tera::Tera::default();
-        renderer.add_raw_templates(structure.load_template_data(&base_dir))?;
+        renderer.add_raw_templates(structure.load_template_data(&base_dir, spec.loglevel))?;
+        dbug!(&spec.loglevel, "Processing templates for {}", hostname);
 
+        let mut config_variants = Vec::new();
+        for variant in &structure.variations {
+            context.insert(variant, &true);
             let mut configs = Vec::new();
             for template_name in &structure.files {
-            verb!(dbg, " | {}", &template_name);
+                dbug!(&spec.loglevel, " | {}.{}", &template_name, variant);
                 if !renderer.get_template_names().any(|n| n == template_name) {
                     continue;
                 }
-
                 match renderer.render(template_name, &context) {
-                Ok(rendered) => configs.push(Configuration {
+                    Ok(data) => configs.push(Configuration {
                         name: template_name.clone(),
-                    data: rendered,
+                        data,
                     }),
                     Err(e) => Self::log_tera_error(template_name, &e),
                 }
             }
+            config_variants.push(ConfigVariant {
+                name: String::from(variant),
+                configs,
+            });
+            context.remove(variant);
+        }
 
         Ok(Self {
             hostname,
-            configs,
+            configs: config_variants,
-            spec: spec.compiled.clone(),
+            spec,
+            platform: structure.platform,
         })
     }
 
@@ -61,13 +74,7 @@ impl RenderedConfig {
             .get("hostname")
             .and_then(|v| v.as_str())
             .map(|s| s.to_string())
-            .unwrap_or_else(|| {
+            .unwrap_or_else(|| spec.components.get_hostname())
-                spec.device
-                    .file_stem()
-                    .and_then(|n| n.to_str())
-                    .unwrap_or("unknown")
-                    .to_string()
-            })
     }
 
     fn get_context(spec: &Specification) -> tera::Context {
@@ -90,33 +97,43 @@ impl RenderedConfig {
         eprintln!("[tera] {}: {}", name, chain);
     }
 
-    pub fn output(self: Self, outdir: &str, dbg: LogLevel) {
+    pub fn output_artifacts(self: Self) {
-        info!(dbg, "Writing Output:");
+        info!(&self.spec.loglevel, "Writing Output:");
 
-        let out_path = std::path::Path::new(outdir).join(self.hostname);
+        let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
-        if out_path.exists() {
+        if device_outpath.exists() {
-            std::fs::remove_dir_all(&out_path).ok();
+            std::fs::remove_dir_all(&device_outpath).ok();
         }
+        std::fs::create_dir_all(&device_outpath).ok();
+
+        for variant_configs in &self.configs {
+            let out_path = device_outpath.join(&variant_configs.name);
             std::fs::create_dir_all(&out_path).ok();
 
-        let merged_config_path = out_path.join("all.conf");
+            let merged_config_path =
+                device_outpath.join(format!("all.{}.{}", variant_configs.name, &self.platform));
             let mut all_file: std::fs::File = std::fs::OpenOptions::new()
                 .create(true)
                 .append(true)
                 .open(&merged_config_path)
                 .expect("unable to open");
 
-        for config in &self.configs {
+            for config in &variant_configs.configs {
-            let template_path = out_path.join(format!("{}.tera", &config.name));
+                let config_outpath = out_path.join(format!("{}.{}", &config.name, &self.platform));
-            verb!(dbg, " | {}", &template_path.display());
+                std::fs::create_dir_all(&config_outpath).ok();
-            std::fs::write(&template_path, &config.data).ok();
+                verb!(&self.spec.loglevel, " | {}", &config_outpath.display());
+                std::fs::write(&config_outpath, &config.data).ok();
                 all_file.write_all(&config.data.as_bytes()).ok();
             }
+            info!(&self.spec.loglevel, " | {} ", &merged_config_path.display());
+        }
 
-        let spec: String = yaml_serde::to_string(&self.spec).unwrap();
+        let compiled_spec_path = device_outpath.join("context.yaml");
-        let compiled_spec_path = out_path.join("compiled-spec.yaml");
+        verb!(&self.spec.loglevel, " | {}", &compiled_spec_path.display());
-        verb!(dbg, " | {}", &compiled_spec_path.display());
+        std::fs::write(
-        std::fs::write(&compiled_spec_path, spec).ok();
+            &compiled_spec_path,
-        info!(dbg, " | {} ", &merged_config_path.display());
+            yaml_serde::to_string(&self.spec.compiled).unwrap(),
+        )
+        .ok();
     }
 }

src/spec.rs

@@ -1,19 +1,17 @@
+use crate::cli::Args;
 use crate::{LogLevel, info, verb};
 use regex::Regex;
 use serde_json::{Map, Value, json};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
-use std::{env, fmt, fs};
 use walkdir::WalkDir;
 use yaml_serde;
 
 #[derive(Debug)]
 pub struct Specification {
-    pub partitional: PathBuf,
-    pub regional: PathBuf,
-    pub common: PathBuf,
-    pub zonal: PathBuf,
-    pub device: PathBuf,
     pub compiled: Value,
+    pub components: Components,
+    pub loglevel: LogLevel,
+    pub outpath: PathBuf,
 }
 
 impl Specification {
@@ -23,10 +21,12 @@ impl Specification {
         common: PathBuf,
         zonal: PathBuf,
         device: PathBuf,
+        loglevel: LogLevel,
+        outpath: PathBuf,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let json_values: Vec<Value> = [&partitional, &regional, &common, &zonal, &device]
             .iter()
-            .filter_map(|path| fs::read_to_string(path).ok())
+            .filter_map(|path| std::fs::read_to_string(path).ok())
             .map(|content| yaml_serde::from_str::<Value>(&content))
             .collect::<Result<Vec<Value>, _>>()?;
 
@@ -40,17 +40,22 @@ impl Specification {
         let compiled = json!({"data": merged_map});
 
         Ok(Self {
+            compiled,
+            components: Components {
                 partitional,
                 regional,
                 common,
                 zonal,
                 device,
-            compiled,
+            },
+            loglevel,
+            outpath,
         })
     }
 
     pub fn get_layer(&self) -> String {
-        self.device
+        self.components
+            .device
             .parent()
             .unwrap()
             .file_name()
@@ -60,48 +65,35 @@ impl Specification {
             .filter(|c| !c.is_numeric())
             .collect::<String>()
     }
-}
 
-impl fmt::Display for Specification {
+    /// compiles specifications from regex and paths provided by Args
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    pub fn compile(args: &Args) -> Vec<Specification> {
-        write!(
+        let spec_list: Vec<PathBuf> = Self::filter_specs(
-            f,
+            &args.devices,
-            "Specification:\nPartitional: {}\nRegional: {}\nCommon: {}\nZonal: {}\nDevice: {}\nCompiled:\n{}",
+            Self::get_specs(&args.env.spec_path, args.loglevel),
-            self.partitional.display(),
+        );
-            self.regional.display(),
-            self.common.display(),
-            self.zonal.display(),
-            self.device.display(),
-            self.compiled
-        )
-    }
-}
-
-pub fn compile(pattern: &Regex, spec_path: &String, dbg: LogLevel) -> Vec<Specification> {
-    let spec_list: Vec<PathBuf> = filter_specs(pattern, get_specs(spec_path, dbg));
         info!(
-        dbg,
+            &args.loglevel,
             "Matched {} devices against '{}'",
             &spec_list.len(),
-        &pattern
+            &args.devices
         );
-    for spec in &spec_list {
+        spec_list
-        verb!(dbg, " | {}", spec.display());
+            .into_iter()
-    }
+            .map(|spec| {
-    let mut specifications: Vec<Specification> = Vec::new();
+                verb!(&args.loglevel, " | {}", spec.display());
-    for spec in spec_list {
+                Self::build(
-        let zonal = get_zonal(&spec);
+                    Self::get_partitional(&Self::get_common(&spec), &Self::get_regional(&spec)),
-        let common = get_common(&spec);
+                    Self::get_regional(&spec),
-        let regional = get_regional(&spec);
+                    Self::get_common(&spec),
-        let partitional = get_partitional(&common, &regional);
+                    Self::get_zonal(&spec),
-        specifications.push(
+                    spec,
-            match Specification::build(partitional, regional, common, zonal, spec) {
+                    args.loglevel,
-                Ok(compiled_spec) => compiled_spec,
+                    std::path::PathBuf::from(&args.env.out_path),
-                Err(e) => panic!("failed to build Specification: {}", e),
-            },
                 )
-    }
+                .unwrap_or_else(|e| panic!("failed to build Specification: {}", e))
-    specifications
+            })
+            .collect()
     }
 
     fn get_partitional(common: &PathBuf, regional: &PathBuf) -> PathBuf {
@@ -110,7 +102,7 @@ fn get_partitional(common: &PathBuf, regional: &PathBuf) -> PathBuf {
         [common, regional]
             .iter()
             .find_map(|path| {
-            fs::read_to_string(path).ok().and_then(|contents| {
+                std::fs::read_to_string(path).ok().and_then(|contents| {
                     re.captures(&contents).and_then(|captures| {
                         captures.get(1).map(|matched| {
                             PathBuf::from(format!("./spec/common/{}.yaml", matched.as_str().trim()))
@@ -164,9 +156,9 @@ fn filter_specs(pattern: &Regex, spec_list: Vec<PathBuf>) -> Vec<PathBuf> {
             .collect()
     }
 
-fn get_specs(spec_path: &String, dbg: LogLevel) -> Vec<PathBuf> {
+    fn get_specs(spec_path: &PathBuf, dbg: LogLevel) -> Vec<PathBuf> {
         let mut result = Vec::new();
-    let root = Path::new(&spec_path);
+        let root = spec_path.as_path();
         for entry in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
             if let Ok(path) = entry.path().strip_prefix(root) {
                 let path_str = path.to_string_lossy().to_string();
@@ -179,7 +171,51 @@ fn get_specs(spec_path: &String, dbg: LogLevel) -> Vec<PathBuf> {
             dbg,
             "Skyforge found {} renderable devices in {}",
             result.len(),
-        env::current_dir().unwrap().display()
+            std::env::current_dir().unwrap().display()
         );
         result
     }
+}
+
+impl std::fmt::Display for Specification {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "Compiled Specification: {}\nLogLevel: {}\nComponents: {}",
+            self.compiled, self.loglevel, self.components,
+        )
+    }
+}
+
+#[derive(Debug)]
+pub struct Components {
+    partitional: PathBuf,
+    regional: PathBuf,
+    common: PathBuf,
+    zonal: PathBuf,
+    device: PathBuf,
+}
+
+impl Components {
+    pub fn get_hostname(self: &Self) -> String {
+        self.device
+            .file_stem()
+            .and_then(|n| n.to_str())
+            .unwrap_or("unknown")
+            .to_string()
+    }
+}
+
+impl std::fmt::Display for Components {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            " | Partitional: {}\n | Regional: {}\n | Common: {}\n | Zonal: {}\n | Device: {}\n",
+            self.partitional.display(),
+            self.regional.display(),
+            self.common.display(),
+            self.zonal.display(),
+            self.device.display(),
+        )
+    }
+}

src/tmpl.rs

@@ -1,12 +1,11 @@
+use crate::{LogLevel, crit, info};
 use serde::Deserialize;
 use yaml_serde;
 
 #[derive(Deserialize)]
 pub struct Structure {
     pub files: Vec<String>,
-    #[allow(dead_code)] //todo
     pub platform: String,
-    #[allow(dead_code)] //todo
     pub variations: Vec<String>,
 }
 
@@ -17,15 +16,22 @@ impl Structure {
         Ok(yaml_serde::from_str(&data)?)
     }
 
-    pub fn load_template_data(&self, base_dir: &std::path::Path) -> Vec<(String, String)> {
+    pub fn load_template_data(
+        &self,
+        tmpl_dir: &std::path::Path,
+        dbg: LogLevel,
+    ) -> Vec<(String, String)> {
         self.files
             .iter()
             .filter_map(|name| {
-                let path = base_dir.join(format!("{}.tera", name));
+                let template = tmpl_dir.join(format!("{}.tera", name));
-                match std::fs::read_to_string(&path) {
+                match std::fs::read_to_string(&template) {
-                    Ok(content) => Some((name.clone(), content)),
+                    Ok(content) => {
+                        info!(dbg, " | {}", template.display());
+                        Some((name.clone(), content))
+                    }
                     Err(e) => {
-                        eprintln!("Skipping {}: {}", path.display(), e);
+                        crit!(dbg, "Skipping {}: {}", template.display(), e);
                         None
                     }
                 }