init commit

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "skyforge"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+clap = { version = "4.5.11" }
+regex = "1.10.5"
+serde = { version = "1.0", features = ["derive"] }
+serde_yml = "0.0.10"
+serde_json = "1.0.112"
+tera = "1.20.0"
+thiserror = "1.0"
+walkdir = "2.3"

README.md

@@ -0,0 +1,28 @@
+# Skyforge
+
+## Brief
+
+Skyforge was designed to assist in render thousands of device configurations across the globe.
+
+## Assumptions
+
+- Partitions are groups of regions
+- Regions are groups of zones
+- Zones are groups of devices
+- Layers are groups of common devices and facilitate template mapping
+
+## Functionality
+
+Skyforge takes a user provide regex pattern, performs a walk on a `./spec` dir, and matches a list of devices specifications that do not have the word "common" in their path.
+ All group files are labeled with common and mappable from the file itself.
+
+For each file_path matched, Skyforge then maps to all consituent files:
+
+- Layer - from  by the common file in parent dir and maps the region
+- Zonal - from the first group of chars in filename up to a `-` (region_id + zone_id)
+- Regional - from the region_id of containing folder
+- Partitional - from either layer (common.yaml) or regional yaml
+
+Once all files are found, a compiled_specifcation is built.
+ This spec is then passed to Tera as context.
+ Tera then loads the template files for that layer and renders the configuration file.

demo/spec/common/eu.yaml

@@ -0,0 +1,2 @@
+partitional:
+  location: european union

demo/spec/common/us.yaml

@@ -0,0 +1,2 @@
+partitional:
+  location: united states

demo/spec/xyz/common/ex.yaml

@@ -0,0 +1,2 @@
+regional:
+  partition: us

demo/spec/xyz/ex-edge-r1/common.yaml

@@ -0,0 +1,2 @@
+common:
+  layer: ex-edge-r

demo/spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz1-ex-edge-r101

demo/spec/xyz/ex-edge-r1/xyz1-ex-edge-r102.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz1-ex-edge-r102

demo/spec/xyz/ex-edge-r1/xyz1.common.yaml

@@ -0,0 +1,2 @@
+zonal:
+  az: 1

demo/spec/xyz/ex-edge-r1/xyz2-ex-edge-r101.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz2-ex-edge-r101

demo/spec/xyz/ex-edge-r1/xyz2-ex-edge-r102.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz2-ex-edge-r102

demo/spec/xyz/ex-edge-r1/xyz2.common.yaml

@@ -0,0 +1,2 @@
+zonal:
+  az: 2

demo/spec/xyz/ex-edge-r2/common.yaml

@@ -0,0 +1,3 @@
+common:
+  partition: eu
+  layer: ex-edge-r

demo/spec/xyz/ex-edge-r2/xyz1-ex-edge-r201.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz1-ex-edge-r201

demo/spec/xyz/ex-edge-r2/xyz1-ex-edge-r202.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz1-ex-edge-r202

demo/spec/xyz/ex-edge-r2/xyz1.common.yaml

@@ -0,0 +1,2 @@
+zonal:
+  az: 1

demo/spec/xyz/ex-edge-r2/xyz2-ex-edge-r201.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz2-ex-edge-r201

demo/spec/xyz/ex-edge-r2/xyz2-ex-edge-r202.yaml

@@ -0,0 +1,2 @@
+device:
+  hostname: xyz2-ex-edge-r202

demo/spec/xyz/ex-edge-r2/xyz2.common.yaml

@@ -0,0 +1,2 @@
+zonal:
+  az: 2

demo/tmpl/common/ex/system.tmpl

@@ -0,0 +1,4 @@
+system {
+  hostname {{ hostname }};
+  location "{{ location }}";
+}

demo/tmpl/ex-edge-r/chassis.tmpl

@@ -0,0 +1,2 @@
+chassis {
+}

demo/tmpl/ex-edge-r/interfaces.tmpl

@@ -0,0 +1,2 @@
+interfaces {
+}

demo/tmpl/ex-edge-r/protocols.tmpl

@@ -0,0 +1,2 @@
+protocols {
+}

demo/tmpl/ex-edge-r/structure.yaml

@@ -0,0 +1,5 @@
+files:
+  - system
+  - chassis
+  - interfaces
+  - protocols

demo/tmpl/ex-edge-r/system.tmpl

@@ -0,0 +1 @@
+../common/ex/system.tmpl

src/cli.rs

@@ -0,0 +1,130 @@
+use crate::log::LogLevel;
+use clap::{Arg, ArgAction, ArgGroup, Command};
+use regex::Regex;
+use std::fmt;
+
+#[derive(Debug)]
+pub struct Args {
+    pub devices: Regex,
+    pub loglevel: LogLevel,
+    pub env: EnvVars,
+}
+
+impl fmt::Display for Args {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "devices: {}, loglevel: {}, env: {}",
+            self.devices, self.loglevel, self.env
+        )
+    }
+}
+
+#[derive(Debug)]
+pub struct EnvVars {
+    pub spec_path: String,
+    pub tmpl_path: String,
+    pub out_path: String,
+    pub log_path: String,
+}
+
+impl fmt::Display for EnvVars {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "spec_path: {}, tmpl_path: {}, out_path: {}, log_path: {}",
+            self.spec_path, self.tmpl_path, self.out_path, self.log_path
+        )
+    }
+}
+
+/// loads `command line arguments` and `environment variables` using custom `clap`
+pub fn parse_args() -> Args {
+    let matches: clap::ArgMatches = build().get_matches();
+
+    let raw_devices = matches.get_one::<String>("devices").unwrap().to_string();
+    let devices: Regex =
+        Regex::new(&raw_devices).expect("Invalid regex pattern provided for `devices`");
+
+    let loglevel: LogLevel = match matches.get_flag("debug") {
+        true => LogLevel::Debug,
+        false => match matches.get_flag("verbose") {
+            true => LogLevel::Verbose,
+            false => LogLevel::Info,
+        },
+    };
+
+    let env: EnvVars = parse_env();
+
+    Args {
+        devices,
+        loglevel,
+        env,
+    }
+}
+
+/// loads only environment variables
+pub fn parse_env() -> EnvVars {
+    EnvVars {
+        spec_path: match std::env::var("SF_SPEC_PATH") {
+            Ok(path) => path.trim_end_matches('/').to_string(),
+            Err(_) => String::from("./spec"),
+        },
+        tmpl_path: match std::env::var("SF_TMPL_PATH") {
+            Ok(path) => path.trim_end_matches('/').to_string(),
+            Err(_) => String::from("./tmpl"),
+        },
+        out_path: match std::env::var("SF_OUT_PATH") {
+            Ok(path) => path.trim_end_matches('/').to_string(),
+            Err(_) => String::from("./out"),
+        },
+        log_path: match std::env::var("SF_LOG_PATH") {
+            Ok(path) => path.trim_end_matches('/').to_string(),
+            Err(_) => String::from("./log"),
+        },
+    }
+}
+
+const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
+const ENV_MSG: &str = r#"Environment:
+    SF_SPEC_PATH    Path to the directory containing templates.        Defaults to "./spec".
+    SF_TMPL_PATH    Path to the directory containing specifications.   Defaults to "./tmpl".
+    SF_OUT_PATH     Path to the directory for command output.          Defaults to "./out".
+    SF_LOG_PATH     Path to the directory for log output.              Defaults to "./log".
+"#;
+
+/// builds a custom command line argument parser
+fn build() -> Command {
+    Command::new("app")
+        .about(ABOUT_MSG)
+        .version(env!("CARGO_PKG_VERSION"))
+        .author("rskntroot")
+        .arg(
+            Arg::new("devices")
+                .long("devices")
+                .short('d')
+                .help("A regular expression pattern")
+                .required(true),
+        )
+        .arg(
+            Arg::new("debug")
+                .long("debug")
+                .help("Print debug information")
+                .action(ArgAction::SetTrue)
+                .required(false),
+        )
+        .arg(
+            Arg::new("verbose")
+                .short('v')
+                .long("verbose")
+                .help("Print verbose information")
+                .action(ArgAction::SetTrue)
+                .required(false),
+        )
+        .group(
+            ArgGroup::new("loglevel")
+                .args(&["debug", "verbose"])
+                .required(false),
+        )
+        .after_help(ENV_MSG)
+}

src/log.rs

@@ -0,0 +1,78 @@
+#![allow(dead_code)]
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum LogLevel {
+    Debug,
+    Verbose,
+    Info,
+    Warning,
+    Critical,
+    None,
+}
+
+impl LogLevel {
+    pub fn value(&self) -> u8 {
+        match self {
+            LogLevel::Debug => u8::MIN,
+            LogLevel::Verbose => u8::from(30),
+            LogLevel::Info => u8::from(60),
+            LogLevel::Warning => u8::from(90),
+            LogLevel::Critical => u8::from(120),
+            LogLevel::None => u8::MAX,
+        }
+    }
+}
+
+impl std::fmt::Display for LogLevel {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+#[macro_export]
+macro_rules! info {
+    ($current_level:expr, $($msg:expr),*) => {
+        if LogLevel::Info.value() >= $current_level.value() {
+            println!($($msg),*);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! verb {
+    ($current_level:expr, $($msg:expr),*) => {
+        if LogLevel::Verbose.value() >= $current_level.value() {
+            println!($($msg),*);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! dbug {
+    ($current_level:expr, $($msg:expr),*) => {
+        if LogLevel::Debug.value() >= $current_level.value() {
+            let message = format!($($msg),*);
+            for line in message.lines() {
+                println!(" |D| {}", line);
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! warn {
+    ($current_level:expr, $($msg:expr),*) => {
+        if LogLevel::Warning.value() >= $current_level.value() {
+            eprintln!($($msg),*);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! crit {
+    ($current_level:expr, $($msg:expr),*) => {
+        if LogLevel::Critical.value() >= $current_level.value() {
+            eprintln!($($msg),*);
+        }
+    };
+}

src/main.rs

@@ -0,0 +1,31 @@
+mod cli;
+mod log;
+mod specs;
+mod tmpls;
+
+use log::LogLevel;
+
+fn main() {
+    let args: cli::Args = cli::parse_args();
+    let dbg: LogLevel = args.loglevel;
+
+    dbug!(dbg, "{}", &args);
+
+    let specifications: Vec<specs::Specification> =
+        specs::compile(&args.devices, &args.env.spec_path, dbg);
+
+    for spec in specifications {
+        let result = tmpls::process_templates(&spec, dbg).ok().unwrap();
+        verb!(
+            dbg,
+            "Compiled Spec:\n{}",
+            serde_json::to_string_pretty(&spec.compiled).unwrap()
+        );
+        info!(dbg, "Rendered Config:");
+        for line in result {
+            if line != "\n" {
+                print!("{}", line)
+            }
+        }
+    }
+}

src/specs.rs

@@ -0,0 +1,232 @@
+use crate::{info, verb, LogLevel};
+use regex::Regex;
+use serde_json::{json, Map, Value};
+use serde_yml;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+use std::{env, fmt, fs};
+use walkdir::WalkDir;
+
+#[derive(Debug)]
+pub struct Specification {
+    pub partitional: String,
+    pub regional: String,
+    pub common: String,
+    pub zonal: String,
+    pub device: String,
+    pub compiled: Value,
+}
+
+impl Specification {
+    pub fn build(
+        partitional: &str,
+        regional: &str,
+        common: &str,
+        zonal: &str,
+        device: &str,
+    ) -> Result<Self, Box<dyn std::error::Error>> {
+        // Read all YAML files
+        let yaml_contents: Vec<String> = [partitional, regional, common, zonal, device]
+            .iter()
+            .filter_map(|path| fs::read_to_string(path).ok())
+            .collect();
+
+        // Convert each YAML to JSON Value and collect them
+        let json_values: Vec<Value> = yaml_contents
+            .iter()
+            .map(|content| serde_yml::from_str(content))
+            .collect::<Result<Vec<Value>, _>>()?;
+
+        // Merge all objects under a single key
+        let mut merged_map = Map::new();
+        for value in json_values {
+            if let Some(obj) = value.as_object() {
+                for (_key, value) in obj {
+                    if let Some(inner_obj) = value.as_object() {
+                        for (k, v) in inner_obj {
+                            merged_map.insert(k.clone(), v.clone());
+                        }
+                    }
+                }
+            }
+        }
+
+        // Create the final merged object
+        let compiled = json!({"data": merged_map});
+
+        Ok(Self {
+            partitional: String::from(partitional),
+            regional: String::from(regional),
+            common: String::from(common),
+            zonal: String::from(zonal),
+            device: String::from(device),
+            compiled,
+        })
+    }
+
+    pub fn get_layer(&self) -> String {
+        PathBuf::from(&self.device)
+            .parent()
+            .unwrap()
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+            .chars()
+            .filter(|c| !c.is_numeric())
+            .collect::<String>()
+    }
+
+    pub fn get_hostname(&self) -> String {
+        PathBuf::from(&self.device)
+            .file_name()
+            .unwrap()
+            .to_string_lossy()
+            .into_owned()
+    }
+}
+
+impl fmt::Display for Specification {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Specification:\n\
+             Partitional: {}\n\
+             Regional: {}\n\
+             Common: {}\n\
+             Zonal: {}\n\
+             Device: {}\n\
+             Compiled:\n{}",
+            self.partitional, self.regional, self.common, self.zonal, self.device, self.compiled
+        )
+    }
+}
+
+pub fn compile(pattern: &Regex, spec_path: &String, dbg: LogLevel) -> Vec<Specification> {
+    let spec_list: Vec<String> = filter_specs(pattern, get_specs(spec_path, dbg));
+    info!(
+        dbg,
+        "Matched {} devices against '{}'",
+        &spec_list.len(),
+        &pattern
+    );
+    for spec in &spec_list {
+        verb!(dbg, " | {}", spec);
+    }
+    let mut specifications: Vec<Specification> = Vec::new();
+    for spec in spec_list {
+        let zonal: String = get_zonal(&spec);
+        let common: String = get_common(&spec);
+        let regional: String = get_regional(&spec);
+        let partitional: String = get_partional(&common, &regional);
+        specifications.push(
+            match Specification::build(&partitional, &regional, &common, &zonal, &spec) {
+                Ok(compiled_spec) => {
+                    verb!(
+                        dbg,
+                        "Compiled Spec for '{}'\n | {}\n | {}\n | {}\n | {}\n | {}",
+                        compiled_spec.get_hostname(),
+                        compiled_spec.partitional,
+                        compiled_spec.regional,
+                        compiled_spec.common,
+                        compiled_spec.zonal,
+                        compiled_spec.device
+                    );
+                    compiled_spec
+                }
+                Err(e) => panic!("failed to build Specification: {}", e),
+            },
+        )
+    }
+    specifications
+}
+
+fn get_partional(common: &str, regional: &str) -> String {
+    let re = Regex::new(r"partition: ([^\n\r$]+)").expect("failed");
+
+    [common, regional]
+        .iter()
+        .find_map(|&path| {
+            fs::read_to_string(path).ok().and_then(|contents| {
+                re.captures(&contents).and_then(|captures| {
+                    captures
+                        .get(1)
+                        .map(|matched| format!("./spec/common/{}.yaml", matched.as_str().trim()))
+                })
+            })
+        })
+        .unwrap_or_else(|| format!("./spec/common/default.yaml"))
+}
+
+fn get_regional(path: &str) -> String {
+    let dir_name = PathBuf::from(path)
+        .parent()
+        .unwrap()
+        .file_name()
+        .unwrap()
+        .to_string_lossy()
+        .into_owned();
+
+    let new_path = PathBuf::from(path)
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .join("common")
+        .join(format!("{}.yaml", &dir_name[0..2]));
+
+    String::from(new_path.to_str().unwrap())
+}
+
+fn get_common(spec: &str) -> String {
+    let path: PathBuf = PathBuf::from_str(&spec).expect("Path was invalid");
+    String::from(path.with_file_name("common.yaml").to_str().unwrap())
+}
+
+fn get_zonal(spec: &str) -> String {
+    let path: PathBuf = PathBuf::from_str(&spec).expect("Path was invalid");
+
+    String::from(format!(
+        "{}.common.yaml",
+        path.file_name()
+            .and_then(|filename| filename.to_str())
+            .and_then(|filename| filename.split_once('-'))
+            .map(|(zone, _)| path.with_file_name(zone))
+            .unwrap_or_else(|| path.with_file_name("common.yaml"))
+            .to_str()
+            .unwrap()
+            .to_string(),
+    ))
+}
+
+fn filter_specs(pattern: &Regex, spec_list: Vec<String>) -> Vec<String> {
+    spec_list
+        .into_iter()
+        .filter(|spec| {
+            Path::new(spec)
+                .file_name()
+                .and_then(|name| name.to_str())
+                .map_or(false, |name| pattern.is_match(name))
+        })
+        .collect()
+}
+
+fn get_specs(spec_path: &String, dbg: LogLevel) -> Vec<String> {
+    let mut result = Vec::new();
+    let root = Path::new(&spec_path);
+    for entry in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
+        if let Ok(path) = entry.path().strip_prefix(root) {
+            let path_str = path.to_string_lossy().to_string();
+
+            if !path_str.contains("common") && path_str.ends_with("yaml") {
+                result.push(format!("{}/{}", spec_path, path_str));
+            }
+        }
+    }
+    info!(
+        dbg,
+        "Skyforge found {} renderable devices in {}",
+        result.len(),
+        env::current_dir().unwrap().display()
+    );
+    result
+}

src/tmpls.rs

@@ -0,0 +1,56 @@
+use crate::specs::Specification;
+use crate::{verb, LogLevel};
+use serde_json::Value;
+use serde_yml;
+use std::fs;
+use tera::{Context, Tera};
+
+#[derive(serde::Deserialize)]
+struct TemplateConfig {
+    files: Vec<String>,
+}
+
+pub fn process_templates(
+    spec: &Specification,
+    dbg: LogLevel,
+) -> Result<Vec<String>, Box<dyn std::error::Error>> {
+    // Create Tera context from spec.compiled
+    let mut context = Context::new();
+    if let Value::Object(map) = &spec.compiled {
+        if let Some(Value::Object(data_map)) = map.get("data") {
+            for (key, value) in data_map {
+                context.insert(key, value);
+            }
+        }
+    }
+
+    // Get the base directory path
+    let base_dir = format!("./tmpl/{}", spec.get_layer());
+
+    // Read structure.yaml
+    let structure_path = format!("{}/structure.yaml", base_dir);
+    let structure_content = fs::read_to_string(&structure_path)?;
+    let config: TemplateConfig = serde_yml::from_str(&structure_content)?;
+
+    // Initialize Tera with the specific directory
+    let mut tera = Tera::default();
+
+    // Process each template
+    verb!(dbg, "Rendering Templates");
+    let mut rendered_templates = Vec::new();
+    for template_name in config.files {
+        // Read the template file directly
+        let template_path = format!("{}/{}.tmpl", base_dir, template_name);
+        verb!(dbg, " | {}", &template_path);
+        let template_content = fs::read_to_string(&template_path)?;
+
+        // Add this specific template to Tera
+        tera.add_raw_template(&template_name, &template_content)?;
+
+        // Render the template
+        let rendered = tera.render(&template_name, &context)?;
+        rendered_templates.push(rendered);
+    }
+
+    Ok(rendered_templates)
+}