feat output artifacts to tmp; add bench & profiling; minor fixes
This commit is contained in:
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,2 +1 @@
|
|||||||
/target
|
/target
|
||||||
/demo/out
|
|
||||||
|
|||||||
2185
Cargo.lock
generated
2185
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
12
Cargo.toml
12
Cargo.toml
@@ -3,6 +3,10 @@ name = "skyforge"
|
|||||||
version = "0.1.112"
|
version = "0.1.112"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
|
[profile.profiling]
|
||||||
|
inherits = "release"
|
||||||
|
debug = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
clap = { version = "4.5" }
|
clap = { version = "4.5" }
|
||||||
@@ -13,3 +17,11 @@ yaml_serde = "0"
|
|||||||
tera = "1"
|
tera = "1"
|
||||||
thiserror = "1"
|
thiserror = "1"
|
||||||
walkdir = "2"
|
walkdir = "2"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
criterion = "0.5"
|
||||||
|
flamegraph = "0.6"
|
||||||
|
|
||||||
|
[[bench]]
|
||||||
|
name = "benchmark"
|
||||||
|
harness = false
|
||||||
|
|||||||
141
README.md
141
README.md
@@ -9,6 +9,7 @@ skyforge was designed to assist in rendering thousands of device configurations
|
|||||||
- Partitions are groups of regions
|
- Partitions are groups of regions
|
||||||
- Regions are groups of zones
|
- Regions are groups of zones
|
||||||
- Zones are groups of devices
|
- Zones are groups of devices
|
||||||
|
- Fabrics are groups of layers
|
||||||
- Layers are groups of common devices and facilitate template mapping
|
- Layers are groups of common devices and facilitate template mapping
|
||||||
|
|
||||||
## Functionality
|
## Functionality
|
||||||
@@ -20,13 +21,13 @@ Skyforge takes a user provided regex pattern, performs a walk on a `./spec` dir,
|
|||||||
For each "device" matched, skyforge then maps to all consituent files:
|
For each "device" matched, skyforge then maps to all consituent files:
|
||||||
|
|
||||||
- Layer - from the `common.yaml` file in parent dir and maps the region
|
- Layer - from the `common.yaml` file in parent dir and maps the region
|
||||||
- Zonal - from the first group of chars in filename up to a `-` which is expected to be region and zone
|
- Zonal - from the first group of chars in filename up to a `-`
|
||||||
- Regional - from the `<region>/common/<network>.yaml` of containing folder where network matches the layer info
|
- Regional - from value of `common.fabric` field in `Layer` file
|
||||||
- Partitional - from either layer (common.yaml) or regional yaml
|
- Partitional - from value of `regional.partition` field in `Regional` file
|
||||||
|
|
||||||
Once all files are found, a compiled specifcation is built.
|
Once all files are found, a compiled specifcation is built.
|
||||||
This spec is then passed to Tera as context.
|
This spec is then passed to Tera as context.
|
||||||
Tera then loads the template files for that layer and renders the configuration files.
|
Tera loads the template files for that layer and renders the configuration files.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -47,68 +48,112 @@ Options:
|
|||||||
-V, --version Print version
|
-V, --version Print version
|
||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
SF_SPEC_PATH Directory containing templates
|
SKYFORGE_SPECDIR Directory containing templates
|
||||||
SF_TMPL_PATH Directory containing specifications
|
SKYFORGE_TMPLDIR Directory containing specifications
|
||||||
SF_OUT_PATH Directory for command output
|
SKYFORGE_OUTDIR Directory for command output
|
||||||
SF_LOG_PATH Directory for log output
|
SKYFORGE_LOGDIR Directory for log output
|
||||||
```
|
```
|
||||||
|
|
||||||
### Standard
|
### Standard
|
||||||
|
|
||||||
``` bash
|
``` bash
|
||||||
$ skyforge -d xyz1-ex-edge-r101
|
$ skyforge -d xyz1-ex-edge-r101
|
||||||
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
|
Skyforge found 9 renderable devices
|
||||||
Matched 1 devices against 'xyz1-ex-edge-r101'
|
Matched 1 devices against 'xyz1-ex-edge-r101'
|
||||||
Rendering xyz1-ex-edge-r101
|
Processing templates for 'xyz1-ex-edge-r101'
|
||||||
Writing Output
|
Writing Output:
|
||||||
| out/xyz1-ex-edge-r101/all.conf
|
| ./out/xyz1-ex-edge-r101/all.live.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.shifted.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.init.junos
|
||||||
```
|
```
|
||||||
|
|
||||||
### Verbose
|
### Verbose
|
||||||
|
|
||||||
``` bash
|
``` bash
|
||||||
$ skyforge -d xyz1-ex-edge-r101 -v
|
$ skyforge -d xyz1-ex-edge-r101 --verbose
|
||||||
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
|
Skyforge found 9 renderable devices
|
||||||
Matched 1 devices against 'xyz1-ex-edge-r101'
|
Matched 1 devices against 'xyz1-ex-edge-r101'
|
||||||
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
|
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
|
||||||
Rendering xyz1-ex-edge-r101
|
Processing templates for 'xyz1-ex-edge-r101'
|
||||||
| ./tmpl/ex-edge-r/system.tmpl
|
| ./tmpl/ex-edge-r/system.tera
|
||||||
| ./tmpl/ex-edge-r/chassis.tmpl
|
| ./tmpl/ex-edge-r/interfaces.tera
|
||||||
| ./tmpl/ex-edge-r/interfaces.tmpl
|
| ./tmpl/ex-edge-r/protocols.tera
|
||||||
| ./tmpl/ex-edge-r/protocols.tmpl
|
| ./tmpl/ex-edge-r/policy-options.tera
|
||||||
Writing Output
|
Writing Output:
|
||||||
| out/xyz1-ex-edge-r101/system.tmpl
|
| ./out/xyz1-ex-edge-r101/live/system.junos
|
||||||
| out/xyz1-ex-edge-r101/chassis.tmpl
|
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
|
||||||
| out/xyz1-ex-edge-r101/interfaces.tmpl
|
| ./out/xyz1-ex-edge-r101/live/protocols.junos
|
||||||
| out/xyz1-ex-edge-r101/protocols.tmpl
|
| ./out/xyz1-ex-edge-r101/live/policy-options.junos
|
||||||
| out/xyz1-ex-edge-r101/compiled.spec
|
| ./out/xyz1-ex-edge-r101/all.live.junos
|
||||||
| out/xyz1-ex-edge-r101/all.conf
|
| ./out/xyz1-ex-edge-r101/shifted/system.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/interfaces.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/protocols.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/policy-options.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.shifted.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/system.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/interfaces.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/protocols.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.init.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/context.yaml
|
||||||
```
|
```
|
||||||
|
|
||||||
### Debug
|
### Debug
|
||||||
|
|
||||||
``` bash
|
``` bash
|
||||||
$ skyforge -d xyz1-ex-edge-r101 --debug
|
$ skyforge -d xyz1-ex-edge-r101 --debug
|
||||||
devices: xyz1-ex-edge-r101, loglevel: Debug, env: spec_path: ./spec, tmpl_path: ./tmpl, out_path: ./out, log_path: ./log
|
Skyforge found 9 renderable devices
|
||||||
Skyforge found 8 renderable devices in /home/lost/workspace/skyforge/demo
|
|
||||||
Matched 1 devices against 'xyz1-ex-edge-r101'
|
Matched 1 devices against 'xyz1-ex-edge-r101'
|
||||||
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
|
| ./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml
|
||||||
Compiled Spec for 'xyz1-ex-edge-r101.yaml'
|
Components {
|
||||||
| ./spec/common/us.yaml
|
partitional: "./spec/common/us.yaml",
|
||||||
| ./spec/xyz/common/ex.yaml
|
regional: "./spec/xyz/common/ex.yaml",
|
||||||
| ./spec/xyz/ex-edge-r1/common.yaml
|
common: "./spec/xyz/ex-edge-r/common.yaml",
|
||||||
| ./spec/xyz/ex-edge-r1/xyz1.common.yaml
|
zonal: "./spec/xyz/ex-edge-r/xyz1.common.yaml",
|
||||||
| ./spec/xyz/ex-edge-r1/xyz1-ex-edge-r101.yaml
|
device: "./spec/xyz/ex-edge-r/xyz1-ex-edge-r101.yaml",
|
||||||
Rendering xyz1-ex-edge-r101
|
}
|
||||||
| ./tmpl/ex-edge-r/system.tmpl
|
Processing templates for 'xyz1-ex-edge-r101'
|
||||||
| ./tmpl/ex-edge-r/chassis.tmpl
|
| ./tmpl/ex-edge-r/system.tera
|
||||||
| ./tmpl/ex-edge-r/interfaces.tmpl
|
| ./tmpl/ex-edge-r/interfaces.tera
|
||||||
| ./tmpl/ex-edge-r/protocols.tmpl
|
| ./tmpl/ex-edge-r/protocols.tera
|
||||||
Writing Output
|
| ./tmpl/ex-edge-r/policy-options.tera
|
||||||
| out/xyz1-ex-edge-r101/system.tmpl
|
Rendering templates for xyz1-ex-edge-r101
|
||||||
| out/xyz1-ex-edge-r101/chassis.tmpl
|
| system.live
|
||||||
| out/xyz1-ex-edge-r101/interfaces.tmpl
|
| interfaces.live
|
||||||
| out/xyz1-ex-edge-r101/protocols.tmpl
|
| protocols.live
|
||||||
| out/xyz1-ex-edge-r101/compiled.spec
|
| policy-options.live
|
||||||
| out/xyz1-ex-edge-r101/all.conf
|
| system.shifted
|
||||||
|
| interfaces.shifted
|
||||||
|
| protocols.shifted
|
||||||
|
| policy-options.shifted
|
||||||
|
| system.init
|
||||||
|
| interfaces.init
|
||||||
|
| protocols.init
|
||||||
|
| policy-options.init
|
||||||
|
Writing Output:
|
||||||
|
| ./out/xyz1-ex-edge-r101/live/system.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/live/interfaces.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/live/protocols.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/live/policy-options.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.live.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/system.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/interfaces.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/protocols.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/shifted/policy-options.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.shifted.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/system.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/interfaces.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/protocols.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/init/policy-options.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/all.init.junos
|
||||||
|
| ./out/xyz1-ex-edge-r101/context.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Flamegraph
|
||||||
|
|
||||||
|
Assume flamelens is installed, otherwise `cargo install flamelens`
|
||||||
|
|
||||||
|
``` bash
|
||||||
|
cd demo
|
||||||
|
cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devices ".*"
|
||||||
```
|
```
|
||||||
|
|||||||
28
benches/benchmark.rs
Normal file
28
benches/benchmark.rs
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
use criterion::{Criterion, criterion_group, criterion_main};
|
||||||
|
use skyforge::{Args, DeviceConfigBundle, Specification};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
fn benchmark(c: &mut Criterion) {
|
||||||
|
let args = Args {
|
||||||
|
devices: regex::Regex::new(".*").unwrap(),
|
||||||
|
env: skyforge::cli::EnvVars {
|
||||||
|
spec_path: PathBuf::from("./demo/spec"),
|
||||||
|
tmpl_path: PathBuf::from("./demo/tmpl"),
|
||||||
|
out_path: PathBuf::from("./demo/out"),
|
||||||
|
log_path: PathBuf::from("./demo/log"),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
c.bench_function("compile", |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
for spec in Specification::compile(&args) {
|
||||||
|
DeviceConfigBundle::from_spec(spec)
|
||||||
|
.unwrap()
|
||||||
|
.output_artifacts();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
criterion_group!(benches, benchmark);
|
||||||
|
criterion_main!(benches);
|
||||||
3
demo/.gitignore
vendored
Normal file
3
demo/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
flamegraph*
|
||||||
|
perf*
|
||||||
|
out
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
common:
|
common:
|
||||||
partition: us
|
fabric: ex
|
||||||
layer: ex-core-r
|
layer: ex-core-r
|
||||||
protocol: ospf
|
protocol: ospf
|
||||||
uplink: et-0/0/36
|
uplink: et-0/0/36
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
common:
|
common:
|
||||||
|
fabric: ex
|
||||||
layer: ex-edge-r
|
layer: ex-edge-r
|
||||||
protocol: bgp
|
protocol: bgp
|
||||||
uplink: et-0/0/0
|
uplink: et-0/0/0
|
||||||
|
|||||||
18
src/cli.rs
18
src/cli.rs
@@ -5,16 +5,15 @@ use std::path::PathBuf;
|
|||||||
|
|
||||||
const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
|
const ABOUT_MSG: &str = r#"Skyforge Config Generation Engine"#;
|
||||||
const ENV_MSG: &str = r#"Environment:
|
const ENV_MSG: &str = r#"Environment:
|
||||||
SF_SPEC_PATH Directory containing templates
|
SKYFORGE_SPECDIR Directory containing templates
|
||||||
SF_TMPL_PATH Directory containing specifications
|
SKYFORGE_TMPLDIR Directory containing specifications
|
||||||
SF_OUT_PATH Directory for command output
|
SKYFORGE_OUTDIR Directory for command output
|
||||||
SF_LOG_PATH Directory for log output
|
SKYFORGE_LOGDIR Directory for log output
|
||||||
"#;
|
"#;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Args {
|
pub struct Args {
|
||||||
pub devices: Regex,
|
pub devices: Regex,
|
||||||
pub loglevel: LogLevel,
|
|
||||||
pub env: EnvVars,
|
pub env: EnvVars,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,7 +35,6 @@ impl Args {
|
|||||||
|
|
||||||
Args {
|
Args {
|
||||||
devices,
|
devices,
|
||||||
loglevel,
|
|
||||||
env: EnvVars::parse(),
|
env: EnvVars::parse(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -87,16 +85,16 @@ pub struct EnvVars {
|
|||||||
impl EnvVars {
|
impl EnvVars {
|
||||||
pub fn parse() -> Self {
|
pub fn parse() -> Self {
|
||||||
Self {
|
Self {
|
||||||
spec_path: std::env::var("SF_SPEC_PATH")
|
spec_path: std::env::var("SKYFORGE_SPECDIR")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("./spec")),
|
.unwrap_or_else(|_| PathBuf::from("./spec")),
|
||||||
tmpl_path: std::env::var("SF_TMPL_PATH")
|
tmpl_path: std::env::var("SKYFORGE_TMPLDIR")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("./tmpl")),
|
.unwrap_or_else(|_| PathBuf::from("./tmpl")),
|
||||||
out_path: std::env::var("SF_OUT_PATH")
|
out_path: std::env::var("SKYFORGE_OUTDIR")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("./out")),
|
.unwrap_or_else(|_| PathBuf::from("./out")),
|
||||||
log_path: std::env::var("SF_LOG_PATH")
|
log_path: std::env::var("SKYFORGE_LOGDIR")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("./log")),
|
.unwrap_or_else(|_| PathBuf::from("./log")),
|
||||||
}
|
}
|
||||||
|
|||||||
13
src/main.rs
13
src/main.rs
@@ -3,6 +3,19 @@ use skyforge::{Args, DeviceConfigBundle, Specification};
|
|||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let args = Args::parse();
|
let args = Args::parse();
|
||||||
|
|
||||||
|
let exec_id = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_millis();
|
||||||
|
|
||||||
|
let tmp_out = std::env::temp_dir().join(format!("skyforge-{}", exec_id));
|
||||||
|
std::fs::create_dir_all(&tmp_out)?;
|
||||||
|
if args.env.out_path.exists() {
|
||||||
|
std::fs::remove_file(&args.env.out_path)
|
||||||
|
.or_else(|_| std::fs::remove_dir_all(&args.env.out_path))?;
|
||||||
|
}
|
||||||
|
std::os::unix::fs::symlink(&tmp_out, &args.env.out_path)?;
|
||||||
|
|
||||||
for spec in Specification::compile(&args) {
|
for spec in Specification::compile(&args) {
|
||||||
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
|
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,16 +27,15 @@ impl DeviceConfigBundle {
|
|||||||
let mut context = Self::merge_context(&spec);
|
let mut context = Self::merge_context(&spec);
|
||||||
|
|
||||||
let hostname = Self::get_hostname(&context, &spec);
|
let hostname = Self::get_hostname(&context, &spec);
|
||||||
let base_dir = std::path::Path::new("./tmpl").join(spec.get_layer());
|
info!("Processing templates for '{hostname}'");
|
||||||
info!("Loading templates for '{hostname}':");
|
|
||||||
|
|
||||||
let structure_path = base_dir.join("structure.yaml");
|
let structure_path = spec.tmplpath.join("structure.yaml");
|
||||||
let structure = tmpl::Structure::from_file(&structure_path)
|
let structure = tmpl::Structure::from_file(&structure_path)
|
||||||
.map_err(|e| format!("Failed to parse {}: {e}", structure_path.display()))?;
|
.map_err(|e| format!("Failed to parse {:?}: {e}", structure_path))?;
|
||||||
|
|
||||||
let mut renderer = tera::Tera::default();
|
let mut renderer = tera::Tera::default();
|
||||||
renderer.add_raw_templates(structure.load_template_data(base_dir))?;
|
renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
|
||||||
dbug!("Processing templates for {hostname}");
|
dbug!("Rendering templates for {hostname}");
|
||||||
|
|
||||||
let mut failures = false;
|
let mut failures = false;
|
||||||
let mut configs = Vec::new();
|
let mut configs = Vec::new();
|
||||||
@@ -91,9 +90,6 @@ impl DeviceConfigBundle {
|
|||||||
info!("Writing Output:");
|
info!("Writing Output:");
|
||||||
|
|
||||||
let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
|
let device_outpath = std::path::Path::new(&self.spec.outpath).join(self.hostname);
|
||||||
if device_outpath.exists() {
|
|
||||||
std::fs::remove_dir_all(&device_outpath).ok();
|
|
||||||
}
|
|
||||||
std::fs::create_dir_all(&device_outpath).ok();
|
std::fs::create_dir_all(&device_outpath).ok();
|
||||||
|
|
||||||
for variant_configs in self.configs {
|
for variant_configs in self.configs {
|
||||||
|
|||||||
163
src/spec.rs
163
src/spec.rs
@@ -1,16 +1,14 @@
|
|||||||
use crate::cli::Args;
|
use crate::cli::Args;
|
||||||
use crate::{LogLevel, info, verb};
|
use crate::{LogLevel, dbug, info, verb};
|
||||||
use regex::Regex;
|
use serde_json::Value;
|
||||||
use serde_json::{Map, Value, json};
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use walkdir::WalkDir;
|
|
||||||
use yaml_serde;
|
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Specification {
|
pub struct Specification {
|
||||||
pub compiled: Value,
|
pub compiled: Value,
|
||||||
pub components: Components,
|
pub components: Components,
|
||||||
pub outpath: PathBuf,
|
pub outpath: PathBuf,
|
||||||
|
pub tmplpath: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
trait Pipe: Sized {
|
trait Pipe: Sized {
|
||||||
@@ -26,52 +24,48 @@ impl<T> Pipe for T {}
|
|||||||
|
|
||||||
impl Specification {
|
impl Specification {
|
||||||
pub fn build(
|
pub fn build(
|
||||||
partitional: PathBuf,
|
components: Components,
|
||||||
regional: PathBuf,
|
|
||||||
common: PathBuf,
|
|
||||||
zonal: PathBuf,
|
|
||||||
device: PathBuf,
|
|
||||||
outpath: PathBuf,
|
outpath: PathBuf,
|
||||||
|
tmplpath: PathBuf,
|
||||||
) -> Result<Self, Box<dyn std::error::Error>> {
|
) -> Result<Self, Box<dyn std::error::Error>> {
|
||||||
let json_values: Vec<Value> = [&partitional, ®ional, &common, &zonal, &device]
|
dbug!("{:#?}", components);
|
||||||
|
let merged_map: serde_json::Map<String, Value> = [
|
||||||
|
&components.partitional,
|
||||||
|
&components.regional,
|
||||||
|
&components.common,
|
||||||
|
&components.zonal,
|
||||||
|
&components.device,
|
||||||
|
]
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|path| std::fs::read_to_string(path).ok())
|
.filter_map(|path| std::fs::read_to_string(path).ok())
|
||||||
.map(|content| yaml_serde::from_str::<Value>(&content))
|
.map(|content| yaml_serde::from_str::<Value>(&content))
|
||||||
.collect::<Result<Vec<Value>, _>>()?;
|
.collect::<Result<Vec<Value>, _>>()?
|
||||||
|
|
||||||
let merged_map: Map<String, Value> = json_values
|
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter_map(|v| v.as_object().cloned())
|
.filter_map(|v| v.as_object().cloned())
|
||||||
.flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
|
.flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
|
||||||
.flatten()
|
.flatten()
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let compiled = json!({"data": merged_map});
|
let tmplpath = Self::get_templates(&components.device, &tmplpath);
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
compiled,
|
compiled: serde_json::json!({"data": merged_map}),
|
||||||
components: Components {
|
components,
|
||||||
partitional,
|
|
||||||
regional,
|
|
||||||
common,
|
|
||||||
zonal,
|
|
||||||
device,
|
|
||||||
},
|
|
||||||
outpath,
|
outpath,
|
||||||
|
tmplpath,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_layer(&self) -> String {
|
fn get_templates(device: &PathBuf, tmpldir: &PathBuf) -> PathBuf {
|
||||||
self.components
|
tmpldir.join(
|
||||||
.device
|
device
|
||||||
.parent()
|
.parent()
|
||||||
.unwrap()
|
.and_then(|f| f.file_name())
|
||||||
.file_name()
|
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.to_string_lossy()
|
.to_string_lossy()
|
||||||
.chars()
|
.chars()
|
||||||
.filter(|c| !c.is_numeric())
|
.filter(|c| !c.is_numeric())
|
||||||
.collect::<String>()
|
.collect::<String>(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// compiles specifications from regex and paths provided by Args
|
/// compiles specifications from regex and paths provided by Args
|
||||||
@@ -86,52 +80,56 @@ impl Specification {
|
|||||||
.map(|spec| {
|
.map(|spec| {
|
||||||
verb!(" | {}", spec.display());
|
verb!(" | {}", spec.display());
|
||||||
let common = Self::get_common(&spec);
|
let common = Self::get_common(&spec);
|
||||||
let regional = Self::get_regional(&spec);
|
let regional = Self::get_regional(&common);
|
||||||
Self::build(
|
Self::build(
|
||||||
Self::get_partitional(&common, ®ional),
|
Components {
|
||||||
|
partitional: Self::get_partitional(®ional),
|
||||||
regional,
|
regional,
|
||||||
common,
|
common,
|
||||||
Self::get_zonal(&spec),
|
zonal: Self::get_zonal(&spec),
|
||||||
spec,
|
device: spec,
|
||||||
|
},
|
||||||
args.env.out_path.clone(),
|
args.env.out_path.clone(),
|
||||||
|
args.env.tmpl_path.clone(),
|
||||||
)
|
)
|
||||||
.expect("failed to build Specification")
|
.expect("failed to build Specification")
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_partitional(common: &PathBuf, regional: &PathBuf) -> PathBuf {
|
fn get_partitional(regional: &PathBuf) -> PathBuf {
|
||||||
let re = Regex::new(r"partition: ([^\n\r$]+)").expect("failed");
|
let basedir = regional.ancestors().nth(3).unwrap().join("common");
|
||||||
|
match yaml_serde::from_str::<yaml_serde::Value>(
|
||||||
[common, regional]
|
&std::fs::read_to_string(regional).unwrap_or_default(),
|
||||||
.iter()
|
)
|
||||||
.find_map(|path| {
|
.unwrap_or_default()["regional"]["partition"]
|
||||||
std::fs::read_to_string(path).ok().and_then(|contents| {
|
.as_str()
|
||||||
re.captures(&contents).and_then(|captures| {
|
{
|
||||||
captures.get(1).map(|matched| {
|
Some(partition) => basedir.join(format!("{partition}.yaml")),
|
||||||
PathBuf::from(format!("./spec/common/{}.yaml", matched.as_str().trim()))
|
None => {
|
||||||
})
|
verb!("[?] {} missing regional.partition", regional.display());
|
||||||
})
|
basedir.join("default.yaml")
|
||||||
})
|
}
|
||||||
})
|
}
|
||||||
.unwrap_or_else(|| PathBuf::from("./spec/common/default.yaml"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_regional(path: &PathBuf) -> PathBuf {
|
fn get_regional(common: &PathBuf) -> PathBuf {
|
||||||
let dir_name = path
|
let basedir = common.ancestors().nth(2).unwrap().join("common");
|
||||||
.parent()
|
match yaml_serde::from_str::<yaml_serde::Value>(
|
||||||
.unwrap()
|
&std::fs::read_to_string(common).unwrap_or_default(),
|
||||||
.file_name()
|
)
|
||||||
.unwrap()
|
.unwrap_or_default()["common"]["fabric"]
|
||||||
.to_string_lossy()
|
.as_str()
|
||||||
.into_owned();
|
{
|
||||||
|
Some(fabric) => basedir.join(format!("{fabric}.yaml")),
|
||||||
path.parent()
|
None => {
|
||||||
.unwrap()
|
verb!(
|
||||||
.parent()
|
"[?] {} missing common.fabric, falling back to regional default",
|
||||||
.unwrap()
|
common.display()
|
||||||
.join("common")
|
);
|
||||||
.join(format!("{}.yaml", &dir_name[0..2]))
|
basedir.join("default.yaml")
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_common(spec: &PathBuf) -> PathBuf {
|
fn get_common(spec: &PathBuf) -> PathBuf {
|
||||||
@@ -139,17 +137,17 @@ impl Specification {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn get_zonal(spec: &PathBuf) -> PathBuf {
|
fn get_zonal(spec: &PathBuf) -> PathBuf {
|
||||||
let zone = spec
|
spec.parent().unwrap().join(format!(
|
||||||
.file_name()
|
"{}.common.yaml",
|
||||||
|
spec.file_name()
|
||||||
.and_then(|f| f.to_str())
|
.and_then(|f| f.to_str())
|
||||||
.and_then(|f| f.split_once('-'))
|
.and_then(|f| f.split_once('-'))
|
||||||
.map(|(zone, _)| spec.with_file_name(zone))
|
.map(|(zone, _)| zone)
|
||||||
.unwrap_or_else(|| spec.with_file_name("common.yaml"));
|
.unwrap_or("none")
|
||||||
|
))
|
||||||
PathBuf::from(format!("{}.common.yaml", zone.to_str().unwrap()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn filter_specs(pattern: &Regex, spec_list: Vec<PathBuf>) -> Vec<PathBuf> {
|
fn filter_specs(pattern: ®ex::Regex, spec_list: Vec<PathBuf>) -> Vec<PathBuf> {
|
||||||
spec_list
|
spec_list
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|spec| {
|
.filter(|spec| {
|
||||||
@@ -163,7 +161,10 @@ impl Specification {
|
|||||||
fn get_specs(spec_path: &PathBuf) -> Vec<PathBuf> {
|
fn get_specs(spec_path: &PathBuf) -> Vec<PathBuf> {
|
||||||
let mut result = Vec::new();
|
let mut result = Vec::new();
|
||||||
let root = spec_path.as_path();
|
let root = spec_path.as_path();
|
||||||
for entry in WalkDir::new(root).into_iter().filter_map(|e| e.ok()) {
|
for entry in walkdir::WalkDir::new(root)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
{
|
||||||
if let Ok(path) = entry.path().strip_prefix(root) {
|
if let Ok(path) = entry.path().strip_prefix(root) {
|
||||||
let path_str = path.to_string_lossy().to_string();
|
let path_str = path.to_string_lossy().to_string();
|
||||||
if !path_str.contains("common") && path_str.ends_with("yaml") {
|
if !path_str.contains("common") && path_str.ends_with("yaml") {
|
||||||
@@ -179,8 +180,8 @@ impl std::fmt::Display for Specification {
|
|||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||||
write!(
|
write!(
|
||||||
f,
|
f,
|
||||||
"Compiled Specification: {}\nComponents: {}",
|
"| Compiled Specification: {}\n| Components: {:#?}\n| Outpath: {:?}\n",
|
||||||
self.compiled, self.components,
|
self.compiled, self.components, self.outpath,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -203,17 +204,3 @@ impl Components {
|
|||||||
.to_string()
|
.to_string()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for Components {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
|
||||||
write!(
|
|
||||||
f,
|
|
||||||
" | Partitional: {}\n | Regional: {}\n | Common: {}\n | Zonal: {}\n | Device: {}\n",
|
|
||||||
self.partitional.display(),
|
|
||||||
self.regional.display(),
|
|
||||||
self.common.display(),
|
|
||||||
self.zonal.display(),
|
|
||||||
self.device.display(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
use crate::{LogLevel, crit, info};
|
use crate::{LogLevel, crit, verb};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
@@ -17,18 +17,18 @@ impl Structure {
|
|||||||
Ok(yaml_serde::from_str(&data)?)
|
Ok(yaml_serde::from_str(&data)?)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn load_template_data(&self, tmpl_dir: PathBuf) -> Vec<(String, String)> {
|
pub fn load_template_data(&self, tmpl_dir: &PathBuf) -> Vec<(String, String)> {
|
||||||
self.files
|
self.files
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|name| {
|
.filter_map(|name| {
|
||||||
let template = tmpl_dir.join(format!("{}.tera", name));
|
let template = tmpl_dir.join(format!("{}.tera", name));
|
||||||
match std::fs::read_to_string(&template) {
|
match std::fs::read_to_string(&template) {
|
||||||
Ok(content) => {
|
Ok(content) => {
|
||||||
info!(" | {}", template.display());
|
verb!(" | {}", template.display());
|
||||||
Some((name.clone(), content))
|
Some((name.clone(), content))
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
crit!("[!] {}: {e}", template.display());
|
crit!("[!] {:?}: {e}", template);
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user