lost/skyforge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize load spec; update logging; parallelize output

Browse Source
This commit is contained in:
lost
2026-02-27 05:00:28 -07:00
parent d8134095ab
commit 92b36e09b7
12 changed files with 195 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -2136,6 +2136,7 @@ dependencies = [
"clap",
"criterion",
"flamegraph",
"rayon",
"regex",
"serde",
"serde_json",

1
Cargo.toml
View File

@@ -10,6 +10,7 @@ debug = true
[dependencies]
anyhow = "1"
clap = { version = "4.5" }
rayon = "1"
regex = "1"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

3
README.md
View File

@@ -167,6 +167,5 @@ cargo flamegraph --post-process 'flamelens --echo' --profile profiling -- --devi
Using Demo:
``` bash
process_specs time: [1.6096 ms 1.6205 ms 1.6327 ms]
from_spec time: [3.3230 ms 3.3397 ms 3.3575 ms]
process_specs time: [1.2339 ms 1.2399 ms 1.2463 ms]
```

21
benches/benchmark.rs
View File

@@ -16,31 +16,14 @@ fn benchmark(c: &mut Criterion) {
LOG_LEVEL.set(LogLevel::Warning).ok();
let args = Args {
devices: regex::Regex::new(".*").unwrap(),
classic: false,
env: env(),
};
c.bench_function("process_specs", |b| {
b.iter(|| {
DeviceConfigBundle::process_specs(Specification::compile(&args)).unwrap();
DeviceConfigBundle::process_specs(Specification::compile(&args).unwrap()).unwrap();
});
});
}
fn benchmark_classic(c: &mut Criterion) {
LOG_LEVEL.set(LogLevel::Warning).ok();
let args = Args {
devices: regex::Regex::new(".*").unwrap(),
classic: true,
env: env(),
};
c.bench_function("from_spec", |b| {
b.iter(|| {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec).unwrap();
}
});
});
}
criterion_group!(benches, benchmark, benchmark_classic);
criterion_group!(benches, benchmark);
criterion_main!(benches);

10
src/cli.rs
View File

@@ -15,7 +15,6 @@ const ENV_MSG: &str = r#"Environment:
#[derive(Debug)]
pub struct Args {
pub devices: Regex,
pub classic: bool,
pub env: EnvVars,
}
@@ -39,7 +38,6 @@ impl Args {
Args {
devices,
classic: matches.get_flag("classic"),
env: EnvVars::parse(),
}
}
@@ -79,14 +77,6 @@ impl Args {
.action(ArgAction::SetTrue)
.required(false),
)
.arg(
Arg::new("classic")
.short('c')
.long("classic")
.help("creates a new renderer for each device")
.action(ArgAction::SetTrue)
.required(false),
)
.group(
ArgGroup::new("loglevel")
.args(&["debug", "verbose", "silent"])

33
src/error.rs Normal file
View File

@@ -0,0 +1,33 @@
pub trait ErrorExt {
fn chain_msg(&self) -> String;
}
impl<E: std::error::Error> ErrorExt for E {
fn chain_msg(&self) -> String {
let mut out = self.to_string();
let mut source = self.source();
while let Some(s) = source {
out.push_str(&format!("\n > {}", s));
source = s.source();
}
out
}
}
#[derive(Debug)]
pub struct ContextError {
pub message: String,
pub source: Box<dyn std::error::Error>,
}
impl std::fmt::Display for ContextError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.message)
}
}
impl std::error::Error for ContextError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&*self.source)
}
}

1
src/lib.rs
View File

@@ -1,4 +1,5 @@
pub mod cli;
pub mod error;
pub mod log;
pub mod render;
pub mod spec;

10
src/log.rs
View File

@@ -25,7 +25,7 @@ impl std::fmt::Display for LogLevel {
#[macro_export]
macro_rules! info {
($($msg:expr),*) => {
if LogLevel::Info >= $crate::log_level() {
if $crate::log::LogLevel::Info >= $crate::log_level() {
println!($($msg),*);
}
};
@@ -34,7 +34,7 @@ macro_rules! info {
#[macro_export]
macro_rules! verb {
($($msg:expr),*) => {
if LogLevel::Verbose >= $crate::log_level() {
if $crate::log::LogLevel::Verbose >= $crate::log_level() {
println!($($msg),*);
}
};
@@ -43,7 +43,7 @@ macro_rules! verb {
#[macro_export]
macro_rules! dbug {
($($msg:expr),*) => {
if LogLevel::Debug >= $crate::log_level() {
if $crate::log::LogLevel::Debug >= $crate::log_level() {
println!($($msg),*);
}
};
@@ -52,7 +52,7 @@ macro_rules! dbug {
#[macro_export]
macro_rules! warn {
($($msg:expr),*) => {
if LogLevel::Warning >= $crate::log_level() {
if $crate::log::LogLevel::Warning >= $crate::log_level() {
eprintln!($($msg),*);
}
};
@@ -61,7 +61,7 @@ macro_rules! warn {
#[macro_export]
macro_rules! crit {
($($msg:expr),*) => {
if LogLevel::Critical >= $crate::log_level() {
if $crate::log::LogLevel::Critical >= $crate::log_level() {
eprintln!($($msg),*);
}
};

12
src/main.rs
View File

@@ -1,3 +1,4 @@
use rayon::prelude::*;
use skyforge::{Args, DeviceConfigBundle, Specification};
fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -5,15 +6,10 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
args.use_tmp();
if args.classic {
for spec in Specification::compile(&args) {
DeviceConfigBundle::from_spec(spec)?.output_artifacts();
}
} else {
DeviceConfigBundle::process_specs(Specification::compile(&args))?
.into_iter()
let specifications = Specification::compile(&args)?;
DeviceConfigBundle::process_specs(specifications)?
.into_par_iter()
.for_each(|c| c.output_artifacts());
}
println!("Completed Successfully");
Ok(())

120
src/render.rs
View File

@@ -1,8 +1,8 @@
use crate::log::LogLevel;
use crate::error::{ContextError, ErrorExt};
use crate::spec::Specification;
use crate::tmpl;
use crate::{crit, dbug, info, verb};
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::error::Error;
use std::io::Write;
use std::path::PathBuf;
@@ -27,77 +27,45 @@ pub struct Configuration {
pub struct Renderer {
pub engine: Tera,
pub from: PathBuf,
pub structure: tmpl::Structure,
}
impl DeviceConfigBundle {
pub fn from_spec(spec: Specification) -> Result<Self, Box<dyn Error>> {
let structure =
tmpl::Structure::from_file(&spec.tmplpath.join("structure.yaml")).map_err(|e| {
crit!("{}", e);
e
})?;
let mut renderer = Tera::default();
renderer.add_raw_templates(structure.load_template_data(&spec.tmplpath))?;
Self::render_configs(spec, &structure, &renderer)
}
pub fn process_specs(specs: Vec<Specification>) -> Result<Vec<Self>, Box<dyn Error>> {
let mut failed = false;
let renderers: Vec<Renderer> = specs
let renderers: HashMap<PathBuf, Renderer> = specs
.iter()
.map(|s| s.tmplpath.clone())
.collect::<HashSet<PathBuf>>()
.map(|s| &s.tmplpath)
.collect::<HashSet<&PathBuf>>()
.into_iter()
.filter_map(|p| {
let structure = tmpl::Structure::from_file(&p.join("structure.yaml"))
Self::get_renderer(&p)
.map_err(|e| {
crit!("[!] {}", e);
crit!("{}", e.chain_msg());
failed = true;
})
.ok()?;
let mut t = Tera::default();
t.add_raw_templates(structure.load_template_data(&p))
.map_err(|e| {
crit!("While loading {}: {}", &p.display(), e.chain_msg());
failed = true;
})
.ok()?;
Some(Renderer { engine: t, from: p })
.ok()
})
.collect();
if failed {
return Err("Failed to load templates.".into());
}
info!("Rendering configs:");
let results: Vec<Result<Self, _>> = specs
let mut failed = false;
let successes: Vec<Self> = specs
.into_iter()
.map(|s| {
let renderer =
renderers
.iter()
.find(|r| r.from == s.tmplpath)
.ok_or_else(|| {
format!(
"Unexpected. Missing Tera renderer: {}",
s.tmplpath.display()
)
})?;
let structure = tmpl::Structure::from_file(&s.tmplpath.join("structure.yaml"))
.filter_map(|s| {
let renderer = renderers
.get(&s.tmplpath)
.expect("renderer exists for all tmplaths");
Self::render_configs(s, &renderer.structure, &renderer.engine)
.map_err(|e| {
crit!("{}", e);
e
})?;
Self::render_configs(s, &structure, &renderer.engine)
failed = true;
})
.ok()
})
.collect();
failed = results.iter().any(|r| r.is_err());
let successes: Vec<Self> = results
.into_iter()
.filter_map(|r| r.map_err(|e| crit!("{}", e)).ok())
.collect();
if failed {
@@ -107,6 +75,23 @@ impl DeviceConfigBundle {
Ok(successes)
}
fn get_renderer(tmplpath: &PathBuf) -> Result<(PathBuf, Renderer), ContextError> {
let structure = Self::get_structure(&tmplpath)?;
let mut t = Tera::default();
t.add_raw_templates(structure.load_template_data(&tmplpath))
.map_err(|e| ContextError {
message: format!("While loading {}", tmplpath.display()),
source: Box::new(e),
})?;
Ok((
tmplpath.to_owned(),
Renderer {
engine: t,
structure,
},
))
}
fn render_configs(
spec: Specification,
structure: &tmpl::Structure,
@@ -125,7 +110,7 @@ impl DeviceConfigBundle {
dbug!(" |\t{template_name}.{variant}");
match renderer.render(template_name, &context) {
Ok(data) => cfgs.push(Configuration {
name: String::from(template_name),
name: template_name.to_owned(),
data,
}),
Err(e) => {
@@ -135,7 +120,7 @@ impl DeviceConfigBundle {
}
}
configs.push(ConfigVariant {
name: String::from(variant),
name: variant.to_owned(),
configs: cfgs,
});
context.remove(variant);
@@ -149,7 +134,15 @@ impl DeviceConfigBundle {
hostname,
configs,
spec,
platform: structure.platform.clone(),
platform: structure.platform.to_owned(),
})
}
fn get_structure(tmplpath: &PathBuf) -> Result<tmpl::Structure, ContextError> {
let structure_path = &tmplpath.join("structure.yaml");
tmpl::Structure::from_file(structure_path).map_err(|e| ContextError {
message: format!("While loading {}", structure_path.display()),
source: e,
})
}
@@ -218,25 +211,10 @@ impl DeviceConfigBundle {
verb!(" | {}", &compiled_spec_path.display());
if let Err(e) = std::fs::write(
&compiled_spec_path,
yaml_serde::to_string(&self.spec.compiled).unwrap(),
yaml_serde::to_string(&self.spec.compiled)
.expect("spec was successfully compiled and must be serializable"),
) {
crit!("Failed to write {}: {}", compiled_spec_path.display(), e);
}
}
}
trait ErrorExt {
fn chain_msg(&self) -> String;
}
impl ErrorExt for tera::Error {
fn chain_msg(&self) -> String {
let mut out = self.to_string();
let mut source = self.source();
while let Some(s) = source {
out.push_str(&format!("\n > {}", s));
source = s.source();
}
out
}
}

151
src/spec.rs
View File

@@ -1,6 +1,8 @@
use crate::cli::Args;
use crate::{LogLevel, dbug, info, verb};
use crate::error::*;
use crate::{crit, dbug, info, verb};
use serde_json::Value;
use std::collections::HashMap;
use std::path::PathBuf;
#[derive(Debug)]
@@ -23,24 +25,92 @@ trait Pipe: Sized {
impl<T> Pipe for T {}
impl Specification {
pub fn build(
/// compiles specifications from regex and paths provided by Args
pub fn compile(args: &Args) -> Result<Vec<Specification>, Box<dyn std::error::Error>> {
let mut file_cache: HashMap<PathBuf, Value> = HashMap::new();
let mut failed = false;
let specifications: Vec<Specification> = Self::get_specs(&args.env.spec_path)
.pipe(|p| {
info!("Skyforge found {} renderable devices", p.len());
Self::filter_specs(&args.devices, p)
})
.tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices))
.into_iter()
.filter_map(|spec| {
verb!(" | {}", spec.display());
let common = Self::get_common(&spec);
let regional = Self::get_regional(&common);
Self::build_cached(
Components {
partitional: Self::get_partitional(&regional),
regional,
common,
zonal: Self::get_zonal(&spec),
device: spec.to_owned(),
},
args.env.out_path.to_owned(),
args.env.tmpl_path.to_owned(),
&mut file_cache,
)
.map_err(|e| {
crit!(
"Failed to build Specification for {}:\n{}",
spec.display(),
e.chain_msg()
);
failed = true;
})
.ok()
})
.collect();
if failed {
return Err("One or more specifications failed to compile.".into());
}
Ok(specifications)
}
fn build_cached(
components: Components,
outpath: PathBuf,
tmplpath: PathBuf,
) -> Result<Self, Box<dyn std::error::Error>> {
cache: &mut HashMap<PathBuf, Value>,
) -> Result<Self, ContextError> {
dbug!("{:#?}", components);
let merged_map: serde_json::Map<String, Value> = [
&components.partitional,
&components.regional,
&components.common,
&components.zonal,
&components.device,
components.partitional.as_ref(),
Some(&components.regional),
Some(&components.common),
Some(&components.zonal),
Some(&components.device),
]
.iter()
.filter_map(|path| std::fs::read_to_string(path).ok())
.map(|content| yaml_serde::from_str::<Value>(&content))
.collect::<Result<Vec<Value>, _>>()?
.into_iter()
.flatten()
.map(|path| -> Result<Option<Value>, ContextError> {
if let Some(cached) = cache.get(path) {
return Ok(Some(cached.clone()));
}
match std::fs::read_to_string(path) {
Ok(content) => {
let parsed =
yaml_serde::from_str::<Value>(&content).map_err(|e| ContextError {
message: format!("Failed to parse {}", path.display()),
source: Box::new(e),
})?;
cache.insert(path.clone(), parsed.clone());
Ok(Some(parsed))
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(ContextError {
message: format!("Failed to read {}", path.display()),
source: Box::new(e),
}),
}
})
.collect::<Result<Vec<Option<Value>>, _>>()?
.into_iter()
.flatten()
.filter_map(|v| v.as_object().cloned())
.flat_map(|obj| obj.into_values().filter_map(|v| v.as_object().cloned()))
.flatten()
@@ -68,36 +138,7 @@ impl Specification {
)
}
/// compiles specifications from regex and paths provided by Args
pub fn compile(args: &Args) -> Vec<Specification> {
Self::get_specs(&args.env.spec_path)
.pipe(|p| {
info!("Skyforge found {} renderable devices", p.len());
Self::filter_specs(&args.devices, p)
})
.tap(|p| info!("Matched {} devices against '{}'", p.len(), &args.devices))
.into_iter()
.map(|spec| {
verb!(" | {}", spec.display());
let common = Self::get_common(&spec);
let regional = Self::get_regional(&common);
Self::build(
Components {
partitional: Self::get_partitional(&regional),
regional,
common,
zonal: Self::get_zonal(&spec),
device: spec,
},
args.env.out_path.clone(),
args.env.tmpl_path.clone(),
)
.expect("failed to build Specification")
})
.collect()
}
fn get_partitional(regional: &PathBuf) -> PathBuf {
fn get_partitional(regional: &PathBuf) -> Option<PathBuf> {
let basedir = regional.ancestors().nth(3).unwrap().join("common");
match yaml_serde::from_str::<yaml_serde::Value>(
&std::fs::read_to_string(regional).unwrap_or_default(),
@@ -105,10 +146,10 @@ impl Specification {
.unwrap_or_default()["regional"]["partition"]
.as_str()
{
Some(partition) => basedir.join(format!("{partition}.yaml")),
Some(partition) => Some(basedir.join(format!("{partition}.yaml"))),
None => {
verb!("[?] {} missing regional.partition", regional.display());
basedir.join("default.yaml")
None
}
}
}
@@ -159,20 +200,20 @@ impl Specification {
}
fn get_specs(spec_path: &PathBuf) -> Vec<PathBuf> {
let mut result = Vec::new();
let root = spec_path.as_path();
for entry in walkdir::WalkDir::new(root)
walkdir::WalkDir::new(root)
.into_iter()
.filter_map(|e| e.ok())
{
if let Ok(path) = entry.path().strip_prefix(root) {
let path_str = path.to_string_lossy().to_string();
.filter_map(|e| {
let entry = e.ok()?;
let path = entry.path().strip_prefix(root).ok()?;
let path_str = path.to_string_lossy();
if !path_str.contains("common") && path_str.ends_with("yaml") {
result.push(root.join(path));
Some(root.join(path))
} else {
None
}
}
}
result
})
.collect()
}
}
@@ -188,7 +229,7 @@ impl std::fmt::Display for Specification {
#[derive(Debug)]
pub struct Components {
partitional: PathBuf,
partitional: Option<PathBuf>,
regional: PathBuf,
common: PathBuf,
zonal: PathBuf,

2
src/tmpl.rs
View File

@@ -1,4 +1,4 @@
use crate::{LogLevel, crit, dbug};
use crate::{crit, dbug};
use serde::Deserialize;
use std::error::Error;
use std::path::PathBuf;