Hemera/nixie-ci
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add ability to build nix derivations

Browse source 
Signed-off-by: Marcel Müller <neikos@neikos.email>
This commit is contained in:
Marcel Müller 2026-01-11 11:43:24 +01:00
parent 85c6a8f169
commit 8005aeccd9
8 changed files with 296 additions and 97 deletions
Download patch file Download diff file Expand all files Collapse all files

32
nixie-build/foo.rs Normal file
View file

@ -0,0 +1,32 @@
{
"/nix/store/6hv3vg9g9jf9z4bbmkz6b0yixk7h48fv-nixie-ci-0.1.0.drv": null,
"/nix/store/8brm4045abq27fa7i2v68yx2q5d70mdr-zstd-1.5.7.drv": {
"ca": "text:sha256:1r5zh6h09l9a1gzkrvccp6xxi087365wr22pahgz5cm8sg1ssp8a",
"compression": "xz",
"deriver": null,
"downloadHash": "sha256-JkICxO8KGTN9ehMWt3bbEcPPZgb9RlDQscmI0OBJlXw=",
"downloadSize": 1688,
"narHash": "sha256-BPtqm0GZEJseFXwZSGO/lYRWBgdlRsqoyAMEEWozacI=",
"narSize": 3920,
"references": [
"/nix/store/377lw5m13cliaj6328iz9il687qxwahb-source.drv",
"/nix/store/5mv61fp0f6l28m4bvanzxr82lzb7d5s3-file-5.45.drv",
"/nix/store/6k4n3g5jahyxqn6m7bdviydxibzp7xx1-playtests-darwin.patch",
"/nix/store/6li69pci6if48q2bjir0p58qvf8iy703-man-fix.patch.drv",
"/nix/store/hi1hlj16r6gwpzk8kk3wnmq1wjgyf3jf-gnugrep-3.12.drv",
"/nix/store/l622p70vy8k5sh7y5wizi5f2mic6ynpg-source-stdenv.sh",
"/nix/store/pnjvpwgka59d6fwpp9fnz42ll2ai4ffm-stdenv-linux.drv",
"/nix/store/shkw4qm9qcw5sc5n1k5jznc83ny02r39-default-builder.sh",
"/nix/store/vwmk63kc9sysjif65h7fdwnqr5h8jfm6-bash-5.3p3.drv",
"/nix/store/xd9slhxpvs9wl0hgd36x4fkfnp2ybb9i-cmake-minimal-4.1.2.drv"
],
"registrationTime": null,
"signatures": [
"cache.nixos.org-1:6mzDXuIB96xqsrtAl7n8Mh9zgkWzVvpj+bEpo5H6c+KNKME4z7xdR7WYiFH6iERwWFCO3v387ps+AkcY/GQeCQ=="
],
"ultimate": false,
"url": "nar/0z4m97hd1269n7850ipx0rkczhqivdvbf5hkg9yk668axz204hi6.nar.xz"
},
"/nix/store/d41hh410m104pnid9vynwbpng3m4w3gh-nixie-ci-fmt-0.1.0.drv": null
}

243
nixie-build/src/lib.rs
View file

@ -1,18 +1,23 @@
use std::collections::HashMap;
use std::fmt::Write;
use std::collections::HashSet;
use std::process::Stdio;
use displaydoc::Display;
use futures::FutureExt;
use futures::StreamExt;
use futures::stream::BoxStream;
use nix_json::ActivityKind;
use nix_json::NixBuildLogLine;
use nix_json::RawNixDerivationInfoOutput;
use nix_json::ResultKind;
use nix_json::helpers::NixBuildState;
use petgraph::Directed;
use petgraph::Graph;
use petgraph::acyclic::Acyclic;
use petgraph::data::Build;
use petgraph::prelude::NodeIndex;
use thiserror::Error;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
use tokio::process::Command;
#[derive(Debug)]
@ -36,8 +41,11 @@ impl NixBuildResult {
}
}
#[derive(Debug)]
pub enum NixBuildError {}
#[derive(Debug, Error, Display)]
pub enum NixBuildError {
/// No error
NoneYet,
}
pub trait NixInProgressBuild {
fn next_log_line(
@ -59,12 +67,24 @@ pub trait NixBackend {
&self,
derivation: &str,
) -> impl Future<Output = Result<NixBuildGraph, NixBuildError>>;
fn check_if_paths_in_cache(
&self,
store: String,
paths: &[String],
) -> impl Future<Output = Result<HashMap<String, bool>, NixBuildError>>;
}
pub struct NixCliBackend {
command_path: String,
}
impl NixCliBackend {
pub fn new(command_path: String) -> Self {
Self { command_path }
}
}
pub struct NixCliBackendBuild {
child: BoxStream<'static, String>,
}
@ -82,84 +102,117 @@ impl NixInProgressBuild for NixCliBackendBuild {
impl NixBackend for NixCliBackend {
type InProgressBuild<'b> = NixCliBackendBuild;
fn start_build(
async fn start_build(
&self,
derivation: &str,
) -> impl Future<Output = Result<Self::InProgressBuild<'_>, NixBuildError>> {
async move {
let mut cmd = Command::new(&self.command_path)
.args(["build", "--log-format", "internal-json"])
.arg(derivation)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
) -> Result<Self::InProgressBuild<'_>, NixBuildError> {
let mut cmd = Command::new(&self.command_path)
.args(["build", "--log-format", "internal-json"])
.arg(derivation)
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let init = cmd.stderr.take().unwrap();
Ok(NixCliBackendBuild {
child: futures::stream::unfold(
tokio::io::BufReader::new(init),
move |mut state| async move {
let mut buffer = String::new();
let num_bytes = state.read_line(&mut buffer).await.unwrap();
let init = cmd.stderr.take().unwrap();
Ok(NixCliBackendBuild {
child: futures::stream::unfold(
tokio::io::BufReader::new(init),
move |mut state| async move {
let mut buffer = String::new();
let num_bytes = state.read_line(&mut buffer).await.unwrap();
if num_bytes == 0 {
None
} else {
Some((std::mem::take(&mut buffer), state))
}
},
)
.boxed(),
})
}
if num_bytes == 0 {
None
} else {
Some((std::mem::take(&mut buffer), state))
}
},
)
.boxed(),
})
}
fn get_needed_builds(
&self,
derivation: &str,
) -> impl Future<Output = Result<NixBuildGraph, NixBuildError>> {
async move {
let cmd = Command::new(&self.command_path)
.args(["derivation", "show", "--recursive"])
.arg(derivation)
.output()
.await
.unwrap();
async fn get_needed_builds(&self, derivation: &str) -> Result<NixBuildGraph, NixBuildError> {
let cmd = Command::new(&self.command_path)
.args(["derivation", "show", "--recursive"])
.arg(derivation)
.output()
.await
.unwrap();
let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap();
let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap();
let mut build_graph = NixBuildGraph::default();
let mut build_graph = NixBuildGraph::default();
for (path, _info) in output.info() {
let internal_id = build_graph.dependencies.add_node(path.to_string());
build_graph.build_infos.insert(
path.to_string(),
NixBuildInfo {
internal_id,
present_in_binary_cache: false,
for path in output.info().keys() {
let internal_id = build_graph.dependencies.add_node(path.to_string());
build_graph.build_infos.insert(
path.to_string(),
NixBuildInfo {
internal_id,
present_in_binary_cache: false,
},
);
}
for (path, info) in output.info() {
let build_info = &build_graph.build_infos[path];
let cur_node = build_info.internal_id;
for dep_path in info.input_derivations.keys() {
let other_node = build_graph.build_infos[dep_path].internal_id;
build_graph.dependencies.add_edge(
cur_node,
other_node,
NixBuildOutput {
output_name: String::new(),
},
);
}
for (path, info) in output.info() {
let build_info = &build_graph.build_infos[path];
let cur_node = build_info.internal_id;
for (dep_path, _dep_info) in &info.input_derivations {
let other_node = build_graph.build_infos[dep_path].internal_id;
build_graph.dependencies.add_edge(
cur_node,
other_node,
NixBuildOutput {
output_name: String::new(),
},
);
}
}
Ok(build_graph)
}
Ok(build_graph)
}
async fn check_if_paths_in_cache(
&self,
store: String,
paths: &[String],
) -> Result<HashMap<String, bool>, NixBuildError> {
let mut cmd = Command::new(&self.command_path)
.args(["path-info", "--store", &store, "--json", "--stdin"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.unwrap();
let mut stdout = cmd.stdout.take().unwrap();
// Write all to stdin, and then have it drop at the end of the block
{
let paths = paths.join(" ");
let mut stdin = cmd.stdin.take().unwrap();
stdin.write_all(paths.as_ref()).await.unwrap();
stdin.shutdown().await.unwrap();
}
println!("Wrote all to stdin!");
let mut output = vec![];
stdout.read_to_end(&mut output).await.unwrap();
tokio::spawn(async move { cmd.wait().await });
let store_info: HashMap<String, Option<serde_json::Value>> =
serde_json::from_slice(&output).unwrap();
Ok(store_info
.into_iter()
.map(|(k, v)| (k, v.is_some()))
.collect())
}
}
@ -178,14 +231,20 @@ pub struct NixBuildOutput {
output_name: String,
}
impl NixBuildOutput {
pub fn output_name(&self) -> &str {
&self.output_name
}
}
#[derive(Debug, Default)]
pub struct NixBuildGraph {
build_infos: HashMap<String, NixBuildInfo>,
dependencies: Graph<String, NixBuildOutput, Directed, usize>,
dependencies: Acyclic<Graph<String, NixBuildOutput, Directed, usize>>,
}
impl NixBuildGraph {
fn get_non_binary_builds(&self) -> impl Iterator<Item = (String, NixBuildInfo)> {
pub fn get_non_binary_builds(&self) -> impl Iterator<Item = (String, NixBuildInfo)> {
self.build_infos
.iter()
.filter(|(_, value)| !value.present_in_binary_cache)
@ -197,6 +256,17 @@ impl<B> NixBuilder<B>
where
B: NixBackend,
{
pub fn new(backend: B) -> Self {
Self { backend }
}
pub async fn get_needed_builds(
&self,
derivation: &str,
) -> Result<NixBuildGraph, NixBuildError> {
self.backend.get_needed_builds(derivation).await
}
pub async fn build(
&self,
derivation: String,
@ -216,30 +286,23 @@ where
)
}));
let mut id_to_derivation = HashMap::new();
let mut build_state = NixBuildState::default();
while let Some(next_log_line) = started_build.next_log_line().await {
let next_log_line = next_log_line?;
if let NixBuildLogLine::Start(start_log) = next_log_line {
if start_log.kind == ActivityKind::Build {
id_to_derivation.insert(
start_log.id,
start_log.fields[0].as_str().unwrap().to_string(),
);
}
} else if let NixBuildLogLine::Result(result) = next_log_line {
if result.kind == ResultKind::BuildLogLine {
if let Some(build) = actually_built.get_mut(&id_to_derivation[&result.id]) {
writeln!(&mut build.log, "{}", result.fields[0].as_str().unwrap()).unwrap()
} else {
panic!("Could not find correct id");
}
}
}
build_state.handle_log_line(next_log_line);
}
actually_built.retain(|_k, v| !v.log.is_empty());
let mut actually_built_derivations = HashSet::new();
for drv in build_state.derivations() {
let built = actually_built.get_mut(drv.derivation_path()).unwrap();
built.success = drv.is_success();
actually_built_derivations.insert(drv.derivation_path());
}
actually_built.retain(|k, _| actually_built_derivations.contains(&k.as_ref()));
Ok(actually_built)
}