Add ability to build nix derivations

Signed-off-by: Marcel Müller <neikos@neikos.email>

Cargo.lock

@@ -61,6 +61,12 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
+
 [[package]]
 name = "bit-set"
 version = "0.8.0"
@@ -432,6 +438,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "nixie-cli"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "nixie-build",
+ "tokio",
+]
+
+[[package]]
+name = "nixie-server"
+version = "0.1.0"
+
 [[package]]
 name = "once_cell_polyfill"
 version = "1.70.2"

Cargo.toml

@@ -1,6 +1,6 @@
 [workspace]
 
-members = ["nix-json", "nixie-build"]
+members = ["nix-json", "nixie-build", "nixie-cli"]
 
 resolver = "3"
 
@@ -21,9 +21,12 @@ petgraph = "0.8"
 tokio = { features = ["full"], version = "1.48" }
 futures = "0.3"
 datatest-stable = "0.3.3"
+itertools = "0.14"
+anyhow = "1.0.100"
+clap = "4.5.54"
 
 nix-json = { version = "0.1.0", path = "./nix-json" }
-
+nixie-build = { version = "0.1.0", path = "./nixie-build" }
 
 [workspace.metadata.crane]
 name = "nixie-ci"

nix-json/Cargo.toml

@@ -16,7 +16,7 @@ serde_json.workspace = true
 serde_repr.workspace = true
 thiserror = { workspace = true }
 displaydoc = { workspace = true }
-itertools = "0.14"
+itertools = { workspace = true }
 
 [dev-dependencies]
 datatest-stable.workspace = true

nix-json/src/helpers.rs

@@ -17,7 +17,7 @@ pub enum LogBuildState {
 #[derive(Debug)]
 pub struct LogBuildStatus {
     state: LogBuildState,
-    store_path: String,
+    derivation_path: String,
     log_lines: Vec<String>,
 }
 
@@ -29,6 +29,14 @@ impl LogBuildStatus {
     pub fn log_lines(&self) -> &[String] {
         &self.log_lines
     }
+
+    pub fn derivation_path(&self) -> &str {
+        &self.derivation_path
+    }
+
+    pub fn is_success(&self) -> bool {
+        self.state == LogBuildState::Succeeded
+    }
 }
 
 /// Accumulated build status of different nix-builds
@@ -109,7 +117,10 @@ impl NixBuildState {
                     nix_log_start_action.id,
                     LogBuildStatus {
                         state: LogBuildState::Started,
-                        store_path: nix_log_start_action.fields[0].as_str().unwrap().to_string(),
+                        derivation_path: nix_log_start_action.fields[0]
+                            .as_str()
+                            .unwrap()
+                            .to_string(),
                         log_lines: vec![],
                     },
                 );
@@ -171,13 +182,17 @@ impl NixBuildState {
 
     fn handle_log_action(
         &self,
-        nix_log_msg_action: crate::NixLogMsgAction,
+        _nix_log_msg_action: crate::NixLogMsgAction,
     ) -> Option<NixBuildEvent> {
         None
     }
 
     pub fn get_derivation(&self, drv: &str) -> Option<&LogBuildStatus> {
-        self.builds.values().find(|b| b.store_path == drv)
+        self.builds.values().find(|b| b.derivation_path == drv)
+    }
+
+    pub fn derivations(&self) -> impl Iterator<Item = &LogBuildStatus> {
+        self.builds.values()
     }
 }
 

nixie-build/foo.rs

@@ -0,0 +1,32 @@
+
+{
+  "/nix/store/6hv3vg9g9jf9z4bbmkz6b0yixk7h48fv-nixie-ci-0.1.0.drv": null,
+  "/nix/store/8brm4045abq27fa7i2v68yx2q5d70mdr-zstd-1.5.7.drv": {
+    "ca": "text:sha256:1r5zh6h09l9a1gzkrvccp6xxi087365wr22pahgz5cm8sg1ssp8a",
+    "compression": "xz",
+    "deriver": null,
+    "downloadHash": "sha256-JkICxO8KGTN9ehMWt3bbEcPPZgb9RlDQscmI0OBJlXw=",
+    "downloadSize": 1688,
+    "narHash": "sha256-BPtqm0GZEJseFXwZSGO/lYRWBgdlRsqoyAMEEWozacI=",
+    "narSize": 3920,
+    "references": [
+      "/nix/store/377lw5m13cliaj6328iz9il687qxwahb-source.drv",
+      "/nix/store/5mv61fp0f6l28m4bvanzxr82lzb7d5s3-file-5.45.drv",
+      "/nix/store/6k4n3g5jahyxqn6m7bdviydxibzp7xx1-playtests-darwin.patch",
+      "/nix/store/6li69pci6if48q2bjir0p58qvf8iy703-man-fix.patch.drv",
+      "/nix/store/hi1hlj16r6gwpzk8kk3wnmq1wjgyf3jf-gnugrep-3.12.drv",
+      "/nix/store/l622p70vy8k5sh7y5wizi5f2mic6ynpg-source-stdenv.sh",
+      "/nix/store/pnjvpwgka59d6fwpp9fnz42ll2ai4ffm-stdenv-linux.drv",
+      "/nix/store/shkw4qm9qcw5sc5n1k5jznc83ny02r39-default-builder.sh",
+      "/nix/store/vwmk63kc9sysjif65h7fdwnqr5h8jfm6-bash-5.3p3.drv",
+      "/nix/store/xd9slhxpvs9wl0hgd36x4fkfnp2ybb9i-cmake-minimal-4.1.2.drv"
+    ],
+    "registrationTime": null,
+    "signatures": [
+      "cache.nixos.org-1:6mzDXuIB96xqsrtAl7n8Mh9zgkWzVvpj+bEpo5H6c+KNKME4z7xdR7WYiFH6iERwWFCO3v387ps+AkcY/GQeCQ=="
+    ],
+    "ultimate": false,
+    "url": "nar/0z4m97hd1269n7850ipx0rkczhqivdvbf5hkg9yk668axz204hi6.nar.xz"
+  },
+  "/nix/store/d41hh410m104pnid9vynwbpng3m4w3gh-nixie-ci-fmt-0.1.0.drv": null
+}

nixie-build/src/lib.rs

@@ -1,18 +1,23 @@
 use std::collections::HashMap;
-use std::fmt::Write;
+use std::collections::HashSet;
 use std::process::Stdio;
 
+use displaydoc::Display;
 use futures::FutureExt;
 use futures::StreamExt;
 use futures::stream::BoxStream;
-use nix_json::ActivityKind;
 use nix_json::NixBuildLogLine;
 use nix_json::RawNixDerivationInfoOutput;
-use nix_json::ResultKind;
+use nix_json::helpers::NixBuildState;
 use petgraph::Directed;
 use petgraph::Graph;
+use petgraph::acyclic::Acyclic;
+use petgraph::data::Build;
 use petgraph::prelude::NodeIndex;
+use thiserror::Error;
 use tokio::io::AsyncBufReadExt;
+use tokio::io::AsyncReadExt;
+use tokio::io::AsyncWriteExt;
 use tokio::process::Command;
 
 #[derive(Debug)]
@@ -36,8 +41,11 @@ impl NixBuildResult {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Error, Display)]
-pub enum NixBuildError {}
+pub enum NixBuildError {
+    /// No error
+    NoneYet,
+}
 
 pub trait NixInProgressBuild {
     fn next_log_line(
@@ -59,12 +67,24 @@ pub trait NixBackend {
         &self,
         derivation: &str,
     ) -> impl Future<Output = Result<NixBuildGraph, NixBuildError>>;
+
+    fn check_if_paths_in_cache(
+        &self,
+        store: String,
+        paths: &[String],
+    ) -> impl Future<Output = Result<HashMap<String, bool>, NixBuildError>>;
 }
 
 pub struct NixCliBackend {
     command_path: String,
 }
 
+impl NixCliBackend {
+    pub fn new(command_path: String) -> Self {
+        Self { command_path }
+    }
+}
+
 pub struct NixCliBackendBuild {
     child: BoxStream<'static, String>,
 }
@@ -82,84 +102,117 @@ impl NixInProgressBuild for NixCliBackendBuild {
 impl NixBackend for NixCliBackend {
     type InProgressBuild<'b> = NixCliBackendBuild;
 
-    fn start_build(
+    async fn start_build(
         &self,
         derivation: &str,
-    ) -> impl Future<Output = Result<Self::InProgressBuild<'_>, NixBuildError>> {
+    ) -> Result<Self::InProgressBuild<'_>, NixBuildError> {
-        async move {
+        let mut cmd = Command::new(&self.command_path)
-            let mut cmd = Command::new(&self.command_path)
+            .args(["build", "--log-format", "internal-json"])
-                .args(["build", "--log-format", "internal-json"])
+            .arg(derivation)
-                .arg(derivation)
+            .stdout(Stdio::piped())
-                .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
-                .stderr(Stdio::piped())
+            .spawn()
-                .spawn()
+            .unwrap();
-                .unwrap();
 
-            let init = cmd.stderr.take().unwrap();
+        let init = cmd.stderr.take().unwrap();
-            Ok(NixCliBackendBuild {
+        Ok(NixCliBackendBuild {
-                child: futures::stream::unfold(
+            child: futures::stream::unfold(
-                    tokio::io::BufReader::new(init),
+                tokio::io::BufReader::new(init),
-                    move |mut state| async move {
+                move |mut state| async move {
-                        let mut buffer = String::new();
+                    let mut buffer = String::new();
-                        let num_bytes = state.read_line(&mut buffer).await.unwrap();
+                    let num_bytes = state.read_line(&mut buffer).await.unwrap();
 
-                        if num_bytes == 0 {
+                    if num_bytes == 0 {
-                            None
+                        None
-                        } else {
+                    } else {
-                            Some((std::mem::take(&mut buffer), state))
+                        Some((std::mem::take(&mut buffer), state))
-                        }
+                    }
-                    },
+                },
-                )
+            )
-                .boxed(),
+            .boxed(),
-            })
+        })
-        }
     }
 
-    fn get_needed_builds(
+    async fn get_needed_builds(&self, derivation: &str) -> Result<NixBuildGraph, NixBuildError> {
-        &self,
+        let cmd = Command::new(&self.command_path)
-        derivation: &str,
+            .args(["derivation", "show", "--recursive"])
-    ) -> impl Future<Output = Result<NixBuildGraph, NixBuildError>> {
+            .arg(derivation)
-        async move {
+            .output()
-            let cmd = Command::new(&self.command_path)
+            .await
-                .args(["derivation", "show", "--recursive"])
+            .unwrap();
-                .arg(derivation)
-                .output()
-                .await
-                .unwrap();
 
-            let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap();
+        let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap();
 
-            let mut build_graph = NixBuildGraph::default();
+        let mut build_graph = NixBuildGraph::default();
 
-            for (path, _info) in output.info() {
+        for path in output.info().keys() {
-                let internal_id = build_graph.dependencies.add_node(path.to_string());
+            let internal_id = build_graph.dependencies.add_node(path.to_string());
-                build_graph.build_infos.insert(
+            build_graph.build_infos.insert(
-                    path.to_string(),
+                path.to_string(),
-                    NixBuildInfo {
+                NixBuildInfo {
-                        internal_id,
+                    internal_id,
-                        present_in_binary_cache: false,
+                    present_in_binary_cache: false,
+                },
+            );
+        }
+
+        for (path, info) in output.info() {
+            let build_info = &build_graph.build_infos[path];
+            let cur_node = build_info.internal_id;
+
+            for dep_path in info.input_derivations.keys() {
+                let other_node = build_graph.build_infos[dep_path].internal_id;
+                build_graph.dependencies.add_edge(
+                    cur_node,
+                    other_node,
+                    NixBuildOutput {
+                        output_name: String::new(),
                     },
                 );
             }
-
-            for (path, info) in output.info() {
-                let build_info = &build_graph.build_infos[path];
-                let cur_node = build_info.internal_id;
-
-                for (dep_path, _dep_info) in &info.input_derivations {
-                    let other_node = build_graph.build_infos[dep_path].internal_id;
-                    build_graph.dependencies.add_edge(
-                        cur_node,
-                        other_node,
-                        NixBuildOutput {
-                            output_name: String::new(),
-                        },
-                    );
-                }
-            }
-
-            Ok(build_graph)
         }
+
+        Ok(build_graph)
+    }
+
+    async fn check_if_paths_in_cache(
+        &self,
+        store: String,
+        paths: &[String],
+    ) -> Result<HashMap<String, bool>, NixBuildError> {
+        let mut cmd = Command::new(&self.command_path)
+            .args(["path-info", "--store", &store, "--json", "--stdin"])
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::null())
+            .spawn()
+            .unwrap();
+
+        let mut stdout = cmd.stdout.take().unwrap();
+
+        // Write all to stdin, and then have it drop at the end of the block
+        {
+            let paths = paths.join(" ");
+            let mut stdin = cmd.stdin.take().unwrap();
+            stdin.write_all(paths.as_ref()).await.unwrap();
+            stdin.shutdown().await.unwrap();
+        }
+
+        println!("Wrote all to stdin!");
+
+        let mut output = vec![];
+
+        stdout.read_to_end(&mut output).await.unwrap();
+
+        tokio::spawn(async move { cmd.wait().await });
+
+        let store_info: HashMap<String, Option<serde_json::Value>> =
+            serde_json::from_slice(&output).unwrap();
+
+        Ok(store_info
+            .into_iter()
+            .map(|(k, v)| (k, v.is_some()))
+            .collect())
     }
 }
 
@@ -178,14 +231,20 @@ pub struct NixBuildOutput {
     output_name: String,
 }
 
+impl NixBuildOutput {
+    pub fn output_name(&self) -> &str {
+        &self.output_name
+    }
+}
+
 #[derive(Debug, Default)]
 pub struct NixBuildGraph {
     build_infos: HashMap<String, NixBuildInfo>,
-    dependencies: Graph<String, NixBuildOutput, Directed, usize>,
+    dependencies: Acyclic<Graph<String, NixBuildOutput, Directed, usize>>,
 }
 
 impl NixBuildGraph {
-    fn get_non_binary_builds(&self) -> impl Iterator<Item = (String, NixBuildInfo)> {
+    pub fn get_non_binary_builds(&self) -> impl Iterator<Item = (String, NixBuildInfo)> {
         self.build_infos
             .iter()
             .filter(|(_, value)| !value.present_in_binary_cache)
@@ -197,6 +256,17 @@ impl<B> NixBuilder<B>
 where
     B: NixBackend,
 {
+    pub fn new(backend: B) -> Self {
+        Self { backend }
+    }
+
+    pub async fn get_needed_builds(
+        &self,
+        derivation: &str,
+    ) -> Result<NixBuildGraph, NixBuildError> {
+        self.backend.get_needed_builds(derivation).await
+    }
+
     pub async fn build(
         &self,
         derivation: String,
@@ -216,30 +286,23 @@ where
                 )
             }));
 
-        let mut id_to_derivation = HashMap::new();
+        let mut build_state = NixBuildState::default();
 
         while let Some(next_log_line) = started_build.next_log_line().await {
             let next_log_line = next_log_line?;
 
-            if let NixBuildLogLine::Start(start_log) = next_log_line {
+            build_state.handle_log_line(next_log_line);
-                if start_log.kind == ActivityKind::Build {
-                    id_to_derivation.insert(
-                        start_log.id,
-                        start_log.fields[0].as_str().unwrap().to_string(),
-                    );
-                }
-            } else if let NixBuildLogLine::Result(result) = next_log_line {
-                if result.kind == ResultKind::BuildLogLine {
-                    if let Some(build) = actually_built.get_mut(&id_to_derivation[&result.id]) {
-                        writeln!(&mut build.log, "{}", result.fields[0].as_str().unwrap()).unwrap()
-                    } else {
-                        panic!("Could not find correct id");
-                    }
-                }
-            }
         }
 
-        actually_built.retain(|_k, v| !v.log.is_empty());
+        let mut actually_built_derivations = HashSet::new();
+        for drv in build_state.derivations() {
+            let built = actually_built.get_mut(drv.derivation_path()).unwrap();
+
+            built.success = drv.is_success();
+            actually_built_derivations.insert(drv.derivation_path());
+        }
+
+        actually_built.retain(|k, _| actually_built_derivations.contains(&k.as_ref()));
 
         Ok(actually_built)
     }

nixie-cli/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "nixie-cli"
+edition.workspace = true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+readme.workspace = true
+
+[dependencies]
+clap = { workspace = true, features = ["derive"] }
+anyhow = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
+nixie-build.workspace = true

nixie-cli/src/main.rs

@@ -0,0 +1,53 @@
+use clap::Parser;
+use clap::Subcommand;
+use nixie_build::NixBackend;
+use nixie_build::NixBuilder;
+use nixie_build::NixCliBackend;
+
+#[derive(Debug, Parser)]
+#[command(version, about)]
+struct Args {
+    #[clap(subcommand)]
+    command: ArgCommand,
+}
+
+#[derive(Debug, Subcommand)]
+enum ArgCommand {
+    Build { installable: String },
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args = Args::parse();
+
+    run(args).await
+}
+
+async fn run(args: Args) -> anyhow::Result<()> {
+    match args.command {
+        ArgCommand::Build { installable } => {
+            build_and_report(NixCliBackend::new("nix".to_string()), installable).await?;
+        }
+    }
+
+    Ok(())
+}
+
+async fn build_and_report<B: NixBackend>(backend: B, installable: String) -> anyhow::Result<()> {
+    let builder = NixBuilder::new(backend);
+
+    let build_result = builder.build(installable).await?;
+
+    for (drv, result) in build_result {
+        println!(
+            "Built:\t{drv}: {}",
+            if result.success() {
+                "Succesfully"
+            } else {
+                "Errored"
+            }
+        )
+    }
+
+    Ok(())
+}