diff --git a/Cargo.lock b/Cargo.lock index f46def0..1a53144 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,6 +61,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "bit-set" version = "0.8.0" @@ -432,6 +438,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "nixie-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "nixie-build", + "tokio", +] + +[[package]] +name = "nixie-server" +version = "0.1.0" + [[package]] name = "once_cell_polyfill" version = "1.70.2" diff --git a/Cargo.toml b/Cargo.toml index 269c913..adf9e81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = ["nix-json", "nixie-build"] +members = ["nix-json", "nixie-build", "nixie-cli"] resolver = "3" @@ -21,9 +21,12 @@ petgraph = "0.8" tokio = { features = ["full"], version = "1.48" } futures = "0.3" datatest-stable = "0.3.3" +itertools = "0.14" +anyhow = "1.0.100" +clap = "4.5.54" nix-json = { version = "0.1.0", path = "./nix-json" } - +nixie-build = { version = "0.1.0", path = "./nixie-build" } [workspace.metadata.crane] name = "nixie-ci" diff --git a/nix-json/Cargo.toml b/nix-json/Cargo.toml index 462ec03..57233fb 100644 --- a/nix-json/Cargo.toml +++ b/nix-json/Cargo.toml @@ -16,7 +16,7 @@ serde_json.workspace = true serde_repr.workspace = true thiserror = { workspace = true } displaydoc = { workspace = true } -itertools = "0.14" +itertools = { workspace = true } [dev-dependencies] datatest-stable.workspace = true diff --git a/nix-json/src/helpers.rs b/nix-json/src/helpers.rs index 4208152..7cc5d56 100644 --- a/nix-json/src/helpers.rs +++ b/nix-json/src/helpers.rs @@ -17,7 +17,7 @@ pub enum LogBuildState { #[derive(Debug)] pub struct LogBuildStatus { state: LogBuildState, - store_path: String, + derivation_path: String, log_lines: Vec, } @@ -29,6 +29,14 @@ impl LogBuildStatus { pub fn log_lines(&self) -> &[String] { &self.log_lines } + + pub fn derivation_path(&self) -> &str { + &self.derivation_path + } + + pub fn is_success(&self) -> bool { + self.state == LogBuildState::Succeeded + } } /// Accumulated build status of different nix-builds @@ -109,7 +117,10 @@ impl NixBuildState { nix_log_start_action.id, LogBuildStatus { state: LogBuildState::Started, - store_path: nix_log_start_action.fields[0].as_str().unwrap().to_string(), + derivation_path: nix_log_start_action.fields[0] + .as_str() + .unwrap() + .to_string(), log_lines: vec![], }, ); @@ -171,13 +182,17 @@ impl NixBuildState { fn handle_log_action( &self, - nix_log_msg_action: crate::NixLogMsgAction, + _nix_log_msg_action: crate::NixLogMsgAction, ) -> Option { None } pub fn get_derivation(&self, drv: &str) -> Option<&LogBuildStatus> { - self.builds.values().find(|b| b.store_path == drv) + self.builds.values().find(|b| b.derivation_path == drv) + } + + pub fn derivations(&self) -> impl Iterator { + self.builds.values() } } diff --git a/nixie-build/foo.rs b/nixie-build/foo.rs new file mode 100644 index 0000000..fbd3577 --- /dev/null +++ b/nixie-build/foo.rs @@ -0,0 +1,32 @@ + +{ + "/nix/store/6hv3vg9g9jf9z4bbmkz6b0yixk7h48fv-nixie-ci-0.1.0.drv": null, + "/nix/store/8brm4045abq27fa7i2v68yx2q5d70mdr-zstd-1.5.7.drv": { + "ca": "text:sha256:1r5zh6h09l9a1gzkrvccp6xxi087365wr22pahgz5cm8sg1ssp8a", + "compression": "xz", + "deriver": null, + "downloadHash": "sha256-JkICxO8KGTN9ehMWt3bbEcPPZgb9RlDQscmI0OBJlXw=", + "downloadSize": 1688, + "narHash": "sha256-BPtqm0GZEJseFXwZSGO/lYRWBgdlRsqoyAMEEWozacI=", + "narSize": 3920, + "references": [ + "/nix/store/377lw5m13cliaj6328iz9il687qxwahb-source.drv", + "/nix/store/5mv61fp0f6l28m4bvanzxr82lzb7d5s3-file-5.45.drv", + "/nix/store/6k4n3g5jahyxqn6m7bdviydxibzp7xx1-playtests-darwin.patch", + "/nix/store/6li69pci6if48q2bjir0p58qvf8iy703-man-fix.patch.drv", + "/nix/store/hi1hlj16r6gwpzk8kk3wnmq1wjgyf3jf-gnugrep-3.12.drv", + "/nix/store/l622p70vy8k5sh7y5wizi5f2mic6ynpg-source-stdenv.sh", + "/nix/store/pnjvpwgka59d6fwpp9fnz42ll2ai4ffm-stdenv-linux.drv", + "/nix/store/shkw4qm9qcw5sc5n1k5jznc83ny02r39-default-builder.sh", + "/nix/store/vwmk63kc9sysjif65h7fdwnqr5h8jfm6-bash-5.3p3.drv", + "/nix/store/xd9slhxpvs9wl0hgd36x4fkfnp2ybb9i-cmake-minimal-4.1.2.drv" + ], + "registrationTime": null, + "signatures": [ + "cache.nixos.org-1:6mzDXuIB96xqsrtAl7n8Mh9zgkWzVvpj+bEpo5H6c+KNKME4z7xdR7WYiFH6iERwWFCO3v387ps+AkcY/GQeCQ==" + ], + "ultimate": false, + "url": "nar/0z4m97hd1269n7850ipx0rkczhqivdvbf5hkg9yk668axz204hi6.nar.xz" + }, + "/nix/store/d41hh410m104pnid9vynwbpng3m4w3gh-nixie-ci-fmt-0.1.0.drv": null +} diff --git a/nixie-build/src/lib.rs b/nixie-build/src/lib.rs index 8c1c453..b5e3499 100644 --- a/nixie-build/src/lib.rs +++ b/nixie-build/src/lib.rs @@ -1,18 +1,23 @@ use std::collections::HashMap; -use std::fmt::Write; +use std::collections::HashSet; use std::process::Stdio; +use displaydoc::Display; use futures::FutureExt; use futures::StreamExt; use futures::stream::BoxStream; -use nix_json::ActivityKind; use nix_json::NixBuildLogLine; use nix_json::RawNixDerivationInfoOutput; -use nix_json::ResultKind; +use nix_json::helpers::NixBuildState; use petgraph::Directed; use petgraph::Graph; +use petgraph::acyclic::Acyclic; +use petgraph::data::Build; use petgraph::prelude::NodeIndex; +use thiserror::Error; use tokio::io::AsyncBufReadExt; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; use tokio::process::Command; #[derive(Debug)] @@ -36,8 +41,11 @@ impl NixBuildResult { } } -#[derive(Debug)] -pub enum NixBuildError {} +#[derive(Debug, Error, Display)] +pub enum NixBuildError { + /// No error + NoneYet, +} pub trait NixInProgressBuild { fn next_log_line( @@ -59,12 +67,24 @@ pub trait NixBackend { &self, derivation: &str, ) -> impl Future>; + + fn check_if_paths_in_cache( + &self, + store: String, + paths: &[String], + ) -> impl Future, NixBuildError>>; } pub struct NixCliBackend { command_path: String, } +impl NixCliBackend { + pub fn new(command_path: String) -> Self { + Self { command_path } + } +} + pub struct NixCliBackendBuild { child: BoxStream<'static, String>, } @@ -82,84 +102,117 @@ impl NixInProgressBuild for NixCliBackendBuild { impl NixBackend for NixCliBackend { type InProgressBuild<'b> = NixCliBackendBuild; - fn start_build( + async fn start_build( &self, derivation: &str, - ) -> impl Future, NixBuildError>> { - async move { - let mut cmd = Command::new(&self.command_path) - .args(["build", "--log-format", "internal-json"]) - .arg(derivation) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap(); + ) -> Result, NixBuildError> { + let mut cmd = Command::new(&self.command_path) + .args(["build", "--log-format", "internal-json"]) + .arg(derivation) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); - let init = cmd.stderr.take().unwrap(); - Ok(NixCliBackendBuild { - child: futures::stream::unfold( - tokio::io::BufReader::new(init), - move |mut state| async move { - let mut buffer = String::new(); - let num_bytes = state.read_line(&mut buffer).await.unwrap(); + let init = cmd.stderr.take().unwrap(); + Ok(NixCliBackendBuild { + child: futures::stream::unfold( + tokio::io::BufReader::new(init), + move |mut state| async move { + let mut buffer = String::new(); + let num_bytes = state.read_line(&mut buffer).await.unwrap(); - if num_bytes == 0 { - None - } else { - Some((std::mem::take(&mut buffer), state)) - } - }, - ) - .boxed(), - }) - } + if num_bytes == 0 { + None + } else { + Some((std::mem::take(&mut buffer), state)) + } + }, + ) + .boxed(), + }) } - fn get_needed_builds( - &self, - derivation: &str, - ) -> impl Future> { - async move { - let cmd = Command::new(&self.command_path) - .args(["derivation", "show", "--recursive"]) - .arg(derivation) - .output() - .await - .unwrap(); + async fn get_needed_builds(&self, derivation: &str) -> Result { + let cmd = Command::new(&self.command_path) + .args(["derivation", "show", "--recursive"]) + .arg(derivation) + .output() + .await + .unwrap(); - let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap(); + let output: RawNixDerivationInfoOutput = serde_json::from_slice(&cmd.stdout).unwrap(); - let mut build_graph = NixBuildGraph::default(); + let mut build_graph = NixBuildGraph::default(); - for (path, _info) in output.info() { - let internal_id = build_graph.dependencies.add_node(path.to_string()); - build_graph.build_infos.insert( - path.to_string(), - NixBuildInfo { - internal_id, - present_in_binary_cache: false, + for path in output.info().keys() { + let internal_id = build_graph.dependencies.add_node(path.to_string()); + build_graph.build_infos.insert( + path.to_string(), + NixBuildInfo { + internal_id, + present_in_binary_cache: false, + }, + ); + } + + for (path, info) in output.info() { + let build_info = &build_graph.build_infos[path]; + let cur_node = build_info.internal_id; + + for dep_path in info.input_derivations.keys() { + let other_node = build_graph.build_infos[dep_path].internal_id; + build_graph.dependencies.add_edge( + cur_node, + other_node, + NixBuildOutput { + output_name: String::new(), }, ); } - - for (path, info) in output.info() { - let build_info = &build_graph.build_infos[path]; - let cur_node = build_info.internal_id; - - for (dep_path, _dep_info) in &info.input_derivations { - let other_node = build_graph.build_infos[dep_path].internal_id; - build_graph.dependencies.add_edge( - cur_node, - other_node, - NixBuildOutput { - output_name: String::new(), - }, - ); - } - } - - Ok(build_graph) } + + Ok(build_graph) + } + + async fn check_if_paths_in_cache( + &self, + store: String, + paths: &[String], + ) -> Result, NixBuildError> { + let mut cmd = Command::new(&self.command_path) + .args(["path-info", "--store", &store, "--json", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + + let mut stdout = cmd.stdout.take().unwrap(); + + // Write all to stdin, and then have it drop at the end of the block + { + let paths = paths.join(" "); + let mut stdin = cmd.stdin.take().unwrap(); + stdin.write_all(paths.as_ref()).await.unwrap(); + stdin.shutdown().await.unwrap(); + } + + println!("Wrote all to stdin!"); + + let mut output = vec![]; + + stdout.read_to_end(&mut output).await.unwrap(); + + tokio::spawn(async move { cmd.wait().await }); + + let store_info: HashMap> = + serde_json::from_slice(&output).unwrap(); + + Ok(store_info + .into_iter() + .map(|(k, v)| (k, v.is_some())) + .collect()) } } @@ -178,14 +231,20 @@ pub struct NixBuildOutput { output_name: String, } +impl NixBuildOutput { + pub fn output_name(&self) -> &str { + &self.output_name + } +} + #[derive(Debug, Default)] pub struct NixBuildGraph { build_infos: HashMap, - dependencies: Graph, + dependencies: Acyclic>, } impl NixBuildGraph { - fn get_non_binary_builds(&self) -> impl Iterator { + pub fn get_non_binary_builds(&self) -> impl Iterator { self.build_infos .iter() .filter(|(_, value)| !value.present_in_binary_cache) @@ -197,6 +256,17 @@ impl NixBuilder where B: NixBackend, { + pub fn new(backend: B) -> Self { + Self { backend } + } + + pub async fn get_needed_builds( + &self, + derivation: &str, + ) -> Result { + self.backend.get_needed_builds(derivation).await + } + pub async fn build( &self, derivation: String, @@ -216,30 +286,23 @@ where ) })); - let mut id_to_derivation = HashMap::new(); + let mut build_state = NixBuildState::default(); while let Some(next_log_line) = started_build.next_log_line().await { let next_log_line = next_log_line?; - if let NixBuildLogLine::Start(start_log) = next_log_line { - if start_log.kind == ActivityKind::Build { - id_to_derivation.insert( - start_log.id, - start_log.fields[0].as_str().unwrap().to_string(), - ); - } - } else if let NixBuildLogLine::Result(result) = next_log_line { - if result.kind == ResultKind::BuildLogLine { - if let Some(build) = actually_built.get_mut(&id_to_derivation[&result.id]) { - writeln!(&mut build.log, "{}", result.fields[0].as_str().unwrap()).unwrap() - } else { - panic!("Could not find correct id"); - } - } - } + build_state.handle_log_line(next_log_line); } - actually_built.retain(|_k, v| !v.log.is_empty()); + let mut actually_built_derivations = HashSet::new(); + for drv in build_state.derivations() { + let built = actually_built.get_mut(drv.derivation_path()).unwrap(); + + built.success = drv.is_success(); + actually_built_derivations.insert(drv.derivation_path()); + } + + actually_built.retain(|k, _| actually_built_derivations.contains(&k.as_ref())); Ok(actually_built) } diff --git a/nixie-cli/Cargo.toml b/nixie-cli/Cargo.toml new file mode 100644 index 0000000..1689dc7 --- /dev/null +++ b/nixie-cli/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "nixie-cli" +edition.workspace = true +version.workspace = true +license.workspace = true +authors.workspace = true +readme.workspace = true + +[dependencies] +clap = { workspace = true, features = ["derive"] } +anyhow = { workspace = true } +tokio = { workspace = true, features = ["full"] } +nixie-build.workspace = true diff --git a/nixie-cli/src/main.rs b/nixie-cli/src/main.rs new file mode 100644 index 0000000..325e596 --- /dev/null +++ b/nixie-cli/src/main.rs @@ -0,0 +1,53 @@ +use clap::Parser; +use clap::Subcommand; +use nixie_build::NixBackend; +use nixie_build::NixBuilder; +use nixie_build::NixCliBackend; + +#[derive(Debug, Parser)] +#[command(version, about)] +struct Args { + #[clap(subcommand)] + command: ArgCommand, +} + +#[derive(Debug, Subcommand)] +enum ArgCommand { + Build { installable: String }, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + run(args).await +} + +async fn run(args: Args) -> anyhow::Result<()> { + match args.command { + ArgCommand::Build { installable } => { + build_and_report(NixCliBackend::new("nix".to_string()), installable).await?; + } + } + + Ok(()) +} + +async fn build_and_report(backend: B, installable: String) -> anyhow::Result<()> { + let builder = NixBuilder::new(backend); + + let build_result = builder.build(installable).await?; + + for (drv, result) in build_result { + println!( + "Built:\t{drv}: {}", + if result.success() { + "Succesfully" + } else { + "Errored" + } + ) + } + + Ok(()) +}