snix_build_glue/build_state.rs
1use std::{cell::RefCell, io, os::unix::ffi::OsStrExt as _, sync::Arc};
2
3use futures::TryStreamExt as _;
4use nix_compat::{
5 nixhash::CAHash,
6 store_path::{StorePath, StorePathRef},
7};
8use snix_build::buildservice::BuildService;
9use snix_castore::{
10 blobservice::BlobService,
11 directoryservice::{DirectoryService, traversal::descend_to},
12};
13use snix_store::{
14 nar::NarCalculationService, path_info::PathInfo, pathinfoservice::PathInfoService,
15};
16use tracing::{Level, Span, instrument, warn};
17use tracing_indicatif::span_ext::IndicatifSpanExt;
18use url::Url;
19
20use crate::{builder, fetchers::Fetcher, known_paths::KnownPaths};
21
22pub struct BuildState {
23 // This is public so helper functions can interact with the stores directly.
24 pub blob_service: Arc<dyn BlobService>,
25 pub directory_service: Arc<dyn DirectoryService>,
26 pub path_info_service: Arc<dyn PathInfoService>,
27 pub nar_calculation_service: Arc<dyn NarCalculationService>,
28
29 #[allow(dead_code)]
30 build_service: Arc<dyn BuildService>,
31
32 #[allow(clippy::type_complexity)]
33 pub fetcher: Fetcher<
34 Arc<dyn BlobService>,
35 Arc<dyn DirectoryService>,
36 Arc<dyn PathInfoService>,
37 Arc<dyn NarCalculationService>,
38 >,
39
40 // Paths known how to produce, by building or fetching.
41 pub known_paths: RefCell<KnownPaths>,
42}
43
44impl BuildState {
45 pub fn new(
46 blob_service: Arc<dyn BlobService>,
47 directory_service: Arc<dyn DirectoryService>,
48 path_info_service: Arc<dyn PathInfoService>,
49 nar_calculation_service: Arc<dyn NarCalculationService>,
50 build_service: Arc<dyn BuildService>,
51 hashed_mirrors: Vec<Url>,
52 ) -> Self {
53 Self {
54 blob_service: blob_service.clone(),
55 directory_service: directory_service.clone(),
56 path_info_service: path_info_service.clone(),
57 nar_calculation_service: nar_calculation_service.clone(),
58 build_service,
59 fetcher: Fetcher::new(
60 blob_service,
61 directory_service,
62 path_info_service,
63 nar_calculation_service,
64 hashed_mirrors,
65 ),
66 known_paths: Default::default(),
67 }
68 }
69
70 /// for a given [StorePath] and additional [snix_castore::Path] inside the store path,
71 /// look up the [PathInfo], and if it exists, and then uses
72 /// [descend_to] to return the [snix_castore::Node] specified by `sub_path`.
73 ///
74 /// In case there is no PathInfo yet, we check "self.known_paths" (learnt by
75 /// the evaluator)
76 /// - If there's a fetch, we do that and return the resulting PathInfo.
77 /// - If there's a Derivation, we build it and return the resulting
78 /// PathInfo.
79 /// To build it, we need to do a function call to ourselves with all
80 /// inputs of that Derivation, which will trigger fetches / builds of
81 /// inputs, recursively.
82 ///
83 /// This should be replaced with a proper scheduler knowing about a partial
84 /// subgraph at some point, because this design doesn't allow concurrent
85 /// builds yet.
86 #[instrument(skip(self, store_path), fields(store_path=%store_path, indicatif.pb_show=tracing::field::Empty), ret(level = Level::TRACE), err(level = Level::TRACE))]
87 pub async fn store_path_to_path_info(
88 &self,
89 store_path: &StorePathRef<'_>,
90 sub_path: &snix_castore::Path,
91 ) -> io::Result<Option<PathInfo>> {
92 // Find the root node for the store_path.
93 // It asks the PathInfoService first, but in case there was a Derivation
94 // produced that would build it, fall back to triggering the build.
95 // To populate the input nodes, it might recursively trigger builds of
96 // its dependencies too.
97 let mut path_info = if let Some(path_info) = self
98 .path_info_service
99 .as_ref()
100 .get(*store_path.digest())
101 .await
102 .map_err(std::io::Error::other)?
103 {
104 path_info
105 } else {
106 // If there's no PathInfo found, this normally means we have to
107 // trigger the build (and insert into PathInfoService, after
108 // reference scanning).
109 // However, as Snix is (currently) not managing /nix/store itself,
110 // we return Ok(None) to let std_io take over.
111 // While reading from store paths that are not known to Snix during
112 // that evaluation clearly is an impurity, we still need to support
113 // it for things like <nixpkgs> pointing to a store path.
114 // In the future, these things will (need to) have PathInfo.
115 //
116 // The store path doesn't exist yet, so we need to fetch or build it.
117 // We check for fetches first, as we might have both native
118 // fetchers and FODs in KnownPaths, and prefer the former.
119 // This will also find [Fetch] synthesized from
120 // `builtin:fetchurl` Derivations.
121 let maybe_fetch = self
122 .known_paths
123 .borrow()
124 .get_fetch_for_output_path(store_path);
125 if let Some((name, fetch)) = maybe_fetch {
126 let (sp, path_info) = self
127 .fetcher
128 .ingest_and_persist(&name, fetch)
129 .await
130 .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
131
132 debug_assert_eq!(
133 sp.to_absolute_path(),
134 store_path.to_absolute_path(),
135 "store path returned from fetcher must match store path we have in fetchers"
136 );
137
138 path_info
139 } else {
140 // Look up the derivation for this output path.
141 let (drv_path, drv) = {
142 let known_paths = self.known_paths.borrow();
143 if let Some(drv_path) = known_paths.get_drv_path_for_output_path(store_path) {
144 (
145 drv_path.to_owned(),
146 known_paths
147 .get_drv_by_drvpath(&drv_path.as_ref())
148 .unwrap()
149 .to_owned(),
150 )
151 } else {
152 warn!(store_path=%store_path, "no drv found");
153 // let StdIO take over
154 return Ok(None);
155 }
156 };
157 let span = Span::current();
158 span.pb_start();
159 span.pb_set_style(&snix_tracing::PB_SPINNER_STYLE);
160 span.pb_set_message(&format!("⏳Waiting for inputs {}", &store_path));
161
162 // derivation_to_build_request needs castore nodes for all inputs.
163 // Provide them, which means, here is where we recursively build
164 // all dependencies.
165 let resolved_inputs = {
166 let known_paths = &self.known_paths.borrow();
167 builder::get_all_inputs(&drv, known_paths, |path| {
168 Box::pin(async move {
169 self.store_path_to_path_info(&path.as_ref(), snix_castore::Path::ROOT)
170 .await
171 })
172 })
173 }
174 .try_collect()
175 .await?;
176
177 // precompute the `ca` field in the PathInfo, so drv can be sent away owned.
178 let mut ca = drv.fod_digest().map(|fod_digest| {
179 CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest.into()))
180 });
181
182 // synthesize the build request.
183 let build_request = builder::derivation_into_build_request(drv, &resolved_inputs)?;
184
185 // Assemble a mapping table from needle back to store path, as well as a list of all outputs.
186 // The latter is a subset of the former.
187 // We need this to understand the response later.
188 let mut output_paths: Vec<StorePath> =
189 Vec::with_capacity(build_request.outputs.len());
190 let all_possible_refs: Vec<StorePath> = build_request
191 .outputs
192 .iter()
193 .map(|p| {
194 let sp = StorePath::from_bytes(
195 p.strip_prefix(&nix_compat::store_path::STORE_DIR[1..])
196 .expect("output doesn't have expected store_dir prefix")
197 .as_os_str()
198 .as_bytes(),
199 )
200 .expect("Snix bug: cannot parse output as StorePath");
201 output_paths.push(sp.clone());
202
203 sp
204 })
205 .chain(resolved_inputs.keys().cloned())
206 .collect();
207
208 span.pb_set_message(&format!("🔨Building {}", &store_path));
209
210 // create a build
211 let build_result = self
212 .build_service
213 .as_ref()
214 .do_build(build_request)
215 .await
216 .map_err(std::io::Error::other)?;
217
218 let mut out_path_info: Option<PathInfo> = None;
219
220 // For each output, insert a PathInfo.
221 for (output, output_path) in build_result.outputs.into_iter().zip(output_paths) {
222 // calculate the nar representation
223 let (nar_size, nar_sha256) = self
224 .nar_calculation_service
225 .calculate_nar(&output.node)
226 .await
227 .map_err(std::io::Error::other)?;
228
229 // assemble the PathInfo to persist
230 let path_info = PathInfo {
231 store_path: output_path.clone(),
232 node: output.node,
233 references: {
234 let mut references = Vec::with_capacity(output.output_needles.len());
235
236 // Map each output needle index back into a store path.
237 for needle_idx in output.output_needles {
238 let output = all_possible_refs
239 .get(needle_idx as usize)
240 .ok_or(std::io::Error::other("invalid needle_idx"))?
241 .clone();
242 references.push(output);
243 }
244
245 // Produce references sorted by name for consistency with nix narinfos
246 references.sort();
247 references
248 },
249 nar_size,
250 nar_sha256,
251 signatures: vec![],
252 deriver: Some(
253 StorePath::from_name_and_digest_fixed(
254 drv_path
255 .name()
256 .strip_suffix(".drv")
257 .expect("missing .drv suffix"),
258 *drv_path.digest(),
259 )
260 .expect("Snix bug: StorePath without .drv suffix must be valid"),
261 ),
262 // CA derivations only have one output, so this only runs once.
263 ca: ca.take(),
264 };
265
266 self.path_info_service
267 .put(path_info.clone())
268 .await
269 .map_err(std::io::Error::other)?;
270
271 if *store_path == output_path.as_ref() {
272 out_path_info = Some(path_info);
273 }
274 }
275
276 out_path_info.ok_or(io::Error::other("build didn't produce store path"))?
277 }
278 };
279
280 // now with the root_node and sub_path, descend to the node requested.
281 Ok(
282 descend_to(&self.directory_service, path_info.node.clone(), sub_path)
283 .await
284 .map_err(std::io::Error::other)?
285 .map(|node| {
286 path_info.node = node;
287 path_info
288 }),
289 )
290 }
291}