snix_build_glue/build_state.rs
1use std::{cell::RefCell, io, os::unix::ffi::OsStrExt as _, sync::Arc};
2
3use futures::TryStreamExt as _;
4use nix_compat::{
5 nixhash::CAHash,
6 store_path::{StorePath, StorePathRef},
7};
8use snix_build::buildservice::BuildService;
9use snix_castore::{
10 blobservice::BlobService,
11 directoryservice::{DirectoryService, traversal::descend_to},
12};
13use snix_store::{
14 nar::NarCalculationService, path_info::PathInfo, pathinfoservice::PathInfoService,
15};
16use tracing::{Level, Span, instrument, warn};
17use tracing_indicatif::span_ext::IndicatifSpanExt;
18use url::Url;
19
20use crate::{builder, fetchers::Fetcher, known_paths::KnownPaths};
21
22pub struct BuildState {
23 // This is public so helper functions can interact with the stores directly.
24 pub blob_service: Arc<dyn BlobService>,
25 pub directory_service: Arc<dyn DirectoryService>,
26 pub path_info_service: Arc<dyn PathInfoService>,
27 pub nar_calculation_service: Arc<dyn NarCalculationService>,
28
29 #[allow(dead_code)]
30 build_service: Arc<dyn BuildService>,
31
32 #[allow(clippy::type_complexity)]
33 pub fetcher: Fetcher<
34 Arc<dyn BlobService>,
35 Arc<dyn DirectoryService>,
36 Arc<dyn PathInfoService>,
37 Arc<dyn NarCalculationService>,
38 >,
39
40 // Paths known how to produce, by building or fetching.
41 pub known_paths: RefCell<KnownPaths>,
42}
43
44impl BuildState {
45 pub fn new(
46 blob_service: Arc<dyn BlobService>,
47 directory_service: Arc<dyn DirectoryService>,
48 path_info_service: Arc<dyn PathInfoService>,
49 nar_calculation_service: Arc<dyn NarCalculationService>,
50 build_service: Arc<dyn BuildService>,
51 hashed_mirrors: Vec<Url>,
52 ) -> Self {
53 Self {
54 blob_service: blob_service.clone(),
55 directory_service: directory_service.clone(),
56 path_info_service: path_info_service.clone(),
57 nar_calculation_service: nar_calculation_service.clone(),
58 build_service,
59 fetcher: Fetcher::new(
60 blob_service,
61 directory_service,
62 path_info_service,
63 nar_calculation_service,
64 hashed_mirrors,
65 ),
66 known_paths: Default::default(),
67 }
68 }
69
70 /// for a given [StorePath] and additional [snix_castore::Path] inside the store path,
71 /// look up the [PathInfo], and if it exists, and then uses
72 /// [descend_to] to return the [snix_castore::Node] specified by `sub_path`.
73 ///
74 /// In case there is no PathInfo yet, we check "self.known_paths" (learnt by
75 /// the evaluator)
76 /// - If there's a fetch, we do that and return the resulting PathInfo.
77 /// - If there's a Derivation, we build it and return the resulting
78 /// PathInfo.
79 /// To build it, we need to do a function call to ourselves with all
80 /// inputs of that Derivation, which will trigger fetches / builds of
81 /// inputs, recursively.
82 ///
83 /// This should be replaced with a proper scheduler knowing about a partial
84 /// subgraph at some point, because this design doesn't allow concurrent
85 /// builds yet.
86 #[instrument(skip(self, store_path), fields(store_path=%store_path, indicatif.pb_show=tracing::field::Empty), ret(level = Level::TRACE), err(level = Level::TRACE))]
87 pub async fn store_path_to_path_info(
88 &self,
89 store_path: &StorePathRef<'_>,
90 sub_path: &snix_castore::Path,
91 ) -> io::Result<Option<PathInfo>> {
92 // Find the root node for the store_path.
93 // It asks the PathInfoService first, but in case there was a Derivation
94 // produced that would build it, fall back to triggering the build.
95 // To populate the input nodes, it might recursively trigger builds of
96 // its dependencies too.
97 let mut path_info = match self
98 .path_info_service
99 .as_ref()
100 .get(*store_path.digest())
101 .await
102 .map_err(std::io::Error::other)?
103 {
104 Some(path_info) => path_info,
105 // If there's no PathInfo found, this normally means we have to
106 // trigger the build (and insert into PathInfoService, after
107 // reference scanning).
108 // However, as Snix is (currently) not managing /nix/store itself,
109 // we return Ok(None) to let std_io take over.
110 // While reading from store paths that are not known to Snix during
111 // that evaluation clearly is an impurity, we still need to support
112 // it for things like <nixpkgs> pointing to a store path.
113 // In the future, these things will (need to) have PathInfo.
114 None => {
115 // The store path doesn't exist yet, so we need to fetch or build it.
116 // We check for fetches first, as we might have both native
117 // fetchers and FODs in KnownPaths, and prefer the former.
118 // This will also find [Fetch] synthesized from
119 // `builtin:fetchurl` Derivations.
120 let maybe_fetch = self
121 .known_paths
122 .borrow()
123 .get_fetch_for_output_path(store_path);
124
125 match maybe_fetch {
126 Some((name, fetch)) => {
127 let (sp, path_info) = self
128 .fetcher
129 .ingest_and_persist(&name, fetch)
130 .await
131 .map_err(|e| {
132 std::io::Error::new(std::io::ErrorKind::InvalidData, e)
133 })?;
134
135 debug_assert_eq!(
136 sp.to_absolute_path(),
137 store_path.to_absolute_path(),
138 "store path returned from fetcher must match store path we have in fetchers"
139 );
140
141 path_info
142 }
143 None => {
144 // Look up the derivation for this output path.
145 let (drv_path, drv) = {
146 let known_paths = self.known_paths.borrow();
147 match known_paths.get_drv_path_for_output_path(store_path) {
148 Some(drv_path) => (
149 drv_path.to_owned(),
150 known_paths
151 .get_drv_by_drvpath(&drv_path.as_ref())
152 .unwrap()
153 .to_owned(),
154 ),
155 None => {
156 warn!(store_path=%store_path, "no drv found");
157 // let StdIO take over
158 return Ok(None);
159 }
160 }
161 };
162 let span = Span::current();
163 span.pb_start();
164 span.pb_set_style(&snix_tracing::PB_SPINNER_STYLE);
165 span.pb_set_message(&format!("⏳Waiting for inputs {}", &store_path));
166
167 // derivation_to_build_request needs castore nodes for all inputs.
168 // Provide them, which means, here is where we recursively build
169 // all dependencies.
170 let resolved_inputs = {
171 let known_paths = &self.known_paths.borrow();
172 builder::get_all_inputs(&drv, known_paths, |path| {
173 Box::pin(async move {
174 self.store_path_to_path_info(
175 &path.as_ref(),
176 snix_castore::Path::ROOT,
177 )
178 .await
179 })
180 })
181 }
182 .try_collect()
183 .await?;
184
185 // precompute the `ca` field in the PathInfo, so we can send off drv.
186 let mut ca = drv.fod_digest().map(|fod_digest| {
187 CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest))
188 });
189
190 // synthesize the build request.
191 let build_request =
192 builder::derivation_into_build_request(drv, &resolved_inputs)?;
193
194 // Assemble a mapping table from needle back to store path, as well as a list of all outputs.
195 // The latter is a subset of the former.
196 // We need this to understand the response later.
197 let mut output_paths: Vec<StorePath> =
198 Vec::with_capacity(build_request.outputs.len());
199 let all_possible_refs: Vec<StorePath> = build_request
200 .outputs
201 .iter()
202 .map(|p| {
203 let sp = StorePath::from_bytes(
204 p.strip_prefix(&nix_compat::store_path::STORE_DIR[1..])
205 .expect("output doesn't have expected store_dir prefix")
206 .as_os_str()
207 .as_bytes(),
208 )
209 .expect("Snix bug: cannot parse output as StorePath");
210 output_paths.push(sp.clone());
211
212 sp
213 })
214 .chain(resolved_inputs.keys().cloned())
215 .collect();
216
217 span.pb_set_message(&format!("🔨Building {}", &store_path));
218
219 // create a build
220 let build_result = self
221 .build_service
222 .as_ref()
223 .do_build(build_request)
224 .await
225 .map_err(std::io::Error::other)?;
226
227 let mut out_path_info: Option<PathInfo> = None;
228
229 // For each output, insert a PathInfo.
230 for (output, output_path) in
231 build_result.outputs.into_iter().zip(output_paths)
232 {
233 // calculate the nar representation
234 let (nar_size, nar_sha256) = self
235 .nar_calculation_service
236 .calculate_nar(&output.node)
237 .await
238 .map_err(std::io::Error::other)?;
239
240 // assemble the PathInfo to persist
241 let path_info = PathInfo {
242 store_path: output_path.clone(),
243 node: output.node,
244 references: {
245 let mut references =
246 Vec::with_capacity(output.output_needles.len());
247
248 // Map each output needle index back into a store path.
249 for needle_idx in output.output_needles {
250 let output = all_possible_refs
251 .get(needle_idx as usize)
252 .ok_or(std::io::Error::other("invalid needle_idx"))?
253 .clone();
254 references.push(output);
255 }
256
257 // Produce references sorted by name for consistency with nix narinfos
258 references.sort();
259 references
260 },
261 nar_size,
262 nar_sha256,
263 signatures: vec![],
264 deriver: Some(
265 StorePath::from_name_and_digest_fixed(
266 drv_path
267 .name()
268 .strip_suffix(".drv")
269 .expect("missing .drv suffix"),
270 *drv_path.digest(),
271 )
272 .expect(
273 "Snix bug: StorePath without .drv suffix must be valid",
274 ),
275 ),
276 // CA derivations only have one output, so this only runs once.
277 ca: ca.take(),
278 };
279
280 self.path_info_service
281 .put(path_info.clone())
282 .await
283 .map_err(std::io::Error::other)?;
284
285 if *store_path == output_path.as_ref() {
286 out_path_info = Some(path_info);
287 }
288 }
289
290 out_path_info.ok_or(io::Error::other("build didn't produce store path"))?
291 }
292 }
293 }
294 };
295
296 // now with the root_node and sub_path, descend to the node requested.
297 Ok(
298 descend_to(&self.directory_service, path_info.node.clone(), sub_path)
299 .await
300 .map_err(std::io::Error::other)?
301 .map(|node| {
302 path_info.node = node;
303 path_info
304 }),
305 )
306 }
307}