snix_glue/
snix_store_io.rs

1//! This module provides an implementation of EvalIO talking to snix-store.
2use futures::TryStreamExt;
3use nix_compat::{nixhash::CAHash, store_path::StorePath};
4use snix_build::buildservice::BuildService;
5use snix_eval::{EvalIO, FileType, StdIO};
6use snix_store::nar::NarCalculationService;
7use std::{
8    cell::RefCell,
9    io,
10    path::{Path, PathBuf},
11    sync::Arc,
12};
13use tokio_util::io::SyncIoBridge;
14use tracing::{Level, Span, error, instrument, warn};
15use tracing_indicatif::span_ext::IndicatifSpanExt;
16
17use snix_castore::{
18    Node,
19    blobservice::BlobService,
20    directoryservice::{self, DirectoryService},
21};
22use snix_store::pathinfoservice::{PathInfo, PathInfoService};
23
24use crate::fetchers::Fetcher;
25use crate::known_paths::KnownPaths;
26use crate::snix_build::derivation_to_build_request;
27
28/// Implements [EvalIO], asking given [PathInfoService], [DirectoryService]
29/// and [BlobService].
30///
31/// In case the given path does not exist in these stores, we ask StdIO.
32/// This is to both cover cases of syntactically valid store paths, that exist
33/// on the filesystem (still managed by Nix), as well as being able to read
34/// files outside store paths.
35///
36/// This structure is also directly used by the derivation builtins
37/// and tightly coupled to it.
38///
39/// In the future, we may revisit that coupling and figure out how to generalize this interface and
40/// hide this implementation detail of the glue itself so that glue can be used with more than one
41/// implementation of "Snix Store IO" which does not necessarily bring the concept of blob service,
42/// directory service or path info service.
43pub struct SnixStoreIO {
44    // This is public so helper functions can interact with the stores directly.
45    pub(crate) blob_service: Arc<dyn BlobService>,
46    pub(crate) directory_service: Arc<dyn DirectoryService>,
47    pub(crate) path_info_service: Arc<dyn PathInfoService>,
48    pub(crate) nar_calculation_service: Arc<dyn NarCalculationService>,
49
50    std_io: StdIO,
51    #[allow(dead_code)]
52    build_service: Arc<dyn BuildService>,
53    pub(crate) tokio_handle: tokio::runtime::Handle,
54
55    #[allow(clippy::type_complexity)]
56    pub(crate) fetcher: Fetcher<
57        Arc<dyn BlobService>,
58        Arc<dyn DirectoryService>,
59        Arc<dyn PathInfoService>,
60        Arc<dyn NarCalculationService>,
61    >,
62
63    // Paths known how to produce, by building or fetching.
64    pub known_paths: RefCell<KnownPaths>,
65}
66
67impl SnixStoreIO {
68    pub fn new(
69        blob_service: Arc<dyn BlobService>,
70        directory_service: Arc<dyn DirectoryService>,
71        path_info_service: Arc<dyn PathInfoService>,
72        nar_calculation_service: Arc<dyn NarCalculationService>,
73        build_service: Arc<dyn BuildService>,
74        tokio_handle: tokio::runtime::Handle,
75    ) -> Self {
76        Self {
77            blob_service: blob_service.clone(),
78            directory_service: directory_service.clone(),
79            path_info_service: path_info_service.clone(),
80            nar_calculation_service: nar_calculation_service.clone(),
81            std_io: StdIO {},
82            build_service,
83            tokio_handle,
84            fetcher: Fetcher::new(
85                blob_service,
86                directory_service,
87                path_info_service,
88                nar_calculation_service,
89            ),
90            known_paths: Default::default(),
91        }
92    }
93
94    /// for a given [StorePath] and additional [Path] inside the store path,
95    /// look up the [PathInfo], and if it exists, and then use
96    /// [directoryservice::descend_to] to return the
97    /// [Node] specified by `sub_path`.
98    ///
99    /// In case there is no PathInfo yet, this means we need to build it
100    /// (which currently is stubbed out still).
101    #[instrument(skip(self, store_path), fields(store_path=%store_path, indicatif.pb_show=tracing::field::Empty), ret(level = Level::TRACE), err(level = Level::TRACE))]
102    async fn store_path_to_path_info(
103        &self,
104        store_path: &StorePath<String>,
105        sub_path: &Path,
106    ) -> io::Result<Option<PathInfo>> {
107        // Find the root node for the store_path.
108        // It asks the PathInfoService first, but in case there was a Derivation
109        // produced that would build it, fall back to triggering the build.
110        // To populate the input nodes, it might recursively trigger builds of
111        // its dependencies too.
112        let mut path_info = match self
113            .path_info_service
114            .as_ref()
115            .get(*store_path.digest())
116            .await?
117        {
118            Some(path_info) => path_info,
119            // If there's no PathInfo found, this normally means we have to
120            // trigger the build (and insert into PathInfoService, after
121            // reference scanning).
122            // However, as Snix is (currently) not managing /nix/store itself,
123            // we return Ok(None) to let std_io take over.
124            // While reading from store paths that are not known to Snix during
125            // that evaluation clearly is an impurity, we still need to support
126            // it for things like <nixpkgs> pointing to a store path.
127            // In the future, these things will (need to) have PathInfo.
128            None => {
129                // The store path doesn't exist yet, so we need to fetch or build it.
130                // We check for fetches first, as we might have both native
131                // fetchers and FODs in KnownPaths, and prefer the former.
132                // This will also find [Fetch] synthesized from
133                // `builtin:fetchurl` Derivations.
134                let maybe_fetch = self
135                    .known_paths
136                    .borrow()
137                    .get_fetch_for_output_path(store_path);
138
139                match maybe_fetch {
140                    Some((name, fetch)) => {
141                        let (sp, path_info) = self
142                            .fetcher
143                            .ingest_and_persist(&name, fetch)
144                            .await
145                            .map_err(|e| {
146                            std::io::Error::new(std::io::ErrorKind::InvalidData, e)
147                        })?;
148
149                        debug_assert_eq!(
150                            sp.to_absolute_path(),
151                            store_path.as_ref().to_absolute_path(),
152                            "store path returned from fetcher must match store path we have in fetchers"
153                        );
154
155                        path_info
156                    }
157                    None => {
158                        // Look up the derivation for this output path.
159                        let (drv_path, drv) = {
160                            let known_paths = self.known_paths.borrow();
161                            match known_paths.get_drv_path_for_output_path(store_path) {
162                                Some(drv_path) => (
163                                    drv_path.to_owned(),
164                                    known_paths.get_drv_by_drvpath(drv_path).unwrap().to_owned(),
165                                ),
166                                None => {
167                                    warn!(store_path=%store_path, "no drv found");
168                                    // let StdIO take over
169                                    return Ok(None);
170                                }
171                            }
172                        };
173                        let span = Span::current();
174                        span.pb_start();
175                        span.pb_set_style(&snix_tracing::PB_SPINNER_STYLE);
176                        span.pb_set_message(&format!("⏳Waiting for inputs {}", &store_path));
177
178                        // derivation_to_build_request needs castore nodes for all inputs.
179                        // Provide them, which means, here is where we recursively build
180                        // all dependencies.
181                        let resolved_inputs = {
182                            let known_paths = &self.known_paths.borrow();
183                            crate::snix_build::get_all_inputs(&drv, known_paths, |path| {
184                                Box::pin(async move {
185                                    self.store_path_to_path_info(&path, Path::new("")).await
186                                })
187                            })
188                        }
189                        .try_collect()
190                        .await?;
191
192                        span.pb_set_message(&format!("🔨Building {}", &store_path));
193
194                        // synthesize the build request.
195                        let build_request = derivation_to_build_request(&drv, &resolved_inputs)?;
196
197                        let build_request_outputs = build_request.outputs.clone();
198
199                        // create a build
200                        let build_result = self
201                            .build_service
202                            .as_ref()
203                            .do_build(build_request)
204                            .await
205                            .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?;
206
207                        let mut out_path_info: Option<PathInfo> = None;
208
209                        // For each output, insert a PathInfo.
210                        for (output, output_path) in
211                            build_result.outputs.into_iter().zip(build_request_outputs)
212                        {
213                            // convert output_path to StorePath
214                            let output_store_path: StorePath<String> = {
215                                use std::os::unix::ffi::OsStrExt;
216
217                                StorePath::from_bytes(output_path.as_path().as_os_str().as_bytes())
218                                    .map_err(|e| {
219                                        std::io::Error::new(std::io::ErrorKind::InvalidData, e)
220                                    })?
221                            };
222
223                            // calculate the nar representation
224                            let (nar_size, nar_sha256) = self
225                                .nar_calculation_service
226                                .calculate_nar(&output.node)
227                                .await?;
228
229                            // assemble the PathInfo to persist
230                            let path_info = PathInfo {
231                                store_path: output_store_path.clone(),
232                                node: output.node,
233                                references: {
234                                    let all_possible_refs: Vec<_> = drv
235                                        .outputs
236                                        .values()
237                                        .filter_map(|output| output.path.as_ref())
238                                        .chain(resolved_inputs.keys())
239                                        .collect();
240                                    let mut references: Vec<_> = output
241                                        .output_needles
242                                        .iter()
243                                        // Map each output needle index back to the refscan_needle
244                                        .map(|idx| {
245                                            all_possible_refs
246                                                .get(*idx as usize)
247                                                .map(|it| (*it).clone())
248                                                .ok_or(std::io::Error::new(
249                                                    std::io::ErrorKind::Other,
250                                                    "invalid build response",
251                                                ))
252                                        })
253                                        .collect::<Result<_, std::io::Error>>()?;
254                                    // Produce references sorted by name for consistency with nix narinfos
255                                    references.sort();
256                                    references
257                                },
258                                nar_size,
259                                nar_sha256,
260                                signatures: vec![],
261                                deriver: Some(
262                                    StorePath::from_name_and_digest_fixed(
263                                        drv_path
264                                            .name()
265                                            .strip_suffix(".drv")
266                                            .expect("missing .drv suffix"),
267                                        *drv_path.digest(),
268                                    )
269                                    .expect(
270                                        "Snix bug: StorePath without .drv suffix must be valid",
271                                    ),
272                                ),
273                                ca: drv.fod_digest().map(|fod_digest| {
274                                    CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest))
275                                }),
276                            };
277
278                            self.path_info_service
279                                .put(path_info.clone())
280                                .await
281                                .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?;
282
283                            if store_path == &output_store_path {
284                                out_path_info = Some(path_info);
285                            }
286                        }
287
288                        out_path_info.ok_or(io::Error::other("build didn't produce store path"))?
289                    }
290                }
291            }
292        };
293
294        // now with the root_node and sub_path, descend to the node requested.
295        // We convert sub_path to the castore model here.
296        let sub_path = snix_castore::PathBuf::from_host_path(sub_path, true)?;
297
298        Ok(
299            directoryservice::descend_to(&self.directory_service, path_info.node.clone(), sub_path)
300                .await
301                .map_err(|e| std::io::Error::new(io::ErrorKind::Other, e))?
302                .map(|node| {
303                    path_info.node = node;
304                    path_info
305                }),
306        )
307    }
308}
309
310/// Helper function peeking at a [snix_castore::Node] and returning its [FileType]
311fn node_get_type(node: &Node) -> FileType {
312    match node {
313        Node::Directory { .. } => FileType::Directory,
314        Node::File { .. } => FileType::Regular,
315        Node::Symlink { .. } => FileType::Symlink,
316    }
317}
318
319impl EvalIO for SnixStoreIO {
320    #[instrument(skip(self), ret(level = Level::TRACE), err)]
321    fn path_exists(&self, path: &Path) -> io::Result<bool> {
322        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
323            if self
324                .tokio_handle
325                .block_on(self.store_path_to_path_info(&store_path, sub_path))?
326                .is_some()
327            {
328                Ok(true)
329            } else {
330                // As snix-store doesn't manage /nix/store on the filesystem,
331                // we still need to also ask self.std_io here.
332                self.std_io.path_exists(path)
333            }
334        } else {
335            // The store path is no store path, so do regular StdIO.
336            self.std_io.path_exists(path)
337        }
338    }
339
340    #[instrument(skip(self), err)]
341    fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>> {
342        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
343            if let Some(path_info) = self
344                .tokio_handle
345                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
346            {
347                // depending on the node type, treat open differently
348                match path_info.node {
349                    Node::Directory { .. } => {
350                        // This would normally be a io::ErrorKind::IsADirectory (still unstable)
351                        Err(io::Error::new(
352                            io::ErrorKind::Unsupported,
353                            format!("tried to open directory at {:?}", path),
354                        ))
355                    }
356                    Node::File { digest, .. } => {
357                        self.tokio_handle.block_on(async {
358                            let resp = self.blob_service.as_ref().open_read(&digest).await?;
359                            match resp {
360                                Some(blob_reader) => {
361                                    // The VM Response needs a sync [std::io::Reader].
362                                    Ok(Box::new(SyncIoBridge::new(blob_reader))
363                                        as Box<dyn io::Read>)
364                                }
365                                None => {
366                                    error!(
367                                        blob.digest = %digest,
368                                        "blob not found",
369                                    );
370                                    Err(io::Error::new(
371                                        io::ErrorKind::NotFound,
372                                        format!("blob {} not found", &digest),
373                                    ))
374                                }
375                            }
376                        })
377                    }
378                    Node::Symlink { .. } => Err(io::Error::new(
379                        io::ErrorKind::Unsupported,
380                        "open for symlinks is unsupported",
381                    ))?,
382                }
383            } else {
384                // As snix-store doesn't manage /nix/store on the filesystem,
385                // we still need to also ask self.std_io here.
386                self.std_io.open(path)
387            }
388        } else {
389            // The store path is no store path, so do regular StdIO.
390            self.std_io.open(path)
391        }
392    }
393
394    #[instrument(skip(self), ret(level = Level::TRACE), err)]
395    fn file_type(&self, path: &Path) -> io::Result<FileType> {
396        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
397            if let Some(path_info) = self
398                .tokio_handle
399                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
400            {
401                Ok(node_get_type(&path_info.node))
402            } else {
403                self.std_io.file_type(path)
404            }
405        } else {
406            self.std_io.file_type(path)
407        }
408    }
409
410    #[instrument(skip(self), ret(level = Level::TRACE), err)]
411    fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
412        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
413            if let Some(path_info) = self
414                .tokio_handle
415                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
416            {
417                match path_info.node {
418                    Node::Directory { digest, .. } => {
419                        // fetch the Directory itself.
420                        let directory = self
421                            .tokio_handle
422                            .block_on(async { self.directory_service.as_ref().get(&digest).await })?
423                            .ok_or_else(|| {
424                                // If we didn't get the directory node that's linked, that's a store inconsistency!
425                                error!(
426                                    directory.digest = %digest,
427                                    path = ?path,
428                                    "directory not found",
429                                );
430                                io::Error::new(
431                                    io::ErrorKind::NotFound,
432                                    format!("directory {digest} does not exist"),
433                                )
434                            })?;
435
436                        // construct children from nodes
437                        Ok(directory
438                            .into_nodes()
439                            .map(|(name, node)| (name.into(), node_get_type(&node)))
440                            .collect())
441                    }
442                    Node::File { .. } => {
443                        // This would normally be a io::ErrorKind::NotADirectory (still unstable)
444                        Err(io::Error::new(
445                            io::ErrorKind::Unsupported,
446                            "tried to readdir path {:?}, which is a file",
447                        ))?
448                    }
449                    Node::Symlink { .. } => Err(io::Error::new(
450                        io::ErrorKind::Unsupported,
451                        "read_dir for symlinks is unsupported",
452                    ))?,
453                }
454            } else {
455                self.std_io.read_dir(path)
456            }
457        } else {
458            self.std_io.read_dir(path)
459        }
460    }
461
462    #[instrument(skip(self), ret(level = Level::TRACE), err)]
463    fn import_path(&self, path: &Path) -> io::Result<PathBuf> {
464        let path_info = self.tokio_handle.block_on({
465            snix_store::import::import_path_as_nar_ca(
466                path,
467                snix_store::import::path_to_name(path)?,
468                &self.blob_service,
469                &self.directory_service,
470                &self.path_info_service,
471                &self.nar_calculation_service,
472            )
473        })?;
474
475        // From the returned PathInfo, extract the store path and return it.
476        Ok(path_info.store_path.to_absolute_path().into())
477    }
478
479    #[instrument(skip(self), ret(level = Level::TRACE))]
480    fn store_dir(&self) -> Option<String> {
481        Some("/nix/store".to_string())
482    }
483}
484
485#[cfg(test)]
486mod tests {
487    use std::{path::Path, rc::Rc, sync::Arc};
488
489    use bstr::ByteSlice;
490    use clap::Parser;
491    use snix_build::buildservice::DummyBuildService;
492    use snix_eval::{EvalIO, EvaluationResult};
493    use snix_store::utils::{ServiceUrlsMemory, construct_services};
494    use tempfile::TempDir;
495
496    use super::SnixStoreIO;
497    use crate::builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins};
498
499    /// evaluates a given nix expression and returns the result.
500    /// Takes care of setting up the evaluator so it knows about the
501    // `derivation` builtin.
502    fn eval(str: &str) -> EvaluationResult {
503        let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
504        let (blob_service, directory_service, path_info_service, nar_calculation_service) =
505            tokio_runtime
506                .block_on(async {
507                    construct_services(ServiceUrlsMemory::parse_from(std::iter::empty::<&str>()))
508                        .await
509                })
510                .unwrap();
511
512        let io = Rc::new(SnixStoreIO::new(
513            blob_service,
514            directory_service,
515            path_info_service,
516            nar_calculation_service.into(),
517            Arc::<DummyBuildService>::default(),
518            tokio_runtime.handle().clone(),
519        ));
520
521        let mut eval_builder =
522            snix_eval::Evaluation::builder(io.clone() as Rc<dyn EvalIO>).enable_import();
523        eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&io));
524        eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&io));
525        eval_builder = add_import_builtins(eval_builder, io);
526        let eval = eval_builder.build();
527
528        // run the evaluation itself.
529        eval.evaluate(str, None)
530    }
531
532    /// Helper function that takes a &Path, and invokes a snix evaluator coercing that path to a string
533    /// (via "${/this/path}"). The path can be both absolute or not.
534    /// It returns Option<String>, depending on whether the evaluation succeeded or not.
535    fn import_path_and_compare<P: AsRef<Path>>(p: P) -> Option<String> {
536        // Try to import the path using "${/tmp/path/to/test}".
537        // The format string looks funny, the {} passed to Nix needs to be
538        // escaped.
539        let code = format!(r#""${{{}}}""#, p.as_ref().display());
540        let result = eval(&code);
541
542        if !result.errors.is_empty() {
543            return None;
544        }
545
546        let value = result.value.expect("must be some");
547        match value {
548            snix_eval::Value::String(s) => Some(s.to_str_lossy().into_owned()),
549            _ => panic!("unexpected value type: {:?}", value),
550        }
551    }
552
553    /// Import a directory with a zero-sized ".keep" regular file.
554    /// Ensure it matches the (pre-recorded) store path that Nix would produce.
555    #[test]
556    fn import_directory() {
557        let tmpdir = TempDir::new().unwrap();
558
559        // create a directory named "test"
560        let src_path = tmpdir.path().join("test");
561        std::fs::create_dir(&src_path).unwrap();
562
563        // write a regular file `.keep`.
564        std::fs::write(src_path.join(".keep"), vec![]).unwrap();
565
566        // importing the path with .../test at the end.
567        assert_eq!(
568            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
569            import_path_and_compare(&src_path)
570        );
571
572        // importing the path with .../test/. at the end.
573        assert_eq!(
574            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
575            import_path_and_compare(src_path.join("."))
576        );
577    }
578
579    /// Import a file into the store. Nix uses the "recursive"/NAR-based hashing
580    /// scheme for these.
581    #[test]
582    fn import_file() {
583        let tmpdir = TempDir::new().unwrap();
584
585        // write a regular file `empty`.
586        std::fs::write(tmpdir.path().join("empty"), vec![]).unwrap();
587
588        assert_eq!(
589            Some("/nix/store/lx5i78a4izwk2qj1nq8rdc07y8zrwy90-empty".to_string()),
590            import_path_and_compare(tmpdir.path().join("empty"))
591        );
592
593        // write a regular file `hello.txt`.
594        std::fs::write(tmpdir.path().join("hello.txt"), b"Hello World!").unwrap();
595
596        assert_eq!(
597            Some("/nix/store/925f1jb1ajrypjbyq7rylwryqwizvhp0-hello.txt".to_string()),
598            import_path_and_compare(tmpdir.path().join("hello.txt"))
599        );
600    }
601
602    /// Invoke toString on a nonexisting file, and access the .file attribute.
603    /// This should not cause an error, because it shouldn't trigger an import,
604    /// and leave the path as-is.
605    #[test]
606    fn nonexisting_path_without_import() {
607        let result = eval("toString ({ line = 42; col = 42; file = /deep/thought; }.file)");
608
609        assert!(result.errors.is_empty(), "expect evaluation to succeed");
610        let value = result.value.expect("must be some");
611
612        match value {
613            snix_eval::Value::String(s) => {
614                assert_eq!(*s, "/deep/thought");
615            }
616            _ => panic!("unexpected value type: {:?}", value),
617        }
618    }
619}
snix_glue/snix_store_io.rs

snix_glue/
snix_store_io.rs
