snix_glue/
snix_store_io.rs

1//! This module provides an implementation of EvalIO talking to snix-store.
2use futures::TryStreamExt;
3use nix_compat::{nixhash::CAHash, store_path::StorePath};
4use snix_build::buildservice::BuildService;
5use snix_eval::{EvalIO, FileType, StdIO};
6use snix_store::nar::NarCalculationService;
7use std::{
8    cell::RefCell,
9    env,
10    ffi::{OsStr, OsString},
11    io,
12    path::{Path, PathBuf},
13    sync::Arc,
14};
15use tokio_util::io::SyncIoBridge;
16use tracing::{Level, Span, error, instrument, warn};
17use tracing_indicatif::span_ext::IndicatifSpanExt;
18use url::Url;
19
20use snix_castore::{
21    Node,
22    blobservice::BlobService,
23    directoryservice::{self, DirectoryService},
24};
25use snix_store::pathinfoservice::{PathInfo, PathInfoService};
26
27use crate::fetchers::Fetcher;
28use crate::known_paths::KnownPaths;
29use crate::snix_build::derivation_to_build_request;
30
31/// Implements [EvalIO], asking given [PathInfoService], [DirectoryService]
32/// and [BlobService].
33///
34/// In case the given path does not exist in these stores, we ask StdIO.
35/// This is to both cover cases of syntactically valid store paths, that exist
36/// on the filesystem (still managed by Nix), as well as being able to read
37/// files outside store paths.
38///
39/// This structure is also directly used by the derivation builtins
40/// and tightly coupled to it.
41///
42/// In the future, we may revisit that coupling and figure out how to generalize this interface and
43/// hide this implementation detail of the glue itself so that glue can be used with more than one
44/// implementation of "Snix Store IO" which does not necessarily bring the concept of blob service,
45/// directory service or path info service.
46pub struct SnixStoreIO {
47    // This is public so helper functions can interact with the stores directly.
48    pub(crate) blob_service: Arc<dyn BlobService>,
49    pub(crate) directory_service: Arc<dyn DirectoryService>,
50    pub(crate) path_info_service: Arc<dyn PathInfoService>,
51    pub(crate) nar_calculation_service: Arc<dyn NarCalculationService>,
52
53    std_io: StdIO,
54    #[allow(dead_code)]
55    build_service: Arc<dyn BuildService>,
56    pub(crate) tokio_handle: tokio::runtime::Handle,
57
58    #[allow(clippy::type_complexity)]
59    pub(crate) fetcher: Fetcher<
60        Arc<dyn BlobService>,
61        Arc<dyn DirectoryService>,
62        Arc<dyn PathInfoService>,
63        Arc<dyn NarCalculationService>,
64    >,
65
66    // Paths known how to produce, by building or fetching.
67    pub known_paths: RefCell<KnownPaths>,
68}
69
70impl SnixStoreIO {
71    pub fn new(
72        blob_service: Arc<dyn BlobService>,
73        directory_service: Arc<dyn DirectoryService>,
74        path_info_service: Arc<dyn PathInfoService>,
75        nar_calculation_service: Arc<dyn NarCalculationService>,
76        build_service: Arc<dyn BuildService>,
77        tokio_handle: tokio::runtime::Handle,
78        hashed_mirrors: Vec<Url>,
79    ) -> Self {
80        Self {
81            blob_service: blob_service.clone(),
82            directory_service: directory_service.clone(),
83            path_info_service: path_info_service.clone(),
84            nar_calculation_service: nar_calculation_service.clone(),
85            std_io: StdIO {},
86            build_service,
87            tokio_handle,
88            fetcher: Fetcher::new(
89                blob_service,
90                directory_service,
91                path_info_service,
92                nar_calculation_service,
93                hashed_mirrors,
94            ),
95            known_paths: Default::default(),
96        }
97    }
98
99    /// for a given [StorePath] and additional [Path] inside the store path,
100    /// look up the [PathInfo], and if it exists, and then use
101    /// [directoryservice::descend_to] to return the
102    /// [Node] specified by `sub_path`.
103    ///
104    /// In case there is no PathInfo yet, this means we need to build it
105    /// (which currently is stubbed out still).
106    #[instrument(skip(self, store_path), fields(store_path=%store_path, indicatif.pb_show=tracing::field::Empty), ret(level = Level::TRACE), err(level = Level::TRACE))]
107    async fn store_path_to_path_info(
108        &self,
109        store_path: &StorePath<String>,
110        sub_path: &Path,
111    ) -> io::Result<Option<PathInfo>> {
112        // Find the root node for the store_path.
113        // It asks the PathInfoService first, but in case there was a Derivation
114        // produced that would build it, fall back to triggering the build.
115        // To populate the input nodes, it might recursively trigger builds of
116        // its dependencies too.
117        let mut path_info = match self
118            .path_info_service
119            .as_ref()
120            .get(*store_path.digest())
121            .await?
122        {
123            Some(path_info) => path_info,
124            // If there's no PathInfo found, this normally means we have to
125            // trigger the build (and insert into PathInfoService, after
126            // reference scanning).
127            // However, as Snix is (currently) not managing /nix/store itself,
128            // we return Ok(None) to let std_io take over.
129            // While reading from store paths that are not known to Snix during
130            // that evaluation clearly is an impurity, we still need to support
131            // it for things like <nixpkgs> pointing to a store path.
132            // In the future, these things will (need to) have PathInfo.
133            None => {
134                // The store path doesn't exist yet, so we need to fetch or build it.
135                // We check for fetches first, as we might have both native
136                // fetchers and FODs in KnownPaths, and prefer the former.
137                // This will also find [Fetch] synthesized from
138                // `builtin:fetchurl` Derivations.
139                let maybe_fetch = self
140                    .known_paths
141                    .borrow()
142                    .get_fetch_for_output_path(store_path);
143
144                match maybe_fetch {
145                    Some((name, fetch)) => {
146                        let (sp, path_info) = self
147                            .fetcher
148                            .ingest_and_persist(&name, fetch)
149                            .await
150                            .map_err(|e| {
151                            std::io::Error::new(std::io::ErrorKind::InvalidData, e)
152                        })?;
153
154                        debug_assert_eq!(
155                            sp.to_absolute_path(),
156                            store_path.as_ref().to_absolute_path(),
157                            "store path returned from fetcher must match store path we have in fetchers"
158                        );
159
160                        path_info
161                    }
162                    None => {
163                        // Look up the derivation for this output path.
164                        let (drv_path, drv) = {
165                            let known_paths = self.known_paths.borrow();
166                            match known_paths.get_drv_path_for_output_path(store_path) {
167                                Some(drv_path) => (
168                                    drv_path.to_owned(),
169                                    known_paths.get_drv_by_drvpath(drv_path).unwrap().to_owned(),
170                                ),
171                                None => {
172                                    warn!(store_path=%store_path, "no drv found");
173                                    // let StdIO take over
174                                    return Ok(None);
175                                }
176                            }
177                        };
178                        let span = Span::current();
179                        span.pb_start();
180                        span.pb_set_style(&snix_tracing::PB_SPINNER_STYLE);
181                        span.pb_set_message(&format!("⏳Waiting for inputs {}", &store_path));
182
183                        // derivation_to_build_request needs castore nodes for all inputs.
184                        // Provide them, which means, here is where we recursively build
185                        // all dependencies.
186                        let resolved_inputs = {
187                            let known_paths = &self.known_paths.borrow();
188                            crate::snix_build::get_all_inputs(&drv, known_paths, |path| {
189                                Box::pin(async move {
190                                    self.store_path_to_path_info(&path, Path::new("")).await
191                                })
192                            })
193                        }
194                        .try_collect()
195                        .await?;
196
197                        span.pb_set_message(&format!("🔨Building {}", &store_path));
198
199                        // synthesize the build request.
200                        let build_request = derivation_to_build_request(&drv, &resolved_inputs)?;
201
202                        // collect all store paths from the request, sorted.
203                        let output_paths: Vec<StorePath<String>> = build_request
204                            .outputs
205                            .iter()
206                            .map(|output_path| {
207                                // in the case of building nix store paths,
208                                // all outputs are in `inputs_dir`.
209                                // When stripping it, we end up with the store path
210                                // basename.
211                                StorePath::from_bytes(
212                                    output_path
213                                        .strip_prefix(&build_request.inputs_dir)
214                                        .expect("Snix bug: inputs_dir not prefix of request output")
215                                        .as_os_str()
216                                        .as_encoded_bytes(),
217                                )
218                                .expect("Snix bug: unable to parse output path as StorePath")
219                            })
220                            .collect();
221
222                        // create a build
223                        let build_result = self
224                            .build_service
225                            .as_ref()
226                            .do_build(build_request)
227                            .await
228                            .map_err(std::io::Error::other)?;
229
230                        let mut out_path_info: Option<PathInfo> = None;
231
232                        // For each output, insert a PathInfo.
233                        for (output, output_path) in
234                            build_result.outputs.into_iter().zip(output_paths)
235                        {
236                            // calculate the nar representation
237                            let (nar_size, nar_sha256) = self
238                                .nar_calculation_service
239                                .calculate_nar(&output.node)
240                                .await?;
241
242                            // assemble the PathInfo to persist
243                            let path_info = PathInfo {
244                                store_path: output_path.clone(),
245                                node: output.node,
246                                references: {
247                                    let all_possible_refs: Vec<_> = drv
248                                        .outputs
249                                        .values()
250                                        .filter_map(|output| output.path.as_ref())
251                                        .chain(resolved_inputs.keys())
252                                        .collect();
253                                    let mut references: Vec<_> = output
254                                        .output_needles
255                                        .iter()
256                                        // Map each output needle index back to the refscan_needle
257                                        .map(|idx| {
258                                            all_possible_refs
259                                                .get(*idx as usize)
260                                                .map(|it| (*it).clone())
261                                                .ok_or(std::io::Error::other(
262                                                    "invalid build response",
263                                                ))
264                                        })
265                                        .collect::<Result<_, std::io::Error>>()?;
266                                    // Produce references sorted by name for consistency with nix narinfos
267                                    references.sort();
268                                    references
269                                },
270                                nar_size,
271                                nar_sha256,
272                                signatures: vec![],
273                                deriver: Some(
274                                    StorePath::from_name_and_digest_fixed(
275                                        drv_path
276                                            .name()
277                                            .strip_suffix(".drv")
278                                            .expect("missing .drv suffix"),
279                                        *drv_path.digest(),
280                                    )
281                                    .expect(
282                                        "Snix bug: StorePath without .drv suffix must be valid",
283                                    ),
284                                ),
285                                ca: drv.fod_digest().map(|fod_digest| {
286                                    CAHash::Nar(nix_compat::nixhash::NixHash::Sha256(fod_digest))
287                                }),
288                            };
289
290                            self.path_info_service
291                                .put(path_info.clone())
292                                .await
293                                .map_err(std::io::Error::other)?;
294
295                            if store_path == &output_path {
296                                out_path_info = Some(path_info);
297                            }
298                        }
299
300                        out_path_info.ok_or(io::Error::other("build didn't produce store path"))?
301                    }
302                }
303            }
304        };
305
306        // now with the root_node and sub_path, descend to the node requested.
307        // We convert sub_path to the castore model here.
308        let sub_path = snix_castore::PathBuf::from_host_path(sub_path, true)?;
309
310        Ok(
311            directoryservice::descend_to(&self.directory_service, path_info.node.clone(), sub_path)
312                .await
313                .map_err(std::io::Error::other)?
314                .map(|node| {
315                    path_info.node = node;
316                    path_info
317                }),
318        )
319    }
320}
321
322/// Helper function peeking at a [snix_castore::Node] and returning its [FileType]
323fn node_get_type(node: &Node) -> FileType {
324    match node {
325        Node::Directory { .. } => FileType::Directory,
326        Node::File { .. } => FileType::Regular,
327        Node::Symlink { .. } => FileType::Symlink,
328    }
329}
330
331impl EvalIO for SnixStoreIO {
332    #[instrument(skip(self), ret(level = Level::TRACE), err)]
333    fn path_exists(&self, path: &Path) -> io::Result<bool> {
334        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
335            if self
336                .tokio_handle
337                .block_on(self.store_path_to_path_info(&store_path, sub_path))?
338                .is_some()
339            {
340                Ok(true)
341            } else {
342                // As snix-store doesn't manage /nix/store on the filesystem,
343                // we still need to also ask self.std_io here.
344                self.std_io.path_exists(path)
345            }
346        } else {
347            // The store path is no store path, so do regular StdIO.
348            self.std_io.path_exists(path)
349        }
350    }
351
352    #[instrument(skip(self), err)]
353    fn open(&self, path: &Path) -> io::Result<Box<dyn io::Read>> {
354        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
355            if let Some(path_info) = self
356                .tokio_handle
357                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
358            {
359                // depending on the node type, treat open differently
360                match path_info.node {
361                    Node::Directory { .. } => {
362                        // This would normally be a io::ErrorKind::IsADirectory (still unstable)
363                        Err(io::Error::new(
364                            io::ErrorKind::Unsupported,
365                            format!("tried to open directory at {path:?}"),
366                        ))
367                    }
368                    Node::File { digest, .. } => {
369                        self.tokio_handle.block_on(async {
370                            let resp = self.blob_service.as_ref().open_read(&digest).await?;
371                            match resp {
372                                Some(blob_reader) => {
373                                    // The VM Response needs a sync [std::io::Reader].
374                                    Ok(Box::new(SyncIoBridge::new(blob_reader))
375                                        as Box<dyn io::Read>)
376                                }
377                                None => {
378                                    error!(
379                                        blob.digest = %digest,
380                                        "blob not found",
381                                    );
382                                    Err(io::Error::new(
383                                        io::ErrorKind::NotFound,
384                                        format!("blob {} not found", &digest),
385                                    ))
386                                }
387                            }
388                        })
389                    }
390                    Node::Symlink { .. } => Err(io::Error::new(
391                        io::ErrorKind::Unsupported,
392                        "open for symlinks is unsupported",
393                    ))?,
394                }
395            } else {
396                // As snix-store doesn't manage /nix/store on the filesystem,
397                // we still need to also ask self.std_io here.
398                self.std_io.open(path)
399            }
400        } else {
401            // The store path is no store path, so do regular StdIO.
402            self.std_io.open(path)
403        }
404    }
405
406    #[instrument(skip(self), ret(level = Level::TRACE), err)]
407    fn file_type(&self, path: &Path) -> io::Result<FileType> {
408        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
409            if let Some(path_info) = self
410                .tokio_handle
411                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
412            {
413                Ok(node_get_type(&path_info.node))
414            } else {
415                self.std_io.file_type(path)
416            }
417        } else {
418            self.std_io.file_type(path)
419        }
420    }
421
422    #[instrument(skip(self), ret(level = Level::TRACE), err)]
423    fn read_dir(&self, path: &Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
424        if let Ok((store_path, sub_path)) = StorePath::from_absolute_path_full(path) {
425            if let Some(path_info) = self
426                .tokio_handle
427                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
428            {
429                match path_info.node {
430                    Node::Directory { digest, .. } => {
431                        // fetch the Directory itself.
432                        let directory = self
433                            .tokio_handle
434                            .block_on(async { self.directory_service.as_ref().get(&digest).await })?
435                            .ok_or_else(|| {
436                                // If we didn't get the directory node that's linked, that's a store inconsistency!
437                                error!(
438                                    directory.digest = %digest,
439                                    path = ?path,
440                                    "directory not found",
441                                );
442                                io::Error::new(
443                                    io::ErrorKind::NotFound,
444                                    format!("directory {digest} does not exist"),
445                                )
446                            })?;
447
448                        // construct children from nodes
449                        Ok(directory
450                            .into_nodes()
451                            .map(|(name, node)| (name.into(), node_get_type(&node)))
452                            .collect())
453                    }
454                    Node::File { .. } => {
455                        // This would normally be a io::ErrorKind::NotADirectory (still unstable)
456                        Err(io::Error::new(
457                            io::ErrorKind::Unsupported,
458                            "tried to readdir path {:?}, which is a file",
459                        ))?
460                    }
461                    Node::Symlink { .. } => Err(io::Error::new(
462                        io::ErrorKind::Unsupported,
463                        "read_dir for symlinks is unsupported",
464                    ))?,
465                }
466            } else {
467                self.std_io.read_dir(path)
468            }
469        } else {
470            self.std_io.read_dir(path)
471        }
472    }
473
474    #[instrument(skip(self), ret(level = Level::TRACE), err)]
475    fn import_path(&self, path: &Path) -> io::Result<PathBuf> {
476        let path_info = self.tokio_handle.block_on({
477            snix_store::import::import_path_as_nar_ca(
478                path,
479                snix_store::import::path_to_name(path)?,
480                &self.blob_service,
481                &self.directory_service,
482                &self.path_info_service,
483                &self.nar_calculation_service,
484            )
485        })?;
486
487        // From the returned PathInfo, extract the store path and return it.
488        Ok(path_info.store_path.to_absolute_path().into())
489    }
490
491    #[instrument(skip(self), ret(level = Level::TRACE))]
492    fn store_dir(&self) -> Option<String> {
493        Some("/nix/store".to_string())
494    }
495
496    fn get_env(&self, key: &OsStr) -> Option<OsString> {
497        env::var_os(key)
498    }
499}
500
501#[cfg(test)]
502mod tests {
503    use std::{path::Path, rc::Rc, sync::Arc};
504
505    use bstr::ByteSlice;
506    use clap::Parser;
507    use snix_build::buildservice::DummyBuildService;
508    use snix_eval::{EvalIO, EvaluationResult};
509    use snix_store::utils::{ServiceUrlsMemory, construct_services};
510    use tempfile::TempDir;
511
512    use super::SnixStoreIO;
513    use crate::builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins};
514
515    /// evaluates a given nix expression and returns the result.
516    /// Takes care of setting up the evaluator so it knows about the
517    // `derivation` builtin.
518    fn eval(str: &str) -> EvaluationResult {
519        let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
520        let (blob_service, directory_service, path_info_service, nar_calculation_service) =
521            tokio_runtime
522                .block_on(async {
523                    construct_services(ServiceUrlsMemory::parse_from(std::iter::empty::<&str>()))
524                        .await
525                })
526                .unwrap();
527
528        let io = Rc::new(SnixStoreIO::new(
529            blob_service,
530            directory_service,
531            path_info_service,
532            nar_calculation_service.into(),
533            Arc::<DummyBuildService>::default(),
534            tokio_runtime.handle().clone(),
535            Vec::new(),
536        ));
537
538        let mut eval_builder =
539            snix_eval::Evaluation::builder(io.clone() as Rc<dyn EvalIO>).enable_import();
540        eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&io));
541        eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&io));
542        eval_builder = add_import_builtins(eval_builder, io);
543        let eval = eval_builder.build();
544
545        // run the evaluation itself.
546        eval.evaluate(str, None)
547    }
548
549    /// Helper function that takes a &Path, and invokes a snix evaluator coercing that path to a string
550    /// (via "${/this/path}"). The path can be both absolute or not.
551    /// It returns Option<String>, depending on whether the evaluation succeeded or not.
552    fn import_path_and_compare<P: AsRef<Path>>(p: P) -> Option<String> {
553        // Try to import the path using "${/tmp/path/to/test}".
554        // The format string looks funny, the {} passed to Nix needs to be
555        // escaped.
556        let code = format!(r#""${{{}}}""#, p.as_ref().display());
557        let result = eval(&code);
558
559        if !result.errors.is_empty() {
560            return None;
561        }
562
563        let value = result.value.expect("must be some");
564        match value {
565            snix_eval::Value::String(s) => Some(s.to_str_lossy().into_owned()),
566            _ => panic!("unexpected value type: {value:?}"),
567        }
568    }
569
570    /// Import a directory with a zero-sized ".keep" regular file.
571    /// Ensure it matches the (pre-recorded) store path that Nix would produce.
572    #[test]
573    fn import_directory() {
574        let tmpdir = TempDir::new().unwrap();
575
576        // create a directory named "test"
577        let src_path = tmpdir.path().join("test");
578        std::fs::create_dir(&src_path).unwrap();
579
580        // write a regular file `.keep`.
581        std::fs::write(src_path.join(".keep"), vec![]).unwrap();
582
583        // importing the path with .../test at the end.
584        assert_eq!(
585            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
586            import_path_and_compare(&src_path)
587        );
588
589        // importing the path with .../test/. at the end.
590        assert_eq!(
591            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
592            import_path_and_compare(src_path.join("."))
593        );
594    }
595
596    /// Import a file into the store. Nix uses the "recursive"/NAR-based hashing
597    /// scheme for these.
598    #[test]
599    fn import_file() {
600        let tmpdir = TempDir::new().unwrap();
601
602        // write a regular file `empty`.
603        std::fs::write(tmpdir.path().join("empty"), vec![]).unwrap();
604
605        assert_eq!(
606            Some("/nix/store/lx5i78a4izwk2qj1nq8rdc07y8zrwy90-empty".to_string()),
607            import_path_and_compare(tmpdir.path().join("empty"))
608        );
609
610        // write a regular file `hello.txt`.
611        std::fs::write(tmpdir.path().join("hello.txt"), b"Hello World!").unwrap();
612
613        assert_eq!(
614            Some("/nix/store/925f1jb1ajrypjbyq7rylwryqwizvhp0-hello.txt".to_string()),
615            import_path_and_compare(tmpdir.path().join("hello.txt"))
616        );
617    }
618
619    /// Invoke toString on a nonexisting file, and access the .file attribute.
620    /// This should not cause an error, because it shouldn't trigger an import,
621    /// and leave the path as-is.
622    #[test]
623    fn nonexisting_path_without_import() {
624        let result = eval("toString ({ line = 42; col = 42; file = /deep/thought; }.file)");
625
626        assert!(result.errors.is_empty(), "expect evaluation to succeed");
627        let value = result.value.expect("must be some");
628
629        match value {
630            snix_eval::Value::String(s) => {
631                assert_eq!(*s, "/deep/thought");
632            }
633            _ => panic!("unexpected value type: {value:?}"),
634        }
635    }
636}
snix_glue/snix_store_io.rs

snix_glue/
snix_store_io.rs
