snix_glue/
snix_store_io.rs

1//! This module provides an implementation of EvalIO talking to snix-store.
2use nix_compat::store_path::StorePathRef;
3use snix_build::buildservice::BuildService;
4use snix_build_glue::build_state::BuildState;
5use snix_eval::{EvalIO, FileType, StdIO};
6use snix_store::nar::NarCalculationService;
7use std::{
8    env,
9    ffi::{OsStr, OsString},
10    io,
11    sync::Arc,
12};
13use tokio_util::io::SyncIoBridge;
14use tracing::{Level, error, instrument};
15use url::Url;
16
17use snix_castore::{Node, blobservice::BlobService, directoryservice::DirectoryService};
18use snix_store::pathinfoservice::{PathInfo, PathInfoService};
19
20/// Implements [EvalIO], asking given [PathInfoService], [DirectoryService]
21/// and [BlobService].
22///
23/// In case the given path does not exist in these stores, we ask StdIO.
24/// This is to both cover cases of syntactically valid store paths, that exist
25/// on the filesystem (still managed by Nix), as well as being able to read
26/// files outside store paths.
27///
28/// This structure is also directly used by the derivation builtins
29/// and tightly coupled to it.
30///
31/// In the future, we may revisit that coupling and figure out how to generalize this interface and
32/// hide this implementation detail of the glue itself so that glue can be used with more than one
33/// implementation of "Snix Store IO" which does not necessarily bring the concept of blob service,
34/// directory service or path info service.
35pub struct SnixStoreIO {
36    pub build_state: BuildState,
37
38    std_io: StdIO,
39    pub(crate) tokio_handle: tokio::runtime::Handle,
40}
41
42impl SnixStoreIO {
43    pub fn new(
44        blob_service: Arc<dyn BlobService>,
45        directory_service: Arc<dyn DirectoryService>,
46        path_info_service: Arc<dyn PathInfoService>,
47        nar_calculation_service: Arc<dyn NarCalculationService>,
48        build_service: Arc<dyn BuildService>,
49        tokio_handle: tokio::runtime::Handle,
50        hashed_mirrors: Vec<Url>,
51    ) -> Self {
52        Self {
53            build_state: BuildState::new(
54                blob_service,
55                directory_service,
56                path_info_service,
57                nar_calculation_service,
58                build_service,
59                hashed_mirrors,
60            ),
61            std_io: StdIO {},
62            tokio_handle,
63        }
64    }
65
66    /// for a given [StorePath] and additional [snix_castore::Path] inside the store path,
67    /// look up the [PathInfo], and if it exists, and then uses
68    /// [descend_to] to return the [Node] specified by `sub_path`.
69    ///
70    /// In case there is no PathInfo yet, we check "self.known_paths" (learnt by
71    /// the evaluator)
72    ///  - If there's a fetch, we do that and return the resulting PathInfo.
73    ///  - If there's a Derivation, we build it and return the resulting
74    ///    PathInfo.
75    ///    To build it, we need to do a function call to ourselves with all
76    ///    inputs of that Derivation, which will trigger fetches / builds of
77    ///    inputs, recursively.
78    ///
79    /// This should be replaced with a proper scheduler knowing about a partial
80    /// subgraph at some point, because this design doesn't allow concurrent
81    /// builds yet.
82    ///
83    /// [StorePath]: nix_compat::store_path::StorePath
84    /// [descend_to]: snix_castore::directoryservice::traversal::descend_to
85    #[instrument(skip(self, store_path), fields(store_path=%store_path, indicatif.pb_show=tracing::field::Empty), ret(level = Level::TRACE), err(level = Level::TRACE))]
86    async fn store_path_to_path_info(
87        &self,
88        store_path: &StorePathRef<'_>,
89        sub_path: &snix_castore::Path,
90    ) -> io::Result<Option<PathInfo>> {
91        self.build_state
92            .store_path_to_path_info(store_path, sub_path)
93            .await
94    }
95}
96
97/// Helper function peeking at a [snix_castore::Node] and returning its [FileType]
98fn node_get_type(node: &Node) -> FileType {
99    match node {
100        Node::Directory { .. } => FileType::Directory,
101        Node::File { .. } => FileType::Regular,
102        Node::Symlink { .. } => FileType::Symlink,
103    }
104}
105
106// Helper function converting a [std::path::Path] to a [StorePath] and [snix_castore::Path].
107#[cfg(unix)]
108fn parse_store_and_sub_path<'a>(
109    path: &'a std::path::Path,
110) -> io::Result<(StorePathRef<'a>, &'a snix_castore::Path)> {
111    let (store_path, rest) =
112        StorePathRef::from_absolute_path_full(path).map_err(std::io::Error::other)?;
113
114    use std::os::unix::ffi::OsStrExt;
115    let sub_path = snix_castore::Path::from_bytes(rest.as_os_str().as_bytes())
116        .ok_or_else(|| std::io::Error::other("sub_path is no valid path"))?;
117
118    Ok((store_path, sub_path))
119}
120
121impl EvalIO for SnixStoreIO {
122    #[instrument(skip(self), ret(level = Level::TRACE), err)]
123    fn path_exists(&self, path: &std::path::Path) -> io::Result<bool> {
124        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
125            if self
126                .tokio_handle
127                .block_on(self.store_path_to_path_info(&store_path, sub_path))?
128                .is_some()
129            {
130                Ok(true)
131            } else {
132                // As snix-store doesn't manage /nix/store on the filesystem,
133                // we still need to also ask self.std_io here.
134                self.std_io.path_exists(path)
135            }
136        } else {
137            // The store path is no store path, so do regular StdIO.
138            self.std_io.path_exists(path)
139        }
140    }
141
142    #[instrument(skip(self), err)]
143    fn open(&self, path: &std::path::Path) -> io::Result<Box<dyn io::Read>> {
144        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
145            self.tokio_handle.block_on(async {
146                if let Some(path_info) = self.store_path_to_path_info(&store_path, sub_path).await?
147                {
148                    // depending on the node type, treat open differently
149                    match path_info.node {
150                        Node::Directory { .. } => {
151                            // This would normally be a io::ErrorKind::IsADirectory (still unstable)
152                            Err(io::Error::new(
153                                io::ErrorKind::Unsupported,
154                                format!("tried to open directory at {path:?}"),
155                            ))
156                        }
157                        Node::File { digest, .. } => {
158                            let resp = self
159                                .build_state
160                                .blob_service
161                                .as_ref()
162                                .open_read(&digest)
163                                .await?;
164                            match resp {
165                                Some(blob_reader) => {
166                                    // The VM Response needs a sync [std::io::Reader].
167                                    Ok(Box::new(SyncIoBridge::new(blob_reader))
168                                        as Box<dyn io::Read>)
169                                }
170                                None => {
171                                    error!(
172                                        blob.digest = %digest,
173                                        "blob not found",
174                                    );
175                                    Err(io::Error::new(
176                                        io::ErrorKind::NotFound,
177                                        format!("blob {} not found", &digest),
178                                    ))
179                                }
180                            }
181                        }
182                        Node::Symlink { .. } => Err(io::Error::new(
183                            io::ErrorKind::Unsupported,
184                            "open for symlinks is unsupported",
185                        ))?,
186                    }
187                } else {
188                    // As snix-store doesn't manage /nix/store on the filesystem,
189                    // we still need to also ask self.std_io here.
190                    self.std_io.open(path)
191                }
192            })
193        } else {
194            // The store path is no store path, so do regular StdIO.
195            self.std_io.open(path)
196        }
197    }
198
199    #[instrument(skip(self), ret(level = Level::TRACE), err)]
200    fn file_type(&self, path: &std::path::Path) -> io::Result<FileType> {
201        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
202            if let Some(path_info) = self
203                .tokio_handle
204                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
205            {
206                Ok(node_get_type(&path_info.node))
207            } else {
208                self.std_io.file_type(path)
209            }
210        } else {
211            self.std_io.file_type(path)
212        }
213    }
214
215    #[instrument(skip(self), ret(level = Level::TRACE), err)]
216    fn read_dir(&self, path: &std::path::Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
217        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
218            self.tokio_handle.block_on(async {
219                if let Some(path_info) = self.store_path_to_path_info(&store_path, sub_path).await?
220                {
221                    match path_info.node {
222                        Node::Directory { digest, .. } => {
223                            // fetch the Directory itself.
224                            let directory = self
225                                .build_state
226                                .directory_service
227                                .as_ref()
228                                .get(&digest)
229                                .await
230                                .map_err(std::io::Error::other)?
231                                .ok_or_else(|| {
232                                    // If we didn't get the directory node that's linked, that's a store inconsistency!
233                                    error!(
234                                        directory.digest = %digest,
235                                        path = ?path,
236                                        "directory not found",
237                                    );
238                                    io::Error::new(
239                                        io::ErrorKind::NotFound,
240                                        format!("directory {digest} does not exist"),
241                                    )
242                                })?;
243
244                            // construct children from nodes
245                            Ok(directory
246                                .into_nodes()
247                                .map(|(name, node)| (name.into(), node_get_type(&node)))
248                                .collect())
249                        }
250                        Node::File { .. } => {
251                            // This would normally be a io::ErrorKind::NotADirectory (still unstable)
252                            Err(io::Error::new(
253                                io::ErrorKind::Unsupported,
254                                "tried to readdir path {:?}, which is a file",
255                            ))?
256                        }
257                        Node::Symlink { .. } => Err(io::Error::new(
258                            io::ErrorKind::Unsupported,
259                            "read_dir for symlinks is unsupported",
260                        ))?,
261                    }
262                } else {
263                    self.std_io.read_dir(path)
264                }
265            })
266        } else {
267            self.std_io.read_dir(path)
268        }
269    }
270
271    #[instrument(skip(self), ret(level = Level::TRACE), err)]
272    fn import_path(&self, path: &std::path::Path) -> io::Result<std::path::PathBuf> {
273        let file_name = path.file_name().ok_or_else(|| {
274            io::Error::new(
275                io::ErrorKind::InvalidFilename,
276                "path without basename encountered",
277            )
278        })?;
279        let path_info = self.tokio_handle.block_on({
280            snix_store::import::import_path_as_nar_ca(
281                path,
282                nix_compat::store_path::validate_name_from_os_str(file_name)
283                    .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?,
284                &self.build_state.blob_service,
285                &self.build_state.directory_service,
286                &self.build_state.path_info_service,
287                &self.build_state.nar_calculation_service,
288            )
289        })?;
290
291        // From the returned PathInfo, extract the store path and return it.
292        Ok(path_info.store_path.to_absolute_path().into())
293    }
294
295    #[instrument(skip(self), ret(level = Level::TRACE))]
296    fn store_dir(&self) -> Option<String> {
297        Some("/nix/store".to_string())
298    }
299
300    fn get_env(&self, key: &OsStr) -> Option<OsString> {
301        env::var_os(key)
302    }
303}
304
305#[cfg(test)]
306mod tests {
307    use std::{path::Path, rc::Rc, sync::Arc};
308
309    use bstr::ByteSlice;
310    use clap::Parser;
311    use snix_build::buildservice::DummyBuildService;
312    use snix_eval::{EvalIO, EvaluationResult};
313    use snix_store::utils::{ServiceUrlsMemory, construct_services};
314    use tempfile::TempDir;
315
316    use super::SnixStoreIO;
317    use crate::builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins};
318
319    /// evaluates a given nix expression and returns the result.
320    /// Takes care of setting up the evaluator so it knows about the
321    // `derivation` builtin.
322    fn eval(str: &str) -> EvaluationResult {
323        let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
324        let (blob_service, directory_service, path_info_service, nar_calculation_service) =
325            tokio_runtime
326                .block_on(async {
327                    construct_services(ServiceUrlsMemory::parse_from(std::iter::empty::<&str>()))
328                        .await
329                })
330                .unwrap();
331
332        let io = Rc::new(SnixStoreIO::new(
333            blob_service,
334            directory_service,
335            path_info_service,
336            nar_calculation_service,
337            Arc::<DummyBuildService>::default(),
338            tokio_runtime.handle().clone(),
339            Vec::new(),
340        ));
341
342        let mut eval_builder =
343            snix_eval::Evaluation::builder(io.clone() as Rc<dyn EvalIO>).enable_import();
344        eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&io));
345        eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&io));
346        eval_builder = add_import_builtins(eval_builder, io);
347        let eval = eval_builder.build();
348
349        // run the evaluation itself.
350        eval.evaluate(str, None)
351    }
352
353    /// Helper function that takes a &Path, and invokes a snix evaluator coercing that path to a string
354    /// (via "${/this/path}"). The path can be both absolute or not.
355    /// It returns Option<String>, depending on whether the evaluation succeeded or not.
356    fn import_path_and_compare<P: AsRef<Path>>(p: P) -> Option<String> {
357        // Try to import the path using "${/tmp/path/to/test}".
358        // The format string looks funny, the {} passed to Nix needs to be
359        // escaped.
360        let code = format!(r#""${{{}}}""#, p.as_ref().display());
361        let result = eval(&code);
362
363        if !result.errors.is_empty() {
364            return None;
365        }
366
367        let value = result.value.expect("must be some");
368        match value {
369            snix_eval::Value::String(s) => Some(s.to_str_lossy().into_owned()),
370            _ => panic!("unexpected value type: {value:?}"),
371        }
372    }
373
374    /// Import a directory with a zero-sized ".keep" regular file.
375    /// Ensure it matches the (pre-recorded) store path that Nix would produce.
376    #[test]
377    fn import_directory() {
378        let tmpdir = TempDir::new().unwrap();
379
380        // create a directory named "test"
381        let src_path = tmpdir.path().join("test");
382        std::fs::create_dir(&src_path).unwrap();
383
384        // write a regular file `.keep`.
385        std::fs::write(src_path.join(".keep"), vec![]).unwrap();
386
387        // importing the path with .../test at the end.
388        assert_eq!(
389            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
390            import_path_and_compare(&src_path)
391        );
392
393        // importing the path with .../test/. at the end.
394        assert_eq!(
395            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
396            import_path_and_compare(src_path.join("."))
397        );
398    }
399
400    /// Import a file into the store. Nix uses the "recursive"/NAR-based hashing
401    /// scheme for these.
402    #[test]
403    fn import_file() {
404        let tmpdir = TempDir::new().unwrap();
405
406        // write a regular file `empty`.
407        std::fs::write(tmpdir.path().join("empty"), vec![]).unwrap();
408
409        assert_eq!(
410            Some("/nix/store/lx5i78a4izwk2qj1nq8rdc07y8zrwy90-empty".to_string()),
411            import_path_and_compare(tmpdir.path().join("empty"))
412        );
413
414        // write a regular file `hello.txt`.
415        std::fs::write(tmpdir.path().join("hello.txt"), b"Hello World!").unwrap();
416
417        assert_eq!(
418            Some("/nix/store/925f1jb1ajrypjbyq7rylwryqwizvhp0-hello.txt".to_string()),
419            import_path_and_compare(tmpdir.path().join("hello.txt"))
420        );
421    }
422
423    /// Invoke toString on a nonexisting file, and access the .file attribute.
424    /// This should not cause an error, because it shouldn't trigger an import,
425    /// and leave the path as-is.
426    #[test]
427    fn nonexisting_path_without_import() {
428        let result = eval("toString ({ line = 42; col = 42; file = /deep/thought; }.file)");
429
430        assert!(result.errors.is_empty(), "expect evaluation to succeed");
431        let value = result.value.expect("must be some");
432
433        match value {
434            snix_eval::Value::String(s) => {
435                assert_eq!(*s, "/deep/thought");
436            }
437            _ => panic!("unexpected value type: {value:?}"),
438        }
439    }
440}
snix_glue/snix_store_io.rs

snix_glue/
snix_store_io.rs
